diff --git a/BENCHMARK.md b/BENCHMARK.md index 427097936..c1d4c0756 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -222,12 +222,21 @@ versus an LLM-judged setup. Per-tier R@5 (bm25): exact **96.8%** · concept 25.4% · multi_hop 30.0%. +_The `bm25` row key is historical. The row measures the shipping +lexical path — the store's native symbol FTS queried through +`Engine.SearchSymbols`, the same call `search_symbols` makes — not an +in-process BM25 index. The key is kept unchanged so older runs stay +joinable._ + **Headline**: the `search_symbols` text path (`bm25`) lands **R@5 = 55.1%** / **R@20 = 63.5%**, and **96.8%** on exact symbol-name queries — 3.2× ripgrep's R@5 floor. Enabling Porter stemming (`GORTEX_FTS_STEMMING=1`) trades a little exact-tier precision for breadth — R@20 +5.7pp, exact-tier R@5 −3.1pp — so it -ships opt-in. The `semantic` and `rrf` rankers require `--embeddings` +ships opt-in. The mode is read at process start; on a later restart, +gortex automatically rebuilds a repository's native symbol FTS when +its persisted normalization marker differs. The `semantic` and `rrf` +rankers require `--embeddings` and are omitted here; the `graph` ranker scores only graph-traversal fixtures. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 32498cce1..691a51cb5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -236,7 +236,7 @@ internal/ query/ Query engine (BFS traversal, SubGraph) resolver/ Cross-file reference resolution, IMPLEMENTS inference review/ PR review and diff analysis pipeline - search/ BM25, trigram, and semantic search + search/ FTS adapter, trigram, and semantic search server/ HTTP transports and the web dashboard ``` diff --git a/README.md b/README.md index d9dbd6eaa..4228969b6 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ High-quality parsing 257 languages/grammars through tree-sitter AST analysis, in - **Zero external dependencies** — single binary, everything in-process. No network, no model download to get started. Install, start daemon, use. - **Agent integrations (19)** — `gortex init` configures every detected coding assistant on the machine → [docs/agents.md](docs/agents.md) - **100+ MCP tools, 16 resources, 3 prompts** — symbol lookup, call chains, blast radius, dataflow, clone detection, refactoring, code actions → [docs/mcp.md](docs/mcp.md) -- **Semantic search default-on** — baked GloVe-50d (3.8 MB embedded), hybrid BM25 + vector + RRF, zero deps; opt-in MiniLM / Ollama / OpenAI → [docs/semantic-search.md](docs/semantic-search.md) +- **Semantic search default-on** — baked GloVe-50d (3.8 MB embedded), store-native FTS5/BM25 + vector with adaptive alpha fusion, zero deps; opt-in MiniLM / Ollama / OpenAI → [docs/semantic-search.md](docs/semantic-search.md) - **Speculative execution** — `preview_edit` / `simulate_chain` answer "what would change if I applied this WorkspaceEdit?" without touching disk - **Live editor overlays** — push unsaved buffers as a shadow graph; tools read through it. Branching for parallel speculative sessions - **GCX1 wire format** — published, round-trippable. **An additional −27% tokens vs JSON** at same fidelity → [docs/wire-format.md](docs/wire-format.md) diff --git a/bench/fixtures/retrieval.yaml b/bench/fixtures/retrieval.yaml index 2aa84265f..5180dcde2 100644 --- a/bench/fixtures/retrieval.yaml +++ b/bench/fixtures/retrieval.yaml @@ -27,11 +27,11 @@ # Cases are tiered so per-tier weakness is visible: # # tier: exact — symbol-name queries. Tests "find this named -# thing." BM25 should dominate; a retrieval stack -# that can't ace exact tier is broken. +# thing." Store-native lexical search should dominate; +# a retrieval stack that can't ace exact tier is broken. # tier: concept — natural-language paraphrase queries. Tests -# semantic understanding. Where BM25 starts losing -# to semantic / RRF if the embedder is competent. +# semantic understanding. Where lexical retrieval starts +# losing to vector/adaptive fusion if the embedder is competent. # tier: multi_hop — relational queries with several valid expected # IDs (any-hit semantics). Tests graph-aware # retrieval: "things that handle X" / "extractors @@ -63,8 +63,8 @@ cases: - { id: exact-AddNode, tier: exact, query: "AddNode", expected: [internal/graph/graph.go::Graph.AddNode] } - { id: exact-AtomicWriteFile, tier: exact, query: "AtomicWriteFile", expected: [internal/agents/writer.go::AtomicWriteFile] } - { id: exact-WriteIfNotExists, tier: exact, query: "WriteIfNotExists", expected: [internal/agents/writer.go::WriteIfNotExists] } - - { id: exact-BM25Backend, tier: exact, query: "BM25Backend", expected: [internal/search/bm25.go::BM25Backend] } - - { id: exact-NewBM25, tier: exact, query: "NewBM25", expected: [internal/search/bm25.go::NewBM25] } + - { id: exact-SymbolSearcherBackend, tier: exact, query: "SymbolSearcherBackend", expected: [internal/search/symbolsearcher_backend.go::SymbolSearcherBackend] } + - { id: exact-NewSymbolSearcherBackend, tier: exact, query: "NewSymbolSearcherBackend", expected: [internal/search/symbolsearcher_backend.go::NewSymbolSearcherBackend] } - { id: exact-HybridBackend, tier: exact, query: "HybridBackend", expected: [internal/search/hybrid.go::HybridBackend] } - { id: exact-NewHybrid, tier: exact, query: "NewHybrid", expected: [internal/search/hybrid.go::NewHybrid] } - { id: exact-alphaFuse, tier: exact, query: "alphaFuse", expected: [internal/search/hybrid.go::alphaFuse] } @@ -119,10 +119,10 @@ cases: - { id: concept-reindex-one-file, tier: concept, query: "reindex a single file after an edit", expected: [internal/indexer/indexer.go::Indexer.IndexFile] } - { id: concept-evict-file, tier: concept, query: "remove all nodes belonging to a file", expected: [internal/graph/graph.go::Graph.EvictFile] } - { id: concept-evict-repo, tier: concept, query: "drop every node for a repository prefix", expected: [internal/graph/graph.go::Graph.EvictRepo] } - - { id: concept-bm25-index, tier: concept, query: "text search backend with TF-IDF ranking", expected: [internal/search/bm25.go::BM25Backend] } - - { id: concept-hybrid-fuse, tier: concept, query: "combine text and vector search with RRF", expected: [internal/search/hybrid.go::HybridBackend] } - - { id: concept-adaptive-alpha, tier: concept, query: "adaptive alpha weighted reciprocal rank fusion", expected: [internal/search/hybrid.go::alphaFuse] } - - { id: concept-swap-backend, tier: concept, query: "hot-swap the in-memory search backend", expected: [internal/search/swappable.go::Swappable] } + - { id: concept-store-native-text, tier: concept, query: "adapt store-native symbol search into the search backend", expected: [internal/search/symbolsearcher_backend.go::SymbolSearcherBackend] } + - { id: concept-hybrid-fuse, tier: concept, query: "combine store-native text and vector search with adaptive alpha", expected: [internal/search/hybrid.go::HybridBackend] } + - { id: concept-adaptive-alpha, tier: concept, query: "adaptive alpha weighted text and vector fusion", expected: [internal/search/hybrid.go::alphaFuse] } + - { id: concept-swap-backend, tier: concept, query: "atomically replace the vector channel while preserving the live text backend", expected: [internal/search/swappable.go::Swappable.ReplaceHybridVector] } - { id: concept-glove-embed, tier: concept, query: "built-in GloVe word vector embedder", expected: [internal/embedding/static.go::StaticProvider] } - { id: concept-mcp-server, tier: concept, query: "MCP server type holding engine and graph", expected: [internal/mcp/server.go::Server] } - { id: concept-session-state, tier: concept, query: "per-client session activity tracking", expected: [internal/mcp/server.go::sessionState] } @@ -154,7 +154,7 @@ cases: - { id: concept-ts-extract, tier: concept, query: "TypeScript language extractor", expected: [internal/parser/languages/typescript.go::TypeScriptExtractor] } - { id: concept-detect-lang, tier: concept, query: "detect language from file extension", expected: [internal/parser/registry.go::Registry.GetByExtension] } - { id: concept-register-all, tier: concept, query: "register every language extractor", expected: [internal/parser/languages/register.go::RegisterAll] } - - { id: concept-ranker-bm25, tier: concept, query: "eval ranker adapter for BM25", expected: [internal/eval/recall/rankers.go::BM25Ranker] } + - { id: concept-ranker-engine, tier: concept, query: "eval ranker adapter for engine search", expected: [internal/eval/recall/rankers.go::EngineRanker] } - { id: concept-ranker-rrf, tier: concept, query: "eval ranker adapter for RRF hybrid", expected: [internal/eval/recall/rankers.go::RRFRanker] } - { id: concept-ranker-semantic, tier: concept, query: "eval ranker for vector-only semantic", expected: [internal/eval/recall/rankers.go::SemanticRanker] } - { id: concept-ranker-winnow, tier: concept, query: "eval ranker wrapping graph-aware winnow", expected: [internal/eval/recall/rankers.go::WinnowRanker] } @@ -163,7 +163,7 @@ cases: - { id: concept-repo-for-file, tier: concept, query: "find which repo contains a given file", expected: [internal/indexer/multi.go::MultiIndexer.RepoForFile] } - { id: concept-set-embedder, tier: concept, query: "attach embedding provider to the indexer", expected: [internal/indexer/indexer.go::Indexer.SetEmbedder] } - { id: concept-new-server, tier: concept, query: "construct a new MCP server instance", expected: [internal/mcp/server.go::NewServer] } - - { id: concept-new-hybrid, tier: concept, query: "construct a hybrid BM25+vector backend", expected: [internal/search/hybrid.go::NewHybrid] } + - { id: concept-new-hybrid, tier: concept, query: "construct a hybrid store-native text and vector backend", expected: [internal/search/hybrid.go::NewHybrid] } - { id: concept-reindex-all, tier: concept, query: "incremental reindex after file changes", expected: [internal/indexer/indexer.go::Indexer.IncrementalReindexPaths] } - { id: concept-new-vector, tier: concept, query: "HNSW vector index backend", expected: [internal/search/vector.go::VectorBackend] } - { id: concept-api-embedder, tier: concept, query: "OpenAI-compatible embeddings API provider", expected: [internal/embedding/api.go::APIProvider] } @@ -229,7 +229,7 @@ cases: tier: multi_hop query: "search backend implementations" expected: - - internal/search/bm25.go::BM25Backend + - internal/search/null.go::NullBackend - internal/search/symbolsearcher_backend.go::SymbolSearcherBackend - internal/search/hybrid.go::HybridBackend - internal/search/vector.go::VectorBackend @@ -269,7 +269,7 @@ cases: tier: multi_hop query: "ranker adapters in the eval package" expected: - - internal/eval/recall/rankers.go::BM25Ranker + - internal/eval/recall/rankers.go::EngineRanker - internal/eval/recall/rankers.go::SemanticRanker - internal/eval/recall/rankers.go::RRFRanker - internal/eval/recall/rankers.go::WinnowRanker @@ -410,15 +410,15 @@ cases: - internal/config/config.go::Load - internal/config/config.go::Default - - id: mh-bm25-api + - id: mh-null-backend-api tier: multi_hop - query: "BM25 backend public API" + query: "null search backend public API" expected: - - internal/search/bm25.go::BM25Backend.Add - - internal/search/bm25.go::BM25Backend.Remove - - internal/search/bm25.go::BM25Backend.Search - - internal/search/bm25.go::BM25Backend.Count - - internal/search/bm25.go::NewBM25 + - internal/search/null.go::NullBackend.Add + - internal/search/null.go::NullBackend.Remove + - internal/search/null.go::NullBackend.Search + - internal/search/null.go::NullBackend.Count + - internal/search/null.go::NewNull - id: mh-tool-registrations tier: multi_hop diff --git a/bench/fixtures/retrieval_typo.yaml b/bench/fixtures/retrieval_typo.yaml index d048c6d79..d34c92a3a 100644 --- a/bench/fixtures/retrieval_typo.yaml +++ b/bench/fixtures/retrieval_typo.yaml @@ -45,79 +45,79 @@ cases: query: IndexFileNozResolve expected: - internal/indexer/indexer.go::Indexer.IndexFileNoResolve - - id: exact-IncrementalReindex-typo + - id: exact-IncrementalReindexPaths-typo tier: exact - query: IncrementalPeindex + query: IncrementalReigdexPaths expected: - internal/indexer/indexer.go::Indexer.IncrementalReindexPaths - id: exact-SetEmbedder-typo tier: exact - query: SetEmbeeder + query: SetEmbedde expected: - internal/indexer/indexer.go::Indexer.SetEmbedder - id: exact-EvictFile-typo tier: exact - query: EvictFife + query: EvictFilq expected: - internal/graph/graph.go::Graph.EvictFile - id: exact-EvictRepo-typo tier: exact - query: EvkctRepo + query: EiictRepo expected: - internal/graph/graph.go::Graph.EvictRepo - id: exact-AddNode-typo tier: exact - query: AddNode + query: AddoNde expected: - internal/graph/graph.go::Graph.AddNode - id: exact-AtomicWriteFile-typo tier: exact - query: AtomicnWriteFile + query: AtomicWriteile expected: - internal/agents/writer.go::AtomicWriteFile - id: exact-WriteIfNotExists-typo tier: exact - query: WrilteIfNotExists + query: WriteIfNotExisks expected: - internal/agents/writer.go::WriteIfNotExists - - id: exact-BM25Backend-typo + - id: exact-SymbolSearcherBackend-typo tier: exact - query: BM25Backejd + query: SymbolSearcherackend expected: - - internal/search/bm25.go::BM25Backend - - id: exact-NewBM25-typo + - internal/search/symbolsearcher_backend.go::SymbolSearcherBackend + - id: exact-NewSymbolSearcherBackend-typo tier: exact - query: NenBM25 + query: NewSymboSlearcherBackend expected: - - internal/search/bm25.go::NewBM25 + - internal/search/symbolsearcher_backend.go::NewSymbolSearcherBackend - id: exact-HybridBackend-typo tier: exact - query: HybridBfckend + query: HybridUackend expected: - internal/search/hybrid.go::HybridBackend - id: exact-NewHybrid-typo tier: exact - query: NewHybid + query: NtwHybrid expected: - internal/search/hybrid.go::NewHybrid - id: exact-alphaFuse-typo tier: exact - query: alphaFse + query: agphaFuse expected: - internal/search/hybrid.go::alphaFuse - id: exact-VectorBackend-typo tier: exact - query: VectorBcakend + query: VectorBackedd expected: - internal/search/vector.go::VectorBackend - id: exact-SearchBackend-typo tier: exact - query: search.Backeod + query: sewarch.Backend expected: - internal/search/search.go::Backend - id: exact-SearchResult-typo tier: exact - query: SearchResutl + query: SearchResuzt expected: - internal/search/search.go::SearchResult - id: exact-Swappable-typo @@ -335,26 +335,26 @@ cases: query: drop every node for a rpeository prefix expected: - internal/graph/graph.go::Graph.EvictRepo - - id: concept-bm25-index-typo + - id: concept-store-native-text-typo tier: concept - query: text search backend wtih TF-IDF ranking + query: adapt store-native symbol search itno the search backend expected: - - internal/search/bm25.go::BM25Backend + - internal/search/symbolsearcher_backend.go::SymbolSearcherBackend - id: concept-hybrid-fuse-typo tier: concept - query: comlbine text and vector search with RRF + query: combine store-native text and lvector search with adaptive alpha expected: - internal/search/hybrid.go::HybridBackend - id: concept-adaptive-alpha-typo tier: concept - query: adaptive alpha weighted reciprocal randk fusion + query: adaptive alpha weightded text and vector fusion expected: - internal/search/hybrid.go::alphaFuse - id: concept-swap-backend-typo tier: concept - query: hot-swap the in-memory searich backend + query: atomically replace the vectior channel while preserving the live text backend expected: - - internal/search/swappable.go::Swappable + - internal/search/swappable.go::Swappable.ReplaceHybridVector - id: concept-glove-embed-typo tier: concept query: built-in GloVe owrd vector embedder @@ -510,11 +510,11 @@ cases: query: register every language extactor expected: - internal/parser/languages/register.go::RegisterAll - - id: concept-ranker-bm25-typo + - id: concept-ranker-engine-typo tier: concept - query: eval rnker adapter for BM25 + query: eval rnker adapter for engine search expected: - - internal/eval/recall/rankers.go::BM25Ranker + - internal/eval/recall/rankers.go::EngineRanker - id: concept-ranker-rrf-typo tier: concept query: eval ranker dapter for RRF hybrid @@ -557,7 +557,7 @@ cases: - internal/mcp/server.go::NewServer - id: concept-new-hybrid-typo tier: concept - query: construct a hbyrid BM25+vector backend + query: construct a hybrid stroe-native text and vector backend expected: - internal/search/hybrid.go::NewHybrid - id: concept-reindex-all-typo @@ -672,7 +672,7 @@ cases: tier: multi_hop query: search backend implementatsions expected: - - internal/search/bm25.go::BM25Backend + - internal/search/null.go::NullBackend - internal/search/symbolsearcher_backend.go::SymbolSearcherBackend - internal/search/hybrid.go::HybridBackend - internal/search/vector.go::VectorBackend @@ -708,7 +708,7 @@ cases: tier: multi_hop query: ranker adapters in the eval packapge expected: - - internal/eval/recall/rankers.go::BM25Ranker + - internal/eval/recall/rankers.go::EngineRanker - internal/eval/recall/rankers.go::SemanticRanker - internal/eval/recall/rankers.go::RRFRanker - internal/eval/recall/rankers.go::WinnowRanker @@ -832,15 +832,15 @@ cases: expected: - internal/config/config.go::Load - internal/config/config.go::Default - - id: mh-bm25-api-typo + - id: mh-null-backend-api-typo tier: multi_hop - query: BM25 backend pubilc API + query: null seacrh backend public API expected: - - internal/search/bm25.go::BM25Backend.Add - - internal/search/bm25.go::BM25Backend.Remove - - internal/search/bm25.go::BM25Backend.Search - - internal/search/bm25.go::BM25Backend.Count - - internal/search/bm25.go::NewBM25 + - internal/search/null.go::NullBackend.Add + - internal/search/null.go::NullBackend.Remove + - internal/search/null.go::NullBackend.Search + - internal/search/null.go::NullBackend.Count + - internal/search/null.go::NewNull - id: mh-tool-registrations-typo tier: multi_hop query: MCP tool registration functuons diff --git a/cmd/gortex/daemon_controller.go b/cmd/gortex/daemon_controller.go index 3cd7c02a2..8b7a0aff5 100644 --- a/cmd/gortex/daemon_controller.go +++ b/cmd/gortex/daemon_controller.go @@ -622,14 +622,13 @@ type searchBackendInfo struct { // document count, and its heap footprint. // // Real-world unwrap order: Swappable → HybridBackend → (text, vector). -// The text side is itself a concrete BM25/SymbolSearcherBackend. Both -// layers have to be peeled; if we stop early we fall into the default -// branch and the status reports "unknown" — which was the bug users -// saw. When the store implements graph.SymbolSearcher, the indexer -// wires up a *search.SymbolSearcherBackend instead of building an -// in-process BM25 index at all (see initialSearchBackend in -// internal/indexer/indexer.go) — that case has to be matched -// explicitly too, or it falls into the same "unknown" default. +// Both layers have to be peeled; if we stop early we fall into the +// default branch and the status reports "unknown" — which was the bug +// users saw. The text side is a concrete backend that has to be matched +// explicitly, or it lands in that same "unknown" default: the indexer +// wires up a *search.SymbolSearcherBackend when the store implements +// graph.SymbolSearcher and a *search.NullBackend when it does not (see +// initialSearchBackend in internal/indexer/indexer.go). func resolveSearchBackend(b search.Backend) searchBackendInfo { out := searchBackendInfo{} if b == nil { @@ -659,11 +658,6 @@ func resolveSearchBackend(b search.Backend) searchBackendInfo { } switch back := inner.(type) { - case *search.BM25Backend: - out.Name = "bm25" - out.DocCount = back.Count() - out.DocCountKnown = true - out.Bytes = back.SizeBytes() case *search.SymbolSearcherBackend: // The FTS5 index lives inside the graph store's own file, not a // separate in-memory structure — there is no honest byte count @@ -679,6 +673,15 @@ func resolveSearchBackend(b search.Backend) searchBackendInfo { out.Name = "sqlite-fts5" out.DiskResident = true out.DocCount, out.DocCountKnown = back.DocCount() + case *search.NullBackend: + // A store with no native symbol search gets the null text + // backend: it indexes nothing and the engine falls back to its + // substring path. Say so — an empty backend is a fact worth + // reporting, whereas the "unknown" default would imply we failed + // to recognise whatever is serving queries. Doc count and heap + // stay zero because both are honestly zero. + out.Name = "none" + out.DocCountKnown = true default: out.Name = "unknown" out.DocCount = inner.Count() diff --git a/cmd/gortex/daemon_controller_worktree_test.go b/cmd/gortex/daemon_controller_worktree_test.go index e5d3a2a6a..05d7489cb 100644 --- a/cmd/gortex/daemon_controller_worktree_test.go +++ b/cmd/gortex/daemon_controller_worktree_test.go @@ -67,7 +67,7 @@ func buildWorktreeController(t *testing.T) (*realController, *indexer.MultiIndex g := graph.New() reg := parser.NewRegistry() languages.RegisterAll(reg) - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/cmd/gortex/daemon_enrichment_progress_test.go b/cmd/gortex/daemon_enrichment_progress_test.go index a1a998539..816547a68 100644 --- a/cmd/gortex/daemon_enrichment_progress_test.go +++ b/cmd/gortex/daemon_enrichment_progress_test.go @@ -110,7 +110,8 @@ func TestStatusResponse_EnrichmentJSONRoundTrip(t *testing.T) { // TestSearchBackendStats_DiskResidentJSONRoundTrip locks in that the // DiskResident flag survives the wire round trip and is omitted when -// false (the in-process BM25 backend never sets it). +// false (only the store-native FTS index sets it; a backend reporting a +// heap figure leaves it clear). func TestSearchBackendStats_DiskResidentJSONRoundTrip(t *testing.T) { sb := daemon.SearchBackendStats{Name: "sqlite-fts5", DocCount: 48572, DiskResident: true} raw, err := json.Marshal(sb) @@ -121,8 +122,8 @@ func TestSearchBackendStats_DiskResidentJSONRoundTrip(t *testing.T) { require.NoError(t, json.Unmarshal(raw, &round)) assert.True(t, round.DiskResident) - bm25 := daemon.SearchBackendStats{Name: "bm25", DocCount: 10} - raw2, err := json.Marshal(bm25) + heapResident := daemon.SearchBackendStats{Name: "unknown", DocCount: 10} + raw2, err := json.Marshal(heapResident) require.NoError(t, err) assert.NotContains(t, string(raw2), "disk_resident") } diff --git a/cmd/gortex/daemon_search_backend_test.go b/cmd/gortex/daemon_search_backend_test.go index 6deb00381..d3ebe1a6c 100644 --- a/cmd/gortex/daemon_search_backend_test.go +++ b/cmd/gortex/daemon_search_backend_test.go @@ -40,23 +40,21 @@ func TestResolveSearchBackend_SymbolSearcherBackend(t *testing.T) { assert.Equal(t, "sqlite-fts5", info.Name) assert.True(t, info.DiskResident, "the FTS5 index lives inside the graph store, not in-process heap") assert.Zero(t, info.Bytes, "no fabricated byte count for a disk-resident backend") - // Count() is a since-construction Add/Remove delta, not a corpus size — - // it goes negative as soon as an eviction path drops more than the admit - // predicate ever added. A store that cannot answer the real count must - // leave the figure unreported rather than have the delta stand in for it. - assert.False(t, info.DocCountKnown, "the delta must never be reported as a document count") + // Add and Remove are no-ops because the native store owns the corpus. + // A store that cannot answer the authoritative count must leave the figure + // unreported rather than fabricate an adapter-local document count. + assert.False(t, info.DocCountKnown, "an unavailable count must remain unknown") assert.Zero(t, info.DocCount) } func TestResolveSearchBackend_SymbolSearcherBackend_CountFromIndex(t *testing.T) { - // Adds and removes move the adapter's delta. Count combines that delta - // with the persisted baseline, while status still reports the index's own - // authoritative count. + // Adds and removes are no-ops. Count and status both report the native + // index's authoritative corpus size without duplicating write-path deltas. b := search.NewSymbolSearcherBackend(countingSymbolSearcher{count: 48572}) b.Add("node-1") b.Remove("node-2") b.Remove("node-3") - assert.Equal(t, 48571, b.Count(), "Count must combine the persisted baseline with the adapter delta") + assert.Equal(t, 48572, b.Count(), "Count must come from the native index") info := resolveSearchBackend(b) @@ -74,6 +72,29 @@ func TestResolveSearchBackend_SymbolSearcherBackend_ThroughSwappable(t *testing. assert.True(t, info.DiskResident) } +func TestResolveSearchBackend_NullBackend(t *testing.T) { + // A store with no native symbol search carries the null text backend. + // Status must name it, not report it as an unrecognised backend: the + // difference between "nothing is indexing text here" and "we could not + // identify what is" is exactly what a user reads this row for. + info := resolveSearchBackend(search.NewNull()) + + assert.Equal(t, "none", info.Name) + assert.False(t, info.DiskResident, "there is no index on disk either") + assert.Zero(t, info.Bytes) + assert.True(t, info.DocCountKnown, "zero documents is a known count, not an unanswerable one") + assert.Zero(t, info.DocCount) +} + +func TestRenderDaemonHeader_SearchBackendRow_NullBackend(t *testing.T) { + st := sampleStatus() + st.SearchBackend = daemon.SearchBackendStats{Name: "none", DocCountKnown: true} + var buf bytes.Buffer + renderDaemonHeader(&buf, st) + assert.Contains(t, buf.String(), "none docs=0 heap=0 B", + "an empty backend still gets a row, with its zeros stated plainly") +} + func TestRenderDaemonHeader_SearchBackendRow_SymbolSearcher(t *testing.T) { st := sampleStatus() st.SearchBackend = daemon.SearchBackendStats{ diff --git a/cmd/gortex/daemon_status_render_test.go b/cmd/gortex/daemon_status_render_test.go index b437e6257..a20ed2215 100644 --- a/cmd/gortex/daemon_status_render_test.go +++ b/cmd/gortex/daemon_status_render_test.go @@ -101,9 +101,12 @@ func TestRenderDaemonHeader_SearchBackendRow(t *testing.T) { func TestRenderDaemonHeader_SearchBackendRow_HeapBackend(t *testing.T) { st := sampleStatus() - // The in-process BM25 index does have a heap footprint to report. + // The other arm of the row: a backend that is not disk-resident, so + // the heap figure it does report has to be printed. resolveSearchBackend + // reaches it for a backend it cannot identify, where the byte count + // comes from search.BackendSize. st.SearchBackend = daemon.SearchBackendStats{ - Name: "bm25", + Name: "unknown", DocCount: 12000, DocCountKnown: true, Bytes: 200 * 1024 * 1024, @@ -111,7 +114,7 @@ func TestRenderDaemonHeader_SearchBackendRow_HeapBackend(t *testing.T) { var buf bytes.Buffer renderDaemonHeader(&buf, st) out := buf.String() - assert.Contains(t, out, "bm25") + assert.Contains(t, out, "unknown") assert.Contains(t, out, "12000") assert.Contains(t, out, "heap=") } diff --git a/cmd/gortex/daemon_status_repo_counts_test.go b/cmd/gortex/daemon_status_repo_counts_test.go index 19c601d4b..9cb431a04 100644 --- a/cmd/gortex/daemon_status_repo_counts_test.go +++ b/cmd/gortex/daemon_status_repo_counts_test.go @@ -53,7 +53,7 @@ func TestStatus_TrackedRepoRowFollowsTheLiveGraph(t *testing.T) { g := graph.New() reg := parser.NewRegistry() languages.RegisterAll(reg) - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -129,7 +129,7 @@ func TestStatus_TwoReposEachReportTheirOwnBucket(t *testing.T) { // A is indexed while it is the only configured repo. g := graph.New() - mi1 := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi1 := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi1.IndexAll() require.NoError(t, err) var priorA map[string]int64 @@ -141,7 +141,7 @@ func TestStatus_TwoReposEachReportTheirOwnBucket(t *testing.T) { // B is added while the daemon is "down", then a warm restart reconciles // A from its persisted mtimes and tracks B fresh. require.NoError(t, cm.Global().AddRepo(config.RepoEntry{Path: dirB})) - mi2 := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi2 := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi2.ReconcileRepoCtx(context.Background(), config.RepoEntry{Path: dirA}, priorA) require.NoError(t, err) _, err = mi2.TrackRepoCtx(context.Background(), config.RepoEntry{Path: dirB}) diff --git a/cmd/gortex/eval_recall.go b/cmd/gortex/eval_recall.go index 414d8cd0f..73e119ab0 100644 --- a/cmd/gortex/eval_recall.go +++ b/cmd/gortex/eval_recall.go @@ -53,17 +53,19 @@ returned. Cases are tiered (exact / concept / multi_hop) so per-tier recall is broken out separately. Rankers: - bm25 — text-only (default text backend) + bm25 — lexical-only: the store-native symbol FTS queried + through Engine.SearchSymbols (the search_symbols path) semantic — vector-only (requires --embeddings) - rrf — BM25 + vector fused via RRF (requires --embeddings) + rrf — native FTS + vector with adaptive alpha (historical key; + requires --embeddings) winnow — graph-aware constraint chain (MCP winnow_symbols scorer) ripgrep — rg --files-with-matches baseline ("retrieval floor") By default every available ranker runs; narrow with --rankers bm25,rrf. -Without --embeddings, semantic and RRF degrade: semantic reports SKIPPED -and RRF falls back to BM25 inside HybridBackend.Search. Use --embeddings -to enable the built-in static (GloVe) provider, or --embeddings-url to +Without --embeddings, semantic and RRF both report SKIPPED. Use +--embeddings to enable the built-in static (GloVe) provider, or +--embeddings-url to point at an OpenAI-compatible API (e.g. Ollama).`, RunE: runEvalRecall, } @@ -152,7 +154,8 @@ func runEvalRecall(_ *cobra.Command, _ []string) error { // Peel the Swappable wrapper so we can see the real backend. When // embeddings are on, the indexer builds a HybridBackend internally; - // its TextBackend() is the pure lexical side, and the whole + // its TextBackend() is the pure lexical side — an adapter over the + // store's own symbol FTS, not a harness-built index — and the whole // HybridBackend is what RRF queries. inner := idx.Search() if sw, ok := inner.(*search.Swappable); ok { @@ -181,10 +184,12 @@ func runEvalRecall(_ *cobra.Command, _ []string) error { } // The engine-backed lexical row mirrors the MCP search_symbols call - // path (BM25 ranking + substring fallback for camelCase-only queries) - // against the store's own full-text index — the same backend the - // daemon serves, so the row reports what real callers hit rather than - // a harness-only in-process index. Critically: the engine is pointed + // path: the store-native FTS ranking plus the substring fallback for + // camelCase-only queries, run through Engine.SearchSymbols against + // the store's own full-text index — the same backend the daemon + // serves, so the row reports what real callers hit. The "bm25" row + // key is kept only so historical bench artifacts stay joinable; no + // in-process BM25 index is involved. Critically: the engine is pointed // at the PURE text backend even when --embeddings is on — otherwise // the "bm25" row would silently run through HybridBackend.Search (RRF // fusion with vector results) and the measurement would no longer diff --git a/cmd/gortex/eval_stdbench.go b/cmd/gortex/eval_stdbench.go index 24796ede2..aa91602a6 100644 --- a/cmd/gortex/eval_stdbench.go +++ b/cmd/gortex/eval_stdbench.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/cobra" "github.com/zzet/gortex/internal/eval/stdbench" + "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/search" ) @@ -22,8 +23,9 @@ var ( var evalStdbenchCmd = &cobra.Command{ Use: "stdbench", Short: "Run a standardized retrieval benchmark (CoIR / SWE-ContextBench / ContextBench)", - Long: `Runs Gortex's BM25 text retrieval against a standardized code-retrieval -benchmark and reports Recall@K, Precision@K, NDCG@10, and MRR. + Long: `Runs Gortex's shipping text retrieval — the store-native symbol FTS +search_symbols queries — against a standardized code-retrieval benchmark +and reports Recall@K, Precision@K, NDCG@10, and MRR. Benchmarks (--bench): coir CoIR (Code Information Retrieval, ACL 2025). --dataset is @@ -84,22 +86,76 @@ func runEvalStdbench(_ *cobra.Command, _ []string) error { return fmt.Errorf("benchmark %q carries no relevance judgements to score against", evalStdbenchBench) } - // Index the corpus into Gortex's BM25 backend — the same text - // retrieval search_symbols runs — and rank doc IDs per query. - bm := search.NewBM25() + // Index the corpus into a throwaway SQLite store's native symbol FTS — + // the text retrieval search_symbols actually runs — and rank doc IDs + // per query. Note the corpus goes in through the store's FTS writer + // directly, never through search.Backend.Add: the shipping backend is + // an adapter over this same FTS whose Add is a no-op, so routing the + // corpus through it would index nothing and report a silent 0.0. + st, closeStore, err := newEvalStore("stdbench") + if err != nil { + return fmt.Errorf("opening the eval store: %w", err) + } + defer closeStore() + + searcher, ok := st.(graph.SymbolSearcher) + if !ok { + return fmt.Errorf("the eval store does not expose native symbol search") + } + counter, ok := st.(graph.SymbolFTSCounter) + if !ok { + return fmt.Errorf("the eval store does not report its symbol FTS document count") + } + + items := make([]graph.SymbolFTSItem, 0, len(ds.Corpus)) for _, d := range ds.Corpus { - bm.Add(d.ID, d.Text) + // Mirrors the indexer's ftsTokensFor: the write side splits with + // search.Tokenize so the read side's query tokenisation lands on + // the same terms. + tokens := search.NormalizeFTSTokens(search.Tokenize(d.Text)) + items = append(items, graph.SymbolFTSItem{ + NodeID: d.ID, + Tokens: strings.Join(tokens, " "), + }) + } + // One call, not a chunked loop: BulkUpsertSymbolFTS wipes the prefix + // before inserting, so a second call under the same prefix would erase + // the first. It bounds its own INSERT statements internally. + if err := searcher.BulkUpsertSymbolFTS("", items); err != nil { + return fmt.Errorf("indexing the %s corpus: %w", ds.Name, err) + } + if err := searcher.BuildSymbolIndex(); err != nil { + return fmt.Errorf("building the symbol index: %w", err) } + indexed, err := counter.SymbolFTSCount() + if err != nil { + return fmt.Errorf("counting the indexed corpus: %w", err) + } + if indexed != len(ds.Corpus) { + return fmt.Errorf("indexed %d of %d corpus documents — scoring a partial corpus "+ + "would report a retrieval number no user can reproduce", indexed, len(ds.Corpus)) + } + + var retrieveErr error retrieve := func(query string, k int) []string { - hits := bm.Search(query, k) + hits, err := searcher.SearchSymbols(query, k) + if err != nil { + if retrieveErr == nil { + retrieveErr = err + } + return nil + } ids := make([]string, 0, len(hits)) for _, h := range hits { - ids = append(ids, h.ID) + ids = append(ids, h.NodeID) } return ids } metrics := stdbench.Evaluate(ds, retrieve, nil) + if retrieveErr != nil { + return fmt.Errorf("searching the corpus: %w", retrieveErr) + } var rendered string if strings.EqualFold(evalStdbenchFormat, "json") { diff --git a/cmd/gortex/repo_inventory_missing_test.go b/cmd/gortex/repo_inventory_missing_test.go index 3d235ac28..53ff7d2cc 100644 --- a/cmd/gortex/repo_inventory_missing_test.go +++ b/cmd/gortex/repo_inventory_missing_test.go @@ -116,7 +116,7 @@ func newInventoryController(t *testing.T, repos []config.RepoEntry) *realControl g := graph.New() reg := parser.NewRegistry() languages.RegisterAll(reg) - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, _ = mi.IndexAll() return &realController{graph: g, multiIndexer: mi, configManager: cm, logger: zap.NewNop()} diff --git a/docs/04-evaluation/task-set.md b/docs/04-evaluation/task-set.md index b966a449f..1af46a5a9 100644 --- a/docs/04-evaluation/task-set.md +++ b/docs/04-evaluation/task-set.md @@ -33,9 +33,11 @@ order of operations." `parser.ExtractionResult`; `Indexer.processExtraction` writes them into the `graph.Graph` and accumulates incoming-edge tracking for the next phase. -4. `Indexer.buildSearchIndex` (in-process BM25 in tests and evals, - store-native FTS in production) + `idx.embedder` (if set) - populate the search backends. +4. The backing store owns the symbol FTS corpus: shadow drains bulk-publish it, + direct-store passes rebuild it, and incremental mutations upsert or delete + affected rows. The vector pipeline prepares an immutable plan with + `Indexer.prepareSearchIndexForPublication`, then publishes it through + `Indexer.installVectorPlan` after the durable shadow drain succeeds. 5. Semantic enrichment (`internal/semantic`) runs LSP / SCIP providers in parallel; resolved edges get `Origin=lsp_resolved` for tier filtering. diff --git a/docs/features.md b/docs/features.md index 0d1d9f74f..1ad693429 100644 --- a/docs/features.md +++ b/docs/features.md @@ -34,7 +34,7 @@ The full surface, grouped by concern. Each item links to the deeper reference wh ## Search & navigation -- **Semantic search default-on** — hybrid BM25 + vector with RRF fusion, baked GloVe-50d (~3.8 MB embedded in the binary, top 20k tokens), CPU-only, zero native deps. Large symbols are split into AST-aware windows and de-chunked at query time. Equivalence-class vocabulary expansion bridges `auth ≈ authentication ≈ login` without an LLM. A HITS authority/hub signal feeds the rerank pipeline. A keyword-soup query defense detects degenerate OR-soup *and* operator-free phrasing and skips wasted LLM expansion. Markdown documentation is a first-class corpus with its own retrieval channel and prose-tuned ranking. Opt-in `embedding.provider: local` (Hugot MiniLM-L6-v2) or `api` (Ollama / OpenAI). See [semantic-search.md](semantic-search.md). +- **Semantic search default-on** — store-native FTS5/BM25 + vector with adaptive alpha fusion, baked GloVe-50d (~3.8 MB embedded in the binary, top 20k tokens), CPU-only, zero native deps. Large symbols are split into AST-aware windows and de-chunked at query time. Equivalence-class vocabulary expansion bridges `auth ≈ authentication ≈ login` without an LLM. A HITS authority/hub signal feeds the rerank pipeline. A keyword-soup query defense detects degenerate OR-soup *and* operator-free phrasing and skips wasted LLM expansion. Markdown documentation is a first-class corpus with its own retrieval channel and prose-tuned ranking. Opt-in `embedding.provider: local` (Hugot MiniLM-L6-v2) or `api` (Ollama / OpenAI). See [semantic-search.md](semantic-search.md). - **Provenance-aware ranking** — the BM25↔vector balance is scored continuously from query shape; edge-resolution provenance attenuates LSP-inflated framework wiring in centrality and rerank; generated files are ranked below a real same-named implementation; the implementation is lifted above its own test; and a post-rerank pass recovers exact embedding cosine. Zero-result identifier queries are auto-decomposed into leaf terms. - **`context_closure`** — given seed files/symbols, walks the transitive import/dependency closure and packs it under one `token_budget`, ranked by graph distance or seeded random-walk proximity. - **Code search beyond symbols** — `search_text` is a trigram-indexed literal/regex search; `search_ast` runs structural tree-sitter queries; `analyze kind=sast` is a 190-rule, CWE/OWASP-tagged security scan across 8 languages. diff --git a/docs/semantic-search.md b/docs/semantic-search.md index 87e4c186e..6ee9de579 100644 --- a/docs/semantic-search.md +++ b/docs/semantic-search.md @@ -1,6 +1,6 @@ # Semantic search -**Default-on.** A baked GloVe-50d table (~3.8 MB embedded in the binary, top 20k tokens) gives every install hybrid BM25 + vector search out of the box — no flag, no model download, no native dependency. Reciprocal Rank Fusion blends the two channels, and the BM25↔vector balance is scored *continuously* from the query's shape (identifier density, separators, stopwords) rather than bucketed into a discrete class — so a half-identifier query lands between the symbol and natural-language blends instead of jumping a whole tier. After ranking, an optional pure-cosine refinement pass re-scores the top results with the exact embedding distance the rank-based fusion discards. +**Default-on.** A baked GloVe-50d table (~3.8 MB embedded in the binary, top 20k tokens) gives every install hybrid store-native FTS5/BM25 + vector search out of the box — no flag, no model download, no native dependency. Adaptive alpha-weighted rank fusion blends the two channels, and the text↔vector balance is scored *continuously* from the query's shape (identifier density, separators, stopwords) rather than bucketed into a discrete class — so a half-identifier query lands between the symbol and natural-language blends instead of jumping a whole tier. After ranking, an optional pure-cosine refinement pass re-scores the top results with the exact embedding distance the rank-based fusion discards. ## Configuration @@ -63,13 +63,9 @@ Centrality (HITS + PageRank) and a dedicated rerank signal weight call/reference - **Generated-file demotion** — a generated file (`*.pb.go`, `mock_*.go`, `*_pb2.py`, …) is ranked below a real same-named hand-written implementation, but only when one exists. - **Source over test** — when a query surfaces both an implementation and its test, the implementation is lifted above the test (only when both co-occur, so it never shifts the rest of the page). -### Sparse sub-word tokenization (opt-in) - -An optional tokenizer stage emits sub-word n-grams whose split points come from a per-repo boundary table learned from symbol names at index time, trading exact-identifier precision for recall on typo/fragment queries. Off by default (it is reindex-required and precision-sensitive); enable with `GORTEX_SPARSE_NGRAM=1`. Applies to the BM25 backend. - ## Keyword-soup defense -Boolean / OR-soup queries (`A OR B OR 'no access' OR …`) — and operator-free keyword lists (`parse decode unmarshal token jwt cache`) and comma-enumerations — defeat embedding retrieval. The query classifier detects all three, skips wasted LLM expansion, and splits the soup into terms fused via the existing BM25 expansion path; a `query_advice` nudge rides on the response. Genuine natural-language questions stay classified as concept. Tune via `search.keyword_soup_rewrite: split | nudge | off`. +Boolean / OR-soup queries (`A OR B OR 'no access' OR …`) — and operator-free keyword lists (`parse decode unmarshal token jwt cache`) and comma-enumerations — defeat embedding retrieval. The query classifier detects all three, skips wasted LLM expansion, and splits the soup into terms fused via the store-native lexical expansion path; a `query_advice` nudge rides on the response. Genuine natural-language questions stay classified as concept. Tune via `search.keyword_soup_rewrite: split | nudge | off`. ## Prose corpus @@ -115,7 +111,7 @@ Semantic search degrading to text-only (BM25 / FTS5) is always logged — match - `auto` (default) — skips LLM for identifier queries, expands NL queries - `on` — forces expansion + rerank -- `off` — pure BM25 +- `off` — store-native FTS5/BM25 only - `deep` — adds a body-grounded verification pass; +1.5–4 s; quality is highly model-dependent — unreliable on 3B local models, fine on 7B+ or hosted See [llm.md](llm.md) for provider configuration. diff --git a/go.sum b/go.sum index c57fded8b..b09cd001b 100644 --- a/go.sum +++ b/go.sum @@ -450,8 +450,6 @@ github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= -github.com/charmbracelet/x/ansi v0.11.7 h1:kzv1kJvjg2S3r9KHo8hDdHFQLEqn4RBCb39dAYC84jI= -github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9No6p1epAihUZwQ= github.com/charmbracelet/x/ansi v0.11.8 h1:JMFwp0CgDC2+jcOB162HH5k7I3FVbgFSMMYg7dSPBQQ= github.com/charmbracelet/x/ansi v0.11.8/go.mod h1:ZNN+3mXny/516oTQPLMPIBeSINvNJJQ8uQXDgbeJxY0= github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI= @@ -573,8 +571,6 @@ github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 h1:QwWKgMY28TAXaDl+ github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728/go.mod h1:1fEHWurg7pvf5SG6XNE5Q8UZmOwex51Mkx3SLhrW5B4= github.com/lucasb-eyer/go-colorful v1.4.1 h1:1EO+WB73+EH8EVbzlrG3KLAfEypQWVHIBqlTf+2hNss= github.com/lucasb-eyer/go-colorful v1.4.1/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/mark3labs/mcp-go v0.57.0 h1:jzWKyCzdWnwnZt05cvcQQ+ngiUl2RnixXJa7Kj4qP1E= -github.com/mark3labs/mcp-go v0.57.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas= github.com/mark3labs/mcp-go v0.58.0 h1:AWfBk8lgRR0KZYve7PaLbR2MIjpw1oK2eGpBApaNS+Q= github.com/mark3labs/mcp-go v0.58.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= @@ -617,8 +613,6 @@ github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88ee github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= github.com/sahilm/fuzzy v0.1.3 h1:juByESSS32nVD81vr6tHmKmA/8zde7gE+x5CLxrzXPU= github.com/sahilm/fuzzy v0.1.3/go.mod h1:au6//VbVSqu6DFrkL2CfjlJ5iURpNCPeE+1GwY3XsT8= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc= @@ -690,8 +684,6 @@ github.com/tree-sitter/tree-sitter-ruby v0.23.1 h1:T/NKHUA+iVbHM440hFx+lzVOzS4dV github.com/tree-sitter/tree-sitter-ruby v0.23.1/go.mod h1:kUS4kCCQloFcdX6sdpr8p6r2rogbM6ZjTox5ZOQy8cA= github.com/tree-sitter/tree-sitter-rust v0.24.2 h1:NL4nF67ib21RMzzfvkmXlVwe45vvhW10DVyO+D0z/W0= github.com/tree-sitter/tree-sitter-rust v0.24.2/go.mod h1:hfeGWic9BAfgTrc7Xf6FaOAguCFJRo3RBbs7QJ6D7MI= -github.com/tree-sitter/tree-sitter-scala v0.26.0 h1:hpn0hO6cGtAAC9aqyVlp9HDGq9EexUa+6IzcDP6wR44= -github.com/tree-sitter/tree-sitter-scala v0.26.0/go.mod h1:BmDV0f9rgsnGuG9QtKXQZnqJvECyR9fM8wVg984ulBo= github.com/tree-sitter/tree-sitter-scala v0.26.2 h1:ue3zsxnZzKvCAlBqOxIecqgI+NZjjdvrY+hQ8OA3K0Q= github.com/tree-sitter/tree-sitter-scala v0.26.2/go.mod h1:BmDV0f9rgsnGuG9QtKXQZnqJvECyR9fM8wVg984ulBo= github.com/tree-sitter/tree-sitter-typescript v0.23.2 h1:/Odvphn18PniVixb9e97X0DbNVsU6Qocv9mfkyzdXwU= @@ -702,12 +694,8 @@ github.com/viterin/partial v1.1.0 h1:iH1l1xqBlapXsYzADS1dcbizg3iQUKTU1rbwkHv/80E github.com/viterin/partial v1.1.0/go.mod h1:oKGAo7/wylWkJTLrWX8n+f4aDPtQMQ6VG4dd2qur5QA= github.com/viterin/vek v0.4.3 h1:cogdlNjd6EJYtNbmTN0lJCey2htrfSo1AHWpc6DVncQ= github.com/viterin/vek v0.4.3/go.mod h1:A4JRAe8OvbhdzBL5ofzjBS0J29FyUrf95tQogvtHHUc= -github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= -github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xo/terminfo v1.0.0 h1:2ZpYzqWzyyytjk3TP6aJVDhkMAkc99/1xKQdA3TDTBY= github.com/xo/terminfo v1.0.0/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= -github.com/yalue/onnxruntime_go v1.32.0 h1:O4pPw3IT+46CRrfuT0lcHWkczVoFvtfq6kMAO/iIVKc= -github.com/yalue/onnxruntime_go v1.32.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4= github.com/yalue/onnxruntime_go v1.32.1 h1:bjvmp0ONztctKBOkaU6hgphVoKPH3GTZAJvFfPHdtcI= github.com/yalue/onnxruntime_go v1.32.1/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= @@ -727,20 +715,12 @@ go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= -golang.org/x/exp v0.0.0-20260727155853-b88d891fe743 h1:ex206bKw+v3K0dm3andkrIF+ijyQKJG1pLgwQ2PYdQM= -golang.org/x/exp v0.0.0-20260727155853-b88d891fe743/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q= golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY= golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk= -golang.org/x/image v0.44.0 h1:+tDekMZED9+LrtB3G5xzRggpVh9CARjZqROla3R3R+I= -golang.org/x/image v0.44.0/go.mod h1:V8K3KE9KKKE+pLpQDOeN18w9oacNSvy1tDOirTu4xtY= golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= -golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= -golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= @@ -750,18 +730,12 @@ golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= -golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= -golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= gonum.org/v1/plot v0.15.2 h1:Tlfh/jBk2tqjLZ4/P8ZIwGrLEWQSPDLRm/SNWKNXiGI= gonum.org/v1/plot v0.15.2/go.mod h1:DX+x+DWso3LTha+AdkJEv5Txvi+Tql3KAGkehP0/Ubg= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -772,29 +746,22 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= -modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= -modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= -modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= -modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= +modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= -modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= -modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= modernc.org/libc v1.75.3 h1:vCqT5+R0jPXMnvMkGo0T2zXvFNth+lYXVCx5X7CCX/g= modernc.org/libc v1.75.3/go.mod h1:MjAX68G+0oufI+hNuh0QXcK+Ap+sL8bNPPcIC6EqOfo= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= -modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= -modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= modernc.org/memory v1.12.0 h1:twkmYNkGXCvtYWzoux02jtK6eovjZbdI0uHFUYp6kuU= modernc.org/memory v1.12.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= diff --git a/internal/config/config.go b/internal/config/config.go index 59436fb6e..9e9661eaa 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -589,15 +589,14 @@ type IndexConfig struct { // SkipSearch is the effective text-index skip rules resolved from // Semantic.SkipSearch, same propagation pattern as SkipEmbed. // Users configure this under semantic.skip_search; the indexer - // reads it here. Controls what goes into the text search index - // (in-process BM25 in tests and evals, store-native FTS in - // production) — unlike SkipEmbed it doesn't affect the graph or + // reads it here. Controls what goes into the store-native FTS + // text index — unlike SkipEmbed it doesn't affect the graph or // vector index. SkipSearch []SkipEmbedRule `mapstructure:"-" yaml:"-"` // IndexProse is the effective prose-indexing toggle resolved from // Search.IndexProse -- same `-` (not on-disk) propagation pattern // as SkipSearch. When false, Markdown KindDoc prose-section nodes - // are kept out of the BM25 search index. Defaults to true. + // are kept out of the text search index. Defaults to true. IndexProse bool `mapstructure:"-" yaml:"-"` // MaxFileSize skips files larger than this during indexing. Zero // (the default) disables the cap — full coverage is preferred so diff --git a/internal/config/manager.go b/internal/config/manager.go index 68894778e..e0b181e63 100644 --- a/internal/config/manager.go +++ b/internal/config/manager.go @@ -299,11 +299,10 @@ func (cm *ConfigManager) GetRepoConfig(repoPrefix string) *Config { out.Index.SkipEmbed = DefaultSkipEmbed() } // Same plumbing for semantic.skip_search — controls what goes into - // the text search index (in-process BM25 in tests and evals, - // store-native FTS in production). Separate from SkipEmbed so users can - // tune the two filters independently (e.g. a tiny-repo user who - // doesn't care about text-index memory can clear SkipSearch while - // keeping SkipEmbed's embedding-cost savings). + // the store-native text search index. Separate from SkipEmbed so + // users can tune the two filters independently (e.g. a tiny-repo + // user who doesn't care about text-index size can clear SkipSearch + // while keeping SkipEmbed's embedding-cost savings). if len(out.Semantic.SkipSearch) > 0 { out.Index.SkipSearch = out.Semantic.SkipSearch } else { diff --git a/internal/daemon/proto.go b/internal/daemon/proto.go index a097fe83b..f34cf3882 100644 --- a/internal/daemon/proto.go +++ b/internal/daemon/proto.go @@ -468,11 +468,13 @@ type TrigramCacheStats struct { // SearchBackendStats identifies which search backend is currently // serving queries, so users can read the `search_b` column in the -// repo breakdown with the right mental model. The in-process BM25 -// index costs ~2 KiB of heap per document; the store-native FTS index -// lives inside the graph store's own file and costs no heap of its own. +// repo breakdown with the right mental model. The store-native FTS +// index lives inside the graph store's own file and costs no heap of +// its own. "none" means the store exposes no native symbol search, so +// no text index is serving queries at all and the engine answers from +// its substring fallback. type SearchBackendStats struct { - Name string `json:"name"` // "bm25" | "sqlite-fts5" | "unknown" + Name string `json:"name"` // "sqlite-fts5" | "none" | "unknown" DocCount int `json:"doc_count"` // indexed documents across all repos // DocCountKnown distinguishes "the index holds zero documents" from // "this backend cannot report a document count". Backends whose only diff --git a/internal/eval/recall/rankers.go b/internal/eval/recall/rankers.go index de11fcaf8..18df3cbb7 100644 --- a/internal/eval/recall/rankers.go +++ b/internal/eval/recall/rankers.go @@ -14,37 +14,15 @@ import ( "github.com/zzet/gortex/internal/search" ) -// BM25Ranker adapts a plain search.Backend to the Ranker shape. Works -// for either a raw text backend — the in-process BM25 index the evals -// build, or the store-native FTS adapter production wires up — or a -// HybridBackend's text side extracted via HybridBackend.TextBackend(). -// -// Note: Gortex's indexer tokenizes symbol names at ingest time -// (Tokenize — camelCase-aware), but the query side (TokenizeQuery) does -// NOT split camelCase — so `backend.Search("NewServer", ...)` matches -// zero documents because the inverted index has `new` and `server` -// separately. The user-facing MCP search_symbols path avoids this by -// running through Engine.SearchSymbols, which adds a substring -// fallback. Use EngineRanker (below) to measure that full call path; -// use BM25Ranker only when you want the raw backend's behaviour. -func BM25Ranker(name string, backend search.Backend) Ranker { - return Ranker{ - Name: name, - Search: func(query string, limit int) []string { - hits := backend.Search(query, limit) - out := make([]string, len(hits)) - for i, h := range hits { - out[i] = h.ID - } - return out - }, - } -} - // EngineRanker measures what a real MCP caller sees via -// Engine.SearchSymbols — text-backend results + camelCase-friendly -// substring fallback. This is the recommended default for "bm25"- -// style evaluation; it reflects production behaviour. +// Engine.SearchSymbols — store-native FTS results plus the +// camelCase-friendly substring fallback. It is the only lexical +// adapter this package ships, and it backs the report's "bm25" row. +// +// The fallback is load-bearing: the indexer tokenizes symbol names at +// ingest time (camelCase-aware), but the query side does not split +// camelCase, so a raw backend query for "NewServer" matches nothing +// while Engine.SearchSymbols still finds it. func EngineRanker(name string, searchFn func(query string, limit int) []string) Ranker { return Ranker{Name: name, Search: searchFn} } @@ -106,9 +84,9 @@ func SemanticRanker(name string, vector *search.VectorBackend, embedder embeddin } } -// RRFRanker adapts a HybridBackend (BM25 + vector fused via RRF) to the -// Ranker shape. Uses Search() which runs both sides and fuses when the -// vector backend has data — otherwise it degrades to BM25 gracefully. +// RRFRanker preserves the historical eval row key while adapting the live +// HybridBackend (native symbol FTS + vector with adaptive-alpha fusion) to the +// Ranker shape. Callers install it only when the vector backend has data. func RRFRanker(name string, hybrid *search.HybridBackend) Ranker { return Ranker{ Name: name, diff --git a/internal/eval/recall/recall.go b/internal/eval/recall/recall.go index e824d1867..aee4daab9 100644 --- a/internal/eval/recall/recall.go +++ b/internal/eval/recall/recall.go @@ -7,18 +7,18 @@ // Recall is reported as any-hit set-level recall: a retrieval counts as // correct at rank K if *any* of the Expected IDs for a case appears in // the ranker's top-K results. Multiple Expected IDs per case are OK — -// they represent valid alternative targets (e.g. a type and its -// constructor both being reasonable answers to "BM25 backend"). +// they represent valid alternative targets (e.g. a backend type and its +// constructor both being reasonable answers to "store-native text search"). // // Cases are tiered so per-tier weakness is visible: // // - exact: symbol-name queries. Tests the basic "can you find a -// named symbol I already know about" case. BM25 should -// dominate here; a retrieval tool that can't ace exact -// tier is broken. +// named symbol I already know about" case. Store-native lexical +// retrieval should dominate here; a retrieval tool that can't ace +// exact tier is broken. // - concept: natural-language paraphrase queries. Tests semantic -// understanding. This is where BM25 starts losing to -// semantic / RRF. +// understanding. This is where lexical retrieval starts losing to +// vector search and adaptive fusion. // - multi_hop: relational queries accepting several valid expected // IDs (any-hit). Tests graph-aware retrieval. // diff --git a/internal/eval/recall/recall_test.go b/internal/eval/recall/recall_test.go index c687ca31a..6f8841564 100644 --- a/internal/eval/recall/recall_test.go +++ b/internal/eval/recall/recall_test.go @@ -5,8 +5,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - - "github.com/zzet/gortex/internal/search" ) // staticRanker returns a predetermined ranked list regardless of query. @@ -135,26 +133,6 @@ func TestMarkdownSkippedRankerRow(t *testing.T) { assert.Contains(t, md, "skipped: no embedder") } -// TestBM25Ranker_AgainstRealBackend wires the adapter to a live BM25 -// backend and spot-checks ranked output. -func TestBM25Ranker_AgainstRealBackend(t *testing.T) { - backend := search.NewBM25() - backend.Add("pkg/a.go::Foo", "Foo", "pkg/a.go", "") - backend.Add("pkg/b.go::Bar", "Bar", "pkg/b.go", "") - - r := BM25Ranker("bm25", backend) - hits := r.Search("Foo", 5) - assert.NotEmpty(t, hits) - assert.Equal(t, "pkg/a.go::Foo", hits[0]) - - fixture := Fixture{Cases: []Case{ - {Query: "Bar", Tier: TierExact, Expected: []string{"pkg/b.go::Bar"}}, - {Query: "Foo", Tier: TierExact, Expected: []string{"pkg/a.go::Foo"}}, - }} - report := Run(fixture, []Ranker{r}, nil) - assert.Equal(t, 2, report.Rankers[0].Hits[1]) -} - func TestAdaptCasesForFileRanker(t *testing.T) { in := []Case{ {Query: "q", Expected: []string{"a/b.go::Foo", "a/b.go::Bar", "c/d.go::Baz"}}, diff --git a/internal/eval/stdbench/dataset.go b/internal/eval/stdbench/dataset.go index 74de7806e..a993b2dae 100644 --- a/internal/eval/stdbench/dataset.go +++ b/internal/eval/stdbench/dataset.go @@ -4,9 +4,9 @@ // with the textbook Recall@K / Precision@K / NDCG@K / MRR metrics. // // The loaders parse the on-disk formats; the actual retrieval is left -// to the caller (the `gortex eval stdbench` verb wires Gortex's BM25 -// backend in), so the same harness measures whatever retriever is -// handed to Evaluate. +// to the caller (the `gortex eval stdbench` verb wires in the +// store-native symbol FTS search_symbols runs), so the same harness +// measures whatever retriever is handed to Evaluate. package stdbench // Doc is one corpus document — a code snippet, file, or symbol the diff --git a/internal/eval/stdbench/eval_test.go b/internal/eval/stdbench/eval_test.go index 24d26fe38..fb6df6e1e 100644 --- a/internal/eval/stdbench/eval_test.go +++ b/internal/eval/stdbench/eval_test.go @@ -2,10 +2,14 @@ package stdbench import ( "math" + "path/filepath" + "strings" "testing" "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" "github.com/zzet/gortex/internal/search" ) @@ -53,28 +57,54 @@ func TestEvaluate_PerfectRanking(t *testing.T) { require.InDelta(t, 1.0, m.NDCGAt10, 1e-9) } -// TestEvaluate_EndToEndBM25 runs the full evaluation path: load a JSONL -// benchmark, index its corpus into Gortex's real BM25 backend, and -// score retrieval — the same wiring `gortex eval stdbench` uses. -func TestEvaluate_EndToEndBM25(t *testing.T) { +// TestEvaluate_EndToEndFTS runs the full evaluation path: load a JSONL +// benchmark, index its corpus into a real SQLite store's native symbol +// FTS, and score retrieval — the same wiring `gortex eval stdbench` +// uses, which is the same text path search_symbols serves. +func TestEvaluate_EndToEndFTS(t *testing.T) { ds, err := LoadContextBench("testdata/contextbench.jsonl") require.NoError(t, err) - bm := search.NewBM25() + store, err := store_sqlite.Open(filepath.Join(t.TempDir(), "stdbench.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + + items := make([]graph.SymbolFTSItem, 0, len(ds.Corpus)) for _, d := range ds.Corpus { - bm.Add(d.ID, d.Text) + // Same splitter the indexer's write side uses, so the read side's + // query tokenisation lands on the same terms. + items = append(items, graph.SymbolFTSItem{ + NodeID: d.ID, + Tokens: strings.Join(search.Tokenize(d.Text), " "), + }) } + require.NoError(t, store.BulkUpsertSymbolFTS("", items)) + require.NoError(t, store.BuildSymbolIndex()) + + indexed, err := store.SymbolFTSCount() + require.NoError(t, err) + require.Equal(t, len(ds.Corpus), indexed, + "every corpus document must reach the FTS — a partial corpus scores a benchmark nobody ran") + retrieve := func(query string, k int) []string { - hits := bm.Search(query, k) + hits, err := store.SearchSymbols(query, k) + require.NoError(t, err) ids := make([]string, 0, len(hits)) for _, h := range hits { - ids = append(ids, h.ID) + ids = append(ids, h.NodeID) } return ids } m := Evaluate(ds, retrieve, nil) require.Equal(t, 2, m.Scored) - require.Greater(t, m.RecallAtK[5], 0.0, "BM25 should surface the gold candidate") + require.Greater(t, m.RecallAtK[5], 0.0, "the FTS should surface the gold candidate") require.Greater(t, m.MRR, 0.0) + // Both queries are multi-word, so they miss the store's exact-name + // Tier 0 short-circuit and are ranked by FTS bm25 — which puts each + // gold candidate first. Pinning the values keeps the assertion honest: + // a ranking inversion, or a corpus that reached the index untokenised, + // moves them. + require.InDelta(t, 1.0, m.RecallAtK[5], 1e-9) + require.InDelta(t, 1.0, m.MRR, 1e-9) } diff --git a/internal/graph/node_id_parity_test.go b/internal/graph/node_id_parity_test.go index 8a74a7ad0..08c801ab6 100644 --- a/internal/graph/node_id_parity_test.go +++ b/internal/graph/node_id_parity_test.go @@ -220,7 +220,7 @@ func indexFixture(t *testing.T, checkoutName string) fixtureResult { require.NoError(t, err) g := graph.New() - mi := indexer.NewMultiIndexer(g, newParityRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, newParityRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) diff --git a/internal/graph/store.go b/internal/graph/store.go index 4e6efa801..98507f827 100644 --- a/internal/graph/store.go +++ b/internal/graph/store.go @@ -513,10 +513,12 @@ type SymbolFTSItem struct { // SymbolSearcher is an optional interface backends MAY implement to // expose engine-native full-text search over the graph's symbol // names. When the backing store implements it, the daemon's -// search_symbols path routes through the backend FTS instead of -// building a parallel in-process BM25 index — saving ~100MB -// of heap on a vscode-scale repo and putting the search latency in -// the same address space as the rest of the graph. +// search_symbols path routes through the backend FTS; a store that +// does not gets search.NullBackend and the engine's index-free +// substring scan instead. Serving ranked search from the store's own +// index is what keeps ~100MB of heap off a vscode-scale repo and puts +// the search latency in the same address space as the rest of the +// graph. // // Contract: // @@ -572,6 +574,16 @@ type SymbolFTSCounter interface { SymbolFTSCount() (int, error) } +// SymbolFTSNormalizationState persists the normalization mode used to produce +// each repository's native symbol FTS corpus. An indexer must advance the +// marker only after an authoritative replacement succeeds; a crash between +// replacement and marker update is safe because the next warm reconcile +// repeats the idempotent rebuild. +type SymbolFTSNormalizationState interface { + GetSymbolFTSNormalization(repoPrefix string) (mode string, ok bool, err error) + SetSymbolFTSNormalization(repoPrefix, mode string) error +} + // SymbolFTSBatchUpserter is the optional incremental fast path. It is kept // separate from SymbolSearcher so alternate search implementations and test // doubles retain source compatibility. Indexing paths use this capability as diff --git a/internal/graph/store_sqlite/fts_normalization_test.go b/internal/graph/store_sqlite/fts_normalization_test.go new file mode 100644 index 000000000..1261f2dc8 --- /dev/null +++ b/internal/graph/store_sqlite/fts_normalization_test.go @@ -0,0 +1,202 @@ +package store_sqlite + +import ( + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/search" +) + +const ftsNormalizationTestModeEnv = "GORTEX_FTS_NORMALIZATION_TEST_MODE" + +func TestSymbolFTSNormalizationStateLifecycle(t *testing.T) { + s, err := Open(filepath.Join(t.TempDir(), "fts-state.sqlite")) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { _ = s.Close() }) + + if mode, ok, err := s.GetSymbolFTSNormalization("repo-a"); err != nil || ok || mode != "" { + t.Fatalf("missing repo-a marker = (%q, %v, %v), want (\"\", false, nil)", mode, ok, err) + } + if err := s.SetSymbolFTSNormalization("repo-a", "porter-v1"); err != nil { + t.Fatalf("SetSymbolFTSNormalization(repo-a): %v", err) + } + if err := s.SetSymbolFTSNormalization("repo-b", ""); err != nil { + t.Fatalf("SetSymbolFTSNormalization(repo-b): %v", err) + } + const opaqueMode = "future/vendor-v7+opaque" + if err := s.SetSymbolFTSNormalization("repo-opaque", opaqueMode); err != nil { + t.Fatalf("SetSymbolFTSNormalization(repo-opaque): %v", err) + } + assertFTSNormalizationState(t, s, "repo-a", "porter-v1", true) + assertFTSNormalizationState(t, s, "repo-b", "", true) + assertFTSNormalizationState(t, s, "repo-opaque", opaqueMode, true) + + if err := s.SetSymbolFTSNormalization("repo-a", "porter-v2"); err != nil { + t.Fatalf("overwrite repo-a marker: %v", err) + } + assertFTSNormalizationState(t, s, "repo-a", "porter-v2", true) + assertFTSNormalizationState(t, s, "repo-b", "", true) + + if err := s.SetSymbolFTSNormalization("orphan", "porter-v1"); err != nil { + t.Fatalf("set orphan marker: %v", err) + } + if err := s.SetSymbolFTSNormalization("", "porter-v1"); err != nil { + t.Fatalf("set protected empty-prefix marker: %v", err) + } + orphans := s.OrphanRepoPrefixes([]string{"repo-a", "repo-b", "repo-opaque"}) + if !slices.Equal(orphans, []string{"orphan"}) { + t.Fatalf("OrphanRepoPrefixes = %v, want [orphan] (empty prefix must stay protected)", orphans) + } + + if err := s.PurgeRepo("repo-a"); err != nil { + t.Fatalf("PurgeRepo(repo-a): %v", err) + } + assertFTSNormalizationState(t, s, "repo-a", "", false) + assertFTSNormalizationState(t, s, "repo-b", "", true) + + if err := s.SetSymbolFTSNormalization("old", "porter-v1"); err != nil { + t.Fatalf("set old marker: %v", err) + } + if err := s.SetSymbolFTSNormalization("new", "stale-mode"); err != nil { + t.Fatalf("set destination marker: %v", err) + } + if err := s.RekeyRepoPrefix("old", "new"); err != nil { + t.Fatalf("RekeyRepoPrefix: %v", err) + } + // Rekey drops the symbol FTS corpus because node IDs change. Neither the + // source marker nor a stale destination marker may survive and falsely + // certify that the newly empty corpus was built in a particular mode. + assertFTSNormalizationState(t, s, "old", "", false) + assertFTSNormalizationState(t, s, "new", "", false) +} + +func assertFTSNormalizationState(t *testing.T, s *Store, repo, wantMode string, wantOK bool) { + t.Helper() + mode, ok, err := s.GetSymbolFTSNormalization(repo) + if err != nil { + t.Fatalf("GetSymbolFTSNormalization(%q): %v", repo, err) + } + if mode != wantMode || ok != wantOK { + t.Fatalf("GetSymbolFTSNormalization(%q) = (%q, %v), want (%q, %v)", repo, mode, ok, wantMode, wantOK) + } +} + +func TestFTSNormalizationWriteQueryAndContentParity(t *testing.T) { + if mode := os.Getenv(ftsNormalizationTestModeEnv); mode != "" { + runFTSNormalizationModeAssertions(t, mode == "enabled") + return + } + + for _, tc := range []struct { + name string + stemming string + }{ + {name: "disabled", stemming: "0"}, + {name: "enabled", stemming: "1"}, + } { + t.Run(tc.name, func(t *testing.T) { + cmd := exec.Command(os.Args[0], "-test.run=^TestFTSNormalizationWriteQueryAndContentParity$") + cmd.Env = ftsNormalizationTestEnv(tc.name, tc.stemming) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("subprocess failed: %v\n%s", err, out) + } + }) + } +} + +func ftsNormalizationTestEnv(mode, stemming string) []string { + env := make([]string, 0, len(os.Environ())+2) + for _, entry := range os.Environ() { + if strings.HasPrefix(entry, ftsNormalizationTestModeEnv+"=") || strings.HasPrefix(entry, "GORTEX_FTS_STEMMING=") { + continue + } + env = append(env, entry) + } + return append(env, ftsNormalizationTestModeEnv+"="+mode, "GORTEX_FTS_STEMMING="+stemming) +} + +func runFTSNormalizationModeAssertions(t *testing.T, enabled bool) { + wantMode := "" + wantSymbolMatch := `"the"* OR "manager"*` + if enabled { + wantMode = "porter-v1" + wantSymbolMatch = `"manag"*` + } + if got := search.FTSNormalizationMode(); got != wantMode { + t.Fatalf("FTSNormalizationMode = %q, want %q", got, wantMode) + } + + s, err := Open(filepath.Join(t.TempDir(), "fts-mode.sqlite")) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { _ = s.Close() }) + + if got := s.buildFTSMatch("the manager", true); got != wantSymbolMatch { + t.Fatalf("normalized symbol MATCH = %q, want %q", got, wantSymbolMatch) + } + if got := s.buildFTSMatch("the manager", false); got != `"the"* OR "manager"*` { + t.Fatalf("raw content MATCH = %q, want unnormalised terms", got) + } + + const symbolID = "repo/worker.go::WorkerCatalog" + s.AddNode(&graph.Node{ + ID: symbolID, Kind: graph.KindType, Name: "WorkerCatalog", + FilePath: "repo/worker.go", Language: "go", RepoPrefix: "repo", + }) + writeTokens := search.NormalizeFTSTokens(search.Tokenize("the manage")) + if err := s.UpsertSymbolFTS(symbolID, strings.Join(writeTokens, " ")); err != nil { + t.Fatalf("UpsertSymbolFTS: %v", err) + } + + hits, err := s.SearchSymbols("manager", 10) + if err != nil { + t.Fatalf("SearchSymbols(manager): %v", err) + } + if enabled && (len(hits) != 1 || hits[0].NodeID != symbolID) { + t.Fatalf("enabled stemming hits = %+v, want %q", hits, symbolID) + } + if !enabled && len(hits) != 0 { + t.Fatalf("disabled stemming hits = %+v, want no morphological match", hits) + } + + hits, err = s.SearchSymbols("the", 10) + if err != nil { + t.Fatalf("SearchSymbols(stopword): %v", err) + } + if enabled && len(hits) != 0 { + t.Fatalf("enabled stopword query hits = %+v, want none", hits) + } + if !enabled && (len(hits) != 1 || hits[0].NodeID != symbolID) { + t.Fatalf("disabled stopword query hits = %+v, want raw-token hit", hits) + } + + const contentBody = "the managers are running" + const contentID = "repo/readme.md::doc:section-0" + if err := s.AppendContent("repo", []graph.ContentFTSItem{{ + NodeID: contentID, FilePath: "repo/readme.md", Body: contentBody, + }}); err != nil { + t.Fatalf("AppendContent: %v", err) + } + contentHits, err := s.SearchContent("the", "repo", 10) + if err != nil { + t.Fatalf("SearchContent(stopword): %v", err) + } + if len(contentHits) != 1 || contentHits[0].NodeID != contentID { + t.Fatalf("content stopword hits = %+v, want raw content hit in both modes", contentHits) + } + var storedBody string + if err := s.db.QueryRow(`SELECT body FROM content_fts WHERE node_id = ?`, contentID).Scan(&storedBody); err != nil { + t.Fatalf("read raw content_fts body: %v", err) + } + if storedBody != contentBody { + t.Fatalf("content_fts body = %q, want raw %q", storedBody, contentBody) + } +} diff --git a/internal/graph/store_sqlite/schema.go b/internal/graph/store_sqlite/schema.go index 34b856178..4d0321881 100644 --- a/internal/graph/store_sqlite/schema.go +++ b/internal/graph/store_sqlite/schema.go @@ -768,6 +768,15 @@ CREATE INDEX IF NOT EXISTS blame_by_repo ON blame_enrichment(repo_prefix) WHERE -- on its next open + reindex. CREATE VIRTUAL TABLE IF NOT EXISTS symbol_fts USING fts5(node_id UNINDEXED, repo_prefix UNINDEXED, tokens); +-- symbol_fts_state records which deterministic token-normalization mode built +-- each repository's durable symbol corpus. The marker advances only after an +-- authoritative replacement succeeds, so a crash can cause a harmless repeat +-- rebuild but can never certify a corpus written with a different mode. +CREATE TABLE IF NOT EXISTS symbol_fts_state ( + repo_prefix TEXT PRIMARY KEY, + normalization TEXT NOT NULL DEFAULT '' +) WITHOUT ROWID; + -- symbol_fts_rowid maps a node_id to the rowid (FTS5 docid) of its row in -- symbol_fts. node_id is UNINDEXED in the FTS5 vtable, so deleting a node's -- prior row with "DELETE … WHERE node_id = ?" full-scans the entire index diff --git a/internal/graph/store_sqlite/schema_fts_state_migration_test.go b/internal/graph/store_sqlite/schema_fts_state_migration_test.go new file mode 100644 index 000000000..1897b21cf --- /dev/null +++ b/internal/graph/store_sqlite/schema_fts_state_migration_test.go @@ -0,0 +1,94 @@ +package store_sqlite + +import ( + "database/sql" + "path/filepath" + "testing" + + "github.com/zzet/gortex/internal/graph" +) + +func TestOpenV10AddsSymbolFTSStateWithoutRebuildingCorpus(t *testing.T) { + if currentSchemaVersion != 11 { + t.Fatalf("currentSchemaVersion = %d, want 11 for the symbol FTS state migration", currentSchemaVersion) + } + var v11 *schemaMigration + for i := range schemaMigrations { + if schemaMigrations[i].version == 11 { + v11 = &schemaMigrations[i] + break + } + } + if v11 == nil || v11.rebuild || v11.inPlace == nil { + t.Fatalf("v11 migration = %+v, want a registered in-place step", v11) + } + + path := filepath.Join(t.TempDir(), "v10.sqlite") + seed, err := Open(path) + if err != nil { + t.Fatalf("create current store: %v", err) + } + + const ( + symbolID = "repo/legacy.go::LegacyWidget" + callerID = "repo/caller.go::UseLegacyWidget" + ) + seed.AddBatch([]*graph.Node{ + {ID: symbolID, Kind: graph.KindType, Name: "LegacyWidget", FilePath: "repo/legacy.go", RepoPrefix: "repo"}, + {ID: callerID, Kind: graph.KindFunction, Name: "UseLegacyWidget", FilePath: "repo/caller.go", RepoPrefix: "repo"}, + }, []*graph.Edge{{ + From: callerID, To: symbolID, Kind: graph.EdgeCalls, FilePath: "repo/caller.go", Line: 7, + }}) + if err := seed.UpsertSymbolFTS(symbolID, "migration sentinel"); err != nil { + t.Fatalf("seed symbol FTS: %v", err) + } + if err := seed.Close(); err != nil { + t.Fatalf("close seed store: %v", err) + } + + // Recreate the exact v10 shape: the graph and native FTS corpus exist, but + // the per-repository normalization marker introduced by v11 does not. + withRawDB(t, path, func(db *sql.DB) { + if _, err := db.Exec(`DROP TABLE symbol_fts_state`); err != nil { + t.Fatalf("drop post-v10 marker table: %v", err) + } + if _, err := db.Exec(`PRAGMA user_version = 10`); err != nil { + t.Fatalf("stamp v10: %v", err) + } + }) + + migrated, err := Open(path) + if err != nil { + t.Fatalf("open v10 store: %v", err) + } + defer migrated.Close() + + if migrated.NeedsRebuild() { + t.Fatal("additive v10->v11 state-table migration requested a rebuild") + } + if version, err := readUserVersion(migrated.db); err != nil || version != currentSchemaVersion { + t.Fatalf("migrated user_version = %d (err %v), want %d", version, err, currentSchemaVersion) + } + var stateTables int + if err := migrated.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'symbol_fts_state'`).Scan(&stateTables); err != nil { + t.Fatalf("probe symbol_fts_state: %v", err) + } + if stateTables != 1 { + t.Fatalf("symbol_fts_state table count = %d, want 1", stateTables) + } + assertFTSNormalizationState(t, migrated, "repo", "", false) + + if got := migrated.NodeCount(); got != 2 { + t.Fatalf("node count after migration = %d, want 2", got) + } + if got := migrated.EdgeCount(); got != 1 { + t.Fatalf("edge count after migration = %d, want 1", got) + } + hits, err := migrated.SearchSymbols("sentinel", 10) + if err != nil { + t.Fatalf("search preserved v10 FTS corpus: %v", err) + } + if len(hits) != 1 || hits[0].NodeID != symbolID { + t.Fatalf("preserved FTS hits = %+v, want %q", hits, symbolID) + } +} diff --git a/internal/graph/store_sqlite/schema_version.go b/internal/graph/store_sqlite/schema_version.go index 9163a632c..ad5d11152 100644 --- a/internal/graph/store_sqlite/schema_version.go +++ b/internal/graph/store_sqlite/schema_version.go @@ -32,7 +32,7 @@ import ( // index changes in a way an old on-disk DB would not already have, and append a // matching schemaMigrations entry describing how to bring an older store // forward (in place, or by rebuild). -const currentSchemaVersion = 10 +const currentSchemaVersion = 11 // schemaMigration is one forward step. Exactly one strategy applies: // - rebuild=true: the change introduces structure/data that can only come @@ -78,6 +78,18 @@ var schemaMigrations = []schemaMigration{ // the legacy (node_id, dims, vec) rows for every ID shape. Rebuild only the // derived vector sidecar rather than discarding otherwise-valid topology. {version: 10, name: "rebuild vector corpus ownership and parents", inPlace: rebuildVectorCorpusSchema}, + {version: 11, name: "add symbol FTS normalization state", inPlace: createSymbolFTSNormalizationStateTable}, +} + +// createSymbolFTSNormalizationStateTable is the explicit v11 migration for +// existing stores. schemaSQL owns the canonical fresh-store definition; this +// idempotent step makes the additive table part of the versioned contract. +func createSymbolFTSNormalizationStateTable(tx *sql.Tx) error { + _, err := tx.Exec(`CREATE TABLE IF NOT EXISTS symbol_fts_state ( + repo_prefix TEXT PRIMARY KEY, + normalization TEXT NOT NULL DEFAULT '' + ) WITHOUT ROWID`) + return err } // dropUnusedSemanticPendingIndex removes an experimental index for a query diff --git a/internal/graph/store_sqlite/store_content_fts.go b/internal/graph/store_sqlite/store_content_fts.go index db194a659..a88f0304b 100644 --- a/internal/graph/store_sqlite/store_content_fts.go +++ b/internal/graph/store_sqlite/store_content_fts.go @@ -502,7 +502,7 @@ func (s *Store) SearchContent(query, repoPrefix string, limit int) ([]graph.Cont if limit <= 0 { limit = 20 } - match := s.buildFTSMatch(query) + match := s.buildFTSMatch(query, false) if match == "" { return nil, nil } diff --git a/internal/graph/store_sqlite/store_fts.go b/internal/graph/store_sqlite/store_fts.go index 3853d744e..c351758e5 100644 --- a/internal/graph/store_sqlite/store_fts.go +++ b/internal/graph/store_sqlite/store_fts.go @@ -40,7 +40,7 @@ import ( // Compile-time assertions: *Store satisfies the symbol-search // capabilities. The indexer auto-engages these when the active backend // implements them, routing search_symbols through on-disk FTS5 instead -// of the in-process BM25 index. +// of the engine's index-free substring fallback. var ( _ graph.SymbolSearcher = (*Store)(nil) _ graph.SymbolFTSBatchUpserter = (*Store)(nil) @@ -664,7 +664,7 @@ func (s *Store) SearchSymbolsRepoScoped(query string, repoAllow []string, limit } } - match := s.buildFTSMatch(query) + match := s.buildFTSMatch(query, true) if match == "" { return nil, nil } @@ -730,17 +730,22 @@ func tier0ShortCircuitKind(k graph.NodeKind) bool { // buildFTSMatch tokenises the query with the write-side splitter and // builds an FTS5 MATCH expression: each token becomes a quoted prefix // term ("tok"*) and the terms are OR-joined so any token match counts. +// normalize selects the symbol-FTS normalization pass. Content FTS keeps raw +// bodies for snippets and scans, so its query path must remain unnormalised. // Returns "" when the query degenerates to no tokens. -func (s *Store) buildFTSMatch(query string) string { +func (s *Store) buildFTSMatch(query string, normalize bool) string { tokens := search.Tokenize(query) if len(tokens) == 0 { // Fallback: when Tokenize drops everything (e.g. a single // sub-2-char token like "go"), use the looser query tokeniser so // the search still reaches the engine instead of returning empty. tokens = search.TokenizeQuery(query) - if len(tokens) == 0 { - return "" - } + } + if normalize { + tokens = search.NormalizeFTSTokens(tokens) + } + if len(tokens) == 0 { + return "" } parts := make([]string, 0, len(tokens)) for _, t := range tokens { diff --git a/internal/graph/store_sqlite/store_fts_state.go b/internal/graph/store_sqlite/store_fts_state.go new file mode 100644 index 000000000..65925e120 --- /dev/null +++ b/internal/graph/store_sqlite/store_fts_state.go @@ -0,0 +1,40 @@ +package store_sqlite + +import ( + "context" + "database/sql" + + "github.com/zzet/gortex/internal/graph" +) + +var _ graph.SymbolFTSNormalizationState = (*Store)(nil) + +// GetSymbolFTSNormalization returns the mode that produced repoPrefix's +// durable symbol FTS corpus. A missing row means the historical unnormalised +// mode; callers still receive ok=false so an enabled mode can trigger its +// one-time authoritative rebuild. +func (s *Store) GetSymbolFTSNormalization(repoPrefix string) (string, bool, error) { + var mode string + err := s.db.QueryRow(`SELECT normalization FROM symbol_fts_state WHERE repo_prefix = ?`, repoPrefix).Scan(&mode) + if err == sql.ErrNoRows { + return "", false, nil + } + if err != nil { + return "", false, err + } + return mode, true, nil +} + +// SetSymbolFTSNormalization advances repoPrefix's marker after the caller has +// committed an authoritative corpus replacement. Keeping this write separate +// and ordered after replacement is deliberately fail-closed: a crash between +// them leaves the old marker and makes the next warm reconcile repeat the +// idempotent rebuild. +func (s *Store) SetSymbolFTSNormalization(repoPrefix, mode string) error { + s.writeMu.Lock() + defer s.writeMu.Unlock() + _, err := s.execActiveWriteLocked(context.Background(), ` +INSERT INTO symbol_fts_state (repo_prefix, normalization) VALUES (?, ?) +ON CONFLICT(repo_prefix) DO UPDATE SET normalization = excluded.normalization`, repoPrefix, mode) + return err +} diff --git a/internal/graph/store_sqlite/store_purge.go b/internal/graph/store_sqlite/store_purge.go index fda669884..4ef3c154a 100644 --- a/internal/graph/store_sqlite/store_purge.go +++ b/internal/graph/store_sqlite/store_purge.go @@ -35,6 +35,7 @@ import ( var purgeSidecarTables = []string{ "file_mtimes", "repo_index_state", + "symbol_fts_state", "enrichment_state", "contract_state", "semantic_binding_types", @@ -143,6 +144,7 @@ var orphanScanTables = []string{ "nodes", "file_mtimes", "repo_index_state", + "symbol_fts_state", "enrichment_state", "files", "semantic_binding_types", @@ -303,6 +305,17 @@ func (s *Store) RekeyRepoPrefix(oldPrefix, newPrefix string) error { changed = true } } + // The symbol FTS corpus above is dropped rather than relabeled because its + // node IDs change. Invalidate both possible markers in the same transaction: + // moving the old marker would falsely certify the now-empty destination, + // while retaining a prior destination marker would do the same after merge. + stateRes, err := tx.Exec(`DELETE FROM symbol_fts_state WHERE repo_prefix IN (?, ?)`, oldPrefix, newPrefix) + if err != nil { + return fmt.Errorf("store_sqlite: RekeyRepoPrefix invalidate symbol FTS normalization: %w", err) + } + if n, rowsErr := stateRes.RowsAffected(); rowsErr == nil && n > 0 { + changed = true + } // Vectors are handled explicitly instead of joining rekeyDropTables because // that shared list is also used by the historical v6→v7 migration, whose // vector schema predates repo_prefix. At the current schema the old node and diff --git a/internal/indexer/batch_transition_coordinator_test.go b/internal/indexer/batch_transition_coordinator_test.go index 8d9f48eca..5370838f4 100644 --- a/internal/indexer/batch_transition_coordinator_test.go +++ b/internal/indexer/batch_transition_coordinator_test.go @@ -26,7 +26,7 @@ func newBatchTransitionTestMulti(g graph.Store) *MultiIndexer { if g == nil { g = graph.New() } - return NewMultiIndexer(g, parser.NewRegistry(), search.NewBM25(), nil, zap.NewNop()) + return NewMultiIndexer(g, parser.NewRegistry(), search.NewNull(), nil, zap.NewNop()) } func requireBatchTransitionBlocked(t *testing.T, done <-chan struct{}) { @@ -51,7 +51,7 @@ func TestIndexAllWorkersDoNotReenterRegistryLock(t *testing.T) { cm.Global().Repos = append(cm.Global().Repos, config.RepoEntry{Path: root, Name: name}) } - mi := NewMultiIndexer(g, reg, search.NewAuto(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) done := make(chan error, 1) go func() { _, err := mi.IndexAll() diff --git a/internal/indexer/chunk_index_test.go b/internal/indexer/chunk_index_test.go index 246ca092c..88f6f30f0 100644 --- a/internal/indexer/chunk_index_test.go +++ b/internal/indexer/chunk_index_test.go @@ -42,8 +42,8 @@ func (stubEmbedder) EmbedBatch(_ context.Context, texts []string) ([][]float32, func (stubEmbedder) Dimensions() int { return 4 } func (stubEmbedder) Close() error { return nil } -// indexedVectorBackend indexes dir with the given chunk options and -// returns the vector backend buildSearchIndex produced. +// indexedVectorBackend indexes dir with the given chunk options and returns +// the published vector backend, pinned for the remainder of the test. func indexedVectorBackend(t *testing.T, dir string, opts embedding.ChunkOptions) *search.VectorBackend { t.Helper() g := graph.New() @@ -62,16 +62,17 @@ func indexedVectorBackend(t *testing.T, dir string, opts embedding.ChunkOptions) sw, ok := idx.Search().(*search.Swappable) require.True(t, ok) - hyb, ok := sw.Inner().(*search.HybridBackend) + backend, release := sw.AcquireBackend() + t.Cleanup(release) + hyb, ok := backend.(*search.HybridBackend) require.True(t, ok, "an embedder-equipped index must produce a HybridBackend") return hyb.VectorIndex() } -// TestBuildSearchIndex_LongSymbolYieldsMultipleChunks proves the -// pipeline change: a function whose source span exceeds the chunk -// threshold is split into several chunk vectors, and the chunk → -// parent mapping is recorded on the vector backend. -func TestBuildSearchIndex_LongSymbolYieldsMultipleChunks(t *testing.T) { +// TestVectorPublication_LongSymbolYieldsMultipleChunks proves that a function +// whose source span exceeds the chunk threshold is split into several chunk +// vectors and the chunk → parent mapping is recorded on the vector backend. +func TestVectorPublication_LongSymbolYieldsMultipleChunks(t *testing.T) { dir := t.TempDir() var b strings.Builder b.WriteString("package main\n\nfunc BigFunc() {\n") @@ -100,8 +101,8 @@ func TestBuildSearchIndex_LongSymbolYieldsMultipleChunks(t *testing.T) { "BigFunc must be split into at least two chunk vectors") } -// bigFuncChunkID builds the synthetic chunk ID buildSearchIndex stamps -// on BigFunc's i-th window. +// bigFuncChunkID builds the synthetic chunk ID vector preparation stamps on +// BigFunc's i-th window. func bigFuncChunkID(i int) string { return "big.go::BigFunc#chunk" + itoa(i) } @@ -118,10 +119,10 @@ func itoa(i int) string { return string(digits) } -// TestBuildSearchIndex_ShortSymbolStaysWhole proves the inverse: a file -// of only small functions produces no chunk vectors — every symbol is -// embedded under its own ID. -func TestBuildSearchIndex_ShortSymbolStaysWhole(t *testing.T) { +// TestVectorPublication_ShortSymbolStaysWhole proves the inverse: a file of +// only small functions produces no chunk vectors — every symbol is embedded +// under its own ID. +func TestVectorPublication_ShortSymbolStaysWhole(t *testing.T) { dir := t.TempDir() src := `package main @@ -141,10 +142,10 @@ func AlsoTiny() { assert.Greater(t, vec.Count(), 0, "the symbols must still be embedded") } -// TestBuildSearchIndex_ChunkedSymbolNotDuplicatedInSearch proves the -// end-to-end de-chunk contract through a real index: searching for a -// long symbol returns it once, and no synthetic chunk ID surfaces. -func TestBuildSearchIndex_ChunkedSymbolNotDuplicatedInSearch(t *testing.T) { +// TestVectorPublication_ChunkedSymbolNotDuplicatedInSearch proves the end-to-end +// de-chunk contract through a real index: searching for a long symbol returns +// it once, and no synthetic chunk ID surfaces. +func TestVectorPublication_ChunkedSymbolNotDuplicatedInSearch(t *testing.T) { dir := t.TempDir() var b strings.Builder b.WriteString("package main\n\nfunc ValidateRequestPayload() {\n") @@ -154,16 +155,31 @@ func TestBuildSearchIndex_ChunkedSymbolNotDuplicatedInSearch(t *testing.T) { b.WriteString("}\n\nfunc checkField() {}\n") require.NoError(t, os.WriteFile(filepath.Join(dir, "v.go"), []byte(b.String()), 0o644)) - g := graph.New() reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) cfg := config.Default().Index cfg.Workers = 1 - idx := New(g, reg, cfg, zap.NewNop()) - idx.SetEmbedder(stubEmbedder{}) - idx.SetEmbeddingChunkOptions(embedding.ChunkOptions{ThresholdLines: 20, WindowLines: 15}) - _, err := idx.Index(dir) + idx, store := newFTSIndexer(t, dir, reg, cfg, func(idx *Indexer) { + idx.SetEmbedder(stubEmbedder{}) + idx.SetEmbeddingChunkOptions(embedding.ChunkOptions{ThresholdLines: 20, WindowLines: 15}) + }) + + const symbolID = "v.go::ValidateRequestPayload" + + // Probe the text half of the hybrid on its own first. The fused result + // below cannot carry this claim: the stub embedder maps every text to the + // same direction, so the vector channel returns the whole corpus for any + // query and would answer "is the symbol retrievable" no matter what the + // text side did. The query is multi-word on purpose — an identifier-shaped + // query is short-circuited on an exact FindNodesByName hit (store_fts.go + // tier 0) and never reaches the ranked FTS tier. + ftsHits, err := store.SearchSymbols("validate request payload", 20) require.NoError(t, err) + ftsIDs := make([]string, 0, len(ftsHits)) + for _, h := range ftsHits { + ftsIDs = append(ftsIDs, h.NodeID) + } + require.Contains(t, ftsIDs, symbolID, "the chunked symbol must be in the symbol FTS corpus") results := idx.Search().Search("ValidateRequestPayload", 20) require.NotEmpty(t, results) @@ -174,6 +190,6 @@ func TestBuildSearchIndex_ChunkedSymbolNotDuplicatedInSearch(t *testing.T) { "a chunk ID must never appear in search output") seen[r.ID]++ } - assert.LessOrEqual(t, seen["v.go::ValidateRequestPayload"], 1, - "the chunked symbol must not be returned more than once") + assert.Equal(t, 1, seen[symbolID], + "the chunked symbol must be returned exactly once") } diff --git a/internal/indexer/contract_bridge_test.go b/internal/indexer/contract_bridge_test.go index 47b10788d..aab216cef 100644 --- a/internal/indexer/contract_bridge_test.go +++ b/internal/indexer/contract_bridge_test.go @@ -197,7 +197,7 @@ func TestContractBridge_TwoRepoIntegration(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) @@ -386,7 +386,7 @@ func TestReconcileContractEdges_ConcurrentNoRaceOrTear(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) diff --git a/internal/indexer/deferred_enrich_gate_test.go b/internal/indexer/deferred_enrich_gate_test.go index d18f15893..30c0cc26b 100644 --- a/internal/indexer/deferred_enrich_gate_test.go +++ b/internal/indexer/deferred_enrich_gate_test.go @@ -86,7 +86,7 @@ func newEmptyMultiIndexer(t *testing.T, g graph.Store) *MultiIndexer { require.NoError(t, gc.Save()) cm, err := config.NewConfigManager(tmpCfg) require.NoError(t, err) - return NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + return NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) } // TestMultiIndexer_RunDeferredEnrich_GatesUnchangedRepos is the core diff --git a/internal/indexer/direct_mutation_api_test.go b/internal/indexer/direct_mutation_api_test.go index cda79c44d..e36e7e4d0 100644 --- a/internal/indexer/direct_mutation_api_test.go +++ b/internal/indexer/direct_mutation_api_test.go @@ -94,7 +94,7 @@ func TestDirectMutationAPIsUseStableLaneAndCurrentIndexer(t *testing.T) { reg.Register(languages.NewGoExtractor()) cfg := config.Default() cfg.Index.Workers = 1 - mi := NewMultiIndexer(g, reg, search.NewAuto(), nil, zap.NewNop()) + mi := NewMultiIndexer(g, reg, search.NewNull(), nil, zap.NewNop()) current := mi.newPerRepoIndexer(cfg.Index) current.SetRepoPrefix("repo") // Fixture construction mirrors TrackRepo/IndexRepo: the per-repository diff --git a/internal/indexer/embed_pool_test.go b/internal/indexer/embed_pool_test.go index b2e2f5f06..ab7efdcac 100644 --- a/internal/indexer/embed_pool_test.go +++ b/internal/indexer/embed_pool_test.go @@ -161,10 +161,10 @@ func TestEmbedAllChunks_AbortsOnError(t *testing.T) { assert.Contains(t, err.Error(), "t37") } -// TestBuildSearchIndex_AbortOnEmbedErrorKeepsTextOnly asserts the -// end-to-end abort contract: when embedding fails, buildSearchIndex -// leaves the search backend text-only — no HybridBackend is swapped in. -func TestBuildSearchIndex_AbortOnEmbedErrorKeepsTextOnly(t *testing.T) { +// TestBuildSearchIndexCtx_AbortOnEmbedErrorKeepsTextOnly asserts the end-to-end +// abort contract: when embedding fails, vector publication leaves the search +// backend text-only — no HybridBackend is swapped in. +func TestBuildSearchIndexCtx_AbortOnEmbedErrorKeepsTextOnly(t *testing.T) { g := graph.New() // Two function nodes; their embed metadata text is // "function ...". poolEmbedder fails on an exact text @@ -176,15 +176,18 @@ func TestBuildSearchIndex_AbortOnEmbedErrorKeepsTextOnly(t *testing.T) { emb := &poolEmbedder{failOnText: "function Alpha a.go"} idx.SetEmbedder(emb) - idx.buildSearchIndex() + require.NoError(t, idx.buildSearchIndexCtx(context.Background())) + require.Error(t, idx.LastVectorBuildError()) // The backend must NOT be a HybridBackend — embedding aborted, so // the search stays text-only. sw, ok := idx.Search().(*search.Swappable) require.True(t, ok) - _, isHybrid := sw.Inner().(*search.HybridBackend) + backend, release := sw.AcquireBackend() + defer release() + _, isHybrid := backend.(*search.HybridBackend) assert.False(t, isHybrid, - "buildSearchIndex must not install a HybridBackend when embedding fails") + "vector publication must not install a HybridBackend when embedding fails") } // TestEmbedAllChunks_DeterministicRegardlessOfOrder runs the pool many diff --git a/internal/indexer/external_revert_reindex_test.go b/internal/indexer/external_revert_reindex_test.go index f555ec6db..58082374e 100644 --- a/internal/indexer/external_revert_reindex_test.go +++ b/internal/indexer/external_revert_reindex_test.go @@ -32,7 +32,7 @@ func setupRevertWatcher(t *testing.T) (w *Watcher, g graph.Store, defPath, defID g = newSqliteGraph(t) idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) diff --git a/internal/indexer/extractor_version_restage_test.go b/internal/indexer/extractor_version_restage_test.go index f18880456..98505206d 100644 --- a/internal/indexer/extractor_version_restage_test.go +++ b/internal/indexer/extractor_version_restage_test.go @@ -50,7 +50,7 @@ func TestIncrementalReindex_NonMerkleExtractorBumpRestagesLanguage(t *testing.T) reg := parser.NewRegistry() reg.Register(languages.NewCSharpExtractor()) idx := New(store, reg, config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) @@ -87,7 +87,7 @@ func TestIncrementalReindex_NonMerkleVersionCurrentNoRestage(t *testing.T) { reg := parser.NewRegistry() reg.Register(languages.NewCSharpExtractor()) idx := New(store, reg, config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) diff --git a/internal/indexer/fts_normalization.go b/internal/indexer/fts_normalization.go new file mode 100644 index 000000000..92783e44b --- /dev/null +++ b/internal/indexer/fts_normalization.go @@ -0,0 +1,80 @@ +package indexer + +import ( + "fmt" + + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/progress" + "github.com/zzet/gortex/internal/search" +) + +const symbolFTSNormalizationRebuildPending = "__gortex_rebuild_pending__" + +// setSymbolFTSNormalization persists a mode marker for backends with a durable +// native symbol index. Other backends have no corpus whose mode can drift. +func (idx *Indexer) setSymbolFTSNormalization(target graph.Store, mode string) error { + state, ok := target.(graph.SymbolFTSNormalizationState) + if !ok { + return nil + } + if err := state.SetSymbolFTSNormalization(idx.RepoPrefix(), mode); err != nil { + return fmt.Errorf("persist symbol FTS normalization: %w", err) + } + return nil +} + +// markSymbolFTSNormalization records the current mode only after the caller has +// completed an authoritative symbol-FTS replacement and finalization. +func (idx *Indexer) markSymbolFTSNormalization(target graph.Store) error { + return idx.setSymbolFTSNormalization(target, search.FTSNormalizationMode()) +} + +// markSymbolFTSNormalizationPending invalidates any previously certified mode +// before a destructive replacement starts. The sentinel is deliberately not a +// valid normalization mode, so any failure forces the next warm reconcile to +// repeat the rebuild even when stemming is disabled. +func (idx *Indexer) markSymbolFTSNormalizationPending(target graph.Store) error { + return idx.setSymbolFTSNormalization(target, symbolFTSNormalizationRebuildPending) +} + +// reconcileSymbolFTSNormalization repairs a warm durable corpus whose +// persisted normalization marker differs from this process's immutable mode. +// The marker advances only after populateSymbolFTS atomically replaces the +// repository corpus. A crash between those operations leaves the old marker, +// so the next reconcile safely repeats the rebuild instead of serving a +// silently mismatched index. +func (idx *Indexer) reconcileSymbolFTSNormalization(reporter progress.Reporter) (bool, error) { + state, ok := idx.graph.(graph.SymbolFTSNormalizationState) + if !ok { + return false, nil + } + want := search.FTSNormalizationMode() + got, marked, err := state.GetSymbolFTSNormalization(idx.RepoPrefix()) + if err != nil { + return false, fmt.Errorf("read symbol FTS normalization: %w", err) + } + if marked && got == want { + return false, nil + } + // A missing marker is a corpus written before normalization existed. That + // historical corpus is already correct for the disabled (empty) mode. + if marked || want != "" { + if err := idx.markSymbolFTSNormalizationPending(idx.graph); err != nil { + return false, err + } + if err := idx.populateSymbolFTS(reporter); err != nil { + return false, err + } + } + if err := idx.markSymbolFTSNormalization(idx.graph); err != nil { + return false, err + } + idx.logger.Info("indexer: symbol FTS normalization reconciled", + zap.String("repo", idx.RepoPrefix()), + zap.String("from", got), + zap.String("to", want), + zap.Bool("rebuilt", marked || want != "")) + return marked || want != "", nil +} diff --git a/internal/indexer/fts_normalization_lifecycle_test.go b/internal/indexer/fts_normalization_lifecycle_test.go new file mode 100644 index 000000000..b59c33ff5 --- /dev/null +++ b/internal/indexer/fts_normalization_lifecycle_test.go @@ -0,0 +1,386 @@ +package indexer + +import ( + "bytes" + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/search" +) + +var errFTSLifecycleWrite = errors.New("injected FTS replacement failure") +var errFTSLifecycleReset = errors.New("injected FTS reset failure") +var errFTSLifecycleBatch = errors.New("injected FTS batch failure") +var errFTSLifecycleFinalize = errors.New("injected FTS finalization failure") + +type ftsLifecycleProbeStore struct { + *store_sqlite.Store + replaceCalls int + markerCalls int + resetCalls int + batchCalls int + replaceErr error + resetErr error + batchFailAt int + finalizeErr error +} + +func (s *ftsLifecycleProbeStore) ReplaceSymbolFTS( + repoPrefix string, + produce func(emit func([]graph.SymbolFTSItem) error) error, +) error { + s.replaceCalls++ + if s.replaceErr != nil { + return s.replaceErr + } + return s.Store.ReplaceSymbolFTS(repoPrefix, produce) +} + +func (s *ftsLifecycleProbeStore) SetSymbolFTSNormalization(repoPrefix, mode string) error { + s.markerCalls++ + return s.Store.SetSymbolFTSNormalization(repoPrefix, mode) +} + +func (s *ftsLifecycleProbeStore) ResetSymbolFTS(repoPrefix string) error { + s.resetCalls++ + if s.resetErr != nil { + return s.resetErr + } + return s.Store.ResetSymbolFTS(repoPrefix) +} + +func (s *ftsLifecycleProbeStore) BatchUpsertSymbolFTS(items []graph.SymbolFTSItem) error { + s.batchCalls++ + if s.batchFailAt > 0 && s.batchCalls == s.batchFailAt { + return errFTSLifecycleBatch + } + return s.Store.BatchUpsertSymbolFTS(items) +} + +func (s *ftsLifecycleProbeStore) BuildSymbolIndex() error { + if s.finalizeErr != nil { + return s.finalizeErr + } + return s.Store.BuildSymbolIndex() +} + +func TestFTSNormalizationMarkerFollowsAuthoritativeIndexModes(t *testing.T) { + cases := []struct { + name string + shadowMaxFiles string + streaming string + wantReplacement bool + }{ + {name: "direct", shadowMaxFiles: "0", streaming: "0", wantReplacement: true}, + {name: "streaming", shadowMaxFiles: "0", streaming: "1", wantReplacement: true}, + {name: "cold_shadow", shadowMaxFiles: "1000000", streaming: "0"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("GORTEX_SHADOW_MAX_FILES", tc.shadowMaxFiles) + t.Setenv("GORTEX_STREAMING_FLUSH", tc.streaming) + t.Setenv("GORTEX_STREAMING_CHUNK_SIZE", "1") + + repo := t.TempDir() + writeFile(t, filepath.Join(repo, "orders.go"), `package shop + +func ReconcilePonies() int { return 1 } +`) + base := openFTSLifecycleStore(t) + probe := &ftsLifecycleProbeStore{Store: base} + idx := newTestIndexer(probe) + + _, err := idx.Index(repo) + require.NoError(t, err) + + got, ok, err := base.GetSymbolFTSNormalization(idx.RepoPrefix()) + require.NoError(t, err) + require.True(t, ok, "authoritative index did not persist an FTS normalization marker") + require.Equal(t, search.FTSNormalizationMode(), got) + require.Positive(t, probe.markerCalls) + if tc.wantReplacement { + require.Positive(t, probe.replaceCalls, "direct durable path must publish an authoritative corpus") + } + }) + } +} + +func TestFTSNormalizationCleanWarmReconcileIsNoOp(t *testing.T) { + base := openFTSLifecycleStore(t) + probe := &ftsLifecycleProbeStore{Store: base} + idx := newTestIndexer(probe) + require.NoError(t, base.SetSymbolFTSNormalization(idx.RepoPrefix(), search.FTSNormalizationMode())) + + _, err := idx.cleanCensusResult(context.Background(), 0, time.Now()) + require.NoError(t, err) + require.Zero(t, probe.replaceCalls, "a current warm corpus must not be rebuilt") + require.Zero(t, probe.markerCalls, "a current marker must not be rewritten") +} + +func TestFTSNormalizationAuthoritativeFailureLeavesPendingMarkerForRetry(t *testing.T) { + t.Setenv("GORTEX_SHADOW_MAX_FILES", "0") + t.Setenv("GORTEX_STREAMING_FLUSH", "0") + + repo := t.TempDir() + writeFile(t, filepath.Join(repo, "orders.go"), `package shop + +func ReconcilePonies() int { return 1 } +`) + base := openFTSLifecycleStore(t) + probe := &ftsLifecycleProbeStore{Store: base, finalizeErr: errFTSLifecycleFinalize} + idx := newTestIndexer(probe) + current := search.FTSNormalizationMode() + require.NoError(t, base.SetSymbolFTSNormalization(idx.RepoPrefix(), current)) + + _, err := idx.Index(repo) + require.ErrorIs(t, err, errFTSLifecycleFinalize) + pending, ok, readErr := base.GetSymbolFTSNormalization(idx.RepoPrefix()) + require.NoError(t, readErr) + require.True(t, ok) + require.NotEqual(t, current, pending, "failed authoritative write left a falsely-current marker") + + probe.finalizeErr = nil + probe.replaceCalls = 0 + probe.markerCalls = 0 + rebuilt, err := idx.reconcileSymbolFTSNormalization(nil) + require.NoError(t, err) + require.True(t, rebuilt, "pending marker did not force an authoritative retry") + require.Positive(t, probe.replaceCalls) + require.Positive(t, probe.markerCalls) + got, ok, readErr := base.GetSymbolFTSNormalization(idx.RepoPrefix()) + require.NoError(t, readErr) + require.True(t, ok) + require.Equal(t, current, got) +} + +func TestFTSNormalizationShadowFailureLeavesPendingMarkerForRetry(t *testing.T) { + t.Setenv("GORTEX_SHADOW_MAX_FILES", "1000000") + t.Setenv("GORTEX_STREAMING_FLUSH", "0") + + for _, tc := range []struct { + name string + resetErr error + batchFailAt int + finalizeErr error + wantErr error + }{ + {name: "reset", resetErr: errFTSLifecycleReset, wantErr: errFTSLifecycleReset}, + {name: "batch", batchFailAt: 1, wantErr: errFTSLifecycleBatch}, + {name: "finalization", finalizeErr: errFTSLifecycleFinalize, wantErr: errFTSLifecycleFinalize}, + } { + t.Run(tc.name, func(t *testing.T) { + repo := t.TempDir() + writeFile(t, filepath.Join(repo, "orders.go"), `package shop + +func ReconcilePonies() int { return 1 } +`) + base := openFTSLifecycleStore(t) + probe := &ftsLifecycleProbeStore{ + Store: base, + resetErr: tc.resetErr, + batchFailAt: tc.batchFailAt, + finalizeErr: tc.finalizeErr, + } + idx := newTestIndexer(probe) + current := search.FTSNormalizationMode() + require.NoError(t, base.SetSymbolFTSNormalization(idx.RepoPrefix(), current)) + + _, err := idx.Index(repo) + require.ErrorIs(t, err, tc.wantErr) + pending, ok, readErr := base.GetSymbolFTSNormalization(idx.RepoPrefix()) + require.NoError(t, readErr) + require.True(t, ok) + require.NotEqual(t, current, pending, "failed shadow publication left a falsely-current marker") + + probe.resetErr = nil + probe.batchFailAt = 0 + probe.finalizeErr = nil + probe.replaceCalls = 0 + probe.markerCalls = 0 + rebuilt, err := idx.reconcileSymbolFTSNormalization(nil) + require.NoError(t, err) + require.True(t, rebuilt, "pending shadow marker did not force an authoritative retry") + require.Positive(t, probe.replaceCalls) + got, ok, readErr := base.GetSymbolFTSNormalization(idx.RepoPrefix()) + require.NoError(t, readErr) + require.True(t, ok) + require.Equal(t, current, got) + }) + } +} + +func TestFTSNormalizationMarkerDoesNotAdvanceOnRebuildFailure(t *testing.T) { + cases := []struct { + name string + replaceErr error + finalizeErr error + wantErr error + }{ + {name: "write", replaceErr: errFTSLifecycleWrite, wantErr: errFTSLifecycleWrite}, + {name: "finalization", finalizeErr: errFTSLifecycleFinalize, wantErr: errFTSLifecycleFinalize}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + base := openFTSLifecycleStore(t) + probe := &ftsLifecycleProbeStore{ + Store: base, + replaceErr: tc.replaceErr, + finalizeErr: tc.finalizeErr, + } + idx := newTestIndexer(probe) + prior := mismatchedFTSNormalizationMode() + require.NoError(t, base.SetSymbolFTSNormalization(idx.RepoPrefix(), prior)) + + rebuilt, err := idx.reconcileSymbolFTSNormalization(nil) + require.ErrorIs(t, err, tc.wantErr) + require.False(t, rebuilt) + require.Positive(t, probe.markerCalls, "rebuild did not invalidate the old marker before writing") + got, ok, readErr := base.GetSymbolFTSNormalization(idx.RepoPrefix()) + require.NoError(t, readErr) + require.True(t, ok) + require.NotEqual(t, search.FTSNormalizationMode(), got, "incomplete rebuild left a falsely-current marker") + }) + } +} + +func TestFTSNormalizationReconcilesBeforeIncrementalMutation(t *testing.T) { + if os.Getenv("GORTEX_TEST_FTS_NORMALIZATION_CHILD") == "1" { + runFTSNormalizationIncrementalChild(t) + return + } + + exe, err := os.Executable() + require.NoError(t, err) + cmd := exec.Command(exe, "-test.run=^TestFTSNormalizationReconcilesBeforeIncrementalMutation$") + cmd.Env = append(envWithoutFTSLifecycleKeys(os.Environ()), + "GORTEX_TEST_FTS_NORMALIZATION_CHILD=1", + "GORTEX_FTS_STEMMING=1", + ) + var output bytes.Buffer + cmd.Stdout = &output + cmd.Stderr = &output + require.NoError(t, cmd.Run(), output.String()) +} + +func runFTSNormalizationIncrementalChild(t *testing.T) { + require.Equal(t, "porter-v1", search.FTSNormalizationMode()) + + for _, tc := range []struct { + name string + paths func(repo, changed string) []string + mode incrementalPathMode + }{ + { + name: "scoped", + paths: func(_ string, changed string) []string { return []string{changed} }, + mode: incrementalPathMode{forceExplicitFiles: true}, + }, + { + name: "full_root", + paths: func(_, _ string) []string { return nil }, + mode: incrementalPathMode{detectDeletions: true}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("GORTEX_SHADOW_MAX_FILES", "1000000") + t.Setenv("GORTEX_STREAMING_FLUSH", "0") + + repo := t.TempDir() + changed := filepath.Join(repo, "first.go") + writeFile(t, changed, `package herd + +func PoniesFirst() int { return 1 } +`) + writeFile(t, filepath.Join(repo, "second.go"), `package herd + +func PoniesSecond() int { return 2 } +`) + + base := openFTSLifecycleStore(t) + probe := &ftsLifecycleProbeStore{Store: base} + idx := newTestIndexer(probe) + _, err := idx.Index(repo) + require.NoError(t, err) + + oldItems := make([]graph.SymbolFTSItem, 0, 2) + for _, file := range []string{"first.go", "second.go"} { + for _, node := range base.GetFileNodes(file) { + if node != nil && node.Kind == graph.KindFunction && strings.HasPrefix(node.Name, "Ponies") { + oldItems = append(oldItems, graph.SymbolFTSItem{NodeID: node.ID, Tokens: "ponies " + strings.ToLower(node.Name)}) + } + } + } + require.Len(t, oldItems, 2) + require.NoError(t, base.ReplaceSymbolFTS(idx.RepoPrefix(), func(emit func([]graph.SymbolFTSItem) error) error { + return emit(oldItems) + })) + require.NoError(t, base.BuildSymbolIndex()) + require.NoError(t, base.SetSymbolFTSNormalization(idx.RepoPrefix(), "")) + probe.replaceCalls = 0 + probe.markerCalls = 0 + + writeFile(t, changed, `package herd + +func PoniesFirst() int { return 11 } +`) + future := time.Now().Add(2 * time.Second) + require.NoError(t, os.Chtimes(changed, future, future)) + + result, err := idx.incrementalReindexPathsMode(repo, tc.paths(repo, changed), tc.mode) + require.NoError(t, err) + require.Equal(t, 1, result.StaleFileCount) + require.Positive(t, probe.replaceCalls, "mode mismatch was not repaired before the incremental patch") + + gotMode, ok, err := base.GetSymbolFTSNormalization(idx.RepoPrefix()) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, search.FTSNormalizationMode(), gotMode) + + names := make([]string, 0, 2) + for _, hit := range searchAll(t, base, "pony") { + if node := base.GetNode(hit.NodeID); node != nil { + names = append(names, node.Name) + } + } + require.Contains(t, names, "PoniesFirst") + require.Contains(t, names, "PoniesSecond", "unchanged old-mode posting survived beside the patched corpus") + }) + } +} + +func openFTSLifecycleStore(t *testing.T) *store_sqlite.Store { + t.Helper() + store, err := store_sqlite.Open(filepath.Join(t.TempDir(), "fts-lifecycle.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + return store +} + +func mismatchedFTSNormalizationMode() string { + if search.FTSNormalizationMode() == "" { + return "porter-v1" + } + return "" +} + +func envWithoutFTSLifecycleKeys(env []string) []string { + out := make([]string, 0, len(env)) + for _, entry := range env { + if strings.HasPrefix(entry, "GORTEX_TEST_FTS_NORMALIZATION_CHILD=") || + strings.HasPrefix(entry, "GORTEX_FTS_STEMMING=") { + continue + } + out = append(out, entry) + } + return out +} diff --git a/internal/indexer/full_topology_transaction_test.go b/internal/indexer/full_topology_transaction_test.go index 19662908a..93ba34022 100644 --- a/internal/indexer/full_topology_transaction_test.go +++ b/internal/indexer/full_topology_transaction_test.go @@ -27,7 +27,7 @@ func TestIndexCtxStaleHandleUsesCurrentIndexerAndMetadata(t *testing.T) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) cm := newTestConfigManager(t) - mi := NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) entry := config.RepoEntry{Path: root, Name: "repo"} beforeTrack := reach.BuildCounter() @@ -81,13 +81,13 @@ func TestReconcileTopologyGenerationChangesOnlyForRealMutation(t *testing.T) { reg.Register(languages.NewGoExtractor()) cm := newTestConfigManager(t) entry := config.RepoEntry{Path: root, Name: "repo"} - seed := NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + seed := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err := seed.TrackRepo(entry) require.NoError(t, err) prior := seed.GetIndexer("repo").FileMtimes() writeFile(t, filepath.Join(root, "added.go"), "package sample\nfunc Added() {}\n") - changed := NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + changed := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) beforeChanged := reach.BuildCounter() changedResult, err := changed.ReconcileRepoCtx(context.Background(), entry, prior) require.NoError(t, err) @@ -95,7 +95,7 @@ func TestReconcileTopologyGenerationChangesOnlyForRealMutation(t *testing.T) { require.Greater(t, reach.BuildCounter(), beforeChanged) unchangedPrior := changed.GetIndexer("repo").FileMtimes() - unchanged := NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + unchanged := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) beforeUnchanged := reach.BuildCounter() unchangedResult, err := unchanged.ReconcileRepoCtx(context.Background(), entry, unchangedPrior) require.NoError(t, err) @@ -148,7 +148,7 @@ func TestIndexAllHoldsEveryLaneAndTopologyThroughPublication(t *testing.T) { cm.Global().Repos = append(cm.Global().Repos, config.RepoEntry{Path: root, Name: name}) } store := &armableBlockingAddBatchStore{Store: graph.New()} - mi := NewMultiIndexer(store, reg, search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(store, reg, search.NewNull(), cm, zap.NewNop()) _, err := mi.IndexAll() require.NoError(t, err) oldA := mi.GetIndexer("repo-a") @@ -260,7 +260,7 @@ func TestIndexAllDeduplicatesIdenticalRepositoryEntries(t *testing.T) { cm.Global().Repos = append(cm.Global().Repos, config.RepoEntry{Path: root, Name: "repo"}) } store := &countingIndexAllStore{Store: graph.New()} - mi := NewMultiIndexer(store, reg, search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(store, reg, search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexAll() require.NoError(t, err) require.Len(t, results, 1) @@ -279,7 +279,7 @@ func TestTrackRepoRechecksSamePathAcrossDifferentPrefixLanes(t *testing.T) { writeFile(t, filepath.Join(root, "main.go"), "package sample\nfunc Existing() {}\n") reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) - mi := NewMultiIndexer(graph.New(), reg, search.NewBM25(), newTestConfigManager(t), zap.NewNop()) + mi := NewMultiIndexer(graph.New(), reg, search.NewNull(), newTestConfigManager(t), zap.NewNop()) arrived := make(chan struct{}, 2) release := make(chan struct{}) diff --git a/internal/indexer/git_watcher_integration_test.go b/internal/indexer/git_watcher_integration_test.go index a7f70aed9..281018a30 100644 --- a/internal/indexer/git_watcher_integration_test.go +++ b/internal/indexer/git_watcher_integration_test.go @@ -60,7 +60,7 @@ func TestGitWatcher_BranchSwitchReconciles(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err := idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) @@ -83,7 +83,7 @@ func TestGitWatcher_BranchSwitchReconciles(t *testing.T) { // started on main" by re-indexing explicitly. g2 := graph.New() idx2 := New(g2, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx2.search = search.NewBM25() + idx2.search = search.NewNull() idx2.SetRootPath(repoDir) _, err = idx2.IndexCtx(testCtx(), repoDir) require.NoError(t, err) @@ -146,7 +146,7 @@ func TestGitWatcher_ReconcileSingleFlight(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err := idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) @@ -229,7 +229,7 @@ func TestGitWatcher_NoopWhenHeadUnchanged(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err := idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) @@ -308,7 +308,7 @@ func TestGitWatcher_UntrackedFileStaysIndexed(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err := idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) diff --git a/internal/indexer/git_watcher_scoped_resolve_test.go b/internal/indexer/git_watcher_scoped_resolve_test.go index 556dae756..5714097e5 100644 --- a/internal/indexer/git_watcher_scoped_resolve_test.go +++ b/internal/indexer/git_watcher_scoped_resolve_test.go @@ -65,7 +65,7 @@ func TestGitWatcher_SmallCommitResolvesIncoming(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err := idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) diff --git a/internal/indexer/incremental_resolve_revert_test.go b/internal/indexer/incremental_resolve_revert_test.go index 022e29d52..bd666a3c6 100644 --- a/internal/indexer/incremental_resolve_revert_test.go +++ b/internal/indexer/incremental_resolve_revert_test.go @@ -68,7 +68,7 @@ func TestIncrementalReindex_DoubleCycle_Sqlite_IntraFileCaller(t *testing.T) { g := newSqliteGraph(t) idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) @@ -113,7 +113,7 @@ func TestReresolveFileScoped_RebindsDroppedIncomingEdges(t *testing.T) { g := newSqliteGraph(t) idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) diff --git a/internal/indexer/incremental_resolve_test.go b/internal/indexer/incremental_resolve_test.go index b5b152b0f..0f4a1c830 100644 --- a/internal/indexer/incremental_resolve_test.go +++ b/internal/indexer/incremental_resolve_test.go @@ -71,7 +71,7 @@ func TestIncrementalReindex_PreservesIncomingCallerEdges(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) @@ -119,7 +119,7 @@ func TestIncrementalReindex_DoubleCycle_PreservesIncomingTier(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) @@ -164,7 +164,7 @@ func TestIncrementalReindex_DeletedDefinition_NoStaleTierOnStub(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) @@ -189,7 +189,7 @@ func TestIncrementalReindex_DeletedDefinition_NoStaleTierOnStub(t *testing.T) { // cascade: deleting a file drops its nodes' churn/coverage/blame // sidecar rows, leaving no orphan enrichment. func TestEvictFile_DropsEnrichmentSidecars(t *testing.T) { - idx, _ := newToggleIndexer(t) + idx, _, _ := newToggleIndexer(t) dir := t.TempDir() idx.SetRootPath(dir) g := idx.graph @@ -227,7 +227,7 @@ func TestIncrementalReuse_SameFileEdge_KeepsTier(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) diff --git a/internal/indexer/incremental_visibility_test.go b/internal/indexer/incremental_visibility_test.go index 97fd3d77b..3047c542c 100644 --- a/internal/indexer/incremental_visibility_test.go +++ b/internal/indexer/incremental_visibility_test.go @@ -51,7 +51,7 @@ func newCSVisIndexer(t *testing.T, dir string) (graph.Store, *Indexer) { reg := parser.NewRegistry() reg.Register(languages.NewCSharpExtractor()) idx := New(g, reg, config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 8ddef4e6c..5954c2004 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -298,14 +298,14 @@ type Indexer struct { // Set during the shadow swap, cleared when idx.graph is restored. contractStateSink graph.ContractStateStore - // embedChunkOpts tunes the AST sub-chunking buildSearchIndex applies - // to large symbols before embedding. The zero value makes the - // chunker fall back to its package defaults. + // embedChunkOpts tunes the AST sub-chunking applied while preparing a + // vector publication plan. The zero value makes the chunker fall back to + // its package defaults. embedChunkOpts embedding.ChunkOptions - // embedMaxSymbols overrides the built-in cap on how many texts the - // vector index will hold before buildSearchIndex skips the embed - // pass. Zero keeps the built-in default. + // embedMaxSymbols overrides the built-in cap on how many texts the vector + // preparation pass accepts before skipping embeddings. Zero keeps the + // built-in default. embedMaxSymbols int // embedAPIConcurrency bounds how many embedding requests run in @@ -314,8 +314,8 @@ type Indexer struct { // inference mutex. embedAPIConcurrency int - // lastVectorBuildErr records why the most recent buildSearchIndex pass - // shipped text-only instead of a vector index (chunk-embed failure, + // lastVectorBuildErr records why the most recent vector prepare/install + // pass did not publish a vector index (chunk-embed failure, // all-vectors-invalid, or the symbol-count guard). Nil after a build that // produced a vector index. Read via LastVectorBuildError once a build has // finished — it lets `gortex eval embedders` report the real cause instead @@ -501,7 +501,11 @@ type contractCacheEntry struct { // Any backend (in-memory, SQLite-on-disk, remote) is acceptable — the // indexer's mutation paths go through the Store interface methods only, // so swapping backends is a zero-code-change configuration choice for -// callers. +// callers. Text search belongs to the store: a store implementing +// graph.SymbolSearcher answers queries from its own FTS, and every other +// store gets a null text backend whose empty corpus routes the query +// engine to its substring fallback. The indexer builds no text index of +// its own for either. func New(g graph.Store, reg *parser.Registry, cfg config.IndexConfig, logger *zap.Logger) *Indexer { idx := &Indexer{ graph: g, @@ -514,13 +518,11 @@ func New(g graph.Store, reg *parser.Registry, cfg config.IndexConfig, logger *za // Subsequent reassignments to idx.search should use the swap // helpers below. // - // When the backing store implements graph.SymbolSearcher - // (today only store_sqlite), the initial backend is a thin - // adapter that forwards Search to the store's native FTS. - // The in-process BM25 build path is then bypassed entirely — - // saving ~100MB heap on a Vscode-scale repo and putting - // search in the same address space as the rest of the graph - // queries. + // initialSearchBackend picks the text side: the store's own FTS + // when it implements graph.SymbolSearcher (today only + // store_sqlite), otherwise the null backend. Neither holds a + // corpus in this process, so search costs no heap beyond what the + // store already spends on the graph itself. search: search.NewSwappable(initialSearchBackend(g)), config: cfg, transforms: newTransformPipeline(cfg.Transforms, logger), @@ -646,10 +648,10 @@ func docSummary(doc string) string { return doc } -// vectorSearcherDelegate is the search.VectorDelegate-shaped -// adapter the indexer hands to VectorBackend.SetDelegate when the -// underlying store implements graph.VectorSearcher. SimilarTo just -// forwards — search.VectorDelegate is defined to return +// vectorSearcherDelegate is the search.VectorDelegate-shaped adapter the +// indexer passes to search.NewDelegatedVector when the underlying store +// implements graph.VectorSearcher. SimilarTo just forwards — +// search.VectorDelegate is defined to return // graph.VectorHit slices directly, so there's no translation work // here, just a small struct so the in-process search package // doesn't depend on graph.VectorSearcher's full surface. @@ -667,47 +669,30 @@ func (d *vectorSearcherDelegate) SimilarTo(vec []float32, limit int) ([]graph.Ve // initialSearchBackend picks the search.Backend the indexer wraps // in its Swappable on construction. When the underlying store // implements graph.SymbolSearcher (today only store_sqlite), a -// thin adapter routes Search calls through the store's native FTS -// — the in-process BM25 build path is bypassed entirely. Otherwise -// falls through to search.NewAuto's in-memory BM25 index. +// thin adapter routes Search calls through the store's native FTS. +// Every other store gets the null backend: it indexes nothing and +// reports an empty corpus, so the query engine answers from its own +// substring scan rather than from a second, in-process copy of the +// corpus. Those are the only two shapes, which is why no code path +// builds a text index in this process. func initialSearchBackend(g graph.Store) search.Backend { if s, ok := g.(graph.SymbolSearcher); ok { return search.NewSymbolSearcherBackend(s) } - return search.NewAuto() -} - -// isSymbolSearcherBackend reports whether the swappable's currently -// active backend is the SymbolSearcher adapter. Used to suppress the -// in-process index builds — if the active backend is already a native -// FTS, re-indexing the same corpus into a parallel in-process index -// would defeat the FTS path and pin the ~100MB heap the FTS -// integration was meant to release. -func isSymbolSearcherBackend(b search.Backend) bool { - switch backend := b.(type) { - case *search.SymbolSearcherBackend: - return true - case *search.Swappable: - inner, release := backend.AcquireBackend() - defer release() - return isSymbolSearcherBackend(inner) - case *search.HybridBackend: - return isSymbolSearcherBackend(backend.TextBackend()) - default: - return false - } + return search.NewNull() } // ftsTokensFor produces the pre-tokenised text the backend FTS path // indexes. Mirrors searchIndexFields' field selection but joins // every field through search.Tokenize (camelCase / snake_case / -// path-segment splitter) so the resulting token list matches the -// in-process BM25 corpus contract — the same query produces the -// same recall against either backend. Joined with spaces so the +// path-segment splitter) — the same splitter buildFTSMatch runs over +// an incoming query, so a query token can only match a document token +// that was split the same way. Diverge here and identifier queries +// stop reaching the symbols that carry them. Joined with spaces so the // downstream COPY FROM sees a single STRING column value. func ftsTokensFor(n *graph.Node, projectName string) string { // searchIndexFields includes the resolver qualifier or its retrieval-only - // replacement, so both BM25 backends and embeddings see the same token bag. + // replacement, so both the FTS documents and embeddings see the same token bag. fields := searchIndexFields(n, projectName) tokens := make([]string, 0, 16) for _, f := range fields { @@ -716,6 +701,7 @@ func ftsTokensFor(n *graph.Node, projectName string) string { } tokens = append(tokens, search.Tokenize(f)...) } + tokens = search.NormalizeFTSTokens(tokens) if len(tokens) == 0 { return "" } @@ -741,8 +727,9 @@ const ( // Nodes stream through the store's bounded scoped projection instead of a // whole-repository read: the caller reached this path precisely because the // repository does not fit in memory. Backends without the replace/projection -// capabilities (test fixtures) are a no-op — their search index is populated -// by buildSearchIndex instead. +// capabilities (test fixtures and in-memory stores) are a no-op — their +// search backend stays empty and the query engine uses its graph substring +// fallback. // // The replacement is one atomic unit: a rebuild that fails part-way leaves the // corpus the repository already had. Wiping first and appending chunk by chunk @@ -824,8 +811,8 @@ func (idx *Indexer) populateSymbolFTS(reporter progress.Reporter) error { // searchable symbols. Beyond that, config.SkipSearch filters out // (language, kind) pairs that would only add noise — JSON/YAML/TOML // keys, CSS tokens, Terraform blocks, shell/build variables. Every -// text-index call site (buildSearchIndex bulk loop, indexFile -// incremental add) must go through this predicate so they can't drift. +// FTS writer (shadow drain, direct rebuild, and incremental mutation) must go +// through this predicate so the persisted corpora cannot drift. func (idx *Indexer) shouldIndexForSearch(n *graph.Node) bool { // Cross-daemon proxy-edge nodes stand in for remote symbols; they // are never surfaced in local name search. Inert until @@ -1480,8 +1467,8 @@ func (idx *Indexer) SetProjectID(id string) { idx.projectID = id } // ProjectID returns the project slug this indexer stamps on nodes. func (idx *Indexer) ProjectID() string { return idx.projectID } -// SetEmbedder sets the embedding provider for semantic search. -// When set, buildSearchIndex will create a HybridBackend with vector search. +// SetEmbedder sets the embedding provider for semantic search. When set, the +// vector prepare/install pipeline publishes a HybridBackend with vector search. func (idx *Indexer) SetEmbedder(p embedding.Provider) { idx.embedder = p } // SetEmbeddingChunkOptions tunes the AST sub-chunking applied to large @@ -1492,8 +1479,8 @@ func (idx *Indexer) SetEmbeddingChunkOptions(opts embedding.ChunkOptions) { } // SetEmbeddingMaxSymbols overrides the cap on how many texts the vector -// index will hold before buildSearchIndex skips the embed pass. Zero -// keeps the built-in default. +// preparation pass accepts before skipping embeddings. Zero keeps the built-in +// default. func (idx *Indexer) SetEmbeddingMaxSymbols(n int) { idx.embedMaxSymbols = n } // SetEmbeddingAPIConcurrency overrides how many embedding requests run @@ -2660,6 +2647,9 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index // into the durable store, which is far cheaper than writing each node // and edge through as it is parsed. idx.indexCount.Add(1) + if err := idx.markSymbolFTSNormalizationPending(idx.graph); err != nil { + return nil, err + } diskTarget = idx.graph inMemShadow = idx.newStructuralIntegrityShadow(diskTarget, graph.StructuralPathShadowCold) idx.graph = inMemShadow @@ -2836,6 +2826,11 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index retErr = fmt.Errorf("indexer: finalize backend FTS: %w", ferr) } } + if retErr == nil && ftsReady { + if err := idx.markSymbolFTSNormalization(diskTarget); err != nil { + retErr = err + } + } reporter.Report("building symbol fts", 1, 1) } reporter.Report("persisting bulk graph", 1, 1) @@ -2889,8 +2884,16 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index if retErr != nil { return } + if err := idx.markSymbolFTSNormalizationPending(idx.graph); err != nil { + retErr = err + return + } if err := idx.populateSymbolFTS(reporter); err != nil { retErr = err + return + } + if err := idx.markSymbolFTSNormalization(idx.graph); err != nil { + retErr = err } }() } @@ -3978,8 +3981,9 @@ func (idx *Indexer) repoNodeEdgeCount() (int, int) { // cleanCensusResult publishes the same zero-delta state as the full-root // incremental pipeline after ChangedSinceMtimes has already proved the tree -// unchanged. It preserves the one necessary side effect for non-persistent -// search backends without repeating filesystem discovery. +// unchanged. The store already owns the native text corpus, but the process-local +// vector channel must still be restored from durable corpus statistics (or rebuilt +// once when migration left that corpus empty). func (idx *Indexer) cleanCensusResult(ctx context.Context, detected int, started time.Time) (*IndexResult, error) { if idx.totalDetected == 0 { idx.totalDetected = detected @@ -3997,13 +4001,14 @@ func (idx *Indexer) cleanCensusResult(ctx context.Context, detected int, started idx.lastVectorBuildErr = restoreErr idx.logger.Warn("restore durable vector corpus failed; rebuilding", zap.Error(restoreErr)) } - if restored { - idx.rebuildTextSearchIndex() - } else if !isSymbolSearcherBackend(idx.search) || idx.embedder != nil { + if !restored && idx.embedder != nil { if err := idx.buildSearchIndexCtx(ctx); err != nil { return nil, err } } + if _, err := idx.reconcileSymbolFTSNormalization(nil); err != nil { + return nil, err + } nodes, edges := idx.repoNodeEdgeCount() fileCount := idx.trackedFileCount() @@ -4956,8 +4961,8 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) { // embeddingDimsOrDefault returns the embedder's reported vector width, // falling back to a neutral placeholder only when the provider cannot // state its width yet (Dimensions() == 0, the APIProvider-before-first- -// call case). The fallback is never persisted: buildSearchIndex -// overwrites it with the true width taken from a real vector. Kept as a +// call case). The fallback is never persisted: vector-plan preparation +// replaces it with the true width taken from a real vector. Kept as a // named helper so the vector-dimension default has one definition // instead of a scattered magic number. func embeddingDimsOrDefault(p embedding.Provider) int { @@ -5281,16 +5286,6 @@ func flattenEmbedResults(results [][][]float32) [][]float32 { return out } -// buildSearchIndex is the compatibility entry point for direct and focused -// callers. Full indexing uses the context-aware preparation/publication split -// so a cold shadow can defer publication until its durable drain succeeds. -func (idx *Indexer) buildSearchIndex() { - if err := idx.buildSearchIndexCtx(context.Background()); err != nil { - idx.lastVectorBuildErr = err - idx.logger.Warn("vector index build canceled", zap.Error(err)) - } -} - // dirIgnoreFiles are the per-directory ignore-file basenames honored by // the index walk, siblings to .gitignore: Gortex's own .gortexignore // plus ripgrep's .ignore and .rgignore. Patterns in each file are @@ -5698,6 +5693,13 @@ func (idx *Indexer) incrementalReindexPathsMode( } idx.storeRootPath(absRoot) + // Reconcile the complete durable corpus before any scoped mutation writes + // rows with this process's normalization mode. Doing this after a partial + // update would leave unchanged symbols in the previous mode. + if _, err := idx.reconcileSymbolFTSNormalization(nil); err != nil { + return nil, err + } + // scopeRels holds the repo-relative slash-paths the caller asked to // reindex — used both to drive the discovery walk and to bound // deletion detection to the scoped subtree. @@ -5944,15 +5946,10 @@ func (idx *Indexer) incrementalReindexPathsMode( idx.reresolveCSharpGlobalUsingDependents(evictedGlobalsFiles, evicted) } - // Structural and metadata refreshes maintain both the in-memory search - // backend and persistent FTS one symbol at a time; deletions remove the - // prior symbols above. Rebuilding the whole corpus after a tiny edit would - // re-embed every unchanged symbol. The only rebuild retained here is the - // zero-delta bootstrap for a non-persistent backend restored beside an - // already-populated graph. - if len(staleFiles) == 0 && len(deletedFiles) == 0 && !isSymbolSearcherBackend(idx.search) { - idx.buildSearchIndex() - } + // No search rebuild here. Structural and metadata refreshes maintain the + // store's FTS one symbol at a time and deletions remove the prior symbols + // above, so the text corpus is already current; rebuilding it after a tiny + // edit would only re-embed every unchanged symbol. if len(staleFiles) > 0 || len(deletedFiles) > 0 { // Contract extraction is file-bounded even for body-only and metadata diff --git a/internal/indexer/indexer_test.go b/internal/indexer/indexer_test.go index 0ead3a056..4dccc89fa 100644 --- a/internal/indexer/indexer_test.go +++ b/internal/indexer/indexer_test.go @@ -13,6 +13,7 @@ import ( "github.com/zzet/gortex/internal/config" "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" "github.com/zzet/gortex/internal/parser" "github.com/zzet/gortex/internal/parser/languages" ) @@ -72,6 +73,54 @@ func newTestIndexer(g graph.Store) *Indexer { return New(g, reg, cfg, zap.NewNop()) } +// newFTSStore opens a throwaway sqlite store — the only Store that carries a +// native symbol FTS, and the one production always runs on. +func newFTSStore(t *testing.T) *store_sqlite.Store { + t.Helper() + store, err := store_sqlite.Open(filepath.Join(t.TempDir(), "fts.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + return store +} + +// newFTSIndexer indexes dir into a real sqlite store and returns both. It is +// the fixture for every assertion about what indexing makes SEARCHABLE: +// production always runs on a store whose native FTS answers symbol search, so +// idx.Search() here is the SymbolSearcherBackend reading store.SearchSymbols +// rather than an in-process index the daemon never builds. +// +// prepare runs on the indexer before Index, for the SetEmbedder / +// SetEmbeddingChunkOptions wiring some fixtures need. +func newFTSIndexer( + t *testing.T, + dir string, + reg *parser.Registry, + cfg config.IndexConfig, + prepare ...func(*Indexer), +) (*Indexer, *store_sqlite.Store) { + t.Helper() + store := newFTSStore(t) + idx := New(store, reg, cfg, zap.NewNop()) + for _, p := range prepare { + p(idx) + } + _, err := idx.Index(dir) + require.NoError(t, err) + requireSymbolFTS(t, store) + return idx, store +} + +// requireSymbolFTS fails unless the store's symbol corpus actually holds +// something. An empty corpus satisfies every "must NOT be searchable" +// assertion for the wrong reason and turns the whole fixture into a silent +// pass, so no search test may run without this gate. +func requireSymbolFTS(t *testing.T, store *store_sqlite.Store) { + t.Helper() + count, err := store.SymbolFTSCount() + require.NoError(t, err) + require.Greater(t, count, 0, "fixture left the symbol FTS empty — search assertions would be vacuous") +} + // newTestIndexerGoJava registers both the Go and Java extractors — used by // the cross-language Temporal join tests. func newTestIndexerGoJava(g graph.Store) *Indexer { diff --git a/internal/indexer/lifecycle_acceptance_test.go b/internal/indexer/lifecycle_acceptance_test.go index 59911baca..cfa6c8185 100644 --- a/internal/indexer/lifecycle_acceptance_test.go +++ b/internal/indexer/lifecycle_acceptance_test.go @@ -29,7 +29,7 @@ func newLifecycleTestMultiIndexer(t *testing.T) *MultiIndexer { return NewMultiIndexer( graph.New(), newTestRegistry(), - search.NewBM25(), + search.NewNull(), newTestConfigManager(t), zap.NewNop(), ) diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index cf6ce0ef2..2074c6dae 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -201,7 +201,7 @@ type MultiIndexer struct { // SetEmbedder installs the embedding provider every per-repo indexer // should use. Must be called before IndexAll / TrackRepo for vectors // to land in the graph — without this the fresh Indexer created per -// repo has embedder=nil and buildSearchIndex skips the vector pass. +// repo has embedder=nil and vector-plan preparation skips the embedding pass. // Safe to call zero or one times; subsequent calls silently replace. func (mi *MultiIndexer) SetEmbedder(e embedding.Provider) { mi.mu.Lock() diff --git a/internal/indexer/multi_case_identity_test.go b/internal/indexer/multi_case_identity_test.go index 370fc2724..4ee9829f0 100644 --- a/internal/indexer/multi_case_identity_test.go +++ b/internal/indexer/multi_case_identity_test.go @@ -103,7 +103,7 @@ func TestScopeForCWD_CaseMismatchedWorkspaceRoot(t *testing.T) { require.NoError(t, gc.Save()) cm, err := config.NewConfigManager(tmpCfg) require.NoError(t, err) - mi := NewMultiIndexer(graph.New(), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(graph.New(), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/indexer/multi_cold_orchestration_test.go b/internal/indexer/multi_cold_orchestration_test.go index 2ca5cfe9e..db827db35 100644 --- a/internal/indexer/multi_cold_orchestration_test.go +++ b/internal/indexer/multi_cold_orchestration_test.go @@ -48,7 +48,7 @@ func coldOrchestrationRepos(t *testing.T, count int) []config.RepoEntry { } func newColdOrchestrationMulti(store graph.Store, logger *zap.Logger) *MultiIndexer { - return NewMultiIndexer(store, coldOrchestrationRegistry(), search.NewAuto(), nil, logger) + return NewMultiIndexer(store, coldOrchestrationRegistry(), search.NewNull(), nil, logger) } type coldOrchestrationProbeStore struct { diff --git a/internal/indexer/multi_contract_edges_test.go b/internal/indexer/multi_contract_edges_test.go index 9c7177922..fe42c857e 100644 --- a/internal/indexer/multi_contract_edges_test.go +++ b/internal/indexer/multi_contract_edges_test.go @@ -111,7 +111,7 @@ func TestReconcileContractEdges_BridgesConsumerToProvider(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) @@ -205,7 +205,7 @@ func TestReconcileContractEdges_TemplateLiteralConsumer(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) @@ -287,7 +287,7 @@ func TestReconcileContractEdges_DartConsumer(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) @@ -447,7 +447,7 @@ func TestInlineWrappers_TuckShape(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) @@ -553,7 +553,7 @@ func TestReconcileContractEdges_TopicBridge(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) @@ -607,7 +607,7 @@ func TestEnvConsumer_SymbolIDSet(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) @@ -724,7 +724,7 @@ func TestReconcileContractEdges_GRPCBridge(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) @@ -840,7 +840,7 @@ func TestReconcileContractEdges_OpenAPIBridge(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) @@ -911,7 +911,7 @@ func TestReconcileContractEdges_PurgesStaleOnUntrack(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) diff --git a/internal/indexer/multi_global_passes_test.go b/internal/indexer/multi_global_passes_test.go index c2869d845..02328a35b 100644 --- a/internal/indexer/multi_global_passes_test.go +++ b/internal/indexer/multi_global_passes_test.go @@ -82,7 +82,7 @@ func TestMultiIndexer_IndexAll_GlobalPassesProduceEdges(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexAll() require.NoError(t, err) @@ -134,7 +134,7 @@ func TestMultiIndexer_GlobalGraphPassPipeline_Idempotent(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -173,7 +173,7 @@ func TestMultiIndexer_BeginEndBatch_DefersGlobalPasses(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) mi.BeginBatch() @@ -213,7 +213,7 @@ func TestMultiIndexer_TrackRepoCtx_NoBatch_RunsGlobalPassesInline(t *testing.T) require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.TrackRepoCtx(context.Background(), config.RepoEntry{Path: repoA, Name: "repo-a"}) require.NoError(t, err) diff --git a/internal/indexer/multi_node_id_test.go b/internal/indexer/multi_node_id_test.go index 3ede32256..3d2bc71c6 100644 --- a/internal/indexer/multi_node_id_test.go +++ b/internal/indexer/multi_node_id_test.go @@ -87,7 +87,7 @@ func TestMultiRepo_ResolvesCallEdges(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) @@ -164,7 +164,7 @@ func TestTrackRepoCtx_FirstOfManyStillGetsPrefix(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) // Simulate warmupDaemonState's loop: TrackRepoCtx each config'd repo // in order. The first call is the one that used to skip prefixing. @@ -275,7 +275,7 @@ rules: g := graph.New() registry := newTestRegistry() registry.Register(languages.NewYAMLExtractor()) - mi := NewMultiIndexer(g, registry, search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, registry, search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "tracking %s", entry.Name) diff --git a/internal/indexer/multi_reconcile_sqlite_test.go b/internal/indexer/multi_reconcile_sqlite_test.go index 89760ee8d..6594ab352 100644 --- a/internal/indexer/multi_reconcile_sqlite_test.go +++ b/internal/indexer/multi_reconcile_sqlite_test.go @@ -41,7 +41,7 @@ func TestReconcileRepoCtx_Sqlite_FullRetrackFlag(t *testing.T) { // First "daemon run": index the repo on the disk-backed store and // capture mtimes as if we were writing a warm-restart snapshot. - mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -54,7 +54,7 @@ func TestReconcileRepoCtx_Sqlite_FullRetrackFlag(t *testing.T) { // down the whole-repo re-track branch. writeFile(t, filepath.Join(repoPath, "b.go"), "package main\nfunc Beta() {}\n") - mi2 := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi2 := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) result, err := mi2.ReconcileRepoCtx(context.Background(), config.RepoEntry{Path: repoPath, Name: "repo"}, priorMtimes) require.NoError(t, err) require.NotNil(t, result) @@ -70,7 +70,7 @@ func TestReconcileRepoCtx_Sqlite_FullRetrackFlag(t *testing.T) { require.NotNil(t, meta2) unchangedMtimes := mi2.FileMtimes("repo") - mi3 := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi3 := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) result2, err := mi3.ReconcileRepoCtx(context.Background(), config.RepoEntry{Path: repoPath, Name: "repo"}, unchangedMtimes) require.NoError(t, err) require.NotNil(t, result2) diff --git a/internal/indexer/multi_reconcile_test.go b/internal/indexer/multi_reconcile_test.go index e58c9a75f..6a4cc1b45 100644 --- a/internal/indexer/multi_reconcile_test.go +++ b/internal/indexer/multi_reconcile_test.go @@ -48,7 +48,7 @@ func TestReconcileRepoCtx_EvictsOfflineDeletions(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = s.Close() }) g := graph.Store(s) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -66,7 +66,7 @@ func TestReconcileRepoCtx_EvictsOfflineDeletions(t *testing.T) { // Second "daemon run": fresh MultiIndexer, graph already populated // from the "snapshot", reconcile with prior mtimes. - mi2 := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi2 := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi2.ReconcileRepoCtx(context.Background(), config.RepoEntry{Path: repoPath, Name: "repo"}, priorMtimes) require.NoError(t, err) @@ -97,7 +97,7 @@ func TestReconcileRepoCtx_DoesNotDuplicateUnchanged(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -105,7 +105,7 @@ func TestReconcileRepoCtx_DoesNotDuplicateUnchanged(t *testing.T) { priorMtimes := mi.FileMtimes("repo") // Simulate restart: fresh MultiIndexer on the same graph, reconcile. - mi2 := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi2 := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi2.ReconcileRepoCtx(context.Background(), config.RepoEntry{Path: repoPath, Name: "repo"}, priorMtimes) require.NoError(t, err) @@ -140,7 +140,7 @@ func TestReconcileRepoCtx_RunsDerivedPassesForOfflineChange(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = s.Close() }) g := graph.Store(s) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) priorMtimes := mi.FileMtimes("repo") @@ -150,7 +150,7 @@ import "os/exec" func Run() error { return exec.Command("true").Run() } `) - mi2 := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi2 := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) result, err := mi2.ReconcileRepoCtx( context.Background(), config.RepoEntry{Path: repoPath, Name: "repo"}, priorMtimes, ) @@ -178,7 +178,7 @@ func TestReconcileAll_RunsDerivedPassesForMissedChange(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -215,7 +215,7 @@ func TestReconcileAllCtx_PreservesExistingBatchFlags(t *testing.T) { cm, err := config.NewConfigManager(cfgPath) require.NoError(t, err) - mi := NewMultiIndexer(graph.New(), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(graph.New(), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -268,7 +268,7 @@ func TestReconcileAll_CatchesJanitorTargets(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/indexer/multi_refresh_config_test.go b/internal/indexer/multi_refresh_config_test.go index b42a70c0f..cc2cc7913 100644 --- a/internal/indexer/multi_refresh_config_test.go +++ b/internal/indexer/multi_refresh_config_test.go @@ -35,7 +35,7 @@ func Dropped() {} require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) require.NotEmpty(t, g.FindNodesByName("Dropped"), "precondition: drop.go is indexed") @@ -58,6 +58,6 @@ func Dropped() {} } func TestMultiIndexer_RefreshRepoConfigs_NoConfigManager(t *testing.T) { - mi := NewMultiIndexer(graph.New(), newTestRegistry(), search.NewBM25(), nil, zap.NewNop()) + mi := NewMultiIndexer(graph.New(), newTestRegistry(), search.NewNull(), nil, zap.NewNop()) assert.Equal(t, 0, mi.RefreshRepoConfigs()) } diff --git a/internal/indexer/multi_scoped_test.go b/internal/indexer/multi_scoped_test.go index 900b86b84..fc0281449 100644 --- a/internal/indexer/multi_scoped_test.go +++ b/internal/indexer/multi_scoped_test.go @@ -40,7 +40,7 @@ func TestMultiIndexer_IndexScoped_WorkspaceFromGortexYAML(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexScoped("alpha", "") require.NoError(t, err) @@ -72,7 +72,7 @@ func TestMultiIndexer_IndexScoped_RepoEntryOverride(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) // Override wins over .gortex.yaml: "upstream" matches nothing. _, err = mi.IndexScoped("upstream", "") @@ -105,7 +105,7 @@ func TestMultiIndexer_IndexScoped_FallsBackToRepoPrefix(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexScoped("repo-b", "") require.NoError(t, err) @@ -139,7 +139,7 @@ func TestMultiIndexer_IndexScoped_ProjectNarrowsInsideWorkspace(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexScoped("shared", "api") require.NoError(t, err) @@ -164,7 +164,7 @@ func TestMultiIndexer_IndexScoped_NoMatchErrors(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexScoped("nonexistent", "") require.Error(t, err) @@ -195,7 +195,7 @@ func TestMultiIndexer_IndexScoped_EmptyFiltersEqualsIndexAll(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexScoped("", "") require.NoError(t, err) diff --git a/internal/indexer/multi_singlerepo_resolve_test.go b/internal/indexer/multi_singlerepo_resolve_test.go index f466ffe23..fdab98bf3 100644 --- a/internal/indexer/multi_singlerepo_resolve_test.go +++ b/internal/indexer/multi_singlerepo_resolve_test.go @@ -29,7 +29,7 @@ func indexSingleRepoForTest(t *testing.T) (*MultiIndexer, string) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) return mi, dir @@ -54,7 +54,7 @@ func indexTwoReposForTest(t *testing.T) (*MultiIndexer, string, string) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) return mi, repoA, repoB @@ -135,7 +135,7 @@ func TestMultiIndexer_RunPreEnrichResolve_BindsInboundCrossRepoEdge(t *testing.T inbound := &graph.Edge{From: "repo-a/a.go::Caller", To: "unresolved::Foo", Kind: graph.EdgeCalls, FilePath: "repo-a/a.go", Line: 5} g.AddEdge(inbound) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) // Scoped warm restart: only the provider repo-b re-indexed. require.NoError(t, mi.RunPreEnrichResolve(context.Background(), map[string]struct{}{"repo-b": {}}, nil)) diff --git a/internal/indexer/multi_test.go b/internal/indexer/multi_test.go index a9e9c09ec..e80357687 100644 --- a/internal/indexer/multi_test.go +++ b/internal/indexer/multi_test.go @@ -51,7 +51,7 @@ func Hello() {} func TestNewMultiIndexer(t *testing.T) { g := graph.New() reg := newTestRegistry() - s := search.NewBM25() + s := search.NewNull() cm := newTestConfigManager(t) mi := NewMultiIndexer(g, reg, s, cm, zap.NewNop()) @@ -76,7 +76,7 @@ func TestMultiIndexer_IndexAll_SingleRepo(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexAll() require.NoError(t, err) @@ -122,7 +122,7 @@ func TestMultiIndexer_IndexAll_MultiRepo(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexAll() require.NoError(t, err) @@ -160,7 +160,7 @@ func TestMultiIndexer_IndexAll_SingleRepoLoadsWorkspaceExclude(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -196,7 +196,7 @@ func TestMultiIndexer_IndexAll_MultiRepoLoadsWorkspaceExclude(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -230,7 +230,7 @@ func TestMultiIndexer_IndexRepo(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -258,7 +258,7 @@ func TestMultiIndexer_IndexRepo(t *testing.T) { func TestMultiIndexer_IndexRepo_NotFound(t *testing.T) { g := graph.New() cm := newTestConfigManager(t) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err := mi.IndexRepo("nonexistent") assert.Error(t, err) @@ -270,7 +270,7 @@ func TestMultiIndexer_TrackRepo(t *testing.T) { g := graph.New() cm := newTestConfigManager(t) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) result, err := mi.TrackRepo(config.RepoEntry{Path: dir, Name: "tracked"}) require.NoError(t, err) @@ -290,7 +290,7 @@ func TestMultiIndexer_TrackRepo(t *testing.T) { } // TestMultiIndexer_TrackRepo_SearchSpansAllRepos verifies that scoping -// buildSearchIndex to the current repo (the perf fix that drops the +// native symbol FTS writes to the current repo (the perf fix that drops the // O(N²) re-index of every prior repo's nodes on every TrackRepo call) // does not regress search recall. After three repos are tracked, the // shared search backend must still find symbols defined in the first, @@ -320,8 +320,8 @@ func TestMultiIndexer_TrackRepo_SearchSpansAllRepos(t *testing.T) { cm, err := config.NewConfigManager(tmpCfg) require.NoError(t, err) - g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + store := newFTSStore(t) + mi := NewMultiIndexer(store, newTestRegistry(), search.NewSymbolSearcherBackend(store), cm, zap.NewNop()) for _, e := range []config.RepoEntry{ {Path: dirA, Name: "repo-aaa"}, @@ -331,10 +331,13 @@ func TestMultiIndexer_TrackRepo_SearchSpansAllRepos(t *testing.T) { _, err := mi.TrackRepo(e) require.NoError(t, err) } + requireSymbolFTS(t, store) // Query the camelCase-split tokens individually — that's how the - // BM25 backend stores them at Add time, and Search.TokenizeQuery - // doesn't perform the same camelCase split. + // write-side tokenizer stores them, and the query tokenizer doesn't + // perform the same camelCase split. Each is also a token no node + // carries as its whole name, so the exact-name short-circuit + // (store_fts.go tier 0) misses and the ranked FTS tier answers. for _, want := range []struct{ query, prefix string }{ {"alpha", "repo-aaa"}, {"beta", "repo-bbb"}, @@ -366,7 +369,7 @@ func TestMultiIndexer_TrackRepo_EmptyAfterPopulated(t *testing.T) { g := graph.New() cm := newTestConfigManager(t) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) first, err := mi.TrackRepo(config.RepoEntry{Path: populated, Name: "populated"}) require.NoError(t, err) @@ -405,7 +408,7 @@ func TestMultiIndexer_TrackRepo_EmptyAfterPopulated(t *testing.T) { func TestMultiIndexer_TrackRepo_InvalidPath(t *testing.T) { g := graph.New() cm := newTestConfigManager(t) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err := mi.TrackRepo(config.RepoEntry{Path: "/nonexistent/path/xyz"}) assert.Error(t, err) @@ -418,7 +421,7 @@ func TestMultiIndexer_TrackRepo_NotADirectory(t *testing.T) { g := graph.New() cm := newTestConfigManager(t) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err := mi.TrackRepo(config.RepoEntry{Path: tmpFile}) assert.Error(t, err) @@ -443,7 +446,7 @@ func TestMultiIndexer_UntrackRepo(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -467,7 +470,7 @@ func TestMultiIndexer_UntrackRepo(t *testing.T) { func TestMultiIndexer_UntrackRepo_NotFound(t *testing.T) { g := graph.New() cm := newTestConfigManager(t) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) nodesRemoved, edgesRemoved := mi.UntrackRepo("nonexistent") assert.Equal(t, 0, nodesRemoved) @@ -492,7 +495,7 @@ func TestMultiIndexer_RepoForFile(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -519,7 +522,7 @@ func TestMultiIndexer_GetIndexer(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -532,7 +535,7 @@ func TestMultiIndexer_GetIndexer(t *testing.T) { func TestMultiIndexer_IndexAll_EmptyRepos(t *testing.T) { cm := newTestConfigManager(t) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexAll() require.NoError(t, err) @@ -586,7 +589,7 @@ func TestPropertyNodeIDFormat(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexAll() require.NoError(t, err) @@ -619,7 +622,7 @@ func TestPropertyNodeIDFormat(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexAll() require.NoError(t, err) @@ -694,7 +697,7 @@ func TestPropertyReindexIsolation(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -779,7 +782,7 @@ func TestLoneRepo_ConfigCompat(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -850,7 +853,7 @@ guards: require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -893,7 +896,7 @@ func TestGrowWorkspaceFromOneRepoToTwo(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -931,7 +934,7 @@ func TestGrowWorkspaceFromOneRepoToTwo(t *testing.T) { require.NoError(t, err) g2 := graph.New() - mi2 := NewMultiIndexer(g2, newTestRegistry(), search.NewBM25(), cm2, zap.NewNop()) + mi2 := NewMultiIndexer(g2, newTestRegistry(), search.NewNull(), cm2, zap.NewNop()) results, err := mi2.IndexAll() require.NoError(t, err) @@ -971,7 +974,7 @@ func TestScopedReindexPreservesRepoMetadataFileCount(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/indexer/multi_topic_edges_test.go b/internal/indexer/multi_topic_edges_test.go index 66b06505e..49f31a3de 100644 --- a/internal/indexer/multi_topic_edges_test.go +++ b/internal/indexer/multi_topic_edges_test.go @@ -74,7 +74,7 @@ func TestReconcileContractEdges_TopicEdges_KafkaPair(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) @@ -141,7 +141,7 @@ func TestReconcileContractEdges_TopicEdges_CrossBrokerIsolation(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) @@ -188,7 +188,7 @@ func TestReconcileContractEdges_TopicEdges_MultiConsumerFanout(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) @@ -249,7 +249,7 @@ func TestReconcileContractEdges_TopicEdges_CrossWorkspaceIsolation(t *testing.T) require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) diff --git a/internal/indexer/multi_transition_test.go b/internal/indexer/multi_transition_test.go index 287adfca1..c4263cdff 100644 --- a/internal/indexer/multi_transition_test.go +++ b/internal/indexer/multi_transition_test.go @@ -28,7 +28,7 @@ func TestTrackRepo_FirstRepoIsPrefixedFromTheStart(t *testing.T) { cm := newTestConfigManager(t) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) ctx := context.Background() _, err := mi.TrackRepoCtx(ctx, config.RepoEntry{Path: dirA, Name: "repo-a"}) @@ -58,7 +58,7 @@ func TestUntrackRepo_EvictsTheReposNodes(t *testing.T) { cm := newTestConfigManager(t) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err := mi.TrackRepoCtx(context.Background(), config.RepoEntry{Path: dir, Name: "repo-a"}) require.NoError(t, err) @@ -92,7 +92,7 @@ func TestRepoRoot_EmptyPrefixResolvesTheSoleRepo(t *testing.T) { cm := newTestConfigManager(t) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) ctx := context.Background() _, err := mi.TrackRepoCtx(ctx, config.RepoEntry{Path: dirA, Name: "repo-a"}) @@ -135,7 +135,7 @@ func TestResolveFilePath_RepoNameMatchingOwnSubdirectory(t *testing.T) { cm := newTestConfigManager(t) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err := mi.TrackRepoCtx(context.Background(), config.RepoEntry{Path: dir, Name: "api"}) require.NoError(t, err) diff --git a/internal/indexer/multi_watcher_test.go b/internal/indexer/multi_watcher_test.go index 4fc8dec03..28dcf593b 100644 --- a/internal/indexer/multi_watcher_test.go +++ b/internal/indexer/multi_watcher_test.go @@ -50,7 +50,7 @@ func HelloB() {} {Path: repoBDir, Name: "repo-b"}, } - mi := NewMultiIndexer(g, reg, search.NewAuto(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err := mi.IndexAll() require.NoError(t, err) @@ -111,7 +111,7 @@ func Hello() {} {Path: repoDir, Name: "valid"}, } - mi := NewMultiIndexer(g, reg, search.NewAuto(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err := mi.IndexAll() require.NoError(t, err) diff --git a/internal/indexer/poller_test.go b/internal/indexer/poller_test.go index c91009417..28a555aa8 100644 --- a/internal/indexer/poller_test.go +++ b/internal/indexer/poller_test.go @@ -101,7 +101,7 @@ func TestPoller_DetectsFilesystemChangeMissedByFsnotify(t *testing.T) { g := graph.New() idx := newTestIndexer(g) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.Index(dir) require.NoError(t, err) @@ -138,7 +138,7 @@ func TestPoller_DetectsDeletedFileMissedByFsnotify(t *testing.T) { g := graph.New() idx := newTestIndexer(g) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.Index(dir) require.NoError(t, err) @@ -184,7 +184,7 @@ func TestPoller_DetectsGitHeadMoveMissedByFsnotify(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err := idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) @@ -345,7 +345,7 @@ func TestPoller_SweepHookReportsWork(t *testing.T) { g := graph.New() idx := newTestIndexer(g) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.Index(dir) require.NoError(t, err) @@ -461,7 +461,7 @@ func TestPoller_SweepUnionsGitAndFilesystemPathsIntoOneBatch(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err := idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) @@ -519,7 +519,7 @@ func TestPoller_GitSHAAdvancesOnlyAfterSuccessfulBatch(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err := idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) diff --git a/internal/indexer/pre_enrich_resolve_hooks_test.go b/internal/indexer/pre_enrich_resolve_hooks_test.go index 091786c44..9bbf2d075 100644 --- a/internal/indexer/pre_enrich_resolve_hooks_test.go +++ b/internal/indexer/pre_enrich_resolve_hooks_test.go @@ -36,7 +36,7 @@ func TestRunPreEnrichResolveFiresComputeDoneHook(t *testing.T) { inbound := &graph.Edge{From: "repo-a/a.go::Caller", To: "unresolved::Foo", Kind: graph.EdgeCalls, FilePath: "repo-a/a.go", Line: 5} g.AddEdge(inbound) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) var order []string require.NoError(t, mi.RunPreEnrichResolve(context.Background(), nil, diff --git a/internal/indexer/pre_enrich_slug_backfill_test.go b/internal/indexer/pre_enrich_slug_backfill_test.go index 8a42fc711..7ca3cf097 100644 --- a/internal/indexer/pre_enrich_slug_backfill_test.go +++ b/internal/indexer/pre_enrich_slug_backfill_test.go @@ -106,7 +106,7 @@ func newPreEnrichSlugFixture(t *testing.T, stamped bool) (*MultiIndexer, *graph. call := &graph.Edge{From: "a/a.go::Call", To: "unresolved::Serve", Kind: graph.EdgeCalls, FilePath: "a/a.go", Line: 3} base.AddEdge(call) store := base - mi := NewMultiIndexer(store, newTestRegistry(), search.NewBM25(), configMgr, zap.NewNop()) + mi := NewMultiIndexer(store, newTestRegistry(), search.NewNull(), configMgr, zap.NewNop()) mi.repos = map[string]*RepoMetadata{ "a": {RepoPrefix: "a", RootPath: roots["a"]}, "b": {RepoPrefix: "b", RootPath: roots["b"]}, diff --git a/internal/indexer/realtime_reliability_test.go b/internal/indexer/realtime_reliability_test.go index 193dba6a2..31e7aa9b2 100644 --- a/internal/indexer/realtime_reliability_test.go +++ b/internal/indexer/realtime_reliability_test.go @@ -19,6 +19,7 @@ import ( "github.com/zzet/gortex/internal/config" "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" "github.com/zzet/gortex/internal/parser" "github.com/zzet/gortex/internal/search" ) @@ -78,19 +79,33 @@ func (e *toggleExtractor) Extract(filePath string, src []byte) (*parser.Extracti return &parser.ExtractionResult{Nodes: nodes}, nil } -func newToggleIndexer(t *testing.T) (*Indexer, *toggleExtractor) { +func newToggleIndexer(t *testing.T) (*Indexer, *toggleExtractor, *store_sqlite.Store) { t.Helper() ext := &toggleExtractor{} reg := parser.NewRegistry() reg.Register(ext) - g := graph.New() - idx := New(g, reg, config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() - return idx, ext + // A real sqlite store, so idx.search is the production + // SymbolSearcherBackend over the store's native symbol FTS: the + // "search entry survived / was evicted" assertions below then pin the + // corpus the daemon actually serves. The store rides out so a test can + // gate on a non-empty corpus before trusting any of them. + store := newFTSStore(t) + idx := New(store, reg, config.IndexConfig{Workers: 1}, zap.NewNop()) + return idx, ext, store } +// tier0Dodge is appended to every searchHasID query. The sqlite backend +// short-circuits an identifier-shaped query on an exact FindNodesByName hit +// and returns before the ranked FTS tier runs (store_fts.go tier 0), so +// asking for "Alpha" would be answered straight out of the graph — these +// assertions would then restate the GetNode check on the line above them and +// prove nothing about the search corpus. A second whitespace-separated token +// makes the query non-identifier-shaped. It matches no document and the MATCH +// expression is a prefix-OR, so the real term still decides the answer. +const tier0Dodge = " zznosuchtokenzz" + func searchHasID(idx *Indexer, query, id string) bool { - for _, r := range idx.search.Search(query, 50) { + for _, r := range idx.search.Search(query+tier0Dodge, 50) { if r.ID == id { return true } @@ -108,7 +123,7 @@ func searchHasID(idx *Indexer, query, id string) bool { // evicted the graph + search entries before parsing and returned early // on result == nil, leaving the file at zero nodes. func TestIndexFile_ParseFailureKeepsPriorNodes(t *testing.T) { - idx, ext := newToggleIndexer(t) + idx, ext, store := newToggleIndexer(t) dir := t.TempDir() path := filepath.Join(dir, "main.fk") idx.SetRootPath(dir) @@ -121,6 +136,7 @@ func TestIndexFile_ParseFailureKeepsPriorNodes(t *testing.T) { funcID := "main.fk::Alpha" require.NotNil(t, idx.graph.GetNode(funcID), "Alpha must be indexed before the bad edit") + requireSymbolFTS(t, store) require.True(t, searchHasID(idx, "Alpha", funcID), "Alpha must be in the search index before the bad edit") nodesBefore := len(idx.graph.GetFileNodes("main.fk")) require.Equal(t, 2, nodesBefore, "file node + Alpha") @@ -164,7 +180,7 @@ func TestIndexFile_ParseFailureKeepsPriorNodes(t *testing.T) { // must leave the file's prior nodes / search entries intact, and a clean // modify must still swap. Against the pre-fix patchGraph this FAILS. func TestPatchGraphModify_ParseFailureKeepsPriorNodes(t *testing.T) { - idx, ext := newToggleIndexer(t) + idx, ext, store := newToggleIndexer(t) dir := t.TempDir() idx.SetRootPath(dir) path := filepath.Join(dir, "main.fk") @@ -180,6 +196,7 @@ func TestPatchGraphModify_ParseFailureKeepsPriorNodes(t *testing.T) { funcID := "main.fk::Alpha" require.NotNil(t, idx.graph.GetNode(funcID), "Alpha must be indexed via the create patch") + requireSymbolFTS(t, store) require.True(t, searchHasID(idx, "Alpha", funcID)) nodesBefore := len(idx.graph.GetFileNodes("main.fk")) require.Equal(t, 2, nodesBefore, "file node + Alpha") @@ -237,7 +254,7 @@ func TestPollGitHead_DiffFailureRetriesRange(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err = idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) @@ -290,7 +307,7 @@ func haveGit(t *testing.T) bool { // seam stands in for full-tree reconciliation so the assertion is // deterministic and platform-independent. func TestWatcher_OverflowEventTriggersReconcile(t *testing.T) { - idx, _ := newToggleIndexer(t) + idx, _, _ := newToggleIndexer(t) idx.SetRootPath(t.TempDir()) w, err := NewWatcher(idx, config.WatchConfig{Enabled: true, DebounceMs: 10}, zap.NewNop()) require.NoError(t, err) @@ -321,7 +338,7 @@ func TestWatcher_OverflowEventTriggersReconcile(t *testing.T) { // signals collapses into at most one reconcile in flight — the loop is // never blocked and the tree isn't re-walked per dropped event. func TestWatcher_OverflowReconcileCoalesces(t *testing.T) { - idx, _ := newToggleIndexer(t) + idx, _, _ := newToggleIndexer(t) idx.SetRootPath(t.TempDir()) w, err := NewWatcher(idx, config.WatchConfig{Enabled: true, DebounceMs: 10}, zap.NewNop()) require.NoError(t, err) @@ -378,7 +395,7 @@ func TestWatcher_OverflowReconcileIndexesMissedFile(t *testing.T) { reg.Register(ext) g := graph.New() idx := New(g, reg, config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) ext.setFail(false) @@ -432,7 +449,7 @@ func TestWatcher_NewSubdirScanIndexesPreWatchFile(t *testing.T) { reg.Register(ext) g := graph.New() idx := New(g, reg, config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) ext.setFail(false) @@ -477,7 +494,7 @@ func TestWatcher_NewSubdirScanIndexesPreWatchFile(t *testing.T) { // changes inside an existing directory fire their own file events. Uses // the scanFn seam. func TestWatcher_DirEventScanGating(t *testing.T) { - idx, _ := newToggleIndexer(t) + idx, _, _ := newToggleIndexer(t) dir := t.TempDir() idx.SetRootPath(dir) subdir := filepath.Join(dir, "sub") @@ -545,7 +562,7 @@ func TestWatcher_PatchPanicRecoveredNotCrash(t *testing.T) { reg.Register(ext) store := &panicOnReadStore{Store: graph.New()} idx := New(store, reg, config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() dir := t.TempDir() idx.SetRootPath(dir) path := filepath.Join(dir, "main.fk") diff --git a/internal/indexer/reconcile_clean_census_manifest_test.go b/internal/indexer/reconcile_clean_census_manifest_test.go index ab09f3fb7..b53ee59df 100644 --- a/internal/indexer/reconcile_clean_census_manifest_test.go +++ b/internal/indexer/reconcile_clean_census_manifest_test.go @@ -27,7 +27,7 @@ func TestReconcileRepoCtxRoutesManifestOnlyChurnScopedAndConverges(t *testing.T) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) }) - seed := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + seed := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = seed.IndexAll() require.NoError(t, err) prior := seed.GetIndexer("repo").FileMtimes() @@ -35,7 +35,7 @@ func TestReconcileRepoCtxRoutesManifestOnlyChurnScopedAndConverges(t *testing.T) assert.False(t, manifestTrackedByFullIndex) core, logs := observer.New(zap.DebugLevel) - firstRestart := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewBM25(), cm, zap.New(core)) + firstRestart := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewNull(), cm, zap.New(core)) result, err := firstRestart.ReconcileRepoCtx(t.Context(), entry, prior) require.NoError(t, err) require.NotNil(t, result) @@ -49,7 +49,7 @@ func TestReconcileRepoCtxRoutesManifestOnlyChurnScopedAndConverges(t *testing.T) assert.True(t, manifestTracked) core, logs = observer.New(zap.DebugLevel) - secondRestart := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewBM25(), cm, zap.New(core)) + secondRestart := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewNull(), cm, zap.New(core)) result, err = secondRestart.ReconcileRepoCtx(t.Context(), entry, convergedMtimes) require.NoError(t, err) require.NotNil(t, result) diff --git a/internal/indexer/reconcile_clean_census_merkle_route_test.go b/internal/indexer/reconcile_clean_census_merkle_route_test.go index a2611b7ce..96c581c8f 100644 --- a/internal/indexer/reconcile_clean_census_merkle_route_test.go +++ b/internal/indexer/reconcile_clean_census_merkle_route_test.go @@ -29,7 +29,7 @@ func TestReconcileRepoCtxKeepsMerkleFallbackForCleanMtimeCensus(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) }) - seed := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + seed := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = seed.IndexAll() require.NoError(t, err) prior := seed.GetIndexer("repo").FileMtimes() @@ -45,7 +45,7 @@ func TestReconcileRepoCtxKeepsMerkleFallbackForCleanMtimeCensus(t *testing.T) { // remain disabled for every Merkle-enabled repository. require.NoError(t, os.Remove(merkleTreeFile(root))) core, logs := observer.New(zap.DebugLevel) - restarted := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewBM25(), cm, zap.New(core)) + restarted := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewNull(), cm, zap.New(core)) result, err := restarted.ReconcileRepoCtx(t.Context(), entry, prior) require.NoError(t, err) require.NotNil(t, result) diff --git a/internal/indexer/reconcile_clean_census_test.go b/internal/indexer/reconcile_clean_census_test.go index 3b8b48f74..9bf781fb3 100644 --- a/internal/indexer/reconcile_clean_census_test.go +++ b/internal/indexer/reconcile_clean_census_test.go @@ -59,28 +59,6 @@ func TestChangedSinceMtimesCensusDeletesNewlyExcludedTrackedFile(t *testing.T) { assert.Zero(t, detected) } -func TestCleanCensusResultBootstrapsNonPersistentSearch(t *testing.T) { - g := graph.New() - g.AddNode(&graph.Node{ - ID: "function::Alpha", - Kind: graph.KindFunction, - Name: "Alpha", - FilePath: "a.go", - }) - idx := newTestIndexer(g) - idx.search = search.NewBM25() - idx.SetFileMtimes(map[string]int64{"a.go": 1}) - - result, err := idx.cleanCensusResult(t.Context(), 1, time.Now()) - require.NoError(t, err) - require.NotNil(t, result) - assert.Equal(t, 1, result.FileCount) - assert.Equal(t, 1, result.NodeCount) - assert.Equal(t, 1, idx.TotalDetected()) - assert.Equal(t, 1, idx.search.Count()) - require.NotEmpty(t, idx.search.Search("Alpha", 10)) -} - func TestCleanCensusRestoresDurableVectorsWithoutEmbedding(t *testing.T) { root := vectorPersistFixture(t, 1) store, err := store_sqlite.Open(filepath.Join(t.TempDir(), "store.sqlite")) @@ -171,14 +149,14 @@ func TestReconcileRepoCtxUsesCleanCensusNoOp(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) }) - seed := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + seed := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = seed.IndexAll() require.NoError(t, err) prior := seed.GetIndexer("repo").FileMtimes() before := store.Stats() core, logs := observer.New(zap.DebugLevel) - restarted := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewBM25(), cm, zap.New(core)) + restarted := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewNull(), cm, zap.New(core)) result, err := restarted.ReconcileRepoCtx(t.Context(), entry, prior) require.NoError(t, err) require.NotNil(t, result) diff --git a/internal/indexer/reconcile_scoped_routing_test.go b/internal/indexer/reconcile_scoped_routing_test.go index ff20d7be6..3a34ef1d4 100644 --- a/internal/indexer/reconcile_scoped_routing_test.go +++ b/internal/indexer/reconcile_scoped_routing_test.go @@ -94,7 +94,7 @@ func TestReconcileRepoCtx_ScopedEqualsFullIndex(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = s.Close() }) - mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) priorMtimes := mi.FileMtimes("repo") @@ -108,7 +108,7 @@ func TestReconcileRepoCtx_ScopedEqualsFullIndex(t *testing.T) { // Second "daemon run": a fresh MultiIndexer over the same persisted // store reconciles from the snapshot. - mi2 := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi2 := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) res, err := mi2.ReconcileRepoCtx(context.Background(), config.RepoEntry{Path: repoPath, Name: "repo"}, priorMtimes) require.NoError(t, err) require.NotNil(t, res) @@ -134,7 +134,7 @@ func TestReconcileRepoCtx_ScopedEqualsFullIndex(t *testing.T) { s2, err := store_sqlite.Open(filepath.Join(t.TempDir(), "golden.sqlite")) require.NoError(t, err) t.Cleanup(func() { _ = s2.Close() }) - miGold := NewMultiIndexer(graph.Store(s2), newTestRegistry(), search.NewBM25(), cm2, zap.NewNop()) + miGold := NewMultiIndexer(graph.Store(s2), newTestRegistry(), search.NewNull(), cm2, zap.NewNop()) _, err = miGold.IndexAll() require.NoError(t, err) @@ -166,7 +166,7 @@ func TestReconcileRepoCtx_Routing(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = s.Close() }) - mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) return s, cm, mi.FileMtimes("repo") @@ -174,7 +174,7 @@ func TestReconcileRepoCtx_Routing(t *testing.T) { reconcile := func(t *testing.T, s *store_sqlite.Store, cm *config.ConfigManager, repoPath string, prior map[string]int64) *IndexResult { t.Helper() - mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) res, err := mi.ReconcileRepoCtx(context.Background(), config.RepoEntry{Path: repoPath, Name: "repo"}, prior) require.NoError(t, err) require.NotNil(t, res) diff --git a/internal/indexer/repo_prefix_parity_test.go b/internal/indexer/repo_prefix_parity_test.go index 14c942ddf..4dec91d96 100644 --- a/internal/indexer/repo_prefix_parity_test.go +++ b/internal/indexer/repo_prefix_parity_test.go @@ -64,7 +64,7 @@ func TestRepoPrefixParityAfterIndexing(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, e := range entries { _, err := mi.TrackRepoCtx(context.Background(), e) require.NoError(t, err) diff --git a/internal/indexer/repository_topology_batch_test.go b/internal/indexer/repository_topology_batch_test.go index 010934183..8127fc030 100644 --- a/internal/indexer/repository_topology_batch_test.go +++ b/internal/indexer/repository_topology_batch_test.go @@ -53,7 +53,7 @@ func TestRunRepositoryTopologyBatchAllowsParallelTrackAndBlocksReach(t *testing. registry := parser.NewRegistry() registry.Register(&topologyBatchBlockingExtractor{entered: entered, release: release}) store := graph.New() - mi := NewMultiIndexer(store, registry, search.NewBM25(), configManager, zap.NewNop()) + mi := NewMultiIndexer(store, registry, search.NewNull(), configManager, zap.NewNop()) mi.BeginParallelBatch() defer mi.ResetBatch() diff --git a/internal/indexer/reresolve_guard_test.go b/internal/indexer/reresolve_guard_test.go index df5b5d61c..f0d5ecaf6 100644 --- a/internal/indexer/reresolve_guard_test.go +++ b/internal/indexer/reresolve_guard_test.go @@ -17,7 +17,7 @@ import ( // edge demoted from a concrete target to a stub keeps the incident-edge total // identical. func TestCountResolvedFileEdges(t *testing.T) { - idx, _ := newToggleIndexer(t) + idx, _, _ := newToggleIndexer(t) g := idx.graph g.AddBatch([]*graph.Node{ {ID: "a.go", Kind: graph.KindFile, Name: "a.go", FilePath: "a.go"}, @@ -39,7 +39,7 @@ func TestCountResolvedFileEdges(t *testing.T) { // enqueues a forced scoped re-resolve and bumps the regression counter, while // below-floor / symbol-removal / modest-drop cases stay quiet. func TestGuardResolvedEdgeRegression(t *testing.T) { - idx, _ := newToggleIndexer(t) + idx, _, _ := newToggleIndexer(t) w, err := NewWatcher(idx, config.WatchConfig{Enabled: true, DebounceMs: 10}, zap.NewNop()) require.NoError(t, err) diff --git a/internal/indexer/scope_for_cwd_test.go b/internal/indexer/scope_for_cwd_test.go index b6ffff12b..61bd4bc0a 100644 --- a/internal/indexer/scope_for_cwd_test.go +++ b/internal/indexer/scope_for_cwd_test.go @@ -47,7 +47,7 @@ func TestScopeForCWD_And_ReposInWorkspace(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexScoped("", "") // empty scope → index every configured repo require.NoError(t, err) @@ -146,7 +146,7 @@ func TestScopeForCWD_WorkspaceRoot(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) @@ -231,7 +231,7 @@ func TestContainedReposScope(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) @@ -313,7 +313,7 @@ func TestContainedReposScope_RefusesOverbroadRoots(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) @@ -365,7 +365,7 @@ func TestScopeForCWD_RefusesOverbroadRootSharedWorkspace(t *testing.T) { cm, err := config.NewConfigManager(tmpCfg) require.NoError(t, err) - mi := NewMultiIndexer(graph.New(), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(graph.New(), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) diff --git a/internal/indexer/search_backend_native_test.go b/internal/indexer/search_backend_native_test.go index c519e20ac..4b45a52f0 100644 --- a/internal/indexer/search_backend_native_test.go +++ b/internal/indexer/search_backend_native_test.go @@ -1,6 +1,7 @@ package indexer import ( + "context" "testing" "github.com/stretchr/testify/assert" @@ -19,29 +20,13 @@ func (s *countingSearchStore) AllNodes() []*graph.Node { return s.Store.AllNodes() } -func TestBuildSearchIndex_NativeTextWithoutVectorsSkipsNodeCensus(t *testing.T) { +func TestBuildSearchIndexCtx_WithoutVectorsSkipsNodeCensus(t *testing.T) { store := &countingSearchStore{Store: graph.New()} backend := search.NewSwappable(search.NewSymbolSearcherBackend(nil)) defer backend.Close() idx := &Indexer{graph: store, search: backend} - idx.buildSearchIndex() + assert.NoError(t, idx.buildSearchIndexCtx(context.Background())) assert.Zero(t, store.allNodesCalls) } - -func TestIsSymbolSearcherBackend_UnwrapsProductionLayers(t *testing.T) { - native := search.NewSymbolSearcherBackend(nil) - assert.True(t, isSymbolSearcherBackend(native)) - - swappable := search.NewSwappable(native) - assert.True(t, isSymbolSearcherBackend(swappable)) - - hybrid := search.NewHybrid(native, search.NewVector(1), nil) - assert.True(t, isSymbolSearcherBackend(hybrid)) - assert.True(t, isSymbolSearcherBackend(search.NewSwappable(hybrid))) - - bm25 := search.NewBM25() - defer bm25.Close() - assert.False(t, isSymbolSearcherBackend(bm25)) -} diff --git a/internal/indexer/search_count_symmetry_test.go b/internal/indexer/search_count_symmetry_test.go index 85122a592..7a45aa310 100644 --- a/internal/indexer/search_count_symmetry_test.go +++ b/internal/indexer/search_count_symmetry_test.go @@ -11,9 +11,9 @@ import ( ) // countingSearch records Add/Remove calls so a test can assert that a node -// admitted by shouldIndexForSearch is the only kind ever removed. The real -// backend's Remove is an unconditional decrement, so an eviction predicate -// broader than the admit predicate silently corrupts its count. +// admitted by shouldIndexForSearch is the only kind ever removed. Production +// backends differ in how they store search state, but their admit and eviction +// predicates must remain symmetric. type countingSearch struct { added []string removed []string @@ -45,7 +45,7 @@ func TestRemoveFromSearchOnlyEvictsWhatWouldBeIndexed(t *testing.T) { idx.removeFromSearch(n) } require.Empty(t, spy.removed, - "a node the admit predicate rejects must never reach Remove — the backend decrements unconditionally") + "a node the admit predicate rejects must never reach Remove") admitted := &graph.Node{ID: "pkg/a.go::Fn", Kind: graph.KindFunction, Name: "Fn", Language: "go"} require.True(t, idx.shouldIndexForSearch(admitted)) @@ -53,9 +53,9 @@ func TestRemoveFromSearchOnlyEvictsWhatWouldBeIndexed(t *testing.T) { require.Equal(t, []string{"pkg/a.go::Fn"}, spy.removed) } -// The count only stays honest if the two predicates agree. Simulating a -// reconcile — evict the prior nodes, re-add the fresh ones — must be -// count-neutral for an unchanged file. +// Search membership only stays stable if the two predicates agree. Simulating +// a reconcile — evict the prior nodes, re-add the fresh ones — must be neutral +// for an unchanged file. func TestReconcileOfUnchangedFileIsCountNeutral(t *testing.T) { spy := &countingSearch{} idx := &Indexer{search: spy, config: config.IndexConfig{}} diff --git a/internal/indexer/shadow_admission_test.go b/internal/indexer/shadow_admission_test.go index aede8664e..57a9f6fc4 100644 --- a/internal/indexer/shadow_admission_test.go +++ b/internal/indexer/shadow_admission_test.go @@ -193,8 +193,8 @@ func TestShadowAdmissionIsProcessWideAcrossMultiIndexers(t *testing.T) { g := graph.New() reg := parser.NewRegistry() logger := zap.NewNop() - first := NewMultiIndexer(g, reg, search.NewBM25(), nil, logger) - second := NewMultiIndexer(g, reg, search.NewBM25(), nil, logger) + first := NewMultiIndexer(g, reg, search.NewNull(), nil, logger) + second := NewMultiIndexer(g, reg, search.NewNull(), nil, logger) standalone := New(g, reg, config.IndexConfig{}, logger) if first.shadowAdmission == nil || first.shadowAdmission != second.shadowAdmission || first.shadowAdmission != standalone.shadowAdmission { diff --git a/internal/indexer/skip_search_test.go b/internal/indexer/skip_search_test.go index 65b123229..6d9475afa 100644 --- a/internal/indexer/skip_search_test.go +++ b/internal/indexer/skip_search_test.go @@ -7,20 +7,39 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/zap" "github.com/zzet/gortex/internal/config" - "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/parser" "github.com/zzet/gortex/internal/parser/languages" ) +// searchIDs runs a query through the indexer's search backend and returns the +// hit IDs. +func searchIDs(idx *Indexer, query string, limit int) []string { + hits := idx.Search().Search(query, limit) + ids := make([]string, 0, len(hits)) + for _, h := range hits { + ids = append(ids, h.ID) + } + return ids +} + // Indexing a directory with JSON + Go files must keep JSON keys out -// of the text search index by default (regression guard for the +// of the symbol search corpus by default (regression guard for the // search-backend memory blowup) while still leaving them reachable // via graph queries. Uses lowercase-only single-word identifiers so -// the BM25 query tokenizer (which does not split camelCase) hits -// exactly the tokens produced by the Add-path tokenizer. +// the query tokenizer (which does not split camelCase) hits exactly +// the tokens produced by the write-side tokenizer. +// +// The probe is ONE two-word query rather than two identifier queries, and +// that shape is load-bearing: the sqlite backend short-circuits an +// identifier-shaped query on an exact FindNodesByName hit and returns +// before the ranked FTS tier runs (store_fts.go tier 0). Asking for +// "uniquejsonkeyzzz" alone would therefore be answered out of the GRAPH — +// where the JSON key legitimately lives — and the exclusion would look +// broken even when it works. A whitespace-separated query skips tier 0, and +// the MATCH expression is a prefix-OR, so one query proves both halves at +// once: the Go symbol is in the corpus, the JSON key is not. func TestIndex_JSONVariablesSkippedFromSearchByDefault(t *testing.T) { dir := t.TempDir() @@ -35,8 +54,6 @@ func TestIndex_JSONVariablesSkippedFromSearchByDefault(t *testing.T) { func uniquegosymbolzzz() {} `), 0o644)) - g := graph.New() - reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) reg.Register(languages.NewJSONExtractor()) @@ -46,28 +63,30 @@ func uniquegosymbolzzz() {} // Mirror what ConfigManager.GetRepoConfig would do for a real run. cfg.SkipSearch = config.DefaultSkipSearch() - idx := New(g, reg, cfg, zap.NewNop()) - _, err := idx.Index(dir) - require.NoError(t, err) + idx, store := newFTSIndexer(t, dir, reg, cfg) - // Graph still carries the JSON key — SkipSearch is about the text - // index, not the graph. Users looking up a config key via + // Graph still carries the JSON key — SkipSearch is about the search + // corpus, not the graph. Users looking up a config key via // FindNodesByName / get_symbol must still find it. - jsonNodes := g.FindNodesByName("uniquejsonkeyzzz") + jsonNodes := store.FindNodesByName("uniquejsonkeyzzz") require.NotEmpty(t, jsonNodes, "JSON variable node should exist in graph") + jsonID := jsonNodes[0].ID - // Search index must include the Go symbol but NOT the JSON key. - goHits := idx.Search().Search("uniquegosymbolzzz", 10) - assert.NotEmpty(t, goHits, "Go symbol should be text-indexed") - - jsonHits := idx.Search().Search("uniquejsonkeyzzz", 10) - assert.Empty(t, jsonHits, - "JSON variable node must be excluded from text index by default (SkipSearch)") + ids := searchIDs(idx, "uniquegosymbolzzz uniquejsonkeyzzz", 10) + assert.Contains(t, ids, "main.go::uniquegosymbolzzz", + "Go symbol should be in the symbol search corpus") + assert.NotContains(t, ids, jsonID, + "JSON variable node must be excluded from the search corpus by default (SkipSearch)") } -// With SkipSearch cleared, JSON variable nodes are text-searchable -// again. Guards the config surface: users who actually want to search -// their package.json keys can opt in by overriding SkipSearch. +// With SkipSearch cleared, JSON variable nodes are searchable again. Guards +// the config surface: users who actually want to search their package.json +// keys can opt in by overriding SkipSearch. +// +// The trailing "absentneedlezzz" token exists only to make the query +// non-identifier-shaped, for the tier-0 reason spelled out above; it appears +// in no document, so whether the JSON key comes back is decided entirely by +// the ranked FTS tier. func TestIndex_JSONVariablesSearchableWhenSkipSearchCleared(t *testing.T) { dir := t.TempDir() @@ -75,8 +94,6 @@ func TestIndex_JSONVariablesSearchableWhenSkipSearchCleared(t *testing.T) { "overridekeyzzz": "value" }`), 0o644)) - g := graph.New() - reg := parser.NewRegistry() reg.Register(languages.NewJSONExtractor()) @@ -84,11 +101,12 @@ func TestIndex_JSONVariablesSearchableWhenSkipSearchCleared(t *testing.T) { cfg.Workers = 1 cfg.SkipSearch = nil // override: index everything - idx := New(g, reg, cfg, zap.NewNop()) - _, err := idx.Index(dir) - require.NoError(t, err) + idx, store := newFTSIndexer(t, dir, reg, cfg) + + nodes := store.FindNodesByName("overridekeyzzz") + require.NotEmpty(t, nodes, "JSON variable node should exist in graph") - hits := idx.Search().Search("overridekeyzzz", 10) - assert.NotEmpty(t, hits, - "JSON variable node should be text-indexed when SkipSearch is cleared") + ids := searchIDs(idx, "overridekeyzzz absentneedlezzz", 10) + assert.Contains(t, ids, nodes[0].ID, + "JSON variable node should be searchable when SkipSearch is cleared") } diff --git a/internal/indexer/skip_telemetry_test.go b/internal/indexer/skip_telemetry_test.go index 5c2f734d2..e9eec5d7a 100644 --- a/internal/indexer/skip_telemetry_test.go +++ b/internal/indexer/skip_telemetry_test.go @@ -180,7 +180,7 @@ func TestParseFailedSkipResult_RecordsError(t *testing.T) { // file's prior nodes through a transient failure (see // TestPatchGraphModify_ParseFailureKeepsPriorNodes). func TestIndex_ParseFailedSkipTelemetry(t *testing.T) { - idx, ext := newToggleIndexer(t) + idx, ext, _ := newToggleIndexer(t) ext.setFail(true) // every extraction returns an error dir := t.TempDir() diff --git a/internal/indexer/spec_launch_acceptance_test.go b/internal/indexer/spec_launch_acceptance_test.go index ef06888f2..b9ec8386d 100644 --- a/internal/indexer/spec_launch_acceptance_test.go +++ b/internal/indexer/spec_launch_acceptance_test.go @@ -94,7 +94,7 @@ func TestSpecLaunch_4_5_OrphansAreReportedPerWorkspace(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) @@ -159,7 +159,7 @@ func TestSpecLaunch_4_5_NoCrossWorkspacePairs(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) @@ -202,7 +202,7 @@ func TestSpecLaunch_4_5_FindUsagesScopedToWorkspace(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) @@ -274,7 +274,7 @@ func TestSpecLaunch_4_5_CrossWorkspaceHappyPath(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) @@ -326,7 +326,7 @@ func TestSpecLaunch_4_5_MonorepoProjectBoundary(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) diff --git a/internal/indexer/store_hygiene_indexer_test.go b/internal/indexer/store_hygiene_indexer_test.go index 61cab283c..8062eaa2b 100644 --- a/internal/indexer/store_hygiene_indexer_test.go +++ b/internal/indexer/store_hygiene_indexer_test.go @@ -37,7 +37,7 @@ func newSqliteMultiIndexer(t *testing.T, repos []config.RepoEntry) (*MultiIndexe require.NoError(t, err) t.Cleanup(func() { _ = s.Close() }) - mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) return mi, s } diff --git a/internal/indexer/storm_test.go b/internal/indexer/storm_test.go index 3774f1cf1..6bb44730e 100644 --- a/internal/indexer/storm_test.go +++ b/internal/indexer/storm_test.go @@ -202,7 +202,7 @@ func TestIndexer_IndexFileNoResolve_SkipsResolver(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) // IndexFileNoResolve populates the graph but defers cross-file diff --git a/internal/indexer/unicode_path_test.go b/internal/indexer/unicode_path_test.go index c038ec946..b79936acf 100644 --- a/internal/indexer/unicode_path_test.go +++ b/internal/indexer/unicode_path_test.go @@ -353,7 +353,7 @@ func TestGitWatcher_NonASCIIFileBranchSwitch(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err := idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) diff --git a/internal/indexer/vector_ingest_test.go b/internal/indexer/vector_ingest_test.go index 036221cb9..bde616f27 100644 --- a/internal/indexer/vector_ingest_test.go +++ b/internal/indexer/vector_ingest_test.go @@ -103,16 +103,18 @@ func indexWithEmbedder(t *testing.T, emb interface { return idx } -// TestBuildSearchIndex_DropsInvalidVectors: a mix of valid/nil/short vectors +// TestVectorPublication_DropsInvalidVectors: a mix of valid/nil/short vectors // yields a vector index populated with only the valid ones — the bad vectors // are dropped rather than poisoning the index or aborting a viable build. -func TestBuildSearchIndex_DropsInvalidVectors(t *testing.T) { +func TestVectorPublication_DropsInvalidVectors(t *testing.T) { emb := &mixedEmbedder{} idx := indexWithEmbedder(t, emb) sw, ok := idx.Search().(*search.Swappable) require.True(t, ok) - hybrid, ok := sw.Inner().(*search.HybridBackend) + backend, release := sw.AcquireBackend() + defer release() + hybrid, ok := backend.(*search.HybridBackend) require.True(t, ok, "a viable subset of vectors must still produce a HybridBackend") require.NotNil(t, hybrid.VectorIndex()) @@ -124,14 +126,16 @@ func TestBuildSearchIndex_DropsInvalidVectors(t *testing.T) { "a partially-valid build is a success, not a recorded failure") } -// TestBuildSearchIndex_AllInvalidAbortsToTextOnly: when every vector is invalid +// TestVectorPublication_AllInvalidAbortsToTextOnly: when every vector is invalid // the build must abort to text-only search, not ship a silently empty index. -func TestBuildSearchIndex_AllInvalidAbortsToTextOnly(t *testing.T) { +func TestVectorPublication_AllInvalidAbortsToTextOnly(t *testing.T) { idx := indexWithEmbedder(t, nilEmbedder{}) sw, ok := idx.Search().(*search.Swappable) require.True(t, ok) - _, isHybrid := sw.Inner().(*search.HybridBackend) + backend, release := sw.AcquireBackend() + defer release() + _, isHybrid := backend.(*search.HybridBackend) assert.False(t, isHybrid, "an all-invalid embedding pass must leave a text-only backend, not an empty vector index") assert.Error(t, idx.LastVectorBuildError(), diff --git a/internal/indexer/vector_plan.go b/internal/indexer/vector_plan.go index 50f66a12c..0fc969bf8 100644 --- a/internal/indexer/vector_plan.go +++ b/internal/indexer/vector_plan.go @@ -51,54 +51,23 @@ func (p *preparedVectorPlan) Release() { p.embeddedText = 0 } -// prepareSearchIndex updates the process-local text index, then prepares the -// vector corpus without mutating either the durable vector store or the active -// vector backend. The caller decides when publication is safe. +// prepareSearchIndex prepares the vector corpus without mutating either the +// durable vector store or the active vector backend. The caller decides when +// publication is safe. func (idx *Indexer) prepareSearchIndex(ctx context.Context) (*preparedVectorPlan, error) { if ctx == nil { ctx = context.Background() } idx.lastVectorBuildErr = nil - - // Install learned sub-word boundaries before populating an in-process BM25 - // backend. Native SQLite FTS needs neither the census nor duplicate Adds. - search.BuildAndInstallNgramBoundaries(idx.search, idx.graph) - nativeText := isSymbolSearcherBackend(idx.search) - buildVectors := idx.embedder != nil - if nativeText && !buildVectors { + if idx.embedder == nil { return nil, nil } // Content sections belong to content search, not symbol/vector search. nodes := graph.RepoCodeNodes(idx.graph, idx.repoPrefix) - if !nativeText { - for _, n := range nodes { - if idx.shouldIndexForSearch(n) { - idx.search.Add(n.ID, searchIndexFields(n, idx.projectName)...) - } - } - } - if !buildVectors { - return nil, nil - } return idx.prepareVectorPlan(ctx, nodes) } -// rebuildTextSearchIndex restores only the non-persistent text channel. It is -// used on warm startup after a durable vector corpus has been published, so a -// paid embedding pass is not repeated merely to reconstruct BM25. -func (idx *Indexer) rebuildTextSearchIndex() { - search.BuildAndInstallNgramBoundaries(idx.search, idx.graph) - if isSymbolSearcherBackend(idx.search) { - return - } - for _, n := range graph.RepoCodeNodes(idx.graph, idx.repoPrefix) { - if idx.shouldIndexForSearch(n) { - idx.search.Add(n.ID, searchIndexFields(n, idx.projectName)...) - } - } -} - func (idx *Indexer) prepareVectorPlan(ctx context.Context, nodes []*graph.Node) (*preparedVectorPlan, error) { if err := ctx.Err(); err != nil { return nil, err @@ -230,9 +199,9 @@ func (idx *Indexer) prepareVectorPlan(ctx context.Context, nodes []*graph.Node) return plan, nil } -// prepareSearchIndexForPublication preserves the historical text-only -// degradation policy for provider/cap/validation failures. Parent cancellation -// is different: it aborts the index operation and must not publish anything. +// prepareSearchIndexForPublication retains the previous vector publication on +// provider, cap, or validation failures. Parent cancellation is different: it +// aborts the index operation and must not publish anything. func (idx *Indexer) prepareSearchIndexForPublication(ctx context.Context) (*preparedVectorPlan, error) { plan, err := idx.prepareSearchIndex(ctx) if err == nil { diff --git a/internal/indexer/watcher_inert_test.go b/internal/indexer/watcher_inert_test.go index 0faed697d..34e34b50b 100644 --- a/internal/indexer/watcher_inert_test.go +++ b/internal/indexer/watcher_inert_test.go @@ -27,7 +27,7 @@ func inertTestWatcher(t *testing.T, fileName, content string) (string, *Indexer, g := graph.New() idx := newTestIndexer(g) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.Index(dir) require.NoError(t, err) @@ -265,7 +265,7 @@ func TestWatcher_OldDatabaseFirstPatchIsConservative(t *testing.T) { path := filepath.Join(dir, "main.go") writeTestFile(t, path, "package main\n\nfunc Stable() {}\n") idx := newTestIndexer(graph.New()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.Index(dir) require.NoError(t, err) diff --git a/internal/indexer/worktree_gc_test.go b/internal/indexer/worktree_gc_test.go index 2a195761f..3c15dbea4 100644 --- a/internal/indexer/worktree_gc_test.go +++ b/internal/indexer/worktree_gc_test.go @@ -67,7 +67,7 @@ func TestGCVanishedWorktrees_EvictsRemovedWorktree(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -132,7 +132,7 @@ func TestGCVanishedWorktrees_LeavesVanishedMainCheckout(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) require.False(t, mi.GetMetadata("plain").IsWorktree, @@ -180,7 +180,7 @@ func TestLinkedWorktreeRoots(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/indexer/worktree_instance_test.go b/internal/indexer/worktree_instance_test.go index ca8a21e64..a3b115a5c 100644 --- a/internal/indexer/worktree_instance_test.go +++ b/internal/indexer/worktree_instance_test.go @@ -60,7 +60,7 @@ func newWorktreeTestIndexer(t *testing.T, repos ...config.RepoEntry) (*graph.Gra cm, err := config.NewConfigManager(cfgPath) require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) return g, mi, cm } diff --git a/internal/llm/svc/assist_e2e_test.go b/internal/llm/svc/assist_e2e_test.go index db6f3d193..641114034 100644 --- a/internal/llm/svc/assist_e2e_test.go +++ b/internal/llm/svc/assist_e2e_test.go @@ -141,9 +141,9 @@ func TestE2E_AssistAgainstRealModel(t *testing.T) { t.Logf("VERDICT(hash-passwords): kept HashPassword=%v, dropped hashDiagnostics=%v", kept, dropped) }) - t.Run("VerifyRelevance_BM25", func(t *testing.T) { - query := "how does the BM25 search rank symbols" - cands := verifyBM25Scenario() + t.Run("VerifyRelevance_SymbolSearch", func(t *testing.T) { + query := "how does symbol search rank results" + cands := verifySymbolSearchScenario() t0 := time.Now() got, err := svcInst.VerifyRelevance(ctx, query, cands) t.Logf("VerifyRelevance %q → kept=%v dropped=%d (%v)", query, got.Keep, len(cands)-len(got.Keep), time.Since(t0)) @@ -152,11 +152,11 @@ func TestE2E_AssistAgainstRealModel(t *testing.T) { } assertVerifySubset(t, got.Keep, cands) - expectKept := "real.NewBM25" + expectKept := "real.NewSymbolSearcherBackend" expectDropped := "synthetic.unrelated.parseTSConfig" kept := containsID(got.Keep, expectKept) dropped := !containsID(got.Keep, expectDropped) - t.Logf("VERDICT(BM25): kept NewBM25=%v, dropped parseTSConfig=%v", kept, dropped) + t.Logf("VERDICT(symbol-search): kept NewSymbolSearcherBackend=%v, dropped parseTSConfig=%v", kept, dropped) }) } @@ -244,39 +244,36 @@ func verifyHashPasswordsScenario() []llm.VerifyCandidate { } } -// verifyBM25Scenario tests the model's ability to keep the genuine -// BM25 ranking implementation while dropping unrelated parsers / +// verifySymbolSearchScenario tests the model's ability to keep the +// genuine symbol-search ranking path while dropping unrelated parsers / // fixtures. -func verifyBM25Scenario() []llm.VerifyCandidate { +func verifySymbolSearchScenario() []llm.VerifyCandidate { return []llm.VerifyCandidate{ { - ID: "real.NewBM25", - Name: "NewBM25", - Signature: "func NewBM25() *BM25Backend", - Body: `func NewBM25() *BM25Backend { - return &BM25Backend{ - inverted: make(map[string][]posting), - bigrams: make(map[string]map[string]struct{}), - docs: make(map[string]doc), - } + ID: "real.NewSymbolSearcherBackend", + Name: "NewSymbolSearcherBackend", + Signature: "func NewSymbolSearcherBackend(s graph.SymbolSearcher) *SymbolSearcherBackend", + Body: `func NewSymbolSearcherBackend(s graph.SymbolSearcher) *SymbolSearcherBackend { + return &SymbolSearcherBackend{s: s} }`, Callers: []llm.CallerInfo{ {Name: "indexer.New", Signature: "func New(g *graph.Graph, ...) *Indexer"}, }, }, { - ID: "real.BM25Backend.Search", + ID: "real.SymbolSearcherBackend.Search", Name: "Search", - Signature: "func (b *BM25Backend) Search(query string, limit int) []scored", - Body: `func (b *BM25Backend) Search(query string, limit int) []scored { - terms := tokenize(query) - scores := map[string]float64{} - for _, t := range terms { - for _, p := range b.inverted[t] { - scores[p.id] += b.bm25Score(p, t) - } + Signature: "func (b *SymbolSearcherBackend) Search(query string, limit int) []SearchResult", + Body: `func (b *SymbolSearcherBackend) Search(query string, limit int) []SearchResult { + hits, err := b.s.SearchSymbols(query, limit) + if err != nil || len(hits) == 0 { + return nil } - return topK(scores, limit) + out := make([]SearchResult, len(hits)) + for i, h := range hits { + out[i] = SearchResult{ID: h.NodeID, Score: h.Score} + } + return out }`, Callers: []llm.CallerInfo{ {Name: "Engine.SearchSymbolsScoped", Signature: "func (e *Engine) SearchSymbolsScoped(q string, limit int, opts QueryOptions) []*graph.Node"}, diff --git a/internal/mcp/analyze_scope_test.go b/internal/mcp/analyze_scope_test.go index 1dfaf3c25..79484cc8d 100644 --- a/internal/mcp/analyze_scope_test.go +++ b/internal/mcp/analyze_scope_test.go @@ -80,7 +80,7 @@ func newAnalyzeServer(t *testing.T, flagOn bool, repos ...analyzeRepoSpec) (*Ser g := graph.New() reg := testRegistry() - bm := search.NewBM25() + bm := search.NewNull() mi := indexer.NewMultiIndexer(g, reg, bm, cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) diff --git a/internal/mcp/change_contract_edit_paths_test.go b/internal/mcp/change_contract_edit_paths_test.go index dd8c21460..0f505b87a 100644 --- a/internal/mcp/change_contract_edit_paths_test.go +++ b/internal/mcp/change_contract_edit_paths_test.go @@ -106,7 +106,7 @@ func setupContractMultiRepoServer(t *testing.T, names ...string) (*Server, *grap require.NoError(t, err) g := graph.New() - multi := indexer.NewMultiIndexer(g, testRegistry(), search.NewBM25(), manager, zap.NewNop()) + multi := indexer.NewMultiIndexer(g, testRegistry(), search.NewNull(), manager, zap.NewNop()) _, err = multi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/diff_repo_scope_test.go b/internal/mcp/diff_repo_scope_test.go index 734f13163..c70f4ee80 100644 --- a/internal/mcp/diff_repo_scope_test.go +++ b/internal/mcp/diff_repo_scope_test.go @@ -41,7 +41,7 @@ func TestDiffRepoScope(t *testing.T) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/enclosing_test.go b/internal/mcp/enclosing_test.go index 8754b0016..306fa0e1e 100644 --- a/internal/mcp/enclosing_test.go +++ b/internal/mcp/enclosing_test.go @@ -73,7 +73,7 @@ func TestFindUsages_GroupByFile(t *testing.T) { g.AddEdge(&graph.Edge{From: b1.ID, To: target.ID, Kind: graph.EdgeCalls, FilePath: "pkg/b.go", Line: 5}) eng := query.NewEngine(g) - eng.SetSearch(search.NewBM25()) + eng.SetSearch(search.NewNull()) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) req := mcplib.CallToolRequest{} @@ -117,7 +117,7 @@ func TestFindUsages_FlatByDefault(t *testing.T) { g.AddEdge(&graph.Edge{From: caller.ID, To: target.ID, Kind: graph.EdgeCalls, FilePath: "pkg/a.go", Line: 3}) eng := query.NewEngine(g) - eng.SetSearch(search.NewBM25()) + eng.SetSearch(search.NewNull()) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) req := mcplib.CallToolRequest{} diff --git a/internal/mcp/ensure_fresh_self_heal_test.go b/internal/mcp/ensure_fresh_self_heal_test.go index dc1e8476b..193cf393a 100644 --- a/internal/mcp/ensure_fresh_self_heal_test.go +++ b/internal/mcp/ensure_fresh_self_heal_test.go @@ -43,7 +43,7 @@ func TestEnsureFresh_MultiRepoSelfHealsStaleFile(t *testing.T) { reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) require.True(t, mi.IsMultiRepo()) diff --git a/internal/mcp/explore_source_literal_test.go b/internal/mcp/explore_source_literal_test.go index 6c79975bd..406594a39 100644 --- a/internal/mcp/explore_source_literal_test.go +++ b/internal/mcp/explore_source_literal_test.go @@ -1046,7 +1046,7 @@ func TestSearchExploreSourceLiteralDoesNotCrossConfiguredRepository(t *testing.T configPath := filepath.Join(t.TempDir(), "config.yaml") manager, err := config.NewConfigManager(configPath) require.NoError(t, err) - multi := indexer.NewMultiIndexer(store, registry, search.NewAuto(), manager, zap.NewNop()) + multi := indexer.NewMultiIndexer(store, registry, search.NewNull(), manager, zap.NewNop()) otherRoot := filepath.Join(t.TempDir(), "repo-a") require.NoError(t, os.MkdirAll(otherRoot, 0o755)) _, err = multi.TrackRepoCtx(context.Background(), config.RepoEntry{Path: otherRoot, Force: true}) diff --git a/internal/mcp/facade_tools_test.go b/internal/mcp/facade_tools_test.go index bcb2d3436..6a822e3ea 100644 --- a/internal/mcp/facade_tools_test.go +++ b/internal/mcp/facade_tools_test.go @@ -1609,12 +1609,12 @@ func TestBatchEditPreflightsAllItemsBeforeFirstWrite(t *testing.T) { func TestFacadeReadResolvesOnlyUniqueSymbolShorthand(t *testing.T) { g := graph.New() - bm := search.NewBM25() first := &graph.Node{ID: "pkg/a.go::UniqueReadTarget", Name: "UniqueReadTarget", Kind: graph.KindFunction, FilePath: "pkg/a.go"} g.AddNode(first) - bm.Add(first.ID, first.Name, first.FilePath, first.Name) eng := query.NewEngine(g) - eng.SetSearch(bm) + // The shorthand resolver reads FindNodesByName off the graph, never + // the text backend, so an inert one keeps the fixture honest. + eng.SetSearch(search.NewNull()) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) resolved, ambiguous := srv.resolveFacadeSymbolShorthand(context.Background(), "UniqueReadTarget") @@ -1623,7 +1623,6 @@ func TestFacadeReadResolvesOnlyUniqueSymbolShorthand(t *testing.T) { second := &graph.Node{ID: "pkg/b.go::UniqueReadTarget", Name: "UniqueReadTarget", Kind: graph.KindFunction, FilePath: "pkg/b.go"} g.AddNode(second) - bm.Add(second.ID, second.Name, second.FilePath, second.Name) resolved, ambiguous = srv.resolveFacadeSymbolShorthand(context.Background(), "UniqueReadTarget") require.Equal(t, "UniqueReadTarget", resolved) require.ElementsMatch(t, []string{first.ID, second.ID}, ambiguous) diff --git a/internal/mcp/find_usages_context_test.go b/internal/mcp/find_usages_context_test.go index dd918f154..6def86d04 100644 --- a/internal/mcp/find_usages_context_test.go +++ b/internal/mcp/find_usages_context_test.go @@ -31,7 +31,7 @@ func usagesContextServer(t *testing.T) (*Server, string) { g.AddEdge(&graph.Edge{From: field.ID, To: foo.ID, Kind: graph.EdgeTypedAs, FilePath: "pkg/b.go", Line: 3}) eng := query.NewEngine(g) - eng.SetSearch(search.NewBM25()) + eng.SetSearch(search.NewNull()) return NewServer(eng, g, nil, nil, zap.NewNop(), nil), foo.ID } diff --git a/internal/mcp/find_usages_summary_test.go b/internal/mcp/find_usages_summary_test.go index dab0d1211..fa4af0b60 100644 --- a/internal/mcp/find_usages_summary_test.go +++ b/internal/mcp/find_usages_summary_test.go @@ -42,7 +42,7 @@ func usagesSummaryServer(t *testing.T) (srv *Server, fooID, unusedID string) { g.AddEdge(&graph.Edge{From: testUse.ID, To: foo.ID, Kind: graph.EdgeCalls, FilePath: "pkg/foo_test.go", Line: 12}) eng := query.NewEngine(g) - eng.SetSearch(search.NewBM25()) + eng.SetSearch(search.NewNull()) return NewServer(eng, g, nil, nil, zap.NewNop(), nil), foo.ID, unused.ID } diff --git a/internal/mcp/freshness_rider_test.go b/internal/mcp/freshness_rider_test.go index 0fa2e7908..dca6a327f 100644 --- a/internal/mcp/freshness_rider_test.go +++ b/internal/mcp/freshness_rider_test.go @@ -105,7 +105,7 @@ func TestMultiRepoRiderAndMissingFileFlag(t *testing.T) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/instructions_test.go b/internal/mcp/instructions_test.go index b4989b7d2..c6302fc16 100644 --- a/internal/mcp/instructions_test.go +++ b/internal/mcp/instructions_test.go @@ -85,7 +85,7 @@ func TestStateAwareInstructionsVariants(t *testing.T) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/ordered_backend_test.go b/internal/mcp/ordered_backend_test.go new file mode 100644 index 000000000..02b6e1259 --- /dev/null +++ b/internal/mcp/ordered_backend_test.go @@ -0,0 +1,142 @@ +package mcp + +import ( + "strings" + "sync" + "unicode" + + "github.com/zzet/gortex/internal/search" +) + +// orderedBackend is a search.Backend that answers with a literal, +// hand-written candidate ordering instead of a ranked one. +// +// The MCP fixtures that use it are not testing a ranker: they test +// handler-layer logic (post-filter escalation, soup split/merge, +// equivalence rewrite, over-fetch retention, name diversification) +// GIVEN some candidate ordering. Encoding that ordering directly makes +// each fixture's premise readable and stable, instead of leaving it to +// emerge from term frequencies in a hand-tuned corpus. +// +// Retrieval model, deliberately crude: the query is split on +// whitespace, each field lowercased and stripped of edge punctuation, +// and the ordered ID lists of every matching token are unioned in +// token order (first token's list first, deduped). A query whose +// tokens are all unknown falls back to the optional default list — +// so a fixture can either key retrieval on specific tokens (the +// rewrite is then the thing under test: a symbol is reachable only +// if the handler actually queried one of its tokens) or hand the same +// ordering to every query. +// +// Note the tokens are whole whitespace fields, NOT camelCase parts: +// "WidgetExtensions" is the single token "widgetextensions", so a +// fixture can give a compound name and one of its parts different +// orderings. +type orderedBackend struct { + mu sync.Mutex + byToken map[string][]string // query token -> ordered node IDs + def []string // answer when no token matched + limits []int // every limit Search was called with +} + +func newOrderedBackend() *orderedBackend { + return &orderedBackend{byToken: make(map[string][]string)} +} + +// put appends ids to the ordering token answers with. Call order is +// the ranking: the first ID put is the top hit. +func (o *orderedBackend) put(token string, ids ...string) { + key := strings.ToLower(token) + o.byToken[key] = append(o.byToken[key], ids...) +} + +// putDefault appends ids to the ordering returned for a query whose +// tokens are all unknown. +func (o *orderedBackend) putDefault(ids ...string) { + o.def = append(o.def, ids...) +} + +// searchLimits returns every limit the engine asked this backend for, +// oldest first. +func (o *orderedBackend) searchLimits() []int { + o.mu.Lock() + defer o.mu.Unlock() + return append([]int(nil), o.limits...) +} + +// Add / Remove are no-ops: the ordering is declared by the fixture, not +// accumulated from indexed text. +func (o *orderedBackend) Add(string, ...string) {} +func (o *orderedBackend) Remove(string) {} +func (o *orderedBackend) Close() {} + +// Count reports the number of distinct IDs the backend can return. It +// must be positive for a populated fixture: the engine routes to its +// substring fallback when the backend reports an empty corpus. +func (o *orderedBackend) Count() int { + seen := make(map[string]bool) + for _, ids := range o.byToken { + for _, id := range ids { + seen[id] = true + } + } + for _, id := range o.def { + seen[id] = true + } + return len(seen) +} + +// Search returns the first limit IDs of the matching ordering, scored +// descending so the caller sees a plausible ranked shape. (The engine +// keeps the rank, not the score, so the numbers are cosmetic.) +func (o *orderedBackend) Search(query string, limit int) []search.SearchResult { + o.mu.Lock() + o.limits = append(o.limits, limit) + o.mu.Unlock() + + ids := o.ordering(query) + if limit > 0 && len(ids) > limit { + ids = ids[:limit] + } + out := make([]search.SearchResult, 0, len(ids)) + for i, id := range ids { + out = append(out, search.SearchResult{ID: id, Score: float64(len(ids) - i)}) + } + return out +} + +func (o *orderedBackend) ordering(query string) []string { + var ( + seen = make(map[string]bool) + ids []string + ) + for _, tok := range orderedQueryTokens(query) { + for _, id := range o.byToken[tok] { + if seen[id] { + continue + } + seen[id] = true + ids = append(ids, id) + } + } + if len(ids) == 0 { + return o.def + } + return ids +} + +// orderedQueryTokens lowercases the query, splits it on whitespace and +// trims edge punctuation from each field. +func orderedQueryTokens(query string) []string { + fields := strings.Fields(strings.ToLower(query)) + out := make([]string, 0, len(fields)) + for _, f := range fields { + f = strings.TrimFunc(f, func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) + if f != "" { + out = append(out, f) + } + } + return out +} diff --git a/internal/mcp/path_scope_test.go b/internal/mcp/path_scope_test.go index f355c7c8d..8e963b47f 100644 --- a/internal/mcp/path_scope_test.go +++ b/internal/mcp/path_scope_test.go @@ -86,7 +86,7 @@ func TestResolvePathFilter_Sources(t *testing.T) { t.Setenv("GORTEX_SCOPES_PATH", filepath.Join(tempSidecarDir(t), "scopes.json")) g := graph.New() eng := query.NewEngine(g) - eng.SetSearch(search.NewBM25()) + eng.SetSearch(search.NewNull()) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) require.NoError(t, srv.scopeStoreOrInit().put(SavedScope{ Name: "billing", Repos: []string{"r"}, Paths: []string{"services/billing"}, @@ -120,22 +120,32 @@ func TestResolvePathFilter_Sources(t *testing.T) { func pathScopeServer(t *testing.T) *Server { t.Helper() g := graph.New() - bm := search.NewBM25() + // Every symbol is retrievable by its whole name and by each of the + // camelCase words in it, so the queries below reach all three and + // only the path filter can confine the result. + ob := newOrderedBackend() files := map[string]string{ "services/billing/Invoice.go": "BillingInvoice", "services/auth/Login.go": "AuthLogin", "libs/money/Amount.go": "MoneyAmount", } + tokens := map[string][]string{ + "BillingInvoice": {"billinginvoice", "billing", "invoice"}, + "AuthLogin": {"authlogin", "auth", "login"}, + "MoneyAmount": {"moneyamount", "money", "amount"}, + } for path, name := range files { id := path + "::" + name g.AddNode(&graph.Node{ ID: id, Kind: graph.KindFunction, Name: name, FilePath: path, StartLine: 1, EndLine: 5, Language: "go", }) - bm.Add(id, name, path, "") + for _, tok := range tokens[name] { + ob.put(tok, id) + } } eng := query.NewEngine(g) - eng.SetSearch(bm) + eng.SetSearch(ob) return NewServer(eng, g, nil, nil, zap.NewNop(), nil) } diff --git a/internal/mcp/rename_recovery_test.go b/internal/mcp/rename_recovery_test.go index 6eafeafda..98783f09f 100644 --- a/internal/mcp/rename_recovery_test.go +++ b/internal/mcp/rename_recovery_test.go @@ -234,7 +234,7 @@ func setupMultiRepoRenameRecoveryServer( registry := testRegistry() store := graph.New() - multi := indexer.NewMultiIndexer(store, registry, search.NewBM25(), manager, zap.NewNop()) + multi := indexer.NewMultiIndexer(store, registry, search.NewNull(), manager, zap.NewNop()) _, err = multi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/return_usage_test.go b/internal/mcp/return_usage_test.go index 5f6882cc4..8ea9e8523 100644 --- a/internal/mcp/return_usage_test.go +++ b/internal/mcp/return_usage_test.go @@ -60,7 +60,7 @@ func returnUsageServer(t *testing.T) (*Server, string) { }) eng := query.NewEngine(g) - eng.SetSearch(search.NewBM25()) + eng.SetSearch(search.NewNull()) return NewServer(eng, g, nil, nil, zap.NewNop(), nil), fetch.ID } @@ -233,7 +233,7 @@ func relay() int { } eng := query.NewEngine(g) - eng.SetSearch(search.NewBM25()) + eng.SetSearch(search.NewNull()) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) edges := findUsagesEdges(t, srv, map[string]any{"id": "main.go::helper"}) diff --git a/internal/mcp/scope_resolve_test.go b/internal/mcp/scope_resolve_test.go index b28f2df5d..0a0abbd6c 100644 --- a/internal/mcp/scope_resolve_test.go +++ b/internal/mcp/scope_resolve_test.go @@ -19,6 +19,7 @@ import ( "github.com/zzet/gortex/internal/config" "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" "github.com/zzet/gortex/internal/indexer" "github.com/zzet/gortex/internal/query" "github.com/zzet/gortex/internal/search" @@ -76,7 +77,7 @@ func newSharedWorkspaceServer(t *testing.T, flagOn bool) sharedWSOptions { g := graph.New() reg := testRegistry() - bm := search.NewBM25() + bm := search.NewNull() mi := indexer.NewMultiIndexer(g, reg, bm, cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) @@ -114,7 +115,7 @@ func newSplitProjectWorkspaceServer(t *testing.T, flagOn bool) sharedWSOptions { g := graph.New() reg := testRegistry() - bm := search.NewBM25() + bm := search.NewNull() mi := indexer.NewMultiIndexer(g, reg, bm, cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) @@ -693,7 +694,7 @@ func newLoneRepoServer(t *testing.T, flagOn bool) (*Server, string) { g := graph.New() reg := testRegistry() - bm := search.NewBM25() + bm := search.NewNull() mi := indexer.NewMultiIndexer(g, reg, bm, cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) @@ -828,18 +829,24 @@ func newTwoRepoServer(t *testing.T) (*Server, string) { cm, err := config.NewConfigManager(tmpCfg) require.NoError(t, err) - g := graph.New() - reg := testRegistry() - bm := search.NewBM25() - mi := indexer.NewMultiIndexer(g, reg, bm, cm, zap.NewNop()) + // A real sqlite store: search runs off its native symbol FTS, the same + // corpus the daemon serves, so the scope narrowing is applied to + // production-shaped hits. + store, err := store_sqlite.Open(filepath.Join(t.TempDir(), "fts.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + + backend := search.NewSymbolSearcherBackend(store) + mi := indexer.NewMultiIndexer(store, testRegistry(), backend, cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) + requireSymbolFTS(t, store) - eng := query.NewEngine(g) - eng.SetSearch(bm) + eng := query.NewEngine(store) + eng.SetSearch(backend) flagVal := true - srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ + srv := NewServer(eng, store, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ MultiIndexer: mi, ConfigManager: cm, ScopeIntentDefaults: &flagVal, diff --git a/internal/mcp/search_corpus_scope_test.go b/internal/mcp/search_corpus_scope_test.go index 697c7858a..5e51acdd0 100644 --- a/internal/mcp/search_corpus_scope_test.go +++ b/internal/mcp/search_corpus_scope_test.go @@ -57,7 +57,7 @@ func newWorkspaceRootBoundServer(t *testing.T) (s *Server, root string) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) - mi := indexer.NewMultiIndexer(store, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(store, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_contract_bridge_test.go b/internal/mcp/tools_contract_bridge_test.go index 50c22e0be..61417e9f8 100644 --- a/internal/mcp/tools_contract_bridge_test.go +++ b/internal/mcp/tools_contract_bridge_test.go @@ -126,7 +126,7 @@ func fetchUsers() { preg := testRegistry() g := graph.New() - mi := indexer.NewMultiIndexer(g, preg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, preg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_contracts_filter_test.go b/internal/mcp/tools_contracts_filter_test.go index 9515a15c5..a61dd188e 100644 --- a/internal/mcp/tools_contracts_filter_test.go +++ b/internal/mcp/tools_contracts_filter_test.go @@ -69,7 +69,7 @@ func TestHandleContracts_FiltersByProjectAndRef(t *testing.T) { preg := testRegistry() g := graph.New() - mi := indexer.NewMultiIndexer(g, preg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, preg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_contracts_test.go b/internal/mcp/tools_contracts_test.go index c3d292b6c..ae9450ed1 100644 --- a/internal/mcp/tools_contracts_test.go +++ b/internal/mcp/tools_contracts_test.go @@ -91,7 +91,7 @@ func TestHandleContracts_ReflectsRuntimeTrackedRepos(t *testing.T) { preg := testRegistry() g := graph.New() - mi := indexer.NewMultiIndexer(g, preg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, preg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -159,7 +159,7 @@ func TestHandleContracts_MatchesGraphContractCount(t *testing.T) { preg := testRegistry() g := graph.New() - mi := indexer.NewMultiIndexer(g, preg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, preg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_core_index_test.go b/internal/mcp/tools_core_index_test.go index 51ed1ad5a..b996f5d6e 100644 --- a/internal/mcp/tools_core_index_test.go +++ b/internal/mcp/tools_core_index_test.go @@ -59,7 +59,7 @@ func TestHandleIndexRepository_MultiRepoRoutesThroughMultiIndexer(t *testing.T) reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) require.True(t, mi.IsMultiRepo()) @@ -145,7 +145,7 @@ func TestHandleIndexRepository_ReconcilesFromPersistedConfig(t *testing.T) { g := graph.New() // Intentionally DO NOT call IndexAll / TrackRepoCtx — simulates the // warmup-drift state: config persisted, but MultiIndexer empty. - mi := indexer.NewMultiIndexer(g, preg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, preg, search.NewNull(), cm, zap.NewNop()) require.Empty(t, mi.AllMetadata(), "precondition: MultiIndexer must start empty to reproduce drift") @@ -195,7 +195,7 @@ func TestHandleIndexRepository_MultiRepoRejectsUntrackedPath(t *testing.T) { reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_core_reindex_test.go b/internal/mcp/tools_core_reindex_test.go index 1e6ddd12b..f83d440bf 100644 --- a/internal/mcp/tools_core_reindex_test.go +++ b/internal/mcp/tools_core_reindex_test.go @@ -226,7 +226,7 @@ func TestHandleReindexRepository_MultiRepoRoutesByPrefix(t *testing.T) { reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) require.True(t, mi.IsMultiRepo()) @@ -284,7 +284,7 @@ func TestHandleReindexRepository_MultiRepoPathScoped(t *testing.T) { reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -338,7 +338,7 @@ func TestHandleReindexRepository_MultiRepoRejectsUntrackedPath(t *testing.T) { reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_explore_test.go b/internal/mcp/tools_explore_test.go index fba5cd304..afa58ae98 100644 --- a/internal/mcp/tools_explore_test.go +++ b/internal/mcp/tools_explore_test.go @@ -13,7 +13,6 @@ import ( "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/query" - "github.com/zzet/gortex/internal/search" "github.com/zzet/gortex/internal/search/rerank" ) @@ -711,7 +710,7 @@ func TestRenderExploreLimitsFullBodiesToTopTargets(t *testing.T) { targets = append(targets, exploreTarget{ node: &graph.Node{ Name: name, Kind: graph.KindFunction, - FilePath: fmt.Sprintf("cand_%02d.go", index), + FilePath: fmt.Sprintf("cand_%02d.go", index), StartLine: 1, EndLine: 4, Language: "go", }, source: fmt.Sprintf("func %s() int {\n\treturn 9900 + %d\n}", name, index), @@ -812,7 +811,14 @@ func TestExploreHelpers(t *testing.T) { // repeated same-name leaves must not crowd out a differently-named code target. func TestFacadeExploreDemotesRepeatedDataLeafNames(t *testing.T) { g := graph.New() - bm := search.NewBM25() + // Retrieval ordering, stated rather than ranked: every query in this + // fixture retrieves the 30 short exact-name `client` declarations + // first and the differently-named callable last. That is the shape a + // ranker produces for this task -- the literal name matches win the + // head, the callable that actually explains the area matches on prose + // and lands behind them -- and it is the shape explore's over-fetch + // window must retain so name diversification can promote it. + ob := newOrderedBackend() for i := 0; i < 30; i++ { id := fmt.Sprintf("pkg/service%d.go::client", i) n := &graph.Node{ @@ -821,7 +827,7 @@ func TestFacadeExploreDemotesRepeatedDataLeafNames(t *testing.T) { Meta: map[string]any{"signature": "var client *Transport"}, } g.AddNode(n) - bm.Add(id, n.Name, n.FilePath, "trace client coordinated transport") + ob.putDefault(id) } relevant := &graph.Node{ ID: "pkg/coordinator.go::TransportCoordinator", Name: "TransportCoordinator", @@ -829,14 +835,10 @@ func TestFacadeExploreDemotesRepeatedDataLeafNames(t *testing.T) { Meta: map[string]any{"signature": "func TransportCoordinator()"}, } g.AddNode(relevant) - // It matches the whole task, but its longer prose and non-literal symbol - // name rank below the short exact-name `client` declarations. The concept - // over-fetch window must retain it so name diversification can promote it. - relevantText := strings.Repeat("architecture routing plumbing lifecycle ", 40) + "trace client coordinated transport coordinator" - bm.Add(relevant.ID, relevant.Name, relevant.FilePath, relevantText) + ob.putDefault(relevant.ID) eng := query.NewEngine(g) - eng.SetSearch(bm) + eng.SetSearch(ob) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) task := "trace how the client is coordinated" searchQuery := shapeExploreQuery(task) diff --git a/internal/mcp/tools_fileops_singlerepo_test.go b/internal/mcp/tools_fileops_singlerepo_test.go index 80f7a3b61..d58865145 100644 --- a/internal/mcp/tools_fileops_singlerepo_test.go +++ b/internal/mcp/tools_fileops_singlerepo_test.go @@ -37,7 +37,7 @@ func newSingleRepoServer(t *testing.T) (*Server, *graph.Graph, string) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -115,7 +115,7 @@ func TestResolveFilePath_MultiRepoBareRelativeStillAmbiguous(t *testing.T) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_fileops_worktree_test.go b/internal/mcp/tools_fileops_worktree_test.go index a4a4f481a..f20d086a4 100644 --- a/internal/mcp/tools_fileops_worktree_test.go +++ b/internal/mcp/tools_fileops_worktree_test.go @@ -86,7 +86,7 @@ func setupWorktreePair(t *testing.T) (mainRepo, worktree string, srv *Server) { reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) require.True(t, mi.IsMultiRepo()) diff --git a/internal/mcp/tools_find_declaration_test.go b/internal/mcp/tools_find_declaration_test.go index e3db62067..9cecd6569 100644 --- a/internal/mcp/tools_find_declaration_test.go +++ b/internal/mcp/tools_find_declaration_test.go @@ -250,7 +250,7 @@ func consumer() int { return uniqueDecl() + uniqueDecl() } reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) require.True(t, mi.IsMultiRepo()) diff --git a/internal/mcp/tools_multi_worktree_test.go b/internal/mcp/tools_multi_worktree_test.go index 6b12e795c..147b36fa6 100644 --- a/internal/mcp/tools_multi_worktree_test.go +++ b/internal/mcp/tools_multi_worktree_test.go @@ -34,7 +34,7 @@ func newWorktreeMCPServer(t *testing.T, repos ...config.RepoEntry) (*Server, *in preg := testRegistry() g := graph.New() - mi := indexer.NewMultiIndexer(g, preg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, preg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_review_rulepack_test.go b/internal/mcp/tools_review_rulepack_test.go index 09762a120..34f78b1b8 100644 --- a/internal/mcp/tools_review_rulepack_test.go +++ b/internal/mcp/tools_review_rulepack_test.go @@ -47,7 +47,7 @@ func prefixedServerOver(t *testing.T, dir, name string) (*Server, string) { reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_search_assist.go b/internal/mcp/tools_search_assist.go index 152bf9c1a..ad4b36bed 100644 --- a/internal/mcp/tools_search_assist.go +++ b/internal/mcp/tools_search_assist.go @@ -286,8 +286,7 @@ func anchorTermsToVocabulary(terms []string, ac *search.AutoConcepts) []string { // expansion hits append in their own BM25 order with duplicates // skipped. // -// Both BM25 backends (BM25Backend and the on-disk backend's FTS) -// treat a multi-token query as an OR-style union +// The store's FTS treats a multi-token query as an OR-style union // with a single global BM25 score, so one combined call replaces // the prior N per-term fan-out (the N+1 round-trip pattern dominated // the search hot path on disk backends). @@ -649,9 +648,9 @@ func verifyWithLLM(ctx context.Context, s *Server, query string, nodes []*graph. // // Before partitioning into head/tail, nodes are re-sorted so callable // kinds (function / method) come before everything else — preserving -// BM25 order within each bucket. Without this, a high-scoring param -// or field node (e.g. `BM25Backend.Search#param:limit`) can pre-empt -// the enclosing method (`BM25Backend.Search`) inside the rerank +// retrieval order within each bucket. Without this, a high-scoring +// param or field node (e.g. `Engine.SearchSymbols#param:limit`) can +// pre-empt the enclosing method (`Engine.SearchSymbols`) inside the rerank // window, leaving the model unable to surface the real callable. func rerankWithLLM(ctx context.Context, s *Server, query string, nodes []*graph.Node) []*graph.Node { if s.llmService == nil || !s.llmService.Enabled() || len(nodes) < 2 { diff --git a/internal/mcp/tools_search_corpus_test.go b/internal/mcp/tools_search_corpus_test.go index 99d0047c6..12b74191a 100644 --- a/internal/mcp/tools_search_corpus_test.go +++ b/internal/mcp/tools_search_corpus_test.go @@ -12,13 +12,46 @@ import ( "go.uber.org/zap" "github.com/zzet/gortex/internal/config" - "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" "github.com/zzet/gortex/internal/indexer" "github.com/zzet/gortex/internal/query" ) +// newFTSServer indexes dir into a real sqlite store and returns a Server whose +// search reads that store's native symbol FTS — the corpus production actually +// serves. Everything the daemon's search_symbols path touches (the engine, the +// retrieval channels, the rerank) then sees the same documents and the same +// ranking it would see in a live workspace. +// +// The SymbolFTSCount gate is load-bearing: an empty corpus answers every +// "must not be returned" assertion correctly and leaves the "must be returned" +// ones as the only thing between the fixture and a silent pass. +func newFTSServer(t *testing.T, dir string, cfg config.IndexConfig) *Server { + t.Helper() + store, err := store_sqlite.Open(filepath.Join(t.TempDir(), "fts.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + + idx := indexer.New(store, testRegistry(), cfg, zap.NewNop()) + _, err = idx.Index(dir) + require.NoError(t, err) + requireSymbolFTS(t, store) + + eng := query.NewEngine(store) + eng.SetSearchProvider(idx.Search) + return NewServer(eng, store, idx, nil, zap.NewNop(), nil) +} + +// requireSymbolFTS fails unless the store's symbol corpus holds something. +func requireSymbolFTS(t *testing.T, store *store_sqlite.Store) { + t.Helper() + count, err := store.SymbolFTSCount() + require.NoError(t, err) + require.Greater(t, count, 0, "fixture left the symbol FTS empty — search assertions would be vacuous") +} + // corpusTestServer indexes a repo containing a Go file and a Markdown -// doc, so the BM25 index holds both code symbols and prose-section +// doc, so the symbol FTS corpus holds both code symbols and prose-section // nodes. func corpusTestServer(t *testing.T) *Server { return corpusTestServerProse(t, true) @@ -39,16 +72,9 @@ func corpusTestServerProse(t *testing.T, indexProse bool) *Server { "and apply the kubernetes manifest.\n\n"+ "## Troubleshooting\n\nCheck the logs when a request times out.\n"), 0o644)) - g := graph.New() - reg := testRegistry() cfg := config.Default() cfg.Index.IndexProse = indexProse - idx := indexer.New(g, reg, cfg.Index, zap.NewNop()) - _, err := idx.Index(dir) - require.NoError(t, err) - eng := query.NewEngine(g) - eng.SetSearchProvider(idx.Search) - return NewServer(eng, g, idx, nil, zap.NewNop(), nil) + return newFTSServer(t, dir, cfg.Index) } func corpusSearch(t *testing.T, srv *Server, args map[string]any) []map[string]any { diff --git a/internal/mcp/tools_search_docchannel_test.go b/internal/mcp/tools_search_docchannel_test.go index ac3279f7a..dba8fa45b 100644 --- a/internal/mcp/tools_search_docchannel_test.go +++ b/internal/mcp/tools_search_docchannel_test.go @@ -8,12 +8,8 @@ import ( "testing" "github.com/stretchr/testify/require" - "go.uber.org/zap" "github.com/zzet/gortex/internal/config" - "github.com/zzet/gortex/internal/graph" - "github.com/zzet/gortex/internal/indexer" - "github.com/zzet/gortex/internal/query" ) // docChannelServer indexes MANY code symbols whose names share the @@ -38,15 +34,7 @@ func docChannelServer(t *testing.T) *Server { []byte("# Guide\n\n## Deployment\n\n"+ "To deploy the service push the container image and apply the manifest.\n"), 0o644)) - g := graph.New() - reg := testRegistry() - cfg := config.Default() - idx := indexer.New(g, reg, cfg.Index, zap.NewNop()) - _, err := idx.Index(dir) - require.NoError(t, err) - eng := query.NewEngine(g) - eng.SetSearchProvider(idx.Search) - return NewServer(eng, g, idx, nil, zap.NewNop(), nil) + return newFTSServer(t, dir, config.Default().Index) } // TestSearchSymbols_DocChannelRescuesCrowdedProse is the core feature diff --git a/internal/mcp/tools_search_equivalence_test.go b/internal/mcp/tools_search_equivalence_test.go index c43fd4e59..844c52bd9 100644 --- a/internal/mcp/tools_search_equivalence_test.go +++ b/internal/mcp/tools_search_equivalence_test.go @@ -3,6 +3,7 @@ package mcp import ( "context" "encoding/json" + "strings" "testing" mcplib "github.com/mark3labs/mcp-go/mcp" @@ -12,26 +13,39 @@ import ( "github.com/zzet/gortex/internal/config" "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/query" - "github.com/zzet/gortex/internal/search" + "github.com/zzet/gortex/internal/search/rerank" ) // equivalenceTestServer builds a single-repo server with no LLM // provider, so any expansion observed comes purely from the // deterministic equivalence channel. +// +// Retrieval ordering: a symbol answers to its own lowercased name and +// to each of that name's camelCase/snake_case parts ("LoginService" -> +// loginservice, login, service), in declaration order, and to nothing +// else. The vocabulary the query itself uses ("auth", "delete", +// "blast") is deliberately absent, so a bridged symbol is reachable +// only if the rewrite actually emitted one of its tokens — the rewrite +// stays the thing under test rather than a ranker's tolerance. func equivalenceTestServer(t *testing.T, names []string, equivEnabled *bool) *Server { t.Helper() g := graph.New() - bm := search.NewBM25() + ob := newOrderedBackend() for i, n := range names { id := "pkg/" + n + ".go::" + n g.AddNode(&graph.Node{ ID: id, Kind: graph.KindFunction, Name: n, FilePath: "pkg/" + n + ".go", StartLine: i + 1, EndLine: i + 5, Language: "go", }) - bm.Add(id, n, "pkg/"+n+".go", "") + ob.put(n, id) + for _, tok := range rerank.Tokenize(n) { + if tok != strings.ToLower(n) { + ob.put(tok, id) + } + } } eng := query.NewEngine(g) - eng.SetSearch(bm) + eng.SetSearch(ob) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) srv.SetSearchConfig(config.SearchConfig{EquivalenceClasses: equivEnabled}) srv.RunAnalysis() // builds the auto-concept vocabulary @@ -73,7 +87,7 @@ func TestSearchSymbols_EquivalenceBridgesVocabulary(t *testing.T) { } // TestSearchSymbols_EquivalenceDeleteRemove confirms the delete/remove -// bridge works in both directions through the BM25 OR-merge. +// bridge works in both directions through the expansion OR-merge. func TestSearchSymbols_EquivalenceDeleteRemove(t *testing.T) { srv := equivalenceTestServer(t, []string{ "RemoveUser", "DropTable", "FetchProfile", diff --git a/internal/mcp/tools_search_escalation_test.go b/internal/mcp/tools_search_escalation_test.go index 90129fbd4..4956ff2e5 100644 --- a/internal/mcp/tools_search_escalation_test.go +++ b/internal/mcp/tools_search_escalation_test.go @@ -9,27 +9,29 @@ import ( "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/query" - "github.com/zzet/gortex/internal/search" ) -// floodTestServer builds a server whose BM25 head for the token -// "extensions" is fully occupied by doc-section nodes: one real code -// symbol (WidgetExtensions) plus docCount KindDoc nodes that all -// outrank it (single-token name, tripled term frequency). With the -// default corpus=code filter, every fetched candidate is dropped — -// the shape a suffix-convention query takes on a repo whose doc/junk -// files share the naming tokens. +// floodTestServer builds a server whose candidate ordering for the +// token "extensions" is docCount KindDoc sections and nothing else: +// the junk sections own the token outright, so with the default +// corpus=code filter every candidate a bounded fetch reaches is +// dropped — the shape a suffix-convention query takes on a repo whose +// doc/junk files share the naming tokens. The one real code symbol +// (WidgetExtensions) is NOT in that ordering; it carries the token +// only as part of its full name, so it is reachable solely through +// the engine's substring fill, whose budget is the fetch depth. That +// is what makes the escalation depth observable — a fetch deep enough +// to leave slack past the doc flood is the only one that rescues it. func floodTestServer(t *testing.T, docCount int) *Server { t.Helper() g := graph.New() - bm := search.NewBM25() + ob := newOrderedBackend() id := "pkg/WidgetExtensions.go::WidgetExtensions" g.AddNode(&graph.Node{ ID: id, Kind: graph.KindType, Name: "WidgetExtensions", FilePath: "pkg/WidgetExtensions.go", StartLine: 1, EndLine: 5, Language: "go", }) - bm.Add(id, "WidgetExtensions", "pkg/WidgetExtensions.go", "") for i := 0; i < docCount; i++ { docID := fmt.Sprintf("junk/list%d.txt::sec%d", i, i) @@ -37,11 +39,14 @@ func floodTestServer(t *testing.T, docCount int) *Server { ID: docID, Kind: graph.KindDoc, Name: "Extensions", FilePath: fmt.Sprintf("junk/list%d.txt", i), StartLine: 1, EndLine: 3, Language: "text", }) - bm.Add(docID, "Extensions Extensions Extensions") + ob.put("extensions", docID) } + // The spelled-out name is its own token and answers with the code + // symbol alone — a query for it is never flooded. + ob.put("widgetextensions", id) eng := query.NewEngine(g) - eng.SetSearch(bm) + eng.SetSearch(ob) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) srv.RunAnalysis() return srv @@ -50,11 +55,15 @@ func floodTestServer(t *testing.T, docCount int) *Server { // floodTestServerN is floodTestServer with codeCount rescuable code // symbols behind the doc flood — the multi-page rescue shape from the // PR review: a partially-successful shallow rescue must not strand a -// later cursor page a deeper fetch would fill. +// later cursor page a deeper fetch would fill. Same ordering premise as +// floodTestServer — the doc sections are the whole "extensions" +// ordering and the code symbols ride the substring fill — so the depth +// a fetch reaches decides how many of them survive the corpus filter: +// docCount+5 of budget rescues 5, a deeper fetch rescues all. func floodTestServerN(t *testing.T, docCount, codeCount int) *Server { t.Helper() g := graph.New() - bm := search.NewBM25() + ob := newOrderedBackend() for i := 0; i < codeCount; i++ { id := fmt.Sprintf("pkg/w%d.go::WidgetExtensions%d", i, i) @@ -62,7 +71,6 @@ func floodTestServerN(t *testing.T, docCount, codeCount int) *Server { ID: id, Kind: graph.KindType, Name: fmt.Sprintf("WidgetExtensions%d", i), FilePath: fmt.Sprintf("pkg/w%d.go", i), StartLine: 1, EndLine: 5, Language: "go", }) - bm.Add(id, fmt.Sprintf("WidgetExtensions%d", i), fmt.Sprintf("pkg/w%d.go", i), "") } for i := 0; i < docCount; i++ { docID := fmt.Sprintf("junk/list%d.txt::sec%d", i, i) @@ -70,11 +78,11 @@ func floodTestServerN(t *testing.T, docCount, codeCount int) *Server { ID: docID, Kind: graph.KindDoc, Name: "Extensions", FilePath: fmt.Sprintf("junk/list%d.txt", i), StartLine: 1, EndLine: 3, Language: "text", }) - bm.Add(docID, "Extensions Extensions Extensions") + ob.put("extensions", docID) } eng := query.NewEngine(g) - eng.SetSearch(bm) + eng.SetSearch(ob) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) srv.RunAnalysis() return srv @@ -101,21 +109,22 @@ func TestSearchSymbols_EscalationReachesCursorWindow(t *testing.T) { // TestSearchSymbols_EscalationSkipsDuplicateDepth: once the depth cap // clamps a multiplier, the next multiplier collapses to the same // effective depth — an identical re-query cannot change the outcome and -// must be skipped, not paid a second time. +// must be skipped, not paid a second time. The ordering is all-doc, so +// no depth can ever rescue a code symbol and the escalation loop runs +// to its own stopping rule. func TestSearchSymbols_EscalationSkipsDuplicateDepth(t *testing.T) { g := graph.New() - bm := search.NewBM25() + ob := newOrderedBackend() for i := 0; i < 2100; i++ { docID := fmt.Sprintf("junk/list%d.txt::sec%d", i, i) g.AddNode(&graph.Node{ ID: docID, Kind: graph.KindDoc, Name: "Extensions", FilePath: fmt.Sprintf("junk/list%d.txt", i), StartLine: 1, EndLine: 3, Language: "text", }) - bm.Add(docID, "Extensions Extensions Extensions") + ob.put("extensions", docID) } - counter := &countingBackend{BM25Backend: bm} eng := query.NewEngine(g) - eng.SetSearch(counter) + eng.SetSearch(ob) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) srv.RunAnalysis() @@ -123,27 +132,16 @@ func TestSearchSymbols_EscalationSkipsDuplicateDepth(t *testing.T) { "query": "Extensions", "limit": 100, "cursor": encodeCursor(400), }) require.Empty(t, respIDs(resp), "an all-doc corpus yields no code page") - require.NotEmpty(t, counter.limits, "counting backend must intercept Search calls") + limits := ob.searchLimits() + require.NotEmpty(t, limits, "the ordered backend must have intercepted the Search calls") deep := 0 - for _, l := range counter.limits { + for _, l := range limits { if l >= 2000 { deep++ } } require.LessOrEqualf(t, deep, 1, - "capped escalation depths must not repeat an identical query; deep-call limits: %v", counter.limits) -} - -// countingBackend wraps the BM25 backend and records every Search -// limit, so a test can prove an identical capped depth is not re-paid. -type countingBackend struct { - *search.BM25Backend - limits []int -} - -func (c *countingBackend) Search(q string, limit int) []search.SearchResult { - c.limits = append(c.limits, limit) - return c.BM25Backend.Search(q, limit) + "capped escalation depths must not repeat an identical query; call limits: %v", limits) } // TestSearchSymbols_EscalatesWhenPostFilterEmptiesFetch is the core diff --git a/internal/mcp/tools_search_soup_test.go b/internal/mcp/tools_search_soup_test.go index 6f1862530..8f5ea94f4 100644 --- a/internal/mcp/tools_search_soup_test.go +++ b/internal/mcp/tools_search_soup_test.go @@ -12,27 +12,29 @@ import ( "github.com/zzet/gortex/internal/config" "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/query" - "github.com/zzet/gortex/internal/search" ) -// soupTestServer builds a single-repo server whose BM25 index holds a -// handful of distinct, single-token symbols so a soup query's split -// disjuncts each retrieve a known node. +// soupTestServer builds a single-repo server whose retrieval ordering +// is one token per symbol: each symbol answers to its own lowercased +// name and to nothing else. So a disjunct that the soup handling +// actually queries retrieves exactly its own node, and a symbol no +// disjunct names ("Unrelated") is unreachable — the merged result is a +// readout of which terms were queried, not of how they ranked. func soupTestServer(t *testing.T, soupMode string) *Server { t.Helper() g := graph.New() names := []string{"AuthHandler", "LoginService", "SigninFlow", "CredentialStore", "Unrelated"} - bm := search.NewBM25() + ob := newOrderedBackend() for _, n := range names { id := "pkg/" + n + ".go::" + n g.AddNode(&graph.Node{ ID: id, Kind: graph.KindFunction, Name: n, FilePath: "pkg/" + n + ".go", StartLine: 1, EndLine: 5, Language: "go", }) - bm.Add(id, n, "pkg/"+n+".go", "") + ob.put(n, id) } eng := query.NewEngine(g) - eng.SetSearch(bm) + eng.SetSearch(ob) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) srv.SetSearchConfig(config.SearchConfig{KeywordSoupRewrite: soupMode}) return srv @@ -54,8 +56,8 @@ func runSoupSearch(t *testing.T, srv *Server, args map[string]any) map[string]an // TestSearchSymbols_SoupSplitMerge confirms a degenerate OR-soup query // in the default "split" mode (a) reports the keyword_soup class, (b) // attaches a query_advice nudge with the split disjuncts, and (c) the -// BM25 OR-merge over the disjuncts surfaces every targeted symbol -- -// none of which the raw soup string would rank well on its own. +// OR-merge over those disjuncts surfaces every targeted symbol and +// nothing else -- a symbol none of the disjuncts names stays out. func TestSearchSymbols_SoupSplitMerge(t *testing.T) { srv := soupTestServer(t, config.KeywordSoupSplit) resp := runSoupSearch(t, srv, map[string]any{ @@ -83,6 +85,8 @@ func TestSearchSymbols_SoupSplitMerge(t *testing.T) { } { require.Truef(t, ids[want], "soup split-merge missed %s; got %v", want, ids) } + require.Falsef(t, ids["pkg/Unrelated.go::Unrelated"], + "no disjunct names Unrelated, so the merge must not surface it; got %v", ids) } // TestSearchSymbols_SoupOffMode confirms KeywordSoupRewrite:"off" diff --git a/internal/mcp/tools_search_text_pathsep_test.go b/internal/mcp/tools_search_text_pathsep_test.go index dae6edaef..c070cf7b6 100644 --- a/internal/mcp/tools_search_text_pathsep_test.go +++ b/internal/mcp/tools_search_text_pathsep_test.go @@ -58,7 +58,7 @@ func nestedRepoServer(t *testing.T, entries []config.RepoEntry) (*Server, *graph reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_search_text_test.go b/internal/mcp/tools_search_text_test.go index 4099d50c8..948ecaa31 100644 --- a/internal/mcp/tools_search_text_test.go +++ b/internal/mcp/tools_search_text_test.go @@ -180,7 +180,7 @@ func TestSearchText_MultiRepoFanout(t *testing.T) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) require.True(t, mi.IsMultiRepo()) @@ -236,7 +236,7 @@ func TestSearchText_MultiRepoScopedGrepDoesNotLetOutOfScopeRepoConsumeLimit(t *t reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -294,7 +294,7 @@ func TestFilterTextMatchesByResolvedScope_FailsClosed(t *testing.T) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -364,7 +364,7 @@ func TestGetFileSummary_MultiRepoRelativePath(t *testing.T) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) require.True(t, mi.IsMultiRepo()) diff --git a/internal/mcp/workspace_isolation_test.go b/internal/mcp/workspace_isolation_test.go index f24b60fa4..6801f1000 100644 --- a/internal/mcp/workspace_isolation_test.go +++ b/internal/mcp/workspace_isolation_test.go @@ -57,7 +57,7 @@ func newIsolationServer(t *testing.T) (srv *Server, repoA, repoB string) { g := graph.New() reg := testRegistry() - bm := search.NewBM25() + bm := search.NewNull() mi := indexer.NewMultiIndexer(g, reg, bm, cm, zap.NewNop()) _, err = mi.IndexScoped("", "") // index every configured repo require.NoError(t, err) diff --git a/internal/mcp/workspace_root_scope_test.go b/internal/mcp/workspace_root_scope_test.go index af82f7086..386e9df04 100644 --- a/internal/mcp/workspace_root_scope_test.go +++ b/internal/mcp/workspace_root_scope_test.go @@ -65,7 +65,7 @@ func newContainedFixture(t *testing.T) containedFixture { require.NoError(t, err) g := graph.New() - bm := search.NewBM25() + bm := search.NewNull() mi := indexer.NewMultiIndexer(g, testRegistry(), bm, cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) diff --git a/internal/query/engine.go b/internal/query/engine.go index 70572fdd7..acab97517 100644 --- a/internal/query/engine.go +++ b/internal/query/engine.go @@ -16,7 +16,8 @@ import ( // SearchProvider is a function that returns the current search backend. // This allows the engine to always use the latest backend even when the -// indexer replaces it (e.g., wrapping BM25 in HybridBackend for embeddings). +// indexer replaces it (e.g., wrapping the text backend in HybridBackend +// for embeddings). type SearchProvider func() search.Backend // Engine provides higher-level query operations over the graph. @@ -492,9 +493,8 @@ func (e *Engine) GetCluster(nodeID string, opts QueryOptions) *SubGraph { // SearchSymbols performs full-text search across all nodes. // When a search backend is configured, uses that backend's ranking — -// the in-process BM25 index in tests and evals, the store-native FTS -// index in production — with camelCase-aware tokenization. Falls back -// to substring matching otherwise. +// the store-native FTS index — with camelCase-aware tokenization. +// Falls back to substring matching otherwise. func (e *Engine) SearchSymbols(query string, limit int) []*graph.Node { return e.SearchSymbolsScoped(query, limit, QueryOptions{}) } @@ -682,9 +682,9 @@ func repoAllowList(allow map[string]bool) []string { return out } -// gatherBackendCandidates fetches BM25 + (optional) vector results, +// gatherBackendCandidates fetches text + (optional) vector results, // dedups them across channels, and supplements with exact-name / -// substring / bigram-rescue matches. Each candidate carries its +// substring matches. Each candidate carries its // 0-based TextRank and VectorRank (or -1 when the channel didn't // return it) so the rerank pipeline can score per channel. // @@ -932,7 +932,7 @@ func (e *Engine) gatherBackendCandidates(query string, limit int, opts QueryOpti insert(id, -1, rank) } - // Stop early when the BM25 + vector union has already exceeded the + // Stop early when the native text + vector union has already exceeded the // requested width; the supplementary tiers below are a fill, not a // boost. if len(cands) >= limit*2 { @@ -999,56 +999,9 @@ func (e *Engine) gatherBackendCandidates(query string, limit int, opts QueryOpti } } - // Bigram-overlap typo rescue. Same gates as the legacy path: - // nothing else surfaced, query is one indivisible 4+ char token, - // backend can provide candidates. The bigram backend also returns - // raw IDs — batch-materialise them too rather than fall back to - // per-id GetNode. A query with a separator OR an internal-uppercase - // camelCase boundary is decomposable, so it is left for the handler's - // leaf-decomposition rescue (more precise than fuzzy bigram overlap) — - // the bigram tier serves true atomic-token typos only. - if len(cands) == 0 && len(query) >= 4 && !strings.ContainsAny(query, " /.:_-") && !hasInternalUppercase(query) { - if bg, ok := backend.(bigramProvider); ok { - keys := len(query) - 1 - minOverlap := (keys + 1) / 2 - if minOverlap < 3 { - minOverlap = 3 - } - bigramIDs := bg.BigramCandidates(query, minOverlap) - // Skip the batch fetch entirely when the bigram backend - // returned nothing — otherwise we'd issue an empty query - // round-trip. - if len(bigramIDs) > 0 { - bigramNodes := e.g.GetNodesByIDs(bigramIDs) - for _, id := range bigramIDs { - if _, seen := idx[id]; seen { - continue - } - node := bigramNodes[id] - if node == nil || node.Kind == graph.KindFile || node.Kind == graph.KindImport { - continue - } - idx[id] = len(cands) - cands = append(cands, &rerank.Candidate{Node: node, TextRank: -1, VectorRank: -1}) - if len(cands) >= limit { - break - } - } - } - } - } - return cands } -// bigramProvider is satisfied by backends that expose a typo-tolerant -// rescue list. Declared here (not in search) so the engine can adopt -// rescue without the search interface changing; any backend that can -// provide bigram candidates just has to implement this method. -type bigramProvider interface { - BigramCandidates(query string, minOverlap int) []string -} - const substringSearchPageSize = 256 type substringCandidate struct { @@ -1286,27 +1239,14 @@ func (e *Engine) Stats() *graph.GraphStats { return &s } -// bfs performs breadth-first traversal from nodeID. -// If forward is true, follows outgoing edges; if false, follows incoming. -// If edgeKinds is nil, follows all edge kinds bidirectionally (for cluster). -// hasInternalUppercase reports whether s carries a camelCase boundary — an -// uppercase letter anywhere but the first byte. Such a query decomposes into -// multiple leaf tokens, so the bigram typo-rescue tier defers to the handler's -// leaf-decomposition rescue for it. -func hasInternalUppercase(s string) bool { - for i := 1; i < len(s); i++ { - if s[i] >= 'A' && s[i] <= 'Z' { - return true - } - } - return false -} - // defaultDispatchFanout bounds how many overriders one interface/abstract // method expands to during polymorphic dispatch expansion, so a hub interface // with hundreds of implementors cannot blow up a call-chain walk. const defaultDispatchFanout = 24 +// bfs performs breadth-first traversal from nodeID. +// If forward is true, follows outgoing edges; if false, follows incoming. +// If edgeKinds is nil, follows all edge kinds bidirectionally (for cluster). func (e *Engine) bfs(nodeID string, opts QueryOptions, forward bool, edgeKinds []graph.EdgeKind) *SubGraph { if opts.Depth <= 0 { opts.Depth = 3 diff --git a/internal/query/engine_corpus_gate_test.go b/internal/query/engine_corpus_gate_test.go index 707bc63f9..1bb695a89 100644 --- a/internal/query/engine_corpus_gate_test.go +++ b/internal/query/engine_corpus_gate_test.go @@ -147,6 +147,42 @@ type unknownCountBackend struct{ warmStoreBackend } func (b *unknownCountBackend) DocCount() (int, bool) { return 12345, false } +// TestGatherSymbolCandidates_VectorOnlyHybridUsesBackend proves that the corpus +// gate recognizes a populated vector channel even when its text side is the +// deliberately empty NullBackend. The concept query shares no substring with +// the symbol name, so the candidate can only arrive through semantic search. +func TestGatherSymbolCandidates_VectorOnlyHybridUsesBackend(t *testing.T) { + g := graph.New() + n := &graph.Node{ + ID: "app/snapshot.go::SnapshotCoordinator", + Name: "SnapshotCoordinator", + Kind: graph.KindType, + RepoPrefix: "app", + } + g.AddNode(n) + + vector := search.NewVector(2) + vector.Add(n.ID, []float32{1, 0}) + backend := search.NewSwappable(search.NewNull()) + backend.ReplaceHybridVector(vector, &fakeEmbedder{queryVec: []float32{1, 0}}) + defer backend.Close() + + engine := NewEngine(g) + engine.SetSearch(backend) + got := engine.GatherSymbolCandidates( + "durable state persistence", + 5, + QueryOptions{SkipInnerRerank: true}, + nil, + ) + if len(got) != 1 || got[0].Node.ID != n.ID { + t.Fatalf("vector-only hybrid was bypassed (substring fallback?); got %#v", got) + } + if got[0].TextRank != -1 || got[0].VectorRank != 0 { + t.Fatalf("vector-only hit must preserve channel ranks; got %#v", got[0]) + } +} + // TestGatherSymbolCandidates_EmptyBackendStillFallsBack: no delta // count AND no doc count keeps the substring fallback for genuinely // empty in-process backends. @@ -155,7 +191,7 @@ func TestGatherSymbolCandidates_EmptyBackendStillFallsBack(t *testing.T) { n := &graph.Node{ID: "app/w.go::WidgetExtensions", Name: "WidgetExtensions", Kind: graph.KindType, RepoPrefix: "app"} g.AddNode(n) engine := NewEngine(g) - engine.SetSearch(search.NewBM25()) // empty: Count 0, no corpus + engine.SetSearch(search.NewNull()) // empty: Count 0, no corpus got := engine.GatherSymbolCandidates("WidgetExtensions", 5, QueryOptions{SkipInnerRerank: true, SkipVectorChannel: true}, nil) if len(got) != 1 || got[0].Node.ID != n.ID { diff --git a/internal/search/bench_test.go b/internal/search/bench_test.go index 53bc274f6..a15bb7776 100644 --- a/internal/search/bench_test.go +++ b/internal/search/bench_test.go @@ -1,7 +1,6 @@ package search import ( - "fmt" "testing" ) @@ -48,79 +47,3 @@ func BenchmarkTokenizeQuery(b *testing.B) { }) } } - -// ============================================================================= -// BM25 scaled benchmarks — memory and latency at different corpus sizes -// ============================================================================= - -func buildBM25(size int) *BM25Backend { - b := NewBM25() - for i := range size { - b.Add( - fmt.Sprintf("pkg%d/file%d.go::func%d", i/100, i/10, i), - fmt.Sprintf("getUserById%d", i%50), - fmt.Sprintf("internal/pkg%d/service%d.go", i/100, i/10), - fmt.Sprintf("func getUserById%d(id string) User", i%50), - ) - } - return b -} - -func BenchmarkBM25_Search_Scaled(b *testing.B) { - sizes := []struct { - name string - n int - }{ - {"100_symbols", 100}, - {"1K_symbols", 1_000}, - {"5K_symbols", 5_000}, - {"10K_symbols", 10_000}, - } - - for _, sz := range sizes { - b.Run(sz.name, func(b *testing.B) { - b.ReportAllocs() - backend := buildBM25(sz.n) - b.ResetTimer() - for b.Loop() { - backend.Search("get user auth", 20) - } - }) - } -} - -func BenchmarkBM25_Add(b *testing.B) { - b.ReportAllocs() - backend := NewBM25() - for i := range b.N { - backend.Add( - fmt.Sprintf("file%d.go::func%d", i/10, i), - fmt.Sprintf("processRequest%d", i), - fmt.Sprintf("pkg/file%d.go", i/10), - fmt.Sprintf("func processRequest%d()", i), - ) - } -} - -func BenchmarkBM25_Remove(b *testing.B) { - backend := buildBM25(5000) - b.ReportAllocs() - b.ResetTimer() - for i := range b.N { - backend.Remove(fmt.Sprintf("pkg%d/file%d.go::func%d", (i%5000)/100, (i%5000)/10, i%5000)) - } -} - -func BenchmarkBM25_ConcurrentSearch(b *testing.B) { - backend := buildBM25(5000) - queries := []string{"get user", "server handle", "graph node", "parse extract", "resolve import"} - b.ReportAllocs() - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - i := 0 - for pb.Next() { - backend.Search(queries[i%len(queries)], 20) - i++ - } - }) -} diff --git a/internal/search/bigram.go b/internal/search/bigram.go deleted file mode 100644 index 45d46791c..000000000 --- a/internal/search/bigram.go +++ /dev/null @@ -1,236 +0,0 @@ -package search - -import ( - "os" - "sort" - "strings" - "sync" -) - -// bigramIndexEnabled reports whether the bigram side index should be -// built on Add/Remove and consulted by the engine's typo-rescue tier. -// Default ON: the typo-rescue tier only fires when the primary search -// returns ZERO results (see the rescue-on-empty caller), so the index -// earns its keep — an empty result set is exactly when a typo is the -// likely cause and the bigram overlap is the cheapest way to recover -// recall. A perf-sensitive operator on a very large index can opt OUT -// with GORTEX_BIGRAM_TYPOS=0 (or false / no / off). Read once at backend -// construction so the flag can't toggle mid-session. -func bigramIndexEnabled() bool { - switch strings.ToLower(strings.TrimSpace(os.Getenv("GORTEX_BIGRAM_TYPOS"))) { - case "0", "false", "no", "off", "n": - return false - } - return true -} - -// bigramIndex is an inverted index from bigram key → docID set, built -// alongside BM25's primary index. Its single purpose is typo-tolerant -// recall: when BM25 returns zero hits for a query, the caller can fall -// back to bigram overlap to find the nearest-matching symbols. The index -// tracks both consecutive bigrams (chars i, i+1) and skip-1 bigrams -// (chars i, i+2) — the latter is the FFF trick that makes "validat_e" -// transpositions match "validate". -type bigramIndex struct { - mu sync.RWMutex - // bigrams keys each bigram (hi<<8 | lo) to the set of docIDs that - // contain it at least once in any indexed token. We use a set - // (map[string]struct{}) rather than a slice so Remove is O(1) and the - // per-bigram densities at compress time come from a simple len(). - bigrams map[uint16]map[string]struct{} - // perDoc tracks the bigrams each doc contributed, for clean Remove. - perDoc map[string][]uint16 -} - -func newBigramIndex() *bigramIndex { - return &bigramIndex{ - bigrams: make(map[uint16]map[string]struct{}), - perDoc: make(map[string][]uint16), - } -} - -// bigramize yields every consecutive and skip-1 bigram key for one -// lowercase token. Non-ASCII bytes are silently skipped so the key space -// stays in uint16 (one byte each side) — good enough for symbol names -// which are almost universally ASCII in real codebases. -func bigramize(token string) []uint16 { - b := []byte(strings.ToLower(token)) - if len(b) < 2 { - return nil - } - out := make([]uint16, 0, 2*len(b)) - // Consecutive pairs. - for i := 1; i < len(b); i++ { - a, c := b[i-1], b[i] - if a > 127 || c > 127 { - continue - } - out = append(out, uint16(a)<<8|uint16(c)) - } - // Skip-1 pairs — typo resilience for single-char substitutions and - // transpositions. FFF encodes these in a separate column; we pool - // them into the same key space which costs some density but halves - // the index footprint. - for i := 2; i < len(b); i++ { - a, c := b[i-2], b[i] - if a > 127 || c > 127 { - continue - } - out = append(out, uint16(a)<<8|uint16(c)) - } - return out -} - -// Add indexes one doc under all bigrams found in any of its tokens. -// Called from BM25Backend.Add with the same tokens that feed the BM25 -// posting lists, so the two indexes stay in lockstep without a second -// tokenization pass. -func (bi *bigramIndex) Add(docID string, tokens []string) { - if bi == nil || docID == "" || len(tokens) == 0 { - return - } - seen := make(map[uint16]struct{}, 16) - for _, t := range tokens { - for _, k := range bigramize(t) { - seen[k] = struct{}{} - } - } - if len(seen) == 0 { - return - } - - bi.mu.Lock() - defer bi.mu.Unlock() - - // Drop any prior presence before re-indexing so a re-Add doesn't leave - // orphan bigrams from the old token set. - bi.removeLocked(docID) - - keys := make([]uint16, 0, len(seen)) - for k := range seen { - set, ok := bi.bigrams[k] - if !ok { - set = make(map[string]struct{}) - bi.bigrams[k] = set - } - set[docID] = struct{}{} - keys = append(keys, k) - } - bi.perDoc[docID] = keys -} - -// Remove deletes a doc from every bigram's set and clears its perDoc list. -func (bi *bigramIndex) Remove(docID string) { - if bi == nil || docID == "" { - return - } - bi.mu.Lock() - defer bi.mu.Unlock() - bi.removeLocked(docID) -} - -func (bi *bigramIndex) removeLocked(docID string) { - keys := bi.perDoc[docID] - for _, k := range keys { - if set, ok := bi.bigrams[k]; ok { - delete(set, docID) - if len(set) == 0 { - delete(bi.bigrams, k) - } - } - } - delete(bi.perDoc, docID) -} - -// Candidates returns docIDs whose token bigram set overlaps the query by -// at least minOverlap distinct bigrams. Density-filtered: bigrams that -// appear in highDocPct of all docs are ignored — very -// rare bigrams are noise, very common ones add no signal. Same defaults -// FFF uses (~3% / 90%). -func (bi *bigramIndex) Candidates(query string, minOverlap int) []string { - if bi == nil || query == "" { - return nil - } - keys := bigramize(query) - if len(keys) == 0 { - return nil - } - if minOverlap < 1 { - minOverlap = 1 - } - - bi.mu.RLock() - defer bi.mu.RUnlock() - - total := len(bi.perDoc) - if total == 0 { - return nil - } - - // Density thresholds. - const ( - lowDocPct = 3 - highDocPct = 90 - ) - loBound := (total * lowDocPct) / 100 - if loBound < 1 { - loBound = 1 - } - hiBound := (total * highDocPct) / 100 - if hiBound < 1 { - hiBound = total - } - - // Overlap count per candidate doc. - overlap := make(map[string]int) - for _, k := range keys { - set, ok := bi.bigrams[k] - if !ok { - continue - } - if len(set) < loBound || len(set) > hiBound { - continue - } - for docID := range set { - overlap[docID]++ - } - } - - // Collect candidates above threshold along with their overlap count, - // then sort by overlap descending so the caller can take top-N by - // similarity rather than by Go's random map-iteration order — the - // latter buried the best match past rank 20 on typo'd exact queries. - type cand struct { - id string - count int - } - cands := make([]cand, 0, len(overlap)) - for id, c := range overlap { - if c >= minOverlap { - cands = append(cands, cand{id, c}) - } - } - sort.Slice(cands, func(i, j int) bool { - if cands[i].count != cands[j].count { - return cands[i].count > cands[j].count - } - // Equal-overlap candidates tie-break on ID for stable ordering. - return cands[i].id < cands[j].id - }) - out := make([]string, len(cands)) - for i, c := range cands { - out[i] = c.id - } - return out -} - -// Size reports the number of distinct bigrams currently indexed. -// Exposed for tests and stats. -func (bi *bigramIndex) Size() int { - if bi == nil { - return 0 - } - bi.mu.RLock() - defer bi.mu.RUnlock() - return len(bi.bigrams) -} diff --git a/internal/search/bigram_test.go b/internal/search/bigram_test.go deleted file mode 100644 index 33ae47375..000000000 --- a/internal/search/bigram_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package search - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestBigramize_ConsecutiveAndSkip(t *testing.T) { - keys := bigramize("abcd") - // Consecutive: ab, bc, cd. Skip-1: ac, bd. Total = 5. - require.Len(t, keys, 5) - - // Shorter-than-2 input yields nothing. - assert.Empty(t, bigramize("a")) - assert.Empty(t, bigramize("")) -} - -func TestBigramIndex_RoundTrip(t *testing.T) { - bi := newBigramIndex() - bi.Add("a", []string{"validate", "token"}) - bi.Add("b", []string{"valid", "input"}) - bi.Add("c", []string{"completely", "unrelated"}) - - // The doc count is 3; density filter permits bigrams appearing in - // [1, 2] docs — exclude any that hit all 3. None should hit all 3 - // here, so the filter is a no-op and the test is deterministic. - cands := bi.Candidates("validate", 4) - assert.Contains(t, cands, "a", "validate's own bigrams should put it top") - - // Removed doc disappears. - bi.Remove("a") - cands = bi.Candidates("validate", 4) - assert.NotContains(t, cands, "a") -} - -func TestBM25_BigramCandidates_ExplicitAPI(t *testing.T) { - // Feature is opt-in via GORTEX_BIGRAM_TYPOS — default-off backends - // allocate no bigram index and return nil from BigramCandidates. - t.Setenv("GORTEX_BIGRAM_TYPOS", "1") - // This test asserts the strict-Search contract: a clean BM25 miss - // returns empty. The sub-word n-gram gate deliberately relaxes that - // (a typo shares sub-word grams with the symbol), so pin it off here - // regardless of any ambient GORTEX_SPARSE_NGRAM — the same hermetic - // pattern withStemming uses for the FTS gate. - withSparseNgram(t, false) - - b := NewBM25() - defer b.Close() - - b.Add("auth/token.go::validateToken", "validateToken", "auth/token.go") - b.Add("api/handler.go::handleRequest", "handleRequest", "api/handler.go") - b.Add("db/store.go::openDB", "openDB", "db/store.go") - - ids := b.BigramCandidates("valiadate", 4) - assert.Contains(t, ids, "auth/token.go::validateToken") - - // Plain Search stays strict — rescue is the engine's job, not the - // backend's. Backend returns empty on a clean BM25 miss. - assert.Empty(t, b.Search("valiadate", 10)) -} - -func TestBM25_BigramOptOut(t *testing.T) { - // The bigram index defaults ON now — the typo-rescue tier fires only on - // zero-result queries, so it earns its keep. A perf-sensitive operator - // opts OUT with GORTEX_BIGRAM_TYPOS=0, restoring the zero-cost path where - // Add/Remove cost nothing extra and BigramCandidates returns nil. - t.Setenv("GORTEX_BIGRAM_TYPOS", "0") - - b := NewBM25() - defer b.Close() - b.Add("auth/token.go::validateToken", "validateToken", "auth/token.go") - - assert.Nil(t, b.BigramCandidates("valiadate", 4)) -} diff --git a/internal/search/bm25.go b/internal/search/bm25.go deleted file mode 100644 index 6701e400c..000000000 --- a/internal/search/bm25.go +++ /dev/null @@ -1,292 +0,0 @@ -package search - -import ( - "math" - "sort" - "sync" -) - -// BM25Backend is a custom in-memory inverted index with BM25 scoring. -// Optimal for repos up to ~50k symbols. Zero external dependencies. -type BM25Backend struct { - mu sync.RWMutex - docs map[string]*doc // docID -> document - inverted map[string][]posting // term -> postings list - totalLen int // sum of all doc lengths (for avgLen) - bigrams *bigramIndex // side index for typo-tolerant fallback recall - // ngrams is the optional learned sub-word boundary source consulted - // by the sparse-ngram emission stage. nil until SetNgramBoundaries - // wires one in; the stage degrades to fixed character n-grams while - // it is nil. Read under mu alongside the postings so Add and Search - // always see the same source — the symmetry the sparse-ngram gate - // depends on. The whole stage is a no-op unless GORTEX_SPARSE_NGRAM - // is set. - ngrams NgramBoundaries -} - -// SetNgramBoundaries installs the learned sub-word boundary source the -// sparse-ngram stage consults. Passing nil (or an empty source) reverts -// the stage to fixed character n-grams. Safe to call while the backend -// is live; it takes the write lock so an in-flight Search never sees a -// half-swapped source. Callers that rebuild the table on every index -// pass (mirroring auto-concept mining) re-install it here. -// -// NOTE: when the gate is on, changing the boundary source changes which -// sub-word grams a token emits. To keep the index and query paths in -// lockstep the source should be installed before the backend is -// populated and then left stable for the backend's lifetime — exactly -// how the per-repo table is built once per RunAnalysis pass and handed -// to a freshly (re)built backend. -func (b *BM25Backend) SetNgramBoundaries(src NgramBoundaries) { - b.mu.Lock() - defer b.mu.Unlock() - b.ngrams = src -} - -// boundarySource returns the currently installed sub-word boundary -// source under the read lock, so the sparse-ngram stage on the index -// and query paths reads a consistent value even if SetNgramBoundaries -// races with an in-flight Add / Search. -func (b *BM25Backend) boundarySource() NgramBoundaries { - b.mu.RLock() - defer b.mu.RUnlock() - return b.ngrams -} - -type doc struct { - id string - len int - terms map[string]int // term -> frequency in this doc -} - -type posting struct { - docID string - freq int -} - -// BM25 parameters. -const ( - bm25K1 = 1.2 - bm25B = 0.75 -) - -// SizeBytes is a rough memory estimate for the BM25 in-memory index: -// every document stores an ID + term-frequency map, and every term in -// the inverted index carries a postings list. The per-doc and per-term -// constants are calibrated against live indexes and land within ~25% -// of actual heap delta. -func (b *BM25Backend) SizeBytes() uint64 { - b.mu.RLock() - defer b.mu.RUnlock() - var bytes uint64 - for _, d := range b.docs { - // doc struct + id string + terms map header - bytes += 96 + uint64(len(d.id)) + 48 - // Each term entry: key string header + ~8 bytes for the int frequency. - for term := range d.terms { - bytes += uint64(len(term)) + 24 - } - } - for term, postings := range b.inverted { - // term string + slice header + postings - bytes += uint64(len(term)) + 24 - bytes += uint64(len(postings)) * 32 // docID string hdr + freq int + ptr - } - return bytes -} - -// NewBM25 creates a new BM25 search backend. The bigram side index for -// typo-tolerant rescue is built only when GORTEX_BIGRAM_TYPOS is set; -// leaving it nil is cheap — every bigram method is nil-safe and returns -// the zero-cost branch, so the engine's typo-rescue tier becomes a no-op -// automatically. -func NewBM25() *BM25Backend { - b := &BM25Backend{ - docs: make(map[string]*doc), - inverted: make(map[string][]posting), - } - if bigramIndexEnabled() { - b.bigrams = newBigramIndex() - } - return b -} - -func (b *BM25Backend) Add(id string, fields ...string) { - // Tokenize all fields together. - var allTokens []string - for _, f := range fields { - allTokens = append(allTokens, Tokenize(f)...) - } - - // Stopword-filter + Porter-stem the posting tokens. Search() - // runs the same normalization, so stemmed postings are always probed - // with stemmed query terms. The bigram side index keeps the raw - // tokens — its typo rescue bigramizes the raw query string, so - // raw-against-raw stays consistent there. - ftsTokens := NormalizeFTSTokens(allTokens) - - // Optional sub-word n-gram expansion. Search() runs the identical - // stage on the same normalized tokens with the same boundary source, - // so n-grammed postings are always probed with n-grammed query - // terms. A no-op unless GORTEX_SPARSE_NGRAM is set; the original - // word tokens are preserved, so an exact match still scores. - ftsTokens = ExpandSparseNgrams(ftsTokens, b.boundarySource()) - - termFreq := make(map[string]int) - for _, t := range ftsTokens { - termFreq[t]++ - } - - b.mu.Lock() - defer b.mu.Unlock() - - // Remove old version if exists. - b.removeLocked(id) - - d := &doc{ - id: id, - len: len(ftsTokens), - terms: termFreq, - } - b.docs[id] = d - b.totalLen += d.len - - for term, freq := range termFreq { - b.inverted[term] = append(b.inverted[term], posting{id, freq}) - } - - // Keep the bigram side index in lockstep — same token set, same doc ID. - b.bigrams.Add(id, allTokens) -} - -func (b *BM25Backend) Remove(id string) { - b.mu.Lock() - defer b.mu.Unlock() - b.removeLocked(id) -} - -func (b *BM25Backend) removeLocked(id string) { - d, ok := b.docs[id] - if !ok { - return - } - - b.totalLen -= d.len - - // Remove from inverted index. - for term := range d.terms { - postings := b.inverted[term] - for i, p := range postings { - if p.docID == id { - b.inverted[term] = append(postings[:i], postings[i+1:]...) - break - } - } - if len(b.inverted[term]) == 0 { - delete(b.inverted, term) - } - } - - delete(b.docs, id) - b.bigrams.Remove(id) -} - -func (b *BM25Backend) Search(query string, limit int) []SearchResult { - queryTokens := NormalizeFTSTokens(TokenizeQuery(query)) - // Mirror the index path's sub-word n-gram expansion exactly — same - // stage, same normalized tokens, same boundary source — so a query - // probes the same n-grammed terms that Add wrote into the postings. - // A no-op unless GORTEX_SPARSE_NGRAM is set. - queryTokens = ExpandSparseNgrams(queryTokens, b.boundarySource()) - if len(queryTokens) == 0 { - return nil - } - - b.mu.RLock() - defer b.mu.RUnlock() - - docCount := len(b.docs) - if docCount == 0 { - return nil - } - - avgLen := float64(b.totalLen) / float64(docCount) - scores := make(map[string]float64) - - for _, term := range queryTokens { - postings, ok := b.inverted[term] - if !ok { - continue - } - df := float64(len(postings)) - idf := math.Log((float64(docCount)-df+0.5)/(df+0.5) + 1) - - for _, p := range postings { - d := b.docs[p.docID] - if d == nil { - continue - } - tf := float64(p.freq) - dl := float64(d.len) - score := idf * (tf * (bm25K1 + 1)) / (tf + bm25K1*(1-bm25B+bm25B*dl/avgLen)) - scores[p.docID] += score - } - } - - if len(scores) == 0 { - // The engine layer has its own fallback chain — exact-name match - // then substring contains — that handles queries like "NewServer" - // which the backend's camelCase-split tokenization misses. We stay - // strict here so those higher-precision fallbacks can run; typo - // rescue via bigram overlap belongs one level up, after those. - return nil - } - - // Sort by score descending. - type scored struct { - id string - score float64 - } - results := make([]scored, 0, len(scores)) - for id, score := range scores { - results = append(results, scored{id, score}) - } - sort.Slice(results, func(i, j int) bool { - if results[i].score != results[j].score { - return results[i].score > results[j].score - } - // Tie-break on doc ID so an equal-score run ships in a stable - // order across calls — Go's map iteration is otherwise random. - return results[i].id < results[j].id - }) - - if len(results) > limit { - results = results[:limit] - } - - out := make([]SearchResult, len(results)) - for i, r := range results { - out[i] = SearchResult{ID: r.id, Score: r.score} - } - return out -} - -// BigramCandidates exposes the bigram-overlap list for explicit typo-mode -// callers. minOverlap gates how similar a doc must be — the caller picks -// the strictness. -func (b *BM25Backend) BigramCandidates(query string, minOverlap int) []string { - if b.bigrams == nil { - return nil - } - return b.bigrams.Candidates(query, minOverlap) -} - -func (b *BM25Backend) Count() int { - b.mu.RLock() - defer b.mu.RUnlock() - return len(b.docs) -} - -func (b *BM25Backend) Close() { - // No-op for in-memory backend. -} diff --git a/internal/search/chunk_dechunk_test.go b/internal/search/chunk_dechunk_test.go index 82c234f6c..75fd2e6b8 100644 --- a/internal/search/chunk_dechunk_test.go +++ b/internal/search/chunk_dechunk_test.go @@ -2,12 +2,59 @@ package search import ( "context" + "slices" + "sort" + "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +// mapText is a map-backed text Backend for the hybrid fixtures. The +// production text side is the store-native FTS, which lives behind a +// graph.SymbolSearcher this package must not reach for; these tests +// only need *a* text channel next to the vector one, so the double +// keeps the corpus in a map and ranks by how many query tokens a +// document's own tokens cover, ties broken by ID for determinism. +type mapText struct{ docs map[string][]string } + +func newMapText() *mapText { return &mapText{docs: make(map[string][]string)} } + +func (m *mapText) Add(id string, fields ...string) { + m.docs[id] = Tokenize(strings.Join(fields, " ")) +} + +func (m *mapText) Remove(id string) { delete(m.docs, id) } +func (m *mapText) Count() int { return len(m.docs) } +func (m *mapText) Close() {} + +func (m *mapText) Search(query string, limit int) []SearchResult { + terms := TokenizeQuery(query) + var out []SearchResult + for id, tokens := range m.docs { + matched := 0 + for _, tok := range tokens { + if slices.Contains(terms, tok) { + matched++ + } + } + if matched > 0 { + out = append(out, SearchResult{ID: id, Score: float64(matched)}) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].Score != out[j].Score { + return out[i].Score > out[j].Score + } + return out[i].ID < out[j].ID + }) + if limit > 0 && len(out) > limit { + out = out[:limit] + } + return out +} + // fixedEmbedder is a deterministic test embedding provider: every query // embeds to the same constant vector, so HybridBackend.Search exercises // the vector channel without any model. The vector backend is what @@ -54,7 +101,7 @@ func TestHybridSearch_DeChunksToParent(t *testing.T) { "big.go::Big#chunk2": "big.go::Big", }) - text := NewBM25() + text := newMapText() text.Add("big.go::Big", "Big", "big.go", "") text.Add("small.go::Small", "Small", "small.go", "") @@ -87,9 +134,8 @@ func TestHybridSearch_DeChunkPreservesOrder(t *testing.T) { "b.go::B#chunk0": "b.go::B", }) - // Keep construction realistic; this assertion exercises the vector - // de-chunk order directly, before channel fusion. - h := NewHybrid(NewBM25(), vec, fixedEmbedder{dims: dims}) + // Empty text backend so only the vector channel decides ordering. + h := NewHybrid(newMapText(), vec, fixedEmbedder{dims: dims}) got := h.dechunkVectorIDs(vec.Search([]float32{1, 0, 0}, 8), 8) require.Len(t, got, 2) @@ -106,7 +152,7 @@ func TestHybridSearch_NoChunkMapUnaffected(t *testing.T) { vec.Add("b.go::B", []float32{0, 1, 0}) require.False(t, vec.HasChunks(), "no SetChunkMap → HasChunks must be false") - h := NewHybrid(NewBM25(), vec, fixedEmbedder{dims: dims}) + h := NewHybrid(newMapText(), vec, fixedEmbedder{dims: dims}) results := h.Search("anything", 10) for _, r := range results { assert.NotContains(t, r.ID, "#chunk") diff --git a/internal/search/equivalence.go b/internal/search/equivalence.go index e8a0f51f4..690338cbf 100644 --- a/internal/search/equivalence.go +++ b/internal/search/equivalence.go @@ -18,7 +18,7 @@ import ( // whose members are genuinely interchangeable in code identifiers // belong here. Domain-bearing words that mean different things in // different codebases are left out -- a false synonym inflates the -// BM25 candidate pool with noise. +// text candidate pool with noise. type EquivalenceTable struct { // member maps each lowercased word to the index of its class in // classes. A word in two classes keeps the first; the curated diff --git a/internal/search/fts_normalization_mode_test.go b/internal/search/fts_normalization_mode_test.go new file mode 100644 index 000000000..06656f9bf --- /dev/null +++ b/internal/search/fts_normalization_mode_test.go @@ -0,0 +1,21 @@ +package search + +import "testing" + +func TestFTSNormalizationModeTracksConfiguredCorpusMode(t *testing.T) { + for _, tc := range []struct { + name string + on bool + want string + }{ + {name: "disabled", on: false, want: ""}, + {name: "enabled", on: true, want: "porter-v1"}, + } { + t.Run(tc.name, func(t *testing.T) { + withStemming(t, tc.on) + if got := FTSNormalizationMode(); got != tc.want { + t.Fatalf("FTSNormalizationMode() = %q, want %q", got, tc.want) + } + }) + } +} diff --git a/internal/search/fts_normalize.go b/internal/search/fts_normalize.go index 89959d2a0..01b2c497a 100644 --- a/internal/search/fts_normalize.go +++ b/internal/search/fts_normalize.go @@ -9,14 +9,15 @@ import ( ) // ftsStemmingEnabled gates the token-normalization pass — stopword -// removal plus Porter stemming — applied to the full-text-search index -// and query paths. Default OFF: on the recall fixture stemming trades +// removal plus Porter stemming — applied to the native symbol FTS index +// and query paths. Content FTS intentionally keeps raw body tokens. +// Default OFF: on the recall fixture stemming trades // exact-symbol-lookup precision (exact-tier R@5 −3.1pp) for broader // recall (R@20 +5.7pp), so it ships as an opt-in rather than quietly // reranking every identifier query. Enable it with // GORTEX_FTS_STEMMING=1 (also true / yes / on). // -// Read once at process start, like the bigram-typo flag: the index +// Read once at process start: the index // built during a daemon's lifetime and every query against it share a // single setting, so a mid-session toggle can't desynchronise stemmed // postings from stemmed query terms. When enabled, the same @@ -24,6 +25,20 @@ import ( // two never disagree. var ftsStemmingEnabled = ftsStemmingFromEnv() +const ftsNormalizationPorterV1 = "porter-v1" + +// FTSNormalizationMode returns the stable identifier persisted beside each +// repository's native symbol FTS corpus. The empty string is the historical +// unnormalised mode, so stores created before the marker existed remain +// compatible while stemming is disabled. A non-empty change requires the +// indexer to rebuild the repository's symbol FTS before serving it. +func FTSNormalizationMode() string { + if ftsStemmingEnabled { + return ftsNormalizationPorterV1 + } + return "" +} + func ftsStemmingFromEnv() bool { switch strings.ToLower(strings.TrimSpace(os.Getenv("GORTEX_FTS_STEMMING"))) { case "1", "true", "yes", "on", "y": @@ -47,11 +62,11 @@ var ftsStopWords = map[string]struct{}{ "it": {}, "its": {}, "so": {}, "such": {}, "via": {}, "per": {}, } -// NormalizeFTSTokens applies the FR63 stopword filter and Porter stemmer -// to a token list produced by Tokenize / TokenizeQuery. The index path -// (BM25Backend.Add) and the query path (BM25Backend.Search) both -// call it, so a stemmed -// posting list is always probed with stemmed query terms. +// NormalizeFTSTokens applies the stopword filter and Porter stemmer to +// a token list produced by Tokenize / TokenizeQuery. Every producer of +// FTS terms routes through it — both the tokens written into the index +// and the tokens a query is lowered into — so a stemmed posting list is +// always probed with stemmed query terms. // // Stopwords are dropped before stemming so a stemmed form can never // collide with a stopword entry. The result is a freshly allocated diff --git a/internal/search/fts_normalize_test.go b/internal/search/fts_normalize_test.go index 9d85e0e86..a90016d31 100644 --- a/internal/search/fts_normalize_test.go +++ b/internal/search/fts_normalize_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) // withStemming pins the FTS normalization gate to a known value for @@ -62,23 +61,9 @@ func TestStemFTSToken_LeavesShortAndNonAlphaTokens(t *testing.T) { } } -func TestBM25_StemmedRecall(t *testing.T) { +func TestNormalizeFTSTokens_AllStopWordsYieldsNoTokens(t *testing.T) { withStemming(t, true) - b := NewBM25() - defer b.Close() - b.Add("svc::UserService", "UserService", "user/service.go") - // A query in a different grammatical number still reaches it. - res := b.Search("users", 10) - require.NotEmpty(t, res) - assert.Equal(t, "svc::UserService", res[0].ID) -} - -func TestBM25_StopWordQueryYieldsNoTokens(t *testing.T) { - withStemming(t, true) - b := NewBM25() - defer b.Close() - b.Add("a::Auth", "Auth", "auth.go") - // An all-stopword query produces no FTS tokens, so BM25 returns - // nothing and the engine layer's substring fallback takes over. - assert.Empty(t, b.Search("the and of", 10)) + // An all-stopword query normalizes to nothing, so the caller issues + // no FTS match at all and the engine's substring fallback takes over. + assert.Empty(t, NormalizeFTSTokens([]string{"the", "and", "of"})) } diff --git a/internal/search/hybrid.go b/internal/search/hybrid.go index ce5eccc1c..ce7169102 100644 --- a/internal/search/hybrid.go +++ b/internal/search/hybrid.go @@ -9,11 +9,11 @@ import ( "github.com/zzet/gortex/internal/search/rerank" ) -// HybridBackend combines text search (BM25 or the store-native FTS -// adapter) with vector search (HNSW) using query-adaptive, α-weighted -// Reciprocal Rank Fusion (RRF). Identifier-shaped queries lean toward BM25, -// where exact-token matches are most reliable; natural-language queries give -// semantic similarity more weight so synonymous wording can surface. +// HybridBackend combines text search (the store-native FTS adapter) +// with vector search (HNSW) using query-adaptive, α-weighted +// Reciprocal Rank Fusion (RRF). Identifier-shaped queries lean toward the +// text channel, where exact-token matches are most reliable; natural-language +// queries give semantic similarity more weight so synonymous wording can surface. type HybridBackend struct { text Backend vector *VectorBackend @@ -31,16 +31,12 @@ func NewHybrid(text Backend, vector *VectorBackend, embedder embedding.Provider) } } -// Add indexes a symbol in both text and vector backends. +// Add forwards a symbol update to the text backend. Vector corpora are +// prepared and published atomically by the indexer. func (h *HybridBackend) Add(id string, fields ...string) { h.text.Add(id, fields...) } -// AddVector adds a vector for a symbol to the vector backend. -func (h *HybridBackend) AddVector(id string, vector []float32) { - h.vector.Add(id, vector) -} - // Remove removes a symbol from the text backend. func (h *HybridBackend) Remove(id string) { h.text.Remove(id) @@ -50,7 +46,7 @@ func (h *HybridBackend) Remove(id string) { } // Search runs both text and vector search and fuses them with adaptive -// α-weighted RRF. Identifier queries lean toward BM25; natural-language +// α-weighted RRF. Identifier queries lean toward text search; natural-language // queries give semantic similarity more weight. func (h *HybridBackend) Search(query string, limit int) []SearchResult { textResults, vecIDs, _ := h.searchChannels(query, limit) @@ -63,7 +59,7 @@ func (h *HybridBackend) Search(query string, limit int) []SearchResult { return alphaFuse(textResults, vecIDs, rerank.AlphaFor(query), h.k, limit) } -// SearchChannels returns the raw per-channel results — BM25 ranks +// SearchChannels returns the raw per-channel results — text ranks // (with scores) and the parallel vector-search ID list — without // RRF fusion. The rerank pipeline calls this so each channel can // contribute as a separate Signal instead of being collapsed into a @@ -83,7 +79,7 @@ type ChannelTimings struct { } // VectorChannelOnly returns the vector-channel IDs (embedder + ANN -// search) WITHOUT re-running the text BM25 path. Used by the engine +// search) WITHOUT re-running the text search. Used by the engine // when the text channel has already been satisfied via the bundle // path — the bundle returns Nodes + edges + scores already, so // re-running text Search would double-pay the FTS cost. Returns @@ -112,7 +108,7 @@ func (h *HybridBackend) VectorChannelOnly(query string, limit int) ([]string, Ch } // SearchChannelsTimed is SearchChannels with a per-phase timing -// breakdown so callers can prove which sub-step (text BM25 vs +// breakdown so callers can prove which sub-step (text FTS vs // vector embed vs vector ANN) actually cost wall-clock time. // Used by the MCP search_symbols handler's debug-log // instrumentation; production callers that don't care just use @@ -131,7 +127,7 @@ func (h *HybridBackend) SearchChannelsTimed(query string, limit int) ([]SearchRe // HybridBackend wires both channels together in production, so the // engine's bundle-detection step type-asserts on the outer // HybridBackend through Swappable; this is what makes the bundle -// path available when the daemon's search is the BM25 + vector +// path available when the daemon's search is the FTS + vector // stack instead of a bare SymbolSearcherBackend. func (h *HybridBackend) SearchSymbolBundles(query string, limit int) []SymbolBundle { if h == nil || h.text == nil { @@ -220,8 +216,23 @@ func (h *HybridBackend) dechunkVectorIDs(rawIDs []string, want int) []string { return out } -// Count returns the text backend document count. -func (h *HybridBackend) Count() int { return h.text.Count() } +// Count reports the corpus visible to hybrid retrieval. A positive text count +// remains authoritative; vector-only hybrids fall back to their vector count. +// The channel counts are alternatives, not additive views of the same symbols. +func (h *HybridBackend) Count() int { + if h == nil { + return 0 + } + if h.text != nil { + if count := h.text.Count(); count > 0 { + return count + } + } + if h.vector != nil { + return h.vector.Count() + } + return 0 +} // Close releases resources owned by the hybrid. The embedding provider and a // delegated vector searcher are externally owned; VectorBackend.Close only @@ -275,9 +286,9 @@ func (h *HybridBackend) VectorSizeBytes() uint64 { return h.vector.SizeBytes() } // alphaFuse combines text and vector results with an α-weighted blend // of their reciprocal-rank contributions. Higher α gives the vector // channel more weight (good for natural-language queries where -// semantic similarity catches synonyms); lower α gives BM25 more -// weight (good for identifier queries where exact-token matches are -// the most reliable signal). +// semantic similarity catches synonyms); lower α gives the text channel +// more weight (good for identifier queries where exact-token matches +// are the most reliable signal). // // Formula: // diff --git a/internal/search/hybrid_test.go b/internal/search/hybrid_test.go index 54ad10718..ee8940bcd 100644 --- a/internal/search/hybrid_test.go +++ b/internal/search/hybrid_test.go @@ -7,6 +7,33 @@ import ( "github.com/stretchr/testify/require" ) +type hybridCountTextBackend struct { + count int +} + +func (*hybridCountTextBackend) Add(string, ...string) {} +func (*hybridCountTextBackend) Remove(string) {} +func (*hybridCountTextBackend) Search(string, int) []SearchResult { return nil } +func (b *hybridCountTextBackend) Count() int { return b.count } +func (*hybridCountTextBackend) Close() {} + +func TestHybridCountUsesVectorWhenTextIsEmpty(t *testing.T) { + vector := NewVector(2) + vector.Add("semantic-only", []float32{1, 0}) + + hybrid := NewHybrid(NewNull(), vector, nil) + require.Equal(t, 1, hybrid.Count(), "a populated vector channel is a searchable corpus") + + textAuthoritative := NewHybrid(&hybridCountTextBackend{count: 3}, vector, nil) + assert.Equal(t, 3, textAuthoritative.Count(), "text and vector counts describe the same corpus and must not be summed") + + empty := NewHybrid(NewNull(), nil, nil) + assert.Zero(t, empty.Count(), "an empty hybrid must report no corpus") + + var nilHybrid *HybridBackend + assert.Zero(t, nilHybrid.Count(), "a nil hybrid must report no corpus") +} + func TestAlphaFuse_EqualWeights(t *testing.T) { textResults := []SearchResult{ {ID: "a", Score: 10}, diff --git a/internal/search/ngram_weights.go b/internal/search/ngram_weights.go deleted file mode 100644 index d1e25b650..000000000 --- a/internal/search/ngram_weights.go +++ /dev/null @@ -1,287 +0,0 @@ -package search - -import ( - "sort" - "unicode" - - "github.com/zzet/gortex/internal/graph" -) - -// NgramTable is a per-repository, LLM-free table of learned sub-word -// boundary weights mined from the symbol names in the graph. Where the -// fixed character n-gram stage cuts every token at a fixed width, this -// table cuts each token at *high-information* boundaries — positions -// where the adjacent character pair is rare across the repo's symbol -// vocabulary, which is where a name tends to seam (the "tk" in -// "validateTokenizer" is far rarer than the "to"/"ke"/"en" inside -// "token", so the split lands at the seam, not mid-word). -// -// Go has no compile-time table generation, so this is NOT a comptime -// literal map: it is computed at index / analysis time, once per -// RunAnalysis pass, exactly like the auto-concept vocabulary. The build -// is one tokenizing pass over node names plus a bounded character-pair -// count, cheap enough to recompute on every reindex. -// -// The table feeds the sparse-ngram tokenizer (see ExpandSparseNgrams): -// when a non-empty table is installed on the backend, the tokenizer -// asks it where to Split each word token instead of slicing at a fixed -// n. A nil or empty table degrades the tokenizer to fixed character -// n-grams, so the search path is identical whether or not the table has -// been built yet. -type NgramTable struct { - // boundaryPairs holds the character bigrams whose normalized - // corpus frequency is low enough to count as a split seam. A pair - // is packed as (hi<<16 | lo) of its two runes; only pairs over the - // ASCII-letter alphabet are tracked, which covers essentially all - // real symbol names. Presence in the set means "split between these - // two characters". The set is derived deterministically from the - // sorted frequency table at build time and never mutated after. - boundaryPairs map[uint32]struct{} -} - -// Ngram boundary-mining bounds. Mirrors the auto-concept caps in spirit -// — keep the pair count and the boundary set bounded on a large -// monorepo, and require enough evidence before trusting a seam. -const ( - // ngramMinTokenChars is the shortest token the boundary miner will - // split. A token this short or shorter has no interior seam worth - // learning; the tokenizer keeps it whole (or, in fixed mode, emits - // its single full-length gram). - ngramMinTokenChars = 5 - // ngramMinCorpusPairs is the minimum number of distinct adjacent - // character pairs the corpus must yield before the table trusts its - // frequency distribution. Below this the sample is too thin to tell - // a rare seam from noise, and the table stays empty so the - // tokenizer falls back to fixed n-grams. - ngramMinCorpusPairs = 8 - // ngramBoundaryPercentile selects the rarest adjacent pairs as - // seams: a pair counts as a boundary when its frequency rank falls - // in the bottom this-many percent of all observed pairs. Lower = - // fewer, higher-confidence seams. - ngramBoundaryPercentile = 25 -) - -// BuildNgramBoundaries mines the per-repo sub-word boundary table from a -// graph. Only named code symbols (the same kinds auto-concept mining -// uses) contribute their names. A nil or empty graph — or one whose -// symbol names yield too few distinct character pairs to be -// trustworthy — yields an empty, safe-to-query table that reports -// Empty() == true, so the tokenizer degrades to fixed character -// n-grams. -// -// The build is deterministic: every map is drained into a sorted slice -// before any threshold is derived or any boundary is selected, so two -// builds over the same graph produce byte-identical boundary sets. -func BuildNgramBoundaries(g graph.Reader) *NgramTable { - t := &NgramTable{boundaryPairs: map[uint32]struct{}{}} - if g == nil { - return t - } - - // Pass 1: count how often each adjacent character pair occurs - // inside the repo's symbol-name word tokens. We reuse the - // auto-concept tokenizer so the boundary table is learned over the - // same word vocabulary the rest of search tokenizes on. - pairCount := map[uint32]int{} - for _, n := range g.AllNodes() { - if !autoConceptEligible(n.Kind) { - continue - } - for _, tok := range autoConceptTokens(n.Name) { - countAdjacentPairs(tok, pairCount) - } - } - - // Too thin a sample: keep the table empty rather than learn noise. - if len(pairCount) < ngramMinCorpusPairs { - return t - } - - // Pass 2: rank the pairs by frequency, ascending, with a stable - // tie-break on the packed key so the ordering — and therefore the - // percentile cut — is identical across runs regardless of Go's - // random map iteration. The rarest pairs are the high-information - // seams. - type pairFreq struct { - key uint32 - count int - } - ranked := make([]pairFreq, 0, len(pairCount)) - for k, c := range pairCount { - ranked = append(ranked, pairFreq{k, c}) - } - sort.Slice(ranked, func(i, j int) bool { - if ranked[i].count != ranked[j].count { - return ranked[i].count < ranked[j].count - } - return ranked[i].key < ranked[j].key - }) - - // Select the bottom percentile as boundaries. At least one seam is - // kept whenever the sample cleared the minimum, so a small but - // valid corpus still learns something. - cut := (len(ranked) * ngramBoundaryPercentile) / 100 - if cut < 1 { - cut = 1 - } - for _, pf := range ranked[:cut] { - t.boundaryPairs[pf.key] = struct{}{} - } - return t -} - -// InstallNgramBoundaries walks a search backend down to the BM25 layer -// and installs the learned boundary table on it, so the sparse-ngram -// tokenizer's split decisions become data-driven. The production -// backend is a Swappable wrapping either a HybridBackend (text+vector) -// or a bare BM25Backend; this unwraps both. Backends with no BM25 layer -// (SymbolSearcher) do not run the sparse-ngram stage, so there -// is nothing to install and the call is a harmless no-op returning -// false. -// -// Symmetry contract: install the table before the backend is populated -// and leave it stable for the backend's lifetime. The index and query -// paths both read the installed table per call, so a single stable -// table keeps n-grammed postings and n-grammed query terms in lockstep. -// Installing a different table after postings exist would desynchronise -// them while the gate is on — callers re-install only as part of a -// fresh (re)index, never against a live, already-populated index. -func InstallNgramBoundaries(backend Backend, table NgramBoundaries) bool { - if swappable, ok := backend.(*Swappable); ok { - inner, release := swappable.AcquireBackend() - defer release() - return InstallNgramBoundaries(inner, table) - } - bm := bm25Of(backend) - if bm == nil { - return false - } - bm.SetNgramBoundaries(table) - return true -} - -// BuildAndInstallNgramBoundaries mines and installs a boundary table only -// when backend actually contains a BM25 layer. Capability detection must -// precede BuildNgramBoundaries: the native SQLite FTS never consumes the -// table, and walking the whole graph for them is pure allocation and I/O. -func BuildAndInstallNgramBoundaries(backend Backend, g graph.Reader) bool { - if swappable, ok := backend.(*Swappable); ok { - inner, release := swappable.AcquireBackend() - defer release() - return BuildAndInstallNgramBoundaries(inner, g) - } - bm := bm25Of(backend) - if bm == nil { - return false - } - bm.SetNgramBoundaries(BuildNgramBoundaries(g)) - return true -} - -// bm25Of unwraps a non-swappable backend down to its *BM25Backend, or -// returns nil when the backend has no BM25 layer. Swappable callers are -// handled by the public installation functions so the returned pointer is -// consumed while its AcquireBackend pin remains held. -func bm25Of(backend Backend) *BM25Backend { - switch b := backend.(type) { - case *BM25Backend: - return b - case *HybridBackend: - return bm25Of(b.TextBackend()) - default: - return nil - } -} - -// countAdjacentPairs tallies every adjacent ASCII-letter character pair -// in one lowercase token into counts. Pairs touching a non-ASCII or -// non-letter rune are skipped — digits and symbols are not part of the -// learned alphabet, mirroring how the FTS stemmer leaves digit-bearing -// tokens alone. The token is assumed already lowercased by -// autoConceptTokens. -func countAdjacentPairs(tok string, counts map[uint32]int) { - r := []rune(tok) - for i := 1; i < len(r); i++ { - a, b := r[i-1], r[i] - if !isLearnableRune(a) || !isLearnableRune(b) { - continue - } - counts[packPair(a, b)]++ - } -} - -// isLearnableRune reports whether a rune participates in the learned -// pair alphabet: ASCII letters only. -func isLearnableRune(r rune) bool { - return r <= unicode.MaxASCII && unicode.IsLetter(r) -} - -// packPair packs two runes into a single uint32 key (hi<<16 | lo). -// Both runes are ASCII here, so they fit in 16 bits each with room to -// spare. -func packPair(a, b rune) uint32 { - return uint32(a)<<16 | uint32(b) -} - -// Empty reports whether the table learned any boundaries. A nil table, -// or one mined from an empty / too-thin graph, is empty — callers MUST -// treat an empty table as "no learned boundaries" and degrade to fixed -// behaviour rather than splitting on nothing. Nil-safe so a typed-nil -// *NgramTable stored in the NgramBoundaries interface still answers -// correctly. -func (t *NgramTable) Empty() bool { - return t == nil || len(t.boundaryPairs) == 0 -} - -// BoundaryCount reports the number of learned seam pairs. Used by tests -// and diagnostics. -func (t *NgramTable) BoundaryCount() int { - if t == nil { - return 0 - } - return len(t.boundaryPairs) -} - -// Split cuts a token's runes at the learned high-information boundaries -// and returns the resulting segments left-to-right. A split is taken -// between positions i-1 and i when the adjacent character pair is a -// learned seam. The token is never split so finely that a segment falls -// below the minimum gram length: a candidate seam that would strand a -// sub-minimal segment on either side is skipped, so Split always yields -// segments the tokenizer can use. When the table is empty Split returns -// the whole token as a single segment, leaving the fixed-n fallback to -// the caller. -// -// Split is deterministic — it scans left to right and consults only the -// immutable boundary set — and never mutates the input. -func (t *NgramTable) Split(runes []rune) []string { - if t.Empty() || len(runes) < ngramMinTokenChars { - return []string{string(runes)} - } - - var ( - segs []string - start int - ) - for i := 1; i < len(runes); i++ { - a, b := runes[i-1], runes[i] - if !isLearnableRune(a) || !isLearnableRune(b) { - continue - } - if _, seam := t.boundaryPairs[packPair(a, b)]; !seam { - continue - } - // Only take the seam if both the segment it closes and the - // remainder it opens can still carry a usable gram — guards - // against shredding the token into sub-minimal fragments. - left := i - start - right := len(runes) - i - if left < sparseNgramMinN || right < sparseNgramMinN { - continue - } - segs = append(segs, string(runes[start:i])) - start = i - } - segs = append(segs, string(runes[start:])) - return segs -} diff --git a/internal/search/ngram_weights_test.go b/internal/search/ngram_weights_test.go deleted file mode 100644 index e29e1b707..000000000 --- a/internal/search/ngram_weights_test.go +++ /dev/null @@ -1,218 +0,0 @@ -package search - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/zzet/gortex/internal/graph" -) - -// boundaryFixtureGraph builds a graph whose function names share a -// dominant interior vocabulary ("token", "validate", "handler") so the -// adjacent-pair frequency distribution has a clear common core and rare -// seams. Mirrors fixtureGraph in auto_concepts_test.go. -func boundaryFixtureGraph(names []string) *graph.Graph { - g := graph.New() - for i, n := range names { - g.AddNode(&graph.Node{ - ID: "pkg/f.go::" + n, - Kind: graph.KindFunction, - Name: n, - FilePath: "pkg/f.go", - StartLine: i + 1, - EndLine: i + 2, - Language: "go", - }) - } - return g -} - -type countingNgramReader struct { - graph.Reader - allNodesCalls int -} - -func (r *countingNgramReader) AllNodes() []*graph.Node { - r.allNodesCalls++ - return r.Reader.AllNodes() -} - -func TestBuildNgramBoundaries_NilAndEmpty(t *testing.T) { - // nil graph -> empty table -> tokenizer degrades to fixed behavior. - require.True(t, BuildNgramBoundaries(nil).Empty()) - if c := BuildNgramBoundaries(nil).BoundaryCount(); c != 0 { - t.Errorf("nil graph should yield 0 boundaries, got %d", c) - } - // Empty graph: no names, no pairs, empty table. - require.True(t, BuildNgramBoundaries(graph.New()).Empty()) - // A typed-nil table must answer Empty() true without panicking. - var tnil *NgramTable - require.True(t, tnil.Empty()) -} - -func TestNgramTable_EmptyTableSplitIsWhole(t *testing.T) { - // An empty table splits nothing: the token comes back whole, leaving - // the fixed-n fallback to the tokenizer. - tbl := BuildNgramBoundaries(graph.New()) - require.True(t, tbl.Empty()) - got := tbl.Split([]rune("validateToken")) - assert.Equal(t, []string{"validateToken"}, got) -} - -func TestBuildNgramBoundaries_Deterministic(t *testing.T) { - names := []string{ - "validateToken", "validateTokenizer", "tokenValidator", - "handleToken", "tokenHandler", "parseToken", "tokenParser", - "refreshToken", "tokenRefresher", "revokeToken", - "validateHandler", "handlerValidator", "buildHandler", - } - a := BuildNgramBoundaries(boundaryFixtureGraph(names)) - b := BuildNgramBoundaries(boundaryFixtureGraph(names)) - // Two builds over equivalent graphs must produce byte-identical - // boundary sets — no Go map-iteration nondeterminism may leak into - // the percentile cut. - require.Equal(t, a.BoundaryCount(), b.BoundaryCount()) - for k := range a.boundaryPairs { - _, ok := b.boundaryPairs[k] - assert.Truef(t, ok, "boundary %d present in build A but not build B", k) - } - // And, crucially, the Split decisions must agree token for token. - for _, tok := range names { - assert.Equalf(t, a.Split([]rune(tok)), b.Split([]rune(tok)), - "Split(%q) differs between two deterministic builds", tok) - } -} - -func TestBuildNgramBoundaries_LearnsAReasonableSeam(t *testing.T) { - // A corpus dominated by the substring "token": the pairs inside - // "token" (to, ok, ke, en) are frequent, so a rare cross-word pair - // is selected as a seam and at least one multi-token name splits. - names := []string{ - "tokenAlpha", "tokenBeta", "tokenGamma", "tokenDelta", - "tokenEpsilon", "tokenZeta", "alphaToken", "betaToken", - "gammaToken", "deltaToken", "tokenize", "tokenizer", - "retoken", "subtoken", "tokenList", "tokenMap", - } - tbl := BuildNgramBoundaries(boundaryFixtureGraph(names)) - require.False(t, tbl.Empty(), "a corpus this size should learn boundaries") - - // At least one eligible token must split into >1 segment, otherwise - // the table learned nothing usable. - splitSomething := false - for _, tok := range names { - if len(tbl.Split([]rune(tok))) > 1 { - splitSomething = true - break - } - } - assert.True(t, splitSomething, "the learned table should split at least one token") -} - -func TestNgramTable_SplitNeverStrandsSubMinimalSegments(t *testing.T) { - names := []string{ - "validateToken", "validateTokenizer", "tokenValidator", - "handleToken", "tokenHandler", "parseToken", "tokenParser", - "refreshToken", "tokenRefresher", "revokeToken", - } - tbl := BuildNgramBoundaries(boundaryFixtureGraph(names)) - if tbl.Empty() { - t.Skip("fixture learned no boundaries; nothing to assert") - } - for _, tok := range names { - segs := tbl.Split([]rune(tok)) - if len(segs) == 1 { - continue // unsplit token: the whole thing, fine. - } - for _, s := range segs { - assert.GreaterOrEqualf(t, len([]rune(s)), sparseNgramMinN, - "Split(%q) stranded a sub-minimal segment %q", tok, s) - } - } -} - -func TestNgramTable_SplitShortTokenWhole(t *testing.T) { - tbl := BuildNgramBoundaries(boundaryFixtureGraph([]string{ - "tokenAlpha", "tokenBeta", "tokenGamma", "tokenDelta", - "alphaToken", "betaToken", "gammaToken", "deltaToken", - "tokenize", "tokenizer", - })) - // A token shorter than the minimum is never split. - got := tbl.Split([]rune("go")) - assert.Equal(t, []string{"go"}, got) -} - -// TestInstallNgramBoundaries_BM25Chain verifies the install helper finds -// the BM25 layer through the production Swappable wrapper and that a -// non-BM25 backend is a harmless no-op. -func TestInstallNgramBoundaries_BM25Chain(t *testing.T) { - bm := NewBM25() - defer bm.Close() - tbl := BuildNgramBoundaries(boundaryFixtureGraph([]string{ - "tokenAlpha", "tokenBeta", "tokenGamma", "tokenDelta", - "alphaToken", "betaToken", "tokenize", "tokenizer", - })) - - // Bare BM25. - assert.True(t, InstallNgramBoundaries(bm, tbl)) - - // Wrapped in a Swappable, as in production. - sw := NewSwappable(NewBM25()) - defer sw.Close() - assert.True(t, InstallNgramBoundaries(sw, tbl)) - - // A backend with no BM25 layer: no-op, returns false. - assert.False(t, InstallNgramBoundaries(nonBM25Backend{}, tbl)) -} - -// nonBM25Backend is an inert Backend with no BM25 anywhere in its -// unwrap chain — the shape the ngram installers must refuse. -type nonBM25Backend struct{} - -func (nonBM25Backend) Add(string, ...string) {} -func (nonBM25Backend) Remove(string) {} -func (nonBM25Backend) Search(string, int) []SearchResult { return nil } -func (nonBM25Backend) Count() int { return 0 } -func (nonBM25Backend) Close() {} - -func TestBuildAndInstallNgramBoundaries_ChecksCapabilityBeforeGraphScan(t *testing.T) { - g := &countingNgramReader{Reader: boundaryFixtureGraph([]string{ - "tokenAlpha", "tokenBeta", "tokenGamma", "tokenDelta", - "alphaToken", "betaToken", "tokenize", "tokenizer", - })} - - assert.False(t, BuildAndInstallNgramBoundaries(nonBM25Backend{}, g)) - assert.Zero(t, g.allNodesCalls, "a backend without BM25 must not enumerate the graph") - - bm := NewBM25() - defer bm.Close() - assert.True(t, BuildAndInstallNgramBoundaries(bm, g)) - assert.Equal(t, 1, g.allNodesCalls, "BM25 must mine exactly one graph census") -} - -// TestSparseNgram_TableDrivenVsFixed proves the learned table actually -// changes the tokenizer's emitted grams versus the fixed-n fallback, -// and that the index/query symmetry still holds with a table installed: -// a query routed through the same table reaches a doc it shares a -// learned segment with. -func TestSparseNgram_TableDrivenVsFixed(t *testing.T) { - withSparseNgram(t, true) - tbl := BuildNgramBoundaries(boundaryFixtureGraph([]string{ - "tokenAlpha", "tokenBeta", "tokenGamma", "tokenDelta", - "tokenEpsilon", "alphaToken", "betaToken", "gammaToken", - "tokenize", "tokenizer", "retoken", "subtoken", - })) - require.False(t, tbl.Empty()) - - b := NewBM25() - defer b.Close() - require.True(t, InstallNgramBoundaries(b, tbl)) - b.Add("svc::tokenAlpha", "tokenAlpha", "x.go") - - // The same table drives both Add and Search, so a query that shares - // a learned segment with the indexed symbol reaches it. - res := b.Search("token", 10) - require.NotEmpty(t, res) - assert.Equal(t, "svc::tokenAlpha", res[0].ID) -} diff --git a/internal/search/null.go b/internal/search/null.go new file mode 100644 index 000000000..98aa893d5 --- /dev/null +++ b/internal/search/null.go @@ -0,0 +1,55 @@ +package search + +// NullBackend is the text backend for graph stores that expose no +// native symbol search. It indexes nothing and answers nothing: Add, +// Remove and Close are no-ops, Search returns no hits, and Count +// always reports zero. +// +// Reporting an empty text corpus is the point. When used on its own, the query +// Engine gates its ranked path on the backend having something to answer with +// (Engine.backendHasCorpus) and otherwise falls through to its own substring +// scan over the graph, which needs no text index at all. When NullBackend is the +// text side of a HybridBackend, the hybrid's vector count can still establish a +// searchable corpus. A bare NullBackend therefore routes such a store onto +// exactly the path an Engine with no search backend at all takes in production — +// see the Engine built by pkg/gortex.New, which wires query.NewEngine +// and never calls SetSearch. +// +// NewNull hands back a distinct pointer per call so independent backend +// constructions retain ordinary object identity instead of collapsing onto +// runtime.zerobase. +// +// It deliberately implements Backend and nothing else. Satisfying +// DocCounter would let a disk-corpus count claim a corpus that does +// not exist; Sizer and ChannelSearcher would advertise memory and +// per-channel retrieval it does not have. +type NullBackend struct { + // A field-less struct has size zero, and the language only + // promises distinct addresses for variables of non-zero size: every + // escaping &NullBackend{} would share runtime.zerobase, so any two + // null backends would compare equal. One byte buys each + // construction a real address, which is what the identity promise + // on NewNull rests on. Nothing reads it — the blank name says so. + _ byte +} + +// NewNull returns a Backend that indexes and answers nothing. Every call +// allocates its own instance, so independently constructed null backends have +// distinct identity just like every other backend implementation. +func NewNull() Backend { return &NullBackend{} } + +// Add discards the symbol; nothing is indexed. +func (*NullBackend) Add(string, ...string) {} + +// Remove is a no-op — nothing was ever indexed. +func (*NullBackend) Remove(string) {} + +// Search always returns no hits, so callers fall through to whatever +// index-free path they keep for an empty corpus. +func (*NullBackend) Search(string, int) []SearchResult { return nil } + +// Count reports an empty corpus. +func (*NullBackend) Count() int { return 0 } + +// Close is a no-op — there are no resources to release. +func (*NullBackend) Close() {} diff --git a/internal/search/null_test.go b/internal/search/null_test.go new file mode 100644 index 000000000..810d0c804 --- /dev/null +++ b/internal/search/null_test.go @@ -0,0 +1,72 @@ +package search + +import "testing" + +// The null backend must satisfy Backend itself — the optional +// interfaces it must NOT satisfy are asserted below at run time. +var _ Backend = (*NullBackend)(nil) + +// TestNullBackend_ImplementsBackendAndNothingElse pins the property the +// fallback seam rests on. The query engine treats a backend as having a +// corpus when Count() is positive OR when it satisfies DocCounter and +// reports a positive disk count; Sizer makes daemon status attribute +// heap to it and ChannelSearcher makes the rerank pipeline ask it for a +// text channel. A NullBackend that grew any of those optional +// interfaces would start claiming a corpus it does not have, and the +// engine would rank against nothing instead of falling back to its +// substring scan. +func TestNullBackend_ImplementsBackendAndNothingElse(t *testing.T) { + b := NewNull() + + if _, ok := b.(DocCounter); ok { + t.Error("NullBackend must not implement DocCounter — it would claim a corpus it has no documents for") + } + if _, ok := b.(Sizer); ok { + t.Error("NullBackend must not implement Sizer — it holds no memory to attribute") + } + if _, ok := b.(ChannelSearcher); ok { + t.Error("NullBackend must not implement ChannelSearcher — it has no text channel to contribute") + } + if got := BackendSize(b); got != 0 { + t.Errorf("BackendSize = %d, want 0", got) + } +} + +// TestNullBackend_StaysEmptyAfterAdd is the behaviour the seam promises: +// indexing into the null backend never produces a searchable corpus, so +// callers gated on Count() keep taking their index-free path. +func TestNullBackend_StaysEmptyAfterAdd(t *testing.T) { + b := NewNull() + defer b.Close() + + b.Add("pkg/a.go::Alpha", "Alpha", "pkg/a.go", "") + b.Add("pkg/b.go::Beta", "Beta", "pkg/b.go", "") + + if got := b.Count(); got != 0 { + t.Errorf("Count after Add = %d, want 0", got) + } + if got := b.Search("Alpha", 10); len(got) != 0 { + t.Errorf("Search returned %d hits, want 0", len(got)) + } + + b.Remove("pkg/a.go::Alpha") + if got := b.Count(); got != 0 { + t.Errorf("Count after Remove = %d, want 0", got) + } +} + +// nullIdentityEscapeSink forces both returned pointers to escape. Without it, +// inlining may give two zero-sized values distinct stack addresses and let a +// field-less NullBackend pass the identity assertion accidentally. +var nullIdentityEscapeSink [2]Backend + +// TestNewNullReturnsDistinctInstances pins NewNull's identity contract +// directly. NullBackend intentionally carries one byte so independent +// constructions cannot collapse onto runtime.zerobase. +func TestNewNullReturnsDistinctInstances(t *testing.T) { + nullIdentityEscapeSink = [2]Backend{NewNull(), NewNull()} + t.Cleanup(func() { nullIdentityEscapeSink = [2]Backend{} }) + if nullIdentityEscapeSink[0] == nullIdentityEscapeSink[1] { + t.Fatal("NewNull handed back one shared instance") + } +} diff --git a/internal/search/project_name_test.go b/internal/search/project_name_test.go index 623ec08d1..f324dcfcb 100644 --- a/internal/search/project_name_test.go +++ b/internal/search/project_name_test.go @@ -57,41 +57,3 @@ func mustWrite(t *testing.T, path, content string) { t.Fatal(err) } } - -// TestTypoRescueOnEmpty proves the bigram typo-rescue index is built by default -// (no GORTEX_BIGRAM_TYPOS needed) so a typo'd query that returns zero BM25 hits -// can still be rescued by bigram overlap — and that the opt-out flag disables -// it for a perf-sensitive operator. -func TestTypoRescueOnEmpty(t *testing.T) { - t.Run("default_on", func(t *testing.T) { - t.Setenv("GORTEX_BIGRAM_TYPOS", "") // simulate "unset" → default ON - b := NewBM25() - b.Add("doc1", "validateToken") - b.Add("doc2", "parseRequest") - - // A transposed/truncated query misses BM25 entirely... - if hits := b.Search("validat", 10); len(hits) > 0 { - t.Logf("note: BM25 returned %d hits for the typo (stemming); rescue still applies on true empties", len(hits)) - } - // ...but the bigram rescue tier recovers the nearest symbol. - cands := b.BigramCandidates("validat", 1) - found := false - for _, c := range cands { - if c == "doc1" { - found = true - } - } - if !found { - t.Errorf("typo rescue did not recover doc1 from %q; candidates=%v", "validat", cands) - } - }) - - t.Run("opt_out", func(t *testing.T) { - t.Setenv("GORTEX_BIGRAM_TYPOS", "0") - b := NewBM25() - b.Add("doc1", "validateToken") - if cands := b.BigramCandidates("validat", 1); cands != nil { - t.Errorf("GORTEX_BIGRAM_TYPOS=0 should disable the bigram index, got candidates %v", cands) - } - }) -} diff --git a/internal/search/rerank/pipeline.go b/internal/search/rerank/pipeline.go index 8ade05dfa..89d7ee75b 100644 --- a/internal/search/rerank/pipeline.go +++ b/internal/search/rerank/pipeline.go @@ -21,10 +21,9 @@ import ( type Candidate struct { Node *graph.Node - // TextRank is the 0-based BM25 rank. -1 means the candidate did - // not appear in the text-search result list (e.g. a substring or - // bigram-rescue fallback hit, or a candidate added by another - // retrieval channel). + // TextRank is the 0-based text-search rank. -1 means the candidate + // did not appear in the text-search result list (e.g. a substring + // fallback hit, or a candidate added by another retrieval channel). TextRank int // VectorRank is the 0-based vector-search rank. -1 means absent. VectorRank int diff --git a/internal/search/rerank/retriever.go b/internal/search/rerank/retriever.go index 28042e79c..33a43e34f 100644 --- a/internal/search/rerank/retriever.go +++ b/internal/search/rerank/retriever.go @@ -7,7 +7,8 @@ import ( ) // Retriever is the pluggable candidate-producer protocol. The existing -// hybrid pipeline (BM25 + vector + RRF) is one implementation; others +// hybrid pipeline (native symbol FTS + vector + adaptive fusion) is one +// implementation; others // — graph_completion, an in-house embedding model, a research-grade // LLM-as-retriever — plug in by implementing this interface. // diff --git a/internal/search/search.go b/internal/search/search.go index 9418056f5..c723487ed 100644 --- a/internal/search/search.go +++ b/internal/search/search.go @@ -1,10 +1,16 @@ // Package search provides full-text search over code symbols with -// camelCase/snake_case-aware tokenization and BM25 ranking. +// camelCase/snake_case-aware tokenization. // -// Production search runs on SymbolSearcherBackend, a thin adapter over -// the graph store's own FTS index — no parallel in-process corpus. -// BM25Backend, a self-contained in-memory inverted index, is the -// fallback for stores that expose no native symbol search. +// The package owns no text index of its own. Search runs on +// SymbolSearcherBackend, a thin adapter over the graph store's own FTS +// index, and this package contributes the tokenization and query +// normalization both sides of that index agree on. A store that +// exposes no native symbol search gets NullBackend, whose empty corpus +// routes the query engine to its substring fallback. +// +// HybridBackend layers the optional vector channel on top of whichever +// text Backend it is given; Swappable lets the indexer replace the +// backend under a live engine. package search // SearchResult is a single search hit. @@ -33,10 +39,10 @@ type Backend interface { // ChannelSearcher is an optional interface a Backend can implement to // expose its per-channel raw retrieval output. The rerank pipeline -// queries it so BM25 and semantic (vector) ranks can contribute as +// queries it so text and semantic (vector) ranks can contribute as // separate signals instead of being collapsed via RRF before scoring. -// Backends that only do text search (BM25, the store-native FTS -// adapter) don't satisfy this interface; callers fall through to plain +// Backends that only do text search (the store-native FTS adapter) +// don't satisfy this interface; callers fall through to plain // Search(). type ChannelSearcher interface { SearchChannels(query string, limit int) (textResults []SearchResult, vectorIDs []string) @@ -61,10 +67,3 @@ func BackendSize(b Backend) uint64 { } return 0 } - -// NewAuto returns the default in-process text backend. Reached only -// when the graph store exposes no native symbol search; otherwise the -// indexer wires up a SymbolSearcherBackend over the store's own FTS. -func NewAuto() Backend { - return NewBM25() -} diff --git a/internal/search/search_test.go b/internal/search/search_test.go index 11e3a053d..bdb6b3203 100644 --- a/internal/search/search_test.go +++ b/internal/search/search_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestTokenize(t *testing.T) { @@ -36,113 +35,3 @@ func TestTokenizeQuery(t *testing.T) { tokens = TokenizeQuery("go test") assert.Equal(t, []string{"go", "test"}, tokens) } - -// runBackendTests runs the same test suite on any Backend implementation. -func runBackendTests(t *testing.T, name string, backend Backend) { - // This conformance suite asserts baseline exact-match / no-match - // semantics shared by every backend. The opt-in sub-word n-gram gate - // deliberately adds fuzzy recall paths that would turn a "no match" - // assertion into a match, so pin it off here regardless of any - // ambient GORTEX_SPARSE_NGRAM — sub-word recall is exercised in the - // dedicated sparse-ngram tests, not here. - withSparseNgram(t, false) - t.Run(name+"/BasicSearch", func(t *testing.T) { - backend.Add("auth/token.go::validateToken", "validateToken", "auth/token.go") - backend.Add("auth/token.go::parseJWT", "parseJWT", "auth/token.go") - backend.Add("api/handler.go::handleRequest", "handleRequest", "api/handler.go") - backend.Add("db/query.go::buildQuery", "buildQuery", "db/query.go") - backend.Add("payment/charge.go::chargeCard", "chargeCard", "payment/charge.go") - - results := backend.Search("validate token", 10) - require.NotEmpty(t, results) - assert.Equal(t, "auth/token.go::validateToken", results[0].ID) - }) - - t.Run(name+"/CamelCaseSearch", func(t *testing.T) { - results := backend.Search("handle request", 10) - require.NotEmpty(t, results) - assert.Equal(t, "api/handler.go::handleRequest", results[0].ID) - }) - - t.Run(name+"/PathSearch", func(t *testing.T) { - results := backend.Search("payment", 10) - require.NotEmpty(t, results) - assert.Equal(t, "payment/charge.go::chargeCard", results[0].ID) - }) - - t.Run(name+"/Remove", func(t *testing.T) { - backend.Add("tmp.go::tempFunc", "tempFunc", "tmp.go") - results := backend.Search("temp func", 10) - require.NotEmpty(t, results) - - backend.Remove("tmp.go::tempFunc") - results = backend.Search("temp func", 10) - // Should not find the removed symbol. - for _, r := range results { - assert.NotEqual(t, "tmp.go::tempFunc", r.ID) - } - }) - - t.Run(name+"/Count", func(t *testing.T) { - assert.Equal(t, 5, backend.Count()) - }) - - t.Run(name+"/EmptyQuery", func(t *testing.T) { - results := backend.Search("", 10) - assert.Empty(t, results) - }) - - t.Run(name+"/NoMatch", func(t *testing.T) { - results := backend.Search("xyznonexistent", 10) - assert.Empty(t, results) - }) -} - -func TestBM25Backend(t *testing.T) { - backend := NewBM25() - defer backend.Close() - runBackendTests(t, "BM25", backend) -} - -func TestBM25_RankingQuality(t *testing.T) { - b := NewBM25() - defer b.Close() - - // Add symbols with varying relevance to "auth token" - b.Add("auth/token.go::validateToken", "validateToken", "auth/token.go", "func validateToken(token string) bool") - b.Add("auth/session.go::createSession", "createSession", "auth/session.go", "func createSession(userID string) Session") - b.Add("config/config.go::loadConfig", "loadConfig", "config/config.go", "func loadConfig(path string) Config") - b.Add("api/handler.go::tokenHandler", "tokenHandler", "api/handler.go", "func tokenHandler(w http.ResponseWriter)") - - results := b.Search("auth token", 10) - // OR ranking: every doc matching any query token is scored. - // validateToken has both terms so it ranks first; createSession - // and tokenHandler each match one; loadConfig matches neither - // and is dropped. - require.Len(t, results, 3) - assert.Equal(t, "auth/token.go::validateToken", results[0].ID) -} - -func TestBM25_DuplicateTokensCollapse(t *testing.T) { - b := NewBM25() - defer b.Close() - b.Add("a", "token token", "a.go") - b.Add("b", "token parser", "b.go") - - results := b.Search("token token token", 10) - require.Len(t, results, 2) -} - -func BenchmarkBM25_Search(b *testing.B) { - backend := NewBM25() - for i := 0; i < 10000; i++ { - backend.Add( - "pkg/file.go::func"+string(rune('A'+i%26))+string(rune('0'+i%10)), - "getUserById", "internal/auth/service.go", "func getUserById(id string) User", - ) - } - b.ResetTimer() - for b.Loop() { - backend.Search("get user auth", 20) - } -} diff --git a/internal/search/sparse_ngram.go b/internal/search/sparse_ngram.go deleted file mode 100644 index cf86096c5..000000000 --- a/internal/search/sparse_ngram.go +++ /dev/null @@ -1,153 +0,0 @@ -package search - -import ( - "os" - "strings" -) - -// sparseNgramEnabled gates the optional sub-word n-gram emission stage -// layered over the fixed-rule word tokens that Tokenize / TokenizeQuery -// already produce. Default OFF: emitting character n-grams for every -// word token multiplies the posting set, and on identifier-heavy -// queries the extra sub-word noise can demote an exact match — a -// precision regression we are not willing to ship enabled by default -// until it proves out against the recall fixture. Opt in with -// GORTEX_SPARSE_NGRAM=1 (also true / yes / on / y). -// -// Read once at process start, exactly like the FTS-stemming and -// bigram-typo flags. The index built during a daemon's lifetime and -// every query against it share a single setting, so a mid-session -// toggle can never desynchronise the n-grammed postings from the -// n-grammed query terms: the same emission runs on the index path -// (BM25Backend.Add) and the query path (BM25Backend.Search) because -// both route their word tokens through ExpandSparseNgrams. -var sparseNgramEnabled = sparseNgramFromEnv() - -func sparseNgramFromEnv() bool { - switch strings.ToLower(strings.TrimSpace(os.Getenv("GORTEX_SPARSE_NGRAM"))) { - case "1", "true", "yes", "on", "y": - return true - } - return false -} - -// Sub-word n-gram bounds. Character n-grams in this closed range are -// emitted over each word token when no learned boundary table is -// supplied. n=3..4 is the usual sweet spot for code identifiers: short -// enough to bridge a morphological or typo gap ("valid" reaches -// "validate" via shared trigrams), long enough to stay discriminative -// (bigrams collide across nearly every token, which is why the typo -// side index keeps them separate from the BM25 postings). -const ( - sparseNgramMinN = 3 - sparseNgramMaxN = 4 - // sparseNgramMinTokenLen skips tokens no longer than the smallest - // n-gram — a token of length <= sparseNgramMinN has at most one - // n-gram equal to itself, so it carries no extra sub-word signal - // and only inflates the posting set. - sparseNgramMinTokenLen = sparseNgramMinN + 1 -) - -// NgramBoundaries is the data-driven split source the sparse-ngram -// stage consults when one is available. A learned boundary table built -// at index time (see BuildNgramBoundaries) satisfies it, but the -// tokenizer depends only on this abstraction so it compiles and runs -// with a nil source — degrading cleanly to fixed character n-grams. -// -// Empty reports whether the source carries any learned boundaries; an -// empty source is treated exactly like a nil one. Split cuts a token's -// runes at the source's boundaries and returns the resulting segments -// left-to-right; it must be deterministic for a given input. -type NgramBoundaries interface { - Empty() bool - Split(runes []rune) []string -} - -// ExpandSparseNgrams returns the input word tokens unchanged when the -// sparse-ngram gate is off, and otherwise returns the input tokens -// followed by their emitted sub-word n-grams. The original word tokens -// are always preserved and always come first, so enabling the gate can -// only ADD recall paths — an exact word match still scores through the -// untouched word token, and the appended sub-word grams open additional -// fuzzy-match paths. -// -// The same function runs on both the index and query paths, so the -// postings written for a symbol and the terms a query probes them with -// are produced by identical logic and can never disagree. -// -// When a non-nil, non-empty boundary source is supplied the split -// points are data-driven: each token is cut at the source's -// high-information boundaries and the resulting segments are emitted -// alongside the original token. When the source is nil or empty the -// stage degrades to fixed character n-grams in -// [sparseNgramMinN, sparseNgramMaxN], so the tokenizer behaves -// identically whether or not a learned table has been built yet. -// -// The result is a freshly allocated slice; the input is left untouched. -// Emission is deterministic: for a given token and source the n-grams -// are produced left-to-right in a fixed order with no map iteration. -func ExpandSparseNgrams(tokens []string, table NgramBoundaries) []string { - if !sparseNgramEnabled || len(tokens) == 0 { - return tokens - } - // Preserve the original word tokens verbatim, in order, then append - // the sub-word grams. Over-allocate modestly for the common case. - out := make([]string, 0, len(tokens)*3) - out = append(out, tokens...) - for _, tok := range tokens { - out = appendSparseNgrams(out, tok, table) - } - return out -} - -// appendSparseNgrams appends the sub-word n-grams of one lowercase word -// token to dst and returns the grown slice. A learned boundary source, -// when present and non-empty, drives the split; otherwise the token is -// sliced into fixed-width character n-grams. Tokens too short to carry -// any sub-word signal contribute nothing. Duplicate grams within a -// single token are collapsed so a token like "aaaa" does not emit the -// same gram twice. -func appendSparseNgrams(dst []string, tok string, table NgramBoundaries) []string { - r := []rune(tok) - if len(r) < sparseNgramMinTokenLen { - return dst - } - - seen := make(map[string]struct{}, len(r)) - emit := func(g string) { - if g == "" || g == tok { - return - } - if _, dup := seen[g]; dup { - return - } - seen[g] = struct{}{} - dst = append(dst, g) - } - - if table != nil && !table.Empty() { - // Data-driven: split the token at the learned high-information - // boundaries, then emit each segment. Segments shorter than the - // minimum n are dropped — they collide across too many tokens to - // be useful sub-word keys. - for _, seg := range table.Split(r) { - if len([]rune(seg)) >= sparseNgramMinN { - emit(seg) - } - } - return dst - } - - // Fixed character n-grams in [min, max]. Cap the upper n at the - // token length so a short token still yields its single full-length - // gram rather than nothing. - for n := sparseNgramMinN; n <= sparseNgramMaxN; n++ { - if n > len(r) { - break - } - for i := 0; i+n <= len(r); i++ { - emit(string(r[i : i+n])) - } - } - return dst -} diff --git a/internal/search/sparse_ngram_test.go b/internal/search/sparse_ngram_test.go deleted file mode 100644 index 18313fea7..000000000 --- a/internal/search/sparse_ngram_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package search - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// withSparseNgram pins the sparse-ngram gate to a known value for the -// duration of a test and restores it afterwards, so an ambient -// GORTEX_SPARSE_NGRAM in the environment can't make the suite flaky. -// Mirrors withStemming in fts_normalize_test.go. -func withSparseNgram(t *testing.T, on bool) { - t.Helper() - prev := sparseNgramEnabled - sparseNgramEnabled = on - t.Cleanup(func() { sparseNgramEnabled = prev }) -} - -func TestExpandSparseNgrams_FlagOffIsNoOp(t *testing.T) { - withSparseNgram(t, false) - in := []string{"validate", "token"} - got := ExpandSparseNgrams(in, nil) - // Gate off: the exact same slice is returned, no grams appended. - assert.Equal(t, in, got) -} - -func TestExpandSparseNgrams_FlagOffEmptyInput(t *testing.T) { - withSparseNgram(t, false) - assert.Empty(t, ExpandSparseNgrams(nil, nil)) -} - -func TestExpandSparseNgrams_PreservesWordTokensFirst(t *testing.T) { - withSparseNgram(t, true) - got := ExpandSparseNgrams([]string{"validate", "token"}, nil) - // The original word tokens must survive verbatim and lead the - // output, so an exact word match still scores through them. - require.GreaterOrEqual(t, len(got), 2) - assert.Equal(t, "validate", got[0]) - assert.Equal(t, "token", got[1]) -} - -func TestExpandSparseNgrams_EmitsFixedCharNgrams(t *testing.T) { - withSparseNgram(t, true) - got := ExpandSparseNgrams([]string{"token"}, nil) - set := map[string]struct{}{} - for _, g := range got { - set[g] = struct{}{} - } - // "token" -> trigrams tok,oke,ken and 4-grams toke,oken. - for _, want := range []string{"tok", "oke", "ken", "toke", "oken"} { - _, ok := set[want] - assert.Truef(t, ok, "expected sub-word gram %q in %v", want, got) - } - // The whole token is preserved but never re-emitted as a gram equal - // to itself. - assert.Equal(t, "token", got[0]) -} - -func TestExpandSparseNgrams_ShortTokenEmitsNoGrams(t *testing.T) { - withSparseNgram(t, true) - // "go" is below the minimum gram length, so it yields only itself. - got := ExpandSparseNgrams([]string{"go"}, nil) - assert.Equal(t, []string{"go"}, got) -} - -func TestExpandSparseNgrams_CollapsesDuplicateGrams(t *testing.T) { - withSparseNgram(t, true) - // "aaaa": trigram "aaa" appears twice and the 4-gram is the token - // itself (suppressed). The duplicate "aaa" must collapse to one. - got := ExpandSparseNgrams([]string{"aaaa"}, nil) - count := 0 - for _, g := range got { - if g == "aaa" { - count++ - } - } - assert.Equal(t, 1, count, "duplicate grams within a token must collapse: %v", got) -} - -func TestExpandSparseNgrams_Deterministic(t *testing.T) { - withSparseNgram(t, true) - a := ExpandSparseNgrams([]string{"validate", "handler"}, nil) - b := ExpandSparseNgrams([]string{"validate", "handler"}, nil) - // No map iteration leaks into the emission order. - assert.Equal(t, a, b) -} - -// TestBM25_SparseNgramSymmetry is the load-bearing invariant: every -// term a Search query emits for a doc must be present in the postings -// Add wrote for that doc. We verify it by tokenizing both sides through -// the exact stages the backend uses and asserting the query token set -// is a subset of the indexed token set for an overlapping query. -func TestBM25_SparseNgramSymmetry(t *testing.T) { - withSparseNgram(t, true) - - // Index side: what Add writes for the symbol name "validateToken". - indexTokens := ExpandSparseNgrams( - NormalizeFTSTokens(Tokenize("validateToken")), nil) - indexed := map[string]struct{}{} - for _, tok := range indexTokens { - indexed[tok] = struct{}{} - } - - // Query side: a prefix query "valid" that shares sub-word grams. - queryTokens := ExpandSparseNgrams( - NormalizeFTSTokens(TokenizeQuery("valid")), nil) - require.NotEmpty(t, queryTokens) - - // At least one query gram must hit an indexed gram — otherwise the - // expansion bought no recall. (It must, since "valid" is a prefix of - // "validate" and they share trigrams val/ali/lid.) - overlap := 0 - for _, tok := range queryTokens { - if _, ok := indexed[tok]; ok { - overlap++ - } - } - assert.Positivef(t, overlap, "query grams %v share no indexed gram with %v", - queryTokens, indexTokens) -} - -// TestBM25_SparseNgramRecall exercises the full backend: with the gate -// on, a sub-word query reaches a symbol it would miss on pure word -// tokenization. -func TestBM25_SparseNgramRecall(t *testing.T) { - withSparseNgram(t, true) - b := NewBM25() - defer b.Close() - b.Add("svc::validateToken", "validateToken", "auth/token.go") - - // "valid" is no word token of "validateToken" (camelCase splits to - // validate/token), but it shares sub-word grams with "validate". - res := b.Search("valid", 10) - require.NotEmpty(t, res) - assert.Equal(t, "svc::validateToken", res[0].ID) -} - -// TestBM25_SparseNgramFlagOffUnchanged confirms the gate-off backend -// behaves exactly as before: a sub-word-only query finds nothing. -func TestBM25_SparseNgramFlagOffUnchanged(t *testing.T) { - withSparseNgram(t, false) - b := NewBM25() - defer b.Close() - b.Add("svc::validateToken", "validateToken", "auth/token.go") - - // With the gate off there is no sub-word path, so a fragment that is - // not a whole word token returns nothing from the backend. - res := b.Search("valid", 10) - assert.Empty(t, res) -} diff --git a/internal/search/swappable.go b/internal/search/swappable.go index 6951fb344..488289964 100644 --- a/internal/search/swappable.go +++ b/internal/search/swappable.go @@ -27,7 +27,7 @@ type Swappable struct { } // NewSwappable wraps b. Panics if b is nil — every Indexer must start -// with a real backend, even if it's the in-memory NewAuto() default. +// with a real backend, even if it's the NewNull() no-op default. func NewSwappable(b Backend) *Swappable { if b == nil { panic("search.NewSwappable: nil backend") @@ -107,16 +107,6 @@ func (s *Swappable) AcquireBackend() (backend Backend, release func()) { } } -// Inner returns an unpinned snapshot of the currently-active backend. It is -// retained for tests and diagnostics only: production callers must not keep or -// dereference the result because replacement may retire it immediately after -// this method returns. Use AcquireBackend or a forwarded capability instead. -func (s *Swappable) Inner() Backend { - s.mu.RLock() - defer s.mu.RUnlock() - return s.inner -} - // Embedder returns the active hybrid's externally-owned embedding provider. // The lookup is protected by the Swappable read lock; replacement never closes // the provider, so the returned provider remains under its original owner's @@ -230,7 +220,7 @@ func (s *Swappable) SearchSymbolBundlesScoped(query string, repoAllow []string, // VectorChannelOnly forwards to the inner backend when it implements // the vector-only channel pull (today: HybridBackend). Lets the -// engine fetch the vector channel without re-running text BM25 — +// engine fetch the vector channel without re-running the text search — // the bundle path already has the text hits. Returns (nil, zero // timings) when the inner backend isn't vector-aware. func (s *Swappable) VectorChannelOnly(query string, limit int) ([]string, ChannelTimings) { diff --git a/internal/search/symbolsearcher_backend.go b/internal/search/symbolsearcher_backend.go index 8941eeba9..51164a167 100644 --- a/internal/search/symbolsearcher_backend.go +++ b/internal/search/symbolsearcher_backend.go @@ -2,8 +2,6 @@ package search import ( "strings" - "sync" - "sync/atomic" "github.com/zzet/gortex/internal/graph" ) @@ -11,14 +9,15 @@ import ( // SymbolSearcherBackend adapts a graph.SymbolSearcher into the // search.Backend the daemon's search-symbols path consumes. // Engine.gatherBackendCandidates and the rerank pipeline don't need -// to know whether the backend is BM25 or native FTS — they -// see a plain search.Backend and call Search on it. +// to know where the ranking comes from — they see a plain +// search.Backend and call Search on it. // // Production wiring: when the indexer detects that the backing // graph.Store also implements graph.SymbolSearcher, it constructs -// this adapter as the initial -// search.Backend wrapped by search.NewSwappable. The in-process -// BM25 build path is then bypassed entirely. +// this adapter as the initial search.Backend wrapped by +// search.NewSwappable. A store without that capability gets +// NullBackend instead, and the engine falls back to its substring +// scan — no text index is ever built in this process. // // Add / Remove are no-ops on the adapter because the indexer // already drives the SymbolSearcher writes directly: @@ -35,13 +34,6 @@ import ( // happen through the direct SymbolSearcher surface. type SymbolSearcherBackend struct { s graph.SymbolSearcher - - // count is lazily seeded from the authoritative persisted FTS count on its - // first read, then follows the indexer's incremental Add/Remove deltas. Lazy - // initialization avoids a discarded global count query for each per-repo - // Indexer constructed by MultiIndexer. - countInit sync.Once - count atomic.Int64 } // NewSymbolSearcherBackend wraps a SymbolSearcher in the @@ -153,48 +145,28 @@ func (b *SymbolSearcherBackend) Search(query string, limit int) []SearchResult { } // Add is a no-op — the indexer drives UpsertSymbolFTS on the wrapped -// SymbolSearcher directly. count is bumped immediately so deltas that arrive -// before the first Count call are preserved when the persisted snapshot is -// added. -func (b *SymbolSearcherBackend) Add(id string, _ ...string) { - if b == nil || id == "" { - return - } - b.count.Add(1) -} +// SymbolSearcher directly. +func (b *SymbolSearcherBackend) Add(string, ...string) {} -// Remove is a no-op for the same reason as Add — the per-call -// removal path (when one lands) routes through SymbolSearcher -// directly, not through the search.Backend contract. count is -// decremented so the Count() figure stays roughly consistent. -func (b *SymbolSearcherBackend) Remove(id string) { - if b == nil || id == "" { - return - } - b.count.Add(-1) -} +// Remove is a no-op for the same reason as Add — the removal path routes +// through SymbolSearcher directly, not through the search.Backend contract. +func (b *SymbolSearcherBackend) Remove(string) {} -// Count returns the persisted corpus snapshot observed on its first call plus -// subsequent Add/Remove deltas. It is suitable for readiness gates and rough -// magnitude only; DocCount reads the authoritative current size. +// Count returns the authoritative persisted corpus size when the wrapped store +// exposes it. Add and Remove deliberately do not maintain a second local count: +// their corresponding FTS writes happen directly on the store, so applying the +// same deltas here would double-count them. func (b *SymbolSearcherBackend) Count() int { - if b == nil { + count, ok := b.DocCount() + if !ok { return 0 } - b.countInit.Do(func() { - if counter, ok := b.s.(graph.SymbolFTSCounter); ok { - if count, err := counter.SymbolFTSCount(); err == nil && count > 0 { - b.count.Add(int64(count)) - } - } - }) - return int(b.count.Load()) + return count } -// DocCounter is the capability interface for the authoritative corpus -// size — distinct from Backend.Count(), which is a process-local -// Add/Remove delta. The engine's has-corpus gate asserts this; -// Swappable and HybridBackend forward it. +// DocCounter is the capability interface for the authoritative corpus size. +// The engine's has-corpus gate asserts this; Swappable and HybridBackend +// forward it. type DocCounter interface { DocCount() (int, bool) } @@ -202,10 +174,8 @@ type DocCounter interface { // DocCount returns the authoritative number of indexed documents, straight // from the underlying index, and reports whether it could be obtained. // -// Count() must not be used for this: its cached readiness snapshot can drift as -// direct store maintenance and best-effort Add/Remove deltas diverge. Anything -// user-facing asks here and omits the figure when the authoritative answer is -// unavailable. +// Count delegates to this method as well, so readiness checks and user-facing +// status observe the same store-owned value. func (b *SymbolSearcherBackend) DocCount() (int, bool) { if b == nil || b.s == nil { return 0, false diff --git a/internal/search/symbolsearcher_backend_test.go b/internal/search/symbolsearcher_backend_test.go index 944bb1f2e..0a699264e 100644 --- a/internal/search/symbolsearcher_backend_test.go +++ b/internal/search/symbolsearcher_backend_test.go @@ -30,30 +30,25 @@ func (s *countedSymbolSearcherStub) SymbolFTSCount() (int, error) { return s.count, s.err } -func TestNewSymbolSearcherBackendSeedsPersistedCountLazily(t *testing.T) { +func TestSymbolSearcherBackendCountUsesAuthoritativeCorpus(t *testing.T) { store := &countedSymbolSearcherStub{count: 37} backend := NewSymbolSearcherBackend(store) if store.calls != 0 { t.Fatalf("constructor SymbolFTSCount calls = %d, want 0", store.calls) } - backend.Add("new-before-first-count") - if got := backend.Count(); got != 38 { - t.Fatalf("first Count() = %d, want persisted count plus prior delta 38", got) - } - if store.calls != 1 { - t.Fatalf("SymbolFTSCount calls = %d, want 1", store.calls) - } - if got := backend.Count(); got != 38 { - t.Fatalf("second Count() = %d, want cached count 38", got) - } - if store.calls != 1 { - t.Fatalf("repeated Count SymbolFTSCount calls = %d, want 1", store.calls) + backend.Add("already-persisted") + if got := backend.Count(); got != 37 { + t.Fatalf("Count() = %d, want persisted count 37", got) } - backend.Remove("new-before-first-count") - if got := backend.Count(); got != 37 { - t.Fatalf("Count() after balanced delta = %d, want 37", got) + store.count = 41 + backend.Remove("already-removed") + if got := backend.Count(); got != 41 { + t.Fatalf("Count() after store update = %d, want 41", got) + } + if store.calls != 2 { + t.Fatalf("SymbolFTSCount calls = %d, want one authoritative read per Count", store.calls) } } diff --git a/internal/search/vector.go b/internal/search/vector.go index 0c4c36787..c2ad249a1 100644 --- a/internal/search/vector.go +++ b/internal/search/vector.go @@ -20,10 +20,10 @@ type VectorDelegate interface { // VectorBackend stores and searches embedding vectors using HNSW index. // -// When a delegate is installed, the in-process HNSW is absent: Add tracks -// compatibility-path writes without allocating, and Search forwards to the -// delegate's SimilarTo. Durable hits carry their parent IDs, so chunk results -// can be collapsed without retaining a process-local chunk map. +// When a delegate is installed, the in-process HNSW is absent: Add ignores +// process-local writes, while Search forwards to the delegate's SimilarTo. +// Durable hits carry their parent IDs, so chunk results can be collapsed +// without retaining a process-local chunk map. type VectorBackend struct { graph *hnsw.Graph[string] count int @@ -112,11 +112,10 @@ func (v *VectorBackend) Add(id string, vector []float32) { v.mu.Lock() defer v.mu.Unlock() if v.delegate != nil { - // Legacy compatibility for callers that install a delegate before - // pushing vectors themselves. New publication must construct the - // backend with NewDelegatedVector and complete durable statistics. - // Either way Add never allocates an in-process HNSW in delegated mode. - v.delegateCount++ + // The durable corpus is authoritative. Publication replaces it through + // the store and constructs a fresh backend with NewDelegatedVector; + // accepting a process-local Add here would fabricate Count without + // persisting searchable data. return } v.graph.Add(hnsw.Node[string]{ @@ -126,26 +125,6 @@ func (v *VectorBackend) Add(id string, vector []float32) { v.count++ } -// SetDelegate is the legacy compatibility switch for callers that select a -// delegate before writing their corpus. It releases all prior heap and chunk -// state atomically; subsequent Add calls only advance this process's accepted -// write count. New and warm-start publication must use NewDelegatedVector so -// Count and HasChunks describe the complete durable corpus immediately. -func (v *VectorBackend) SetDelegate(d VectorDelegate) { - v.mu.Lock() - defer v.mu.Unlock() - - // Switching modes must also release the old heap index. Merely routing - // reads to d would leave the complete HNSW graph, its count, and chunk - // ownership map retained for the lifetime of the daemon. - v.graph = nil - v.count = 0 - v.chunkMap = nil - v.delegate = d - v.delegateCount = 0 - v.delegateChunkCount = 0 -} - // Search returns the k nearest neighbors to the query vector. func (v *VectorBackend) Search(query []float32, k int) []string { if k <= 0 { diff --git a/internal/search/vector_publication_test.go b/internal/search/vector_publication_test.go index 1ed7f710d..26e5269ca 100644 --- a/internal/search/vector_publication_test.go +++ b/internal/search/vector_publication_test.go @@ -87,6 +87,13 @@ func TestNewDelegatedVectorHasNoHeapIndexAndUsesDurableStats(t *testing.T) { if got := backend.Count(); got != 17 { t.Fatalf("Count() = %d, want complete durable count 17", got) } + backend.Add("process-local-write", []float32{1, 0}) + if got := backend.Count(); got != 17 { + t.Fatalf("Count() = %d after delegated Add, want unchanged durable count 17", got) + } + if backend.graph != nil { + t.Fatal("delegated Add allocated an in-process HNSW graph") + } if !backend.HasChunks() { t.Fatal("HasChunks() = false, want durable chunk metadata") } @@ -136,44 +143,6 @@ func TestDelegatedVectorSearchRejectsNonPositiveLimitWithoutDelegateCall(t *test } } -func TestSetDelegateReleasesHeapAndChunkState(t *testing.T) { - backend := NewVector(2) - backend.Add("symbol-a#chunk0", []float32{1, 0}) - backend.Add("symbol-b", []float32{0, 1}) - backend.SetChunkMap(map[string]string{"symbol-a#chunk0": "symbol-a"}) - if backend.SizeBytes() == 0 { - t.Fatal("heap backend reported no retained vector bytes before delegation") - } - - backend.SetDelegate(&publicationTestDelegate{}) - - if backend.graph != nil { - t.Fatal("SetDelegate retained the old HNSW graph") - } - if backend.chunkMap != nil { - t.Fatal("SetDelegate retained the old chunk ownership map") - } - if got := backend.Count(); got != 0 { - t.Fatalf("Count() = %d after mode switch, want fresh delegated count", got) - } - if backend.HasChunks() { - t.Fatal("SetDelegate retained old chunk statistics") - } - if got := backend.SizeBytes(); got != 0 { - t.Fatalf("SizeBytes() = %d after delegation, want zero", got) - } - - // Preserve the legacy build path: Add records successfully accepted - // delegate-side writes without rebuilding an HNSW graph. - backend.Add("symbol-c", []float32{1, 0}) - if got := backend.Count(); got != 1 { - t.Fatalf("Count() = %d after delegated Add, want 1", got) - } - if backend.graph != nil { - t.Fatal("delegated Add allocated an HNSW graph") - } -} - func TestSerializeVectorUpdateSerializesConcurrentCallbacks(t *testing.T) { swappable := NewSwappable(&publicationTestTextBackend{}) firstStarted := make(chan struct{}) @@ -366,16 +335,20 @@ func TestReplaceHybridVectorWaitsForReadersAndTransfersTextOwnership(t *testing. t.Fatalf("new reader saw %v, want complete new result %v", got, want) } - current, ok := swappable.Inner().(*HybridBackend) - if !ok { - t.Fatalf("active backend is %T, want *HybridBackend", swappable.Inner()) - } - if current.TextBackend() != text { - t.Fatal("replacement did not retain the original text backend") - } - if _, nested := current.TextBackend().(*HybridBackend); nested { - t.Fatal("replacement nested a HybridBackend inside another HybridBackend") - } + func() { + backend, release := swappable.AcquireBackend() + defer release() + current, ok := backend.(*HybridBackend) + if !ok { + t.Fatalf("active backend is %T, want *HybridBackend", backend) + } + if current.TextBackend() != text { + t.Fatal("replacement did not retain the original text backend") + } + if _, nested := current.TextBackend().(*HybridBackend); nested { + t.Fatal("replacement nested a HybridBackend inside another HybridBackend") + } + }() swappable.Close() if got := text.closes(); got != 1 { @@ -438,9 +411,11 @@ func vectorIDs(swappable *Swappable, query string) []string { func assertSingleHybrid(t *testing.T, swappable *Swappable, wantText Backend) { t.Helper() - hybrid, ok := swappable.Inner().(*HybridBackend) + backend, release := swappable.AcquireBackend() + defer release() + hybrid, ok := backend.(*HybridBackend) if !ok { - t.Fatalf("active backend is %T, want *HybridBackend", swappable.Inner()) + t.Fatalf("active backend is %T, want *HybridBackend", backend) } if hybrid.TextBackend() != wantText { t.Fatalf("hybrid text backend is %T, want original %T", hybrid.TextBackend(), wantText)