Merge pull request #191 from Wolfvin/fix/issue-188-semantic-query-graph-nodes

fix(semantic-query): read symbols from graph_nodes, not empty symbols table (closes #188)

Author: Wolfvin

scripts/semantic_search_engine.py

@@ -13,10 +13,13 @@
   or any ~80 MB embedding model. Fast to import, deterministic, no native
   deps. Good enough for the majority of agent "find the right file" queries.
 * **Reads from the existing SQLite registry.** Symbol text is built from
-  fields already populated by ``persistent_registry`` — ``name``,
-  ``signature``, ``kind``, ``file_path``, ``language`` — so the index is
-  always in sync with the last ``scan`` result. No separate index file to
-  keep consistent.
+  the ``graph_nodes`` table populated by ``scan`` via
+  :func:`graph_model.populate_graph_tables` — fields ``name``,
+  ``node_type``, ``file``, ``line``, ``extra_json`` — so the index is
+  always in sync with the last ``scan`` result. No separate index file
+  to keep consistent. (Issue #188: previously read from the ``symbols``
+  table, which is never populated by the scan flow — see
+  :func:`_load_symbols_from_db` for details.)
 * **In-memory index, cached per (db_path, mtime).** Building the vocabulary
   is O(N_symbols) — for CodeLens's own 3000-node graph this is <100 ms.
   Cached so repeated ``semantic_query`` calls within a session are ~1 ms.
@@ -379,8 +382,34 @@ def clear_cache() -> None:
 def _load_symbols_from_db(db_path: str) -> List[Dict[str, Any]]:
     """Load all symbols from the SQLite registry at ``db_path``.
 
-    Returns an empty list if the database doesn't exist, the ``symbols``
-    table is missing, or SQLite is unavailable. Never raises — semantic
+    Reads from the ``graph_nodes`` table populated by ``scan`` via
+    :func:`graph_model.populate_graph_tables`. The legacy ``symbols``
+    table declared by ``persistent_registry`` is never populated by the
+    scan flow (issue #188) — only the graph tables are — so reading from
+    ``symbols`` returned an empty result set on every real workspace.
+
+    Column mapping (graph_nodes -> symbol dict consumed by this module):
+
+        node_id   -> id          (string node id, e.g. "auth/jwt.py:42")
+        name      -> name        (symbol name)
+        file      -> file_path   (relative file path)
+        node_type -> kind        (function|class|file|module|route|type|interface)
+        line      -> line_start  (1-indexed line number)
+        extra_json-> extra_json  (preserves async/impl_for/status/...)
+
+    Fields that ``graph_nodes`` does not carry (``signature``,
+    ``language``, ``line_end``, ``hash``) are defaulted to empty/None so
+    downstream consumers (:func:`_build_symbol_text`, :func:`semantic_query`
+    result construction) keep working unchanged. Signatures and language
+    are not part of the flat-registry node format produced by parsers
+    (see ``scripts/parsers/python_parser.py`` for a representative node),
+    so there is no signal loss in practice — the TF-IDF document is still
+    built from name + file_path + kind + extra_json, which is enough for
+    "find by meaning" queries.
+
+    Returns an empty list if the database doesn't exist, the
+    ``graph_nodes`` table is missing (e.g. workspace initialized but
+    never scanned), or SQLite is unavailable. Never raises — semantic
     search is a non-breaking add-on and should degrade gracefully to
     "no results" rather than crash the host command.
 
@@ -398,21 +427,47 @@ def _load_symbols_from_db(db_path: str) -> List[Dict[str, Any]]:
         conn = sqlite3.connect(db_path)
         try:
             conn.row_factory = sqlite3.Row
-            # Sanity-check that the symbols table exists. If the workspace
-            # has been initialized but never scanned, the db file exists
-            # but has no tables.
+            # Sanity-check that the graph_nodes table exists. If the
+            # workspace has been initialized but never scanned, the db
+            # file exists but has no graph tables.
             table = conn.execute(
                 "SELECT name FROM sqlite_master "
-                "WHERE type='table' AND name='symbols'"
+                "WHERE type='table' AND name='graph_nodes'"
             ).fetchone()
             if table is None:
                 return []
-            rows = conn.execute("SELECT * FROM symbols").fetchall()
-            return [dict(r) for r in rows]
+            # Map graph_nodes columns to the symbol-dict shape expected
+            # by _build_symbol_text and semantic_query's result builder.
+            # Columns not present in graph_nodes are filled with defaults
+            # so the rest of the engine does not need to know about the
+            # source table.
+            rows = conn.execute(
+                """
+                SELECT
+                    node_id   AS id,
+                    name      AS name,
+                    node_type AS kind,
+                    file      AS file_path,
+                    line      AS line_start,
+                    extra_json AS extra_json
+                FROM graph_nodes
+                """
+            ).fetchall()
+            symbols: List[Dict[str, Any]] = []
+            for r in rows:
+                d = dict(r)
+                # Fill defaults for fields graph_nodes does not carry so
+                # _build_symbol_text / result construction keep working.
+                d.setdefault("line_end", None)
+                d.setdefault("language", "")
+                d.setdefault("signature", "")
+                d.setdefault("hash", "")
+                symbols.append(d)
+            return symbols
         finally:
             conn.close()
     except sqlite3.Error as e:
-        logger.debug(f"semantic_search_engine: failed to read symbols from {db_path}: {e}")
+        logger.debug(f"semantic_search_engine: failed to read graph_nodes from {db_path}: {e}")
         return []
 
 

tests/test_semantic_search_engine.py

@@ -20,7 +20,7 @@
 * IDF weighting causes rare, discriminative terms (e.g. ``jwt``) to rank
   higher than ubiquitous terms (e.g. ``function``).
 * The engine degrades gracefully — empty query, missing db, missing
-  ``symbols`` table, and ``top_k=0`` all return well-formed responses
+  ``graph_nodes`` table, and ``top_k=0`` all return well-formed responses
   rather than raising.
 * The cache invalidates on db mtime change, so a re-scan picks up new
   symbols.
@@ -59,50 +59,58 @@
 def _make_db(workspace: str, symbols):
     """Create a fake ``.codelens/codelens.db`` with the given symbol rows.
 
-    ``symbols`` is a list of dicts with keys: name, kind, file_path,
-    line_start, language, signature, extra_json (dict, will be
-    JSON-encoded).
+    Mirrors what ``scan`` actually produces: rows in the ``graph_nodes``
+    table (issue #188). The engine reads from ``graph_nodes``, not the
+    legacy ``symbols`` table declared by ``persistent_registry`` (which
+    the scan flow never populates).
+
+    ``symbols`` is a list of dicts with keys: name, kind (maps to
+    node_type), file_path (maps to file), line_start (maps to line),
+    language, signature, extra_json (dict, will be JSON-encoded).
     """
     codelens_dir = os.path.join(workspace, ".codelens")
     os.makedirs(codelens_dir, exist_ok=True)
     db_path = os.path.join(codelens_dir, "codelens.db")
     conn = sqlite3.connect(db_path)
     try:
+        # Schema mirrors scripts/graph_model.py:_CREATE_GRAPH_NODES so the
+        # engine's SELECT AS mapping has real columns to read from.
         conn.execute(
             """
-            CREATE TABLE symbols (
+            CREATE TABLE graph_nodes (
                 id INTEGER PRIMARY KEY AUTOINCREMENT,
+                node_id TEXT NOT NULL UNIQUE,
+                node_type TEXT NOT NULL DEFAULT 'function',
                 name TEXT NOT NULL,
-                kind TEXT NOT NULL DEFAULT 'function',
-                file_path TEXT,
-                line_start INTEGER,
-                line_end INTEGER,
-                language TEXT,
-                signature TEXT,
-                hash TEXT,
+                file TEXT,
+                line INTEGER,
                 extra_json TEXT
             )
             """
         )
         for i, s in enumerate(symbols, start=1):
             extra = s.get("extra_json") or {}
+            name = s["name"]
+            file_path = s.get("file_path", "")
+            line = s.get("line_start") or 0
+            # Synthesize a node_id matching the flat-registry convention
+            # (file:line) — what populate_graph_tables would store. This
+            # keeps tests realistic and would let future graph-traversal
+            # integration reuse the same fixtures.
+            node_id = s.get("id") or f"{file_path}:{line}:{name}"
             conn.execute(
                 """
-                INSERT INTO symbols
-                    (id, name, kind, file_path, line_start, line_end,
-                     language, signature, hash, extra_json)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                INSERT INTO graph_nodes
+                    (id, node_id, node_type, name, file, line, extra_json)
+                VALUES (?, ?, ?, ?, ?, ?, ?)
                 """,
                 (
                     i,
-                    s["name"],
+                    node_id,
                     s.get("kind", "function"),
-                    s.get("file_path", ""),
-                    s.get("line_start"),
-                    s.get("line_end"),
-                    s.get("language", ""),
-                    s.get("signature", ""),
-                    s.get("hash", ""),
+                    name,
+                    file_path,
+                    line,
                     json.dumps(extra) if extra else None,
                 ),
             )
@@ -255,7 +263,7 @@ class TestSemanticQuery:
     def test_finds_auth_symbol_by_concept(self, auth_workspace):
         """The core issue #11 use case: a query token that doesn't appear
         in any symbol NAME still surfaces relevant symbols because file
-        paths, signatures, and kinds are all part of the TF-IDF document.
+        paths and kinds are part of the TF-IDF document.
 
         Here, the query ``"auth"`` doesn't appear in any symbol name
         (``verify_jwt_claims``, ``loginUser``, ``format_date``, etc.), but
@@ -380,13 +388,14 @@ def test_cache_invalidates_on_mtime_change(self, workspace):
         idx1 = build_index(db_path)
         assert len(idx1.symbols) == 1
 
-        # Force mtime change by sleeping then writing new symbols
+        # Force mtime change by sleeping then writing a new graph_nodes
+        # row (what a real re-scan would do).
         time.sleep(0.05)
         conn = sqlite3.connect(db_path)
         try:
             conn.execute(
-                "INSERT INTO symbols (name, kind, file_path, line_start, language) "
-                "VALUES ('bar', 'function', 'b.py', 2, 'python')"
+                "INSERT INTO graph_nodes (node_id, node_type, name, file, line) "
+                "VALUES ('b.py:2:bar', 'function', 'bar', 'b.py', 2)"
             )
             conn.commit()
         finally:
@@ -413,8 +422,9 @@ def test_missing_db_returns_empty_results(self, workspace):
         assert result["stats"]["total_symbols"] == 0
         assert result["stats"]["returned"] == 0
 
-    def test_db_without_symbols_table_returns_empty(self, workspace):
-        # Create the db file but no `symbols` table
+    def test_db_without_graph_nodes_table_returns_empty(self, workspace):
+        # Create the db file but no `graph_nodes` table — mirrors a
+        # workspace that has been `codelens init`-ed but never scanned.
         codelens_dir = os.path.join(workspace, ".codelens")
         os.makedirs(codelens_dir, exist_ok=True)
         db_path = os.path.join(codelens_dir, "codelens.db")
@@ -427,7 +437,7 @@ def test_db_without_symbols_table_returns_empty(self, workspace):
         assert result["status"] == "ok"
         assert result["results"] == []
 
-    def test_empty_symbols_table_returns_empty(self, workspace):
+    def test_empty_graph_nodes_table_returns_empty(self, workspace):
         _make_db(workspace, [])
         result = semantic_query(workspace, "anything", top_k=10)
         assert result["status"] == "ok"