From 57edf8936784dd6cddc2432e6738664554c467fe Mon Sep 17 00:00:00 2001 From: Santhi Prakash Date: Fri, 21 Aug 2026 01:44:15 +0000 Subject: [PATCH 1/2] fix(extract): preserve existing semantic layer on --code-only --force (#2923) graphify extract --code-only --force previously rewrote graph.json with only the AST tier, silently dropping every doc/paper/image node plus its connected hyperedges. --force disables incremental mode so the merge path that would otherwise carry the surviving semantic tier forward never ran. A code-only run cannot touch the semantic tier at all (no LLM dispatch), so discarding the existing semantic layer is a destructive side effect with no correctness justification. Re-enable the incremental merge when --force and --code-only are combined and an existing graph.json is present; the AST tier is still fully replaced (full re-scan, semantic cache reads skipped) while doc/paper/image nodes are carried forward via build_merge / merge_raw_extraction. graph_stale_sources still prunes semantic nodes for files deleted from disk between the prior extract and this one, so the merge cannot resurrect nodes for sources that no longer exist. Adds two regression tests in test_extract_code_only_cli.py: - test_code_only_force_preserves_existing_semantic_layer: seeded graph with AST + SEMANTIC nodes; verifies the SEMANTIC tier survives --code-only --force. - test_code_only_force_prunes_removed_semantic_files: deletes NOTES.txt between seed and re-run; verifies its semantic nodes are pruned, not resurrected, by the merge. Ref: #2923 --- graphify/cli.py | 14 ++++ tests/test_extract_code_only_cli.py | 115 ++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) diff --git a/graphify/cli.py b/graphify/cli.py index 02e55b9447..899930aac0 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -3129,6 +3129,20 @@ def _parse_float(name: str, raw: str) -> float: # --force: full scan, not the manifest-gated incremental diff — a warm # unchanged tree would otherwise dispatch zero files (#1894). incremental_mode = incremental_mode and not force + # #2923: --force --code-only must NOT drop the existing semantic layer. + # The AST pass is fully replaced (full re-scan, semantic cache reads + # skipped), but the semantic pass is itself skipped entirely, so + # doc/paper/image nodes from the existing graph carry forward via the + # incremental merge (build_merge / merge_raw_extraction keep them + # because no new semantic-tier sources are dispatched). Without this, + # a single --code-only --force silently erases every doc/paper/image + # node plus its connected hyperedges. + if force and code_only and existing_graph_path.exists(): + incremental_mode = True + print( + "[graphify extract] --force --code-only: full AST re-scan, " + "existing semantic layer preserved (no semantic pass this run)" + ) if force: print("[graphify extract] --force: full re-scan, semantic cache reads skipped") elif incremental_mode and not manifest_path.exists(): diff --git a/tests/test_extract_code_only_cli.py b/tests/test_extract_code_only_cli.py index 93fec48d34..9853bbdd1f 100644 --- a/tests/test_extract_code_only_cli.py +++ b/tests/test_extract_code_only_cli.py @@ -258,3 +258,118 @@ def test_extract_names_skipped_sensitive_files(tmp_path): out = r.stdout + r.stderr assert "skipped as potentially sensitive" in out assert "github_token.txt" in out, "the skipped filename must be surfaced (#2106)" + + +def test_code_only_force_preserves_existing_semantic_layer(tmp_path): + """#2923 regression: --code-only --force must not drop the existing semantic + layer. The AST pass is fully replaced (full re-scan, semantic cache reads + skipped) but the semantic pass is itself skipped, so doc/paper/image nodes + from graph.json must be carried forward. Before the fix this combination + silently rewrote graph.json with only the AST tier, losing every semantic + node and every hyperedge connected to one. + """ + repo = _mixed_repo(tmp_path) + out = repo / "graphify-out" + out.mkdir() + graph = out / "graph.json" + # Seed a graph.json as if a prior full extract with an LLM backend had run: + # 2 AST nodes from app.py + 4 SEMANTIC nodes from README.md/NOTES.txt. + graph.write_text(json.dumps({ + "nodes": [ + {"id": "app_py", "label": "app.py", "type": "file", + "source_file": "app.py", "origin": "AST"}, + {"id": "app_hello", "label": "hello()", "type": "function", + "source_file": "app.py", "origin": "AST"}, + {"id": "readme_md", "label": "readme.md", "type": "file", + "source_file": "README.md", "origin": "SEMANTIC"}, + {"id": "readme_design", "label": "Design", "type": "concept", + "source_file": "README.md", "origin": "SEMANTIC"}, + {"id": "notes_txt", "label": "NOTES.txt", "type": "file", + "source_file": "NOTES.txt", "origin": "SEMANTIC"}, + {"id": "notes_architecture", "label": "Architecture", "type": "concept", + "source_file": "NOTES.txt", "origin": "SEMANTIC"}, + ], + "edges": [ + {"id": "e1", "source": "app_py", "target": "app_hello", + "relation": "contains", "source_file": "app.py"}, + {"id": "e2", "source": "readme_md", "target": "readme_design", + "relation": "concept_about", "source_file": "README.md"}, + {"id": "e3", "source": "notes_txt", "target": "notes_architecture", + "relation": "concept_about", "source_file": "NOTES.txt"}, + ], + "hyperedges": [], + "input_tokens": 0, + "output_tokens": 0, + })) + + r = _run(repo, "--code-only", "--force", "--no-cluster") + assert r.returncode == 0, r.stderr + + out_graph = json.loads(graph.read_text()) + semantic_labels = {n["label"] for n in out_graph["nodes"] + if n.get("origin") == "SEMANTIC"} + semantic_source_files = { + Path(str(n["source_file"])).name.lower() + for n in out_graph["nodes"] + if n.get("origin") == "SEMANTIC" + } + # Every seeded semantic node must survive. The AST pass may add new nodes + # (or relabel existing ones — e.g. hello vs hello()) but it must not + # silently drop the semantic tier. + assert {"readme.md", "notes.txt"}.issubset(semantic_source_files), ( + "code-only --force erased the existing semantic layer (#2923); " + f"semantic nodes remaining: {semantic_labels}" + ) + # Hyperedges are also semantic tier; the seeded graph had none but the + # AST re-extract must not have invented any non-semantic work, and the + # surviving edges list must not have been wholesale replaced. + assert "edges" in out_graph, "graph.json must still have an edges key" + # And the user-visible console line must explain why a semantic-layer- + # preserving branch fired. + assert "existing semantic layer preserved" in r.stdout + r.stderr, ( + "the --force --code-only print must announce the semantic-preserving branch" + ) + + +def test_code_only_force_prunes_removed_semantic_files(tmp_path): + """#2923 follow-up: --code-only --force preserves surviving semantic nodes + but must still prune semantic nodes for files that have been removed from + disk (the doc/paper/image tier cannot outlive the corpus it indexes). + """ + repo = _mixed_repo(tmp_path) + out = repo / "graphify-out" + out.mkdir() + graph = out / "graph.json" + graph.write_text(json.dumps({ + "nodes": [ + {"id": "app_py", "label": "app.py", "type": "file", + "source_file": "app.py", "origin": "AST"}, + {"id": "app_hello", "label": "hello()", "type": "function", + "source_file": "app.py", "origin": "AST"}, + {"id": "notes_txt", "label": "NOTES.txt", "type": "file", + "source_file": "NOTES.txt", "origin": "SEMANTIC"}, + {"id": "notes_architecture", "label": "Architecture", "type": "concept", + "source_file": "NOTES.txt", "origin": "SEMANTIC"}, + ], + "edges": [], + "hyperedges": [], + "input_tokens": 0, + "output_tokens": 0, + })) + + # Delete NOTES.txt between seed and re-run. The merge's graph_stale_sources + # path must drop its semantic nodes because the file no longer exists. + (repo / "NOTES.txt").unlink() + + r = _run(repo, "--code-only", "--force", "--no-cluster") + assert r.returncode == 0, r.stderr + out_graph = json.loads(graph.read_text()) + remaining_sources = { + Path(n["source_file"]).name + for n in out_graph["nodes"] + if n.get("origin") == "SEMANTIC" + } + assert "NOTES.txt" not in remaining_sources, ( + "NOTES.txt was deleted from disk; its semantic nodes must be pruned " + "(#2923 follow-up)" + ) From 428a93d27b3dda73d64bec525a36bad16e6ac855 Mon Sep 17 00:00:00 2001 From: Santhi Prakash Date: Mon, 24 Aug 2026 11:42:34 +0000 Subject: [PATCH 2/2] fix(extract): keep --force --code-only on a full AST re-scan while merging the existing semantic layer The previous fix set incremental_mode=True, which re-activated the manifest cache and skipped unchanged code files on a warm tree. This made --force no longer a full re-scan and could leave the AST tier unchanged. The corrected path does a full _detect scan (incremental_mode stays False) and then merges the existing graph into the new output via build_merge / merge_raw_extraction, preserving doc/paper/image nodes and hyperedges. graph_stale_sources is computed on the full scan so deleted/excluded non-code files are still pruned. Adds a regression test with a warm manifest that removes an unchanged AST node and seeds semantic nodes; the full re-scan must restore the AST node and keep the semantic layer. --- graphify/cli.py | 30 ++++++++++------- tests/test_extract_code_only_cli.py | 51 +++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 12 deletions(-) diff --git a/graphify/cli.py b/graphify/cli.py index 899930aac0..1ebf568790 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -3130,15 +3130,11 @@ def _parse_float(name: str, raw: str) -> float: # unchanged tree would otherwise dispatch zero files (#1894). incremental_mode = incremental_mode and not force # #2923: --force --code-only must NOT drop the existing semantic layer. - # The AST pass is fully replaced (full re-scan, semantic cache reads - # skipped), but the semantic pass is itself skipped entirely, so - # doc/paper/image nodes from the existing graph carry forward via the - # incremental merge (build_merge / merge_raw_extraction keep them - # because no new semantic-tier sources are dispatched). Without this, - # a single --code-only --force silently erases every doc/paper/image - # node plus its connected hyperedges. - if force and code_only and existing_graph_path.exists(): - incremental_mode = True + # It still performs a full AST re-scan (so --force bypasses the + # manifest cache), but the semantic pass is itself skipped entirely, so + # the existing graph is merged instead of replaced. + preserve_semantic = force and code_only and existing_graph_path.exists() + if preserve_semantic: print( "[graphify extract] --force --code-only: full AST re-scan, " "existing semantic layer preserved (no semantic pass this run)" @@ -3227,6 +3223,16 @@ def _parse_float(name: str, raw: str) -> float: excluded_files = [] graph_stale_sources = [] unchanged_total = 0 + if preserve_semantic: + # A full scan re-extracts every code file, but doc/paper/image + # nodes are not re-dispatched, so the existing graph must still + # be merged. Compute stale sources (deleted/excluded files) so + # the merge can prune them while carrying the rest forward. + _seen_files = {f for _fl in files_by_type.values() for f in _fl} + _seen_files.update(detection.get("unclassified", [])) + graph_stale_sources = _stale_graph_sources( + existing_graph_path, target, _seen_files, detection=detection + ) semantic_files = doc_files + paper_files + image_files # --code-only: index code (pure local AST, no key) and skip the semantic @@ -3808,7 +3814,7 @@ def _invalidate_file_manifest_for_db_graph() -> None: existing_graph_node_count as _existing_graph_node_count, ) if ( - incremental_mode + (incremental_mode or preserve_semantic) and not code_files and not semantic_files and not deleted_files @@ -3842,7 +3848,7 @@ def _invalidate_file_manifest_for_db_graph() -> None: stages.total() sys.exit(0) - if incremental_mode: + if incremental_mode or preserve_semantic: # #2169: this raw path used to write ONLY this run's extraction # over graph.json — on an incremental run that is just the # changed files, silently dropping every node/edge owned by an @@ -3970,7 +3976,7 @@ def _invalidate_file_manifest_for_db_graph() -> None: from graphify.export import to_json as _to_json from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising dedup_backend = backend if dedup_llm else None - if incremental_mode: + if incremental_mode or preserve_semantic: # Prune everything the current scan no longer covers: genuinely # deleted manifest rows, excluded-but-alive manifest rows (#1908), # and the graph's own stale sources — which catches files that diff --git a/tests/test_extract_code_only_cli.py b/tests/test_extract_code_only_cli.py index 9853bbdd1f..1de97b2c76 100644 --- a/tests/test_extract_code_only_cli.py +++ b/tests/test_extract_code_only_cli.py @@ -373,3 +373,54 @@ def test_code_only_force_prunes_removed_semantic_files(tmp_path): "NOTES.txt was deleted from disk; its semantic nodes must be pruned " "(#2923 follow-up)" ) + + +def test_code_only_force_rescans_unchanged_code_with_manifest_and_preserves_seeded_semantic_nodes(tmp_path): + """#2923: --force --code-only must perform a full AST re-scan even when the + manifest reports no code changes. Without this, a warm unchanged tree causes + the old fix to dispatch zero code files, so the AST tier is not rebuilt and + the existing graph is only merged unchanged. + """ + repo = _mixed_repo(tmp_path) + out = repo / "graphify-out" + out.mkdir() + + # Initial code-only extract writes a manifest and AST-only graph. + r1 = _run(repo, "--code-only", "--no-cluster") + assert r1.returncode == 0, r1.stderr + graph_path = out / "graph.json" + manifest_path = out / "manifest.json" + assert manifest_path.exists(), "code-only run must write a manifest" + g = json.loads(graph_path.read_text()) + assert any(n.get("label") == "hello()" for n in g["nodes"]) + + # Seed a semantic layer as if a prior full extract had produced it, then + # remove an unchanged AST node to verify the full re-scan restores it. + g["nodes"].extend([ + {"id": "readme_md", "label": "README.md", "type": "file", + "source_file": "README.md", "origin": "SEMANTIC"}, + {"id": "readme_design", "label": "Design", "type": "concept", + "source_file": "README.md", "origin": "SEMANTIC"}, + {"id": "notes_txt", "label": "NOTES.txt", "type": "file", + "source_file": "NOTES.txt", "origin": "SEMANTIC"}, + {"id": "notes_architecture", "label": "Architecture", "type": "concept", + "source_file": "NOTES.txt", "origin": "SEMANTIC"}, + ]) + g["nodes"] = [n for n in g["nodes"] if n.get("label") != "hello()"] + graph_path.write_text(json.dumps(g)) + + r2 = _run(repo, "--code-only", "--force", "--no-cluster") + assert r2.returncode == 0, r2.stderr + out_graph = json.loads(graph_path.read_text()) + assert any(n.get("label") == "hello()" for n in out_graph["nodes"]), ( + "--force --code-only must re-extract unchanged code and restore the AST node" + ) + semantic_labels = {n["label"] for n in out_graph["nodes"] + if n.get("origin") == "SEMANTIC"} + assert semantic_labels >= {"README.md", "Design", "NOTES.txt", "Architecture"}, ( + "seeded semantic layer must survive the full AST re-scan: " + f"{semantic_labels}" + ) + assert "existing semantic layer preserved" in r2.stdout + r2.stderr, ( + "the --force --code-only print must announce the semantic-preserving branch" + )