From c6196922d6a30fbfa93666f4228de28d251d45ea Mon Sep 17 00:00:00 2001 From: StressTestor <212606152+StressTestor@users.noreply.github.com> Date: Mon, 20 Jul 2026 11:06:12 -0600 Subject: [PATCH 1/2] fix(rag): skip hidden and junk directories when indexing (#5559) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit index_personal_documents walked the whole tree with no pruning, so pointing RAG at a real-world folder silently swept in .obsidian/ plugin JS, .git/ internals, node_modules/, and __pycache__/ — multiplying indexing time and polluting retrieval with junk chunks. Prune hidden directories and well-known junk directories from the walk, and skip hidden files. The explicitly passed root is exempt, so a user who deliberately indexes a hidden directory still gets its contents. --- src/rag_vector.py | 17 ++++++- tests/test_rag_index_hidden_dirs.py | 78 +++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 tests/test_rag_index_hidden_dirs.py diff --git a/src/rag_vector.py b/src/rag_vector.py index 9a4c67cfa..d4774224a 100644 --- a/src/rag_vector.py +++ b/src/rag_vector.py @@ -34,6 +34,12 @@ '.csv', '.html', '.css', '.js', '.pdf' } +# Tool-internal directories that match DEFAULT_FILE_EXTENSIONS but are never +# the user's content: an Obsidian vault's .obsidian/ is minified plugin JS, +# .git/ is repo internals, node_modules/ and __pycache__/ are build output. +# Hidden entries are pruned by name; these cover the common non-hidden junk. +EXCLUDED_DIR_NAMES: Set[str] = {'node_modules', '__pycache__', 'venv'} + VECTOR_WEIGHT = 0.7 KEYWORD_WEIGHT = 0.3 @@ -497,8 +503,17 @@ def index_personal_documents( failed = 0 try: - for root, _, files in os.walk(directory): + for root, dirs, files in os.walk(directory): + # Prune in place so os.walk never descends into hidden or + # junk directories (#5559). The passed-in root is exempt: a + # user who deliberately targets a hidden directory gets it. + dirs[:] = [ + d for d in dirs + if not d.startswith('.') and d not in EXCLUDED_DIR_NAMES + ] for fname in files: + if fname.startswith('.'): + continue fpath = os.path.join(root, fname) ext = Path(fname).suffix.lower() if ext not in file_extensions: diff --git a/tests/test_rag_index_hidden_dirs.py b/tests/test_rag_index_hidden_dirs.py new file mode 100644 index 000000000..0c21f1c12 --- /dev/null +++ b/tests/test_rag_index_hidden_dirs.py @@ -0,0 +1,78 @@ +"""Regression guard for #5559 — directory indexing must skip hidden directories, +hidden files, and well-known junk directories. + +VectorRAG.index_personal_documents walked the whole tree with no pruning, so +pointing RAG at a real-world folder (an Obsidian vault, a git repo) swept in +`.obsidian/` plugin JavaScript, `.git/` internals, `node_modules/`, etc. The +junk multiplied indexing time and polluted retrieval. + +These tests are hermetic — no chromadb; VectorRAG is created via __new__ (skip +Chroma connect) with add_document stubbed to record which files get indexed. +""" +import os + +os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:") + +import src.rag_vector as rag_vector + + +def _make_rag(recorded_sources): + rag = rag_vector.VectorRAG.__new__(rag_vector.VectorRAG) # skip Chroma connect + + def _record(text, metadata): + recorded_sources.add(metadata["source"]) + return True + + rag.add_document = _record + return rag + + +def _write(path, content="some real content"): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def test_index_skips_hidden_and_junk_directories(tmp_path): + _write(tmp_path / "note.md") + _write(tmp_path / "sub" / "deeper.md") + _write(tmp_path / ".obsidian" / "plugins" / "plugin.js") + _write(tmp_path / ".git" / "hooks.js") + _write(tmp_path / "node_modules" / "lib" / "index.js") + _write(tmp_path / "__pycache__" / "cached.py") + _write(tmp_path / "venv" / "lib" / "site.py") + + recorded = set() + rag = _make_rag(recorded) + result = rag.index_personal_documents(str(tmp_path)) + + assert result["success"] is True + indexed = {os.path.relpath(p, str(tmp_path)) for p in recorded} + assert indexed == {"note.md", os.path.join("sub", "deeper.md")} + + +def test_index_skips_hidden_files(tmp_path): + _write(tmp_path / "visible.md") + _write(tmp_path / ".hidden.md") + _write(tmp_path / "sub" / ".secret.txt") + + recorded = set() + rag = _make_rag(recorded) + rag.index_personal_documents(str(tmp_path)) + + indexed = {os.path.relpath(p, str(tmp_path)) for p in recorded} + assert indexed == {"visible.md"} + + +def test_explicitly_passed_hidden_root_is_still_indexed(tmp_path): + """Pruning applies to children only — a user who deliberately points RAG at + a hidden directory gets its contents, minus nested hidden/junk dirs.""" + root = tmp_path / ".notes" + _write(root / "idea.md") + _write(root / ".obsidian" / "plugin.js") + + recorded = set() + rag = _make_rag(recorded) + rag.index_personal_documents(str(root)) + + indexed = {os.path.relpath(p, str(root)) for p in recorded} + assert indexed == {"idea.md"} From efed44176095851bb80e201904aad37992f4004a Mon Sep 17 00:00:00 2001 From: StressTestor <212606152+StressTestor@users.noreply.github.com> Date: Thu, 23 Jul 2026 02:59:20 -0600 Subject: [PATCH 2/2] fix(rag): prune hidden/junk dirs in the keyword index too, via a shared helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #5559 fix pruned only VectorRAG.index_personal_documents (the vector index). The parallel keyword index built by PersonalDocsManager.refresh_index -> load_personal_index walked the same tree unpruned, so .obsidian/, .git/, node_modules/ etc. still swept into keyword retrieval and the file listing — the 'end-to-end' guarantee was only half true. Single-source the pruning policy in src/index_walk (prune_index_dirs + is_indexable_file) and use it from both walkers so they cannot drift again. The junk-dir match is now case-insensitive, so a Node_Modules on a case-insensitive filesystem is pruned too. Tests: keyword-path regressions covering hidden/junk dirs, hidden files, junk at depth (not just top level), case-insensitive junk, and the explicit-hidden- root exemption. The existing vector tests still pass against the shared helper. --- src/index_walk.py | 35 +++++++++++++ src/personal_docs.py | 17 +++++-- src/rag_vector.py | 21 ++++---- tests/test_personal_index_hidden_dirs.py | 63 ++++++++++++++++++++++++ 4 files changed, 121 insertions(+), 15 deletions(-) create mode 100644 src/index_walk.py create mode 100644 tests/test_personal_index_hidden_dirs.py diff --git a/src/index_walk.py b/src/index_walk.py new file mode 100644 index 000000000..ec249778f --- /dev/null +++ b/src/index_walk.py @@ -0,0 +1,35 @@ +"""Shared directory-walk pruning for personal-document indexing (#5559). + +Single source of the hidden-dir / junk-dir / hidden-file skip so the vector +index (``rag_vector.index_personal_documents``) and the keyword index +(``personal_docs.load_personal_index``) apply the exact same policy and cannot +drift — the drift is what left the keyword path sweeping in `.obsidian/`, +`.git/`, and `node_modules/` after the vector path was fixed. +""" +from typing import List, Set + +# Well-known non-hidden junk directories to skip. Matched case-insensitively so +# a `Node_Modules` on a case-insensitive filesystem (macOS default) is still +# pruned. Hidden directories (dot-prefixed) are pruned separately. Kept +# deliberately small: over-pruning would silently drop a user's real content +# (e.g. a notes directory legitimately named "build"). +EXCLUDED_DIR_NAMES: Set[str] = {'node_modules', '__pycache__', 'venv'} + + +def prune_index_dirs(dirs: List[str]) -> None: + """In-place ``os.walk`` (topdown) directory prune: drop hidden and known + junk directories so the walk never descends into them. + + The explicitly-targeted walk root is never a member of ``dirs`` (it is the + ``dirpath`` argument), so it stays exempt — a user who deliberately points + indexing at a hidden directory gets its contents, minus nested junk. + """ + dirs[:] = [ + d for d in dirs + if not d.startswith('.') and d.lower() not in EXCLUDED_DIR_NAMES + ] + + +def is_indexable_file(name: str) -> bool: + """A file is indexable only if it is not hidden (dot-prefixed).""" + return not name.startswith('.') diff --git a/src/personal_docs.py b/src/personal_docs.py index 0e8335357..0a60bf62e 100644 --- a/src/personal_docs.py +++ b/src/personal_docs.py @@ -6,6 +6,8 @@ from typing import List, Dict, Set, Any, Tuple from dataclasses import dataclass +from src.index_walk import prune_index_dirs, is_indexable_file + from src.markitdown_runtime import MARKITDOWN_EXTS logger = logging.getLogger(__name__) @@ -94,13 +96,22 @@ def tokenize(s: str) -> Set[str]: return set(t for t in tokens if t not in config.STOP_WORDS and len(t) > 1) def load_personal_index( - personal_dir: str, + personal_dir: str, extensions: Tuple[str, ...] = config.DEFAULT_EXTENSIONS ) -> List[Dict[str, Any]]: - """Load and index personal documents.""" + """Load and index personal documents. + + Skips hidden and junk directories and hidden files via the shared + ``index_walk`` policy, so the keyword index matches the vector index and a + real vault/repo does not sweep in ``.obsidian/`` / ``.git/`` / + ``node_modules/`` content (#5559). + """ files = [] - for root, _, names in os.walk(personal_dir): + for root, dirs, names in os.walk(personal_dir): + prune_index_dirs(dirs) for name in sorted(names): + if not is_indexable_file(name): + continue p = os.path.join(root, name) if not os.path.isfile(p): continue diff --git a/src/rag_vector.py b/src/rag_vector.py index d4774224a..0e967bd87 100644 --- a/src/rag_vector.py +++ b/src/rag_vector.py @@ -14,6 +14,7 @@ from typing import List, Dict, Any, Optional, Set from src.constants import CHROMA_DIR +from src.index_walk import prune_index_dirs, is_indexable_file from pathlib import Path from src.embedding_lanes import ( @@ -35,10 +36,8 @@ } # Tool-internal directories that match DEFAULT_FILE_EXTENSIONS but are never -# the user's content: an Obsidian vault's .obsidian/ is minified plugin JS, -# .git/ is repo internals, node_modules/ and __pycache__/ are build output. -# Hidden entries are pruned by name; these cover the common non-hidden junk. -EXCLUDED_DIR_NAMES: Set[str] = {'node_modules', '__pycache__', 'venv'} +# Directory-walk pruning is single-sourced in src.index_walk so the vector and +# keyword indexers apply the same hidden/junk policy and cannot drift (#5559). VECTOR_WEIGHT = 0.7 KEYWORD_WEIGHT = 0.3 @@ -504,15 +503,13 @@ def index_personal_documents( try: for root, dirs, files in os.walk(directory): - # Prune in place so os.walk never descends into hidden or - # junk directories (#5559). The passed-in root is exempt: a - # user who deliberately targets a hidden directory gets it. - dirs[:] = [ - d for d in dirs - if not d.startswith('.') and d not in EXCLUDED_DIR_NAMES - ] + # Prune in place so os.walk never descends into hidden or junk + # directories (#5559), via the shared index_walk policy. The + # passed-in root is exempt: a user who deliberately targets a + # hidden directory gets it. + prune_index_dirs(dirs) for fname in files: - if fname.startswith('.'): + if not is_indexable_file(fname): continue fpath = os.path.join(root, fname) ext = Path(fname).suffix.lower() diff --git a/tests/test_personal_index_hidden_dirs.py b/tests/test_personal_index_hidden_dirs.py new file mode 100644 index 000000000..9fd63f060 --- /dev/null +++ b/tests/test_personal_index_hidden_dirs.py @@ -0,0 +1,63 @@ +"""Regression guard for #5559 — the KEYWORD index (load_personal_index, which +PersonalDocsManager.refresh_index builds from) must skip hidden dirs, hidden +files, and junk dirs at ANY depth, the same as the vector index. Both walkers +share one pruning helper (src/index_walk) so they cannot drift again. +""" +import os + +os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:") + +from src.personal_docs import load_personal_index + + +def _write(path, content="real content"): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _indexed(root): + return {rec["name"] for rec in load_personal_index(str(root))} + + +def test_keyword_index_skips_hidden_and_junk_dirs(tmp_path): + _write(tmp_path / "note.md") + _write(tmp_path / "sub" / "deeper.md") + _write(tmp_path / ".obsidian" / "workspace.json") + _write(tmp_path / ".git" / "hooks.md") + _write(tmp_path / "node_modules" / "lib" / "readme.md") + _write(tmp_path / "__pycache__" / "cached.txt") + _write(tmp_path / "venv" / "lib" / "site.txt") + assert _indexed(tmp_path) == {"note.md", os.path.join("sub", "deeper.md")} + + +def test_keyword_index_skips_hidden_files(tmp_path): + _write(tmp_path / "visible.md") + _write(tmp_path / ".hidden.md") + _write(tmp_path / "sub" / ".secret.txt") + assert _indexed(tmp_path) == {"visible.md"} + + +def test_keyword_index_prunes_junk_at_depth(tmp_path): + """Pruning must apply at every level, not just the first (the vector test's + fixtures only nested one level under the root).""" + _write(tmp_path / "a" / "b" / "keep.md") + _write(tmp_path / "a" / "b" / "node_modules" / "dep.md") + _write(tmp_path / "a" / ".obsidian" / "deep.json") + assert _indexed(tmp_path) == {os.path.join("a", "b", "keep.md")} + + +def test_keyword_index_junk_match_is_case_insensitive(tmp_path): + """A case-variant junk dir must still be pruned (macOS default FS is + case-insensitive, so `Node_Modules` and `node_modules` are the same dir).""" + _write(tmp_path / "keep.md") + _write(tmp_path / "Node_Modules" / "dep.md") + assert _indexed(tmp_path) == {"keep.md"} + + +def test_keyword_index_explicit_hidden_root_still_indexed(tmp_path): + """Children-only pruning: pointing indexing at a hidden dir gets its + contents, minus nested hidden/junk.""" + root = tmp_path / ".notes" + _write(root / "idea.md") + _write(root / ".obsidian" / "plugin.json") + assert {rec["name"] for rec in load_personal_index(str(root))} == {"idea.md"}