From c81d999f08f0374e1250c58b56a75df91b0eac75 Mon Sep 17 00:00:00 2001 From: Mrbaeksang Date: Fri, 5 Jun 2026 09:18:42 +0900 Subject: [PATCH] fix(retriever): cap per-document content so small local models can finish a report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The StealthRetriever fed full Bypassed pages into the synthesis prompt, so a 16k-context local model overflowed (~56k–68k tokens) and the report step never ran — only the 🛡️ badge was written. This broke the README's "local-first, no API key" promise on a typical small local model. Cap each document to `max_chars` (default 2000) before it reaches the research loop. Verified live against a local 16k llama-server (Qwen) + SearXNG: the same run that previously errored now Bypasses 20 Cloudflare/Turnstile walls AND writes a substantive cited report (prompt dropped 68k -> well under 16k). Pure `_cap_content` helper is unit-tested (langchain-free, CI-safe). Co-Authored-By: Claude Opus 4.8 --- src/deepcloak/retriever.py | 17 ++++++++++++++++- tests/test_retriever.py | 25 +++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 tests/test_retriever.py diff --git a/src/deepcloak/retriever.py b/src/deepcloak/retriever.py index ebcaad7..af90639 100644 --- a/src/deepcloak/retriever.py +++ b/src/deepcloak/retriever.py @@ -39,6 +39,18 @@ def searxng_search(base_url: str, query: str, max_results: int = 8) -> list[dict return out +def _cap_content(text: str, max_chars: int | None) -> str: + """Cap a document's text so many Bypassed pages still fit a model's context. + + A small local model (e.g. a 16k-context llama-server) overflows when the + retriever feeds full pages into the synthesis prompt, so the report step + never runs. ``max_chars`` of ``None``/``0`` disables the cap. + """ + if max_chars and len(text) > max_chars: + return text[:max_chars] + return text + + def _extract_text(html: str) -> str: """Best-effort readable-text extraction (reuses LDR's extractor if present).""" try: @@ -59,6 +71,7 @@ def build_stealth_retriever( searxng_url: str, mode: str = "auto", max_results: int = 8, + max_chars: int = 2000, evidence_log: Any = None, on_event: Any = None, respect_robots: bool = False, @@ -94,7 +107,9 @@ def _get_relevant_documents(self, query: str, *, run_manager=None): # noqa: D40 if result.content: docs.append( Document( - page_content=_extract_text(result.content), + page_content=_cap_content( + _extract_text(result.content), max_chars + ), metadata={"url": url, "title": hit.get("title", "")}, ) ) diff --git a/tests/test_retriever.py b/tests/test_retriever.py new file mode 100644 index 0000000..25a3bba --- /dev/null +++ b/tests/test_retriever.py @@ -0,0 +1,25 @@ +"""Unit tests for the StealthRetriever per-document content cap. + +The retriever itself needs ``langchain_core`` (only present when LDR is +installed), but the per-document cap is a pure helper we can test in isolation. +It is what keeps a research prompt within a small local model's context window: +without it, the retriever feeds full Bypassed pages into the synthesis prompt and +a 16k-context local model overflows before it can write the report. +""" + +from deepcloak.retriever import _cap_content + + +def test_long_content_is_capped_to_max_chars(): + assert len(_cap_content("x" * 10_000, 6000)) == 6000 + + +def test_short_content_is_left_untouched(): + text = "only a little readable text" + assert _cap_content(text, 6000) == text + + +def test_zero_or_none_disables_the_cap(): + text = "y" * 10_000 + assert _cap_content(text, 0) == text + assert _cap_content(text, None) == text