fix(retriever): cap per-document content so small local models can finish a report

src/deepcloak/retriever.py

@@ -39,6 +39,18 @@ def searxng_search(base_url: str, query: str, max_results: int = 8) -> list[dict
     return out
 
 
+def _cap_content(text: str, max_chars: int | None) -> str:
+    """Cap a document's text so many Bypassed pages still fit a model's context.
+
+    A small local model (e.g. a 16k-context llama-server) overflows when the
+    retriever feeds full pages into the synthesis prompt, so the report step
+    never runs. ``max_chars`` of ``None``/``0`` disables the cap.
+    """
+    if max_chars and len(text) > max_chars:
+        return text[:max_chars]
+    return text
+
+
 def _extract_text(html: str) -> str:
     """Best-effort readable-text extraction (reuses LDR's extractor if present)."""
     try:
@@ -59,6 +71,7 @@ def build_stealth_retriever(
     searxng_url: str,
     mode: str = "auto",
     max_results: int = 8,
+    max_chars: int = 2000,
     evidence_log: Any = None,
     on_event: Any = None,
     respect_robots: bool = False,
@@ -94,7 +107,9 @@ def _get_relevant_documents(self, query: str, *, run_manager=None):  # noqa: D40
                 if result.content:
                     docs.append(
                         Document(
-                            page_content=_extract_text(result.content),
+                            page_content=_cap_content(
+                                _extract_text(result.content), max_chars
+                            ),
                             metadata={"url": url, "title": hit.get("title", "")},
                         )
                     )

tests/test_retriever.py

@@ -0,0 +1,25 @@
+"""Unit tests for the StealthRetriever per-document content cap.
+
+The retriever itself needs ``langchain_core`` (only present when LDR is
+installed), but the per-document cap is a pure helper we can test in isolation.
+It is what keeps a research prompt within a small local model's context window:
+without it, the retriever feeds full Bypassed pages into the synthesis prompt and
+a 16k-context local model overflows before it can write the report.
+"""
+
+from deepcloak.retriever import _cap_content
+
+
+def test_long_content_is_capped_to_max_chars():
+    assert len(_cap_content("x" * 10_000, 6000)) == 6000
+
+
+def test_short_content_is_left_untouched():
+    text = "only a little readable text"
+    assert _cap_content(text, 6000) == text
+
+
+def test_zero_or_none_disables_the_cap():
+    text = "y" * 10_000
+    assert _cap_content(text, 0) == text
+    assert _cap_content(text, None) == text