diff --git a/src/deepcloak/retriever.py b/src/deepcloak/retriever.py index ebcaad7..af90639 100644 --- a/src/deepcloak/retriever.py +++ b/src/deepcloak/retriever.py @@ -39,6 +39,18 @@ def searxng_search(base_url: str, query: str, max_results: int = 8) -> list[dict return out +def _cap_content(text: str, max_chars: int | None) -> str: + """Cap a document's text so many Bypassed pages still fit a model's context. + + A small local model (e.g. a 16k-context llama-server) overflows when the + retriever feeds full pages into the synthesis prompt, so the report step + never runs. ``max_chars`` of ``None``/``0`` disables the cap. + """ + if max_chars and len(text) > max_chars: + return text[:max_chars] + return text + + def _extract_text(html: str) -> str: """Best-effort readable-text extraction (reuses LDR's extractor if present).""" try: @@ -59,6 +71,7 @@ def build_stealth_retriever( searxng_url: str, mode: str = "auto", max_results: int = 8, + max_chars: int = 2000, evidence_log: Any = None, on_event: Any = None, respect_robots: bool = False, @@ -94,7 +107,9 @@ def _get_relevant_documents(self, query: str, *, run_manager=None): # noqa: D40 if result.content: docs.append( Document( - page_content=_extract_text(result.content), + page_content=_cap_content( + _extract_text(result.content), max_chars + ), metadata={"url": url, "title": hit.get("title", "")}, ) ) diff --git a/tests/test_retriever.py b/tests/test_retriever.py new file mode 100644 index 0000000..25a3bba --- /dev/null +++ b/tests/test_retriever.py @@ -0,0 +1,25 @@ +"""Unit tests for the StealthRetriever per-document content cap. + +The retriever itself needs ``langchain_core`` (only present when LDR is +installed), but the per-document cap is a pure helper we can test in isolation. +It is what keeps a research prompt within a small local model's context window: +without it, the retriever feeds full Bypassed pages into the synthesis prompt and +a 16k-context local model overflows before it can write the report. +""" + +from deepcloak.retriever import _cap_content + + +def test_long_content_is_capped_to_max_chars(): + assert len(_cap_content("x" * 10_000, 6000)) == 6000 + + +def test_short_content_is_left_untouched(): + text = "only a little readable text" + assert _cap_content(text, 6000) == text + + +def test_zero_or_none_disables_the_cap(): + text = "y" * 10_000 + assert _cap_content(text, 0) == text + assert _cap_content(text, None) == text