From 7d8f4095c9cfc0e10a9f1f3a5445f142c4ac0fcf Mon Sep 17 00:00:00 2001 From: Tavily PR Agent Date: Tue, 16 Jun 2026 14:04:41 +0000 Subject: [PATCH 1/2] feat: add Tavily as configurable search engine option --- pyproject.toml | 1 + src/deepcloak/cli.py | 2 +- src/deepcloak/config.py | 6 +++++- src/deepcloak/research_core.py | 21 ++++++++++++++++++++- src/deepcloak/retriever.py | 32 ++++++++++++++++++++++++++++---- 5 files changed, 55 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e57fe72..6ae9356 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ [project.optional-dependencies] mcp = ["mcp>=1.0"] +tavily = ["tavily-python>=0.5"] dev = ["pytest>=8.0", "ruff>=0.6"] [project.scripts] diff --git a/src/deepcloak/cli.py b/src/deepcloak/cli.py index 719a5e7..3a4c024 100644 --- a/src/deepcloak/cli.py +++ b/src/deepcloak/cli.py @@ -22,7 +22,7 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument("--version", action="version", version=f"deepcloak {__version__}") p.add_argument("query", nargs="?", help="The research question.") p.add_argument("--depth", choices=["quick", "detailed", "report"], default="detailed") - p.add_argument("--engine", choices=["duckduckgo", "searxng", "auto"], default=None) + p.add_argument("--engine", choices=["duckduckgo", "searxng", "tavily", "auto"], default=None) p.add_argument("--searxng-url", dest="searxng_url", default=None, help="SearXNG instance URL (for --engine searxng)") p.add_argument("--stealth", choices=["auto", "always", "off"], default=None) diff --git a/src/deepcloak/config.py b/src/deepcloak/config.py index f1a3600..9014739 100644 --- a/src/deepcloak/config.py +++ b/src/deepcloak/config.py @@ -29,7 +29,7 @@ class ConfigError(Exception): _VALID_STEALTH = {"auto", "always", "off"} _VALID_DEPTH = {"quick", "detailed", "report"} -_VALID_ENGINE = {"duckduckgo", "searxng", "auto"} +_VALID_ENGINE = {"duckduckgo", "searxng", "tavily", "auto"} _DEFAULT_MODEL = { "openai": "gpt-4.1", @@ -63,6 +63,7 @@ class Settings: out: str | None proxy: str | None searxng_url: str | None + tavily_api_key: str | None = None base_url: str | None = None # for provider "openai-endpoint" (local OpenAI-compatible) def to_ldr_env(self) -> dict[str, str]: @@ -146,6 +147,8 @@ def resolve(cli: Mapping, env: Mapping) -> Settings: "LDR_SEARCH_ENGINE_WEB_SEARXNG_DEFAULT_PARAMS_INSTANCE_URL" ) + tavily_api_key = env.get("TAVILY_API_KEY") + base_url = cli.get("base_url") or env.get("LDR_LLM_OPENAI_ENDPOINT_URL") if provider == "openai-endpoint" and not base_url: raise ConfigError( @@ -164,5 +167,6 @@ def resolve(cli: Mapping, env: Mapping) -> Settings: out=cli.get("out"), proxy=cli.get("proxy"), searxng_url=searxng_url, + tavily_api_key=tavily_api_key, base_url=base_url, ) diff --git a/src/deepcloak/research_core.py b/src/deepcloak/research_core.py index fb59abb..982c5ec 100644 --- a/src/deepcloak/research_core.py +++ b/src/deepcloak/research_core.py @@ -54,7 +54,26 @@ def _run_ldr( # full pages we fetched through the stealth path (Bypassing walls), # instead of search snippets. This is what makes a research run actually # read bot-walled sources. - if settings.searxng_url: + if settings.tavily_api_key and settings.search_engine == "tavily": + try: + from functools import partial + + from .retriever import build_stealth_retriever, tavily_search + + fn_kwargs["retrievers"] = { + "stealth": build_stealth_retriever( + search_fn=partial(tavily_search, settings.tavily_api_key), + mode=settings.stealth_mode, + evidence_log=evidence_log, + on_event=on_event, + respect_robots=settings.respect_robots, + proxy=settings.proxy, + ) + } + overrides["search.tool"] = "stealth" + except Exception: + pass + elif settings.searxng_url: try: from .retriever import build_stealth_retriever diff --git a/src/deepcloak/retriever.py b/src/deepcloak/retriever.py index af90639..375cd6d 100644 --- a/src/deepcloak/retriever.py +++ b/src/deepcloak/retriever.py @@ -17,7 +17,7 @@ from .fetch_router import fetch from .stealth_downloader import plain_get, stealth_get -__all__ = ["build_stealth_retriever", "searxng_search"] +__all__ = ["build_stealth_retriever", "searxng_search", "tavily_search"] def searxng_search(base_url: str, query: str, max_results: int = 8) -> list[dict]: @@ -39,6 +39,20 @@ def searxng_search(base_url: str, query: str, max_results: int = 8) -> list[dict return out +def tavily_search(api_key: str, query: str, max_results: int = 8) -> list[dict]: + """Query Tavily and return [{url, title}] hits.""" + from tavily import TavilyClient + + client = TavilyClient(api_key=api_key) + response = client.search(query=query, max_results=max_results) + out: list[dict] = [] + for item in response.get("results", []): + url = item.get("url") + if url: + out.append({"url": url, "title": item.get("title", "")}) + return out + + def _cap_content(text: str, max_chars: int | None) -> str: """Cap a document's text so many Bypassed pages still fit a model's context. @@ -68,7 +82,8 @@ def _extract_text(html: str) -> str: def build_stealth_retriever( *, - searxng_url: str, + searxng_url: str | None = None, + search_fn: Any = None, mode: str = "auto", max_results: int = 8, max_chars: int = 2000, @@ -78,11 +93,20 @@ def build_stealth_retriever( robots_ok: Any = None, proxy: str | None = None, ): - """Construct a LangChain BaseRetriever backed by the stealth fetch path.""" + """Construct a LangChain BaseRetriever backed by the stealth fetch path. + + ``search_fn`` is a callable ``(query, max_results) -> [{url, title}]``. + When omitted, defaults to ``searxng_search`` bound to *searxng_url*. + """ from functools import partial from langchain_core.retrievers import BaseRetriever, Document # type: ignore + if search_fn is None: + if searxng_url is None: + raise ValueError("Either search_fn or searxng_url must be provided") + search_fn = partial(searxng_search, searxng_url) + stealth_fetch = partial(stealth_get, proxy=proxy) class StealthRetriever(BaseRetriever): @@ -90,7 +114,7 @@ class StealthRetriever(BaseRetriever): def _get_relevant_documents(self, query: str, *, run_manager=None): # noqa: D401 docs = [] - for hit in searxng_search(searxng_url, query, max_results): + for hit in search_fn(query, max_results): url = hit["url"] result = fetch( url, From 3781aed5c85c5782b1897bef365534644f2d85f1 Mon Sep 17 00:00:00 2001 From: Tavily PR Agent Date: Tue, 16 Jun 2026 14:09:38 +0000 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20address=20review=20feedback=20(attem?= =?UTF-8?q?pt=202)=20=E2=80=94=20Add=20Tavily=20as=20a=20parallel=20search?= =?UTF-8?q?=20engine=20option?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 13 +++++++++++- src/deepcloak/config.py | 4 +++- src/deepcloak/research_core.py | 5 ++++- src/deepcloak/retriever.py | 9 +++++---- tests/test_config.py | 26 ++++++++++++++++++++++++ tests/test_retriever.py | 37 +++++++++++++++++++++++++++++++++- 6 files changed, 86 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index bf758c6..1219c0a 100644 --- a/README.md +++ b/README.md @@ -108,12 +108,23 @@ Then your agent can call `deep_research` and read bot-walled sources directly. P | Flag | Default | Notes | | --- | --- | --- | | `--depth` | `detailed` | `quick` / `detailed` / `report` | -| `--engine` | `duckduckgo` | `searxng` / `auto` | +| `--engine` | `duckduckgo` | `searxng` / `tavily` / `auto` | | `--stealth` | `auto` | `always` / `off` | | `--provider` / `--model` | auto-detected | `OPENAI` → `ANTHROPIC` → `GEMINI`, or `ollama` | | `--respect-robots` | off | honor robots.txt | | `--proxy` | — | SOCKS5 for the Stealth Fetch | +### Tavily search engine + +To use [Tavily](https://tavily.com) as the search backend: + +```bash +export TAVILY_API_KEY=tvly-... +deepcloak "your query" --engine tavily +``` + +Tavily is an optional dependency — install with `pip install deepcloak[tavily]`. + ## ⚠️ Responsible use DeepCloak Bypasses bot-detection. **You are responsible for having the right to access whatever you fetch.** robots.txt is **ignored by default**; pass `--respect-robots` to honor it ([ADR-0002](docs/adr/0002-ignore-robots-by-default.md)). Don't use it to violate sites' terms or the law. diff --git a/src/deepcloak/config.py b/src/deepcloak/config.py index 9014739..6325323 100644 --- a/src/deepcloak/config.py +++ b/src/deepcloak/config.py @@ -97,7 +97,7 @@ def to_ldr_overrides(self) -> dict: o: dict[str, object] = { "llm.provider": _LDR_PROVIDER[self.provider], "search.snippets_only": False, - "search.tool": self.search_engine, + "search.tool": self.search_engine if self.search_engine in {"duckduckgo", "searxng"} else "duckduckgo", } if self.model: o["llm.model"] = self.model @@ -148,6 +148,8 @@ def resolve(cli: Mapping, env: Mapping) -> Settings: ) tavily_api_key = env.get("TAVILY_API_KEY") + if search_engine == "tavily" and not tavily_api_key: + raise ConfigError("--engine tavily requires TAVILY_API_KEY to be set.") base_url = cli.get("base_url") or env.get("LDR_LLM_OPENAI_ENDPOINT_URL") if provider == "openai-endpoint" and not base_url: diff --git a/src/deepcloak/research_core.py b/src/deepcloak/research_core.py index 982c5ec..19df634 100644 --- a/src/deepcloak/research_core.py +++ b/src/deepcloak/research_core.py @@ -58,11 +58,14 @@ def _run_ldr( try: from functools import partial + from tavily import TavilyClient + from .retriever import build_stealth_retriever, tavily_search + tavily_client = TavilyClient(api_key=settings.tavily_api_key) fn_kwargs["retrievers"] = { "stealth": build_stealth_retriever( - search_fn=partial(tavily_search, settings.tavily_api_key), + search_fn=partial(tavily_search, tavily_client), mode=settings.stealth_mode, evidence_log=evidence_log, on_event=on_event, diff --git a/src/deepcloak/retriever.py b/src/deepcloak/retriever.py index 375cd6d..16e9192 100644 --- a/src/deepcloak/retriever.py +++ b/src/deepcloak/retriever.py @@ -39,11 +39,12 @@ def searxng_search(base_url: str, query: str, max_results: int = 8) -> list[dict return out -def tavily_search(api_key: str, query: str, max_results: int = 8) -> list[dict]: - """Query Tavily and return [{url, title}] hits.""" - from tavily import TavilyClient +def tavily_search(client: Any, query: str, max_results: int = 8) -> list[dict]: + """Query Tavily and return [{url, title}] hits. - client = TavilyClient(api_key=api_key) + ``client`` is a pre-constructed ``TavilyClient`` instance — construct it + once in the caller and pass via ``functools.partial``. + """ response = client.search(query=query, max_results=max_results) out: list[dict] = [] for item in response.get("results", []): diff --git a/tests/test_config.py b/tests/test_config.py index bbaf477..880b6a2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -124,6 +124,32 @@ def test_openai_endpoint_needs_no_key_and_maps_env(): assert env["LDR_LLM_MODEL"] == "qwen" +def test_tavily_engine_reads_api_key_from_env(): + s = resolve( + cli={"engine": "tavily"}, + env={"OPENAI_API_KEY": "x", "TAVILY_API_KEY": "tvly-test"}, + ) + assert s.search_engine == "tavily" + assert s.tavily_api_key == "tvly-test" + + +def test_tavily_engine_without_key_raises(): + with pytest.raises(ConfigError) as exc: + resolve(cli={"engine": "tavily"}, env={"OPENAI_API_KEY": "x"}) + assert "TAVILY_API_KEY" in str(exc.value) + + +def test_tavily_to_ldr_overrides_maps_to_safe_default(): + """When search_engine is 'tavily', to_ldr_overrides() should emit a + LDR-known search.tool (duckduckgo), not the raw 'tavily' string.""" + s = resolve( + cli={"engine": "tavily"}, + env={"OPENAI_API_KEY": "x", "TAVILY_API_KEY": "tvly-test"}, + ) + overrides = s.to_ldr_overrides() + assert overrides["search.tool"] == "duckduckgo" + + def test_openai_endpoint_base_url_from_env(): s = resolve( cli={"provider": "openai-endpoint"}, diff --git a/tests/test_retriever.py b/tests/test_retriever.py index 25a3bba..2b42df5 100644 --- a/tests/test_retriever.py +++ b/tests/test_retriever.py @@ -7,7 +7,9 @@ a 16k-context local model overflows before it can write the report. """ -from deepcloak.retriever import _cap_content +from unittest.mock import MagicMock + +from deepcloak.retriever import _cap_content, tavily_search def test_long_content_is_capped_to_max_chars(): @@ -23,3 +25,36 @@ def test_zero_or_none_disables_the_cap(): text = "y" * 10_000 assert _cap_content(text, 0) == text assert _cap_content(text, None) == text + + +def test_tavily_search_returns_url_title_hits(): + mock_client = MagicMock() + mock_client.search.return_value = { + "results": [ + {"url": "https://a.com", "title": "A"}, + {"url": "https://b.com", "title": "B"}, + ] + } + hits = tavily_search(mock_client, "test query", max_results=5) + mock_client.search.assert_called_once_with(query="test query", max_results=5) + assert hits == [ + {"url": "https://a.com", "title": "A"}, + {"url": "https://b.com", "title": "B"}, + ] + + +def test_tavily_search_filters_items_missing_url(): + mock_client = MagicMock() + mock_client.search.return_value = { + "results": [ + {"url": "https://a.com", "title": "A"}, + {"title": "No URL"}, + {"url": "", "title": "Empty URL"}, + {"url": "https://c.com", "title": "C"}, + ] + } + hits = tavily_search(mock_client, "q") + assert hits == [ + {"url": "https://a.com", "title": "A"}, + {"url": "https://c.com", "title": "C"}, + ]