From 5a9e046f201421d05e3d4082c9fd4773bcf0f8a4 Mon Sep 17 00:00:00 2001 From: Tavily PR Agent Date: Mon, 13 Jul 2026 16:08:27 +0000 Subject: [PATCH] feat: add Tavily as optional Phase 1 search backend --- setup.sh | 5 +- skills/literature-review-agent/SKILL.md | 23 ++ .../references/tavily-search-cookbook.md | 260 ++++++++++++++++++ .../scripts/tavily_search.py | 166 +++++++++++ 4 files changed, 453 insertions(+), 1 deletion(-) create mode 100644 skills/literature-review-agent/references/tavily-search-cookbook.md create mode 100644 skills/literature-review-agent/scripts/tavily_search.py diff --git a/setup.sh b/setup.sh index 1ff699f..73bab91 100755 --- a/setup.sh +++ b/setup.sh @@ -94,6 +94,7 @@ set_env_var() { set_env_var "SEMANTIC_SCHOLAR_API_KEY" "Semantic Scholar API key (SEMANTIC_SCHOLAR_API_KEY)" set_env_var "EXA_API_KEY" "Exa API key (EXA_API_KEY)" +set_env_var "TAVILY_API_KEY" "Tavily API key (TAVILY_API_KEY)" set_env_var "PAPERBANANA_PATH" "Path to PaperBanana executable (PAPERBANANA_PATH)" # ── Step 1c: ~/.paperorchestra/config ───────────────────────── @@ -109,6 +110,7 @@ read_env_val() { cat > "$GLOBAL_CONFIG" < **Tavily is opt-in.** The literature-review-agent's default Phase 1 +> path is "use your host agent's native web search tool" (`WebSearch` in +> Claude Code, `@web` in Cursor, the search tool in Antigravity, etc.). +> That requires zero configuration and no API key. Use Tavily only if +> you want to. + +## Why use it + +Tavily fills three gaps: + +1. **Hosts with no built-in search.** Aider, OpenCode, and generic CLI + agents often lack a native web search tool. Tavily gives them one. +2. **LLM-optimized relevance.** Tavily's `search_depth: "advanced"` + mode returns higher relevance results for complex queries. The + `--academic` flag restricts results to academic domains (arxiv.org, + scholar.google.com, semanticscholar.org, aclanthology.org, + openreview.net) for research-focused discovery. +3. **Batch / non-interactive runs.** When you want a deterministic, + scriptable backend rather than going through the host agent's tool + interface. + +Tavily returns up to 20 results per call (the helper clamps to that +range), and each result includes a `title`, `url`, `content` (snippet), +and a relevance `score` which the helper preserves as `_tavily_score` +for debugging. + +## Get a key + +1. Sign up at . +2. Copy your API key (format: `tvly-xxxxxxxxxxxxxxxxxxxxxxxx`). +3. Set it in your environment: + + ```bash + export TAVILY_API_KEY="tvly-your-key-here" + ``` + + Or put it in a `.env` file (which is gitignored — the repo `.gitignore` + blocks `*.env` and `.env*` patterns) and source it: + + ```bash + set -a; source .env; set +a + ``` + +**This repo never commits a key.** The helper reads `TAVILY_API_KEY` from +the environment at runtime. The key is your responsibility to provision +and secure. + +## Run the helper + +```bash +python skills/literature-review-agent/scripts/tavily_search.py \ + --query "Sparse attention long context transformers" \ + --num-results 15 \ + --academic \ + --discovered-for "related_work[2.1]" +``` + +Output (default — normalized to the literature-review-agent candidate +format): + +```json +{ + "candidates": [ + { + "title": "Longformer: The Long-Document Transformer", + "snippet": "We present the Longformer, a self-attention mechanism that scales linearly with sequence length...", + "source_url": "https://arxiv.org/abs/2004.05150", + "discovered_for": ["related_work[2.1]"], + "_tavily_score": 0.92 + }, + ... + ] +} +``` + +This JSON can be merged directly into `workspace/raw_candidates.json` +before the Phase 2 sequential verification step. + +### Useful flags + +| Flag | Default | Purpose | +|---|---|---| +| `--query` | (required) | Search query string | +| `--num-results` | `10` | 1–20; the helper clamps to this range | +| `--topic` | `"general"` | `"general"` or `"news"`; use `"news"` for recent results | +| `--academic` | off | Restrict to academic domains (arxiv.org, scholar.google.com, etc.) | +| `--discovered-for` | `"intro"` | Tag attached to each candidate; use `"related_work[2.1]"` for cluster queries | +| `--raw` | off | Print the full Tavily response JSON instead of normalized candidates | + +## Direct curl recipe + +If you'd rather not use the Python helper (for one-off testing, or to +invoke from a host agent's `Bash` / `WebFetch` tool directly): + +```bash +curl -X POST https://api.tavily.com/search \ + --header "Content-Type: application/json" \ + --header "Authorization: Bearer $TAVILY_API_KEY" \ + --data '{ + "query": "PaperOrchestra automated paper writing", + "max_results": 10, + "search_depth": "advanced", + "topic": "general", + "include_domains": ["arxiv.org", "scholar.google.com"] + }' +``` + +The `$TAVILY_API_KEY` reference assumes the key is in your shell env. +**Do not** paste the literal key into the curl command in shell history +or chat — use the env var. + +## Response shape + +```json +{ + "query": "PaperOrchestra automated paper writing", + "results": [ + { + "title": "PaperOrchestra: A Multi-Agent Framework for ...", + "url": "https://arxiv.org/abs/2604.05018", + "content": "We present PaperOrchestra, a multi-agent framework...", + "score": 0.95 + } + ] +} +``` + +## Mapping Tavily → literature-review-agent candidate format + +Phase 2 verification (Semantic Scholar fuzzy match → cutoff check → dedup) +expects candidates in this shape: + +```json +{ + "title": "...", + "snippet": "...", + "source_url": "...", + "discovered_for": ["intro"] +} +``` + +`tavily_search.py` (the default mode) does this mapping: + +| Tavily field | Candidate field | +|---|---| +| `result.title` | `title` | +| `result.url` | `source_url` | +| `result.content` capped at 1500 chars | `snippet` | +| `--discovered-for` flag | `discovered_for` | +| `result.score` | `_tavily_score` (preserved for debugging) | + +Phase 2 verification still goes through Semantic Scholar regardless of +whether the candidate came from Tavily, Exa, or from the host's native +search. Tavily is ONLY a discovery backend; the verification chain +(`levenshtein_match.py` → `check_cutoff.py` → `dedupe_by_id.py` → +`bibtex_format.py` → `citation_coverage.py`) is unchanged. + +## Query patterns + +Match the literature-review-agent's outline-driven query design. Run one +Tavily call per query, then merge all candidate lists: + +| Query type | Source in `outline.json` | Example query | `--discovered-for` | +|---|---|---|---| +| Macro context | `introduction_strategy.search_directions[i]` | `"Survey of long-context attention mechanisms 2020-2024"` | `"intro"` | +| Foundational | same | `"Foundational papers transformer self-attention scaling laws"` | `"intro"` | +| SOTA scan | `related_work_strategy.subsections[i].sota_investigation_mission` | `"Recent SOTA sparse attention transformers 2024"` | `"related_work[2.1]"` | +| Limitation hunt | `related_work_strategy.subsections[i].limitation_search_queries[j]` | `"Block-sparse attention failure modes long sequences"` | `"related_work[2.1]"` | + +For the related-work cluster queries, the `--discovered-for` tag matters +— the downstream `citation_coverage.py` gate uses it to attribute each +citation to the right cluster when reporting which papers were not yet +integrated. + +**Tip:** Use `--academic` for the SOTA scan and limitation hunt queries +to keep results focused on research papers. For broad intro queries, +omitting `--academic` may yield useful surveys and blog posts that +reference foundational work. + +## Cost and rate limits + +Tavily offers 1,000 free API credits per month (no credit card +required). The `search_depth: "advanced"` mode used by the helper costs +2 credits per query. For a typical paper with ~15-20 search queries +(3-5 intro queries + 10-15 related-work queries), one full Lit Review +Agent run costs ~30-40 credits — well within the free tier. + +For higher volumes, see for paid plans. +Tavily's rate limits are generous; the paper's 10-worker parallel +discovery pattern is well within them. The pipeline's wall-time floor is +still set by Semantic Scholar's 1 QPS verification limit, not by Tavily. + +## SDK alternative + +If `tavily-python` is installed (`pip install tavily-python`), you can +use the SDK directly instead of the bundled helper: + +```python +from tavily import TavilyClient + +client = TavilyClient() # reads TAVILY_API_KEY from env +response = client.search( + query="Sparse attention long context transformers", + max_results=15, + search_depth="advanced", + include_domains=["arxiv.org", "scholar.google.com"], +) +``` + +The bundled `tavily_search.py` helper uses stdlib `urllib` only (like +`exa_search.py`) to avoid mandatory dependencies. The SDK is optional. + +## Security + +- **NEVER commit `TAVILY_API_KEY` to git.** The repo's `.gitignore` + blocks `.env`, `*.env`, and `secrets.json` patterns. Keep your key in + your shell environment or your secrets manager (1Password CLI, op, + doppler, etc.). +- The helper reads the key from the environment only. It does NOT accept + the key as a command-line argument (which would expose it in shell + history). +- Tavily logs requests for billing and quality. Assume your queries are + not private to Tavily themselves. Don't include sensitive draft text + in queries. + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---|---|---| +| `ERROR: TAVILY_API_KEY environment variable not set` | env var missing | `export TAVILY_API_KEY="tvly-..."` | +| `ERROR: Tavily HTTP 401` | invalid or expired key | check your key at https://app.tavily.com | +| `ERROR: Tavily HTTP 429` | rate-limited | back off, lower concurrency | +| `WARN: Tavily returned 0 results` | query too narrow | broaden the query or remove `--academic` | +| `Tavily network error` | no internet, DNS issue | check your connection; the helper uses urllib stdlib only, no proxy support | + +## When to prefer Tavily vs Exa vs the host's native search + +| Use case | Recommended backend | +|---|---| +| Claude Code, Cursor, Antigravity (have native web search) | host's native search (free, integrated) | +| Aider, OpenCode, generic CLI agents | Tavily or Exa (gives them search) | +| Batch reproducible runs | Tavily or Exa (deterministic backend) | +| Research-paper-heavy queries | Exa (`category: "research paper"`) or Tavily (`--academic`) | +| Free tier / budget-conscious | Tavily (1,000 free credits/month) | +| LLM-optimized relevance scoring | Tavily (`search_depth: "advanced"`) | +| One-off interactive runs | host's native search (less friction) | + +You can also mix: use the host's web search for the broad intro queries, +Exa for the narrow limitation-search queries where the research-paper +category filter helps the most, and Tavily for SOTA scan queries where +LLM-optimized relevance scoring shines. diff --git a/skills/literature-review-agent/scripts/tavily_search.py b/skills/literature-review-agent/scripts/tavily_search.py new file mode 100644 index 0000000..505c6c3 --- /dev/null +++ b/skills/literature-review-agent/scripts/tavily_search.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +""" +tavily_search.py — Optional Tavily (https://tavily.com) backend for the +literature review agent's Phase 1 (parallel candidate discovery) step. + +Tavily is a search API designed for LLMs, enabling AI applications to +access real-time web data. It is OPTIONAL — the literature-review-agent +works fine with any host coding agent's native web search tool. Use +Tavily only if: + + - Your host has no built-in web search (e.g., Aider, OpenCode, generic + CLI agents). + - You want an LLM-optimized search backend with high relevance scoring. + - You're running the pipeline in batch / non-interactive mode and want + a deterministic, scriptable backend. + +This helper reads TAVILY_API_KEY from the environment. The key is YOUR +responsibility to provide; this repo never commits one. Get a key at +https://app.tavily.com (1,000 free credits/month). + +Usage: + export TAVILY_API_KEY="tvly-your-key-here" + python tavily_search.py --query "Sparse attention long context" --num-results 15 + python tavily_search.py --query "..." --raw # full JSON + python tavily_search.py --query "..." --discovered-for "related_work[2.1]" + +Default output: JSON candidates in the literature-review-agent format, ready +to be merged into raw_candidates.json before Phase 2 verification. + +Exit codes: + 0 query succeeded + 1 TAVILY_API_KEY missing, HTTP error, network error, or empty results +""" +import argparse +import json +import os +import sys +import urllib.error +import urllib.request + +TAVILY_ENDPOINT = "https://api.tavily.com/search" +DEFAULT_NUM = 10 +MAX_NUM = 20 +SNIPPET_CAP = 1500 + +# Academic domains to boost research-paper results +ACADEMIC_DOMAINS = [ + "arxiv.org", + "scholar.google.com", + "semanticscholar.org", + "aclanthology.org", + "openreview.net", +] + + +def search(query: str, num_results: int, topic: str, + include_domains: list[str] | None) -> dict: + api_key = os.environ.get("TAVILY_API_KEY") + if not api_key: + print( + "ERROR: TAVILY_API_KEY environment variable not set.\n" + "Get a key at https://app.tavily.com and run:\n" + ' export TAVILY_API_KEY="tvly-your-key-here"\n' + "Then retry. The literature-review-agent also works without\n" + "Tavily — see references/discovery-pipeline.md for the default\n" + "host-native web search path.", + file=sys.stderr, + ) + sys.exit(1) + + body: dict = { + "query": query, + "max_results": num_results, + "search_depth": "advanced", + "topic": topic, + } + if include_domains: + body["include_domains"] = include_domains + + req = urllib.request.Request( + TAVILY_ENDPOINT, + data=json.dumps(body).encode("utf-8"), + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {api_key}", + }, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode("utf-8")) + except urllib.error.HTTPError as e: + body_text = e.read().decode("utf-8", errors="replace")[:500] + print(f"ERROR: Tavily HTTP {e.code}: {body_text}", file=sys.stderr) + sys.exit(1) + except urllib.error.URLError as e: + print(f"ERROR: Tavily network error: {e.reason}", file=sys.stderr) + sys.exit(1) + + +def normalize(tavily_response: dict, discovered_for: list[str]) -> list[dict]: + """Convert Tavily results into the literature-review-agent candidate format.""" + candidates: list[dict] = [] + for r in tavily_response.get("results", []): + title = (r.get("title") or "").strip() + url = r.get("url") or "" + content = (r.get("content") or "").strip() + snippet = content[:SNIPPET_CAP] + candidates.append({ + "title": title, + "snippet": snippet, + "source_url": url, + "discovered_for": list(discovered_for), + "_tavily_score": r.get("score"), + }) + return candidates + + +def main() -> int: + p = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument("--query", required=True, help="Search query") + p.add_argument("--num-results", type=int, default=DEFAULT_NUM, + help=f"Number of results to fetch " + f"(default {DEFAULT_NUM}, clamped to [1, {MAX_NUM}])") + p.add_argument("--topic", default="general", + choices=["general", "news"], + help='Tavily topic filter (default "general"; ' + 'use "news" for recent results)') + p.add_argument("--academic", action="store_true", + help="Restrict results to academic domains " + "(arxiv.org, scholar.google.com, etc.)") + p.add_argument("--discovered-for", default="intro", + help='Tag to attach to each candidate ' + '(default "intro"). Use "related_work[2.1]" or ' + 'similar for cluster-specific queries so the ' + 'downstream citation_coverage gate can attribute ' + 'the citation to the right section.') + p.add_argument("--raw", action="store_true", + help="Print the full Tavily response JSON unmodified " + "instead of normalized candidates") + args = p.parse_args() + + n = max(1, min(MAX_NUM, args.num_results)) + include_domains = ACADEMIC_DOMAINS if args.academic else None + + response = search(args.query, n, args.topic, include_domains) + if not response.get("results"): + print(f"WARN: Tavily returned 0 results for query: {args.query!r}", + file=sys.stderr) + return 1 + + if args.raw: + json.dump(response, sys.stdout, indent=2, ensure_ascii=False) + else: + candidates = normalize(response, [args.discovered_for]) + json.dump({"candidates": candidates}, sys.stdout, indent=2, + ensure_ascii=False) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main())