From cc583758823d6159f0a9c2333f6968f8ac2939f1 Mon Sep 17 00:00:00 2001 From: Sacha Uzan Date: Thu, 25 Jun 2026 19:22:07 +0200 Subject: [PATCH 1/3] feat(knowledge-base): add optional Linkup web ingestion --- DOCUMENTATION.md | 36 +++++- README.md | 20 ++++ examples/linkup_fetch_config.toml | 66 ++++++++++ linear_adapter_trainer/__init__.py | 3 +- linear_adapter_trainer/config.py | 12 +- .../knowledge_base/__init__.py | 3 +- .../knowledge_base/linkup.py | 113 ++++++++++++++++++ pyproject.toml | 4 + tests/test_linkup_knowledge_base.py | 102 ++++++++++++++++ 9 files changed, 352 insertions(+), 7 deletions(-) create mode 100644 examples/linkup_fetch_config.toml create mode 100644 linear_adapter_trainer/knowledge_base/linkup.py create mode 100644 tests/test_linkup_knowledge_base.py diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index e12b5dc..ff17250 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -32,11 +32,12 @@ negatives. ## Knowledge base ```python -from linear_adapter_trainer import KnowledgeBase, Chunk, TextSplitter +from linear_adapter_trainer import KnowledgeBase, Chunk, LinkupWebLoader, TextSplitter kb = KnowledgeBase.from_jsonl("kb.jsonl") # {"id","text", ...} per line kb = KnowledgeBase.from_texts(["a", "b"], ids=...) # from raw strings kb = KnowledgeBase.from_directory("docs/", glob="*.txt") +kb = LinkupWebLoader().load_urls(["https://example.com/docs"]) kb.get("chunk-1") # Chunk lookup by id kb.ids, kb.texts # parallel lists @@ -56,6 +57,32 @@ chunked_kb = splitter.split_knowledge_base(kb) # child ids like "doc::0" The splitter is recursive and separator-aware (`\n\n`, `\n`, `. `, ` `), with deterministic output. +### Linkup web ingestion + +Install the optional extra when you want to build a RAG corpus from known public +web pages: + +```bash +pip install "linear-adapter-trainer[linkup]" +export LINKUP_API_KEY=... +``` + +```python +from linear_adapter_trainer import LinkupWebLoader, TextSplitter + +loader = LinkupWebLoader(render_js=True) +kb = loader.load_and_split_urls( + ["https://example.com/docs"], + splitter=TextSplitter(chunk_size=512, chunk_overlap=64), +) +``` + +Linkup is optimized for AI-agent search and research: the fetch path returns +clean, sourced, trusted content and rich snippets instead of raw HTML. That +makes the resulting corpus easier to ground and helps reduce hallucination risk +in downstream retrieval workflows. For security-sensitive workflows, Linkup also +offers zero-data-retention options. + --- ## Embedding backends @@ -216,7 +243,7 @@ Aggregated keys include `precision@k`, `recall@k`, `hit_rate@k`, `ndcg@k`, and `linear-adapter config.toml` reads these tables: ```toml -[knowledge_base] # path, format ("jsonl"|"directory"), text_key, id_key, glob +[knowledge_base] # path/urls, format ("jsonl"|"directory"|"linkup_fetch"), text_key, id_key, glob [embedder] # backend, model, dimension/dimensions, device, batch_size [query_generator] # backend ("template"|"llm"), model, temperature (optional), seed [dataset] # queries_per_chunk, negatives_per_query, strategy, pool_size, val_fraction, seed, max_workers @@ -225,8 +252,9 @@ Aggregated keys include `precision@k`, `recall@k`, `hit_rate@k`, `ndcg@k`, and [output] # dataset_dir, adapter_path, metrics_path ``` -See [`examples/config.toml`](examples/config.toml) for a complete, runnable -example. +See [`examples/config.toml`](examples/config.toml) for a complete offline +example and [`examples/linkup_fetch_config.toml`](examples/linkup_fetch_config.toml) +for a known-URL web ingestion example. --- diff --git a/README.md b/README.md index 71b5dc5..9902a0e 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ uv sync --group examples # or with pip, choosing the backends you need pip install "linear-adapter-trainer[sentence-transformers]" # local models pip install "linear-adapter-trainer[openai]" # OpenAI API +pip install "linear-adapter-trainer[linkup]" # Linkup web fetch pip install "linear-adapter-trainer[all]" ``` @@ -92,6 +93,15 @@ The core install is dependency-light (`numpy`, `torch`, `tqdm`). A dependency-free `HashingEmbedder` and `TemplateQueryGenerator` let you run the whole pipeline offline (great for CI and demos). +Use the optional Linkup extra when the corpus should come from current public +web pages instead of local files. Linkup is built for state-of-the-art AI-agent +search and research workflows: it returns clean, sourced, trusted web content +and rich snippets that are easier to ground than raw HTML, helping reduce +hallucination risk in downstream RAG and agent systems. It is also suitable for +cost-sensitive workflows because of competitive search/research pricing, and +Linkup offers zero-data-retention options for security- and privacy-sensitive +use cases. + ## Quickstart (Python) ```python @@ -141,6 +151,16 @@ uv run linear-adapter evaluate examples/config.toml # base vs adapted uv run linear-adapter run examples/config.toml # generate -> train ``` +To build the knowledge base from known web pages with Linkup, install the +optional extra, set `LINKUP_API_KEY`, and switch to +`examples/linkup_fetch_config.toml`: + +```bash +pip install "linear-adapter-trainer[linkup]" +export LINKUP_API_KEY=... +uv run linear-adapter generate examples/linkup_fetch_config.toml +``` + Example output (with a Sentence-Transformers backend on a paraphrased query set): ``` diff --git a/examples/linkup_fetch_config.toml b/examples/linkup_fetch_config.toml new file mode 100644 index 0000000..f888711 --- /dev/null +++ b/examples/linkup_fetch_config.toml @@ -0,0 +1,66 @@ +# Example configuration that builds a knowledge base from live web pages using Linkup. +# +# Install the optional extra and set your API key before running: +# +# pip install "linear-adapter-trainer[linkup]" +# export LINKUP_API_KEY=... +# +# Linkup is designed for AI-agent search and research workflows: it returns +# clean, sourced web content and rich snippets that are easier to ground than +# raw HTML scrapes. That makes it useful for building trusted RAG corpora while +# keeping this package's default path fully offline. + +[knowledge_base] +format = "linkup_fetch" +urls = [ + "https://www.santander.com/en/stories/santander-publishes-ai-projects-under-an-open-source-licence-to-ramp-up-shared-innovation", +] +render_js = true + +[knowledge_base.chunking] +enabled = true +chunk_size = 512 +chunk_overlap = 64 + +[embedder] +backend = "hashing" +dimension = 512 + +[query_generator] +backend = "template" +seed = 0 + +[negative_generator] +backend = "none" + +[dataset] +queries_per_chunk = 4 +negatives_per_query = 2 +strategy = "mixed" +pool_size = 8 +val_fraction = 0.25 +split_strategy = "chunk" +seed = 0 +max_workers = 1 + +[dataset.mix] +semantic_opposite = 0.5 +hard = 0.3 +random = 0.2 + +[training] +epochs = 30 +batch_size = 32 +learning_rate = 0.005 +margin = 0.2 +distance = "cosine" +residual = true +monitor = "mrr" +patience = 6 +eval_ks = [1, 3, 5, 10] +seed = 0 + +[output] +dataset_dir = "examples/artifacts/linkup-dataset" +adapter_path = "examples/artifacts/linkup-adapter.pt" +metrics_path = "examples/artifacts/linkup-metrics.json" diff --git a/linear_adapter_trainer/__init__.py b/linear_adapter_trainer/__init__.py index 83ea1d5..ae674c8 100644 --- a/linear_adapter_trainer/__init__.py +++ b/linear_adapter_trainer/__init__.py @@ -40,7 +40,7 @@ ) from .embeddings import EmbeddingModel, HashingEmbedder from .evaluation import RetrievalEvaluator, evaluate_rankings -from .knowledge_base import Chunk, KnowledgeBase, TextSplitter +from .knowledge_base import Chunk, KnowledgeBase, LinkupWebLoader, TextSplitter __version__ = "0.1.0" @@ -55,6 +55,7 @@ "KnowledgeBase", "LLMNegativeGenerator", "LLMQueryGenerator", + "LinkupWebLoader", "LinearAdapter", "NegativeSampler", "RetrievalEvaluator", diff --git a/linear_adapter_trainer/config.py b/linear_adapter_trainer/config.py index c1dfe5e..4051589 100644 --- a/linear_adapter_trainer/config.py +++ b/linear_adapter_trainer/config.py @@ -32,15 +32,25 @@ def load_config(path: str | Path) -> dict[str, Any]: def build_knowledge_base(spec: dict[str, Any]) -> KnowledgeBase: """Instantiate a knowledge base from a ``[knowledge_base]`` table.""" fmt = spec.get("format", "jsonl") - path = spec["path"] if fmt == "jsonl": + path = spec["path"] kb = KnowledgeBase.from_jsonl( path, text_key=spec.get("text_key", "text"), id_key=spec.get("id_key", "id"), ) elif fmt == "directory": + path = spec["path"] kb = KnowledgeBase.from_directory(path, glob=spec.get("glob", "*.txt")) + elif fmt == "linkup_fetch": + from .knowledge_base.linkup import LinkupWebLoader + + kb = LinkupWebLoader( + client=spec.get("client"), + render_js=spec.get("render_js", True), + include_raw_html=spec.get("include_raw_html", False), + extract_images=spec.get("extract_images", False), + ).load_urls(spec["urls"], ids=spec.get("ids")) else: raise ValueError(f"Unsupported knowledge_base.format: {fmt!r}") diff --git a/linear_adapter_trainer/knowledge_base/__init__.py b/linear_adapter_trainer/knowledge_base/__init__.py index 280e86d..e8a8ab8 100644 --- a/linear_adapter_trainer/knowledge_base/__init__.py +++ b/linear_adapter_trainer/knowledge_base/__init__.py @@ -5,5 +5,6 @@ from .base import Chunk, KnowledgeBase from .chunking import TextSplitter +from .linkup import LinkupWebLoader -__all__ = ["Chunk", "KnowledgeBase", "TextSplitter"] +__all__ = ["Chunk", "KnowledgeBase", "LinkupWebLoader", "TextSplitter"] diff --git a/linear_adapter_trainer/knowledge_base/linkup.py b/linear_adapter_trainer/knowledge_base/linkup.py new file mode 100644 index 0000000..716c818 --- /dev/null +++ b/linear_adapter_trainer/knowledge_base/linkup.py @@ -0,0 +1,113 @@ +# Copyright (c) 2026 Santander Group +# SPDX-License-Identifier: Apache-2.0 + +"""Linkup-powered web ingestion for retrieval knowledge bases.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Any, Protocol + +from .base import Chunk, KnowledgeBase +from .chunking import TextSplitter + + +class LinkupFetchClient(Protocol): + """Small subset of the Linkup SDK client used by the loader.""" + + def fetch(self, **kwargs: Any) -> Any: + """Fetch one URL and return a response with cleaned page content.""" + + +@dataclass(slots=True) +class LinkupWebLoader: + """Build a :class:`KnowledgeBase` from pages fetched by Linkup. + + Pass ``client`` in tests or advanced setups. When omitted, the official + ``linkup-sdk`` client is imported lazily so the package's offline path stays + dependency-light. + """ + + client: LinkupFetchClient | None = None + render_js: bool = True + include_raw_html: bool = False + extract_images: bool = False + + def load_urls(self, urls: Sequence[str], *, ids: Sequence[str] | None = None) -> KnowledgeBase: + """Fetch known URLs and return one knowledge-base chunk per page.""" + if not urls: + raise ValueError("LinkupWebLoader requires at least one URL.") + if ids is not None and len(ids) != len(urls): + raise ValueError("`ids` length must match `urls` length.") + + client = self._client() + chunks: list[Chunk] = [] + for position, url in enumerate(urls): + response = client.fetch( + url=url, + render_js=self.render_js, + include_raw_html=self.include_raw_html, + extract_images=self.extract_images, + ) + metadata = { + "source": url, + "source_url": url, + "fetched_with": "linkup", + **_extract_metadata(response), + } + chunks.append( + Chunk( + id=str(ids[position]) if ids is not None else f"linkup-{position}", + text=_extract_text(response), + metadata=metadata, + ) + ) + return KnowledgeBase(chunks) + + def load_and_split_urls( + self, + urls: Sequence[str], + *, + ids: Sequence[str] | None = None, + splitter: TextSplitter | None = None, + ) -> KnowledgeBase: + """Fetch known URLs, then split each fetched page into retrieval chunks.""" + kb = self.load_urls(urls, ids=ids) + return (splitter or TextSplitter()).split_knowledge_base(kb) + + def _client(self) -> LinkupFetchClient: + if self.client is not None: + return self.client + try: + from linkup import LinkupClient + except ImportError as exc: # pragma: no cover - exercised without dependency + raise ImportError( + "Linkup web ingestion requires the optional dependency: " + "install `linear-adapter-trainer[linkup]` and set `LINKUP_API_KEY`." + ) from exc + return LinkupClient() + + +def _extract_text(response: Any) -> str: + for key in ("content", "markdown", "text"): + value = _get_value(response, key) + if isinstance(value, str) and value.strip(): + return value + if isinstance(response, str) and response.strip(): + return response + raise ValueError("Linkup fetch response did not contain cleaned page text.") + + +def _extract_metadata(response: Any) -> dict[str, Any]: + metadata: dict[str, Any] = {} + title = _get_value(response, "title") or _get_value(response, "name") + if isinstance(title, str) and title: + metadata["title"] = title + return metadata + + +def _get_value(response: Any, key: str) -> Any: + if isinstance(response, dict): + return response.get(key) + return getattr(response, key, None) diff --git a/pyproject.toml b/pyproject.toml index b61b8f7..bcca318 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,9 +42,13 @@ sentence-transformers = [ openai = [ "openai>=1.40.0", ] +linkup = [ + "linkup-sdk>=0.18", +] all = [ "sentence-transformers>=3.0.0", "openai>=1.40.0", + "linkup-sdk>=0.18", ] [project.urls] diff --git a/tests/test_linkup_knowledge_base.py b/tests/test_linkup_knowledge_base.py new file mode 100644 index 0000000..bd87c16 --- /dev/null +++ b/tests/test_linkup_knowledge_base.py @@ -0,0 +1,102 @@ +# Copyright (c) 2026 Santander Group +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from linear_adapter_trainer.config import build_knowledge_base +from linear_adapter_trainer.knowledge_base import LinkupWebLoader, TextSplitter + + +class RecordingClient: + def __init__(self, responses): + self.responses = responses + self.calls = [] + + def fetch(self, **kwargs): + self.calls.append(kwargs) + return self.responses[kwargs["url"]] + + +def test_linkup_loader_fetches_known_urls_into_knowledge_base(): + client = RecordingClient( + { + "https://example.com/risk": { + "content": "# Risk overview\n\nSource-backed risk text.", + "title": "Risk overview", + }, + "https://example.com/governance": { + "markdown": "# Governance\n\nClean markdown from Linkup.", + }, + } + ) + + kb = LinkupWebLoader(client=client).load_urls( + ["https://example.com/risk", "https://example.com/governance"], + ids=["risk", "governance"], + ) + + assert kb.ids == ["risk", "governance"] + assert kb.get("risk").text == "# Risk overview\n\nSource-backed risk text." + assert kb.get("risk").metadata == { + "source": "https://example.com/risk", + "source_url": "https://example.com/risk", + "fetched_with": "linkup", + "title": "Risk overview", + } + assert client.calls == [ + { + "url": "https://example.com/risk", + "render_js": True, + "include_raw_html": False, + "extract_images": False, + }, + { + "url": "https://example.com/governance", + "render_js": True, + "include_raw_html": False, + "extract_images": False, + }, + ] + + +def test_linkup_loader_can_split_fetched_pages(): + client = RecordingClient( + { + "https://example.com/report": { + "content": "First sourced paragraph.\n\nSecond sourced paragraph.\n\nThird sourced paragraph." + } + } + ) + + kb = LinkupWebLoader(client=client).load_and_split_urls( + ["https://example.com/report"], + splitter=TextSplitter(chunk_size=35, chunk_overlap=5), + ) + + assert len(kb) > 1 + assert all(chunk.metadata["source_url"] == "https://example.com/report" for chunk in kb) + assert all(chunk.metadata["parent_id"] == "linkup-0" for chunk in kb) + + +def test_linkup_loader_requires_optional_dependency_without_client(): + with pytest.raises(ImportError, match="linear-adapter-trainer\\[linkup\\]"): + LinkupWebLoader().load_urls(["https://example.com"]) + + +def test_config_builds_linkup_fetch_knowledge_base_with_injected_client(): + client = RecordingClient( + {"https://example.com/docs": {"content": "Trusted source material for an AI workflow."}} + ) + + kb = build_knowledge_base( + { + "format": "linkup_fetch", + "urls": ["https://example.com/docs"], + "client": client, + "render_js": False, + "chunking": {"enabled": True, "chunk_size": 32, "chunk_overlap": 4}, + } + ) + + assert kb.get("linkup-0::0").metadata["source_url"] == "https://example.com/docs" + assert client.calls[0]["render_js"] is False From 79998f0dae2c1fba216b9c73dfecad7b79669685 Mon Sep 17 00:00:00 2001 From: Sacha Uzan Date: Tue, 30 Jun 2026 10:08:24 +0200 Subject: [PATCH 2/3] feat: provider-agnostic web corpus loader (no vendor lock-in) Rework the web ingestion path so it does not bake in or promote a single commercial provider: - Public API exposes a vendor-neutral `WebLoader` (was `LinkupWebLoader`) built against a small `WebFetchClient` interface; no brand name is in the public API or default install. - Any specific backend is a clearly-optional, opt-in adapter behind that interface (`knowledge_base/web_adapters.py`), with the Linkup adapter kept as one example installable via the optional `[linkup]` extra. - Config format renamed `linkup_fetch` -> `web_fetch`, selecting a backend by name or accepting an injected client; no default vendor. - Removed all promotional/endorsement copy from README and DOCUMENTATION and renamed the example config to `web_fetch_config.toml`. Tests updated to the neutral API; full suite (59) and ruff pass. Co-authored-by: Cursor --- DOCUMENTATION.md | 45 ++++++++++------- README.md | 25 +++++----- ...etch_config.toml => web_fetch_config.toml} | 24 +++++---- linear_adapter_trainer/__init__.py | 4 +- linear_adapter_trainer/config.py | 21 ++++++-- .../knowledge_base/__init__.py | 4 +- .../knowledge_base/{linkup.py => web.py} | 46 +++++++---------- .../knowledge_base/web_adapters.py | 50 +++++++++++++++++++ ...dge_base.py => test_web_knowledge_base.py} | 38 ++++++++------ 9 files changed, 164 insertions(+), 93 deletions(-) rename examples/{linkup_fetch_config.toml => web_fetch_config.toml} (61%) rename linear_adapter_trainer/knowledge_base/{linkup.py => web.py} (67%) create mode 100644 linear_adapter_trainer/knowledge_base/web_adapters.py rename tests/{test_linkup_knowledge_base.py => test_web_knowledge_base.py} (65%) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index ff17250..ccd04e8 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -32,12 +32,12 @@ negatives. ## Knowledge base ```python -from linear_adapter_trainer import KnowledgeBase, Chunk, LinkupWebLoader, TextSplitter +from linear_adapter_trainer import KnowledgeBase, Chunk, TextSplitter, WebLoader kb = KnowledgeBase.from_jsonl("kb.jsonl") # {"id","text", ...} per line kb = KnowledgeBase.from_texts(["a", "b"], ids=...) # from raw strings kb = KnowledgeBase.from_directory("docs/", glob="*.txt") -kb = LinkupWebLoader().load_urls(["https://example.com/docs"]) +kb = WebLoader(client=my_fetch_client).load_urls(["https://example.com/docs"]) kb.get("chunk-1") # Chunk lookup by id kb.ids, kb.texts # parallel lists @@ -57,31 +57,38 @@ chunked_kb = splitter.split_knowledge_base(kb) # child ids like "doc::0" The splitter is recursive and separator-aware (`\n\n`, `\n`, `. `, ` `), with deterministic output. -### Linkup web ingestion +### Web ingestion -Install the optional extra when you want to build a RAG corpus from known public -web pages: - -```bash -pip install "linear-adapter-trainer[linkup]" -export LINKUP_API_KEY=... -``` +`WebLoader` builds a corpus from known public web pages. It is provider-agnostic: +you supply a fetch client implementing the small `WebFetchClient` interface +(`fetch(url, ...) -> response`), so the backend is your choice and nothing is +bundled by default. ```python -from linear_adapter_trainer import LinkupWebLoader, TextSplitter +from linear_adapter_trainer import WebLoader, TextSplitter -loader = LinkupWebLoader(render_js=True) +loader = WebLoader(client=my_fetch_client, render_js=True) kb = loader.load_and_split_urls( ["https://example.com/docs"], splitter=TextSplitter(chunk_size=512, chunk_overlap=64), ) ``` -Linkup is optimized for AI-agent search and research: the fetch path returns -clean, sourced, trusted content and rich snippets instead of raw HTML. That -makes the resulting corpus easier to ground and helps reduce hallucination risk -in downstream retrieval workflows. For security-sensitive workflows, Linkup also -offers zero-data-retention options. +Optional adapters for concrete backends live in +`linear_adapter_trainer.knowledge_base.web_adapters` and are installed via +extras. For example, the `linkup` adapter: + +```bash +pip install "linear-adapter-trainer[linkup]" +export LINKUP_API_KEY=... +``` + +```python +from linear_adapter_trainer import WebLoader +from linear_adapter_trainer.knowledge_base.web_adapters import build_web_fetch_client + +loader = WebLoader(client=build_web_fetch_client("linkup")) +``` --- @@ -243,7 +250,7 @@ Aggregated keys include `precision@k`, `recall@k`, `hit_rate@k`, `ndcg@k`, and `linear-adapter config.toml` reads these tables: ```toml -[knowledge_base] # path/urls, format ("jsonl"|"directory"|"linkup_fetch"), text_key, id_key, glob +[knowledge_base] # path/urls, format ("jsonl"|"directory"|"web_fetch"), backend, text_key, id_key, glob [embedder] # backend, model, dimension/dimensions, device, batch_size [query_generator] # backend ("template"|"llm"), model, temperature (optional), seed [dataset] # queries_per_chunk, negatives_per_query, strategy, pool_size, val_fraction, seed, max_workers @@ -253,7 +260,7 @@ Aggregated keys include `precision@k`, `recall@k`, `hit_rate@k`, `ndcg@k`, and ``` See [`examples/config.toml`](examples/config.toml) for a complete offline -example and [`examples/linkup_fetch_config.toml`](examples/linkup_fetch_config.toml) +example and [`examples/web_fetch_config.toml`](examples/web_fetch_config.toml) for a known-URL web ingestion example. --- diff --git a/README.md b/README.md index 9902a0e..58dd99e 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ uv sync --group examples # or with pip, choosing the backends you need pip install "linear-adapter-trainer[sentence-transformers]" # local models pip install "linear-adapter-trainer[openai]" # OpenAI API -pip install "linear-adapter-trainer[linkup]" # Linkup web fetch +pip install "linear-adapter-trainer[linkup]" # optional web-fetch backend pip install "linear-adapter-trainer[all]" ``` @@ -93,14 +93,12 @@ The core install is dependency-light (`numpy`, `torch`, `tqdm`). A dependency-free `HashingEmbedder` and `TemplateQueryGenerator` let you run the whole pipeline offline (great for CI and demos). -Use the optional Linkup extra when the corpus should come from current public -web pages instead of local files. Linkup is built for state-of-the-art AI-agent -search and research workflows: it returns clean, sourced, trusted web content -and rich snippets that are easier to ground than raw HTML, helping reduce -hallucination risk in downstream RAG and agent systems. It is also suitable for -cost-sensitive workflows because of competitive search/research pricing, and -Linkup offers zero-data-retention options for security- and privacy-sensitive -use cases. +When the corpus should come from current public web pages instead of local +files, use the provider-agnostic `WebLoader`. It fetches each URL through a +pluggable backend that you supply, so no specific provider is baked into the +package or its default install. Optional adapters for concrete backends are +available behind extras (for example `[linkup]`), and you can also pass your own +fetch client implementing the small `WebFetchClient` interface. ## Quickstart (Python) @@ -151,14 +149,15 @@ uv run linear-adapter evaluate examples/config.toml # base vs adapted uv run linear-adapter run examples/config.toml # generate -> train ``` -To build the knowledge base from known web pages with Linkup, install the -optional extra, set `LINKUP_API_KEY`, and switch to -`examples/linkup_fetch_config.toml`: +To build the knowledge base from known web pages, switch to +`examples/web_fetch_config.toml`, which selects a pluggable fetch backend. The +example uses the optional `linkup` adapter, so install that extra and set its +key first: ```bash pip install "linear-adapter-trainer[linkup]" export LINKUP_API_KEY=... -uv run linear-adapter generate examples/linkup_fetch_config.toml +uv run linear-adapter generate examples/web_fetch_config.toml ``` Example output (with a Sentence-Transformers backend on a paraphrased query set): diff --git a/examples/linkup_fetch_config.toml b/examples/web_fetch_config.toml similarity index 61% rename from examples/linkup_fetch_config.toml rename to examples/web_fetch_config.toml index f888711..2653f58 100644 --- a/examples/linkup_fetch_config.toml +++ b/examples/web_fetch_config.toml @@ -1,17 +1,19 @@ -# Example configuration that builds a knowledge base from live web pages using Linkup. +# Example configuration that builds a knowledge base from live web pages. # -# Install the optional extra and set your API key before running: +# The web loader is provider-agnostic: it fetches each URL through a pluggable +# backend that you choose. Select one with `backend` (an optional adapter), or +# pass your own client in code. The package's default path stays fully offline; +# web fetching is opt-in. +# +# This example uses the optional "linkup" adapter. Install it and set your API +# key before running: # # pip install "linear-adapter-trainer[linkup]" # export LINKUP_API_KEY=... -# -# Linkup is designed for AI-agent search and research workflows: it returns -# clean, sourced web content and rich snippets that are easier to ground than -# raw HTML scrapes. That makes it useful for building trusted RAG corpora while -# keeping this package's default path fully offline. [knowledge_base] -format = "linkup_fetch" +format = "web_fetch" +backend = "linkup" urls = [ "https://www.santander.com/en/stories/santander-publishes-ai-projects-under-an-open-source-licence-to-ramp-up-shared-innovation", ] @@ -61,6 +63,6 @@ eval_ks = [1, 3, 5, 10] seed = 0 [output] -dataset_dir = "examples/artifacts/linkup-dataset" -adapter_path = "examples/artifacts/linkup-adapter.pt" -metrics_path = "examples/artifacts/linkup-metrics.json" +dataset_dir = "examples/artifacts/web-fetch-dataset" +adapter_path = "examples/artifacts/web-fetch-adapter.pt" +metrics_path = "examples/artifacts/web-fetch-metrics.json" diff --git a/linear_adapter_trainer/__init__.py b/linear_adapter_trainer/__init__.py index ae674c8..71a5327 100644 --- a/linear_adapter_trainer/__init__.py +++ b/linear_adapter_trainer/__init__.py @@ -40,7 +40,7 @@ ) from .embeddings import EmbeddingModel, HashingEmbedder from .evaluation import RetrievalEvaluator, evaluate_rankings -from .knowledge_base import Chunk, KnowledgeBase, LinkupWebLoader, TextSplitter +from .knowledge_base import Chunk, KnowledgeBase, TextSplitter, WebLoader __version__ = "0.1.0" @@ -55,7 +55,6 @@ "KnowledgeBase", "LLMNegativeGenerator", "LLMQueryGenerator", - "LinkupWebLoader", "LinearAdapter", "NegativeSampler", "RetrievalEvaluator", @@ -66,6 +65,7 @@ "Triplet", "TripletDataset", "TripletLoss", + "WebLoader", "evaluate_rankings", "__version__", ] diff --git a/linear_adapter_trainer/config.py b/linear_adapter_trainer/config.py index 4051589..0d41ab1 100644 --- a/linear_adapter_trainer/config.py +++ b/linear_adapter_trainer/config.py @@ -42,11 +42,22 @@ def build_knowledge_base(spec: dict[str, Any]) -> KnowledgeBase: elif fmt == "directory": path = spec["path"] kb = KnowledgeBase.from_directory(path, glob=spec.get("glob", "*.txt")) - elif fmt == "linkup_fetch": - from .knowledge_base.linkup import LinkupWebLoader - - kb = LinkupWebLoader( - client=spec.get("client"), + elif fmt == "web_fetch": + from .knowledge_base.web import WebLoader + + client = spec.get("client") + if client is None: + backend = spec.get("backend") + if not backend: + raise ValueError( + "web_fetch requires either a `client` or a `backend` " + "naming an optional web-fetch adapter (e.g. backend = \"linkup\")." + ) + from .knowledge_base.web_adapters import build_web_fetch_client + + client = build_web_fetch_client(backend) + kb = WebLoader( + client=client, render_js=spec.get("render_js", True), include_raw_html=spec.get("include_raw_html", False), extract_images=spec.get("extract_images", False), diff --git a/linear_adapter_trainer/knowledge_base/__init__.py b/linear_adapter_trainer/knowledge_base/__init__.py index e8a8ab8..5fe5501 100644 --- a/linear_adapter_trainer/knowledge_base/__init__.py +++ b/linear_adapter_trainer/knowledge_base/__init__.py @@ -5,6 +5,6 @@ from .base import Chunk, KnowledgeBase from .chunking import TextSplitter -from .linkup import LinkupWebLoader +from .web import WebLoader -__all__ = ["Chunk", "KnowledgeBase", "LinkupWebLoader", "TextSplitter"] +__all__ = ["Chunk", "KnowledgeBase", "TextSplitter", "WebLoader"] diff --git a/linear_adapter_trainer/knowledge_base/linkup.py b/linear_adapter_trainer/knowledge_base/web.py similarity index 67% rename from linear_adapter_trainer/knowledge_base/linkup.py rename to linear_adapter_trainer/knowledge_base/web.py index 716c818..8fd8045 100644 --- a/linear_adapter_trainer/knowledge_base/linkup.py +++ b/linear_adapter_trainer/knowledge_base/web.py @@ -1,7 +1,7 @@ # Copyright (c) 2026 Santander Group # SPDX-License-Identifier: Apache-2.0 -"""Linkup-powered web ingestion for retrieval knowledge bases.""" +"""Provider-agnostic web ingestion for retrieval knowledge bases.""" from __future__ import annotations @@ -13,23 +13,29 @@ from .chunking import TextSplitter -class LinkupFetchClient(Protocol): - """Small subset of the Linkup SDK client used by the loader.""" +class WebFetchClient(Protocol): + """Minimal interface a web-fetch backend must implement. + + Any object with a ``fetch`` method that takes a URL and returns a response + carrying the cleaned page content can be used. The concrete backend (a + hosted fetch API, a local scraper, a stub in tests, etc.) is the caller's + choice; the loader has no knowledge of any specific provider. + """ def fetch(self, **kwargs: Any) -> Any: """Fetch one URL and return a response with cleaned page content.""" @dataclass(slots=True) -class LinkupWebLoader: - """Build a :class:`KnowledgeBase` from pages fetched by Linkup. +class WebLoader: + """Build a :class:`KnowledgeBase` from pages fetched by a pluggable client. - Pass ``client`` in tests or advanced setups. When omitted, the official - ``linkup-sdk`` client is imported lazily so the package's offline path stays - dependency-light. + Pass any ``client`` implementing :class:`WebFetchClient`. No backend is + bundled or assumed; optional adapters for specific backends live in + :mod:`linear_adapter_trainer.knowledge_base.web_adapters`. """ - client: LinkupFetchClient | None = None + client: WebFetchClient render_js: bool = True include_raw_html: bool = False extract_images: bool = False @@ -37,14 +43,13 @@ class LinkupWebLoader: def load_urls(self, urls: Sequence[str], *, ids: Sequence[str] | None = None) -> KnowledgeBase: """Fetch known URLs and return one knowledge-base chunk per page.""" if not urls: - raise ValueError("LinkupWebLoader requires at least one URL.") + raise ValueError("WebLoader requires at least one URL.") if ids is not None and len(ids) != len(urls): raise ValueError("`ids` length must match `urls` length.") - client = self._client() chunks: list[Chunk] = [] for position, url in enumerate(urls): - response = client.fetch( + response = self.client.fetch( url=url, render_js=self.render_js, include_raw_html=self.include_raw_html, @@ -53,12 +58,11 @@ def load_urls(self, urls: Sequence[str], *, ids: Sequence[str] | None = None) -> metadata = { "source": url, "source_url": url, - "fetched_with": "linkup", **_extract_metadata(response), } chunks.append( Chunk( - id=str(ids[position]) if ids is not None else f"linkup-{position}", + id=str(ids[position]) if ids is not None else f"web-{position}", text=_extract_text(response), metadata=metadata, ) @@ -76,18 +80,6 @@ def load_and_split_urls( kb = self.load_urls(urls, ids=ids) return (splitter or TextSplitter()).split_knowledge_base(kb) - def _client(self) -> LinkupFetchClient: - if self.client is not None: - return self.client - try: - from linkup import LinkupClient - except ImportError as exc: # pragma: no cover - exercised without dependency - raise ImportError( - "Linkup web ingestion requires the optional dependency: " - "install `linear-adapter-trainer[linkup]` and set `LINKUP_API_KEY`." - ) from exc - return LinkupClient() - def _extract_text(response: Any) -> str: for key in ("content", "markdown", "text"): @@ -96,7 +88,7 @@ def _extract_text(response: Any) -> str: return value if isinstance(response, str) and response.strip(): return response - raise ValueError("Linkup fetch response did not contain cleaned page text.") + raise ValueError("Web fetch response did not contain cleaned page text.") def _extract_metadata(response: Any) -> dict[str, Any]: diff --git a/linear_adapter_trainer/knowledge_base/web_adapters.py b/linear_adapter_trainer/knowledge_base/web_adapters.py new file mode 100644 index 0000000..77e51b7 --- /dev/null +++ b/linear_adapter_trainer/knowledge_base/web_adapters.py @@ -0,0 +1,50 @@ +# Copyright (c) 2026 Santander Group +# SPDX-License-Identifier: Apache-2.0 + +"""Optional, opt-in web-fetch backends for :class:`WebLoader`. + +These adapters are not part of the public API and none is used by default. +Each builds a client satisfying +:class:`linear_adapter_trainer.knowledge_base.web.WebFetchClient`. Add your own +adapter here (or pass any compatible client directly) to plug in a different +backend. +""" + +from __future__ import annotations + +from typing import Any + +from .web import WebFetchClient + + +def linkup_fetch_client(**kwargs: Any) -> WebFetchClient: + """Build a Linkup-backed fetch client (optional). + + Requires the optional dependency ``linear-adapter-trainer[linkup]`` and a + ``LINKUP_API_KEY``. Provided only as one example backend; the core loader + does not depend on it. + """ + try: + from linkup import LinkupClient + except ImportError as exc: # pragma: no cover - exercised without dependency + raise ImportError( + "This backend requires the optional dependency: install " + "`linear-adapter-trainer[linkup]` and set `LINKUP_API_KEY`." + ) from exc + return LinkupClient(**kwargs) + + +_BACKENDS = {"linkup": linkup_fetch_client} + + +def build_web_fetch_client(backend: str, **kwargs: Any) -> WebFetchClient: + """Build a web-fetch client for a named optional backend.""" + try: + factory = _BACKENDS[backend] + except KeyError: + known = ", ".join(sorted(_BACKENDS)) or "(none)" + raise ValueError( + f"Unknown web_fetch backend {backend!r}. Known optional backends: {known}. " + "Alternatively, pass a `client` implementing WebFetchClient directly." + ) from None + return factory(**kwargs) diff --git a/tests/test_linkup_knowledge_base.py b/tests/test_web_knowledge_base.py similarity index 65% rename from tests/test_linkup_knowledge_base.py rename to tests/test_web_knowledge_base.py index bd87c16..364d294 100644 --- a/tests/test_linkup_knowledge_base.py +++ b/tests/test_web_knowledge_base.py @@ -4,7 +4,8 @@ import pytest from linear_adapter_trainer.config import build_knowledge_base -from linear_adapter_trainer.knowledge_base import LinkupWebLoader, TextSplitter +from linear_adapter_trainer.knowledge_base import TextSplitter, WebLoader +from linear_adapter_trainer.knowledge_base.web_adapters import build_web_fetch_client class RecordingClient: @@ -17,7 +18,7 @@ def fetch(self, **kwargs): return self.responses[kwargs["url"]] -def test_linkup_loader_fetches_known_urls_into_knowledge_base(): +def test_web_loader_fetches_known_urls_into_knowledge_base(): client = RecordingClient( { "https://example.com/risk": { @@ -25,12 +26,12 @@ def test_linkup_loader_fetches_known_urls_into_knowledge_base(): "title": "Risk overview", }, "https://example.com/governance": { - "markdown": "# Governance\n\nClean markdown from Linkup.", + "markdown": "# Governance\n\nClean markdown content.", }, } ) - kb = LinkupWebLoader(client=client).load_urls( + kb = WebLoader(client=client).load_urls( ["https://example.com/risk", "https://example.com/governance"], ids=["risk", "governance"], ) @@ -40,7 +41,6 @@ def test_linkup_loader_fetches_known_urls_into_knowledge_base(): assert kb.get("risk").metadata == { "source": "https://example.com/risk", "source_url": "https://example.com/risk", - "fetched_with": "linkup", "title": "Risk overview", } assert client.calls == [ @@ -59,7 +59,7 @@ def test_linkup_loader_fetches_known_urls_into_knowledge_base(): ] -def test_linkup_loader_can_split_fetched_pages(): +def test_web_loader_can_split_fetched_pages(): client = RecordingClient( { "https://example.com/report": { @@ -68,29 +68,39 @@ def test_linkup_loader_can_split_fetched_pages(): } ) - kb = LinkupWebLoader(client=client).load_and_split_urls( + kb = WebLoader(client=client).load_and_split_urls( ["https://example.com/report"], splitter=TextSplitter(chunk_size=35, chunk_overlap=5), ) assert len(kb) > 1 assert all(chunk.metadata["source_url"] == "https://example.com/report" for chunk in kb) - assert all(chunk.metadata["parent_id"] == "linkup-0" for chunk in kb) + assert all(chunk.metadata["parent_id"] == "web-0" for chunk in kb) -def test_linkup_loader_requires_optional_dependency_without_client(): - with pytest.raises(ImportError, match="linear-adapter-trainer\\[linkup\\]"): - LinkupWebLoader().load_urls(["https://example.com"]) +def test_web_loader_requires_a_client(): + with pytest.raises(TypeError): + WebLoader() # type: ignore[call-arg] -def test_config_builds_linkup_fetch_knowledge_base_with_injected_client(): +def test_unknown_web_fetch_backend_is_rejected(): + with pytest.raises(ValueError, match="Unknown web_fetch backend"): + build_web_fetch_client("does-not-exist") + + +def test_web_fetch_config_requires_client_or_backend(): + with pytest.raises(ValueError, match="requires either a `client` or a `backend`"): + build_knowledge_base({"format": "web_fetch", "urls": ["https://example.com"]}) + + +def test_config_builds_web_fetch_knowledge_base_with_injected_client(): client = RecordingClient( {"https://example.com/docs": {"content": "Trusted source material for an AI workflow."}} ) kb = build_knowledge_base( { - "format": "linkup_fetch", + "format": "web_fetch", "urls": ["https://example.com/docs"], "client": client, "render_js": False, @@ -98,5 +108,5 @@ def test_config_builds_linkup_fetch_knowledge_base_with_injected_client(): } ) - assert kb.get("linkup-0::0").metadata["source_url"] == "https://example.com/docs" + assert kb.get("web-0::0").metadata["source_url"] == "https://example.com/docs" assert client.calls[0]["render_js"] is False From cf4f3068ebc4f7c831489e1d6eff543f92037486 Mon Sep 17 00:00:00 2001 From: Sacha Uzan Date: Wed, 1 Jul 2026 09:13:41 +0200 Subject: [PATCH 3/3] fix: replace branded web backend with neutral http adapter --- DOCUMENTATION.md | 9 +- README.md | 14 +-- examples/web_fetch_config.toml | 17 +-- linear_adapter_trainer/config.py | 2 +- linear_adapter_trainer/knowledge_base/web.py | 5 +- .../knowledge_base/web_adapters.py | 105 +++++++++++++----- pyproject.toml | 4 - tests/test_web_knowledge_base.py | 39 +++++++ 8 files changed, 135 insertions(+), 60 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index ccd04e8..9b0228d 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -74,20 +74,17 @@ kb = loader.load_and_split_urls( ) ``` -Optional adapters for concrete backends live in -`linear_adapter_trainer.knowledge_base.web_adapters` and are installed via -extras. For example, the `linkup` adapter: +For simple public pages, the package includes a dependency-free `http` backend: ```bash -pip install "linear-adapter-trainer[linkup]" -export LINKUP_API_KEY=... +uv run linear-adapter generate examples/web_fetch_config.toml ``` ```python from linear_adapter_trainer import WebLoader from linear_adapter_trainer.knowledge_base.web_adapters import build_web_fetch_client -loader = WebLoader(client=build_web_fetch_client("linkup")) +loader = WebLoader(client=build_web_fetch_client("http")) ``` --- diff --git a/README.md b/README.md index 58dd99e..c54c0a0 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,6 @@ uv sync --group examples # or with pip, choosing the backends you need pip install "linear-adapter-trainer[sentence-transformers]" # local models pip install "linear-adapter-trainer[openai]" # OpenAI API -pip install "linear-adapter-trainer[linkup]" # optional web-fetch backend pip install "linear-adapter-trainer[all]" ``` @@ -96,9 +95,9 @@ whole pipeline offline (great for CI and demos). When the corpus should come from current public web pages instead of local files, use the provider-agnostic `WebLoader`. It fetches each URL through a pluggable backend that you supply, so no specific provider is baked into the -package or its default install. Optional adapters for concrete backends are -available behind extras (for example `[linkup]`), and you can also pass your own -fetch client implementing the small `WebFetchClient` interface. +package or its default install. A dependency-free `http` adapter is included for +simple cases, and you can also pass your own fetch client implementing the small +`WebFetchClient` interface. ## Quickstart (Python) @@ -150,13 +149,10 @@ uv run linear-adapter run examples/config.toml # generate -> train ``` To build the knowledge base from known web pages, switch to -`examples/web_fetch_config.toml`, which selects a pluggable fetch backend. The -example uses the optional `linkup` adapter, so install that extra and set its -key first: +`examples/web_fetch_config.toml`, which selects the neutral `http` fetch +backend: ```bash -pip install "linear-adapter-trainer[linkup]" -export LINKUP_API_KEY=... uv run linear-adapter generate examples/web_fetch_config.toml ``` diff --git a/examples/web_fetch_config.toml b/examples/web_fetch_config.toml index 2653f58..a70bae8 100644 --- a/examples/web_fetch_config.toml +++ b/examples/web_fetch_config.toml @@ -1,21 +1,14 @@ # Example configuration that builds a knowledge base from live web pages. # -# The web loader is provider-agnostic: it fetches each URL through a pluggable -# backend that you choose. Select one with `backend` (an optional adapter), or -# pass your own client in code. The package's default path stays fully offline; -# web fetching is opt-in. -# -# This example uses the optional "linkup" adapter. Install it and set your API -# key before running: -# -# pip install "linear-adapter-trainer[linkup]" -# export LINKUP_API_KEY=... +# The web loader is provider-agnostic. This example uses the dependency-free +# "http" backend; library users can also pass their own client in code. The +# package's default path stays fully offline; web fetching is opt-in. [knowledge_base] format = "web_fetch" -backend = "linkup" +backend = "http" urls = [ - "https://www.santander.com/en/stories/santander-publishes-ai-projects-under-an-open-source-licence-to-ramp-up-shared-innovation", + "https://example.com/", ] render_js = true diff --git a/linear_adapter_trainer/config.py b/linear_adapter_trainer/config.py index 0d41ab1..a8c4ba9 100644 --- a/linear_adapter_trainer/config.py +++ b/linear_adapter_trainer/config.py @@ -51,7 +51,7 @@ def build_knowledge_base(spec: dict[str, Any]) -> KnowledgeBase: if not backend: raise ValueError( "web_fetch requires either a `client` or a `backend` " - "naming an optional web-fetch adapter (e.g. backend = \"linkup\")." + "naming a web-fetch adapter (e.g. backend = \"http\")." ) from .knowledge_base.web_adapters import build_web_fetch_client diff --git a/linear_adapter_trainer/knowledge_base/web.py b/linear_adapter_trainer/knowledge_base/web.py index 8fd8045..45c7f74 100644 --- a/linear_adapter_trainer/knowledge_base/web.py +++ b/linear_adapter_trainer/knowledge_base/web.py @@ -30,9 +30,8 @@ def fetch(self, **kwargs: Any) -> Any: class WebLoader: """Build a :class:`KnowledgeBase` from pages fetched by a pluggable client. - Pass any ``client`` implementing :class:`WebFetchClient`. No backend is - bundled or assumed; optional adapters for specific backends live in - :mod:`linear_adapter_trainer.knowledge_base.web_adapters`. + Pass any ``client`` implementing :class:`WebFetchClient`, or build a neutral + adapter from :mod:`linear_adapter_trainer.knowledge_base.web_adapters`. """ client: WebFetchClient diff --git a/linear_adapter_trainer/knowledge_base/web_adapters.py b/linear_adapter_trainer/knowledge_base/web_adapters.py index 77e51b7..97f4507 100644 --- a/linear_adapter_trainer/knowledge_base/web_adapters.py +++ b/linear_adapter_trainer/knowledge_base/web_adapters.py @@ -1,50 +1,105 @@ # Copyright (c) 2026 Santander Group # SPDX-License-Identifier: Apache-2.0 -"""Optional, opt-in web-fetch backends for :class:`WebLoader`. - -These adapters are not part of the public API and none is used by default. -Each builds a client satisfying -:class:`linear_adapter_trainer.knowledge_base.web.WebFetchClient`. Add your own -adapter here (or pass any compatible client directly) to plug in a different -backend. -""" +"""Neutral, opt-in web-fetch backends for :class:`WebLoader`.""" from __future__ import annotations +import urllib.request +from dataclasses import dataclass +from html.parser import HTMLParser from typing import Any from .web import WebFetchClient -def linkup_fetch_client(**kwargs: Any) -> WebFetchClient: - """Build a Linkup-backed fetch client (optional). +@dataclass(slots=True) +class HttpWebFetchClient: + """Fetch pages with Python's standard-library HTTP client.""" - Requires the optional dependency ``linear-adapter-trainer[linkup]`` and a - ``LINKUP_API_KEY``. Provided only as one example backend; the core loader - does not depend on it. - """ - try: - from linkup import LinkupClient - except ImportError as exc: # pragma: no cover - exercised without dependency - raise ImportError( - "This backend requires the optional dependency: install " - "`linear-adapter-trainer[linkup]` and set `LINKUP_API_KEY`." - ) from exc - return LinkupClient(**kwargs) + timeout: float = 10.0 + user_agent: str = "linear-adapter-trainer/0.1" + + def fetch(self, **kwargs: Any) -> dict[str, str]: + url = kwargs["url"] + include_raw_html = kwargs.get("include_raw_html", False) + request = urllib.request.Request(url, headers={"User-agent": self.user_agent}) + + with urllib.request.urlopen(request, timeout=self.timeout) as response: + charset = response.headers.get_content_charset() or "utf-8" + raw_html = response.read().decode(charset, errors="replace") + + parser = _HTMLTextParser() + parser.feed(raw_html) + result = {"content": parser.content or raw_html.strip()} + if parser.title: + result["title"] = parser.title + if include_raw_html: + result["raw_html"] = raw_html + return result + + +def http_fetch_client(**kwargs: Any) -> WebFetchClient: + """Build a dependency-free HTTP fetch client.""" + return HttpWebFetchClient(**kwargs) -_BACKENDS = {"linkup": linkup_fetch_client} +_BACKENDS = {"http": http_fetch_client} def build_web_fetch_client(backend: str, **kwargs: Any) -> WebFetchClient: - """Build a web-fetch client for a named optional backend.""" + """Build a web-fetch client for a named backend.""" try: factory = _BACKENDS[backend] except KeyError: known = ", ".join(sorted(_BACKENDS)) or "(none)" raise ValueError( - f"Unknown web_fetch backend {backend!r}. Known optional backends: {known}. " + f"Unknown web_fetch backend {backend!r}. Known backends: {known}. " "Alternatively, pass a `client` implementing WebFetchClient directly." ) from None return factory(**kwargs) + + +class _HTMLTextParser(HTMLParser): + def __init__(self) -> None: + super().__init__() + self._parts: list[str] = [] + self._title_parts: list[str] = [] + self._in_title = False + self._in_head = False + self._skip_depth = 0 + + @property + def content(self) -> str: + return " ".join(" ".join(self._parts).split()) + + @property + def title(self) -> str: + return " ".join(" ".join(self._title_parts).split()) + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + tag = tag.lower() + if tag == "head": + self._in_head = True + elif tag == "title": + self._in_title = True + elif tag in {"script", "style", "noscript"}: + self._skip_depth += 1 + + def handle_endtag(self, tag: str) -> None: + tag = tag.lower() + if tag == "head": + self._in_head = False + elif tag == "title": + self._in_title = False + elif tag in {"script", "style", "noscript"} and self._skip_depth: + self._skip_depth -= 1 + + def handle_data(self, data: str) -> None: + text = data.strip() + if not text: + return + if self._in_title: + self._title_parts.append(text) + elif not self._in_head and not self._skip_depth: + self._parts.append(text) diff --git a/pyproject.toml b/pyproject.toml index bcca318..b61b8f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,13 +42,9 @@ sentence-transformers = [ openai = [ "openai>=1.40.0", ] -linkup = [ - "linkup-sdk>=0.18", -] all = [ "sentence-transformers>=3.0.0", "openai>=1.40.0", - "linkup-sdk>=0.18", ] [project.urls] diff --git a/tests/test_web_knowledge_base.py b/tests/test_web_knowledge_base.py index 364d294..6141033 100644 --- a/tests/test_web_knowledge_base.py +++ b/tests/test_web_knowledge_base.py @@ -1,6 +1,8 @@ # Copyright (c) 2026 Santander Group # SPDX-License-Identifier: Apache-2.0 +import io + import pytest from linear_adapter_trainer.config import build_knowledge_base @@ -88,6 +90,43 @@ def test_unknown_web_fetch_backend_is_rejected(): build_web_fetch_client("does-not-exist") +def test_http_web_fetch_backend_fetches_html_with_stdlib(monkeypatch): + class FakeResponse(io.BytesIO): + def __init__(self): + super().__init__( + b"Example" + b"

Heading

Clean page text.

" + ) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return False + + @property + def headers(self): + return self + + def get_content_charset(self): + return "utf-8" + + def fake_urlopen(request, *, timeout): + assert request.full_url == "https://example.com/docs" + assert request.get_header("User-agent") == "linear-adapter-trainer/0.1" + assert timeout == 3.0 + return FakeResponse() + + monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + + client = build_web_fetch_client("http", timeout=3.0) + response = client.fetch(url="https://example.com/docs", include_raw_html=True) + + assert response["title"] == "Example" + assert response["content"] == "Heading Clean page text." + assert response["raw_html"].startswith("") + + def test_web_fetch_config_requires_client_or_backend(): with pytest.raises(ValueError, match="requires either a `client` or a `backend`"): build_knowledge_base({"format": "web_fetch", "urls": ["https://example.com"]})