diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index e12b5dc..9b0228d 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -32,11 +32,12 @@ negatives. ## Knowledge base ```python -from linear_adapter_trainer import KnowledgeBase, Chunk, TextSplitter +from linear_adapter_trainer import KnowledgeBase, Chunk, TextSplitter, WebLoader kb = KnowledgeBase.from_jsonl("kb.jsonl") # {"id","text", ...} per line kb = KnowledgeBase.from_texts(["a", "b"], ids=...) # from raw strings kb = KnowledgeBase.from_directory("docs/", glob="*.txt") +kb = WebLoader(client=my_fetch_client).load_urls(["https://example.com/docs"]) kb.get("chunk-1") # Chunk lookup by id kb.ids, kb.texts # parallel lists @@ -56,6 +57,36 @@ chunked_kb = splitter.split_knowledge_base(kb) # child ids like "doc::0" The splitter is recursive and separator-aware (`\n\n`, `\n`, `. `, ` `), with deterministic output. +### Web ingestion + +`WebLoader` builds a corpus from known public web pages. It is provider-agnostic: +you supply a fetch client implementing the small `WebFetchClient` interface +(`fetch(url, ...) -> response`), so the backend is your choice and nothing is +bundled by default. + +```python +from linear_adapter_trainer import WebLoader, TextSplitter + +loader = WebLoader(client=my_fetch_client, render_js=True) +kb = loader.load_and_split_urls( + ["https://example.com/docs"], + splitter=TextSplitter(chunk_size=512, chunk_overlap=64), +) +``` + +For simple public pages, the package includes a dependency-free `http` backend: + +```bash +uv run linear-adapter generate examples/web_fetch_config.toml +``` + +```python +from linear_adapter_trainer import WebLoader +from linear_adapter_trainer.knowledge_base.web_adapters import build_web_fetch_client + +loader = WebLoader(client=build_web_fetch_client("http")) +``` + --- ## Embedding backends @@ -216,7 +247,7 @@ Aggregated keys include `precision@k`, `recall@k`, `hit_rate@k`, `ndcg@k`, and `linear-adapter config.toml` reads these tables: ```toml -[knowledge_base] # path, format ("jsonl"|"directory"), text_key, id_key, glob +[knowledge_base] # path/urls, format ("jsonl"|"directory"|"web_fetch"), backend, text_key, id_key, glob [embedder] # backend, model, dimension/dimensions, device, batch_size [query_generator] # backend ("template"|"llm"), model, temperature (optional), seed [dataset] # queries_per_chunk, negatives_per_query, strategy, pool_size, val_fraction, seed, max_workers @@ -225,8 +256,9 @@ Aggregated keys include `precision@k`, `recall@k`, `hit_rate@k`, `ndcg@k`, and [output] # dataset_dir, adapter_path, metrics_path ``` -See [`examples/config.toml`](examples/config.toml) for a complete, runnable -example. +See [`examples/config.toml`](examples/config.toml) for a complete offline +example and [`examples/web_fetch_config.toml`](examples/web_fetch_config.toml) +for a known-URL web ingestion example. --- diff --git a/README.md b/README.md index 71b5dc5..c54c0a0 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,13 @@ The core install is dependency-light (`numpy`, `torch`, `tqdm`). A dependency-free `HashingEmbedder` and `TemplateQueryGenerator` let you run the whole pipeline offline (great for CI and demos). +When the corpus should come from current public web pages instead of local +files, use the provider-agnostic `WebLoader`. It fetches each URL through a +pluggable backend that you supply, so no specific provider is baked into the +package or its default install. A dependency-free `http` adapter is included for +simple cases, and you can also pass your own fetch client implementing the small +`WebFetchClient` interface. + ## Quickstart (Python) ```python @@ -141,6 +148,14 @@ uv run linear-adapter evaluate examples/config.toml # base vs adapted uv run linear-adapter run examples/config.toml # generate -> train ``` +To build the knowledge base from known web pages, switch to +`examples/web_fetch_config.toml`, which selects the neutral `http` fetch +backend: + +```bash +uv run linear-adapter generate examples/web_fetch_config.toml +``` + Example output (with a Sentence-Transformers backend on a paraphrased query set): ``` diff --git a/examples/web_fetch_config.toml b/examples/web_fetch_config.toml new file mode 100644 index 0000000..a70bae8 --- /dev/null +++ b/examples/web_fetch_config.toml @@ -0,0 +1,61 @@ +# Example configuration that builds a knowledge base from live web pages. +# +# The web loader is provider-agnostic. This example uses the dependency-free +# "http" backend; library users can also pass their own client in code. The +# package's default path stays fully offline; web fetching is opt-in. + +[knowledge_base] +format = "web_fetch" +backend = "http" +urls = [ + "https://example.com/", +] +render_js = true + +[knowledge_base.chunking] +enabled = true +chunk_size = 512 +chunk_overlap = 64 + +[embedder] +backend = "hashing" +dimension = 512 + +[query_generator] +backend = "template" +seed = 0 + +[negative_generator] +backend = "none" + +[dataset] +queries_per_chunk = 4 +negatives_per_query = 2 +strategy = "mixed" +pool_size = 8 +val_fraction = 0.25 +split_strategy = "chunk" +seed = 0 +max_workers = 1 + +[dataset.mix] +semantic_opposite = 0.5 +hard = 0.3 +random = 0.2 + +[training] +epochs = 30 +batch_size = 32 +learning_rate = 0.005 +margin = 0.2 +distance = "cosine" +residual = true +monitor = "mrr" +patience = 6 +eval_ks = [1, 3, 5, 10] +seed = 0 + +[output] +dataset_dir = "examples/artifacts/web-fetch-dataset" +adapter_path = "examples/artifacts/web-fetch-adapter.pt" +metrics_path = "examples/artifacts/web-fetch-metrics.json" diff --git a/linear_adapter_trainer/__init__.py b/linear_adapter_trainer/__init__.py index 83ea1d5..71a5327 100644 --- a/linear_adapter_trainer/__init__.py +++ b/linear_adapter_trainer/__init__.py @@ -40,7 +40,7 @@ ) from .embeddings import EmbeddingModel, HashingEmbedder from .evaluation import RetrievalEvaluator, evaluate_rankings -from .knowledge_base import Chunk, KnowledgeBase, TextSplitter +from .knowledge_base import Chunk, KnowledgeBase, TextSplitter, WebLoader __version__ = "0.1.0" @@ -65,6 +65,7 @@ "Triplet", "TripletDataset", "TripletLoss", + "WebLoader", "evaluate_rankings", "__version__", ] diff --git a/linear_adapter_trainer/config.py b/linear_adapter_trainer/config.py index c1dfe5e..a8c4ba9 100644 --- a/linear_adapter_trainer/config.py +++ b/linear_adapter_trainer/config.py @@ -32,15 +32,36 @@ def load_config(path: str | Path) -> dict[str, Any]: def build_knowledge_base(spec: dict[str, Any]) -> KnowledgeBase: """Instantiate a knowledge base from a ``[knowledge_base]`` table.""" fmt = spec.get("format", "jsonl") - path = spec["path"] if fmt == "jsonl": + path = spec["path"] kb = KnowledgeBase.from_jsonl( path, text_key=spec.get("text_key", "text"), id_key=spec.get("id_key", "id"), ) elif fmt == "directory": + path = spec["path"] kb = KnowledgeBase.from_directory(path, glob=spec.get("glob", "*.txt")) + elif fmt == "web_fetch": + from .knowledge_base.web import WebLoader + + client = spec.get("client") + if client is None: + backend = spec.get("backend") + if not backend: + raise ValueError( + "web_fetch requires either a `client` or a `backend` " + "naming a web-fetch adapter (e.g. backend = \"http\")." + ) + from .knowledge_base.web_adapters import build_web_fetch_client + + client = build_web_fetch_client(backend) + kb = WebLoader( + client=client, + render_js=spec.get("render_js", True), + include_raw_html=spec.get("include_raw_html", False), + extract_images=spec.get("extract_images", False), + ).load_urls(spec["urls"], ids=spec.get("ids")) else: raise ValueError(f"Unsupported knowledge_base.format: {fmt!r}") diff --git a/linear_adapter_trainer/knowledge_base/__init__.py b/linear_adapter_trainer/knowledge_base/__init__.py index 280e86d..5fe5501 100644 --- a/linear_adapter_trainer/knowledge_base/__init__.py +++ b/linear_adapter_trainer/knowledge_base/__init__.py @@ -5,5 +5,6 @@ from .base import Chunk, KnowledgeBase from .chunking import TextSplitter +from .web import WebLoader -__all__ = ["Chunk", "KnowledgeBase", "TextSplitter"] +__all__ = ["Chunk", "KnowledgeBase", "TextSplitter", "WebLoader"] diff --git a/linear_adapter_trainer/knowledge_base/web.py b/linear_adapter_trainer/knowledge_base/web.py new file mode 100644 index 0000000..45c7f74 --- /dev/null +++ b/linear_adapter_trainer/knowledge_base/web.py @@ -0,0 +1,104 @@ +# Copyright (c) 2026 Santander Group +# SPDX-License-Identifier: Apache-2.0 + +"""Provider-agnostic web ingestion for retrieval knowledge bases.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Any, Protocol + +from .base import Chunk, KnowledgeBase +from .chunking import TextSplitter + + +class WebFetchClient(Protocol): + """Minimal interface a web-fetch backend must implement. + + Any object with a ``fetch`` method that takes a URL and returns a response + carrying the cleaned page content can be used. The concrete backend (a + hosted fetch API, a local scraper, a stub in tests, etc.) is the caller's + choice; the loader has no knowledge of any specific provider. + """ + + def fetch(self, **kwargs: Any) -> Any: + """Fetch one URL and return a response with cleaned page content.""" + + +@dataclass(slots=True) +class WebLoader: + """Build a :class:`KnowledgeBase` from pages fetched by a pluggable client. + + Pass any ``client`` implementing :class:`WebFetchClient`, or build a neutral + adapter from :mod:`linear_adapter_trainer.knowledge_base.web_adapters`. + """ + + client: WebFetchClient + render_js: bool = True + include_raw_html: bool = False + extract_images: bool = False + + def load_urls(self, urls: Sequence[str], *, ids: Sequence[str] | None = None) -> KnowledgeBase: + """Fetch known URLs and return one knowledge-base chunk per page.""" + if not urls: + raise ValueError("WebLoader requires at least one URL.") + if ids is not None and len(ids) != len(urls): + raise ValueError("`ids` length must match `urls` length.") + + chunks: list[Chunk] = [] + for position, url in enumerate(urls): + response = self.client.fetch( + url=url, + render_js=self.render_js, + include_raw_html=self.include_raw_html, + extract_images=self.extract_images, + ) + metadata = { + "source": url, + "source_url": url, + **_extract_metadata(response), + } + chunks.append( + Chunk( + id=str(ids[position]) if ids is not None else f"web-{position}", + text=_extract_text(response), + metadata=metadata, + ) + ) + return KnowledgeBase(chunks) + + def load_and_split_urls( + self, + urls: Sequence[str], + *, + ids: Sequence[str] | None = None, + splitter: TextSplitter | None = None, + ) -> KnowledgeBase: + """Fetch known URLs, then split each fetched page into retrieval chunks.""" + kb = self.load_urls(urls, ids=ids) + return (splitter or TextSplitter()).split_knowledge_base(kb) + + +def _extract_text(response: Any) -> str: + for key in ("content", "markdown", "text"): + value = _get_value(response, key) + if isinstance(value, str) and value.strip(): + return value + if isinstance(response, str) and response.strip(): + return response + raise ValueError("Web fetch response did not contain cleaned page text.") + + +def _extract_metadata(response: Any) -> dict[str, Any]: + metadata: dict[str, Any] = {} + title = _get_value(response, "title") or _get_value(response, "name") + if isinstance(title, str) and title: + metadata["title"] = title + return metadata + + +def _get_value(response: Any, key: str) -> Any: + if isinstance(response, dict): + return response.get(key) + return getattr(response, key, None) diff --git a/linear_adapter_trainer/knowledge_base/web_adapters.py b/linear_adapter_trainer/knowledge_base/web_adapters.py new file mode 100644 index 0000000..97f4507 --- /dev/null +++ b/linear_adapter_trainer/knowledge_base/web_adapters.py @@ -0,0 +1,105 @@ +# Copyright (c) 2026 Santander Group +# SPDX-License-Identifier: Apache-2.0 + +"""Neutral, opt-in web-fetch backends for :class:`WebLoader`.""" + +from __future__ import annotations + +import urllib.request +from dataclasses import dataclass +from html.parser import HTMLParser +from typing import Any + +from .web import WebFetchClient + + +@dataclass(slots=True) +class HttpWebFetchClient: + """Fetch pages with Python's standard-library HTTP client.""" + + timeout: float = 10.0 + user_agent: str = "linear-adapter-trainer/0.1" + + def fetch(self, **kwargs: Any) -> dict[str, str]: + url = kwargs["url"] + include_raw_html = kwargs.get("include_raw_html", False) + request = urllib.request.Request(url, headers={"User-agent": self.user_agent}) + + with urllib.request.urlopen(request, timeout=self.timeout) as response: + charset = response.headers.get_content_charset() or "utf-8" + raw_html = response.read().decode(charset, errors="replace") + + parser = _HTMLTextParser() + parser.feed(raw_html) + result = {"content": parser.content or raw_html.strip()} + if parser.title: + result["title"] = parser.title + if include_raw_html: + result["raw_html"] = raw_html + return result + + +def http_fetch_client(**kwargs: Any) -> WebFetchClient: + """Build a dependency-free HTTP fetch client.""" + return HttpWebFetchClient(**kwargs) + + +_BACKENDS = {"http": http_fetch_client} + + +def build_web_fetch_client(backend: str, **kwargs: Any) -> WebFetchClient: + """Build a web-fetch client for a named backend.""" + try: + factory = _BACKENDS[backend] + except KeyError: + known = ", ".join(sorted(_BACKENDS)) or "(none)" + raise ValueError( + f"Unknown web_fetch backend {backend!r}. Known backends: {known}. " + "Alternatively, pass a `client` implementing WebFetchClient directly." + ) from None + return factory(**kwargs) + + +class _HTMLTextParser(HTMLParser): + def __init__(self) -> None: + super().__init__() + self._parts: list[str] = [] + self._title_parts: list[str] = [] + self._in_title = False + self._in_head = False + self._skip_depth = 0 + + @property + def content(self) -> str: + return " ".join(" ".join(self._parts).split()) + + @property + def title(self) -> str: + return " ".join(" ".join(self._title_parts).split()) + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + tag = tag.lower() + if tag == "head": + self._in_head = True + elif tag == "title": + self._in_title = True + elif tag in {"script", "style", "noscript"}: + self._skip_depth += 1 + + def handle_endtag(self, tag: str) -> None: + tag = tag.lower() + if tag == "head": + self._in_head = False + elif tag == "title": + self._in_title = False + elif tag in {"script", "style", "noscript"} and self._skip_depth: + self._skip_depth -= 1 + + def handle_data(self, data: str) -> None: + text = data.strip() + if not text: + return + if self._in_title: + self._title_parts.append(text) + elif not self._in_head and not self._skip_depth: + self._parts.append(text) diff --git a/tests/test_web_knowledge_base.py b/tests/test_web_knowledge_base.py new file mode 100644 index 0000000..6141033 --- /dev/null +++ b/tests/test_web_knowledge_base.py @@ -0,0 +1,151 @@ +# Copyright (c) 2026 Santander Group +# SPDX-License-Identifier: Apache-2.0 + +import io + +import pytest + +from linear_adapter_trainer.config import build_knowledge_base +from linear_adapter_trainer.knowledge_base import TextSplitter, WebLoader +from linear_adapter_trainer.knowledge_base.web_adapters import build_web_fetch_client + + +class RecordingClient: + def __init__(self, responses): + self.responses = responses + self.calls = [] + + def fetch(self, **kwargs): + self.calls.append(kwargs) + return self.responses[kwargs["url"]] + + +def test_web_loader_fetches_known_urls_into_knowledge_base(): + client = RecordingClient( + { + "https://example.com/risk": { + "content": "# Risk overview\n\nSource-backed risk text.", + "title": "Risk overview", + }, + "https://example.com/governance": { + "markdown": "# Governance\n\nClean markdown content.", + }, + } + ) + + kb = WebLoader(client=client).load_urls( + ["https://example.com/risk", "https://example.com/governance"], + ids=["risk", "governance"], + ) + + assert kb.ids == ["risk", "governance"] + assert kb.get("risk").text == "# Risk overview\n\nSource-backed risk text." + assert kb.get("risk").metadata == { + "source": "https://example.com/risk", + "source_url": "https://example.com/risk", + "title": "Risk overview", + } + assert client.calls == [ + { + "url": "https://example.com/risk", + "render_js": True, + "include_raw_html": False, + "extract_images": False, + }, + { + "url": "https://example.com/governance", + "render_js": True, + "include_raw_html": False, + "extract_images": False, + }, + ] + + +def test_web_loader_can_split_fetched_pages(): + client = RecordingClient( + { + "https://example.com/report": { + "content": "First sourced paragraph.\n\nSecond sourced paragraph.\n\nThird sourced paragraph." + } + } + ) + + kb = WebLoader(client=client).load_and_split_urls( + ["https://example.com/report"], + splitter=TextSplitter(chunk_size=35, chunk_overlap=5), + ) + + assert len(kb) > 1 + assert all(chunk.metadata["source_url"] == "https://example.com/report" for chunk in kb) + assert all(chunk.metadata["parent_id"] == "web-0" for chunk in kb) + + +def test_web_loader_requires_a_client(): + with pytest.raises(TypeError): + WebLoader() # type: ignore[call-arg] + + +def test_unknown_web_fetch_backend_is_rejected(): + with pytest.raises(ValueError, match="Unknown web_fetch backend"): + build_web_fetch_client("does-not-exist") + + +def test_http_web_fetch_backend_fetches_html_with_stdlib(monkeypatch): + class FakeResponse(io.BytesIO): + def __init__(self): + super().__init__( + b"Example" + b"

Heading

Clean page text.

" + ) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return False + + @property + def headers(self): + return self + + def get_content_charset(self): + return "utf-8" + + def fake_urlopen(request, *, timeout): + assert request.full_url == "https://example.com/docs" + assert request.get_header("User-agent") == "linear-adapter-trainer/0.1" + assert timeout == 3.0 + return FakeResponse() + + monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + + client = build_web_fetch_client("http", timeout=3.0) + response = client.fetch(url="https://example.com/docs", include_raw_html=True) + + assert response["title"] == "Example" + assert response["content"] == "Heading Clean page text." + assert response["raw_html"].startswith("") + + +def test_web_fetch_config_requires_client_or_backend(): + with pytest.raises(ValueError, match="requires either a `client` or a `backend`"): + build_knowledge_base({"format": "web_fetch", "urls": ["https://example.com"]}) + + +def test_config_builds_web_fetch_knowledge_base_with_injected_client(): + client = RecordingClient( + {"https://example.com/docs": {"content": "Trusted source material for an AI workflow."}} + ) + + kb = build_knowledge_base( + { + "format": "web_fetch", + "urls": ["https://example.com/docs"], + "client": client, + "render_js": False, + "chunking": {"enabled": True, "chunk_size": 32, "chunk_overlap": 4}, + } + ) + + assert kb.get("web-0::0").metadata["source_url"] == "https://example.com/docs" + assert client.calls[0]["render_js"] is False