feat: add vendor-neutral web corpus loader

DOCUMENTATION.md

@@ -32,11 +32,12 @@ negatives.
 ## Knowledge base
 
 ```python
-from linear_adapter_trainer import KnowledgeBase, Chunk, TextSplitter
+from linear_adapter_trainer import KnowledgeBase, Chunk, TextSplitter, WebLoader
 
 kb = KnowledgeBase.from_jsonl("kb.jsonl")           # {"id","text", ...} per line
 kb = KnowledgeBase.from_texts(["a", "b"], ids=...)  # from raw strings
 kb = KnowledgeBase.from_directory("docs/", glob="*.txt")
+kb = WebLoader(client=my_fetch_client).load_urls(["https://example.com/docs"])
 
 kb.get("chunk-1")        # Chunk lookup by id
 kb.ids, kb.texts         # parallel lists
@@ -56,6 +57,36 @@ chunked_kb = splitter.split_knowledge_base(kb)   # child ids like "doc::0"
 The splitter is recursive and separator-aware (`\n\n`, `\n`, `. `, ` `), with
 deterministic output.
 
+### Web ingestion
+
+`WebLoader` builds a corpus from known public web pages. It is provider-agnostic:
+you supply a fetch client implementing the small `WebFetchClient` interface
+(`fetch(url, ...) -> response`), so the backend is your choice and nothing is
+bundled by default.
+
+```python
+from linear_adapter_trainer import WebLoader, TextSplitter
+
+loader = WebLoader(client=my_fetch_client, render_js=True)
+kb = loader.load_and_split_urls(
+    ["https://example.com/docs"],
+    splitter=TextSplitter(chunk_size=512, chunk_overlap=64),
+)
+```
+
+For simple public pages, the package includes a dependency-free `http` backend:
+
+```bash
+uv run linear-adapter generate examples/web_fetch_config.toml
+```
+
+```python
+from linear_adapter_trainer import WebLoader
+from linear_adapter_trainer.knowledge_base.web_adapters import build_web_fetch_client
+
+loader = WebLoader(client=build_web_fetch_client("http"))
+```
+
 ---
 
 ## Embedding backends
@@ -216,7 +247,7 @@ Aggregated keys include `precision@k`, `recall@k`, `hit_rate@k`, `ndcg@k`, and
 `linear-adapter <command> config.toml` reads these tables:
 
 ```toml
-[knowledge_base]   # path, format ("jsonl"|"directory"), text_key, id_key, glob
+[knowledge_base]   # path/urls, format ("jsonl"|"directory"|"web_fetch"), backend, text_key, id_key, glob
 [embedder]         # backend, model, dimension/dimensions, device, batch_size
 [query_generator]  # backend ("template"|"llm"), model, temperature (optional), seed
 [dataset]          # queries_per_chunk, negatives_per_query, strategy, pool_size, val_fraction, seed, max_workers
@@ -225,8 +256,9 @@ Aggregated keys include `precision@k`, `recall@k`, `hit_rate@k`, `ndcg@k`, and
 [output]           # dataset_dir, adapter_path, metrics_path
 ```
 
-See [`examples/config.toml`](examples/config.toml) for a complete, runnable
-example.
+See [`examples/config.toml`](examples/config.toml) for a complete offline
+example and [`examples/web_fetch_config.toml`](examples/web_fetch_config.toml)
+for a known-URL web ingestion example.
 
 ---
 

README.md

@@ -92,6 +92,13 @@ The core install is dependency-light (`numpy`, `torch`, `tqdm`). A
 dependency-free `HashingEmbedder` and `TemplateQueryGenerator` let you run the
 whole pipeline offline (great for CI and demos).
 
+When the corpus should come from current public web pages instead of local
+files, use the provider-agnostic `WebLoader`. It fetches each URL through a
+pluggable backend that you supply, so no specific provider is baked into the
+package or its default install. A dependency-free `http` adapter is included for
+simple cases, and you can also pass your own fetch client implementing the small
+`WebFetchClient` interface.
+
 ## Quickstart (Python)
 
 ```python
@@ -141,6 +148,14 @@ uv run linear-adapter evaluate examples/config.toml   # base vs adapted
 uv run linear-adapter run      examples/config.toml   # generate -> train
 ```
 
+To build the knowledge base from known web pages, switch to
+`examples/web_fetch_config.toml`, which selects the neutral `http` fetch
+backend:
+
+```bash
+uv run linear-adapter generate examples/web_fetch_config.toml
+```
+
 Example output (with a Sentence-Transformers backend on a paraphrased query set):
 
 ```

examples/web_fetch_config.toml

@@ -0,0 +1,61 @@
+# Example configuration that builds a knowledge base from live web pages.
+#
+# The web loader is provider-agnostic. This example uses the dependency-free
+# "http" backend; library users can also pass their own client in code. The
+# package's default path stays fully offline; web fetching is opt-in.
+
+[knowledge_base]
+format = "web_fetch"
+backend = "http"
+urls = [
+    "https://example.com/",
+]
+render_js = true
+
+[knowledge_base.chunking]
+enabled = true
+chunk_size = 512
+chunk_overlap = 64
+
+[embedder]
+backend = "hashing"
+dimension = 512
+
+[query_generator]
+backend = "template"
+seed = 0
+
+[negative_generator]
+backend = "none"
+
+[dataset]
+queries_per_chunk = 4
+negatives_per_query = 2
+strategy = "mixed"
+pool_size = 8
+val_fraction = 0.25
+split_strategy = "chunk"
+seed = 0
+max_workers = 1
+
+[dataset.mix]
+semantic_opposite = 0.5
+hard = 0.3
+random = 0.2
+
+[training]
+epochs = 30
+batch_size = 32
+learning_rate = 0.005
+margin = 0.2
+distance = "cosine"
+residual = true
+monitor = "mrr"
+patience = 6
+eval_ks = [1, 3, 5, 10]
+seed = 0
+
+[output]
+dataset_dir = "examples/artifacts/web-fetch-dataset"
+adapter_path = "examples/artifacts/web-fetch-adapter.pt"
+metrics_path = "examples/artifacts/web-fetch-metrics.json"

linear_adapter_trainer/__init__.py

@@ -40,7 +40,7 @@
 )
 from .embeddings import EmbeddingModel, HashingEmbedder
 from .evaluation import RetrievalEvaluator, evaluate_rankings
-from .knowledge_base import Chunk, KnowledgeBase, TextSplitter
+from .knowledge_base import Chunk, KnowledgeBase, TextSplitter, WebLoader
 
 __version__ = "0.1.0"
 
@@ -65,6 +65,7 @@
     "Triplet",
     "TripletDataset",
     "TripletLoss",
+    "WebLoader",
     "evaluate_rankings",
     "__version__",
 ]

linear_adapter_trainer/config.py

@@ -32,15 +32,36 @@ def load_config(path: str | Path) -> dict[str, Any]:
 def build_knowledge_base(spec: dict[str, Any]) -> KnowledgeBase:
     """Instantiate a knowledge base from a ``[knowledge_base]`` table."""
     fmt = spec.get("format", "jsonl")
-    path = spec["path"]
     if fmt == "jsonl":
+        path = spec["path"]
         kb = KnowledgeBase.from_jsonl(
             path,
             text_key=spec.get("text_key", "text"),
             id_key=spec.get("id_key", "id"),
         )
     elif fmt == "directory":
+        path = spec["path"]
         kb = KnowledgeBase.from_directory(path, glob=spec.get("glob", "*.txt"))
+    elif fmt == "web_fetch":
+        from .knowledge_base.web import WebLoader
+
+        client = spec.get("client")
+        if client is None:
+            backend = spec.get("backend")
+            if not backend:
+                raise ValueError(
+                    "web_fetch requires either a `client` or a `backend` "
+                    "naming a web-fetch adapter (e.g. backend = \"http\")."
+                )
+            from .knowledge_base.web_adapters import build_web_fetch_client
+
+            client = build_web_fetch_client(backend)
+        kb = WebLoader(
+            client=client,
+            render_js=spec.get("render_js", True),
+            include_raw_html=spec.get("include_raw_html", False),
+            extract_images=spec.get("extract_images", False),
+        ).load_urls(spec["urls"], ids=spec.get("ids"))
     else:
         raise ValueError(f"Unsupported knowledge_base.format: {fmt!r}")
 

linear_adapter_trainer/knowledge_base/__init__.py

@@ -5,5 +5,6 @@
 
 from .base import Chunk, KnowledgeBase
 from .chunking import TextSplitter
+from .web import WebLoader
 
-__all__ = ["Chunk", "KnowledgeBase", "TextSplitter"]
+__all__ = ["Chunk", "KnowledgeBase", "TextSplitter", "WebLoader"]

linear_adapter_trainer/knowledge_base/web.py

@@ -0,0 +1,104 @@
+# Copyright (c) 2026 Santander Group
+# SPDX-License-Identifier: Apache-2.0
+
+"""Provider-agnostic web ingestion for retrieval knowledge bases."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+from .base import Chunk, KnowledgeBase
+from .chunking import TextSplitter
+
+
+class WebFetchClient(Protocol):
+    """Minimal interface a web-fetch backend must implement.
+
+    Any object with a ``fetch`` method that takes a URL and returns a response
+    carrying the cleaned page content can be used. The concrete backend (a
+    hosted fetch API, a local scraper, a stub in tests, etc.) is the caller's
+    choice; the loader has no knowledge of any specific provider.
+    """
+
+    def fetch(self, **kwargs: Any) -> Any:
+        """Fetch one URL and return a response with cleaned page content."""
+
+
+@dataclass(slots=True)
+class WebLoader:
+    """Build a :class:`KnowledgeBase` from pages fetched by a pluggable client.
+
+    Pass any ``client`` implementing :class:`WebFetchClient`, or build a neutral
+    adapter from :mod:`linear_adapter_trainer.knowledge_base.web_adapters`.
+    """
+
+    client: WebFetchClient
+    render_js: bool = True
+    include_raw_html: bool = False
+    extract_images: bool = False
+
+    def load_urls(self, urls: Sequence[str], *, ids: Sequence[str] | None = None) -> KnowledgeBase:
+        """Fetch known URLs and return one knowledge-base chunk per page."""
+        if not urls:
+            raise ValueError("WebLoader requires at least one URL.")
+        if ids is not None and len(ids) != len(urls):
+            raise ValueError("`ids` length must match `urls` length.")
+
+        chunks: list[Chunk] = []
+        for position, url in enumerate(urls):
+            response = self.client.fetch(
+                url=url,
+                render_js=self.render_js,
+                include_raw_html=self.include_raw_html,
+                extract_images=self.extract_images,
+            )
+            metadata = {
+                "source": url,
+                "source_url": url,
+                **_extract_metadata(response),
+            }
+            chunks.append(
+                Chunk(
+                    id=str(ids[position]) if ids is not None else f"web-{position}",
+                    text=_extract_text(response),
+                    metadata=metadata,
+                )
+            )
+        return KnowledgeBase(chunks)
+
+    def load_and_split_urls(
+        self,
+        urls: Sequence[str],
+        *,
+        ids: Sequence[str] | None = None,
+        splitter: TextSplitter | None = None,
+    ) -> KnowledgeBase:
+        """Fetch known URLs, then split each fetched page into retrieval chunks."""
+        kb = self.load_urls(urls, ids=ids)
+        return (splitter or TextSplitter()).split_knowledge_base(kb)
+
+
+def _extract_text(response: Any) -> str:
+    for key in ("content", "markdown", "text"):
+        value = _get_value(response, key)
+        if isinstance(value, str) and value.strip():
+            return value
+    if isinstance(response, str) and response.strip():
+        return response
+    raise ValueError("Web fetch response did not contain cleaned page text.")
+
+
+def _extract_metadata(response: Any) -> dict[str, Any]:
+    metadata: dict[str, Any] = {}
+    title = _get_value(response, "title") or _get_value(response, "name")
+    if isinstance(title, str) and title:
+        metadata["title"] = title
+    return metadata
+
+
+def _get_value(response: Any, key: str) -> Any:
+    if isinstance(response, dict):
+        return response.get(key)
+    return getattr(response, key, None)