diff --git a/.github/workflows/catalog-sentinel.yml b/.github/workflows/catalog-sentinel.yml new file mode 100644 index 0000000..1dd679b --- /dev/null +++ b/.github/workflows/catalog-sentinel.yml @@ -0,0 +1,240 @@ +name: Catalog sentinel + +on: + schedule: + - cron: "23 6 * * 3" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: catalog-sentinel + cancel-in-progress: false + +jobs: + public-discovery: + name: Public catalog discovery + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + issues: write + steps: + - name: Check out source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.14" + - name: Install runtime + run: python -m pip install --disable-pip-version-check . + - name: Restore prior lifecycle state + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: .sentinel-state/public.json + key: catalog-sentinel-public-${{ github.run_id }} + restore-keys: | + catalog-sentinel-public- + - name: Run bounded public discovery + shell: bash + run: | + set -euo pipefail + mkdir -p .sentinel-artifacts .sentinel-state + previous=() + if test -f .sentinel-state/public.json; then + previous=(--previous .sentinel-state/public.json) + fi + # Advisory evidence only. Never mutates providers.toml or routing state. + python scripts/catalog_sentinel.py discover \ + --output .sentinel-artifacts/public.json \ + --summary .sentinel-artifacts/public.md \ + --timeout 10 \ + --max-bytes 1000000 \ + "${previous[@]}" + cp .sentinel-artifacts/public.json .sentinel-state/public.json + - name: Prepare bounded drift issue + id: drift + shell: bash + run: | + set +e + python scripts/catalog_sentinel.py issue-body \ + --report .sentinel-artifacts/public.json \ + --output .sentinel-artifacts/issue.md + status=$? + set -e + if test "$status" -eq 3; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + test "$status" -eq 0 + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + - name: Open or update advisory drift issue + if: steps.drift.outputs.changed == 'true' + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + existing=$(gh issue list \ + --repo "$GITHUB_REPOSITORY" \ + --state open \ + --search '"Catalog sentinel drift" in:title' \ + --limit 100 \ + --json number,title,author,body \ + --jq 'map(select( + .title == "Catalog sentinel drift" + and (.author.login == "github-actions[bot]" or .author.login == "github-actions") + and (.body | contains("")) + )) | first | .number // empty') + if [[ "$existing" =~ ^[0-9]+$ ]]; then + gh issue comment "$existing" \ + --repo "$GITHUB_REPOSITORY" \ + --body-file .sentinel-artifacts/issue.md + else + gh issue create \ + --repo "$GITHUB_REPOSITORY" \ + --title "Catalog sentinel drift" \ + --label provider-catalog \ + --label tests \ + --body-file .sentinel-artifacts/issue.md + fi + - name: Upload sanitized public evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: catalog-sentinel-public-${{ github.run_id }} + path: | + .sentinel-artifacts/public.json + .sentinel-artifacts/public.md + if-no-files-found: error + retention-days: 30 + - name: Save public lifecycle state + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: .sentinel-state/public.json + key: catalog-sentinel-public-${{ github.run_id }} + + authenticated-probes: + name: Protected authenticated completion probes + runs-on: ubuntu-latest + timeout-minutes: 20 + environment: catalog-sentinel + permissions: + contents: read + issues: write + steps: + - name: Check out source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.14" + - name: Install runtime + run: python -m pip install --disable-pip-version-check . + - name: Detect protected probe configuration + id: probe-config + env: + SENTINEL_KEYS: ${{ secrets.FREELLMPOOL_SENTINEL_KEYS_JSON }} + shell: bash + run: | + if test -n "$SENTINEL_KEYS"; then + echo "configured=true" >> "$GITHUB_OUTPUT" + else + echo "configured=false" >> "$GITHUB_OUTPUT" + echo "Protected probe secret is not configured; public discovery still completed." + fi + - name: Restore prior probe lifecycle state + if: steps.probe-config.outputs.configured == 'true' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: .sentinel-state/probe.json + key: catalog-sentinel-probe-${{ github.run_id }} + restore-keys: | + catalog-sentinel-probe- + - name: Run bounded authenticated probes + if: steps.probe-config.outputs.configured == 'true' + env: + FREELLMPOOL_SENTINEL_KEYS_JSON: ${{ secrets.FREELLMPOOL_SENTINEL_KEYS_JSON }} + shell: bash + run: | + set -euo pipefail + mkdir -p .sentinel-artifacts .sentinel-state + previous=() + if test -f .sentinel-state/probe.json; then + previous=(--previous .sentinel-state/probe.json) + fi + python scripts/catalog_sentinel.py probe \ + --output .sentinel-artifacts/probe.json \ + --summary .sentinel-artifacts/probe.md \ + --timeout 20 \ + --max-providers 8 \ + --max-models-per-provider 1 \ + "${previous[@]}" + cp .sentinel-artifacts/probe.json .sentinel-state/probe.json + - name: Prepare bounded probe issue + if: steps.probe-config.outputs.configured == 'true' + id: probe-drift + shell: bash + run: | + set +e + python scripts/catalog_sentinel.py issue-body \ + --report .sentinel-artifacts/probe.json \ + --output .sentinel-artifacts/probe-issue.md + status=$? + set -e + if test "$status" -eq 3; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + test "$status" -eq 0 + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + - name: Open or update advisory probe issue + if: steps.probe-drift.outputs.changed == 'true' + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + existing=$(gh issue list \ + --repo "$GITHUB_REPOSITORY" \ + --state open \ + --search '"Catalog sentinel probe findings" in:title' \ + --limit 100 \ + --json number,title,author,body \ + --jq 'map(select( + .title == "Catalog sentinel probe findings" + and (.author.login == "github-actions[bot]" or .author.login == "github-actions") + and (.body | contains("")) + )) | first | .number // empty') + if [[ "$existing" =~ ^[0-9]+$ ]]; then + gh issue comment "$existing" \ + --repo "$GITHUB_REPOSITORY" \ + --body-file .sentinel-artifacts/probe-issue.md + else + gh issue create \ + --repo "$GITHUB_REPOSITORY" \ + --title "Catalog sentinel probe findings" \ + --label provider-catalog \ + --label tests \ + --body-file .sentinel-artifacts/probe-issue.md + fi + - name: Upload sanitized protected evidence + if: steps.probe-config.outputs.configured == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: catalog-sentinel-probes-${{ github.run_id }} + path: | + .sentinel-artifacts/probe.json + .sentinel-artifacts/probe.md + if-no-files-found: error + retention-days: 30 + - name: Save probe lifecycle state + if: steps.probe-config.outputs.configured == 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: .sentinel-state/probe.json + key: catalog-sentinel-probe-${{ github.run_id }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ce9767..6139366 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ All notable changes to this project are documented here. The format is based on ## [Unreleased] ### Added +- A weekly, manually dispatchable catalog sentinel with bounded public + discovery, environment-protected completion probes, sanitized lifecycle + artifacts, and advisory drift issues that never mutate routing. - Advisory proxy operations APIs: public `/livez` and `/readyz`, an authenticated secret-free `/v1/providers` inventory, and `/v1/models?ready=true` filtering. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f27069b..14385b2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,6 +45,9 @@ maintainer commands for filing them. ## Adding a provider The whole catalog is [`src/freellmpool/providers.toml`](src/freellmpool/providers.toml). +The scheduled discovery and protected-probe contract is documented in +[`docs/CATALOG_SENTINEL.md`](docs/CATALOG_SENTINEL.md); sentinel output is +advisory and never authorizes an automatic catalog mutation. Most providers are OpenAI-compatible, so adding one is just a TOML block: ```toml diff --git a/docs/CATALOG_SENTINEL.md b/docs/CATALOG_SENTINEL.md new file mode 100644 index 0000000..6fa0277 --- /dev/null +++ b/docs/CATALOG_SENTINEL.md @@ -0,0 +1,89 @@ +# Catalog sentinel operations + +The `catalog-sentinel` workflow is a weekly and manually dispatchable, +advisory drift detector. It produces a bounded JSON workflow artifact and a +short Markdown artifact. When a public listing contains actionable additions +or an authoritative removal, the workflow opens or comments on a single +maintainer-review issue. + +It never enables or disables a route, edits `providers.toml`, purchases +credits, or changes runtime routing. A maintainer must reproduce the evidence, +check provider terms and billing behavior, run completion probes, and submit a +normal reviewed pull request before catalog state changes. + +## Public discovery + +The public job sends unauthenticated `GET` requests only to model-list +endpoints derived from the packaged catalog. Redirects are disabled, each +request has a timeout, and decoded bodies are capped at 1 MB. User catalog +overrides are deliberately ignored. + +Unknown and partial listing scopes can identify new candidates, but missing +rows are recorded as unconfirmed absences. They are not retirement evidence. +An empty response, malformed JSON, 429 rate limit, 402 billing/credit response, +provider-wide authentication failure, timeout, and transient 5xx response +never cause a retirement recommendation. + +## Protected completion probes + +Authenticated canaries run in the GitHub `catalog-sentinel` environment. Turn +on environment protection and require a maintainer reviewer before configuring +the environment secret: + +```text +FREELLMPOOL_SENTINEL_KEYS_JSON +``` + +Its value is a bounded JSON object that maps the catalog's environment-variable +names to their values. For example, configure it through GitHub's encrypted +environment-secret UI; never commit the value: + +```json +{"GROQ_API_KEY":"...","CLOUDFLARE_API_TOKEN":"...","CLOUDFLARE_ACCOUNT_ID":"..."} +``` + +The probe report contains provider IDs, catalog model IDs, HTTP status +classifications, timestamps, and lifecycle counters. It excludes keys, +account identifiers, provider response bodies, exception text, prompts, and +completion text. If the secret is absent, the protected job records that probes +were skipped without weakening public discovery. + +Each canary requests at most eight output tokens and explicitly disables the +normal client convenience that raises reasoning-model budgets. Provider count, +models per provider, request timeout, and the overall protected job are all +bounded independently. + +## Lifecycle and artifacts + +Pinned cache actions restore the preceding sanitized report when available. +The sentinel carries forward only validated timestamps and bounded counters for +matching packaged provider/model identities. Invalid, oversized, stale-schema, +or missing state is ignored. + +Each successful run uploads its current JSON and Markdown workflow artifact +with 30-day retention. Cache loss or artifact expiry resets counters but cannot +change routing. Treat the artifact and generated issue as leads—not proof that +a model is free, healthy, or retired. + +For a local public run: + +```bash +python3 scripts/catalog_sentinel.py discover \ + --output /tmp/catalog-sentinel.json \ + --summary /tmp/catalog-sentinel.md +``` + +For a local protected probe, export the JSON secret map and choose explicit +bounds: + +```bash +python3 scripts/catalog_sentinel.py probe \ + --output /tmp/catalog-probes.json \ + --summary /tmp/catalog-probes.md \ + --max-providers 8 \ + --max-models-per-provider 1 +``` + +Inspect the workflow artifact, reproduce any candidate with +`scripts/vet_catalog.py`, and follow the catalog rules in +[`CONTRIBUTING.md`](../CONTRIBUTING.md). diff --git a/scripts/catalog_sentinel.py b/scripts/catalog_sentinel.py new file mode 100644 index 0000000..c375f4e --- /dev/null +++ b/scripts/catalog_sentinel.py @@ -0,0 +1,748 @@ +#!/usr/bin/env python3 +"""Advisory-only catalog drift discovery and bounded completion canaries.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import re +import sys +from collections.abc import Callable +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import httpx + +ROOT = Path(__file__).resolve().parent.parent +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from freellmpool import client as flp_client # noqa: E402 +from freellmpool.config import load_catalog # noqa: E402 +from freellmpool.errors import ProviderHTTPError # noqa: E402 +from freellmpool.models import Provider # noqa: E402 + +_MAX_BODY_BYTES = 1_000_000 +_MAX_MODEL_ID = 200 +_MAX_SECRET_JSON_BYTES = 32_000 +_MAX_PREVIOUS_BYTES = 2_000_000 +_MAX_ISSUE_BODY_BYTES = 60_000 +_PING = [{"role": "user", "content": "Reply with the single word: pong"}] +_SAFE_MODEL_ID = re.compile(r"^[A-Za-z0-9@][A-Za-z0-9._:/@+-]{0,199}$") +# Only endpoints verified to expose a complete public chat-model listing belong +# here. Everything else is useful for additions, but absences remain unconfirmed. +_AUTHORITATIVE_PUBLIC_LISTINGS = frozenset({"pollinations"}) + + +def _timestamp(value: datetime) -> str: + return value.astimezone(UTC).replace(microsecond=0).isoformat().replace( + "+00:00", "Z" + ) + + +def classify_http(status: int | None) -> str: + if status == 200: + return "ok" + if status in {401, 403}: + return "auth_required" + if status == 402: + return "billing_or_credit" + if status == 404: + return "listing_unsupported" + if status == 429: + return "rate_limited" + if status is None: + return "network_or_timeout" + if 500 <= status < 600: + return "transient_provider_error" + return "other_provider_error" + + +def normalize_model_listing(payload: Any) -> tuple[str, ...]: + rows = payload.get("data") if isinstance(payload, dict) else payload + if not isinstance(rows, list): + return () + found: set[str] = set() + for row in rows: + candidates: list[Any] + if isinstance(row, dict): + candidates = [row.get("id") or row.get("name")] + aliases = row.get("aliases") + if isinstance(aliases, list): + candidates.extend(aliases) + else: + candidates = [row] + for model_id in candidates: + if not isinstance(model_id, str): + continue + if ( + not 1 <= len(model_id) <= _MAX_MODEL_ID + or not _SAFE_MODEL_ID.fullmatch(model_id) + ): + continue + found.add(model_id) + return tuple(sorted(found)) + + +def _public_models_url(provider: Provider) -> str: + if provider.id == "pollinations": + return "https://text.pollinations.ai/models" + return f"{provider.base_url.rstrip('/')}/models" + + +async def _bounded_json_get_async( + url: str, + *, + timeout: float, + max_bytes: int, +) -> tuple[int, Any]: + async with httpx.AsyncClient( + follow_redirects=False, + timeout=httpx.Timeout( + timeout, + connect=min(timeout, 3.0), + read=min(timeout, 2.0), + write=min(timeout, 2.0), + pool=min(timeout, 2.0), + ), + headers={"User-Agent": "freellmpool-catalog-sentinel/1"}, + ) as client: + async with client.stream("GET", url) as response: + chunks: list[bytes] = [] + size = 0 + async for chunk in response.aiter_bytes(): + size += len(chunk) + if size > max_bytes: + return response.status_code, None + chunks.append(chunk) + raw = b"".join(chunks) + if not raw: + return response.status_code, None + try: + return response.status_code, json.loads(raw) + except (UnicodeError, json.JSONDecodeError): + return response.status_code, None + + +def _bounded_json_get( + url: str, + *, + timeout: float, + max_bytes: int, +) -> tuple[int | None, Any]: + try: + return asyncio.run( + asyncio.wait_for( + _bounded_json_get_async( + url, + timeout=timeout, + max_bytes=max_bytes, + ), + timeout=timeout, + ) + ) + except (TimeoutError, httpx.HTTPError): + return None, None + + +def discovery_record( + provider: Provider, + *, + status: int | None, + payload: Any, + observed_at: datetime, + listing_authoritative: bool = False, + previous: dict[str, Any] | None = None, +) -> dict[str, Any]: + live = normalize_model_listing(payload) if status == 200 else () + classification = ( + "invalid_or_empty_listing" + if status == 200 and not live + else classify_http(status) + ) + catalog = {model.name for model in provider.models} + listing_complete = status == 200 and bool(live) and listing_authoritative + absences = sorted(catalog - set(live)) if status == 200 and live else [] + timestamp = _timestamp(observed_at) + first_discovered = _safe_timestamp((previous or {}).get("first_discovered")) or timestamp + discovery_count = _bounded_count((previous or {}).get("discovery_count")) + 1 + return { + "provider": provider.id, + "label": provider.label, + "failure_classification": classification, + "status": status, + "listing_complete": listing_complete, + "listing_scope": "authoritative" if listing_complete else "partial_or_unknown", + "live_model_count": len(live), + "observed_models": list(live), + "catalog_gaps": sorted(set(live) - catalog) if live else [], + "catalog_unlisted_models": absences, + "new_models": sorted(set(live) - catalog) if live else [], + "removed_models": absences if listing_complete else [], + "unconfirmed_absences": [] if listing_complete else absences, + "recovered_models": [], + "absence_streaks": {}, + "repeated_absences": [], + "absence_threshold_crossed": [], + "baseline_initialized": bool(live), + # Discovery is evidence for maintainer review, never a routing mutation or + # retirement recommendation. Completion canaries provide separate evidence. + "retirement_candidates": [], + "first_discovered": first_discovered, + "last_discovered": timestamp, + "discovery_count": discovery_count, + "last_verified": None, + "verification_count": 0, + "free_tier_kind": "unknown", + "billing_risk": "review_required", + "region_privacy_notes": "not_recorded", + "advisory_only": True, + } + + +def probe_record( + provider: Provider, + model: str, + *, + ok: bool, + status: int | None, + observed_at: datetime, + previous: dict[str, Any] | None = None, + failure_classification: str | None = None, +) -> dict[str, Any]: + timestamp = _timestamp(observed_at) + first_verified = _safe_timestamp((previous or {}).get("first_verified")) or timestamp + prior_failures = _bounded_count((previous or {}).get("consecutive_failures")) + consecutive_failures = 0 if ok else min(1_000_000_000, prior_failures + 1) + return { + "provider": provider.id, + "model": model, + "ok": ok, + "status": status, + "failure_classification": ( + "ok" if ok else failure_classification or classify_http(status) + ), + "first_verified": first_verified, + "last_verified": timestamp, + "last_successful_verification": ( + timestamp + if ok + else _safe_timestamp((previous or {}).get("last_successful_verification")) + ), + "verification_count": _bounded_count((previous or {}).get("verification_count")) + 1, + "consecutive_failures": consecutive_failures, + "repeated_failure": not ok and consecutive_failures >= 2, + "failure_threshold_crossed": not ok and consecutive_failures == 2, + "recovered": ok and prior_failures > 0, + "retirement_candidate": False, + "advisory_only": True, + } + + +def _safe_timestamp(value: Any) -> str | None: + if not isinstance(value, str) or len(value) > 32: + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + return None + return _timestamp(parsed) + + +def _bounded_count(value: Any) -> int: + if isinstance(value, bool) or not isinstance(value, int): + return 0 + return max(0, min(value, 1_000_000_000)) + + +def _safe_model_set(value: Any) -> set[str] | None: + if not isinstance(value, list): + return None + return set(normalize_model_listing(value)) + + +def _safe_absence_streaks(value: Any) -> dict[str, int]: + if not isinstance(value, dict): + return {} + result: dict[str, int] = {} + for model, count in value.items(): + normalized = normalize_model_listing([model]) + if len(normalized) == 1: + result[normalized[0]] = _bounded_count(count) + return result + + +def _apply_discovery_lifecycle( + record: dict[str, Any], + previous: dict[str, Any] | None, +) -> None: + prior_models = _safe_model_set((previous or {}).get("observed_models")) + prior_streaks = _safe_absence_streaks((previous or {}).get("absence_streaks")) + current_models = set(record["observed_models"]) + listing_worked = record["status"] == 200 and bool(current_models) + + if not listing_worked: + record["observed_models"] = sorted(prior_models or ()) + record["absence_streaks"] = prior_streaks + record["baseline_initialized"] = prior_models is not None + record["new_models"] = [] + record["removed_models"] = [] + record["unconfirmed_absences"] = [] + record["recovered_models"] = [] + record["repeated_absences"] = sorted( + model for model, count in prior_streaks.items() if count >= 2 + ) + record["absence_threshold_crossed"] = [] + return + + record["baseline_initialized"] = True + if prior_models is None: + # The first successful observation establishes state. Catalog gaps stay + # visible in the artifact but are not mislabeled as newly free routes. + if not record["listing_complete"]: + record["new_models"] = [] + record["removed_models"] = [] + record["unconfirmed_absences"] = [] + record["recovered_models"] = [] + record["absence_streaks"] = {} + record["repeated_absences"] = [] + record["absence_threshold_crossed"] = [] + return + + tracked_prior = prior_models | set(prior_streaks) + absent = sorted(tracked_prior - current_models) + recovered = sorted(current_models & set(prior_streaks)) + added = sorted((current_models - prior_models) - set(recovered)) + streaks = { + model: min(1_000_000_000, prior_streaks.get(model, 0) + 1) + for model in absent + } + record["new_models"] = added + record["removed_models"] = absent if record["listing_complete"] else [] + record["unconfirmed_absences"] = [] if record["listing_complete"] else absent + record["recovered_models"] = recovered + record["absence_streaks"] = streaks + record["repeated_absences"] = sorted( + model for model, count in streaks.items() if count >= 2 + ) + record["absence_threshold_crossed"] = sorted( + model + for model, count in streaks.items() + if count == 2 and prior_streaks.get(model, 0) < 2 + ) + + +def _previous_rows( + previous: dict[str, Any] | None, + *, + mode: str, + collection: str, + keys: tuple[str, ...], +) -> dict[tuple[str, ...], dict[str, Any]]: + if ( + not isinstance(previous, dict) + or previous.get("schema_version") != 1 + or previous.get("mode") != mode + ): + return {} + rows = previous.get(collection) + if not isinstance(rows, list): + return {} + result: dict[tuple[str, ...], dict[str, Any]] = {} + for row in rows: + if not isinstance(row, dict): + continue + identity: list[str] = [] + for key in keys: + value = row.get(key) + if not isinstance(value, str): + break + identity.append(value) + else: + result[tuple(identity)] = row + return result + + +def load_previous(path: Path | None) -> dict[str, Any] | None: + if path is None: + return None + try: + raw = path.read_bytes() + except OSError: + return None + if len(raw) > _MAX_PREVIOUS_BYTES: + return None + try: + value = json.loads(raw) + except (UnicodeError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def load_secret_map( + env: dict[str, str] | None = None, + *, + variable: str = "FREELLMPOOL_SENTINEL_KEYS_JSON", +) -> dict[str, str]: + source = env if env is not None else os.environ + raw = source.get(variable, "") + if not raw: + raise ValueError(f"{variable} is required for authenticated probes") + if len(raw.encode("utf-8")) > _MAX_SECRET_JSON_BYTES: + raise ValueError("secret map exceeds size limit") + try: + payload = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError("secret map must be valid JSON") from exc + if not isinstance(payload, dict): + raise ValueError("secret map must be an object") + result: dict[str, str] = {} + for key, value in payload.items(): + if ( + not isinstance(key, str) + or not isinstance(value, str) + or not key + or not value + or len(key) > 128 + or len(value) > 8_192 + ): + raise ValueError("secret map must contain bounded non-empty strings") + result[key] = value + return result + + +def discover( + providers: list[Provider], + *, + timeout: float, + max_bytes: int, + now: datetime, + previous: dict[str, Any] | None = None, + fetch: Callable[..., tuple[int | None, Any]] = _bounded_json_get, +) -> dict[str, Any]: + prior = _previous_rows( + previous, + mode="public_discovery", + collection="providers", + keys=("provider",), + ) + records: list[dict[str, Any]] = [] + for provider in providers: + try: + status, payload = fetch( + _public_models_url(provider), + timeout=timeout, + max_bytes=max_bytes, + ) + except Exception: # noqa: BLE001 - isolate one discovery endpoint + status, payload = None, None + previous_record = prior.get((provider.id,)) + record = discovery_record( + provider, + status=status, + payload=payload, + observed_at=now, + listing_authoritative=provider.id in _AUTHORITATIVE_PUBLIC_LISTINGS, + previous=previous_record, + ) + _apply_discovery_lifecycle(record, previous_record) + records.append(record) + has_changes = any( + record["new_models"] + or record["removed_models"] + or record["recovered_models"] + or record["absence_threshold_crossed"] + for record in records + ) + return { + "schema_version": 1, + "mode": "public_discovery", + "generated_at": _timestamp(now), + "advisory_only": True, + "providers": records, + "drift": {"has_changes": has_changes}, + } + + +def probe( + providers: list[Provider], + secrets: dict[str, str], + *, + timeout: float, + max_providers: int, + max_models_per_provider: int, + now: datetime, + previous: dict[str, Any] | None = None, +) -> dict[str, Any]: + records: list[dict[str, Any]] = [] + prior = _previous_rows( + previous, + mode="authenticated_probe", + collection="probes", + keys=("provider", "model"), + ) + selected = [ + provider + for provider in providers + if provider.is_configured(secrets) + ][:max_providers] + for provider in selected: + models = [model for model in provider.models if model.enabled][ + :max_models_per_provider + ] + for model in models: + try: + reply = flp_client.call( + provider, + model.name, + _PING, + api_key=provider.api_key(secrets), + env=secrets, + max_tokens=8, + temperature=0.0, + timeout=timeout, + enforce_thinking_floor=False, + ) + ok = bool(reply.text.strip()) + status: int | None = 200 + failure_classification = None if ok else "empty_completion" + except ProviderHTTPError as exc: + ok = False + status = exc.status + failure_classification = None + except (httpx.HTTPError, OSError): + ok = False + status = None + failure_classification = None + except Exception: # noqa: BLE001 - never serialize provider exception text + ok = False + status = None + failure_classification = "unexpected_probe_error" + records.append( + probe_record( + provider, + model.name, + ok=ok, + status=status, + observed_at=now, + previous=prior.get((provider.id, model.name)), + failure_classification=failure_classification, + ) + ) + report = { + "schema_version": 1, + "mode": "authenticated_probe", + "generated_at": _timestamp(now), + "advisory_only": True, + "bounds": { + "max_providers": max_providers, + "max_models_per_provider": max_models_per_provider, + }, + "probes": records, + } + report["drift"] = { + "has_changes": any( + record["recovered"] or record["failure_threshold_crossed"] + for record in records + ) + } + return report + + +def render_summary(report: dict[str, Any]) -> str: + lines = [ + "# Catalog sentinel", + "", + "Advisory only. Never mutates providers.toml or routing state.", + "", + ] + if report["mode"] == "public_discovery": + for record in report["providers"]: + lines.append( + f"- `{record['provider']}`: {record['failure_classification']}; " + f"{len(record['new_models'])} new, {len(record['removed_models'])} removed, " + f"{len(record.get('recovered_models', []))} recovered, " + f"{len(record.get('repeated_absences', []))} repeatedly absent" + ) + else: + ok = sum(1 for record in report["probes"] if record["ok"]) + lines.append(f"- bounded probes passing: {ok}/{len(report['probes'])}") + return "\n".join(lines) + "\n" + + +def _write_outputs(report: dict[str, Any], output: Path, summary: Path) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + summary.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + summary.write_text(render_summary(report), encoding="utf-8") + + +def _inline_code(value: str) -> str: + return f"`{value.replace('`', '')}`" + + +def write_issue_body(report: dict[str, Any], output: Path) -> bool: + drift = report.get("drift") if isinstance(report, dict) else None + if not isinstance(drift, dict) or not drift.get("has_changes"): + output.unlink(missing_ok=True) + return False + mode = report.get("mode") + heading = ( + "Catalog sentinel probe findings" + if mode == "authenticated_probe" + else "Catalog sentinel drift" + ) + marker = ( + "" + if mode == "authenticated_probe" + else "" + ) + lines = [ + marker, + "", + f"## {heading}", + "", + f"Generated at {_inline_code(str(report.get('generated_at') or 'unknown'))}.", + "", + "Advisory only. This report never mutates `providers.toml` or routing state.", + "Every catalog change requires maintainer review and live verification.", + "", + ] + if mode == "authenticated_probe": + for record in report.get("probes", []): + if not isinstance(record, dict): + continue + target = f"{record.get('provider')}/{record.get('model')}" + if record.get("failure_threshold_crossed"): + classification = str( + record.get("failure_classification") or "unknown" + ) + lines.append( + "- repeatedly failed " + f"({_inline_code(classification)}; not retirement evidence): " + f"{_inline_code(target)}" + ) + if record.get("recovered"): + lines.append(f"- recovered after prior failures: {_inline_code(target)}") + else: + for record in report.get("providers", []): + if not isinstance(record, dict): + continue + provider = str(record.get("provider") or "") + for model in record.get("new_models", []): + lines.append(f"- new model: {_inline_code(f'{provider}/{model}')}") + for model in record.get("removed_models", []): + lines.append( + "- removed from authoritative listing: " + f"{_inline_code(f'{provider}/{model}')}" + ) + for model in record.get("unconfirmed_absences", []): + lines.append( + "- unconfirmed absence (partial/unknown listing): " + f"{_inline_code(f'{provider}/{model}')}" + ) + for model in record.get("absence_threshold_crossed", []): + lines.append( + "- repeatedly absent, still not retirement evidence: " + f"{_inline_code(f'{provider}/{model}')}" + ) + for model in record.get("recovered_models", []): + lines.append( + f"- recovered in public listing: {_inline_code(f'{provider}/{model}')}" + ) + lines.extend( + [ + "", + "Do not enable, disable, or retire routes from this report alone.", + "", + ] + ) + body = "\n".join(lines) + encoded = body.encode("utf-8") + if len(encoded) > _MAX_ISSUE_BODY_BYTES: + suffix = "\n\n_Report truncated; inspect the workflow artifact for the bounded full report._\n" + body = encoded[: _MAX_ISSUE_BODY_BYTES - len(suffix.encode("utf-8"))].decode( + "utf-8", errors="ignore" + ) + suffix + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(body, encoding="utf-8") + return True + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + discovery = subparsers.add_parser("discover") + discovery.add_argument("--output", type=Path, required=True) + discovery.add_argument("--summary", type=Path, required=True) + discovery.add_argument("--timeout", type=float, default=10.0) + discovery.add_argument("--max-bytes", type=int, default=_MAX_BODY_BYTES) + discovery.add_argument("--previous", type=Path) + + canary = subparsers.add_parser("probe") + canary.add_argument("--output", type=Path, required=True) + canary.add_argument("--summary", type=Path, required=True) + canary.add_argument("--timeout", type=float, default=20.0) + canary.add_argument("--max-providers", type=int, default=8) + canary.add_argument("--max-models-per-provider", type=int, default=1) + canary.add_argument("--previous", type=Path) + + issue = subparsers.add_parser("issue-body") + issue.add_argument("--report", type=Path, required=True) + issue.add_argument("--output", type=Path, required=True) + + args = parser.parse_args(argv) + now = datetime.now(UTC) + if args.command == "issue-body": + report = load_previous(args.report) + if report is None: + parser.error("report must be bounded valid JSON") + return 0 if write_issue_body(report, args.output) else 3 + + # Scheduled automation must use only the reviewed, packaged catalog. Loading + # a user override here would turn a local URL into a workflow SSRF target. + providers = load_catalog(path=SRC / "freellmpool" / "providers.toml") + previous = load_previous(args.previous) + if args.command == "discover": + if not 0 < args.timeout <= 30 or not 1 <= args.max_bytes <= _MAX_BODY_BYTES: + parser.error("discovery bounds are out of range") + report = discover( + providers, + timeout=args.timeout, + max_bytes=args.max_bytes, + now=now, + previous=previous, + ) + else: + if ( + not 0 < args.timeout <= 60 + or not 1 <= args.max_providers <= 24 + or not 1 <= args.max_models_per_provider <= 2 + ): + parser.error("probe bounds are out of range") + try: + secrets = load_secret_map() + except ValueError as exc: + parser.error(str(exc)) + report = probe( + providers, + secrets, + timeout=args.timeout, + max_providers=args.max_providers, + max_models_per_provider=args.max_models_per_provider, + now=now, + previous=previous, + ) + _write_outputs(report, args.output, args.summary) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/freellmpool/client.py b/src/freellmpool/client.py index 6e2a877..1c7b734 100644 --- a/src/freellmpool/client.py +++ b/src/freellmpool/client.py @@ -562,14 +562,17 @@ def call( timeout: float = 90.0, tools: list | None = None, tool_choice=None, + enforce_thinking_floor: bool = True, post: PostFn = default_post, ) -> Reply: """Dispatch one completion to ``provider`` and normalize the response. Routes through the adapter named by ``provider.adapter`` (built-in or plugin-registered). Raises :class:`ProviderHTTPError` on a non-200 status. + Strictly quota-bounded maintenance probes may set + ``enforce_thinking_floor=False``; normal callers retain reasoning headroom. """ - if _is_thinking(model) and max_tokens < _THINKING_FLOOR: + if enforce_thinking_floor and _is_thinking(model) and max_tokens < _THINKING_FLOOR: # Give reasoning models room so hidden reasoning doesn't eat the whole # budget and return empty content. max_tokens = _THINKING_FLOOR diff --git a/tests/test_catalog_sentinel.py b/tests/test_catalog_sentinel.py new file mode 100644 index 0000000..d2f7cdc --- /dev/null +++ b/tests/test_catalog_sentinel.py @@ -0,0 +1,729 @@ +from __future__ import annotations + +import importlib.util +import json +import socketserver +import sys +import threading +import time +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +from freellmpool.errors import ProviderHTTPError +from freellmpool.models import Model, Provider, Reply + +ROOT = Path(__file__).resolve().parent.parent +SPEC = importlib.util.spec_from_file_location( + "catalog_sentinel", + ROOT / "scripts" / "catalog_sentinel.py", +) +assert SPEC is not None and SPEC.loader is not None +SENTINEL = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = SENTINEL +SPEC.loader.exec_module(SENTINEL) + + +def _provider() -> Provider: + return Provider( + id="example", + label="Example", + adapter="openai", + base_url="https://example.test/v1", + key_env="EXAMPLE_KEY", + models=( + Model("kept", enabled=True), + Model("missing", enabled=True), + ), + ) + + +@pytest.mark.parametrize( + ("status", "expected"), + [ + (200, "ok"), + (401, "auth_required"), + (402, "billing_or_credit"), + (403, "auth_required"), + (404, "listing_unsupported"), + (429, "rate_limited"), + (500, "transient_provider_error"), + ], +) +def test_http_failures_are_classified_without_retirement(status, expected): + assert SENTINEL.classify_http(status) == expected + + +def test_model_listing_normalization_is_bounded_and_strict(): + payload = { + "data": [ + {"id": "model-a"}, + {"id": "model-a"}, + {"id": "model/b:free"}, + {"id": "bad\nmodel"}, + {"id": "bad`model"}, + {"id": ""}, + {"id": "x" * 201}, + {"wrong": "ignored"}, + ] + } + + assert SENTINEL.normalize_model_listing(payload) == ( + "model-a", + "model/b:free", + ) + assert SENTINEL.normalize_model_listing({"data": "not-a-list"}) == () + assert SENTINEL.normalize_model_listing(["model-c", {"id": "model-d"}]) == ( + "model-c", + "model-d", + ) + + +def test_pollinations_listing_normalizes_name_and_aliases(): + payload = [ + { + "name": "openai-fast", + "aliases": [ + "openai", + "gpt-oss", + "bad`alias", + "x" * 201, + 42, + ], + } + ] + + assert SENTINEL.normalize_model_listing(payload) == ( + "gpt-oss", + "openai", + "openai-fast", + ) + + +def test_transient_or_incomplete_listing_never_recommends_retirement(): + now = datetime(2026, 7, 29, 12, tzinfo=UTC) + provider = _provider() + + transient = SENTINEL.discovery_record( + provider, + status=429, + payload={}, + observed_at=now, + ) + incomplete = SENTINEL.discovery_record( + provider, + status=200, + payload={"data": []}, + observed_at=now, + ) + + assert transient["failure_classification"] == "rate_limited" + assert transient["removed_models"] == [] + assert transient["retirement_candidates"] == [] + assert incomplete["listing_complete"] is False + assert incomplete["failure_classification"] == "invalid_or_empty_listing" + assert incomplete["removed_models"] == [] + assert incomplete["retirement_candidates"] == [] + + +def test_successful_listing_reports_drift_but_never_mutates_catalog(): + now = datetime(2026, 7, 29, 12, tzinfo=UTC) + provider = _provider() + + record = SENTINEL.discovery_record( + provider, + status=200, + payload={"data": [{"id": "kept"}, {"id": "new-model"}]}, + observed_at=now, + listing_authoritative=True, + ) + + assert record["new_models"] == ["new-model"] + assert record["removed_models"] == ["missing"] + assert record["retirement_candidates"] == [] + assert record["advisory_only"] is True + assert record["last_discovered"] == "2026-07-29T12:00:00Z" + assert record["verification_count"] == 0 + assert record["free_tier_kind"] == "unknown" + assert record["billing_risk"] == "review_required" + + +def test_partial_listing_reports_unconfirmed_absence_not_removal(): + now = datetime(2026, 7, 29, 12, tzinfo=UTC) + provider = _provider() + + record = SENTINEL.discovery_record( + provider, + status=200, + payload={"data": [{"id": "kept"}, {"id": "new-model"}]}, + observed_at=now, + listing_authoritative=False, + ) + + assert record["listing_complete"] is False + assert record["listing_scope"] == "partial_or_unknown" + assert record["new_models"] == ["new-model"] + assert record["removed_models"] == [] + assert record["unconfirmed_absences"] == ["missing"] + assert record["retirement_candidates"] == [] + + +def test_discovery_lifecycle_merges_only_matching_previous_records(): + provider = _provider() + previous = { + "schema_version": 1, + "mode": "public_discovery", + "providers": [ + { + "provider": "example", + "first_discovered": "2026-07-22T12:00:00Z", + "last_discovered": "2026-07-22T12:00:00Z", + "discovery_count": 4, + }, + { + "provider": "other", + "first_discovered": "secret-account-id", + "discovery_count": 999, + }, + ], + } + + report = SENTINEL.discover( + [provider], + timeout=1, + max_bytes=1024, + now=datetime(2026, 7, 29, 12, tzinfo=UTC), + previous=previous, + fetch=lambda _url, **_kwargs: (200, {"data": [{"id": "kept"}]}), + ) + + record = report["providers"][0] + assert record["first_discovered"] == "2026-07-22T12:00:00Z" + assert record["last_discovered"] == "2026-07-29T12:00:00Z" + assert record["discovery_count"] == 5 + assert record["baseline_initialized"] is True + assert record["new_models"] == [] + assert record["catalog_gaps"] == [] + assert "secret-account-id" not in json.dumps(report) + + +def test_discovery_alerts_on_changes_after_baseline_not_all_catalog_gaps(): + provider = _provider() + first = SENTINEL.discover( + [provider], + timeout=1, + max_bytes=1024, + now=datetime(2026, 7, 22, 12, tzinfo=UTC), + fetch=lambda _url, **_kwargs: ( + 200, + {"data": [{"id": "kept"}, {"id": "candidate-a"}]}, + ), + ) + + assert first["drift"]["has_changes"] is False + assert first["providers"][0]["new_models"] == [] + assert first["providers"][0]["catalog_gaps"] == ["candidate-a"] + assert first["providers"][0]["observed_models"] == ["candidate-a", "kept"] + + second = SENTINEL.discover( + [provider], + timeout=1, + max_bytes=1024, + now=datetime(2026, 7, 29, 12, tzinfo=UTC), + previous=first, + fetch=lambda _url, **_kwargs: ( + 200, + {"data": [{"id": "kept"}, {"id": "candidate-a"}, {"id": "candidate-b"}]}, + ), + ) + + assert second["drift"]["has_changes"] is True + assert second["providers"][0]["new_models"] == ["candidate-b"] + assert second["providers"][0]["catalog_gaps"] == [ + "candidate-a", + "candidate-b", + ] + + +def test_authoritative_first_baseline_reports_catalog_additions_and_removals(): + base = _provider() + provider = Provider( + id="pollinations", + label=base.label, + adapter=base.adapter, + base_url=base.base_url, + key_env=base.key_env, + models=base.models, + ) + + report = SENTINEL.discover( + [provider], + timeout=1, + max_bytes=1024, + now=datetime(2026, 7, 29, 12, tzinfo=UTC), + fetch=lambda _url, **_kwargs: ( + 200, + [{"name": "kept", "aliases": ["new-model"]}], + ), + ) + + row = report["providers"][0] + assert row["listing_complete"] is True + assert row["new_models"] == ["new-model"] + assert row["removed_models"] == ["missing"] + assert report["drift"]["has_changes"] is True + + +def test_partial_listing_tracks_repeated_absence_and_recovery_without_retirement(): + provider = _provider() + baseline = { + "schema_version": 1, + "mode": "public_discovery", + "providers": [ + { + "provider": "example", + "observed_models": ["kept", "candidate-a"], + "absence_streaks": {}, + } + ], + } + + first_absence = SENTINEL.discover( + [provider], + timeout=1, + max_bytes=1024, + now=datetime(2026, 7, 22, 12, tzinfo=UTC), + previous=baseline, + fetch=lambda _url, **_kwargs: (200, {"data": [{"id": "kept"}]}), + ) + row = first_absence["providers"][0] + assert row["removed_models"] == [] + assert row["unconfirmed_absences"] == ["candidate-a"] + assert row["absence_streaks"] == {"candidate-a": 1} + assert row["repeated_absences"] == [] + assert first_absence["drift"]["has_changes"] is False + + second_absence = SENTINEL.discover( + [provider], + timeout=1, + max_bytes=1024, + now=datetime(2026, 7, 29, 12, tzinfo=UTC), + previous=first_absence, + fetch=lambda _url, **_kwargs: (200, {"data": [{"id": "kept"}]}), + ) + row = second_absence["providers"][0] + assert row["absence_streaks"] == {"candidate-a": 2} + assert row["repeated_absences"] == ["candidate-a"] + assert second_absence["drift"]["has_changes"] is True + assert row["retirement_candidates"] == [] + + recovered = SENTINEL.discover( + [provider], + timeout=1, + max_bytes=1024, + now=datetime(2026, 8, 5, 12, tzinfo=UTC), + previous=second_absence, + fetch=lambda _url, **_kwargs: ( + 200, + {"data": [{"id": "kept"}, {"id": "candidate-a"}]}, + ), + ) + row = recovered["providers"][0] + assert row["recovered_models"] == ["candidate-a"] + assert row["absence_streaks"] == {} + assert recovered["drift"]["has_changes"] is True + + +def test_probe_lifecycle_increments_and_unexpected_errors_are_sanitized(monkeypatch): + provider = _provider() + previous = { + "schema_version": 1, + "mode": "authenticated_probe", + "probes": [ + { + "provider": "example", + "model": "kept", + "first_verified": "2026-07-22T12:00:00Z", + "verification_count": 2, + } + ], + } + + def fail(*_args, **_kwargs): + raise RuntimeError("secret-provider-response account-123") + + monkeypatch.setattr(SENTINEL.flp_client, "call", fail) + report = SENTINEL.probe( + [provider], + {"EXAMPLE_KEY": "super-secret-provider-key"}, + timeout=1, + max_providers=1, + max_models_per_provider=1, + now=datetime(2026, 7, 29, 12, tzinfo=UTC), + previous=previous, + ) + + record = report["probes"][0] + assert record["first_verified"] == "2026-07-22T12:00:00Z" + assert record["verification_count"] == 3 + assert record["failure_classification"] == "unexpected_probe_error" + serialized = json.dumps(report) + assert "super-secret-provider-key" not in serialized + assert "secret-provider-response" not in serialized + assert "account-123" not in serialized + + +def test_probe_repeated_failure_and_recovery_are_advisory_drift(monkeypatch): + provider = _provider() + + def rate_limited(*_args, **_kwargs): + raise ProviderHTTPError(429, "quota response must not leak", retryable=True) + + monkeypatch.setattr(SENTINEL.flp_client, "call", rate_limited) + first = SENTINEL.probe( + [provider], + {"EXAMPLE_KEY": "secret"}, + timeout=1, + max_providers=1, + max_models_per_provider=1, + now=datetime(2026, 7, 22, 12, tzinfo=UTC), + ) + row = first["probes"][0] + assert row["consecutive_failures"] == 1 + assert row["repeated_failure"] is False + assert row["retirement_candidate"] is False + assert first["drift"]["has_changes"] is False + + second = SENTINEL.probe( + [provider], + {"EXAMPLE_KEY": "secret"}, + timeout=1, + max_providers=1, + max_models_per_provider=1, + now=datetime(2026, 7, 29, 12, tzinfo=UTC), + previous=first, + ) + row = second["probes"][0] + assert row["consecutive_failures"] == 2 + assert row["repeated_failure"] is True + assert row["failure_threshold_crossed"] is True + assert row["failure_classification"] == "rate_limited" + assert row["retirement_candidate"] is False + assert second["drift"]["has_changes"] is True + + monkeypatch.setattr( + SENTINEL.flp_client, + "call", + lambda *_args, **_kwargs: Reply("pong", "example", "kept", {}), + ) + recovered = SENTINEL.probe( + [provider], + {"EXAMPLE_KEY": "secret"}, + timeout=1, + max_providers=1, + max_models_per_provider=1, + now=datetime(2026, 8, 5, 12, tzinfo=UTC), + previous=second, + ) + row = recovered["probes"][0] + assert row["consecutive_failures"] == 0 + assert row["recovered"] is True + assert recovered["drift"]["has_changes"] is True + + +def test_probe_empty_http_success_has_non_success_classification(monkeypatch): + provider = _provider() + monkeypatch.setattr( + SENTINEL.flp_client, + "call", + lambda *_args, **_kwargs: Reply(" \n\t", "example", "kept", {}), + ) + + report = SENTINEL.probe( + [provider], + {"EXAMPLE_KEY": "secret"}, + timeout=1, + max_providers=1, + max_models_per_provider=1, + now=datetime(2026, 7, 29, 12, tzinfo=UTC), + ) + + row = report["probes"][0] + assert row["ok"] is False + assert row["status"] == 200 + assert row["failure_classification"] == "empty_completion" + + +def test_listing_failure_after_threshold_does_not_repeat_drift_event(): + provider = _provider() + previous = { + "schema_version": 1, + "mode": "public_discovery", + "providers": [ + { + "provider": "example", + "observed_models": ["kept"], + "absence_streaks": {"candidate-a": 2}, + } + ], + } + + report = SENTINEL.discover( + [provider], + timeout=1, + max_bytes=1024, + now=datetime(2026, 8, 5, 12, tzinfo=UTC), + previous=previous, + fetch=lambda _url, **_kwargs: (429, None), + ) + + row = report["providers"][0] + assert row["absence_streaks"] == {"candidate-a": 2} + assert row["absence_threshold_crossed"] == [] + assert report["drift"]["has_changes"] is False + + +def test_bounded_get_uses_short_transport_timeouts(monkeypatch): + seen: dict[str, object] = {} + + class Response: + status_code = 200 + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return False + + async def aiter_bytes(self): + for chunk in (b'{"data":', b"[]", b"}"): + yield chunk + + class Client: + def __init__(self, **kwargs): + seen.update(kwargs) + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return False + + def stream(self, *_args, **_kwargs): + return Response() + + monkeypatch.setattr(SENTINEL.httpx, "AsyncClient", Client) + + status, payload = SENTINEL._bounded_json_get( + "https://example.test/models", + timeout=1.0, + max_bytes=1024, + ) + + assert status == 200 + assert payload == {"data": []} + assert seen["timeout"].read <= 1.0 + + +def test_bounded_get_total_deadline_covers_slow_response_headers(): + class SlowHeaderHandler(socketserver.BaseRequestHandler): + def handle(self): + self.request.recv(4096) + self.request.sendall(b"HTTP/1.1 200 OK\r\nX-Slow: ") + for _ in range(20): + time.sleep(0.05) + try: + self.request.sendall(b"a") + except OSError: + break + + class Server(socketserver.ThreadingTCPServer): + allow_reuse_address = True + daemon_threads = True + + server = Server(("127.0.0.1", 0), SlowHeaderHandler) + runner = threading.Thread(target=server.serve_forever, daemon=True) + runner.start() + started = time.monotonic() + try: + status, payload = SENTINEL._bounded_json_get( + f"http://127.0.0.1:{server.server_address[1]}/models", + timeout=0.2, + max_bytes=1024, + ) + elapsed = time.monotonic() - started + finally: + server.shutdown() + server.server_close() + + assert status is None + assert payload is None + assert elapsed < 0.8 + + +def test_issue_body_is_only_generated_for_actionable_drift(tmp_path): + report = { + "mode": "public_discovery", + "generated_at": "2026-07-29T12:00:00Z", + "advisory_only": True, + "providers": [ + { + "provider": "example", + "failure_classification": "ok", + "new_models": ["new-model"], + "removed_models": [], + "unconfirmed_absences": ["missing"], + } + ], + "drift": {"has_changes": True}, + } + output = tmp_path / "issue.md" + + assert SENTINEL.write_issue_body(report, output) is True + body = output.read_text(encoding="utf-8") + assert "Advisory only" in body + assert "`example/new-model`" in body + assert "unconfirmed absence" in body + assert "providers.toml" in body + assert "" in body + + report["drift"]["has_changes"] = False + assert SENTINEL.write_issue_body(report, output) is False + assert not output.exists() + + +def test_probe_issue_body_contains_classification_but_no_response_content(tmp_path): + report = { + "mode": "authenticated_probe", + "generated_at": "2026-07-29T12:00:00Z", + "advisory_only": True, + "probes": [ + { + "provider": "example", + "model": "kept", + "failure_classification": "billing_or_credit", + "repeated_failure": True, + "failure_threshold_crossed": True, + "recovered": False, + "retirement_candidate": False, + } + ], + "drift": {"has_changes": True}, + } + output = tmp_path / "probe-issue.md" + + assert SENTINEL.write_issue_body(report, output) is True + body = output.read_text(encoding="utf-8") + assert "`example/kept`" in body + assert "billing_or_credit" in body + assert "not retirement evidence" in body + assert "response" not in body.lower() + assert "" in body + + +def test_probe_report_contains_no_secret_or_response_content(): + provider = _provider() + secret = "super-secret-provider-key" + response = "account-123 private completion text" + + report = SENTINEL.probe_record( + provider, + "kept", + ok=False, + status=402, + observed_at=datetime(2026, 7, 29, 12, tzinfo=UTC), + ) + serialized = json.dumps(report) + + assert secret not in serialized + assert response not in serialized + assert report["failure_classification"] == "billing_or_credit" + assert report["last_verified"] == "2026-07-29T12:00:00Z" + assert report["verification_count"] == 1 + assert set(report) == { + "provider", + "model", + "ok", + "status", + "failure_classification", + "first_verified", + "last_verified", + "last_successful_verification", + "verification_count", + "consecutive_failures", + "repeated_failure", + "failure_threshold_crossed", + "recovered", + "retirement_candidate", + "advisory_only", + } + + +def test_secret_map_is_strict_and_bounded(monkeypatch): + monkeypatch.setenv( + "FREELLMPOOL_SENTINEL_KEYS_JSON", + json.dumps({"EXAMPLE_KEY": "secret", "CLOUDFLARE_ACCOUNT_ID": "account"}), + ) + assert SENTINEL.load_secret_map() == { + "EXAMPLE_KEY": "secret", + "CLOUDFLARE_ACCOUNT_ID": "account", + } + + monkeypatch.setenv("FREELLMPOOL_SENTINEL_KEYS_JSON", '["not", "an", "object"]') + with pytest.raises(ValueError, match="object"): + SENTINEL.load_secret_map() + + +def test_workflow_is_advisory_least_privilege_and_fork_safe(): + workflow = (ROOT / ".github" / "workflows" / "catalog-sentinel.yml").read_text( + encoding="utf-8" + ) + + assert "schedule:" in workflow + assert "workflow_dispatch:" in workflow + assert "pull_request:" not in workflow + assert "environment: catalog-sentinel" in workflow + assert "issues: write" in workflow + assert "actions: write" not in workflow + assert "contents: write" not in workflow + assert "concurrency:" in workflow + assert "timeout-minutes:" in workflow + assert "actions/cache/restore@" in workflow + assert "actions/cache/save@" in workflow + assert "actions/upload-artifact@" in workflow + assert "uses: actions/checkout@" in workflow + assert "uses: actions/setup-python@" in workflow + assert "gh issue create" in workflow + assert "gh issue comment" in workflow + assert "Catalog sentinel probe findings" in workflow + assert "github-actions[bot]" in workflow + assert "" in workflow + assert "" in workflow + assert ".author.login" in workflow + assert ".body | contains(" in workflow + assert ".title ==" in workflow + assert "--body-file" in workflow + assert "scripts/catalog_sentinel.py discover" in workflow + assert "scripts/catalog_sentinel.py probe" in workflow + assert "--previous" in workflow + assert "FREELLMPOOL_SENTINEL_KEYS_JSON" in workflow + assert "Never mutates providers.toml" in workflow + + +def test_catalog_sentinel_operator_contract_is_documented(): + doc = (ROOT / "docs" / "CATALOG_SENTINEL.md").read_text(encoding="utf-8") + contributing = (ROOT / "CONTRIBUTING.md").read_text(encoding="utf-8") + + for phrase in ( + "catalog-sentinel", + "FREELLMPOOL_SENTINEL_KEYS_JSON", + "environment protection", + "advisory", + "never enables or disables", + "429", + "402", + "workflow artifact", + ): + assert phrase in doc + assert "docs/CATALOG_SENTINEL.md" in contributing diff --git a/tests/test_client.py b/tests/test_client.py index 788b271..6917c86 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -40,6 +40,26 @@ def post(url, headers, body, timeout): assert seen["max_tokens"] >= 4096 # reasoning model got headroom +def test_thinking_model_floor_can_be_disabled_for_strictly_bounded_canary(): + seen = {} + + def post(url, headers, body, timeout): + seen.update(body) + return C.HTTPResult(200, openai_body("ok"), "ok") + + C.call( + P, + "zai-glm-4.7", + [{"role": "user", "content": "hi"}], + api_key="k", + env={}, + max_tokens=8, + enforce_thinking_floor=False, + post=post, + ) + assert seen["max_tokens"] == 8 + + def test_non_thinking_model_keeps_max_tokens(): seen = {}