feat: add advisory catalog sentinel

.github/workflows/catalog-sentinel.yml

@@ -0,0 +1,240 @@
+name: Catalog sentinel
+
+on:
+  schedule:
+    - cron: "23 6 * * 3"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: catalog-sentinel
+  cancel-in-progress: false
+
+jobs:
+  public-discovery:
+    name: Public catalog discovery
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - name: Check out source
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
+        with:
+          python-version: "3.14"
+      - name: Install runtime
+        run: python -m pip install --disable-pip-version-check .
+      - name: Restore prior lifecycle state
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: .sentinel-state/public.json
+          key: catalog-sentinel-public-${{ github.run_id }}
+          restore-keys: |
+            catalog-sentinel-public-
+      - name: Run bounded public discovery
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p .sentinel-artifacts .sentinel-state
+          previous=()
+          if test -f .sentinel-state/public.json; then
+            previous=(--previous .sentinel-state/public.json)
+          fi
+          # Advisory evidence only. Never mutates providers.toml or routing state.
+          python scripts/catalog_sentinel.py discover \
+            --output .sentinel-artifacts/public.json \
+            --summary .sentinel-artifacts/public.md \
+            --timeout 10 \
+            --max-bytes 1000000 \
+            "${previous[@]}"
+          cp .sentinel-artifacts/public.json .sentinel-state/public.json
+      - name: Prepare bounded drift issue
+        id: drift
+        shell: bash
+        run: |
+          set +e
+          python scripts/catalog_sentinel.py issue-body \
+            --report .sentinel-artifacts/public.json \
+            --output .sentinel-artifacts/issue.md
+          status=$?
+          set -e
+          if test "$status" -eq 3; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            test "$status" -eq 0
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+      - name: Open or update advisory drift issue
+        if: steps.drift.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          existing=$(gh issue list \
+            --repo "$GITHUB_REPOSITORY" \
+            --state open \
+            --search '"Catalog sentinel drift" in:title' \
+            --limit 100 \
+            --json number,title,author,body \
+            --jq 'map(select(
+              .title == "Catalog sentinel drift"
+              and (.author.login == "github-actions[bot]" or .author.login == "github-actions")
+              and (.body | contains("<!-- freellmpool-catalog-sentinel:public:v1 -->"))
+            )) | first | .number // empty')
+          if [[ "$existing" =~ ^[0-9]+$ ]]; then
+            gh issue comment "$existing" \
+              --repo "$GITHUB_REPOSITORY" \
+              --body-file .sentinel-artifacts/issue.md
+          else
+            gh issue create \
+              --repo "$GITHUB_REPOSITORY" \
+              --title "Catalog sentinel drift" \
+              --label provider-catalog \
+              --label tests \
+              --body-file .sentinel-artifacts/issue.md
+          fi
+      - name: Upload sanitized public evidence
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: catalog-sentinel-public-${{ github.run_id }}
+          path: |
+            .sentinel-artifacts/public.json
+            .sentinel-artifacts/public.md
+          if-no-files-found: error
+          retention-days: 30
+      - name: Save public lifecycle state
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: .sentinel-state/public.json
+          key: catalog-sentinel-public-${{ github.run_id }}
+
+  authenticated-probes:
+    name: Protected authenticated completion probes
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    environment: catalog-sentinel
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - name: Check out source
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
+        with:
+          python-version: "3.14"
+      - name: Install runtime
+        run: python -m pip install --disable-pip-version-check .
+      - name: Detect protected probe configuration
+        id: probe-config
+        env:
+          SENTINEL_KEYS: ${{ secrets.FREELLMPOOL_SENTINEL_KEYS_JSON }}
+        shell: bash
+        run: |
+          if test -n "$SENTINEL_KEYS"; then
+            echo "configured=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "configured=false" >> "$GITHUB_OUTPUT"
+            echo "Protected probe secret is not configured; public discovery still completed."
+          fi
+      - name: Restore prior probe lifecycle state
+        if: steps.probe-config.outputs.configured == 'true'
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: .sentinel-state/probe.json
+          key: catalog-sentinel-probe-${{ github.run_id }}
+          restore-keys: |
+            catalog-sentinel-probe-
+      - name: Run bounded authenticated probes
+        if: steps.probe-config.outputs.configured == 'true'
+        env:
+          FREELLMPOOL_SENTINEL_KEYS_JSON: ${{ secrets.FREELLMPOOL_SENTINEL_KEYS_JSON }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p .sentinel-artifacts .sentinel-state
+          previous=()
+          if test -f .sentinel-state/probe.json; then
+            previous=(--previous .sentinel-state/probe.json)
+          fi
+          python scripts/catalog_sentinel.py probe \
+            --output .sentinel-artifacts/probe.json \
+            --summary .sentinel-artifacts/probe.md \
+            --timeout 20 \
+            --max-providers 8 \
+            --max-models-per-provider 1 \
+            "${previous[@]}"
+          cp .sentinel-artifacts/probe.json .sentinel-state/probe.json
+      - name: Prepare bounded probe issue
+        if: steps.probe-config.outputs.configured == 'true'
+        id: probe-drift
+        shell: bash
+        run: |
+          set +e
+          python scripts/catalog_sentinel.py issue-body \
+            --report .sentinel-artifacts/probe.json \
+            --output .sentinel-artifacts/probe-issue.md
+          status=$?
+          set -e
+          if test "$status" -eq 3; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            test "$status" -eq 0
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+      - name: Open or update advisory probe issue
+        if: steps.probe-drift.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          existing=$(gh issue list \
+            --repo "$GITHUB_REPOSITORY" \
+            --state open \
+            --search '"Catalog sentinel probe findings" in:title' \
+            --limit 100 \
+            --json number,title,author,body \
+            --jq 'map(select(
+              .title == "Catalog sentinel probe findings"
+              and (.author.login == "github-actions[bot]" or .author.login == "github-actions")
+              and (.body | contains("<!-- freellmpool-catalog-sentinel:probe:v1 -->"))
+            )) | first | .number // empty')
+          if [[ "$existing" =~ ^[0-9]+$ ]]; then
+            gh issue comment "$existing" \
+              --repo "$GITHUB_REPOSITORY" \
+              --body-file .sentinel-artifacts/probe-issue.md
+          else
+            gh issue create \
+              --repo "$GITHUB_REPOSITORY" \
+              --title "Catalog sentinel probe findings" \
+              --label provider-catalog \
+              --label tests \
+              --body-file .sentinel-artifacts/probe-issue.md
+          fi
+      - name: Upload sanitized protected evidence
+        if: steps.probe-config.outputs.configured == 'true'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: catalog-sentinel-probes-${{ github.run_id }}
+          path: |
+            .sentinel-artifacts/probe.json
+            .sentinel-artifacts/probe.md
+          if-no-files-found: error
+          retention-days: 30
+      - name: Save probe lifecycle state
+        if: steps.probe-config.outputs.configured == 'true'
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: .sentinel-state/probe.json
+          key: catalog-sentinel-probe-${{ github.run_id }}

CHANGELOG.md

@@ -7,6 +7,9 @@ All notable changes to this project are documented here. The format is based on
 ## [Unreleased]
 
 ### Added
+- A weekly, manually dispatchable catalog sentinel with bounded public
+  discovery, environment-protected completion probes, sanitized lifecycle
+  artifacts, and advisory drift issues that never mutate routing.
 - Advisory proxy operations APIs: public `/livez` and `/readyz`, an
   authenticated secret-free `/v1/providers` inventory, and
   `/v1/models?ready=true` filtering.

CONTRIBUTING.md

@@ -45,6 +45,9 @@ maintainer commands for filing them.
 ## Adding a provider
 
 The whole catalog is [`src/freellmpool/providers.toml`](src/freellmpool/providers.toml).
+The scheduled discovery and protected-probe contract is documented in
+[`docs/CATALOG_SENTINEL.md`](docs/CATALOG_SENTINEL.md); sentinel output is
+advisory and never authorizes an automatic catalog mutation.
 Most providers are OpenAI-compatible, so adding one is just a TOML block:
 
 ```toml

docs/CATALOG_SENTINEL.md

@@ -0,0 +1,89 @@
+# Catalog sentinel operations
+
+The `catalog-sentinel` workflow is a weekly and manually dispatchable,
+advisory drift detector. It produces a bounded JSON workflow artifact and a
+short Markdown artifact. When a public listing contains actionable additions
+or an authoritative removal, the workflow opens or comments on a single
+maintainer-review issue.
+
+It never enables or disables a route, edits `providers.toml`, purchases
+credits, or changes runtime routing. A maintainer must reproduce the evidence,
+check provider terms and billing behavior, run completion probes, and submit a
+normal reviewed pull request before catalog state changes.
+
+## Public discovery
+
+The public job sends unauthenticated `GET` requests only to model-list
+endpoints derived from the packaged catalog. Redirects are disabled, each
+request has a timeout, and decoded bodies are capped at 1 MB. User catalog
+overrides are deliberately ignored.
+
+Unknown and partial listing scopes can identify new candidates, but missing
+rows are recorded as unconfirmed absences. They are not retirement evidence.
+An empty response, malformed JSON, 429 rate limit, 402 billing/credit response,
+provider-wide authentication failure, timeout, and transient 5xx response
+never cause a retirement recommendation.
+
+## Protected completion probes
+
+Authenticated canaries run in the GitHub `catalog-sentinel` environment. Turn
+on environment protection and require a maintainer reviewer before configuring
+the environment secret:
+
+```text
+FREELLMPOOL_SENTINEL_KEYS_JSON
+```
+
+Its value is a bounded JSON object that maps the catalog's environment-variable
+names to their values. For example, configure it through GitHub's encrypted
+environment-secret UI; never commit the value:
+
+```json
+{"GROQ_API_KEY":"...","CLOUDFLARE_API_TOKEN":"...","CLOUDFLARE_ACCOUNT_ID":"..."}
+```
+
+The probe report contains provider IDs, catalog model IDs, HTTP status
+classifications, timestamps, and lifecycle counters. It excludes keys,
+account identifiers, provider response bodies, exception text, prompts, and
+completion text. If the secret is absent, the protected job records that probes
+were skipped without weakening public discovery.
+
+Each canary requests at most eight output tokens and explicitly disables the
+normal client convenience that raises reasoning-model budgets. Provider count,
+models per provider, request timeout, and the overall protected job are all
+bounded independently.
+
+## Lifecycle and artifacts
+
+Pinned cache actions restore the preceding sanitized report when available.
+The sentinel carries forward only validated timestamps and bounded counters for
+matching packaged provider/model identities. Invalid, oversized, stale-schema,
+or missing state is ignored.
+
+Each successful run uploads its current JSON and Markdown workflow artifact
+with 30-day retention. Cache loss or artifact expiry resets counters but cannot
+change routing. Treat the artifact and generated issue as leads—not proof that
+a model is free, healthy, or retired.
+
+For a local public run:
+
+```bash
+python3 scripts/catalog_sentinel.py discover \
+  --output /tmp/catalog-sentinel.json \
+  --summary /tmp/catalog-sentinel.md
+```
+
+For a local protected probe, export the JSON secret map and choose explicit
+bounds:
+
+```bash
+python3 scripts/catalog_sentinel.py probe \
+  --output /tmp/catalog-probes.json \
+  --summary /tmp/catalog-probes.md \
+  --max-providers 8 \
+  --max-models-per-provider 1
+```
+
+Inspect the workflow artifact, reproduce any candidate with
+`scripts/vet_catalog.py`, and follow the catalog rules in
+[`CONTRIBUTING.md`](../CONTRIBUTING.md).