feat: scheduled evaluations + network resilience (Wave 2)

server/app/main.py

@@ -1,3 +1,4 @@
+import asyncio
 import logging
 from contextlib import asynccontextmanager
 from pathlib import Path
@@ -7,6 +8,7 @@
 from fastapi.responses import FileResponse
 from rich.logging import RichHandler
 
+from . import scheduler
 from .config import settings
 from .runner import load_persisted_runs
 from .task_registry import load_persisted_tasks
@@ -30,12 +32,16 @@ async def lifespan(app):
     load_persisted_runs()
     load_persisted_tools()
     load_persisted_tasks()
+    count = scheduler.load_persisted_schedules()
+    logger.info("Loaded %d schedules", count)
+    scheduler_task = asyncio.create_task(scheduler.run_scheduler())
     logger.info(
         "Sentifish started — results dir: %s (%d runs loaded)",
         results_path.resolve(),
         len(list(results_path.glob("*.json"))),
     )
     yield
+    scheduler_task.cancel()
     logger.info("Sentifish shutting down")
 
 

server/app/models.py

@@ -228,3 +228,19 @@ def cost_by_provider(self) -> dict[str, float]:
         for s in self.scores:
             costs[s.provider] = costs.get(s.provider, 0.0) + s.cost_usd
         return costs
+
+
+class EvalSchedule(BaseModel):
+    """A scheduled recurring evaluation."""
+
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    name: str
+    dataset_name: str
+    providers: list[str]
+    top_k: int = Field(default=10, ge=1, le=100)
+    interval_minutes: int = Field(default=360, ge=5, le=10080)  # 5min to 7 days
+    enabled: bool = True
+    created_at: float = Field(default_factory=time.time)
+    last_run_id: str | None = None
+    last_run_at: float | None = None
+    run_count: int = 0

server/app/scheduler.py

@@ -0,0 +1,119 @@
+"""Simple async scheduler for recurring evaluations."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import time
+from pathlib import Path
+
+from .config import settings
+from .models import EvalSchedule
+
+logger = logging.getLogger(__name__)
+
+_schedules: dict[str, EvalSchedule] = {}
+_SCHEDULES_DIR: Path | None = None
+
+
+def _get_schedules_dir() -> Path:
+    global _SCHEDULES_DIR
+    if _SCHEDULES_DIR is None:
+        _SCHEDULES_DIR = Path(settings.results_dir) / "schedules"
+        _SCHEDULES_DIR.mkdir(parents=True, exist_ok=True)
+    return _SCHEDULES_DIR
+
+
+def list_schedules() -> list[EvalSchedule]:
+    return sorted(_schedules.values(), key=lambda s: s.created_at, reverse=True)
+
+
+def get_schedule(schedule_id: str) -> EvalSchedule | None:
+    return _schedules.get(schedule_id)
+
+
+def create_schedule(schedule: EvalSchedule) -> EvalSchedule:
+    _schedules[schedule.id] = schedule
+    _persist_schedule(schedule)
+    return schedule
+
+
+def delete_schedule(schedule_id: str) -> bool:
+    if schedule_id not in _schedules:
+        return False
+    del _schedules[schedule_id]
+    path = _get_schedules_dir() / f"{schedule_id}.json"
+    path.unlink(missing_ok=True)
+    return True
+
+
+def toggle_schedule(schedule_id: str) -> EvalSchedule | None:
+    schedule = _schedules.get(schedule_id)
+    if schedule is None:
+        return None
+    schedule.enabled = not schedule.enabled
+    _persist_schedule(schedule)
+    return schedule
+
+
+def _persist_schedule(schedule: EvalSchedule) -> None:
+    path = _get_schedules_dir() / f"{schedule.id}.json"
+    path.write_text(schedule.model_dump_json(indent=2))
+
+
+def load_persisted_schedules() -> int:
+    d = _get_schedules_dir()
+    count = 0
+    for path in d.glob("*.json"):
+        try:
+            data = json.loads(path.read_text())
+            schedule = EvalSchedule(**data)
+            _schedules[schedule.id] = schedule
+            count += 1
+        except Exception as exc:
+            logger.warning("Failed to load schedule %s: %s", path.name, exc)
+    return count
+
+
+async def run_scheduler() -> None:
+    """Background loop that checks schedules every 60 seconds."""
+    # Import here to avoid circular imports
+    from . import datasets as ds
+    from . import runner
+
+    logger.info("Scheduler started")
+    while True:
+        await asyncio.sleep(60)
+        now = time.time()
+        for schedule in list(_schedules.values()):
+            if not schedule.enabled:
+                continue
+            # Check if enough time has passed since last run
+            interval_secs = schedule.interval_minutes * 60
+            last = schedule.last_run_at or schedule.created_at
+            if now - last < interval_secs:
+                continue
+            # Time to run
+            try:
+                dataset = ds.load_dataset(schedule.dataset_name)
+            except FileNotFoundError:
+                logger.warning(
+                    "Schedule %s: dataset %r not found, skipping",
+                    schedule.id,
+                    schedule.dataset_name,
+                )
+                continue
+            logger.info(
+                "Schedule %s: triggering run on %s",
+                schedule.name,
+                schedule.dataset_name,
+            )
+            run = runner.create_run(dataset, schedule.providers, schedule.top_k)
+            asyncio.create_task(
+                runner.execute_run(run, dataset, schedule.providers, schedule.top_k)
+            )
+            schedule.last_run_id = run.id
+            schedule.last_run_at = now
+            schedule.run_count += 1
+            _persist_schedule(schedule)

server/app/views.py

@@ -15,11 +15,12 @@
 from . import datasets as ds
 from . import narrator
 from . import runner
+from . import scheduler
 from . import task_registry
 from . import tool_registry
 from .config import settings
 from .metric_recommender import AVAILABLE_METRICS
-from .models import EvalConfig, EvalMetricWeight
+from .models import EvalConfig, EvalMetricWeight, EvalSchedule
 from .providers import PROVIDERS, available_providers
 
 logger = logging.getLogger(__name__)
@@ -563,3 +564,48 @@ def get_run_report(run_id: str):
         "query_winners": query_winners,
         "duration_seconds": (run.completed_at or 0) - run.created_at,
     }
+
+
+# -- Schedules ---------------------------------------------------------------
+
+
+@router.get("/schedules")
+def list_schedules():
+    return {"schedules": [s.model_dump() for s in scheduler.list_schedules()]}
+
+
+@router.post("/schedules", dependencies=[Depends(_require_write_auth)])
+def create_schedule(body: dict):
+    try:
+        schedule = EvalSchedule(**body)
+    except Exception:
+        raise HTTPException(status_code=400, detail="Invalid schedule data")
+    # Validate dataset exists
+    try:
+        ds.load_dataset(schedule.dataset_name)
+    except FileNotFoundError:
+        raise HTTPException(status_code=404, detail=f"Dataset not found: {schedule.dataset_name!r}")
+    # Validate providers
+    unknown = set(schedule.providers) - set(PROVIDERS)
+    if unknown:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Unknown provider(s): {', '.join(sorted(unknown))}",
+        )
+    scheduler.create_schedule(schedule)
+    return {"ok": True, "id": schedule.id}
+
+
+@router.delete("/schedules/{schedule_id}", dependencies=[Depends(_require_write_auth)])
+def delete_schedule_endpoint(schedule_id: str):
+    if not scheduler.delete_schedule(schedule_id):
+        raise HTTPException(status_code=404, detail="Schedule not found")
+    return {"ok": True, "deleted": schedule_id}
+
+
+@router.patch("/schedules/{schedule_id}/toggle", dependencies=[Depends(_require_write_auth)])
+def toggle_schedule_endpoint(schedule_id: str):
+    schedule = scheduler.toggle_schedule(schedule_id)
+    if schedule is None:
+        raise HTTPException(status_code=404, detail="Schedule not found")
+    return {"ok": True, "enabled": schedule.enabled}

server/tests/test_scheduler.py

@@ -0,0 +1,94 @@
+"""Tests for the scheduler module."""
+
+from app.models import EvalSchedule
+from app import scheduler
+
+
+def test_create_and_list_schedule(tmp_path, monkeypatch):
+    """Create a schedule and verify it appears in the list."""
+    monkeypatch.setattr(scheduler, "_schedules", {})
+    monkeypatch.setattr(scheduler, "_SCHEDULES_DIR", tmp_path)
+
+    s = EvalSchedule(
+        name="test-schedule",
+        dataset_name="sample",
+        providers=["brave"],
+        interval_minutes=60,
+    )
+    scheduler.create_schedule(s)
+
+    schedules = scheduler.list_schedules()
+    assert len(schedules) == 1
+    assert schedules[0].name == "test-schedule"
+    assert schedules[0].enabled is True
+
+
+def test_toggle_schedule(tmp_path, monkeypatch):
+    """Toggle a schedule's enabled state."""
+    monkeypatch.setattr(scheduler, "_schedules", {})
+    monkeypatch.setattr(scheduler, "_SCHEDULES_DIR", tmp_path)
+
+    s = EvalSchedule(
+        name="toggle-test",
+        dataset_name="sample",
+        providers=["serper"],
+        interval_minutes=30,
+    )
+    scheduler.create_schedule(s)
+    assert s.enabled is True
+
+    toggled = scheduler.toggle_schedule(s.id)
+    assert toggled is not None
+    assert toggled.enabled is False
+
+    toggled2 = scheduler.toggle_schedule(s.id)
+    assert toggled2 is not None
+    assert toggled2.enabled is True
+
+
+def test_delete_schedule(tmp_path, monkeypatch):
+    """Delete a schedule."""
+    monkeypatch.setattr(scheduler, "_schedules", {})
+    monkeypatch.setattr(scheduler, "_SCHEDULES_DIR", tmp_path)
+
+    s = EvalSchedule(
+        name="delete-test",
+        dataset_name="sample",
+        providers=["tavily"],
+        interval_minutes=360,
+    )
+    scheduler.create_schedule(s)
+    assert len(scheduler.list_schedules()) == 1
+
+    result = scheduler.delete_schedule(s.id)
+    assert result is True
+    assert len(scheduler.list_schedules()) == 0
+
+
+def test_delete_nonexistent(tmp_path, monkeypatch):
+    """Deleting a nonexistent schedule returns False."""
+    monkeypatch.setattr(scheduler, "_schedules", {})
+    assert scheduler.delete_schedule("nonexistent-id") is False
+
+
+def test_load_persisted(tmp_path, monkeypatch):
+    """Schedules persist to disk and reload."""
+    monkeypatch.setattr(scheduler, "_schedules", {})
+    monkeypatch.setattr(scheduler, "_SCHEDULES_DIR", tmp_path)
+
+    s = EvalSchedule(
+        name="persist-test",
+        dataset_name="sample",
+        providers=["brave", "serper"],
+        interval_minutes=720,
+    )
+    scheduler.create_schedule(s)
+
+    # Clear in-memory state
+    monkeypatch.setattr(scheduler, "_schedules", {})
+    assert len(scheduler.list_schedules()) == 0
+
+    # Reload from disk
+    count = scheduler.load_persisted_schedules()
+    assert count == 1
+    assert scheduler.list_schedules()[0].name == "persist-test"

ui/src/App.tsx

@@ -4,18 +4,27 @@ import { Toaster as Sonner } from "@/components/ui/sonner";
 import { Toaster } from "@/components/ui/toaster";
 import { TooltipProvider } from "@/components/ui/tooltip";
 import ErrorBoundary from "./components/ErrorBoundary";
+import NetworkStatus from "./components/NetworkStatus";
 import Landing from "./pages/Landing";
 import Dashboard from "./pages/Dashboard";
 import Leaderboard from "./pages/Leaderboard";
 import Report from "./pages/Report";
 import Configure from "./pages/Configure";
 import NotFound from "./pages/NotFound";
 
-const queryClient = new QueryClient();
+const queryClient = new QueryClient({
+  defaultOptions: {
+    queries: {
+      retry: 3,
+      retryDelay: (attempt) => Math.min(1000 * 2 ** attempt, 10000),
+    },
+  },
+});
 
 const App = () => (
   <QueryClientProvider client={queryClient}>
     <ErrorBoundary>
+      <NetworkStatus />
       <TooltipProvider>
         <Toaster />
         <Sonner />

ui/src/components/NetworkStatus.tsx

@@ -0,0 +1,27 @@
+import { useHealth } from "@/hooks/useApi";
+import { motion, AnimatePresence } from "framer-motion";
+import { WifiOff } from "lucide-react";
+
+export default function NetworkStatus() {
+  const { isError, isLoading } = useHealth();
+  const showBanner = isError && !isLoading;
+
+  return (
+    <AnimatePresence>
+      {showBanner && (
+        <motion.div
+          initial={{ height: 0, opacity: 0 }}
+          animate={{ height: "auto", opacity: 1 }}
+          exit={{ height: 0, opacity: 0 }}
+          transition={{ duration: 0.2 }}
+          className="overflow-hidden"
+        >
+          <div className="flex items-center justify-center gap-2 bg-warning/10 px-4 py-2 text-sm text-warning">
+            <WifiOff className="h-4 w-4" />
+            <span>Unable to reach Sentifish API. Retrying...</span>
+          </div>
+        </motion.div>
+      )}
+    </AnimatePresence>
+  );
+}