feat(timemachine): Model Time Machine — counterfactual replay of your real workload

README.md

@@ -69,6 +69,7 @@
 - [Community Hub](#community-hub)
 - [Real-time Event Stream](#real-time-event-stream)
 - [Run Time Machine](#run-time-machine)
+- [Model Time Machine](#model-time-machine)
 - [Budget Guardrails](#budget-guardrails)
 - [Security](#security)
 - [A2A & AG-UI Protocols](#a2a--ag-ui-protocols)
@@ -1547,6 +1548,40 @@ curl -X POST http://localhost:8080/api/runs/{run_id}/fork \
   }'
 ```
 
+### Model Time Machine
+
+Test any model against your real workload, not benchmarks. The Time Machine takes the runs you actually executed - your prompts, your data, your costs - and answers the question every team asks: *"what would last month have looked like on a different model?"*
+
+```bash
+# Free, instant: price last month's recorded token volume on a local Llama 70B
+sandcastle timemachine --model nim/llama-3.1-70b --since 30d
+
+# Live replay: re-execute the recorded steps for real, score quality with an
+# LLM judge, and measure actual cost + latency - capped by an explicit budget
+sandcastle timemachine --model mistral/small --since 30d --live --budget 5.0
+```
+
+Two modes:
+
+- **Dry run (default)** - zero API calls. Recorded token volumes are priced against the target model's pricing table and extrapolated to a monthly figure. Free and instant.
+- **Live replay (`--live`)** - every recorded LLM step is re-executed on the target model. Old and new outputs are scored 0-10 by a configurable LLM judge (`TIMEMACHINE_JUDGE_MODEL`, default `haiku`), and real cost and latency are measured. A live replay **requires** `--budget`: the replay cost is estimated up front from the recorded token counts and the job refuses to start if the estimate exceeds the cap.
+
+The report breaks down quality, cost, and latency deltas per workflow and ends with a one-line verdict: `Switching to nim/llama-3.1-70b saves $312/mo at -2.1% quality.`
+
+Also available in the dashboard (**Operations → Time Machine**) and over the API:
+
+```bash
+# Start a job (dry-run by default; add "live": true + "budget_usd" for live)
+curl -X POST http://localhost:8080/api/timemachine \
+  -H "Content-Type: application/json" \
+  -d '{ "target_model": "nim/llama-3.1-70b", "since": "30d", "max_cassettes": 20 }'
+
+# Fetch the report
+curl http://localhost:8080/api/timemachine/{job_id}
+```
+
+Reports persist as JSON artifacts under `{data_dir}/timemachine/`.
+
 ---
 
 ## Budget Guardrails

dashboard/src/App.tsx

@@ -36,6 +36,7 @@ const EvolutionPage = lazyWithRetry(() => import("@/pages/EvolutionPage"), "evol
 const SystemHealthPage = lazyWithRetry(() => import("@/pages/SystemHealthPage"), "system-health");
 const CompliancePage = lazyWithRetry(() => import("@/pages/CompliancePage"), "compliance");
 const MemoryPage = lazyWithRetry(() => import("@/pages/MemoryPage"), "memory");
+const TimeMachinePage = lazyWithRetry(() => import("@/pages/TimeMachinePage"), "time-machine");
 const Onboarding = lazyWithRetry(() => import("@/pages/Onboarding"), "onboarding");
 
 /** Wrap a lazy-loaded page in a per-route error boundary so a crash in one
@@ -152,6 +153,7 @@ export default function App() {
               <Route path="/system-health" element={<PageBoundary name="system-health"><SystemHealthPage /></PageBoundary>} />
               <Route path="/compliance" element={<LiteGuard><PageBoundary name="compliance"><CompliancePage /></PageBoundary></LiteGuard>} />
               <Route path="/memory" element={<LiteGuard><PageBoundary name="memory"><MemoryPage /></PageBoundary></LiteGuard>} />
+              <Route path="/time-machine" element={<LiteGuard><PageBoundary name="time-machine"><TimeMachinePage /></PageBoundary></LiteGuard>} />
               <Route path="*" element={<NotFound />} />
             </Route>
           </Routes>

dashboard/src/__tests__/TimeMachinePage.test.tsx

@@ -0,0 +1,168 @@
+/**
+ * TimeMachinePage tests - config form, dry-run job flow with polling,
+ * live-mode budget input, error surfacing, and the empty-selection state.
+ */
+
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { render, screen, fireEvent, waitFor, cleanup } from "@testing-library/react";
+
+const apiGet = vi.fn();
+const apiPost = vi.fn();
+
+vi.mock("@/api/client", () => ({
+  api: {
+    get: (...args: unknown[]) => apiGet(...args),
+    post: (...args: unknown[]) => apiPost(...args),
+    isMockMode: false,
+    onMockChange: vi.fn(() => () => {}),
+  },
+}));
+
+vi.mock("react-router-dom", () => ({
+  useNavigate: () => vi.fn(),
+  useLocation: () => ({ pathname: "/time-machine" }),
+  Link: ({ children, to, ...rest }: { children: React.ReactNode; to: string }) => (
+    <a href={to} {...rest}>{children}</a>
+  ),
+}));
+
+import TimeMachinePage from "@/pages/TimeMachinePage";
+
+const REPORT = {
+  mode: "dry_run",
+  target_model: "nim/llama-3.1-70b",
+  judge_model: null,
+  selection: {
+    runs: 12,
+    steps: 40,
+    workflows: ["lead-enrichment"],
+    original_cost_usd: 20.0,
+    window_days: 28,
+  },
+  cost: { original_usd: 20.0, new_usd: 0.0, delta_usd: -20.0, delta_pct: -100 },
+  quality: null,
+  latency: null,
+  live: null,
+  extrapolation: {
+    window_days: 28,
+    monthly_original_usd: 21.4,
+    monthly_projected_usd: 0,
+    monthly_savings_usd: 21.4,
+  },
+  per_workflow: [
+    {
+      workflow: "lead-enrichment",
+      runs: 12,
+      steps: 40,
+      original_cost_usd: 20.0,
+      new_cost_usd: 0.0,
+      cost_delta_usd: -20.0,
+      cost_delta_pct: -100,
+      quality_old: null,
+      quality_new: null,
+      quality_delta_pct: null,
+      latency_old_seconds: null,
+      latency_new_seconds: null,
+      latency_delta_pct: null,
+    },
+  ],
+  verdict: "Switching to nim/llama-3.1-70b saves $21.40/mo (projected from recorded token volume). Run a live replay to measure quality.",
+};
+
+function mockDefaults(jobReport: unknown = REPORT) {
+  apiGet.mockImplementation((path: string) => {
+    if (path === "/timemachine") return Promise.resolve({ data: [], error: null });
+    if (path === "/workflows") {
+      return Promise.resolve({ data: [{ name: "lead-enrichment" }], error: null });
+    }
+    if (path.startsWith("/timemachine/")) {
+      return Promise.resolve({
+        data: { job_id: "job-1", status: "completed", report: jobReport, error: null },
+        error: null,
+      });
+    }
+    return Promise.resolve({ data: null, error: null });
+  });
+  apiPost.mockResolvedValue({ data: { job_id: "job-1", status: "running" }, error: null });
+}
+
+describe("TimeMachinePage", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mockDefaults();
+  });
+
+  afterEach(() => {
+    cleanup();
+  });
+
+  it("renders the config form and the confident empty state", async () => {
+    render(<TimeMachinePage />);
+    expect(screen.getByText("Model Time Machine")).toBeInTheDocument();
+    expect(screen.getByLabelText("Target model")).toBeInTheDocument();
+    expect(
+      await screen.findByText("Test any model against your real workload")
+    ).toBeInTheDocument();
+    // dry run is the default - the run button says so
+    expect(screen.getByRole("button", { name: /run dry-run estimate/i })).toBeInTheDocument();
+  });
+
+  it("runs a dry-run job and renders the report after polling", async () => {
+    render(<TimeMachinePage />);
+    fireEvent.click(screen.getByRole("button", { name: /run dry-run estimate/i }));
+
+    await waitFor(() => {
+      expect(apiPost).toHaveBeenCalledWith(
+        "/timemachine",
+        expect.objectContaining({
+          target_model: "nim/llama-3.1-70b",
+          since: "30d",
+          live: false,
+        })
+      );
+    });
+
+    // poll interval is 1.5s - wait for the completed report to land
+    expect(await screen.findByTestId("tm-verdict", {}, { timeout: 4000 })).toBeInTheDocument();
+    expect(screen.getAllByText(/saves \$21\.40\/mo/).length).toBeGreaterThan(0);
+    expect(screen.getAllByText("lead-enrichment").length).toBeGreaterThan(1);
+    expect(screen.getByText(/Per-workflow deltas/)).toBeInTheDocument();
+  }, 10000);
+
+  it("reveals the budget input only in live mode and sends budget_usd", async () => {
+    render(<TimeMachinePage />);
+    expect(screen.queryByLabelText(/budget cap/i)).not.toBeInTheDocument();
+
+    fireEvent.click(screen.getByRole("checkbox"));
+    expect(screen.getByLabelText(/budget cap/i)).toBeInTheDocument();
+
+    fireEvent.click(screen.getByRole("button", { name: /run live replay/i }));
+    await waitFor(() => {
+      expect(apiPost).toHaveBeenCalledWith(
+        "/timemachine",
+        expect.objectContaining({ live: true, budget_usd: 5.0 })
+      );
+    });
+  });
+
+  it("surfaces a budget refusal error from the API", async () => {
+    apiPost.mockResolvedValue({
+      data: null,
+      error: { code: "BUDGET_EXCEEDED", message: "Estimated cost $4.20 exceeds budget $1.00" },
+    });
+    render(<TimeMachinePage />);
+    fireEvent.click(screen.getByRole("button", { name: /run dry-run estimate/i }));
+    expect(
+      await screen.findByText(/exceeds budget \$1\.00/)
+    ).toBeInTheDocument();
+  });
+
+  it("shows the empty-selection state when no recorded workload matched", async () => {
+    mockDefaults({ ...REPORT, selection: { ...REPORT.selection, runs: 0, steps: 0 } });
+    render(<TimeMachinePage />);
+    fireEvent.click(screen.getByRole("button", { name: /run dry-run estimate/i }));
+    expect(
+      await screen.findByText("No recorded workload in this window", {}, { timeout: 4000 })
+    ).toBeInTheDocument();
+  }, 10000);
+});

dashboard/src/api/mock.ts

@@ -6879,7 +6879,7 @@
       const limit = Number(_params.limit || 20);
       // Backend RunListItem schema: run_id, workflow_name, status, total_cost_usd,
       // started_at, completed_at, parent_run_id (no risk_level)
       const items = filtered.slice(offset, offset + limit).map(({ risk_level: _, ...rest }) => rest);
       return {
         _data: items,
         _meta: { total: filtered.length, limit, offset },
@@ -7721,7 +7721,7 @@
 
       // If batch was cancelled, return current state without simulating more progress
       if (batch.status === "cancelled") {
         const { _tick: _, ...result } = batch;
         return result;
       }
 
@@ -7785,7 +7785,7 @@
       }
 
       // Return a clean copy without internal _tick
       const { _tick: _, ...result } = batch;
       return result;
     },
   },
@@ -8224,8 +8224,138 @@
       };
     },
   },
+  // --- Model Time Machine (counterfactual replay) ---
+  {
+    match: /^\/timemachine$/,
+    method: "POST",
+    handler: (_params, body) => {
+      const b = body as { target_model?: string; live?: boolean } | undefined;
+      mockTimeMachineLast = {
+        target_model: b?.target_model || "nim/llama-3.1-70b",
+        live: Boolean(b?.live),
+      };
+      return {
+        job_id: "tm-mock-0001",
+        status: "running",
+        mode: mockTimeMachineLast.live ? "live" : "dry_run",
+        target_model: mockTimeMachineLast.target_model,
+      };
+    },
+  },
+  {
+    match: /^\/timemachine$/,
+    handler: () => [
+      {
+        job_id: "tm-mock-0001",
+        status: "completed",
+        created_at: h(2),
+        completed_at: h(2),
+        mode: "dry_run",
+        target_model: "nim/llama-3.1-70b",
+        verdict:
+          "Switching to nim/llama-3.1-70b saves $312.40/mo (projected from recorded token volume). Run a live replay to measure quality.",
+        error: null,
+      },
+      {
+        job_id: "tm-mock-0002",
+        status: "completed",
+        created_at: d(3),
+        completed_at: d(3),
+        mode: "live",
+        target_model: "mistral/small",
+        verdict: "Switching to mistral/small saves $268.10/mo at -3.2% quality.",
+        error: null,
+      },
+    ],
+  },
+  {
+    match: /^\/timemachine\/([^/]+)$/,
+    handler: (params) => ({
+      job_id: params._1,
+      status: "completed",
+      params: {},
+      created_at: h(0.01),
+      completed_at: h(0),
+      error: null,
+      report: buildMockTimeMachineReport(
+        mockTimeMachineLast.target_model,
+        params._1 === "tm-mock-0002" ? true : mockTimeMachineLast.live
+      ),
+    }),
+  },
 ];
 
+let mockTimeMachineLast = { target_model: "nim/llama-3.1-70b", live: false };
+
+function buildMockTimeMachineReport(targetModel: string, live: boolean) {
+  const local = targetModel.startsWith("nim/") || targetModel.startsWith("omlx/") || targetModel === "ollama";
+  const ratio = local ? 0 : 0.22; // new cost as a fraction of original
+  const rows = [
+    { workflow: "lead-enrichment", runs: 9, steps: 34, original: 14.92, lat: 11.2 },
+    { workflow: "competitor-monitor", runs: 6, steps: 18, original: 7.41, lat: 8.6 },
+    { workflow: "seo-audit", runs: 5, steps: 15, original: 5.08, lat: 9.9 },
+  ];
+  const originalTotal = rows.reduce((acc, r) => acc + r.original, 0);
+  const newTotal = originalTotal * ratio;
+  const monthlyOriginal = originalTotal * (30 / 28);
+  const monthlySavings = monthlyOriginal * (1 - ratio);
+  return {
+    mode: live ? "live" : "dry_run",
+    target_model: targetModel,
+    judge_model: live ? "haiku" : null,
+    selection: {
+      runs: rows.reduce((acc, r) => acc + r.runs, 0),
+      steps: rows.reduce((acc, r) => acc + r.steps, 0),
+      workflows: rows.map((r) => r.workflow),
+      original_cost_usd: originalTotal,
+      window_days: 28,
+    },
+    cost: {
+      original_usd: originalTotal,
+      new_usd: newTotal,
+      delta_usd: newTotal - originalTotal,
+      delta_pct: (ratio - 1) * 100,
+    },
+    quality: live ? { old_avg: 8.4, new_avg: 8.1, delta_pct: -3.6 } : null,
+    latency: live
+      ? { old_avg_seconds: 10.4, new_avg_seconds: 6.1, delta_pct: -41.3 }
+      : null,
+    live: live
+      ? {
+          measured_cost_usd: newTotal + 0.31,
+          budget_usd: 5.0,
+          steps_replayed: 67,
+          steps_failed: 0,
+          truncated: false,
+        }
+      : null,
+    extrapolation: {
+      window_days: 28,
+      monthly_original_usd: monthlyOriginal,
+      monthly_projected_usd: monthlyOriginal * ratio,
+      monthly_savings_usd: monthlySavings,
+    },
+    per_workflow: rows.map((r) => ({
+      workflow: r.workflow,
+      runs: r.runs,
+      steps: r.steps,
+      original_cost_usd: r.original,
+      new_cost_usd: r.original * ratio,
+      cost_delta_usd: r.original * ratio - r.original,
+      cost_delta_pct: (ratio - 1) * 100,
+      quality_old: live ? 8.4 : null,
+      quality_new: live ? 8.1 : null,
+      quality_delta_pct: live ? -3.6 : null,
+      latency_old_seconds: live ? r.lat : null,
+      latency_new_seconds: live ? r.lat * 0.59 : null,
+      latency_delta_pct: live ? -41.0 : null,
+    })),
+    verdict: live
+      ? `Switching to ${targetModel} saves $${monthlySavings.toFixed(2)}/mo at -3.6% quality.`
+      : `Switching to ${targetModel} saves $${monthlySavings.toFixed(2)}/mo (projected from recorded token volume). Run a live replay to measure quality.`,
+  };
+}
+
 export function mockFetch(
   path: string,
   params?: Record<string, string>,

dashboard/src/components/layout/Header.tsx

@@ -32,6 +32,7 @@ const PAGE_TITLES: Record<string, string> = {
   "/autopilot": "AutoPilot",
   "/violations": "Violations",
   "/optimizer": "Optimizer",
+  "/time-machine": "Time Machine",
   "/schedules": "Schedules",
   "/dead-letter": "Dead Letter",
   "/api-keys": "API Keys",

dashboard/src/components/layout/Layout.tsx

@@ -98,6 +98,7 @@ const ROUTE_LABELS: Record<string, string> = {
   "/autopilot": "AutoPilot",
   "/violations": "Violations",
   "/optimizer": "Optimizer",
+  "/time-machine": "Time Machine",
   "/schedules": "Schedules",
   "/dead-letter": "Dead Letter",
   "/api-keys": "API Keys",