From ab81a0c8b3a5ae3be780dd4a99044973926452cc Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 22 Apr 2026 16:30:21 -0700 Subject: [PATCH] admin /providers page and dynamic llms settings (PR 5 of provider migration) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Adds `/admin/providers` showing active SCRAPE_TARGETS, per-provider configured status, and a per-target smoke-test button (real call, latency + sample). - Replaces the three hardcoded ChatGPT/Claude/Google cards on `/app/$brand/settings/llms` with one card per active model; toggles write to `brand.enabledModels`. Semantics match `selectTargetsForBrand` that landed in PR 4: null = all enabled, [] = explicit opt-out (nothing runs), [...] = opt-in. Orphaned entries from a shrunk SCRAPE_TARGETS are flagged and dropped on save. - Surfaces the whitelabel report-run coupling: any SCRAPE_TARGETS model not in `WHITELABEL_REPORT_RUNS_PER_MODEL` gets a warning badge on the providers page on `DEPLOYMENT_MODE=whitelabel`. Constant moved into `@workspace/lib/providers` so the web can read it without importing worker code. - Adds "Providers" to the admin sidebar. Completes PR 5 — the provider migration scope from #153 is now closed. --- .changeset/admin-providers-dynamic-llms.md | 7 + apps/web/src/components/app-sidebar.tsx | 7 + apps/web/src/routeTree.gen.ts | 21 + .../src/routes/_authed/admin/providers.tsx | 266 +++++++++++ .../_authed/app/$brand/settings/llms.tsx | 433 ++++++++++++------ apps/web/src/server/admin.ts | 79 +++- apps/web/src/server/brands.ts | 45 ++ apps/worker/src/report-worker.ts | 18 +- packages/lib/src/providers/index.ts | 1 + packages/lib/src/providers/report-runs.ts | 14 + 10 files changed, 734 insertions(+), 157 deletions(-) create mode 100644 .changeset/admin-providers-dynamic-llms.md create mode 100644 apps/web/src/routes/_authed/admin/providers.tsx create mode 100644 packages/lib/src/providers/report-runs.ts diff --git a/.changeset/admin-providers-dynamic-llms.md b/.changeset/admin-providers-dynamic-llms.md new file mode 100644 index 00000000..8e159da8 --- /dev/null +++ b/.changeset/admin-providers-dynamic-llms.md @@ -0,0 +1,7 @@ +--- +"@workspace/web": patch +"@workspace/lib": patch +"@workspace/worker": patch +--- + +Admin `/admin/providers` page shows the active `SCRAPE_TARGETS`, per-provider credential status, and a per-target smoke-test button. Brand `settings/llms` now renders one card per active model and writes to `brand.enabledModels` (matching the `null` / `[]` / opt-in semantics enforced by the worker). Whitelabel deployments get a warning badge for any SCRAPE_TARGETS model missing from the report-run map. diff --git a/apps/web/src/components/app-sidebar.tsx b/apps/web/src/components/app-sidebar.tsx index 5374eb14..5cdd1d66 100644 --- a/apps/web/src/components/app-sidebar.tsx +++ b/apps/web/src/components/app-sidebar.tsx @@ -12,6 +12,7 @@ import { IconReport, IconTimeline, IconTool, + IconPlugConnected, } from "@tabler/icons-react"; import { @@ -128,6 +129,12 @@ export function AppSidebar({ isAdmin = false, hasReportAccess = false, adminOnly icon: IconTimeline, absolute: true, }, + { + title: "Providers", + url: "/admin/providers", + icon: IconPlugConnected, + absolute: true, + }, { title: "Tools", url: "/admin/tools", diff --git a/apps/web/src/routeTree.gen.ts b/apps/web/src/routeTree.gen.ts index 2da05297..5c6db21c 100644 --- a/apps/web/src/routeTree.gen.ts +++ b/apps/web/src/routeTree.gen.ts @@ -28,6 +28,7 @@ import { Route as AuthedAppBrandRouteImport } from './routes/_authed/app/$brand' import { Route as AuthedAdminWorkflowsRouteImport } from './routes/_authed/admin/workflows' import { Route as AuthedAdminWizardTestRouteImport } from './routes/_authed/admin/wizard-test' import { Route as AuthedAdminToolsRouteImport } from './routes/_authed/admin/tools' +import { Route as AuthedAdminProvidersRouteImport } from './routes/_authed/admin/providers' import { Route as ApiV1ReportsIndexRouteImport } from './routes/api/v1/reports/index' import { Route as ApiV1PromptsIndexRouteImport } from './routes/api/v1/prompts/index' import { Route as ApiV1DocsIndexRouteImport } from './routes/api/v1/docs/index' @@ -144,6 +145,11 @@ const AuthedAdminToolsRoute = AuthedAdminToolsRouteImport.update({ path: '/tools', getParentRoute: () => AuthedAdminRoute, } as any) +const AuthedAdminProvidersRoute = AuthedAdminProvidersRouteImport.update({ + id: '/providers', + path: '/providers', + getParentRoute: () => AuthedAdminRoute, +} as any) const ApiV1ReportsIndexRoute = ApiV1ReportsIndexRouteImport.update({ id: '/api/v1/reports/', path: '/api/v1/reports/', @@ -270,6 +276,7 @@ export interface FileRoutesByFullPath { '/auth/login': typeof AuthLoginRoute '/auth/logout': typeof AuthLogoutRoute '/auth/register': typeof AuthRegisterRoute + '/admin/providers': typeof AuthedAdminProvidersRoute '/admin/tools': typeof AuthedAdminToolsRoute '/admin/wizard-test': typeof AuthedAdminWizardTestRoute '/admin/workflows': typeof AuthedAdminWorkflowsRoute @@ -308,6 +315,7 @@ export interface FileRoutesByTo { '/auth/login': typeof AuthLoginRoute '/auth/logout': typeof AuthLogoutRoute '/auth/register': typeof AuthRegisterRoute + '/admin/providers': typeof AuthedAdminProvidersRoute '/admin/tools': typeof AuthedAdminToolsRoute '/admin/wizard-test': typeof AuthedAdminWizardTestRoute '/admin/workflows': typeof AuthedAdminWorkflowsRoute @@ -350,6 +358,7 @@ export interface FileRoutesById { '/auth/login': typeof AuthLoginRoute '/auth/logout': typeof AuthLogoutRoute '/auth/register': typeof AuthRegisterRoute + '/_authed/admin/providers': typeof AuthedAdminProvidersRoute '/_authed/admin/tools': typeof AuthedAdminToolsRoute '/_authed/admin/wizard-test': typeof AuthedAdminWizardTestRoute '/_authed/admin/workflows': typeof AuthedAdminWorkflowsRoute @@ -393,6 +402,7 @@ export interface FileRouteTypes { | '/auth/login' | '/auth/logout' | '/auth/register' + | '/admin/providers' | '/admin/tools' | '/admin/wizard-test' | '/admin/workflows' @@ -431,6 +441,7 @@ export interface FileRouteTypes { | '/auth/login' | '/auth/logout' | '/auth/register' + | '/admin/providers' | '/admin/tools' | '/admin/wizard-test' | '/admin/workflows' @@ -472,6 +483,7 @@ export interface FileRouteTypes { | '/auth/login' | '/auth/logout' | '/auth/register' + | '/_authed/admin/providers' | '/_authed/admin/tools' | '/_authed/admin/wizard-test' | '/_authed/admin/workflows' @@ -660,6 +672,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AuthedAdminToolsRouteImport parentRoute: typeof AuthedAdminRoute } + '/_authed/admin/providers': { + id: '/_authed/admin/providers' + path: '/providers' + fullPath: '/admin/providers' + preLoaderRoute: typeof AuthedAdminProvidersRouteImport + parentRoute: typeof AuthedAdminRoute + } '/api/v1/reports/': { id: '/api/v1/reports/' path: '/api/v1/reports' @@ -811,6 +830,7 @@ declare module '@tanstack/react-router' { } interface AuthedAdminRouteChildren { + AuthedAdminProvidersRoute: typeof AuthedAdminProvidersRoute AuthedAdminToolsRoute: typeof AuthedAdminToolsRoute AuthedAdminWizardTestRoute: typeof AuthedAdminWizardTestRoute AuthedAdminWorkflowsRoute: typeof AuthedAdminWorkflowsRoute @@ -818,6 +838,7 @@ interface AuthedAdminRouteChildren { } const AuthedAdminRouteChildren: AuthedAdminRouteChildren = { + AuthedAdminProvidersRoute: AuthedAdminProvidersRoute, AuthedAdminToolsRoute: AuthedAdminToolsRoute, AuthedAdminWizardTestRoute: AuthedAdminWizardTestRoute, AuthedAdminWorkflowsRoute: AuthedAdminWorkflowsRoute, diff --git a/apps/web/src/routes/_authed/admin/providers.tsx b/apps/web/src/routes/_authed/admin/providers.tsx new file mode 100644 index 00000000..872de031 --- /dev/null +++ b/apps/web/src/routes/_authed/admin/providers.tsx @@ -0,0 +1,266 @@ +/** + * /admin/providers - View SCRAPE_TARGETS configuration, per-model smoke tests, + * and registered-provider credential status. + */ +import { useEffect, useState } from "react"; +import { createFileRoute } from "@tanstack/react-router"; +import { getAppName } from "@/lib/route-head"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@workspace/ui/components/card"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@workspace/ui/components/table"; +import { Badge } from "@workspace/ui/components/badge"; +import { Button } from "@workspace/ui/components/button"; +import { Skeleton } from "@workspace/ui/components/skeleton"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@workspace/ui/components/tooltip"; +import { CheckCircle2, XCircle, AlertTriangle, Play, Loader2, RefreshCw } from "lucide-react"; +import { getProviderStatusFn, testProviderFn, type ProviderStatus } from "@/server/admin"; +import { WHITELABEL_REPORT_RUNS_PER_MODEL, getModelMeta, type ModelConfig, type TestResult } from "@workspace/lib/providers"; + +function formatTarget(cfg: ModelConfig): string { + const parts = [cfg.model, cfg.provider]; + if (cfg.version) parts.push(cfg.version); + if (cfg.webSearch) parts.push("online"); + return parts.join(":"); +} + +function TestCell({ target }: { target: string }) { + const [state, setState] = useState<"idle" | "loading" | TestResult>("idle"); + + const handleTest = async () => { + setState("loading"); + try { + const result = await testProviderFn({ data: { target } }); + setState(result); + } catch (err) { + setState({ + success: false, + latencyMs: 0, + error: err instanceof Error ? err.message : String(err), + }); + } + }; + + if (state === "idle") { + return ( + + ); + } + if (state === "loading") { + return ( + + ); + } + return ( +
+
+ {state.success ? ( + + + Pass + + ) : ( + + + Fail + + )} + {state.latencyMs}ms + +
+ {state.success && state.sampleOutput && ( +

+ {state.sampleOutput} +

+ )} + {!state.success && state.error && ( +

+ {state.error} +

+ )} +
+ ); +} + +export const Route = createFileRoute("/_authed/admin/providers")({ + head: ({ match }) => { + const appName = getAppName(match); + return { + meta: [ + { title: `Providers | ${appName}` }, + { name: "description", content: "SCRAPE_TARGETS configuration and provider connectivity tests." }, + ], + }; + }, + component: ProvidersPage, +}); + +function ProvidersPage() { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + getProviderStatusFn() + .then(setData) + .catch((err) => setError(err instanceof Error ? err.message : "An error occurred")) + .finally(() => setLoading(false)); + }, []); + + if (loading) { + return ( +
+
+ + +
+ + +
+ ); + } + + if (error || !data) { + return ( + + + Error + + +

{error ?? "Failed to load provider status"}

+ + + ); + } + + const isWhitelabel = data.deploymentMode === "whitelabel"; + const whitelabelKeys = new Set(Object.keys(WHITELABEL_REPORT_RUNS_PER_MODEL)); + + return ( +
+
+

Providers

+

+ Current SCRAPE_TARGETS configuration and provider + connectivity. Changing targets requires a redeploy. +

+
+ + + + Active Targets + + {data.activeTargets.length} target{data.activeTargets.length === 1 ? "" : "s"} dispatched by the worker for + each prompt run. Deployment mode: {data.deploymentMode}. + + + + + + + Model + Provider + Version + Web Search + Test + + + + {data.activeTargets.map((cfg) => { + const target = formatTarget(cfg); + const meta = getModelMeta(cfg.model); + const whitelabelMissing = isWhitelabel && !whitelabelKeys.has(cfg.model); + return ( + + +
+ {meta.label} + {cfg.model} + {whitelabelMissing && ( + + + + + Report runs + + + + New models need a run count added to{" "} + WHITELABEL_REPORT_RUNS_PER_MODEL in{" "} + apps/worker/src/report-worker.ts before + report generation will succeed on this deployment. + + + )} +
+ + {cfg.provider} + {cfg.version ?? "—"} + + {cfg.webSearch ? ( + + ) : ( + + )} + + + + + + ); + })} + +
+ + + + + + Registered Providers + + Providers available to SCRAPE_TARGETS. A provider is "configured" when its required credentials are present in + the environment. + + + + + + + ID + Name + Configured + + + + {data.providers.map((p) => ( + + {p.id} + {p.name} + + {p.configured ? ( + + + Ready + + ) : ( + + + Missing credentials + + )} + + + ))} + +
+ + +
+ ); +} diff --git a/apps/web/src/routes/_authed/app/$brand/settings/llms.tsx b/apps/web/src/routes/_authed/app/$brand/settings/llms.tsx index 6e30ee99..bcaf3fc6 100644 --- a/apps/web/src/routes/_authed/app/$brand/settings/llms.tsx +++ b/apps/web/src/routes/_authed/app/$brand/settings/llms.tsx @@ -1,72 +1,49 @@ /** * /app/$brand/settings/llms - LLM configuration page * - * Shows info about tracked LLMs and their web search status. + * Shows every model the worker is configured to dispatch (from SCRAPE_TARGETS) + * and lets brand admins opt in/out per model via brand.enabledModels. + * + * brand.enabledModels semantics (matches selectTargetsForBrand in + * packages/lib/src/providers/runner.ts): + * - null → no override, every configured target runs + * - [] → explicit opt-out, nothing runs for this brand + * - [model,...] → opt-in list; entries must be in current SCRAPE_TARGETS */ +import { useEffect, useMemo, useState } from "react"; +import type { ComponentType, SVGProps } from "react"; import { createFileRoute } from "@tanstack/react-router"; +import { useQueryClient } from "@tanstack/react-query"; import { getAppName, getBrandName, buildTitle } from "@/lib/route-head"; -import { Card, CardContent, CardDescription, CardFooter, CardHeader } from "@workspace/ui/components/card"; +import { Card, CardContent, CardHeader } from "@workspace/ui/components/card"; +import { Button } from "@workspace/ui/components/button"; +import { Checkbox } from "@workspace/ui/components/checkbox"; import { Tooltip, TooltipContent, TooltipTrigger } from "@workspace/ui/components/tooltip"; -import { IconCircleCheck, IconCircleX, IconInfoCircle } from "@tabler/icons-react"; -import { SiOpenai, SiAnthropic, SiGoogle } from "react-icons/si"; - -interface ModelGroupInfo { - id: string; - name: string; - provider: string; - currentModel: string; - description: string; - icon: React.ReactNode; - trackedEnabled: boolean; - webSearchEnabled: boolean; - webSearchStatus: string; - webSearchDetail: string; -} +import { Alert, AlertDescription } from "@workspace/ui/components/alert"; +import { Skeleton } from "@workspace/ui/components/skeleton"; +import { IconInfoCircle, IconCpu } from "@tabler/icons-react"; +import { AlertTriangle, Loader2 } from "lucide-react"; +import { SiOpenai, SiAnthropic, SiGoogle, SiPerplexity, SiX } from "react-icons/si"; +import { useBrand, brandKeys } from "@/hooks/use-brands"; +import { getProviderStatusFn, type ProviderStatus } from "@/server/admin"; +import { updateBrandEnabledModelsFn } from "@/server/brands"; +import { getModelMeta, type ModelConfig } from "@workspace/lib/providers"; -const MODEL_GROUPS: ModelGroupInfo[] = [ - { - id: "chatgpt", - name: "ChatGPT", - provider: "OpenAI", - currentModel: "gpt-5-mini", - description: - "ChatGPT is OpenAI's consumer assistant, and it's often the first place people turn for product recommendations and comparisons. Tracking visibility here shows how your brand appears in everyday ChatGPT responses.", - icon: , - trackedEnabled: true, - webSearchEnabled: true, - webSearchStatus: "Enabled", - webSearchDetail: - "Responses include real-time information from the web, making visibility here especially impactful for brand discovery.", - }, - { - id: "claude", - name: "Claude", - provider: "Anthropic", - currentModel: "claude-sonnet-4-20250514", - description: - "Claude is Anthropic's assistant and is popular with professionals for nuanced, long-form research. Tracking Claude highlights how your brand appears in deeper analysis and decision making.", - icon: , - trackedEnabled: true, - webSearchEnabled: false, - webSearchStatus: "Disabled", - webSearchDetail: - "Responses are based on Claude's training data. Brand mentions reflect how well your brand is represented in publicly available content.", - }, - { - id: "google-ai-mode", - name: "Google AI Overviews", - provider: "Google", - currentModel: "AI Mode", - description: - "Google AI Mode powers AI Overviews in Search. Tracking AI Mode shows how your brand is summarized and cited as Google blends AI answers with traditional results.", - icon: , - trackedEnabled: true, - webSearchEnabled: true, - webSearchStatus: "Enabled", - webSearchDetail: - "Always uses live web data — results reflect real-time Google Search content, including your website, reviews, and mentions across the web.", - }, -]; +type IconComponent = ComponentType>; + +const ICON_BY_ID: Record = { + openai: SiOpenai, + anthropic: SiAnthropic, + google: SiGoogle, + perplexity: SiPerplexity, + x: SiX, +}; + +function ModelIcon({ iconId, className }: { iconId: string; className?: string }) { + const Icon = ICON_BY_ID[iconId]; + if (Icon) return ; + return ; +} export const Route = createFileRoute("/_authed/app/$brand/settings/llms")({ head: ({ matches, match }) => { @@ -83,98 +60,266 @@ export const Route = createFileRoute("/_authed/app/$brand/settings/llms")({ }); function LlmsSettingsPage() { + const { brand, isLoading: brandLoading, revalidate } = useBrand(); + const queryClient = useQueryClient(); + + const [status, setStatus] = useState(null); + const [statusError, setStatusError] = useState(null); + const [statusLoading, setStatusLoading] = useState(true); + + useEffect(() => { + getProviderStatusFn() + .then(setStatus) + .catch((err) => setStatusError(err instanceof Error ? err.message : "Failed to load providers")) + .finally(() => setStatusLoading(false)); + }, []); + + const activeTargets = status?.activeTargets ?? []; + const activeModels = useMemo(() => new Set(activeTargets.map((t) => t.model)), [activeTargets]); + + const persistedEnabled: string[] | null = brand?.enabledModels ?? null; + + // Seed the toggle state from the brand. null (no override) surfaces as "all + // currently-active models selected" so the grid is in a consistent shape; + // saving that state sends back the full array, which the worker enforces. + const [selected, setSelected] = useState>(new Set()); + const [selectionInitialized, setSelectionInitialized] = useState(false); + + useEffect(() => { + if (!brand || statusLoading) return; + if (persistedEnabled === null) { + setSelected(new Set(activeTargets.map((t) => t.model))); + } else { + setSelected(new Set(persistedEnabled.filter((m) => activeModels.has(m)))); + } + setSelectionInitialized(true); + }, [brand?.updatedAt, statusLoading, status]); + + const orphaned = useMemo(() => { + if (!persistedEnabled) return []; + return persistedEnabled.filter((m) => !activeModels.has(m)); + }, [persistedEnabled, activeModels]); + + const [isSubmitting, setIsSubmitting] = useState(false); + const [saveError, setSaveError] = useState(null); + const [saveSuccess, setSaveSuccess] = useState(null); + + const toggle = (model: string) => { + setSaveError(null); + setSaveSuccess(null); + setSelected((prev) => { + const next = new Set(prev); + if (next.has(model)) next.delete(model); + else next.add(model); + return next; + }); + }; + + const persistArray = async (value: string[] | null) => { + if (!brand) return; + setIsSubmitting(true); + setSaveError(null); + setSaveSuccess(null); + try { + await updateBrandEnabledModelsFn({ data: { brandId: brand.id, enabledModels: value } }); + queryClient.invalidateQueries({ queryKey: brandKeys.detail(brand.id) }); + await revalidate(); + setSaveSuccess( + value === null + ? "Reset to default. Every configured model will run." + : value.length === 0 + ? "Saved. No models will run for this brand until you enable one." + : `Saved. ${value.length} model${value.length === 1 ? "" : "s"} enabled.`, + ); + } catch (err) { + setSaveError(err instanceof Error ? err.message : "An error occurred"); + } finally { + setIsSubmitting(false); + } + }; + + const handleSave = () => persistArray([...selected]); + const handleResetDefault = () => persistArray(null); + + if (brandLoading || statusLoading || !selectionInitialized) { + return ( +
+
+ + +
+
+ {[0, 1, 2].map((n) => ( + + ))} +
+
+ ); + } + + if (!brand) { + return ( +
+

LLMs

+

Brand not found

+
+ ); + } + + if (statusError || !status) { + return ( +
+

LLMs

+ + {statusError ?? "Failed to load provider status"} + +
+ ); + } + + const nothingSelected = selected.size === 0; + const usingDefault = persistedEnabled === null; + return (

LLMs

- Your prompts are evaluated against multiple types of AI models to track how your brand appears across different - types of AI search. + Your prompts are evaluated against multiple AI models to track how your brand appears across different types of AI + search. Toggle off any model you don't want included in runs for this brand.

-
- {MODEL_GROUPS.map((group) => ( - - -
-
-
{group.icon}
-
-
- - -
-
-
- Provider -
- {group.provider} -
-
-
- Model - - - - - - Exact model version used for this group. - - -
- {group.currentModel} -
-
-
- Tracked - - - - - - Whether this model group is included in your visibility runs. - - -
-
- {group.trackedEnabled ? ( - - ) : ( - - )} - {group.trackedEnabled ? "Enabled" : "Disabled"} -
-
-
-
- Web search - - - - - - {group.webSearchDetail} - - -
-
- {group.webSearchEnabled ? ( - - ) : ( - - )} - {group.webSearchStatus} -
-
-
- - - {group.description} - - - ))} + {activeTargets.length === 0 && ( + + + No models are configured for this deployment. Ask an admin to set{" "} + SCRAPE_TARGETS. + + + )} + + {orphaned.length > 0 && ( + + + + The following saved models are no longer in this deployment's{" "} + SCRAPE_TARGETS and will cause the worker to throw on next dispatch:{" "} + {orphaned.join(", ")}. Click Save to drop them. + + + )} + + {nothingSelected && activeTargets.length > 0 && ( + + + + No models are selected. If you save now, prompts will stop running for this brand until a model is re-enabled. + + + )} + + {activeTargets.length > 0 && ( +
+ {activeTargets.map((cfg) => ( + toggle(cfg.model)} + /> + ))} +
+ )} + +
+ + {!usingDefault && ( + + )} + {usingDefault && ( +

+ Using deployment default — every configured model runs. Toggle any card off to start an opt-in list. +

+ )} + {saveSuccess &&

{saveSuccess}

} + {saveError &&

{saveError}

}
); } + +function ModelCard({ + config, + enabled, + onToggle, +}: { + config: ModelConfig; + enabled: boolean; + onToggle: () => void; +}) { + const meta = getModelMeta(config.model); + const checkboxId = `llm-toggle-${config.model}`; + return ( + + +
+
+ +
+ {meta.label} + {config.model} +
+
+ +
+ + +
+
+ Provider + {config.provider} +
+
+
+ Version + + + + + + Exact version slug passed to the provider when this model runs. + + +
+ {config.version ?? "—"} +
+
+
+ Web search + + + + + + {config.webSearch + ? "Responses include real-time information from the web." + : "Responses are based on the model's training data only."} + + +
+ {config.webSearch ? "Enabled" : "Disabled"} +
+
+ + + ); +} diff --git a/apps/web/src/server/admin.ts b/apps/web/src/server/admin.ts index bbbdf898..afe7561c 100644 --- a/apps/web/src/server/admin.ts +++ b/apps/web/src/server/admin.ts @@ -17,7 +17,13 @@ import { analyzeWebsite, getCompetitors, generateCandidatePromptsForReports } fr import { DEFAULT_DELAY_HOURS } from "@workspace/lib/constants"; import { sendImmediatePromptJob } from "@/lib/job-scheduler"; import { Client } from "pg"; -import { parseScrapeTargets } from "@workspace/lib/providers"; +import { + parseScrapeTargets, + getAllProviders, + getProvider, + type ModelConfig, + type TestResult, +} from "@workspace/lib/providers"; // ============================================================================ // Admin guard helper @@ -840,3 +846,74 @@ export const getJobLogsFn = createServerFn({ method: "GET" }) return { jobId: data.jobId, logs, count: logs.length }; }); + +// ============================================================================ +// Admin Providers - SCRAPE_TARGETS status + per-target smoke test +// ============================================================================ + +export interface ProviderStatus { + activeTargets: ModelConfig[]; + providers: { id: string; name: string; configured: boolean }[]; + deploymentMode: string; +} + +/** + * Snapshot of what the worker will dispatch: the parsed SCRAPE_TARGETS plus + * which registered providers have credentials configured. Used by + * /admin/providers to render the current configuration. + */ +export const getProviderStatusFn = createServerFn({ method: "GET" }).handler( + async (): Promise => { + await requireAdmin(); + + const activeTargets = parseScrapeTargets(process.env.SCRAPE_TARGETS); + const providers = getAllProviders().map((p) => ({ + id: p.id, + name: p.name, + configured: p.isConfigured(), + })); + + return { + activeTargets, + providers, + deploymentMode: process.env.DEPLOYMENT_MODE ?? "local", + }; + }, +); + +/** + * Run a single SCRAPE_TARGETS entry through its provider with a canned prompt. + * Surfaces credential and connectivity problems before prompts start failing. + */ +export const testProviderFn = createServerFn({ method: "POST" }) + .inputValidator(z.object({ target: z.string() })) + .handler(async ({ data }): Promise => { + await requireAdmin(); + + const configs = parseScrapeTargets(data.target); + if (configs.length !== 1) { + throw new Error( + `testProviderFn expects a single target; got ${configs.length} from "${data.target}"`, + ); + } + const cfg = configs[0]; + const provider = getProvider(cfg.provider); + + const start = Date.now(); + try { + const result = await provider.run(cfg.model, "What is 2+2?", { + webSearch: cfg.webSearch, + version: cfg.version, + }); + const latencyMs = Date.now() - start; + const sample = result.textContent.trim().slice(0, 240); + return { success: true, latencyMs, sampleOutput: sample }; + } catch (err) { + const latencyMs = Date.now() - start; + return { + success: false, + latencyMs, + error: err instanceof Error ? err.message : String(err), + }; + } + }); diff --git a/apps/web/src/server/brands.ts b/apps/web/src/server/brands.ts index 12dea70c..c4c483b7 100644 --- a/apps/web/src/server/brands.ts +++ b/apps/web/src/server/brands.ts @@ -10,6 +10,7 @@ import { brands, prompts, competitors, type BrandWithPrompts, type Brand } from import { eq, and, count, sql } from "drizzle-orm"; import { MAX_COMPETITORS } from "@workspace/lib/constants"; import { cleanAndValidateDomain } from "@/lib/domain-categories"; +import { parseScrapeTargets } from "@workspace/lib/providers"; function getDefaultBrandDomains(): string[] { const raw = process.env.DEFAULT_BRAND_DOMAINS; @@ -218,6 +219,50 @@ export const updateBrandFn = createServerFn({ method: "POST" }) return result[0]; }); +/** + * Update the set of models that should run for this brand. + * + * Semantics (must match `selectTargetsForBrand` in packages/lib): + * - `null` clears the override; every configured SCRAPE_TARGETS model runs. + * - `[]` is an explicit opt-out; no models run for this brand. + * - `[...]` is an opt-in list; every entry must be a model currently in + * SCRAPE_TARGETS, else the worker throws on the next dispatch. Validated + * here so the save surfaces the error instead of a silent future break. + */ +export const updateBrandEnabledModelsFn = createServerFn({ method: "POST" }) + .inputValidator( + z.object({ + brandId: z.string(), + enabledModels: z.array(z.string()).nullable(), + }), + ) + .handler(async ({ data }) => { + const session = await requireAuthSession(); + await requireOrgAccess(session.user.id, data.brandId); + + if (data.enabledModels && data.enabledModels.length > 0) { + const activeModels = new Set( + parseScrapeTargets(process.env.SCRAPE_TARGETS).map((c) => c.model), + ); + const unknown = data.enabledModels.filter((m) => !activeModels.has(m)); + if (unknown.length > 0) { + throw new Error( + `Cannot enable model(s) not in SCRAPE_TARGETS: ${unknown.join(", ")}. ` + + `Available models: ${[...activeModels].join(", ") || "(none)"}.`, + ); + } + } + + const [result] = await db + .update(brands) + .set({ enabledModels: data.enabledModels, updatedAt: new Date() }) + .where(eq(brands.id, data.brandId)) + .returning(); + + if (!result) throw new Error("Brand not found"); + return result; + }); + /** * Get competitors for a brand */ diff --git a/apps/worker/src/report-worker.ts b/apps/worker/src/report-worker.ts index 4a11a956..e6d58039 100644 --- a/apps/worker/src/report-worker.ts +++ b/apps/worker/src/report-worker.ts @@ -2,7 +2,12 @@ import { db } from "@workspace/lib/db/db"; import { reports, type Brand, brands } from "@workspace/lib/db/schema"; import { eq } from "drizzle-orm"; import { RUNS_PER_PROMPT } from "@workspace/lib/constants"; -import { getProvider, parseScrapeTargets, type ModelConfig } from "@workspace/lib/providers"; +import { + getProvider, + parseScrapeTargets, + WHITELABEL_REPORT_RUNS_PER_MODEL, + type ModelConfig, +} from "@workspace/lib/providers"; import { analyzeWebsite, getCompetitors, @@ -22,17 +27,6 @@ const TARGET_PROMPTS_COUNT = 70; const MIN_BRAND_MENTIONS = 14; const MAX_BRAND_MENTIONS = 28; -// Whitelabel deployments preserve the legacy asymmetric per-candidate sample -// counts used before SCRAPE_TARGETS drove dispatch. Any model outside this map -// on a whitelabel deployment is a configuration error (the legacy report flow -// only knew how to sample these three). Other deployment modes use -// RUNS_PER_PROMPT (same frequency as day-to-day prompt tracking). -const WHITELABEL_REPORT_RUNS_PER_MODEL: Record = { - chatgpt: 2, - claude: 1, - "google-ai-mode": 1, -}; - function getReportRunsForModel(model: string): number { if (process.env.DEPLOYMENT_MODE === "whitelabel") { const count = WHITELABEL_REPORT_RUNS_PER_MODEL[model]; diff --git a/packages/lib/src/providers/index.ts b/packages/lib/src/providers/index.ts index 79388063..fe7f9192 100644 --- a/packages/lib/src/providers/index.ts +++ b/packages/lib/src/providers/index.ts @@ -11,6 +11,7 @@ export { KNOWN_MODELS, getModelMeta } from "./models"; export type { ModelMeta } from "./models"; export { parseScrapeTargets, validateScrapeTargets } from "./config"; export { selectTargetsForBrand } from "./runner"; +export { WHITELABEL_REPORT_RUNS_PER_MODEL } from "./report-runs"; const providerMap: Record = { olostep, diff --git a/packages/lib/src/providers/report-runs.ts b/packages/lib/src/providers/report-runs.ts new file mode 100644 index 00000000..b3054403 --- /dev/null +++ b/packages/lib/src/providers/report-runs.ts @@ -0,0 +1,14 @@ +/** + * Whitelabel deployments preserve the legacy asymmetric per-candidate sample + * counts used before SCRAPE_TARGETS drove dispatch. Any model outside this + * map on a whitelabel deployment is a configuration error — the legacy report + * flow only knew how to sample these three. + * + * Exported so the web app can warn admins when a SCRAPE_TARGETS entry would + * crash report generation (see `/admin/providers`). + */ +export const WHITELABEL_REPORT_RUNS_PER_MODEL: Record = { + chatgpt: 2, + claude: 1, + "google-ai-mode": 1, +};