From 25e41219e2724f9cf1ae49efbc896da7f82f5179 Mon Sep 17 00:00:00 2001 From: ananthanarayanan-28 Date: Fri, 26 Jun 2026 00:18:29 +0530 Subject: [PATCH 01/10] feat: add developer metrics dashboard and improve job stream robustness with session validation and abort handling --- .../(dashboard)/prompt-library/[id]/page.tsx | 2 +- .../admin/analytics/developer-metrics.tsx | 350 ++++++++++++++++++ frontend/src/components/admin/view-tab.tsx | 13 +- .../src/components/bridge/transfer-detail.tsx | 2 +- .../components/domain-prompts/domain-card.tsx | 2 +- .../domain-prompts/domain-workspace.tsx | 29 +- frontend/src/components/layout/header.tsx | 9 - frontend/src/components/layout/sidebar.tsx | 33 +- .../src/components/optimize/optimize-chat.tsx | 15 +- .../src/components/optimize/result-card.tsx | 12 +- frontend/src/hooks/use-favorites.ts | 7 +- frontend/src/hooks/use-job-stream.ts | 14 +- frontend/src/lib/schemas.ts | 2 +- frontend/src/types/api.ts | 37 +- frontend/src/types/bridge.ts | 2 + qa-chatbot/src/promptly/admin/api/router.py | 316 +++++++++++++++- qa-chatbot/src/promptly/api/v1/prompts.py | 6 +- .../src/promptly/optimize/api/schemas.py | 4 +- qa-chatbot/src/promptly/schemas/prompt.py | 4 - qa-chatbot/src/promptly/schemas/user.py | 2 +- 20 files changed, 773 insertions(+), 88 deletions(-) create mode 100644 frontend/src/components/admin/analytics/developer-metrics.tsx diff --git a/frontend/src/app/(dashboard)/prompt-library/[id]/page.tsx b/frontend/src/app/(dashboard)/prompt-library/[id]/page.tsx index 4e63ecf..d0b8009 100644 --- a/frontend/src/app/(dashboard)/prompt-library/[id]/page.tsx +++ b/frontend/src/app/(dashboard)/prompt-library/[id]/page.tsx @@ -72,7 +72,7 @@ export default function PromptLibraryDetailPage({ const handleUnstar = async () => { if (!window.confirm('Remove this prompt from your library?')) return; try { - await unlikeMutation.mutateAsync(params.id); + await unlikeMutation.mutateAsync({ id: params.id, promptVersionId: data?.prompt_version_id }); router.push('/prompt-library'); } catch { toast.error('Failed to remove from library'); diff --git a/frontend/src/components/admin/analytics/developer-metrics.tsx b/frontend/src/components/admin/analytics/developer-metrics.tsx new file mode 100644 index 0000000..8265b37 --- /dev/null +++ b/frontend/src/components/admin/analytics/developer-metrics.tsx @@ -0,0 +1,350 @@ +'use client'; + +import { useQuery } from '@tanstack/react-query'; +import { api } from '@/lib/api'; +import type { AnalyticsResponse, AnalyticsPoint } from '@/types/analytics'; +import { getSeries } from '@/types/analytics'; +import { MetricCard } from './metric-card'; +import { StaticCard } from './static-card'; + +// ── Colors for status distribution ─────────────────────────────────────────── + +const STATUS_COLORS: Record = { + completed: '#10b981', + failed: '#f43f5e', + queued: '#f59e0b', + calibrating: '#06b6d4', + extracting_mapping: '#8b5cf6', + adapting: '#3b82f6', + cancelled: '#6b7280', +}; + +// ── Section divider ────────────────────────────────────────────────────────── + +function SectionHeader({ title }: { title: string }) { + return ( +
+ + {title} + +
+
+ ); +} + +// ── Rate badge — green/yellow/red indicator ─────────────────────────────────── + +function RateBadge({ rate, inverse = false }: { rate: number; inverse?: boolean }) { + const isGood = inverse ? rate < 5 : rate >= 95; + const isMid = inverse ? rate < 15 : rate >= 80; + const color = isGood ? '#10b981' : isMid ? '#f59e0b' : '#f43f5e'; + return ( + + {rate}% + + ); +} + +// ── Distribution card (categorical labels, no date parsing) ────────────────── + +interface DistItem { + label: string; + value: number; + color: string; +} + +function DistributionCard({ + title, items, subtitle, +}: { + title: string; + items: DistItem[]; + subtitle?: string; +}) { + const total = items.reduce((s, i) => s + i.value, 0); + return ( +
+
+ + {title} + + {subtitle && ( + {subtitle} + )} +
+ + {items.length === 0 ? ( + No data yet + ) : ( +
+ {items.map(item => { + const pct = total > 0 ? (item.value / total) * 100 : 0; + return ( +
+
+
+
+ + {item.label} + +
+
+ + {item.value.toLocaleString()} + + + {pct.toFixed(1)}% + +
+
+
+
+
+
+ ); + })} +
+ )} +
+ ); +} + +function buildStatusItems(points: AnalyticsPoint[]): DistItem[] { + return points.map(p => ({ + label: p.date.replace(/_/g, ' ').replace(/\b\w/g, c => c.toUpperCase()), + value: p.value, + color: STATUS_COLORS[p.date] ?? '#6b7280', + })); +} + +// ── Main component ──────────────────────────────────────────────────────────── + +export function DeveloperMetrics() { + const { data, isLoading, isError } = useQuery({ + queryKey: ['admin', 'analytics', 'developer_metrics'], + queryFn: async () => { + const res = await api.get<{ data: AnalyticsResponse }>( + '/api/v1/admin/analytics?view=developer_metrics&days=30' + ); + return res.data.data; + }, + staleTime: 5 * 60 * 1000, + }); + + if (isLoading) { + return
Loading…
; + } + if (isError || !data) { + return
Failed to load.
; + } + + const st = data.statics; + const s = (key: string) => getSeries(data, key); + + const bridgeSuccessRate = Number(st.bridge_success_rate_pct ?? 0); + const bridgeFailureRate = Number(st.bridge_failure_rate_pct ?? 0); + const bridgeReuseRate = Number(st.bridge_reuse_rate_pct ?? 0); + const queueDepth = Number(st.bridge_queue_depth ?? 0); + const totalBridgeJobs = Number(st.total_bridge_jobs ?? 0); + const bridgeFailedTotal = Number(st.bridge_failed_all_time ?? 0); + const totalOptSessions = Number(st.total_optimizer_sessions ?? 0); + const incompleteSessions = Number(st.optimizer_incomplete_sessions ?? 0); + const optCompletionRate = Number(st.optimizer_completion_rate_pct ?? 0); + + const bridgeStatusItems = buildStatusItems(s('dev_bridge_status_dist')?.data ?? []); + const bridgeReuseItems: DistItem[] = (s('dev_bridge_reuse_dist')?.data ?? []).map(p => ({ + label: p.date, + value: p.value, + color: p.date === 'Reused' ? '#06b6d4' : '#8b5cf6', + })); + + return ( +
+ + {/* ── Top statics ─────────────────────────────────────────────────── */} +
+ + {/* Bridge success rate with badge */} +
+ + Bridge Success Rate + +
+ = 95 ? '#10b981' : bridgeSuccessRate >= 80 ? '#f59e0b' : '#f43f5e', + lineHeight: 1 }}> + {bridgeSuccessRate}% + +
+ + {(totalBridgeJobs - bridgeFailedTotal).toLocaleString()} / {totalBridgeJobs.toLocaleString()} jobs + +
+ + {/* Failure rate */} +
+ + Bridge Failure Rate + + + {bridgeFailureRate}% + + + {bridgeFailedTotal.toLocaleString()} failed all time + +
+ + {/* Queue depth */} +
+ + Bridge Queue Depth + + 10 ? '#f43f5e' : '#f59e0b', + lineHeight: 1 }}> + {queueDepth} + + + non-terminal jobs right now + +
+ + {/* Optimizer completion rate */} +
+ + Optimizer Completion + + = 95 ? '#10b981' : optCompletionRate >= 80 ? '#f59e0b' : '#f43f5e', + lineHeight: 1 }}> + {optCompletionRate}% + + + {incompleteSessions.toLocaleString()} incomplete of {totalOptSessions.toLocaleString()} + +
+ +
+ + {/* Secondary statics: reuse rate + totals */} +
+ + + +
+ + {/* ── Bridge Pipeline ──────────────────────────────────────────────── */} + + +
+ {s('dev_bridge_jobs_daily') && } + {s('dev_bridge_completed_daily') && } + {s('dev_bridge_failed_daily') && } +
+ +
+ + +
+ + {/* ── Optimizer Pipeline ───────────────────────────────────────────── */} + + +
+ {s('dev_optimizer_sessions_daily') && } + {s('dev_incomplete_sessions_daily') && } +
+ + {/* ── API Call Volume ──────────────────────────────────────────────── */} + + +
+ {s('dev_optimize_events_daily') && } + {s('dev_health_score_daily') && } + {s('dev_advisory_daily') && } +
+ + {/* ── External Tools note ──────────────────────────────────────────── */} +
+ + + + + HTTP-level metrics (4xx/5xx rates, request latency, auth failures) are available in your{' '} + Sentry and{' '} + Grafana dashboards. + This view covers application-layer pipeline health not visible in those tools. + +
+ +
+ ); +} diff --git a/frontend/src/components/admin/view-tab.tsx b/frontend/src/components/admin/view-tab.tsx index 48f7f5a..939bdf8 100644 --- a/frontend/src/components/admin/view-tab.tsx +++ b/frontend/src/components/admin/view-tab.tsx @@ -8,16 +8,18 @@ import { AgentOptimizer } from './analytics/agent-optimizer'; import { AgentSkillOpt } from './analytics/agent-skillopt'; import { AgentDomain } from './analytics/agent-domain'; import { AgentBridge } from './analytics/agent-bridge'; +import { DeveloperMetrics } from './analytics/developer-metrics'; type TopToggle = 'platform' | 'agents'; -type PlatformView = 'feature_engagement' | 'login_activity' | 'user_metrics'; +type PlatformView = 'feature_engagement' | 'login_activity' | 'user_metrics' | 'developer_metrics'; type AgentView = 'prompt_optimizer' | 'skill_builder' | 'domain_pdogepa' | 'bridge'; const PLATFORM_ITEMS: { id: PlatformView; label: string }[] = [ - { id: 'feature_engagement', label: 'Feature Engagement' }, - { id: 'login_activity', label: 'Login Activity' }, - { id: 'user_metrics', label: 'User Metrics' }, + { id: 'feature_engagement', label: 'Feature Engagement' }, + { id: 'login_activity', label: 'Login Activity' }, + { id: 'user_metrics', label: 'User Metrics' }, + { id: 'developer_metrics', label: 'Developer Metrics' }, ]; const AGENT_ITEMS: { id: AgentView; label: string }[] = [ @@ -65,6 +67,8 @@ export function ViewTab() { desc: 'Track login activity and daily, weekly, monthly active user trends' }, user_metrics: { title: 'User Metrics', desc: 'User growth, new signups, and daily/weekly active user trends' }, + developer_metrics: { title: 'Developer Metrics', + desc: 'Token throughput, feature API calls, bridge pipeline health, credit economy, and cost tracking' }, prompt_optimizer: { title: 'Prompt Optimizer', desc: 'Council optimizer runs, token consumption, and model distribution' }, skill_builder: { title: 'Skill Builder', @@ -134,6 +138,7 @@ export function ViewTab() { {toggle === 'platform' && platformView === 'feature_engagement' && } {toggle === 'platform' && platformView === 'login_activity' && } {toggle === 'platform' && platformView === 'user_metrics' && } + {toggle === 'platform' && platformView === 'developer_metrics' && } {toggle === 'agents' && agentView === 'prompt_optimizer' && } {toggle === 'agents' && agentView === 'skill_builder' && } {toggle === 'agents' && agentView === 'domain_pdogepa' && } diff --git a/frontend/src/components/bridge/transfer-detail.tsx b/frontend/src/components/bridge/transfer-detail.tsx index c6ae000..d16b49b 100644 --- a/frontend/src/components/bridge/transfer-detail.tsx +++ b/frontend/src/components/bridge/transfer-detail.tsx @@ -494,7 +494,7 @@ export function TransferDetail({ }}>
- +
{mapping?.avg_target_score != null && ( diff --git a/frontend/src/components/domain-prompts/domain-card.tsx b/frontend/src/components/domain-prompts/domain-card.tsx index 12c1829..25c343e 100644 --- a/frontend/src/components/domain-prompts/domain-card.tsx +++ b/frontend/src/components/domain-prompts/domain-card.tsx @@ -114,7 +114,7 @@ export function DomainCard({ fontSize: 11, color: '#5a5a60', fontFamily: 'var(--font-geist-mono, monospace)', }}> - {domain.dataset.row_count} data sources + {domain.dataset.row_count} Q&A pairs )} {domain.optimized_prompt && ( diff --git a/frontend/src/components/domain-prompts/domain-workspace.tsx b/frontend/src/components/domain-prompts/domain-workspace.tsx index 0122bd3..9733401 100644 --- a/frontend/src/components/domain-prompts/domain-workspace.tsx +++ b/frontend/src/components/domain-prompts/domain-workspace.tsx @@ -1,6 +1,7 @@ 'use client'; import { useState, useEffect, useCallback, useMemo, useRef } from 'react'; +import { toast } from 'sonner'; import { useQuery, useQueryClient, useMutation } from '@tanstack/react-query'; import { api } from '@/lib/api'; import type { DomainPrompt, DomainListResponse, DatasetRowsResponse, QAPair, TournamentState, OptimizationRun, RunListResponse } from '@/types/domain-prompts'; @@ -1545,17 +1546,25 @@ export function DomainWorkspace() { ); setPollingDomainId(capturedDomainId); setPollingJobId(res.data.data.job_id); - } catch { setReoptimizing(false); } + } catch (err: unknown) { + setReoptimizing(false); + const detail = (err as { response?: { data?: { detail?: string } } })?.response?.data?.detail; + toast.error(typeof detail === 'string' ? detail : 'Failed to start optimization — please try again.'); + } }, [selected]); + const [confirmDelete, setConfirmDelete] = useState(false); const handleDelete = useCallback(async () => { if (!selected) return; - if (!window.confirm('Delete this domain and all its data? This cannot be undone.')) return; try { await api.delete(`/api/v1/domain-prompts/${selected.id}`); setSelectedId(null); + setConfirmDelete(false); void qc.invalidateQueries({ queryKey: ['domain-prompts'] }); - } catch { /* ignore */ } + } catch { + toast.error('Failed to delete domain — please try again.'); + setConfirmDelete(false); + } }, [selected, qc]); const [cancelling, setCancelling] = useState(false); @@ -1658,9 +1667,17 @@ export function DomainWorkspace() { {recovering ? 'Restoring…' : 'Restore to Ready'} )} - + {confirmDelete ? ( + <> + Delete? + + + + ) : ( + + )}
); diff --git a/frontend/src/components/layout/sidebar.tsx b/frontend/src/components/layout/sidebar.tsx index 630d734..c4186a4 100644 --- a/frontend/src/components/layout/sidebar.tsx +++ b/frontend/src/components/layout/sidebar.tsx @@ -94,7 +94,22 @@ function formatTokens(n: number): string { return String(n); } -function TokenCard({ tokenBalance }: { tokenBalance: number }) { +function TokenCard({ tokenBalance }: { tokenBalance: number | undefined }) { + if (tokenBalance === undefined) { + return ( +
+
+ Tokens +
+
+
+ +
+
+
+ ); + } + // Clamp display at 0 — never reveal the internal overdraft buffer to users. const displayed = Math.max(0, tokenBalance); const isDepleted = displayed === 0; @@ -123,7 +138,7 @@ function TokenCard({ tokenBalance }: { tokenBalance: number }) { } function RecentSessions() { - const { data } = useQuery({ + const { data, isLoading, isError } = useQuery({ queryKey: ['sessions'], queryFn: async () => { const res = await api.get<{ data: SessionsGrouped }>('/api/v1/chat/sessions'); @@ -132,6 +147,18 @@ function RecentSessions() { staleTime: 60_000, }); + if (isLoading) { + return ( +
+ {[80, 65, 90].map((w, i) => ( +
+ ))} +
+ ); + } + + if (isError) return null; + const sessions: SessionSummary[] = data ? [...data.today, ...data.last_7_days, ...data.last_30_days, ...data.older].slice(0, 5) : []; @@ -185,7 +212,7 @@ export function Sidebar() { staleTime: 1000 * 60 * 5, }); - const tokenBalance = fetchedUser?.token_balance ?? TOKEN_START; + const tokenBalance = fetchedUser?.token_balance; return (