diff --git a/backend/app/routes/api.diarization.ts b/backend/app/routes/api.diarization.ts new file mode 100644 index 00000000..9595c926 --- /dev/null +++ b/backend/app/routes/api.diarization.ts @@ -0,0 +1,172 @@ +/** + * Proxy routes for the diarization/speaker recognition service. + * Forwards requests to the diarization service (default: http://localhost:8085). + */ +import type { Request, Response } from "express"; +import { authenticateOr401 } from "../lib/auth/core.server.ts"; + +// Get diarization service URL from environment or use default +const DIARIZATION_SERVICE_URL = ( + Deno.env.get("DIARIZATION_SERVER_URL") || + Deno.env.get("SPEAKER_SERVICE_URL") || + "http://localhost:8085" +).replace(/\/$/, ""); + +/** + * Proxy handler for diarization service requests. + * Supports both GET and POST requests, forwarding to the speaker recognition service. + */ +export async function apiDiarizationProxyHandler( + req: Request, + res: Response +): Promise { + try { + // Authenticate the request + await authenticateOr401(req, res); + + // Extract the path after /api/diarization + const subPath = req.params[0] || ""; + const targetUrl = `${DIARIZATION_SERVICE_URL}/${subPath}`; + + // Build query string if present + const queryString = new URLSearchParams(req.query as Record).toString(); + const fullUrl = queryString ? `${targetUrl}?${queryString}` : targetUrl; + + console.log(`[diarization-proxy] ${req.method} ${fullUrl}`); + + // Prepare headers (forward auth if needed, but diarization service is internal) + const headers: Record = { + "Accept": "application/json", + }; + + // Handle different request types + let body: BodyInit | undefined; + const contentType = req.headers["content-type"] || ""; + + if (req.method !== "GET" && req.method !== "HEAD") { + if (contentType.includes("multipart/form-data")) { + // For multipart requests, we need to reconstruct the form data + // This is tricky with Express - we'll pass through the raw request + // Actually, since Express parses the body, we need to handle this differently + + // For file uploads, we need to handle this specially + // The frontend should send directly to the diarization service for file uploads + // Or we need to use a middleware like multer + + // For now, let's handle JSON requests and simple form data + res.status(400).json({ + error: "File upload proxy not yet implemented. Please use direct upload to diarization service." + }); + return; + } else if (contentType.includes("application/json")) { + headers["Content-Type"] = "application/json"; + body = JSON.stringify(req.body); + } else if (req.body) { + headers["Content-Type"] = contentType; + body = JSON.stringify(req.body); + } + } + + // Make the proxied request + const proxyResponse = await fetch(fullUrl, { + method: req.method, + headers, + body, + }); + + // Forward the response + const responseContentType = proxyResponse.headers.get("content-type") || "application/json"; + res.status(proxyResponse.status); + res.setHeader("Content-Type", responseContentType); + + if (responseContentType.includes("application/json")) { + const data = await proxyResponse.json(); + res.json(data); + } else { + const buffer = await proxyResponse.arrayBuffer(); + res.send(Buffer.from(buffer)); + } + } catch (error) { + if (error instanceof Error && error.message === "Unauthorized") { + return; // Already sent 401 response + } + + console.error("[diarization-proxy] Error:", error); + + // Check if it's a connection error to the diarization service + if (error instanceof TypeError && error.message.includes("fetch failed")) { + res.status(503).json({ + error: "Diarization service unavailable", + message: `Could not connect to diarization service at ${DIARIZATION_SERVICE_URL}`, + hint: "Make sure the diarization service is running (docker compose up diarization)" + }); + return; + } + + res.status(500).json({ + error: "Internal server error", + message: error instanceof Error ? error.message : String(error) + }); + } +} + +/** + * Dedicated handler for speaker enrollment with file uploads. + * Uses multer or similar for handling multipart/form-data. + */ +export async function apiDiarizationEnrollHandler( + req: Request, + res: Response +): Promise { + try { + await authenticateOr401(req, res); + + // For file uploads, we need to stream the request body + // This is a special case that requires different handling + + const targetUrl = `${DIARIZATION_SERVICE_URL}/enroll/batch`; + console.log(`[diarization-proxy] POST ${targetUrl} (file upload)`); + + // Get the raw content-type header including boundary + const contentType = req.headers["content-type"]; + if (!contentType?.includes("multipart/form-data")) { + res.status(400).json({ error: "Expected multipart/form-data" }); + return; + } + + // For multipart uploads, we need to forward the raw body + // This requires collecting the raw body chunks + const chunks: Uint8Array[] = []; + + await new Promise((resolve, reject) => { + req.on("data", (chunk: Buffer) => { + chunks.push(new Uint8Array(chunk)); + }); + req.on("end", resolve); + req.on("error", reject); + }); + + const bodyBuffer = Buffer.concat(chunks.map(c => Buffer.from(c))); + + const proxyResponse = await fetch(targetUrl, { + method: "POST", + headers: { + "Content-Type": contentType, + }, + body: bodyBuffer, + }); + + const data = await proxyResponse.json(); + res.status(proxyResponse.status).json(data); + } catch (error) { + if (error instanceof Error && error.message === "Unauthorized") { + return; + } + + console.error("[diarization-proxy] Enroll error:", error); + res.status(500).json({ + error: "Enrollment failed", + message: error instanceof Error ? error.message : String(error) + }); + } +} diff --git a/backend/routes.ts b/backend/routes.ts index 651021d6..4cb9028d 100644 --- a/backend/routes.ts +++ b/backend/routes.ts @@ -23,6 +23,7 @@ import { authJwtLoginHandler } from "@/routes/auth.jwt.login.ts"; import { wellKnownOauthAuthorizationServerHandler } from "@/routes/[.]well-known.oauth-authorization-server.ts"; import { wellKnownOauthProtectedResourceHandler } from "@/routes/[.]well-known.oauth-protected-resource.ts"; import { apiChatHandler } from "@/routes/api.chat.ts"; +import { apiDiarizationProxyHandler, apiDiarizationEnrollHandler } from "@/routes/api.diarization.ts"; import { asyncHandler } from "@/middleware/asyncHandler.ts"; export function registerRoutes(app: Express): void { @@ -38,6 +39,8 @@ export function registerRoutes(app: Express): void { app.get("/api/audio/stream", apiAudioStreamHandler); app.post("/api/audio/upload", apiAudioUploadHandler); app.get("/api/audio/wav", apiAudioWavHandler); + // Diarization service proxy routes + app.all("/api/diarization/*", asyncHandler(apiDiarizationProxyHandler)); app.get("/mcp", mcpGetHandler); app.post("/mcp", mcpPostHandler); app.post("/llm/chat/completions", llmChatCompletionsHandler); diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index 7ffe2336..6d3a5bd9 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -1,5 +1,5 @@ import { Link, Navigate, Outlet, useLocation } from "react-router-dom"; -import { Clock, Home, Package, Settings, MessageSquare, Activity, Mic } from "lucide-react"; +import { Clock, Home, Package, Settings, MessageSquare, Activity, Mic, Users } from "lucide-react"; import { useTheme } from "@/hooks/useTheme"; import { useSettingsStore } from "@/stores/settingsStore"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; @@ -92,6 +92,16 @@ const Layout = () => { Pipeline + + + Speakers + { + const navigate = useNavigate(); + const [searchParams] = useSearchParams(); + const { timeFormat } = useSettingsStore(); + const fileInputRef = useRef(null); + + // State + const [speakers, setSpeakers] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + // Enroll dialog state + const [enrollDialogOpen, setEnrollDialogOpen] = useState(false); + const [enrollName, setEnrollName] = useState(""); + const [enrollFiles, setEnrollFiles] = useState([]); + const [enrolling, setEnrolling] = useState(false); + const [enrollError, setEnrollError] = useState(null); + + // Speaker detail sheet state + const [selectedSpeaker, setSelectedSpeaker] = useState(null); + const [detailSheetOpen, setDetailSheetOpen] = useState(false); + + // Timeline selection state for enrollment + const [timelineSelectionMode, setTimelineSelectionMode] = useState(false); + const [selectedSegments, setSelectedSegments] = useState([]); + const [segmentRangeStart, setSegmentRangeStart] = useState(); + const [segmentRangeEnd, setSegmentRangeEnd] = useState(); + const [diarizationSegments, setDiarizationSegments] = useState([]); + const [loadingSegments, setLoadingSegments] = useState(false); + + // Delete confirmation + const [deleteConfirmSpeaker, setDeleteConfirmSpeaker] = useState(null); + const [deleting, setDeleting] = useState(false); + + // Default user ID (could be from auth context in real app) + const userId = 1; + + // Load speakers from diarization service + const loadSpeakers = useCallback(async () => { + setLoading(true); + setError(null); + try { + const response = await apiClient.get<{ speakers: Speaker[] }>( + `${DIARIZATION_SERVICE_URL}/speakers` + ); + // Filter speakers for current user + const userSpeakers = (response.speakers || []).filter( + (s) => s.id.startsWith(`user_${userId}_`) + ); + setSpeakers(userSpeakers); + } catch (err) { + console.error("Failed to load speakers:", err); + setError(err instanceof Error ? err.message : "Failed to load speakers"); + setSpeakers([]); + } finally { + setLoading(false); + } + }, [userId]); + + useEffect(() => { + loadSpeakers(); + }, [loadSpeakers]); + + // Handle file selection for enrollment + const handleFileSelect = (e: React.ChangeEvent) => { + const files = e.target.files; + if (!files) return; + + const newFiles: EnrollmentFile[] = Array.from(files).map((file) => ({ + file, + name: file.name, + })); + setEnrollFiles((prev) => [...prev, ...newFiles]); + }; + + const removeEnrollFile = (index: number) => { + setEnrollFiles((prev) => prev.filter((_, i) => i !== index)); + }; + + // Enroll a new speaker + const handleEnroll = async () => { + if (!enrollName.trim()) { + setEnrollError("Please enter a speaker name"); + return; + } + + if (enrollFiles.length === 0 && selectedSegments.length === 0) { + setEnrollError("Please add at least one audio file or select timeline segments"); + return; + } + + setEnrolling(true); + setEnrollError(null); + + try { + const speakerId = `user_${userId}_${enrollName.toLowerCase().replace(/\s+/g, "_")}`; + const formData = new FormData(); + formData.append("speaker_id", speakerId); + formData.append("speaker_name", enrollName); + + // Add files + enrollFiles.forEach((ef) => { + formData.append("files", ef.file); + }); + + // If we have timeline segments, we need to fetch their audio and add to enrollment + if (selectedSegments.length > 0) { + for (const segment of selectedSegments) { + const originalId = segment.original_id; + const startSec = segment.start.getTime() / 1000; + const endSec = segment.end.getTime() / 1000; + + // Fetch audio for this segment + const blob = await apiClient.getBlob( + `/api/audio/wav?start=${startSec}&end=${endSec}&original_id=${originalId}` + ); + const segmentFile = new File( + [blob], + `segment_${segment.start.getTime()}.wav`, + { type: "audio/wav" } + ); + formData.append("files", segmentFile); + } + } + + // Call enrollment API + const response = await fetch(`${apiClient.baseURL}${DIARIZATION_SERVICE_URL}/enroll/batch`, { + method: "POST", + headers: await apiClient.getAuthHeaders(), + body: formData, + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + throw new Error(errorData.detail || `Enrollment failed: ${response.statusText}`); + } + + // Success - reload speakers and close dialog + await loadSpeakers(); + setEnrollDialogOpen(false); + setEnrollName(""); + setEnrollFiles([]); + setSelectedSegments([]); + } catch (err) { + console.error("Enrollment failed:", err); + setEnrollError(err instanceof Error ? err.message : "Enrollment failed"); + } finally { + setEnrolling(false); + } + }; + + // Delete a speaker + const handleDelete = async (speaker: Speaker) => { + setDeleting(true); + try { + await fetch(`${apiClient.baseURL}${DIARIZATION_SERVICE_URL}/speakers/${speaker.id}`, { + method: "DELETE", + headers: await apiClient.getAuthHeaders(), + }); + await loadSpeakers(); + setDeleteConfirmSpeaker(null); + } catch (err) { + console.error("Delete failed:", err); + } finally { + setDeleting(false); + } + }; + + // Load diarization segments for timeline selection + const loadDiarizationSegments = async () => { + if (!segmentRangeStart || !segmentRangeEnd) return; + + setLoadingSegments(true); + try { + const docs: DiarizationSegment[] = await callResource("mongo", { + action: "find", + collection: "diarizations", + query: { + start: { $lt: segmentRangeEnd }, + end: { $gt: segmentRangeStart }, + }, + options: { sort: { start: 1 }, limit: 500 }, + }); + setDiarizationSegments(docs); + } catch (err) { + console.error("Failed to load segments:", err); + } finally { + setLoadingSegments(false); + } + }; + + const toggleSegmentSelection = (segment: DiarizationSegment) => { + setSelectedSegments((prev) => { + const exists = prev.some( + (s) => s.start.getTime() === segment.start.getTime() && s.end.getTime() === segment.end.getTime() + ); + if (exists) { + return prev.filter( + (s) => !(s.start.getTime() === segment.start.getTime() && s.end.getTime() === segment.end.getTime()) + ); + } + return [...prev, segment]; + }); + }; + + // Render + if (loading) { + return ( +
+

Speakers

+
+ +

Loading speakers...

+
+
+ ); + } + + return ( +
+ {/* Header */} +
+
+

Speakers

+

+ Manage enrolled speaker profiles for voice identification +

+
+
+ + + + + + + + Enroll New Speaker + + Add audio samples to create a voice profile for speaker identification + + + + + + + + Upload Files + + + + From Timeline + + + + {/* Upload Files Tab */} + +
+ + setEnrollName(e.target.value)} + /> +
+ +
+ +
+ +

+ Upload WAV or FLAC files (10-30 seconds each recommended) +

+ + +
+
+ + {/* Selected files list */} + {enrollFiles.length > 0 && ( +
+ +
+ {enrollFiles.map((ef, idx) => ( +
+
+ + + {ef.name} + +
+ +
+ ))} +
+
+ )} + + + {/* Timeline Selection Tab */} + +
+ + setEnrollName(e.target.value)} + /> +
+ +
+
+ + date && setSegmentRangeStart(date)} + placeholder="Select start" + /> +
+
+ + date && setSegmentRangeEnd(date)} + placeholder="Select end" + /> +
+
+ + + + {/* Diarization segments list */} + {diarizationSegments.length > 0 && ( +
+ +
+ {diarizationSegments.map((segment, idx) => { + const isSelected = selectedSegments.some( + (s) => + s.start.getTime() === segment.start.getTime() && + s.end.getTime() === segment.end.getTime() + ); + const color = embeddingToColor(segment.embedding) || "#888"; + const duration = + (segment.end.getTime() - segment.start.getTime()) / 1000; + + return ( +
toggleSegmentSelection(segment)} + > +
+
+
+
+ {formatTime(segment.start, timeFormat)} +
+
+ {formatDuration(duration)} +
+
+
+ {isSelected && ( + + )} +
+ ); + })} +
+
+ )} + + {/* Selected segments summary */} + {selectedSegments.length > 0 && ( +
+

+ {selectedSegments.length} segment(s) selected +

+

+ Total duration:{" "} + {formatDuration( + selectedSegments.reduce( + (acc, s) => acc + (s.end.getTime() - s.start.getTime()) / 1000, + 0 + ) + )} +

+
+ )} + + + + {enrollError && ( +
+ {enrollError} +
+ )} + + + + + + +
+
+
+ + {/* Error display */} + {error && ( +
+ {error} +
+ )} + + {/* Speakers grid */} + {speakers.length === 0 ? ( + + + +

No Speakers Enrolled

+

+ Enroll speakers to enable voice identification in your recordings +

+ + + + ) : ( +
+ {speakers.map((speaker) => { + const color = embeddingToColor(speaker.embedding_data) || "#6366f1"; + return ( + { + setSelectedSpeaker(speaker); + setDetailSheetOpen(true); + }} + > + +
+
+ +
+
+ {speaker.name} + + {speaker.id} + +
+ +
+ + +
+
+ Samples: + + {speaker.audio_sample_count || 0} + +
+
+ Duration: + + {speaker.total_audio_duration + ? formatDuration(speaker.total_audio_duration) + : "N/A"} + +
+
+ + + ); + })} +
+ )} + + {/* Speaker detail sheet */} + + + {selectedSpeaker && ( + <> + + +
+ +
+ {selectedSpeaker.name} + + {selectedSpeaker.id} + + +
+ {/* Stats */} +
+
+
+ Audio Samples +
+
+ {selectedSpeaker.audio_sample_count || 0} +
+
+
+
+ Total Duration +
+
+ {selectedSpeaker.total_audio_duration + ? formatDuration(selectedSpeaker.total_audio_duration) + : "N/A"} +
+
+
+ + {/* Dates */} + {selectedSpeaker.created_at && ( +
+ +

+ {formatTime(new Date(selectedSpeaker.created_at), timeFormat)} +

+
+ )} + {selectedSpeaker.updated_at && ( +
+ +

+ {formatTime(new Date(selectedSpeaker.updated_at), timeFormat)} +

+
+ )} + + {/* Actions */} +
+ + +
+
+ + )} + + + + {/* Delete confirmation dialog */} + !open && setDeleteConfirmSpeaker(null)} + > + + + Delete Speaker + + Are you sure you want to delete "{deleteConfirmSpeaker?.name}"? This + action cannot be undone and will remove all associated voice data. + + + + + + + + +
+ ); +}; + +export default SpeakersPage; diff --git a/frontend/src/router.tsx b/frontend/src/router.tsx index c744f1d5..e2a7a6a5 100644 --- a/frontend/src/router.tsx +++ b/frontend/src/router.tsx @@ -34,6 +34,7 @@ import JobDetailPage from "./pages/JobDetailPage"; import CreateJobPage from "./pages/CreateJobPage"; import ConversationsPage from "./pages/ConversationsPage"; import AudioPipelinePage from "./pages/AudioPipelinePage"; +import SpeakersPage from "./pages/SpeakersPage"; export const router = createBrowserRouter([ { @@ -96,6 +97,10 @@ export const router = createBrowserRouter([ path: "conversations", element: , }, + { + path: "speakers", + element: , + }, { path: "diarizations/:id", element: ,