From 3b03b710b7a30646c0af9d0462b63ce2490bcd04 Mon Sep 17 00:00:00 2001 From: CreatorGhost Date: Mon, 30 Mar 2026 12:08:35 +0530 Subject: [PATCH 1/5] feat: add OpenAI Whisper API support (bring your own key) Add OpenAI Whisper as a cloud transcription option, allowing users to use their own OpenAI API key for speech-to-text. This provides an alternative to Amical Cloud and local models. Changes: - New OpenAI Whisper transcription provider with VAD filtering, WAV encoding, and 60s request timeout - Settings UI for API key configuration with connection validation - Onboarding flow integration with dedicated setup modal - Model selection persistence through app restart (fix race condition in onboarding completion and startup validation) - Proper error handling with CLOUD_TRANSCRIPTION_FAILED error code - i18n support for en, es, ja, zh-TW locales - Unit tests for the provider Co-Authored-By: Claude Opus 4.6 --- apps/desktop/src/constants/models.ts | 31 ++ apps/desktop/src/constants/provider-types.ts | 6 + apps/desktop/src/db/schema.ts | 3 + apps/desktop/src/i18n/locales/en.json | 30 ++ apps/desktop/src/i18n/locales/es.json | 4 + apps/desktop/src/i18n/locales/ja.json | 4 + apps/desktop/src/i18n/locales/zh-TW.json | 4 + .../transcription/openai-whisper-provider.ts | 306 +++++++++++++ .../settings/ai-models/tabs/SpeechTab.tsx | 196 +++++++- .../screens/ModelSelectionScreen.tsx | 19 +- .../components/screens/ModelSetupModal.tsx | 110 +++++ apps/desktop/src/services/model-service.ts | 93 +++- .../src/services/onboarding-service.ts | 47 +- apps/desktop/src/services/settings-service.ts | 31 ++ .../src/services/transcription-service.ts | 64 ++- apps/desktop/src/trpc/routers/models.ts | 10 + apps/desktop/src/trpc/routers/settings.ts | 57 +++ apps/desktop/src/types/error.ts | 1 + apps/desktop/src/types/onboarding.ts | 1 + apps/desktop/src/types/widget-notification.ts | 18 + apps/desktop/src/utils/model-selection.ts | 20 +- .../pipeline/openai-whisper-provider.test.ts | 432 ++++++++++++++++++ 22 files changed, 1447 insertions(+), 40 deletions(-) create mode 100644 apps/desktop/src/pipeline/providers/transcription/openai-whisper-provider.ts create mode 100644 apps/desktop/tests/pipeline/openai-whisper-provider.test.ts diff --git a/apps/desktop/src/constants/models.ts b/apps/desktop/src/constants/models.ts index d3a53495..6f744526 100644 --- a/apps/desktop/src/constants/models.ts +++ b/apps/desktop/src/constants/models.ts @@ -148,6 +148,37 @@ export const AVAILABLE_MODELS: AvailableWhisperModel[] = [ provider: "Amical Cloud", providerIcon: "/assets/icon_logo.svg", }, + { + id: "openai-whisper", + name: "OpenAI Whisper", + type: "whisper", + description: "Cloud transcription using your own OpenAI API key.", + checksum: "", + filename: "", + downloadUrl: "", + size: 0, + sizeFormatted: "Cloud", + modelSize: "Cloud", + features: [ + { + icon: "key-round", + tooltip: "Bring your own API key", + }, + { + icon: "bolt", + tooltip: "Fast cloud transcription", + }, + { + icon: "languages", + tooltip: "Multilingual support", + }, + ], + speed: 4.0, + accuracy: 4.5, + setup: "cloud", + provider: "OpenAI", + providerIcon: "/icons/models/openai_dark.svg", + }, { id: "whisper-tiny", name: "Whisper Tiny", diff --git a/apps/desktop/src/constants/provider-types.ts b/apps/desktop/src/constants/provider-types.ts index cc8d4a79..d3e959f1 100644 --- a/apps/desktop/src/constants/provider-types.ts +++ b/apps/desktop/src/constants/provider-types.ts @@ -6,6 +6,7 @@ export const PROVIDER_TYPES = { openRouter: "openrouter", ollama: "ollama", openAICompatible: "openai-compatible", + openAIWhisper: "openai-whisper", } as const; export type ProviderType = (typeof PROVIDER_TYPES)[keyof typeof PROVIDER_TYPES]; @@ -16,6 +17,7 @@ export const SYSTEM_PROVIDER_INSTANCE_IDS = { openRouter: "system-openrouter", ollama: "system-ollama", openAICompatible: "system-openai-compatible", + openAIWhisper: "system-openai-whisper", } as const; export function getRemoteProviderType(provider: RemoteProvider): ProviderType { @@ -43,6 +45,8 @@ export function getSystemProviderInstanceId( return SYSTEM_PROVIDER_INSTANCE_IDS.ollama; case PROVIDER_TYPES.openAICompatible: return SYSTEM_PROVIDER_INSTANCE_IDS.openAICompatible; + case PROVIDER_TYPES.openAIWhisper: + return SYSTEM_PROVIDER_INSTANCE_IDS.openAIWhisper; } } @@ -58,5 +62,7 @@ export function getProviderDisplayName(providerType: ProviderType): string { return REMOTE_PROVIDERS.ollama; case PROVIDER_TYPES.openAICompatible: return REMOTE_PROVIDERS.openAICompatible; + case PROVIDER_TYPES.openAIWhisper: + return "OpenAI Whisper"; } } diff --git a/apps/desktop/src/db/schema.ts b/apps/desktop/src/db/schema.ts index c7b7ea69..4303be3e 100644 --- a/apps/desktop/src/db/schema.ts +++ b/apps/desktop/src/db/schema.ts @@ -164,6 +164,9 @@ export interface AppSettingsData { apiKey: string; baseURL: string; }; + openAIWhisper?: { + apiKey: string; + }; defaultSpeechModel?: string; // Selection key "::speech::" or legacy speech model ID defaultLanguageModel?: string; // Selection key "::language::" or legacy language model ID defaultEmbeddingModel?: string; // Selection key "::embedding::" or legacy embedding model ID diff --git a/apps/desktop/src/i18n/locales/en.json b/apps/desktop/src/i18n/locales/en.json index 7f1345ec..3d351a8f 100644 --- a/apps/desktop/src/i18n/locales/en.json +++ b/apps/desktop/src/i18n/locales/en.json @@ -181,6 +181,19 @@ "internetLogin": "Needs internet and login" } }, + "openai": { + "title": "OpenAI Whisper", + "subtitle": "Bring your own API key - fast cloud transcription", + "description": "Use your own OpenAI API key for cloud-based Whisper transcription. No login required.", + "pros": { + "accurate": "High accuracy", + "fast": "Fast cloud processing", + "noLogin": "No login needed" + }, + "cons": { + "apiKey": "Requires OpenAI API key" + } + }, "local": { "title": "Local Models", "subtitle": "Private, offline, and free - runs fully on your device.", @@ -212,6 +225,19 @@ "loginStartFailed": "Failed to open login. Please try again." } }, + "openai": { + "title": "Connect OpenAI API Key", + "description": "Enter your OpenAI API key to use cloud-based Whisper transcription.", + "placeholder": "Enter your OpenAI API key (sk-...)", + "connect": "Connect", + "connected": "API key validated and saved!", + "toast": { + "validated": "OpenAI API key validated successfully" + }, + "error": { + "validationFailed": "Failed to validate API key. Please check and try again." + } + }, "local": { "downloadingTitle": "Downloading Local Model", "downloadingDescription": "Setting up local model for private, offline transcription", @@ -326,6 +352,10 @@ "localTranscriptionFailed": { "title": "Local transcription failed", "description": "Whisper audio processing failed" + }, + "cloudTranscriptionFailed": { + "title": "Cloud transcription failed", + "description": "OpenAI Whisper API request failed" } } } diff --git a/apps/desktop/src/i18n/locales/es.json b/apps/desktop/src/i18n/locales/es.json index 6b899f65..688969bf 100644 --- a/apps/desktop/src/i18n/locales/es.json +++ b/apps/desktop/src/i18n/locales/es.json @@ -326,6 +326,10 @@ "localTranscriptionFailed": { "title": "La transcripción local falló", "description": "El procesamiento de audio de Whisper falló" + }, + "cloudTranscriptionFailed": { + "title": "Cloud transcription failed", + "description": "OpenAI Whisper API request failed" } } } diff --git a/apps/desktop/src/i18n/locales/ja.json b/apps/desktop/src/i18n/locales/ja.json index 2e0ccba1..79fb2e38 100644 --- a/apps/desktop/src/i18n/locales/ja.json +++ b/apps/desktop/src/i18n/locales/ja.json @@ -326,6 +326,10 @@ "localTranscriptionFailed": { "title": "ローカル文字起こしに失敗しました", "description": "Whisperの音声処理に失敗しました" + }, + "cloudTranscriptionFailed": { + "title": "Cloud transcription failed", + "description": "OpenAI Whisper API request failed" } } } diff --git a/apps/desktop/src/i18n/locales/zh-TW.json b/apps/desktop/src/i18n/locales/zh-TW.json index 944cb22a..bbc1a054 100644 --- a/apps/desktop/src/i18n/locales/zh-TW.json +++ b/apps/desktop/src/i18n/locales/zh-TW.json @@ -326,6 +326,10 @@ "localTranscriptionFailed": { "title": "本機轉錄失敗", "description": "Whisper 音訊處理失敗" + }, + "cloudTranscriptionFailed": { + "title": "Cloud transcription failed", + "description": "OpenAI Whisper API request failed" } } } diff --git a/apps/desktop/src/pipeline/providers/transcription/openai-whisper-provider.ts b/apps/desktop/src/pipeline/providers/transcription/openai-whisper-provider.ts new file mode 100644 index 00000000..64cf11c1 --- /dev/null +++ b/apps/desktop/src/pipeline/providers/transcription/openai-whisper-provider.ts @@ -0,0 +1,306 @@ +import { + TranscriptionProvider, + TranscribeParams, + TranscribeContext, + TranscriptionOutput, +} from "../../core/pipeline-types"; +import { logger } from "../../../main/logger"; +import { SettingsService } from "../../../services/settings-service"; +import { AppError, ErrorCodes } from "../../../types/error"; +import { extractSpeechFromVad } from "../../utils/vad-audio-filter"; +import { getUserAgent } from "../../../utils/http-client"; + +/** + * Encode raw PCM Float32 samples into a WAV file buffer. + * Output: 16-bit mono PCM WAV at the given sample rate. + */ +function encodeWav(samples: Float32Array, sampleRate: number): Buffer { + const numChannels = 1; + const bitsPerSample = 16; + const byteRate = sampleRate * numChannels * (bitsPerSample / 8); + const blockAlign = numChannels * (bitsPerSample / 8); + const dataSize = samples.length * (bitsPerSample / 8); + const headerSize = 44; + const buffer = Buffer.alloc(headerSize + dataSize); + + // RIFF header + buffer.write("RIFF", 0); + buffer.writeUInt32LE(36 + dataSize, 4); + buffer.write("WAVE", 8); + + // fmt sub-chunk + buffer.write("fmt ", 12); + buffer.writeUInt32LE(16, 16); // Sub-chunk size + buffer.writeUInt16LE(1, 20); // PCM format + buffer.writeUInt16LE(numChannels, 22); + buffer.writeUInt32LE(sampleRate, 24); + buffer.writeUInt32LE(byteRate, 28); + buffer.writeUInt16LE(blockAlign, 32); + buffer.writeUInt16LE(bitsPerSample, 34); + + // data sub-chunk + buffer.write("data", 36); + buffer.writeUInt32LE(dataSize, 40); + + // Convert Float32 [-1, 1] to Int16 + for (let i = 0; i < samples.length; i++) { + const clamped = Math.max(-1, Math.min(1, samples[i])); + const int16 = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff; + buffer.writeInt16LE(Math.round(int16), headerSize + i * 2); + } + + return buffer; +} + +export class OpenAIWhisperProvider implements TranscriptionProvider { + readonly name = "openai-whisper"; + + private settingsService: SettingsService; + + // Frame aggregation state (same pattern as WhisperProvider) + private frameBuffer: Float32Array[] = []; + private frameBufferSpeechProbabilities: number[] = []; + private currentSilenceFrameCount = 0; + + // Configuration + private readonly FRAME_SIZE = 512; // 32ms at 16kHz + private readonly MIN_AUDIO_DURATION_MS = 500; + private readonly MAX_SILENCE_DURATION_MS = 3000; + private readonly SAMPLE_RATE = 16000; + private readonly SPEECH_PROBABILITY_THRESHOLD = 0.2; + private readonly OPENAI_API_URL = + "https://api.openai.com/v1/audio/transcriptions"; + + constructor(settingsService: SettingsService) { + this.settingsService = settingsService; + + logger.transcription.info("OpenAIWhisperProvider initialized"); + } + + /** + * Process an audio chunk - buffers and conditionally transcribes + */ + async transcribe(params: TranscribeParams): Promise { + const { audioData, speechProbability = 1 } = params; + + // Add frame to buffer with speech probability + this.frameBuffer.push(audioData); + this.frameBufferSpeechProbabilities.push(speechProbability); + + // Consider it speech if probability is above threshold + const isSpeech = speechProbability > this.SPEECH_PROBABILITY_THRESHOLD; + + if (isSpeech) { + this.currentSilenceFrameCount = 0; + } else { + this.currentSilenceFrameCount++; + } + + // Only transcribe if speech/silence patterns indicate we should + if (!this.shouldTranscribe()) { + return { text: "" }; + } + + return this.doTranscription(params.context); + } + + /** + * Flush any buffered audio and return transcription + * Called at the end of a recording session + */ + async flush(context: TranscribeContext): Promise { + if (this.frameBuffer.length === 0) { + return { text: "" }; + } + + return this.doTranscription(context); + } + + /** + * Clear internal buffers without transcribing + */ + reset(): void { + this.frameBuffer = []; + this.frameBufferSpeechProbabilities = []; + this.currentSilenceFrameCount = 0; + } + + private shouldTranscribe(): boolean { + const audioDurationMs = + ((this.frameBuffer.length * this.FRAME_SIZE) / this.SAMPLE_RATE) * 1000; + const silenceDurationMs = + ((this.currentSilenceFrameCount * this.FRAME_SIZE) / this.SAMPLE_RATE) * + 1000; + + if ( + audioDurationMs >= this.MIN_AUDIO_DURATION_MS && + silenceDurationMs > this.MAX_SILENCE_DURATION_MS + ) { + return true; + } + + // If buffer is too large (30 seconds), transcribe anyway + if (audioDurationMs > 30000) { + return true; + } + + return false; + } + + private async doTranscription( + context: TranscribeContext, + ): Promise { + try { + // Get API key + const config = await this.settingsService.getOpenAIWhisperConfig(); + if (!config?.apiKey) { + throw new AppError( + "OpenAI API key not configured", + ErrorCodes.AUTH_REQUIRED, + ); + } + + // Capture speech probabilities before reset + const vadProbs = [...this.frameBufferSpeechProbabilities]; + + // Aggregate buffered frames + const rawAudio = this.aggregateFrames(); + + // Clear buffers immediately after aggregation + this.reset(); + + // Apply VAD filtering to extract speech-only portions + const { audio: filteredAudio, segments: speechSegments } = + extractSpeechFromVad(rawAudio, vadProbs); + + if (filteredAudio.length === 0) { + logger.transcription.debug( + "OpenAI Whisper: Skipping - no speech detected by VAD filter", + ); + return { text: "" }; + } + + logger.transcription.debug( + `OpenAI Whisper: VAD filtered ${rawAudio.length} -> ${filteredAudio.length} samples (${speechSegments.length} speech segments, ${((filteredAudio.length / rawAudio.length) * 100).toFixed(0)}% kept)`, + ); + + // Convert to WAV + const wavBuffer = encodeWav(filteredAudio, this.SAMPLE_RATE); + + logger.transcription.info("Sending audio to OpenAI Whisper API", { + audioSamples: filteredAudio.length, + wavSizeBytes: wavBuffer.length, + durationMs: ( + (filteredAudio.length / this.SAMPLE_RATE) * + 1000 + ).toFixed(0), + }); + + // Build form data + const formData = new FormData(); + const wavBlob = new Blob([wavBuffer], { type: "audio/wav" }); + formData.append("file", wavBlob, "audio.wav"); + formData.append("model", "whisper-1"); + + // Pass language hint if available (ISO 639-1 code) + if (context.language && context.language !== "auto") { + formData.append("language", context.language); + } + + // Pass previous transcription as prompt for context continuity + if (context.aggregatedTranscription?.trim()) { + formData.append("prompt", context.aggregatedTranscription); + } + + // Set response format to plain text for simplicity + formData.append("response_format", "text"); + + // Make the API request (60s timeout) + let response: Response; + try { + response = await fetch(this.OPENAI_API_URL, { + method: "POST", + headers: { + Authorization: `Bearer ${config.apiKey}`, + "User-Agent": getUserAgent(), + }, + body: formData, + signal: AbortSignal.timeout(60_000), + }); + } catch (fetchError) { + throw new AppError( + fetchError instanceof Error ? fetchError.message : "Network error", + ErrorCodes.NETWORK_ERROR, + ); + } + + if (!response.ok) { + let errorMessage = `OpenAI API error: ${response.status} ${response.statusText}`; + try { + const errorBody = await response.json(); + if (errorBody?.error?.message) { + errorMessage = errorBody.error.message; + } + } catch { + // Response body wasn't valid JSON + } + + logger.transcription.error("OpenAI Whisper API error:", { + status: response.status, + errorMessage, + }); + + if (response.status === 401) { + throw new AppError(errorMessage, ErrorCodes.AUTH_REQUIRED, 401); + } else if (response.status === 429) { + throw new AppError( + errorMessage, + ErrorCodes.RATE_LIMIT_EXCEEDED, + 429, + ); + } else { + throw new AppError( + errorMessage, + ErrorCodes.CLOUD_TRANSCRIPTION_FAILED, + response.status, + ); + } + } + + // response_format=text returns plain text + const text = (await response.text()).trim(); + + logger.transcription.info("OpenAI Whisper transcription successful", { + textLength: text.length, + preview: text.length > 100 ? text.slice(0, 100) + "…" : text, + }); + + return { text }; + } catch (error) { + logger.transcription.error("OpenAI Whisper transcription error:", error); + if (error instanceof AppError) { + throw error; + } + throw new AppError( + `OpenAI Whisper transcription failed: ${error instanceof Error ? error.message : error}`, + ErrorCodes.LOCAL_TRANSCRIPTION_FAILED, + ); + } + } + + private aggregateFrames(): Float32Array { + const totalLength = this.frameBuffer.reduce( + (sum, frame) => sum + frame.length, + 0, + ); + const aggregated = new Float32Array(totalLength); + + let offset = 0; + for (const frame of this.frameBuffer) { + aggregated.set(frame, offset); + offset += frame.length; + } + + return aggregated; + } +} diff --git a/apps/desktop/src/renderer/main/pages/settings/ai-models/tabs/SpeechTab.tsx b/apps/desktop/src/renderer/main/pages/settings/ai-models/tabs/SpeechTab.tsx index 38befda1..719319b0 100644 --- a/apps/desktop/src/renderer/main/pages/settings/ai-models/tabs/SpeechTab.tsx +++ b/apps/desktop/src/renderer/main/pages/settings/ai-models/tabs/SpeechTab.tsx @@ -22,9 +22,11 @@ import { Trash2, LogIn, Cloud, + KeyRound, } from "lucide-react"; import { DynamicIcon } from "lucide-react/dynamic"; import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; import { TooltipContent, Tooltip, @@ -128,6 +130,14 @@ export default function SpeechTab() { undefined, ); + // OpenAI Whisper API key state + const [openAIApiKey, setOpenAIApiKey] = useState(""); + const [openAIStatus, setOpenAIStatus] = useState< + "connected" | "disconnected" + >("disconnected"); + const [openAIValidating, setOpenAIValidating] = useState(false); + const [openAIValidationError, setOpenAIValidationError] = useState(""); + // tRPC queries const availableModelsQuery = api.models.getAvailableModels.useQuery(); const downloadedModelsQuery = api.models.getDownloadedModels.useQuery(); @@ -198,6 +208,88 @@ export default function SpeechTab() { }, }); + // OpenAI Whisper tRPC queries and mutations + const modelProvidersConfigQuery = + api.settings.getModelProvidersConfig.useQuery(); + + const validateOpenAIWhisperMutation = + api.models.validateOpenAIWhisperConnection.useMutation({ + onSuccess: (result) => { + setOpenAIValidating(false); + if (result.success) { + setOpenAIValidationError(""); + // Save the config + setOpenAIWhisperConfigMutation.mutate({ + apiKey: openAIApiKey.trim(), + }); + toast.success("OpenAI API key validated successfully"); + } else { + setOpenAIValidationError( + result.error || "Failed to validate API key", + ); + toast.error(result.error || "Failed to validate OpenAI API key"); + } + }, + onError: (error) => { + setOpenAIValidating(false); + setOpenAIValidationError(error.message); + toast.error("Failed to validate OpenAI API key"); + }, + }); + + const setOpenAIWhisperConfigMutation = + api.settings.setOpenAIWhisperConfig.useMutation({ + onSuccess: () => { + setOpenAIStatus("connected"); + utils.settings.getModelProvidersConfig.invalidate(); + }, + onError: (error) => { + console.error("Failed to save OpenAI Whisper config:", error); + toast.error("Failed to save OpenAI Whisper configuration"); + }, + }); + + const removeOpenAIWhisperConfigMutation = + api.settings.removeOpenAIWhisperConfig.useMutation({ + onSuccess: () => { + setOpenAIStatus("disconnected"); + setOpenAIApiKey(""); + setOpenAIValidationError(""); + utils.settings.getModelProvidersConfig.invalidate(); + utils.models.getSelectedModel.invalidate(); + toast.success("OpenAI Whisper configuration removed"); + }, + onError: (error) => { + console.error("Failed to remove OpenAI Whisper config:", error); + toast.error("Failed to remove OpenAI Whisper configuration"); + }, + }); + + // Sync OpenAI Whisper state from stored config + useEffect(() => { + const config = modelProvidersConfigQuery.data; + if (config?.openAIWhisper?.apiKey) { + setOpenAIApiKey(config.openAIWhisper.apiKey); + setOpenAIStatus("connected"); + } else { + setOpenAIApiKey(""); + setOpenAIStatus("disconnected"); + } + }, [modelProvidersConfigQuery.data]); + + const handleOpenAIConnect = () => { + const trimmedKey = openAIApiKey.trim(); + if (!trimmedKey) return; + + setOpenAIValidating(true); + setOpenAIValidationError(""); + validateOpenAIWhisperMutation.mutate({ apiKey: trimmedKey }); + }; + + const handleOpenAIRemove = () => { + removeOpenAIWhisperConfigMutation.mutate(); + }; + // Initialize active downloads progress on load useEffect(() => { if (activeDownloadsQuery.data) { @@ -354,6 +446,15 @@ export default function SpeechTab() { // Check if this is a cloud model const model = availableModels.find((m) => m.id === modelId); const isCloudModel = model?.provider === "Amical Cloud"; + const isOpenAIModel = model?.provider === "OpenAI"; + + // If OpenAI model and not configured, show a toast hint + if (isOpenAIModel && openAIStatus !== "connected") { + toast.error( + "Please configure your OpenAI API key above first.", + ); + return; + } // If cloud model and not authenticated, show login dialog if (isCloudModel && !isAuthenticated) { @@ -412,6 +513,79 @@ export default function SpeechTab() { modelType="speech" title={t("settings.aiModels.defaultModels.speech")} /> + + {/* OpenAI Whisper API Key Configuration */} +
+
+
+ + +
+ + + {openAIStatus === "connected" + ? "Connected" + : "Not configured"} + +
+

+ Use your own OpenAI API key for cloud-based Whisper transcription. + No login required. +

+
+ setOpenAIApiKey(e.target.value)} + className="max-w-sm" + disabled={openAIStatus === "connected"} + /> + {openAIStatus === "disconnected" ? ( + + ) : ( + + )} +
+ {openAIValidationError && ( +

+ {openAIValidationError} +

+ )} +
+
diff --git a/apps/desktop/src/renderer/onboarding/components/screens/ModelSetupModal.tsx b/apps/desktop/src/renderer/onboarding/components/screens/ModelSetupModal.tsx index 323cf504..965c5824 100644 --- a/apps/desktop/src/renderer/onboarding/components/screens/ModelSetupModal.tsx +++ b/apps/desktop/src/renderer/onboarding/components/screens/ModelSetupModal.tsx @@ -255,6 +255,7 @@ export function ModelSetupModal({ setOpenAIApiKey(e.target.value)} onKeyDown={(e) => { diff --git a/apps/desktop/src/services/onboarding-service.ts b/apps/desktop/src/services/onboarding-service.ts index d17ee52e..cec67b85 100644 --- a/apps/desktop/src/services/onboarding-service.ts +++ b/apps/desktop/src/services/onboarding-service.ts @@ -164,7 +164,7 @@ export class OnboardingService extends EventEmitter { } if (state.modelRecommendation !== undefined) { stateForDb.modelRecommendation = { - suggested: state.modelRecommendation.suggested as "cloud" | "local", + suggested: state.modelRecommendation.suggested as "cloud" | "local" | "openai-whisper", reason: state.modelRecommendation.reason, followed: state.modelRecommendation.followed, }; @@ -684,46 +684,38 @@ export class OnboardingService extends EventEmitter { try { logger.main.info("Completing onboarding flow"); - // Save the final state - await this.completeOnboarding(finalState); - - // Ensure the selected model is persisted before the app relaunches. + // Ensure the selected model is persisted before saving completion state. // savePreferences (which also calls setSelectedModel) runs as fire-and-forget // from the renderer, so it may not have completed yet. We must set the model - // here synchronously before emitting "completed" (which triggers app.relaunch). - try { - if (finalState.selectedModelType === "cloud") { - await this.modelService.setSelectedModel("amical-cloud"); - logger.main.info( - "Set default speech model to amical-cloud during completion", - ); - } else if (finalState.selectedModelType === "openai-whisper") { - await this.modelService.setSelectedModel("openai-whisper"); - logger.main.info( - "Set default speech model to openai-whisper during completion", - ); - } else if (finalState.selectedModelType === "local") { - const currentModel = await this.modelService.getSelectedModel(); - if (!currentModel) { - const downloadedModels = - await this.modelService.getDownloadedModels(); - const downloadedIds = Object.keys(downloadedModels); - if (downloadedIds.length > 0) { - await this.modelService.setSelectedModel(downloadedIds[0]); - logger.main.info( - `Set default speech model to ${downloadedIds[0]} during completion`, - ); - } + // before emitting "completed" (which triggers app.relaunch). + if (finalState.selectedModelType === "cloud") { + await this.modelService.setSelectedModel("amical-cloud"); + logger.main.info( + "Set default speech model to amical-cloud during completion", + ); + } else if (finalState.selectedModelType === "openai-whisper") { + await this.modelService.setSelectedModel("openai-whisper"); + logger.main.info( + "Set default speech model to openai-whisper during completion", + ); + } else if (finalState.selectedModelType === "local") { + const currentModel = await this.modelService.getSelectedModel(); + if (!currentModel) { + const downloadedModels = + await this.modelService.getDownloadedModels(); + const downloadedIds = Object.keys(downloadedModels); + if (downloadedIds.length > 0) { + await this.modelService.setSelectedModel(downloadedIds[0]); + logger.main.info( + `Set default speech model to ${downloadedIds[0]} during completion`, + ); } } - } catch (modelError) { - logger.main.error( - "Failed to set model during onboarding completion:", - modelError, - ); - // Don't block completion - model can be set manually later } + // Save the final state after model is set + await this.completeOnboarding(finalState); + this.isOnboardingInProgress = false; // Emit event - AppManager listens and handles window transitions diff --git a/apps/desktop/src/services/transcription-service.ts b/apps/desktop/src/services/transcription-service.ts index ad06fde1..2882db0a 100644 --- a/apps/desktop/src/services/transcription-service.ts +++ b/apps/desktop/src/services/transcription-service.ts @@ -207,7 +207,7 @@ export class TranscriptionService { const selectedModelId = await this.modelService.getSelectedModel(); if (selectedModelId) { const model = AVAILABLE_MODELS.find((m) => m.id === selectedModelId); - if (model?.provider === "Amical Cloud") { + if (model?.provider === "Amical Cloud" || model?.provider === "OpenAI") { return true; } } diff --git a/apps/desktop/tests/pipeline/openai-whisper-provider.test.ts b/apps/desktop/tests/pipeline/openai-whisper-provider.test.ts index 4fca48ed..4d4ca541 100644 --- a/apps/desktop/tests/pipeline/openai-whisper-provider.test.ts +++ b/apps/desktop/tests/pipeline/openai-whisper-provider.test.ts @@ -64,7 +64,7 @@ describe("OpenAI Whisper Provider", () => { context: { language: "en" }, }); - expect(result).toBe(""); + expect(result.text).toBe(""); expect(fetchSpy).not.toHaveBeenCalled(); }); @@ -84,17 +84,17 @@ describe("OpenAI Whisper Provider", () => { // Send 3+ seconds of silence to trigger transcription const silenceFrameCount = Math.ceil((3.1 * sampleRate) / frameSize); - let result = ""; + let result = { text: "" }; for (let i = 0; i < silenceFrameCount; i++) { result = await provider.transcribe({ audioData: new Float32Array(frameSize).fill(0.0), speechProbability: 0.01, context: { language: "en" }, }); - if (result) break; + if (result.text) break; } - expect(result).toBe("Hello world"); + expect(result.text).toBe("Hello world"); expect(fetchSpy).toHaveBeenCalledTimes(1); const [url, options] = fetchSpy.mock.calls[0]; @@ -155,13 +155,13 @@ describe("OpenAI Whisper Provider", () => { const result = await provider.flush({ language: "en" }); - expect(result).toBe("Hello world"); + expect(result.text).toBe("Hello world"); expect(fetchSpy).toHaveBeenCalledTimes(1); }); it("should return empty on flush with no buffered audio", async () => { const result = await provider.flush({ language: "en" }); - expect(result).toBe(""); + expect(result.text).toBe(""); expect(fetchSpy).not.toHaveBeenCalled(); }); @@ -179,7 +179,7 @@ describe("OpenAI Whisper Provider", () => { provider.reset(); const result = await provider.flush({ language: "en" }); - expect(result).toBe(""); + expect(result.text).toBe(""); expect(fetchSpy).not.toHaveBeenCalled(); }); }); @@ -426,7 +426,7 @@ describe("OpenAI Whisper Provider - Integration", () => { } const result = await provider.flush({ language: "en" }); - expect(typeof result).toBe("string"); + expect(typeof result.text).toBe("string"); }, 15000); }); }); From 3bf119964dc441c02b657f54c4a77595a48f6c45 Mon Sep 17 00:00:00 2001 From: CreatorGhost Date: Mon, 30 Mar 2026 13:46:05 +0530 Subject: [PATCH 3/5] fix: address second round of code review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Defer buffer reset until after successful transcription; restore buffers on error so audio is not lost on transient failures - Remove plaintext transcription preview from logs (privacy) - Use mutation variables in onSuccess to persist the exact validated key - Mask API key in renderer — don't hydrate raw secret into React state - Add aria-label to settings API key input for accessibility - OpenAI Whisper only counts as available when API key is present (isModelAvailable and initialize cloud checks) Co-Authored-By: Claude Opus 4.6 --- .../transcription/openai-whisper-provider.ts | 23 +++++++++++-------- .../settings/ai-models/tabs/SpeechTab.tsx | 10 ++++---- .../src/services/transcription-service.ts | 18 +++++++++++---- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/apps/desktop/src/pipeline/providers/transcription/openai-whisper-provider.ts b/apps/desktop/src/pipeline/providers/transcription/openai-whisper-provider.ts index ee64c745..54456641 100644 --- a/apps/desktop/src/pipeline/providers/transcription/openai-whisper-provider.ts +++ b/apps/desktop/src/pipeline/providers/transcription/openai-whisper-provider.ts @@ -150,6 +150,10 @@ export class OpenAIWhisperProvider implements TranscriptionProvider { private async doTranscription( context: TranscribeContext, ): Promise { + // Capture buffers before try so they're accessible in catch for restoration + const vadProbs = [...this.frameBufferSpeechProbabilities]; + const rawAudio = this.aggregateFrames(); + try { // Get API key const config = await this.settingsService.getOpenAIWhisperConfig(); @@ -160,20 +164,13 @@ export class OpenAIWhisperProvider implements TranscriptionProvider { ); } - // Capture speech probabilities before reset - const vadProbs = [...this.frameBufferSpeechProbabilities]; - - // Aggregate buffered frames - const rawAudio = this.aggregateFrames(); - - // Clear buffers immediately after aggregation - this.reset(); - // Apply VAD filtering to extract speech-only portions const { audio: filteredAudio, segments: speechSegments } = extractSpeechFromVad(rawAudio, vadProbs); if (filteredAudio.length === 0) { + // No speech detected — safe to clear buffers + this.reset(); logger.transcription.debug( "OpenAI Whisper: Skipping - no speech detected by VAD filter", ); @@ -270,13 +267,19 @@ export class OpenAIWhisperProvider implements TranscriptionProvider { // response_format=text returns plain text const text = (await response.text()).trim(); + // Success — clear buffers now that audio has been processed + this.reset(); + logger.transcription.info("OpenAI Whisper transcription successful", { textLength: text.length, - preview: text.length > 100 ? text.slice(0, 100) + "…" : text, }); return { text }; } catch (error) { + // Restore buffers so audio is not lost on transient failures + this.frameBuffer = [rawAudio]; + this.frameBufferSpeechProbabilities = vadProbs; + logger.transcription.error("OpenAI Whisper transcription error:", error); if (error instanceof AppError) { throw error; diff --git a/apps/desktop/src/renderer/main/pages/settings/ai-models/tabs/SpeechTab.tsx b/apps/desktop/src/renderer/main/pages/settings/ai-models/tabs/SpeechTab.tsx index 85a85547..61b878cd 100644 --- a/apps/desktop/src/renderer/main/pages/settings/ai-models/tabs/SpeechTab.tsx +++ b/apps/desktop/src/renderer/main/pages/settings/ai-models/tabs/SpeechTab.tsx @@ -214,13 +214,13 @@ export default function SpeechTab() { const validateOpenAIWhisperMutation = api.models.validateOpenAIWhisperConnection.useMutation({ - onSuccess: (result) => { + onSuccess: (result, variables) => { setOpenAIValidating(false); if (result.success) { setOpenAIValidationError(""); - // Save the config + // Save the config using the exact key that was validated setOpenAIWhisperConfigMutation.mutate({ - apiKey: openAIApiKey.trim(), + apiKey: variables.apiKey, }); toast.success(t("settings.aiModels.openAIWhisper.toast.validated")); } else { @@ -269,7 +269,8 @@ export default function SpeechTab() { useEffect(() => { const config = modelProvidersConfigQuery.data; if (config?.openAIWhisper?.apiKey) { - setOpenAIApiKey(config.openAIWhisper.apiKey); + // Don't copy the real key into React state — use a masked placeholder + setOpenAIApiKey("••••••••••••••••"); setOpenAIStatus("connected"); } else { setOpenAIApiKey(""); @@ -548,6 +549,7 @@ export default function SpeechTab() { setOpenAIApiKey(e.target.value)} className="max-w-sm" diff --git a/apps/desktop/src/services/transcription-service.ts b/apps/desktop/src/services/transcription-service.ts index 2882db0a..10096b1d 100644 --- a/apps/desktop/src/services/transcription-service.ts +++ b/apps/desktop/src/services/transcription-service.ts @@ -118,12 +118,13 @@ export class TranscriptionService { const model = selectedModelId ? AVAILABLE_MODELS.find((m) => m.id === selectedModelId) : null; - const isCloudModel = - model?.provider === "Amical Cloud" || model?.provider === "OpenAI"; - - // Also check if OpenAI Whisper is configured (even if not yet selected) + // Also check if OpenAI Whisper is configured const openAIConfig = await this.settingsService.getOpenAIWhisperConfig(); + const isOpenAIWithKey = + model?.provider === "OpenAI" && !!openAIConfig?.apiKey; + const isCloudModel = + model?.provider === "Amical Cloud" || isOpenAIWithKey; const hasCloudOption = isCloudModel || !!openAIConfig?.apiKey; // Only preload for local models @@ -207,9 +208,16 @@ export class TranscriptionService { const selectedModelId = await this.modelService.getSelectedModel(); if (selectedModelId) { const model = AVAILABLE_MODELS.find((m) => m.id === selectedModelId); - if (model?.provider === "Amical Cloud" || model?.provider === "OpenAI") { + if (model?.provider === "Amical Cloud") { return true; } + if (model?.provider === "OpenAI") { + const config = + await this.settingsService.getOpenAIWhisperConfig(); + if (config?.apiKey) { + return true; + } + } } // For local models, check if any are downloaded From 2270de6a6428bd042a1421647f4a0556f1214e67 Mon Sep 17 00:00:00 2001 From: CreatorGhost Date: Mon, 30 Mar 2026 14:06:39 +0530 Subject: [PATCH 4/5] fix: use actual sample counts in shouldTranscribe duration calculation After error recovery, frameBuffer may contain a single aggregated Float32Array instead of fixed-size frames. Sum actual buffer lengths instead of assuming FRAME_SIZE per entry. Co-Authored-By: Claude Opus 4.6 --- .../providers/transcription/openai-whisper-provider.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/desktop/src/pipeline/providers/transcription/openai-whisper-provider.ts b/apps/desktop/src/pipeline/providers/transcription/openai-whisper-provider.ts index 54456641..56c32058 100644 --- a/apps/desktop/src/pipeline/providers/transcription/openai-whisper-provider.ts +++ b/apps/desktop/src/pipeline/providers/transcription/openai-whisper-provider.ts @@ -126,8 +126,11 @@ export class OpenAIWhisperProvider implements TranscriptionProvider { } private shouldTranscribe(): boolean { - const audioDurationMs = - ((this.frameBuffer.length * this.FRAME_SIZE) / this.SAMPLE_RATE) * 1000; + const totalSamples = this.frameBuffer.reduce( + (sum, buf) => sum + buf.length, + 0, + ); + const audioDurationMs = (totalSamples / this.SAMPLE_RATE) * 1000; const silenceDurationMs = ((this.currentSilenceFrameCount * this.FRAME_SIZE) / this.SAMPLE_RATE) * 1000; From ab3eb3a92801c664e4feff9ac254d1b1f27bb82e Mon Sep 17 00:00:00 2001 From: CreatorGhost Date: Mon, 30 Mar 2026 22:42:40 +0530 Subject: [PATCH 5/5] fix: align frameBuffer and speechProbabilities after error recovery Collapse vadProbs to a single mean probability to match the single aggregated rawAudio entry, maintaining the 1:1 alignment expected by transcribe() and extractSpeechFromVad(). Co-Authored-By: Claude Opus 4.6 --- .../providers/transcription/openai-whisper-provider.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/desktop/src/pipeline/providers/transcription/openai-whisper-provider.ts b/apps/desktop/src/pipeline/providers/transcription/openai-whisper-provider.ts index 56c32058..0171700e 100644 --- a/apps/desktop/src/pipeline/providers/transcription/openai-whisper-provider.ts +++ b/apps/desktop/src/pipeline/providers/transcription/openai-whisper-provider.ts @@ -279,9 +279,15 @@ export class OpenAIWhisperProvider implements TranscriptionProvider { return { text }; } catch (error) { - // Restore buffers so audio is not lost on transient failures + // Restore buffers so audio is not lost on transient failures. + // Collapse to a single entry to maintain 1:1 alignment between + // frameBuffer and frameBufferSpeechProbabilities. + const meanProb = + vadProbs.length > 0 + ? vadProbs.reduce((a, b) => a + b, 0) / vadProbs.length + : 1; this.frameBuffer = [rawAudio]; - this.frameBufferSpeechProbabilities = vadProbs; + this.frameBufferSpeechProbabilities = [meanProb]; logger.transcription.error("OpenAI Whisper transcription error:", error); if (error instanceof AppError) {