From 39d7379ecdb8e966cd77b6c79d01189c6e527987 Mon Sep 17 00:00:00 2001 From: HadiAssadDiab <92597340+HadiAssadDiab@users.noreply.github.com> Date: Mon, 6 Jul 2026 22:03:06 +0200 Subject: [PATCH] Add ElevenLabs Scribe transcription --- README.md | 8 +- Sources/AppState.swift | 97 +++- Sources/AudioRecorder.swift | 28 +- Sources/RealtimeTranscriptionService.swift | 392 ++++++++++++++- Sources/SettingsView.swift | 284 ++++++++--- Sources/SetupView.swift | 12 +- Sources/TranscriptionService.swift | 547 +++++++++++++-------- 7 files changed, 1073 insertions(+), 295 deletions(-) diff --git a/README.md b/README.md index f7d389d9..59a0780c 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ FreeFlow is a free Mac dictation app inspired by [Wispr Flow](https://wisprflow. - **Custom shortcuts:** Customize both hold-to-talk and toggle dictation shortcuts. If your toggle shortcut extends your hold shortcut, you can start in hold mode and press the extra modifier keys to latch into tap mode without stopping the recording. - **Context-aware cleanup:** FreeFlow can read nearby app context so names, terms, and phrases are spelled correctly when you dictate into email, terminals, docs, and other apps. - **Custom vocabulary:** Add names, jargon, and project-specific words that FreeFlow should preserve during cleanup. -- **OpenAI-compatible providers:** Use Groq by default, or configure a custom model and API URL in settings. +- **Provider choices:** Use Groq by default, select ElevenLabs Scribe v2 for transcription, or configure OpenAI-compatible model IDs and API URLs in settings. ## Edit Mode @@ -46,7 +46,7 @@ Edit Mode lets you highlight existing text and transform it with a spoken instru ## Privacy -There is no FreeFlow server, so FreeFlow does not store or retain your data. The only information that leaves your computer are API calls to your configured transcription and LLM provider. +There is no FreeFlow server, so FreeFlow does not store or retain your data. The only information that leaves your computer are API calls to your selected transcription provider and your configured cleanup/context LLM provider. ## Custom Cleanup @@ -82,6 +82,10 @@ FreeFlow can use OpenAI-compatible local or self-hosted providers instead of Gro Local models are often slower than hosted providers, especially on cold start, long recordings, or busy hardware. +## Using ElevenLabs Scribe + +FreeFlow can use ElevenLabs Scribe v2 for speech-to-text while continuing to use your OpenAI-compatible provider for cleanup, Edit Mode, and context. Open Settings, expand Providers, choose ElevenLabs Scribe as the transcription provider, and enter an ElevenLabs API key. Realtime streaming uses Scribe v2 Realtime when the realtime toggle is enabled. +
Configure longer timeouts for local models diff --git a/Sources/AppState.swift b/Sources/AppState.swift index c65bc809..2d82ff5c 100644 --- a/Sources/AppState.swift +++ b/Sources/AppState.swift @@ -201,8 +201,10 @@ final class AppState: ObservableObject, @unchecked Sendable { private let apiKeyStorageKey = "groq_api_key" private let apiBaseURLStorageKey = "api_base_url" private let transcriptionModelStorageKey = "transcription_model" + private let transcriptionProviderStorageKey = "transcription_provider" private let transcriptionAPIURLStorageKey = "transcription_api_url" private let transcriptionAPIKeyStorageKey = "transcription_api_key" + private let elevenLabsAPIKeyStorageKey = "elevenlabs_api_key" private let postProcessingModelStorageKey = "post_processing_model" private let postProcessingFallbackModelStorageKey = "post_processing_fallback_model" private let contextModelStorageKey = "context_model" @@ -301,6 +303,12 @@ final class AppState: ObservableObject, @unchecked Sendable { } } + @Published var transcriptionProvider: TranscriptionProvider { + didSet { + UserDefaults.standard.set(transcriptionProvider.rawValue, forKey: transcriptionProviderStorageKey) + } + } + @Published var transcriptionAPIURL: String { didSet { persistOptionalAPIValue(transcriptionAPIURL, account: transcriptionAPIURLStorageKey) @@ -313,6 +321,12 @@ final class AppState: ObservableObject, @unchecked Sendable { } } + @Published var elevenLabsAPIKey: String { + didSet { + persistOptionalAPIValue(elevenLabsAPIKey, account: elevenLabsAPIKeyStorageKey) + } + } + @Published var transcriptionModel: String { didSet { UserDefaults.standard.set(transcriptionModel, forKey: transcriptionModelStorageKey) @@ -605,7 +619,7 @@ final class AppState: ObservableObject, @unchecked Sendable { private var pendingManualCommandInvocation = false private var pendingShortcutStartTask: Task? private var pendingShortcutStartMode: RecordingTriggerMode? - private var realtimeService: RealtimeTranscriptionService? + private var realtimeService: RealtimeTranscriptionClient? private var automaticTerminationDisabled = false private var activeAudioInterruption: ActiveAudioInterruption? private var pendingOverlayDismissToken: UUID? @@ -622,9 +636,13 @@ final class AppState: ObservableObject, @unchecked Sendable { let hasCompletedSetup = UserDefaults.standard.bool(forKey: "hasCompletedSetup") let apiKey = Self.loadStoredAPIKey(account: apiKeyStorageKey) let apiBaseURL = Self.loadStoredAPIBaseURL(account: "api_base_url") + let transcriptionProvider = TranscriptionProvider( + rawValue: UserDefaults.standard.string(forKey: transcriptionProviderStorageKey) ?? "" + ) ?? .openAICompatible let transcriptionModel = UserDefaults.standard.string(forKey: transcriptionModelStorageKey) ?? Self.defaultTranscriptionModel let transcriptionAPIURL = Self.loadOptionalStoredAPIValue(account: transcriptionAPIURLStorageKey) let transcriptionAPIKey = Self.loadStoredAPIKey(account: transcriptionAPIKeyStorageKey) + let elevenLabsAPIKey = Self.loadStoredAPIKey(account: elevenLabsAPIKeyStorageKey) let postProcessingModel = UserDefaults.standard.string(forKey: postProcessingModelStorageKey) ?? Self.defaultPostProcessingModel let postProcessingFallbackModel = UserDefaults.standard.string(forKey: postProcessingFallbackModelStorageKey) ?? Self.defaultPostProcessingFallbackModel let contextModel = UserDefaults.standard.string(forKey: contextModelStorageKey) ?? Self.defaultContextModel @@ -724,8 +742,10 @@ final class AppState: ObservableObject, @unchecked Sendable { self.hasCompletedSetup = hasCompletedSetup self.apiKey = apiKey self.apiBaseURL = apiBaseURL + self.transcriptionProvider = transcriptionProvider self.transcriptionAPIURL = transcriptionAPIURL self.transcriptionAPIKey = transcriptionAPIKey + self.elevenLabsAPIKey = elevenLabsAPIKey self.transcriptionModel = transcriptionModel self.postProcessingModel = postProcessingModel self.postProcessingFallbackModel = postProcessingFallbackModel @@ -831,7 +851,7 @@ final class AppState: ObservableObject, @unchecked Sendable { } } - static let defaultAPIBaseURL = "https://api.groq.com/openai/v1" + static let defaultAPIBaseURL = TranscriptionService.defaultOpenAICompatibleBaseURL private struct StoredShortcutConfiguration { let hold: ShortcutBinding @@ -981,17 +1001,24 @@ final class AppState: ObservableObject, @unchecked Sendable { } private var resolvedTranscriptionBaseURL: String { + if transcriptionProvider == .elevenLabs { + return TranscriptionService.defaultElevenLabsBaseURL + } let trimmed = transcriptionAPIURL.trimmingCharacters(in: .whitespacesAndNewlines) return trimmed.isEmpty ? apiBaseURL : trimmed } private var resolvedTranscriptionAPIKey: String { let trimmed = transcriptionAPIKey.trimmingCharacters(in: .whitespacesAndNewlines) + if transcriptionProvider == .elevenLabs { + return elevenLabsAPIKey.trimmingCharacters(in: .whitespacesAndNewlines) + } return trimmed.isEmpty ? apiKey : trimmed } func makeTranscriptionService() throws -> TranscriptionService { try TranscriptionService( + provider: transcriptionProvider, apiKey: resolvedTranscriptionAPIKey, baseURL: resolvedTranscriptionBaseURL, transcriptionModel: transcriptionModel, @@ -999,6 +1026,18 @@ final class AppState: ObservableObject, @unchecked Sendable { ) } + private func transcriptionConfigurationErrorMessage() -> String? { + if resolvedTranscriptionAPIKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + switch transcriptionProvider { + case .openAICompatible: + return "Enter an API key in Settings." + case .elevenLabs: + return "Enter an ElevenLabs API key in Settings." + } + } + return nil + } + private var resolvedTranscriptionLanguage: String? { let normalized = Self.normalizeTranscriptionLanguage(transcriptionLanguage) return normalized.isEmpty ? nil : normalized @@ -1937,6 +1976,16 @@ final class AppState: ObservableObject, @unchecked Sendable { : scheduledManualCommandInvocation, startedAt: t0 ) else { return } + if let configurationError = transcriptionConfigurationErrorMessage() { + errorMessage = configurationError + statusText = "Missing API Key" + activeRecordingTriggerMode = nil + currentSessionIntent = .dictation + shortcutSessionController.reset() + playAlertSound(named: "Basso") + scheduleReadyStatusReset(after: 2, matching: ["Missing API Key"]) + return + } guard ensureMicrophoneAccess() else { return } os_log(.info, log: recordingLog, "mic access check passed: %.3fms", (CFAbsoluteTimeGetCurrent() - t0) * 1000) applyAudioInterruptionIfNeeded() @@ -2043,6 +2092,16 @@ final class AppState: ObservableObject, @unchecked Sendable { selectionSnapshot: pendingSelectionSnapshot, manualCommandRequested: pendingManualCommandRequested ) else { return } + if let configurationError = strongSelf.transcriptionConfigurationErrorMessage() { + strongSelf.errorMessage = configurationError + strongSelf.statusText = "Missing API Key" + strongSelf.activeRecordingTriggerMode = nil + strongSelf.currentSessionIntent = .dictation + strongSelf.shortcutSessionController.reset() + strongSelf.playAlertSound(named: "Basso") + strongSelf.scheduleReadyStatusReset(after: 2, matching: ["Missing API Key"]) + return + } strongSelf.shortcutSessionController.beginManual(mode: .toggle) strongSelf.applyAudioInterruptionIfNeeded() strongSelf.beginRecording(triggerMode: .toggle) @@ -2505,7 +2564,7 @@ final class AppState: ObservableObject, @unchecked Sendable { /// gets a transcript. Runs the realtime commit and file upload in that /// strict order to avoid paying for both when realtime succeeds. private static func resolveRawTranscript( - realtimeService: RealtimeTranscriptionService?, + realtimeService: RealtimeTranscriptionClient?, fileService: TranscriptionService, fileURL: URL ) async throws -> String { @@ -2849,14 +2908,28 @@ final class AppState: ObservableObject, @unchecked Sendable { os_log(.info, log: recordingLog, "realtime streaming requested but base URL is empty — skipping") return } - let model = realtimeStreamingModel.trimmingCharacters(in: .whitespacesAndNewlines) - let config = RealtimeTranscriptionService.Configuration( - baseURL: trimmedBase, - apiKey: resolvedTranscriptionAPIKey, - model: model, - language: resolvedTranscriptionLanguage - ) - let service = RealtimeTranscriptionService(config: config) + + let service: RealtimeTranscriptionClient + switch transcriptionProvider { + case .openAICompatible: + let model = realtimeStreamingModel.trimmingCharacters(in: .whitespacesAndNewlines) + let config = RealtimeTranscriptionService.Configuration( + baseURL: trimmedBase, + apiKey: resolvedTranscriptionAPIKey, + model: model, + language: resolvedTranscriptionLanguage + ) + service = RealtimeTranscriptionService(config: config) + case .elevenLabs: + let config = ElevenLabsRealtimeTranscriptionService.Configuration( + baseURL: trimmedBase, + apiKey: resolvedTranscriptionAPIKey, + model: TranscriptionService.defaultElevenLabsRealtimeModel, + language: resolvedTranscriptionLanguage + ) + service = ElevenLabsRealtimeTranscriptionService(config: config) + } + do { try service.start() } catch { @@ -2864,6 +2937,7 @@ final class AppState: ObservableObject, @unchecked Sendable { return } realtimeService = service + audioRecorder.realtimePCM16SampleRate = service.pcmSampleRate audioRecorder.onPCM16Samples = { [weak service] data in service?.appendPCM16(data) } @@ -2871,6 +2945,7 @@ final class AppState: ObservableObject, @unchecked Sendable { private func tearDownRealtimeService() { audioRecorder.onPCM16Samples = nil + audioRecorder.realtimePCM16SampleRate = 24_000 realtimeService?.cancel() realtimeService = nil } diff --git a/Sources/AudioRecorder.swift b/Sources/AudioRecorder.swift index ec8632f6..cda0f33c 100644 --- a/Sources/AudioRecorder.swift +++ b/Sources/AudioRecorder.swift @@ -95,12 +95,18 @@ final class AudioRecorder: NSObject, ObservableObject, AVCaptureAudioDataOutputS var onRecordingReady: (() -> Void)? var onRecordingFailure: ((Error) -> Void)? - /// Fires on the sample-buffer queue with a 24 kHz mono PCM16 chunk for - /// each incoming audio buffer (matching OpenAI Realtime's default PCM - /// input rate). Set before ``startRecording`` to stream audio out-of-band - /// to a realtime transcription socket. The recorder writes a normalized - /// 16 kHz mono PCM16 WAV file independently for upload-based transcription. + /// Fires on the sample-buffer queue with mono PCM16 chunks for each + /// incoming audio buffer. Set before ``startRecording`` to stream audio + /// out-of-band to a realtime transcription socket. The recorder writes a + /// normalized 16 kHz mono PCM16 WAV file independently for upload-based + /// transcription. var onPCM16Samples: ((Data) -> Void)? + var realtimePCM16SampleRate: Double = 24_000 { + didSet { + guard realtimePCM16SampleRate != oldValue else { return } + pcm16ConverterLock.withLock { $0 = nil } + } + } private let recordingConverterLock = OSAllocatedUnfairLock(initialState: nil) private let pcm16ConverterLock = OSAllocatedUnfairLock(initialState: nil) private let recordingTargetFormat: AVAudioFormat = { @@ -111,14 +117,14 @@ final class AudioRecorder: NSObject, ObservableObject, AVCaptureAudioDataOutputS interleaved: true )! }() - private let pcm16TargetFormat: AVAudioFormat = { + private var pcm16TargetFormat: AVAudioFormat { AVAudioFormat( commonFormat: .pcmFormatInt16, - sampleRate: 24_000, + sampleRate: realtimePCM16SampleRate, channels: 1, interleaved: true )! - }() + } private var readyFired = false private var failureReported = false private static let watchdogTimeout: TimeInterval = 2.0 @@ -187,7 +193,7 @@ final class AudioRecorder: NSObject, ObservableObject, AVCaptureAudioDataOutputS removeSessionObservers() let runtimeObserver = NotificationCenter.default.addObserver( - forName: AVCaptureSession.runtimeErrorNotification, + forName: NSNotification.Name.AVCaptureSessionRuntimeError, object: session, queue: nil ) { [weak self] notification in @@ -199,7 +205,7 @@ final class AudioRecorder: NSObject, ObservableObject, AVCaptureAudioDataOutputS sessionObservers.append(runtimeObserver) let interruptionObserver = NotificationCenter.default.addObserver( - forName: AVCaptureSession.wasInterruptedNotification, + forName: NSNotification.Name.AVCaptureSessionWasInterrupted, object: session, queue: nil ) { [weak self] notification in @@ -208,7 +214,7 @@ final class AudioRecorder: NSObject, ObservableObject, AVCaptureAudioDataOutputS sessionObservers.append(interruptionObserver) let interruptionEndedObserver = NotificationCenter.default.addObserver( - forName: AVCaptureSession.interruptionEndedNotification, + forName: NSNotification.Name.AVCaptureSessionInterruptionEnded, object: session, queue: nil ) { [weak self] notification in diff --git a/Sources/RealtimeTranscriptionService.swift b/Sources/RealtimeTranscriptionService.swift index 37ea24e5..fb9c5027 100644 --- a/Sources/RealtimeTranscriptionService.swift +++ b/Sources/RealtimeTranscriptionService.swift @@ -19,7 +19,15 @@ enum RealtimeTranscriptionError: LocalizedError { } } -final class RealtimeTranscriptionService { +protocol RealtimeTranscriptionClient: AnyObject { + var pcmSampleRate: Double { get } + func start() throws + func cancel() + func appendPCM16(_ data: Data) + func commitAndAwaitFinal() async throws -> String +} + +final class RealtimeTranscriptionService: RealtimeTranscriptionClient { struct Configuration { let baseURL: String let apiKey: String @@ -48,6 +56,7 @@ final class RealtimeTranscriptionService { /// concatenates all `completed` events and currently-streaming `delta` /// events — useful for a live overlay readout. var onPartialUpdate: ((String) -> Void)? + let pcmSampleRate: Double = 24_000 init(config: Configuration, session: URLSession = .shared) { self.config = config @@ -409,3 +418,384 @@ final class RealtimeTranscriptionService { return finalText } } + +final class ElevenLabsRealtimeTranscriptionService: RealtimeTranscriptionClient { + struct Configuration { + let baseURL: String + let apiKey: String + let model: String + let language: String? + } + + private let config: Configuration + private let session: URLSession + private var task: URLSessionWebSocketTask? + private var receiveTask: Task? + + private let stateQueue = DispatchQueue(label: "com.zachlatta.freeflow.realtime.elevenlabs.state") + private var finalText: String = "" + private var partialText: String = "" + private var pendingAudio = Data() + private var hasSentAudio = false + private var finalContinuation: CheckedContinuation? + private var commitSent = false + private var postCommitCompleted = false + private var closed = false + private var terminalError: Error? + + var onPartialUpdate: ((String) -> Void)? + let pcmSampleRate: Double = 16_000 + private let bytesPerSample = 2 + private let targetChunkSeconds = 0.5 + private let heldChunkSeconds = 0.1 + + init(config: Configuration, session: URLSession = .shared) { + self.config = config + self.session = session + } + + func start() throws { + let trimmedKey = config.apiKey.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmedKey.isEmpty else { + throw RealtimeTranscriptionError.serverError( + code: "missing_api_key", + message: "Enter an ElevenLabs API key in Settings." + ) + } + guard let wsURL = Self.deriveWebSocketURL( + baseURL: config.baseURL, + model: config.model, + language: config.language + ) else { + throw RealtimeTranscriptionError.invalidBaseURL(config.baseURL) + } + + var request = URLRequest(url: wsURL) + request.setValue(trimmedKey, forHTTPHeaderField: "xi-api-key") + + let task = session.webSocketTask(with: request) + stateQueue.sync { + self.task = task + } + task.resume() + + receiveTask = Task { [weak self] in + await self?.receiveLoop() + } + } + + func cancel() { + let currentTask: URLSessionWebSocketTask? = stateQueue.sync { + let currentTask = task + task = nil + return currentTask + } + stateQueue.sync { + guard !closed else { return } + closed = true + if let cont = finalContinuation { + finalContinuation = nil + cont.resume(throwing: CancellationError()) + } + } + receiveTask?.cancel() + currentTask?.cancel(with: .normalClosure, reason: nil) + } + + func appendPCM16(_ data: Data) { + guard !data.isEmpty else { return } + var chunks: [Data] = [] + stateQueue.sync { + guard task != nil, !commitSent, !closed else { return } + pendingAudio.append(data) + let targetBytes = Self.byteCount( + seconds: targetChunkSeconds, + sampleRate: pcmSampleRate, + bytesPerSample: bytesPerSample + ) + let heldBytes = Self.byteCount( + seconds: heldChunkSeconds, + sampleRate: pcmSampleRate, + bytesPerSample: bytesPerSample + ) + while pendingAudio.count >= targetBytes + heldBytes { + chunks.append(Data(pendingAudio.prefix(targetBytes))) + pendingAudio.removeFirst(targetBytes) + } + } + for chunk in chunks { + sendAudioChunk(chunk, commit: false) + } + } + + func commitAndAwaitFinal() async throws -> String { + let currentTask: URLSessionWebSocketTask? = stateQueue.sync { + task + } + guard currentTask != nil else { + throw RealtimeTranscriptionError.notConnected + } + + let finalChunk: Data? = stateQueue.sync { + if commitSent { return nil } + commitSent = true + let chunk: Data + if pendingAudio.isEmpty && hasSentAudio { + chunk = Data() + } else if pendingAudio.isEmpty { + chunk = Data(repeating: 0, count: Self.byteCount( + seconds: heldChunkSeconds, + sampleRate: pcmSampleRate, + bytesPerSample: bytesPerSample + )) + } else { + chunk = pendingAudio + } + pendingAudio.removeAll(keepingCapacity: false) + return chunk + } + if let finalChunk { + sendAudioChunk(finalChunk, commit: true) + } + + return try await withCheckedThrowingContinuation { continuation in + var immediateResult: Result? + stateQueue.sync { + if let terminalError { + immediateResult = .failure(terminalError) + return + } + if closed { + immediateResult = .failure(RealtimeTranscriptionError.closedBeforeFinal) + return + } + if let finalText = readyCommittedTranscriptLocked() { + closed = true + immediateResult = .success(finalText) + return + } + finalContinuation = continuation + } + if let immediateResult { + currentTask?.cancel(with: .normalClosure, reason: nil) + continuation.resume(with: immediateResult) + } + } + } + + private func receiveLoop() async { + while !Task.isCancelled { + let currentTask: URLSessionWebSocketTask? = stateQueue.sync { + task + } + guard let currentTask else { break } + do { + let message = try await currentTask.receive() + switch message { + case .string(let text): + handleServerEvent(text) + case .data(let data): + if let text = String(data: data, encoding: .utf8) { + handleServerEvent(text) + } + @unknown default: + break + } + } catch { + finishWithClose() + return + } + } + finishWithClose() + } + + private func finishWithClose() { + stateQueue.sync { + closed = true + if let cont = finalContinuation { + finalContinuation = nil + if postCommitCompleted { + cont.resume(returning: finalText) + } else { + cont.resume(throwing: RealtimeTranscriptionError.closedBeforeFinal) + } + } + } + } + + private func handleServerEvent(_ text: String) { + guard let data = text.data(using: .utf8), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let eventType = json["message_type"] as? String else { + return + } + + switch eventType { + case "partial_transcript": + let text = (json["text"] as? String) ?? (json["partial_transcript"] as? String) ?? "" + updatePartial(text) + case "committed_transcript", "committed_transcript_with_timestamps": + let text = (json["text"] as? String) ?? (json["transcript"] as? String) ?? "" + commitSegment(text) + stateQueue.sync { + if commitSent { + postCommitCompleted = true + } + } + resumeIfReadyAfterCommit() + default: + if eventType.lowercased().contains("error") { + let message = (json["error"] as? String) + ?? (json["message"] as? String) + ?? "unknown realtime error" + os_log(.error, log: realtimeLog, "ElevenLabs server error [%{public}@]: %{public}@", eventType, message) + let error = RealtimeTranscriptionError.serverError(code: eventType, message: message) + stateQueue.sync { + terminalError = error + closed = true + if let cont = finalContinuation { + finalContinuation = nil + cont.resume(throwing: error) + } + } + } + } + } + + private func updatePartial(_ text: String) { + let snapshot: String = stateQueue.sync { + partialText = text + return joinedTranscriptLocked(finalText, partialText) + } + reportPartial(snapshot) + } + + private func commitSegment(_ transcript: String) { + let snapshot: String = stateQueue.sync { + let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmed.isEmpty { + if !finalText.isEmpty { finalText += " " } + finalText += trimmed + } + partialText = "" + return finalText + } + reportPartial(snapshot) + } + + private func sendAudioChunk(_ data: Data, commit: Bool) { + let currentTask: URLSessionWebSocketTask? = stateQueue.sync { + task + } + guard let currentTask else { return } + let message: [String: Any] = [ + "message_type": "input_audio_chunk", + "audio_base_64": data.base64EncodedString(), + "sample_rate": Int(pcmSampleRate), + "commit": commit + ] + send(message, over: currentTask) + stateQueue.sync { + hasSentAudio = true + } + } + + private func send(_ payload: [String: Any], over task: URLSessionWebSocketTask) { + guard let data = try? JSONSerialization.data(withJSONObject: payload), + let text = String(data: data, encoding: .utf8) else { + return + } + task.send(.string(text)) { error in + if let error { + os_log(.error, log: realtimeLog, "ElevenLabs send failed: %{public}@", error.localizedDescription) + } + } + } + + private func resumeIfReadyAfterCommit() { + var pendingResume: (CheckedContinuation, String)? + stateQueue.sync { + guard let cont = finalContinuation, + let finalText = readyCommittedTranscriptLocked() else { + return + } + finalContinuation = nil + closed = true + pendingResume = (cont, finalText) + } + if let (cont, text) = pendingResume { + let currentTask: URLSessionWebSocketTask? = stateQueue.sync { + task + } + currentTask?.cancel(with: .normalClosure, reason: nil) + cont.resume(returning: text) + } + } + + private func readyCommittedTranscriptLocked() -> String? { + guard commitSent, postCommitCompleted else { + return nil + } + return finalText + } + + private func reportPartial(_ text: String) { + guard let handler = onPartialUpdate else { return } + DispatchQueue.main.async { + handler(text) + } + } + + private func joinedTranscriptLocked(_ final: String, _ partial: String) -> String { + if final.isEmpty { return partial } + if partial.isEmpty { return final } + return final + " " + partial + } + + static func deriveWebSocketURL( + baseURL: String, + model: String, + language: String? + ) -> URL? { + let trimmed = baseURL.trimmingCharacters(in: .whitespacesAndNewlines) + guard var components = URLComponents(string: trimmed) else { return nil } + + switch components.scheme?.lowercased() { + case "http": components.scheme = "ws" + case "https": components.scheme = "wss" + case "ws", "wss": break + default: return nil + } + + var path = components.path + if path.hasSuffix("/") { path.removeLast() } + if !path.hasSuffix("/speech-to-text/realtime") { + path += "/speech-to-text/realtime" + } + components.path = path + + var queryItems = components.queryItems ?? [] + func setQueryItem(_ name: String, _ value: String) { + queryItems.removeAll { $0.name == name } + queryItems.append(URLQueryItem(name: name, value: value)) + } + setQueryItem("model_id", model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + ? TranscriptionService.defaultElevenLabsRealtimeModel + : model) + setQueryItem("commit_strategy", "manual") + setQueryItem("audio_format", "pcm_16000") + if let language, !language.isEmpty { + setQueryItem("language_code", language) + } + components.queryItems = queryItems + return components.url + } + + private static func byteCount( + seconds: Double, + sampleRate: Double, + bytesPerSample: Int + ) -> Int { + Int(seconds * sampleRate) * bytesPerSample + } +} diff --git a/Sources/SettingsView.swift b/Sources/SettingsView.swift index d639bf1b..c1059f0f 100644 --- a/Sources/SettingsView.swift +++ b/Sources/SettingsView.swift @@ -51,14 +51,23 @@ struct ProviderSettingsFields: View { @FocusState private var isEditingContextModel: Bool @FocusState private var transcriptionAPIURLFocused: Bool @FocusState private var transcriptionAPIKeyFocused: Bool + @FocusState private var elevenLabsAPIKeyFocused: Bool @State private var transcriptionModelDraft: String = "" @State private var realtimeStreamingModelDraft: String = "" @State private var postProcessingModelDraft: String = "" @State private var postProcessingFallbackModelDraft: String = "" @State private var contextModelDraft: String = "" + @State private var elevenLabsAPIKeyInput: String = "" + @State private var isValidatingTranscriptionAPIKey = false + @State private var transcriptionAPIKeyValidationError: String? + @State private var transcriptionAPIKeyValidationSuccess = false let showsModelDescription: Bool + private var selectedTranscriptionProvider: TranscriptionProvider { + appState.transcriptionProvider + } + private func commitAPIBaseURL() { let trimmed = apiBaseURLInput.trimmingCharacters(in: .whitespacesAndNewlines) let resolvedBaseURL = trimmed.isEmpty ? AppState.defaultAPIBaseURL : trimmed @@ -115,12 +124,44 @@ struct ProviderSettingsFields: View { appState.transcriptionAPIKey = trimmed } + private func commitElevenLabsAPIKey() { + let trimmed = elevenLabsAPIKeyInput.trimmingCharacters(in: .whitespacesAndNewlines) + elevenLabsAPIKeyInput = trimmed + guard appState.elevenLabsAPIKey != trimmed else { return } + appState.elevenLabsAPIKey = trimmed + } + + private func validateElevenLabsTranscriptionAPIKey() { + commitElevenLabsAPIKey() + let key = elevenLabsAPIKeyInput.trimmingCharacters(in: .whitespacesAndNewlines) + guard !key.isEmpty else { return } + isValidatingTranscriptionAPIKey = true + transcriptionAPIKeyValidationError = nil + transcriptionAPIKeyValidationSuccess = false + + Task { + let valid = await TranscriptionService.validateAPIKey( + key, + baseURL: TranscriptionService.defaultElevenLabsBaseURL, + provider: .elevenLabs + ) + await MainActor.run { + isValidatingTranscriptionAPIKey = false + if valid { + transcriptionAPIKeyValidationSuccess = true + } else { + transcriptionAPIKeyValidationError = "Validation failed. Check your ElevenLabs API key and try again." + } + } + } + } + var body: some View { VStack(alignment: .leading, spacing: 12) { - Text("API Base URL") + Text("Cleanup API Base URL") .font(.caption.weight(.semibold)) - Text("Change this to use a different OpenAI-compatible API provider.") + Text("Used for transcript cleanup, Edit Mode, and context. Change this to use a different OpenAI-compatible provider.") .font(.caption) .foregroundStyle(.secondary) @@ -190,18 +231,48 @@ struct ProviderSettingsFields: View { } ) - ModelDropdownView( - title: "Transcription Model", - subtitle: "Used for speech-to-text transcription.", - predefinedModels: ModelConfiguration.transcriptionModels, - defaultModel: AppState.defaultTranscriptionModel, - textDraft: $transcriptionModelDraft, - onCommit: commitTranscriptionModel, - onReset: { - transcriptionModelDraft = AppState.defaultTranscriptionModel - appState.transcriptionModel = AppState.defaultTranscriptionModel + Divider() + + VStack(alignment: .leading, spacing: 6) { + Text("Transcription Provider") + .font(.caption.weight(.semibold)) + Picker("", selection: $appState.transcriptionProvider) { + ForEach(TranscriptionProvider.allCases) { provider in + Text(provider.displayName).tag(provider) + } } - ) + .labelsHidden() + Text(selectedTranscriptionProvider == .elevenLabs + ? "Uses ElevenLabs Scribe for speech-to-text. Cleanup and context still use the API Base URL above." + : "Uses an OpenAI-compatible audio transcription endpoint.") + .font(.caption) + .foregroundStyle(.secondary) + } + + if selectedTranscriptionProvider == .openAICompatible { + ModelDropdownView( + title: "Transcription Model", + subtitle: "Used for speech-to-text transcription.", + predefinedModels: ModelConfiguration.transcriptionModels, + defaultModel: AppState.defaultTranscriptionModel, + textDraft: $transcriptionModelDraft, + onCommit: commitTranscriptionModel, + onReset: { + transcriptionModelDraft = AppState.defaultTranscriptionModel + appState.transcriptionModel = AppState.defaultTranscriptionModel + } + ) + } else { + VStack(alignment: .leading, spacing: 6) { + Text("Transcription Model") + .font(.caption.weight(.semibold)) + Text(TranscriptionService.defaultElevenLabsModel) + .font(.system(.body, design: .monospaced)) + Text("ElevenLabs Scribe v2 is used for upload transcription.") + .font(.caption) + .foregroundStyle(.secondary) + } + } VStack(alignment: .leading, spacing: 6) { Text("Transcription Language") @@ -218,56 +289,113 @@ struct ProviderSettingsFields: View { .foregroundStyle(.secondary) } - VStack(alignment: .leading, spacing: 6) { - Text("Transcription API URL") - .font(.caption.weight(.semibold)) - HStack(spacing: 8) { - TextField("Uses API Base URL when empty", text: $transcriptionAPIURLInput) - .textFieldStyle(.roundedBorder) - .font(.system(.body, design: .monospaced)) - .focused($transcriptionAPIURLFocused) - .onSubmit { - commitTranscriptionAPIURL() - } - .onChange(of: transcriptionAPIURLFocused) { isFocused in - if !isFocused { + if selectedTranscriptionProvider == .openAICompatible { + VStack(alignment: .leading, spacing: 6) { + Text("Transcription API URL") + .font(.caption.weight(.semibold)) + HStack(spacing: 8) { + TextField("Uses Cleanup API Base URL when empty", text: $transcriptionAPIURLInput) + .textFieldStyle(.roundedBorder) + .font(.system(.body, design: .monospaced)) + .focused($transcriptionAPIURLFocused) + .onSubmit { commitTranscriptionAPIURL() } + .onChange(of: transcriptionAPIURLFocused) { isFocused in + if !isFocused { + commitTranscriptionAPIURL() + } + } + if !transcriptionAPIURLInput.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + Button("Clear") { + transcriptionAPIURLInput = "" + appState.transcriptionAPIURL = "" + } + .font(.caption) } - if !transcriptionAPIURLInput.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { - Button("Clear") { - transcriptionAPIURLInput = "" - appState.transcriptionAPIURL = "" - } - .font(.caption) } } } VStack(alignment: .leading, spacing: 6) { - Text("Transcription API Key") + Text(selectedTranscriptionProvider == .elevenLabs ? "ElevenLabs API Key" : "Transcription API Key") .font(.caption.weight(.semibold)) HStack(spacing: 8) { - SecureField("Uses API Key when empty", text: $transcriptionAPIKeyInput) + SecureField( + selectedTranscriptionProvider == .elevenLabs + ? "Required for Scribe" + : "Uses API Key when empty", + text: selectedTranscriptionProvider == .elevenLabs + ? $elevenLabsAPIKeyInput + : $transcriptionAPIKeyInput + ) .textFieldStyle(.roundedBorder) .font(.system(.body, design: .monospaced)) - .focused($transcriptionAPIKeyFocused) + .focused( + selectedTranscriptionProvider == .elevenLabs + ? $elevenLabsAPIKeyFocused + : $transcriptionAPIKeyFocused + ) .onSubmit { - commitTranscriptionAPIKey() + selectedTranscriptionProvider == .elevenLabs + ? commitElevenLabsAPIKey() + : commitTranscriptionAPIKey() } .onChange(of: transcriptionAPIKeyFocused) { isFocused in if !isFocused { commitTranscriptionAPIKey() } } - if !transcriptionAPIKeyInput.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + .onChange(of: elevenLabsAPIKeyFocused) { isFocused in + if !isFocused { + commitElevenLabsAPIKey() + } + } + .onChange(of: transcriptionAPIKeyInput) { _ in + transcriptionAPIKeyValidationError = nil + transcriptionAPIKeyValidationSuccess = false + } + .onChange(of: elevenLabsAPIKeyInput) { _ in + transcriptionAPIKeyValidationError = nil + transcriptionAPIKeyValidationSuccess = false + } + if selectedTranscriptionProvider == .elevenLabs { + Button(isValidatingTranscriptionAPIKey ? "Validating..." : "Validate") { + validateElevenLabsTranscriptionAPIKey() + } + .font(.caption) + .disabled( + elevenLabsAPIKeyInput.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + || isValidatingTranscriptionAPIKey + ) + } + if !(selectedTranscriptionProvider == .elevenLabs + ? elevenLabsAPIKeyInput + : transcriptionAPIKeyInput + ).trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { Button("Clear") { - transcriptionAPIKeyInput = "" - appState.transcriptionAPIKey = "" + if selectedTranscriptionProvider == .elevenLabs { + elevenLabsAPIKeyInput = "" + appState.elevenLabsAPIKey = "" + } else { + transcriptionAPIKeyInput = "" + appState.transcriptionAPIKey = "" + } + transcriptionAPIKeyValidationError = nil + transcriptionAPIKeyValidationSuccess = false } .font(.caption) } } + if let error = transcriptionAPIKeyValidationError { + Label(error, systemImage: "xmark.circle.fill") + .foregroundStyle(.red) + .font(.caption) + } else if transcriptionAPIKeyValidationSuccess { + Label("ElevenLabs API key is valid", systemImage: "checkmark.circle.fill") + .foregroundStyle(.green) + .font(.caption) + } } Divider() @@ -276,36 +404,50 @@ struct ProviderSettingsFields: View { "Stream audio while recording (realtime)", isOn: $appState.realtimeStreamingEnabled ) - Text("Streams audio through the provider's OpenAI-compatible /v1/realtime WebSocket so transcription runs while you speak.") + Text(selectedTranscriptionProvider == .elevenLabs + ? "Streams audio through ElevenLabs Scribe v2 Realtime." + : "Streams audio through the provider's OpenAI-compatible /v1/realtime WebSocket.") .font(.caption) .foregroundStyle(.secondary) - VStack(alignment: .leading, spacing: 6) { - Text("Realtime Transcription Model") - .font(.caption.weight(.semibold)) - HStack(spacing: 8) { - TextField("Required by some providers, e.g. gpt-4o-transcribe", text: $realtimeStreamingModelDraft) - .textFieldStyle(.roundedBorder) - .focused($isEditingRealtimeStreamingModel) - .onSubmit { - commitRealtimeStreamingModel() - } - .onChange(of: isEditingRealtimeStreamingModel) { isEditing in - if !isEditing { + if selectedTranscriptionProvider == .openAICompatible { + VStack(alignment: .leading, spacing: 6) { + Text("Realtime Transcription Model") + .font(.caption.weight(.semibold)) + HStack(spacing: 8) { + TextField("Required by some providers, e.g. gpt-4o-transcribe", text: $realtimeStreamingModelDraft) + .textFieldStyle(.roundedBorder) + .focused($isEditingRealtimeStreamingModel) + .onSubmit { commitRealtimeStreamingModel() } + .onChange(of: isEditingRealtimeStreamingModel) { isEditing in + if !isEditing { + commitRealtimeStreamingModel() + } + } + if !realtimeStreamingModelDraft.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + Button("Reset") { + realtimeStreamingModelDraft = "" + appState.realtimeStreamingModel = "" + } + .font(.caption) } - if !realtimeStreamingModelDraft.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { - Button("Reset") { - realtimeStreamingModelDraft = "" - appState.realtimeStreamingModel = "" - } - .font(.caption) } + Text("Used only for realtime streaming. Leave empty for providers that supply a server default.") + .font(.caption) + .foregroundStyle(.secondary) + } + } else { + VStack(alignment: .leading, spacing: 6) { + Text("Realtime Transcription Model") + .font(.caption.weight(.semibold)) + Text(TranscriptionService.defaultElevenLabsRealtimeModel) + .font(.system(.body, design: .monospaced)) + Text("ElevenLabs Scribe v2 Realtime is used when streaming is enabled.") + .font(.caption) + .foregroundStyle(.secondary) } - Text("Used only for realtime streaming. Leave empty for providers that supply a server default.") - .font(.caption) - .foregroundStyle(.secondary) } } .onAppear { @@ -314,6 +456,7 @@ struct ProviderSettingsFields: View { postProcessingModelDraft = appState.postProcessingModel postProcessingFallbackModelDraft = appState.postProcessingFallbackModel contextModelDraft = appState.contextModel + elevenLabsAPIKeyInput = appState.elevenLabsAPIKey } .onChange(of: appState.transcriptionModel) { value in if !isEditingTranscriptionModel { @@ -325,6 +468,15 @@ struct ProviderSettingsFields: View { realtimeStreamingModelDraft = value } } + .onChange(of: appState.transcriptionProvider) { _ in + transcriptionAPIKeyValidationError = nil + transcriptionAPIKeyValidationSuccess = false + } + .onChange(of: appState.elevenLabsAPIKey) { value in + if !elevenLabsAPIKeyFocused { + elevenLabsAPIKeyInput = value + } + } .onChange(of: appState.postProcessingModel) { value in if !isEditingPostProcessingModel { postProcessingModelDraft = value @@ -645,7 +797,7 @@ struct GeneralSettingsView: View { SettingsCard("Updates", icon: "arrow.triangle.2.circlepath") { updatesSection } - SettingsCard("API Key", icon: "key.fill") { + SettingsCard("Providers", icon: "key.fill") { apiKeySection } SettingsCard("Output Language", icon: "globe") { @@ -902,12 +1054,12 @@ struct GeneralSettingsView: View { private var apiKeySection: some View { VStack(alignment: .leading, spacing: 10) { - Text("\(AppName.displayName) uses the configured transcription model with your selected OpenAI-compatible provider.") + Text("\(AppName.displayName) uses this OpenAI-compatible provider for transcript cleanup, Edit Mode, and context.") .font(.caption) .foregroundStyle(.secondary) HStack(spacing: 8) { - SecureField("Enter your Groq API key", text: $apiKeyInput) + SecureField("Enter your Groq or OpenAI-compatible API key", text: $apiKeyInput) .textFieldStyle(.roundedBorder) .font(.system(.body, design: .monospaced)) .disabled(isValidatingKey) @@ -944,7 +1096,7 @@ struct GeneralSettingsView: View { } } label: { HStack { - Text("Advanced Provider Settings") + Text("Provider Settings") Spacer() } .contentShape(Rectangle()) @@ -974,7 +1126,7 @@ struct GeneralSettingsView: View { appState.apiKey = key keyValidationSuccess = true } else { - keyValidationError = "Validation failed. Please check your API key and provider settings, then try again." + keyValidationError = "Validation failed. Please check your cleanup provider key and settings, then try again." } } } diff --git a/Sources/SetupView.swift b/Sources/SetupView.swift index 048f3e14..d35a9b4f 100644 --- a/Sources/SetupView.swift +++ b/Sources/SetupView.swift @@ -13,9 +13,9 @@ private struct SetupProviderSettingsSheet: View { var body: some View { VStack(spacing: 0) { VStack(alignment: .leading, spacing: 8) { - Text("Advanced Provider Settings") + Text("Provider Settings") .font(.title2.weight(.semibold)) - Text("Use these fields when pointing \(AppName.displayName) at another OpenAI-compatible provider or when you need custom model IDs.") + Text("Configure cleanup models and choose the speech-to-text provider.") .font(.subheadline) .foregroundStyle(.secondary) .fixedSize(horizontal: false, vertical: true) @@ -397,7 +397,7 @@ struct SetupView: View { .font(.title) .fontWeight(.bold) - Text("Enter an API key for your OpenAI-compatible provider. If you are not using Groq, expand the advanced provider settings and enter that provider's base URL and model IDs before continuing.") + Text("Enter an API key for transcript cleanup and context. To use ElevenLabs for speech-to-text, open Provider Settings and choose ElevenLabs Scribe as the transcription provider.") .multilineTextAlignment(.center) .foregroundStyle(.secondary) .fixedSize(horizontal: false, vertical: true) @@ -444,9 +444,9 @@ struct SetupView: View { Image(systemName: "slider.horizontal.3") .foregroundStyle(.secondary) VStack(alignment: .leading, spacing: 2) { - Text("Advanced Provider Settings") + Text("Provider Settings") .foregroundStyle(.primary) - Text("Base URL and model IDs") + Text("Transcription provider, base URL, and model IDs") .font(.caption) .foregroundStyle(.secondary) } @@ -1151,7 +1151,7 @@ struct SetupView: View { currentStep = nextStep(currentStep) } } else { - keyValidationError = "Validation failed. Please check your API key and provider settings, then try again." + keyValidationError = "Validation failed. Please check your cleanup provider key and settings, then try again." } } } diff --git a/Sources/TranscriptionService.swift b/Sources/TranscriptionService.swift index 94c2b43a..7b96f739 100644 --- a/Sources/TranscriptionService.swift +++ b/Sources/TranscriptionService.swift @@ -3,51 +3,73 @@ import os.log private let transcriptionLog = OSLog(subsystem: "com.zachlatta.freeflow", category: "Transcription") +enum TranscriptionProvider: String, CaseIterable, Identifiable { + case openAICompatible = "openai_compatible" + case elevenLabs = "elevenlabs" + + var id: String { rawValue } + + var displayName: String { + switch self { + case .openAICompatible: return "OpenAI-compatible" + case .elevenLabs: return "ElevenLabs Scribe" + } + } +} + +private protocol BatchTranscriptionClient { + func transcribe(fileURL: URL) async throws -> String +} + class TranscriptionService { - private let apiKey: String - private let baseURL: URL - private let transcriptionModel: String - private let language: String? - private let transcriptionResponseFormat = "verbose_json" + static let defaultOpenAICompatibleBaseURL = "https://api.groq.com/openai/v1" + static let defaultElevenLabsBaseURL = "https://api.elevenlabs.io/v1" + static let defaultElevenLabsModel = "scribe_v2" + static let defaultElevenLabsRealtimeModel = "scribe_v2_realtime" + + private let client: BatchTranscriptionClient private var transcriptionTimeoutSeconds: TimeInterval { let override = UserDefaults.standard.double(forKey: "transcription_timeout_seconds") return override > 0 ? override : 20 } init( + provider: TranscriptionProvider = .openAICompatible, apiKey: String, - baseURL: String = "https://api.groq.com/openai/v1", + baseURL: String = TranscriptionService.defaultOpenAICompatibleBaseURL, transcriptionModel: String = "whisper-large-v3", language: String? = nil ) throws { - self.apiKey = apiKey - self.baseURL = try Self.normalizedBaseURL(from: baseURL) - let trimmedModel = transcriptionModel.trimmingCharacters(in: .whitespacesAndNewlines) - self.transcriptionModel = trimmedModel.isEmpty ? "whisper-large-v3" : trimmedModel - let trimmedLanguage = language?.trimmingCharacters(in: .whitespacesAndNewlines) - self.language = (trimmedLanguage?.isEmpty == false) ? trimmedLanguage : nil + switch provider { + case .openAICompatible: + self.client = try OpenAICompatibleTranscriptionClient( + apiKey: apiKey, + baseURL: baseURL, + transcriptionModel: transcriptionModel, + language: language + ) + case .elevenLabs: + self.client = try ElevenLabsTranscriptionClient( + apiKey: apiKey, + baseURL: baseURL, + language: language + ) + } } - // Validate API key by hitting a lightweight endpoint - static func validateAPIKey(_ key: String, baseURL: String = "https://api.groq.com/openai/v1") async -> Bool { - let trimmed = key.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { return false } - guard let baseURL = try? normalizedBaseURL(from: baseURL) else { return false } - - var request = URLRequest(url: baseURL.appendingPathComponent("models")) - request.timeoutInterval = 10 - request.setValue("Bearer \(trimmed)", forHTTPHeaderField: "Authorization") - - do { - let (_, response) = try await LLMAPITransport.data(for: request) - let status = (response as? HTTPURLResponse)?.statusCode ?? 0 - return status == 200 - } catch { - return false + static func validateAPIKey( + _ key: String, + baseURL: String = TranscriptionService.defaultOpenAICompatibleBaseURL, + provider: TranscriptionProvider = .openAICompatible + ) async -> Bool { + switch provider { + case .openAICompatible: + return await OpenAICompatibleTranscriptionClient.validateAPIKey(key, baseURL: baseURL) + case .elevenLabs: + return await ElevenLabsTranscriptionClient.validateAPIKey(key, baseURL: baseURL) } } - // Upload audio file, submit for transcription, poll until done, return text func transcribe(fileURL: URL) async throws -> String { guard !Task.isCancelled else { throw CancellationError() @@ -65,7 +87,7 @@ class TranscriptionService { guard let self else { throw TranscriptionError.transcriptionFailed("Transcription service deallocated") } - let result = try await self.transcribeAudio(fileURL: fileURL) + let result = try await self.client.transcribe(fileURL: fileURL) raceState.finish(.success(result)) } catch { raceState.finish(.failure(Self.transcriptionTimeoutErrorIfNeeded( @@ -92,12 +114,89 @@ class TranscriptionService { } } - // Send audio file for transcription and return text - private func transcribeAudio(fileURL: URL) async throws -> String { - return try await transcribeAudioWithURLSession(fileURL: fileURL) + static func friendlyHTTPMessage(status: Int, host: String?) -> String { + let provider = host ?? "the provider" + switch status { + case 400: + return "Request rejected by \(provider) (HTTP 400). Check provider settings." + case 401: + return "Invalid API key for \(provider). Open Settings to fix it." + case 403: + return "Key lacks permission for this endpoint at \(provider) (HTTP 403). Check the key's scopes." + case 404: + return "Endpoint not found at \(provider) (HTTP 404). Base URL is likely wrong for this provider." + case 413: + return "Audio file too large for \(provider) (HTTP 413). Try a shorter recording." + case 422: + return "Audio request rejected by \(provider) (HTTP 422). Try again with a shorter recording." + case 429: + return "Rate limit reached at \(provider) (HTTP 429). Wait a moment and try again." + case 500..<600: + return "Provider error at \(provider) (HTTP \(status)). Try again in a moment." + default: + return "Request failed at \(provider) (HTTP \(status))." + } + } + + private static func transcriptionTimeoutErrorIfNeeded( + _ error: Error, + timeoutSeconds: TimeInterval + ) -> Error { + if let urlError = error as? URLError, urlError.code == .timedOut { + return TranscriptionError.transcriptionTimedOut(timeoutSeconds) + } + return error } +} + +private final class OpenAICompatibleTranscriptionClient: BatchTranscriptionClient { + private let apiKey: String + private let baseURL: URL + private let transcriptionModel: String + private let language: String? + private let transcriptionResponseFormat = "verbose_json" + private var transcriptionTimeoutSeconds: TimeInterval { + let override = UserDefaults.standard.double(forKey: "transcription_timeout_seconds") + return override > 0 ? override : 20 + } + + init( + apiKey: String, + baseURL: String, + transcriptionModel: String, + language: String? + ) throws { + self.apiKey = apiKey.trimmingCharacters(in: .whitespacesAndNewlines) + self.baseURL = try normalizedBaseURL(from: baseURL) + let trimmedModel = transcriptionModel.trimmingCharacters(in: .whitespacesAndNewlines) + self.transcriptionModel = trimmedModel.isEmpty ? "whisper-large-v3" : trimmedModel + let trimmedLanguage = language?.trimmingCharacters(in: .whitespacesAndNewlines) + self.language = (trimmedLanguage?.isEmpty == false) ? trimmedLanguage : nil + } + + static func validateAPIKey(_ key: String, baseURL: String) async -> Bool { + let trimmed = key.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return false } + guard let baseURL = try? normalizedBaseURL(from: baseURL) else { return false } + + var request = URLRequest(url: baseURL.appendingPathComponent("models")) + request.timeoutInterval = 10 + request.setValue("Bearer \(trimmed)", forHTTPHeaderField: "Authorization") + + do { + let (_, response) = try await LLMAPITransport.data(for: request) + let status = (response as? HTTPURLResponse)?.statusCode ?? 0 + return status == 200 + } catch { + return false + } + } + + func transcribe(fileURL: URL) async throws -> String { + guard !apiKey.isEmpty else { + throw TranscriptionError.submissionFailed("Enter an API key in Settings.") + } - private func transcribeAudioWithURLSession(fileURL: URL) async throws -> String { let url = baseURL .appendingPathComponent("audio") .appendingPathComponent("transcriptions") @@ -109,12 +208,19 @@ class TranscriptionService { request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type") let audioData = try Data(contentsOf: fileURL) + var fields: [(String, String)] = [ + ("model", transcriptionModel), + ("response_format", transcriptionResponseFormat) + ] + if let language { + fields.append(("language", language)) + } + let body = makeMultipartBody( + fields: fields, audioData: audioData, + fileFieldName: "file", fileName: fileURL.lastPathComponent, - model: transcriptionModel, - responseFormat: transcriptionResponseFormat, - language: language, boundary: boundary ) @@ -122,17 +228,7 @@ class TranscriptionService { let (data, response) = try await LLMAPITransport.upload(for: request, from: body) return try validateTranscriptionResponse(data: data, response: response, fileURL: fileURL) } catch { - let nsError = error as NSError - os_log( - .error, - log: transcriptionLog, - "URLSession upload failed for %{public}@ (bytes=%{public}lld): domain=%{public}@ code=%ld desc=%{public}@", - fileURL.lastPathComponent, - fileSizeBytes(for: fileURL), - nsError.domain, - nsError.code, - error.localizedDescription - ) + logUploadFailure(error, fileURL: fileURL) throw error } } @@ -143,17 +239,15 @@ class TranscriptionService { } guard httpResponse.statusCode == 200 else { - let responseBody = String(data: data, encoding: .utf8) ?? "" os_log( .error, log: transcriptionLog, - "URLSession upload returned HTTP %ld for %{public}@ (bytes=%{public}lld) body=%{public}@", + "OpenAI-compatible upload returned HTTP %ld for %{public}@ (bytes=%{public}lld)", httpResponse.statusCode, fileURL.lastPathComponent, - fileSizeBytes(for: fileURL), - responseBody + fileSizeBytes(for: fileURL) ) - throw TranscriptionError.submissionFailed(Self.friendlyHTTPMessage( + throw TranscriptionError.submissionFailed(TranscriptionService.friendlyHTTPMessage( status: httpResponse.statusCode, host: baseURL.host )) @@ -161,139 +255,7 @@ class TranscriptionService { return try parseTranscript(from: data) } - private func audioContentType(for fileName: String) -> String { - if fileName.lowercased().hasSuffix(".wav") { - return "audio/wav" - } - if fileName.lowercased().hasSuffix(".mp3") { - return "audio/mpeg" - } - if fileName.lowercased().hasSuffix(".m4a") { - return "audio/mp4" - } - return "audio/mp4" - } - - private func fileSizeBytes(for fileURL: URL) -> Int64 { - let attributes = try? FileManager.default.attributesOfItem(atPath: fileURL.path) - return (attributes?[.size] as? NSNumber)?.int64Value ?? -1 - } - - private func makeMultipartBody( - audioData: Data, - fileName: String, - model: String, - responseFormat: String, - language: String?, - boundary: String - ) -> Data { - var body = Data() - - func append(_ value: String) { - body.append(Data(value.utf8)) - } - - append("--\(boundary)\r\n") - append("Content-Disposition: form-data; name=\"model\"\r\n\r\n") - append("\(model)\r\n") - - append("--\(boundary)\r\n") - append("Content-Disposition: form-data; name=\"response_format\"\r\n\r\n") - append("\(responseFormat)\r\n") - - if let language, !language.isEmpty { - append("--\(boundary)\r\n") - append("Content-Disposition: form-data; name=\"language\"\r\n\r\n") - append("\(language)\r\n") - } - - append("--\(boundary)\r\n") - append("Content-Disposition: form-data; name=\"file\"; filename=\"\(fileName)\"\r\n") - append("Content-Type: \(audioContentType(for: fileName))\r\n\r\n") - body.append(audioData) - append("\r\n") - append("--\(boundary)--\r\n") - - return body - } - /// Map a non-200 HTTP status into a one-line user-readable message. - /// Used for transcription submission failures so the menu bar shows - /// "Invalid API key for api.openai.com" instead of raw JSON. - static func friendlyHTTPMessage(status: Int, host: String?) -> String { - let provider = host ?? "the provider" - switch status { - case 401: - return "Invalid API key for \(provider). Open Settings to fix it." - case 403: - return "Key lacks permission for this endpoint at \(provider) (HTTP 403). Check the key's scopes." - case 404: - return "Endpoint not found at \(provider) (HTTP 404). Base URL is likely wrong for this provider." - case 413: - return "Audio file too large for \(provider) (HTTP 413). Try a shorter recording." - case 429: - return "Rate limit reached at \(provider) (HTTP 429). Wait a moment and try again." - case 500..<600: - return "Provider error at \(provider) (HTTP \(status)). Try again in a moment." - default: - return "Request failed at \(provider) (HTTP \(status))." - } - } - - private static func transcriptionTimeoutErrorIfNeeded( - _ error: Error, - timeoutSeconds: TimeInterval - ) -> Error { - if let urlError = error as? URLError, urlError.code == .timedOut { - return TranscriptionError.transcriptionTimedOut(timeoutSeconds) - } - return error - } - - private static func normalizedBaseURL(from baseURL: String) throws -> URL { - let trimmed = baseURL.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { - throw TranscriptionError.invalidBaseURL("Provider URL is empty.") - } - - guard var components = URLComponents(string: trimmed) else { - throw TranscriptionError.invalidBaseURL("Provider URL is malformed.") - } - - guard let scheme = components.scheme?.lowercased(), scheme == "http" || scheme == "https" else { - throw TranscriptionError.invalidBaseURL("Provider URL must use http or https.") - } - - guard let host = components.host, !host.isEmpty else { - throw TranscriptionError.invalidBaseURL("Provider URL must include a host.") - } - - components.scheme = scheme - if components.path == "/" { - components.path = "" - } else { - components.path = components.path.replacingOccurrences( - of: "/+$", - with: "", - options: .regularExpression - ) - } - - guard let normalizedURL = components.url else { - throw TranscriptionError.invalidBaseURL("Provider URL is malformed.") - } - - return normalizedURL - } - - // Whisper-large-v3 hallucinates common short phrases on silence/background - // noise. Drop them when whisper itself reports a high no_speech_prob. - // Add a new (phrase, minNoSpeechProb) pair here to filter more hallucinations. - // - // Thresholds tuned on ~500 samples from quiet and noisy environments, including - // both positive cases (real "thank you" speech) and empty-audio cases. Kept - // conservative to minimize false positives (filtering real user speech). - // Normal speech included audios have very low no_speech_prob. private let hallucinationPhrases = [ "thank you", "thank you for watching", @@ -320,9 +282,9 @@ class TranscriptionService { let plainText = String(data: data, encoding: .utf8) ?? "" let text = plainText - .components(separatedBy: .newlines) - .joined(separator: " ") - .trimmingCharacters(in: .whitespacesAndNewlines) + .components(separatedBy: .newlines) + .joined(separator: " ") + .trimmingCharacters(in: .whitespacesAndNewlines) guard !text.isEmpty else { throw TranscriptionError.pollFailed("Invalid response") } @@ -361,6 +323,205 @@ class TranscriptionService { } } +private final class ElevenLabsTranscriptionClient: BatchTranscriptionClient { + private let apiKey: String + private let baseURL: URL + private let language: String? + private var transcriptionTimeoutSeconds: TimeInterval { + let override = UserDefaults.standard.double(forKey: "transcription_timeout_seconds") + return override > 0 ? override : 20 + } + + init(apiKey: String, baseURL: String, language: String?) throws { + self.apiKey = apiKey.trimmingCharacters(in: .whitespacesAndNewlines) + self.baseURL = try normalizedBaseURL(from: baseURL) + let trimmedLanguage = language?.trimmingCharacters(in: .whitespacesAndNewlines) + self.language = (trimmedLanguage?.isEmpty == false) ? trimmedLanguage : nil + } + + static func validateAPIKey(_ key: String, baseURL: String) async -> Bool { + let trimmed = key.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return false } + guard let baseURL = try? normalizedBaseURL(from: baseURL) else { return false } + + var request = URLRequest(url: baseURL.appendingPathComponent("models")) + request.timeoutInterval = 10 + request.setValue(trimmed, forHTTPHeaderField: "xi-api-key") + + do { + let (_, response) = try await LLMAPITransport.data(for: request) + let status = (response as? HTTPURLResponse)?.statusCode ?? 0 + return status == 200 + } catch { + return false + } + } + + func transcribe(fileURL: URL) async throws -> String { + guard !apiKey.isEmpty else { + throw TranscriptionError.submissionFailed("Enter an ElevenLabs API key in Settings.") + } + + let url = baseURL.appendingPathComponent("speech-to-text") + var request = URLRequest(url: url) + request.httpMethod = "POST" + request.timeoutInterval = transcriptionTimeoutSeconds + request.setValue(apiKey, forHTTPHeaderField: "xi-api-key") + let boundary = UUID().uuidString + request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type") + + var fields: [(String, String)] = [ + ("model_id", TranscriptionService.defaultElevenLabsModel), + ("tag_audio_events", "false"), + ("timestamps_granularity", "none") + ] + if let language { + fields.append(("language_code", language)) + } + + let audioData = try Data(contentsOf: fileURL) + let body = makeMultipartBody( + fields: fields, + audioData: audioData, + fileFieldName: "file", + fileName: fileURL.lastPathComponent, + boundary: boundary + ) + + do { + let (data, response) = try await LLMAPITransport.upload(for: request, from: body) + return try validateTranscriptionResponse(data: data, response: response, fileURL: fileURL) + } catch { + logUploadFailure(error, fileURL: fileURL) + throw error + } + } + + private func validateTranscriptionResponse(data: Data, response: URLResponse, fileURL: URL) throws -> String { + guard let httpResponse = response as? HTTPURLResponse else { + throw TranscriptionError.submissionFailed("No response from server") + } + + guard httpResponse.statusCode == 200 else { + os_log( + .error, + log: transcriptionLog, + "ElevenLabs upload returned HTTP %ld for %{public}@ (bytes=%{public}lld)", + httpResponse.statusCode, + fileURL.lastPathComponent, + fileSizeBytes(for: fileURL) + ) + throw TranscriptionError.submissionFailed(TranscriptionService.friendlyHTTPMessage( + status: httpResponse.statusCode, + host: baseURL.host + )) + } + + guard let json = try JSONSerialization.jsonObject(with: data) as? [String: Any], + let text = json["text"] as? String else { + throw TranscriptionError.pollFailed("Invalid ElevenLabs response") + } + return text + } +} + +private func normalizedBaseURL(from baseURL: String) throws -> URL { + let trimmed = baseURL.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { + throw TranscriptionError.invalidBaseURL("Provider URL is empty.") + } + + guard var components = URLComponents(string: trimmed) else { + throw TranscriptionError.invalidBaseURL("Provider URL is malformed.") + } + + guard let scheme = components.scheme?.lowercased(), scheme == "http" || scheme == "https" else { + throw TranscriptionError.invalidBaseURL("Provider URL must use http or https.") + } + + guard let host = components.host, !host.isEmpty else { + throw TranscriptionError.invalidBaseURL("Provider URL must include a host.") + } + + components.scheme = scheme + if components.path == "/" { + components.path = "" + } else { + components.path = components.path.replacingOccurrences( + of: "/+$", + with: "", + options: .regularExpression + ) + } + + guard let normalizedURL = components.url else { + throw TranscriptionError.invalidBaseURL("Provider URL is malformed.") + } + + return normalizedURL +} + +private func makeMultipartBody( + fields: [(String, String)], + audioData: Data, + fileFieldName: String, + fileName: String, + boundary: String +) -> Data { + var body = Data() + + func append(_ value: String) { + body.append(Data(value.utf8)) + } + + for (name, value) in fields { + append("--\(boundary)\r\n") + append("Content-Disposition: form-data; name=\"\(name)\"\r\n\r\n") + append("\(value)\r\n") + } + + append("--\(boundary)\r\n") + append("Content-Disposition: form-data; name=\"\(fileFieldName)\"; filename=\"\(fileName)\"\r\n") + append("Content-Type: \(audioContentType(for: fileName))\r\n\r\n") + body.append(audioData) + append("\r\n") + append("--\(boundary)--\r\n") + + return body +} + +private func audioContentType(for fileName: String) -> String { + if fileName.lowercased().hasSuffix(".wav") { + return "audio/wav" + } + if fileName.lowercased().hasSuffix(".mp3") { + return "audio/mpeg" + } + if fileName.lowercased().hasSuffix(".m4a") { + return "audio/mp4" + } + return "audio/mp4" +} + +private func fileSizeBytes(for fileURL: URL) -> Int64 { + let attributes = try? FileManager.default.attributesOfItem(atPath: fileURL.path) + return (attributes?[.size] as? NSNumber)?.int64Value ?? -1 +} + +private func logUploadFailure(_ error: Error, fileURL: URL) { + let nsError = error as NSError + os_log( + .error, + log: transcriptionLog, + "Transcription upload failed for %{public}@ (bytes=%{public}lld): domain=%{public}@ code=%ld desc=%{public}@", + fileURL.lastPathComponent, + fileSizeBytes(for: fileURL), + nsError.domain, + nsError.code, + error.localizedDescription + ) +} + enum TranscriptionError: LocalizedError { case invalidBaseURL(String) case uploadFailed(String) @@ -441,13 +602,3 @@ private final class TranscriptionTimeoutRaceState { finish(.failure(CancellationError())) } } - -private struct PreparedUploadAudio { - let fileURL: URL - let deleteOnCleanup: Bool - - func cleanup() { - guard deleteOnCleanup else { return } - try? FileManager.default.removeItem(at: fileURL) - } -}