diff --git a/CHANGELOG.md b/CHANGELOG.md index 9988bca9..fc05771b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,17 @@ This project uses semantic versioning for public releases. Use `MAJOR.MINOR.PATC - `MINOR` changes add user-visible features and improvements. - `PATCH` changes fix bugs, polish existing behavior, or make small internal improvements. +## [Unreleased] + +### Improved + +- Faster stop-to-paste latency: API connections to the transcription and post-processing hosts are now opened while you are still speaking, so the upload no longer pays DNS + TLS handshake time after you release the shortcut. +- Transcription uploads and post-processing requests now log their durations, making pipeline latency visible in Console. + +### Fixed + +- Fixed the configured transcription, post-processing, and context timeouts being silently capped at 30 seconds by a session-level resource timeout, which broke long transfers to slow local models. + ## [1.1.0] - 2026-06-03 ### Added diff --git a/Sources/AppState.swift b/Sources/AppState.swift index c65bc809..455cdf06 100644 --- a/Sources/AppState.swift +++ b/Sources/AppState.swift @@ -2131,6 +2131,24 @@ final class AppState: ObservableObject, @unchecked Sendable { automaticTerminationDisabled = false } + /// Opens connections to the transcription and post-processing hosts + /// while the user is speaking, so the stop-to-paste path starts on warm + /// TLS connections instead of paying handshake latency after release. + private func prewarmAPIConnections() { + if let url = URL(string: resolvedTranscriptionBaseURL), url.host != nil { + LLMAPITransport.prewarmUploadConnection( + to: url, + expectedRequestTimeout: TranscriptionService.configuredTimeoutSeconds + ) + } + if let url = URL(string: apiBaseURL), url.host != nil { + LLMAPITransport.prewarmSharedConnection( + to: url, + expectedRequestTimeout: PostProcessingService.configuredTimeoutSeconds + ) + } + } + private func beginRecording(triggerMode: RecordingTriggerMode) { os_log(.info, log: recordingLog, "beginRecording() entered") beginCriticalDictationActivity() @@ -2192,6 +2210,7 @@ final class AppState: ObservableObject, @unchecked Sendable { } startRealtimeStreamingIfEnabled() + prewarmAPIConnections() // Start engine on background thread so UI isn't blocked DispatchQueue.global(qos: .userInitiated).async { [weak self] in diff --git a/Sources/LLMAPITransport.swift b/Sources/LLMAPITransport.swift index e414d13b..2947c1b6 100644 --- a/Sources/LLMAPITransport.swift +++ b/Sources/LLMAPITransport.swift @@ -1,32 +1,114 @@ import Foundation enum LLMAPITransport { - private static let requestSession: URLSession = { - makeEphemeralSession() - }() + /// Floor for the whole-transfer budget so requests that never set an + /// explicit timeout still fail within a reasonable window. + private static let minimumResourceTimeout: TimeInterval = 30 - private static func makeEphemeralSession() -> URLSession { + /// Reusable sessions keyed by resource timeout, so connection reuse is + /// preserved while each request still gets a whole-transfer budget that + /// honors the caller's configured timeout. Only a handful of distinct + /// timeout values ever exist (one per timeout setting). + private static var sessionsByResourceTimeout: [TimeInterval: URLSession] = [:] + private static let sessionsLock = NSLock() + + private static func makeEphemeralSession(resourceTimeout: TimeInterval) -> URLSession { let configuration = URLSessionConfiguration.ephemeral configuration.requestCachePolicy = .reloadIgnoringLocalCacheData configuration.urlCache = nil configuration.timeoutIntervalForRequest = 20 - configuration.timeoutIntervalForResource = 30 + configuration.timeoutIntervalForResource = resourceTimeout return URLSession(configuration: configuration) } + /// timeoutIntervalForResource caps the entire transfer and has no + /// per-request override, so derive it from the caller's configured + /// timeout (e.g. post_processing_timeout_seconds) instead of a fixed cap. + private static func resourceTimeout(for request: URLRequest) -> TimeInterval { + max(request.timeoutInterval, minimumResourceTimeout) + } + + private static func sharedSession(for request: URLRequest) -> URLSession { + let timeout = resourceTimeout(for: request) + sessionsLock.lock() + defer { sessionsLock.unlock() } + if let existing = sessionsByResourceTimeout[timeout] { + return existing + } + let session = makeEphemeralSession(resourceTimeout: timeout) + sessionsByResourceTimeout[timeout] = session + return session + } + + // MARK: - Connection Pre-warming + + /// One-shot session opened while the user is still speaking so the + /// transcription upload does not pay DNS + TCP + TLS setup on the + /// critical stop-to-paste path. Guarded by sessionsLock. + private static var prewarmedUploadSession: (session: URLSession, host: String, resourceTimeout: TimeInterval)? + + /// Opens a connection to `baseURL`'s host on a fresh session and holds + /// that session for the next upload to the same host. Safe to call on + /// every recording start; failures are ignored — the response is + /// irrelevant (401/404 are fine), only the handshake matters. + static func prewarmUploadConnection(to baseURL: URL, expectedRequestTimeout: TimeInterval) { + let timeout = max(expectedRequestTimeout, minimumResourceTimeout) + let session = makeEphemeralSession(resourceTimeout: timeout) + + sessionsLock.lock() + let previous = prewarmedUploadSession?.session + prewarmedUploadSession = (session, baseURL.host ?? "", timeout) + sessionsLock.unlock() + previous?.finishTasksAndInvalidate() + + var request = URLRequest(url: baseURL) + request.httpMethod = "HEAD" + request.timeoutInterval = 10 + Task { + _ = try? await session.data(for: request) + } + } + + /// Warms the shared data session for `baseURL`'s host so the first + /// post-processing call after idle reuses an open connection. The + /// timeout must match the real requests' so the same session is used. + static func prewarmSharedConnection(to baseURL: URL, expectedRequestTimeout: TimeInterval) { + var request = URLRequest(url: baseURL) + request.httpMethod = "HEAD" + request.timeoutInterval = expectedRequestTimeout + Task { + _ = try? await data(for: request) + } + } + + private static func takePrewarmedUploadSession(for request: URLRequest) -> URLSession? { + sessionsLock.lock() + defer { sessionsLock.unlock() } + guard let candidate = prewarmedUploadSession, + candidate.host == request.url?.host, + candidate.resourceTimeout == resourceTimeout(for: request) else { + return nil + } + prewarmedUploadSession = nil + return candidate.session + } + static func data( for request: URLRequest ) async throws -> (Data, URLResponse) { - try await requestSession.data(for: request) + try await sharedSession(for: request).data(for: request) } static func upload( for request: URLRequest, from bodyData: Data ) async throws -> (Data, URLResponse) { - // Use a fresh session for each upload so a bad reused connection cannot + // Prefer the session prewarmed at recording start (skips the TLS + // handshake); otherwise use a fresh session. Either way the session + // serves exactly one upload, so a bad reused connection cannot // poison subsequent transcription uploads. - let session = makeEphemeralSession() + let session = takePrewarmedUploadSession(for: request) + ?? makeEphemeralSession(resourceTimeout: resourceTimeout(for: request)) defer { session.finishTasksAndInvalidate() } return try await session.upload(for: request, from: bodyData) } diff --git a/Sources/PostProcessingService.swift b/Sources/PostProcessingService.swift index 4465a061..21c16c4c 100644 --- a/Sources/PostProcessingService.swift +++ b/Sources/PostProcessingService.swift @@ -1,4 +1,7 @@ import Foundation +import os.log + +private let postProcessingLog = OSLog(subsystem: "com.zachlatta.freeflow", category: "PostProcessing") enum PostProcessingError: LocalizedError { case requestFailed(Int, String) @@ -136,11 +139,15 @@ Behavior: private let defaultFallbackModel = "meta-llama/llama-4-scout-17b-16e-instruct" private let defaultModelReasoningEffort = "low" private let postProcessingMaxCompletionTokens = 4096 - private var postProcessingTimeoutSeconds: TimeInterval { + static var configuredTimeoutSeconds: TimeInterval { let override = UserDefaults.standard.double(forKey: "post_processing_timeout_seconds") return override > 0 ? override : 20 } + private var postProcessingTimeoutSeconds: TimeInterval { + Self.configuredTimeoutSeconds + } + init( apiKey: String, baseURL: String = "https://api.groq.com/openai/v1", @@ -459,7 +466,15 @@ Model: \(model) request.httpBody = try JSONSerialization.data(withJSONObject: payload, options: []) + let t0 = CFAbsoluteTimeGetCurrent() let (data, response) = try await LLMAPITransport.data(for: request) + os_log( + .info, + log: postProcessingLog, + "LLM request completed in %.0fms (model=%{public}@)", + (CFAbsoluteTimeGetCurrent() - t0) * 1000, + model + ) guard let httpResponse = response as? HTTPURLResponse else { throw PostProcessingError.invalidResponse("No HTTP response") } @@ -590,7 +605,15 @@ Model: \(model) request.httpBody = try JSONSerialization.data(withJSONObject: payload, options: []) + let t0 = CFAbsoluteTimeGetCurrent() let (data, response) = try await LLMAPITransport.data(for: request) + os_log( + .info, + log: postProcessingLog, + "LLM request completed in %.0fms (model=%{public}@)", + (CFAbsoluteTimeGetCurrent() - t0) * 1000, + model + ) guard let httpResponse = response as? HTTPURLResponse else { throw PostProcessingError.invalidResponse("No HTTP response") } diff --git a/Sources/TranscriptionService.swift b/Sources/TranscriptionService.swift index 94c2b43a..72e064a1 100644 --- a/Sources/TranscriptionService.swift +++ b/Sources/TranscriptionService.swift @@ -9,11 +9,15 @@ class TranscriptionService { private let transcriptionModel: String private let language: String? private let transcriptionResponseFormat = "verbose_json" - private var transcriptionTimeoutSeconds: TimeInterval { + static var configuredTimeoutSeconds: TimeInterval { let override = UserDefaults.standard.double(forKey: "transcription_timeout_seconds") return override > 0 ? override : 20 } + private var transcriptionTimeoutSeconds: TimeInterval { + Self.configuredTimeoutSeconds + } + init( apiKey: String, baseURL: String = "https://api.groq.com/openai/v1", @@ -119,7 +123,15 @@ class TranscriptionService { ) do { + let t0 = CFAbsoluteTimeGetCurrent() let (data, response) = try await LLMAPITransport.upload(for: request, from: body) + os_log( + .info, + log: transcriptionLog, + "upload+transcription completed in %.0fms (%lld bytes)", + (CFAbsoluteTimeGetCurrent() - t0) * 1000, + Int64(body.count) + ) return try validateTranscriptionResponse(data: data, response: response, fileURL: fileURL) } catch { let nsError = error as NSError