diff --git a/Makefile b/Makefile index 00fda5c5..6f4bb12a 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ APP_EXECUTABLE_TARGET := $(subst $(space),\ ,$(APP_EXECUTABLE)) SOURCES = $(shell find Sources -name '*.swift' -type f | LC_ALL=C sort) TEST_RUNNER = $(BUILD_DIR)/FreeFlowTests +TRANSCRIPTION_TEST_RUNNER = $(BUILD_DIR)/FreeFlowTranscriptionTests RESOURCES = $(CONTENTS)/Resources ARCH ?= $(shell uname -m) @@ -69,8 +70,18 @@ endif @codesign --force --options runtime --sign "$(CODESIGN_IDENTITY)" --entitlements FreeFlow.entitlements "$(APP_BUNDLE)" @echo "Built $(APP_BUNDLE)" -test: $(TEST_RUNNER) +test: $(TEST_RUNNER) $(TRANSCRIPTION_TEST_RUNNER) @$(TEST_RUNNER) + @$(TRANSCRIPTION_TEST_RUNNER) + +$(TRANSCRIPTION_TEST_RUNNER): Sources/TranscriptionService.swift Sources/LLMAPITransport.swift Tests/TranscriptionServiceTests.swift + @mkdir -p "$(BUILD_DIR)" + swiftc \ + -parse-as-library \ + -o "$(TRANSCRIPTION_TEST_RUNNER)" \ + -sdk $(shell xcrun --show-sdk-path) \ + -target $(ARCH)-apple-macosx13.0 \ + Sources/TranscriptionService.swift Sources/LLMAPITransport.swift Tests/TranscriptionServiceTests.swift $(TEST_RUNNER): Sources/AppContextService.swift Sources/LLMAPITransport.swift Sources/ModelConfiguration.swift Tests/AppContextServiceTests.swift @mkdir -p "$(BUILD_DIR)" diff --git a/Sources/TranscriptionService.swift b/Sources/TranscriptionService.swift index 94c2b43a..9110beba 100644 --- a/Sources/TranscriptionService.swift +++ b/Sources/TranscriptionService.swift @@ -307,6 +307,33 @@ class TranscriptionService { "you" ] + // On silence whisper also emits a subtitle-credit line in whatever language it + // guessed ("Undertekster av Ai-Media", "Untertitel von ...", "字幕by..."). The + // wording varies endlessly, so match the credit word instead of the full phrase. + // Compared against diacritic-folded text, and only for short outputs, so a real + // sentence that happens to mention subtitles is not swallowed. + private let hallucinationMarkers = [ + "amara.org", + "subtitles by", "subtitle by", "subs by", "captions by", "captioning by", + "undertekster", "undertitel", "tekstet av", // no/da + "undertext", "textning", // sv + "untertitel", // de + "ondertitel", // nl + "sous-titr", // fr + "subtitulos", "subtitulado", // es + "sottotitoli", // it + "legendas", // pt + "napisy", // pl + "tekstitys", // fi + "altyaz", // tr + "субтитр", // ru + "字幕", // zh/ja + "자막", // ko + "ترجمة" // ar + ] + + private let hallucinationMarkerMaxLength = 60 + private let hallucinationNoSpeechThreshold = 0.1 private func parseTranscript(from data: Data) throws -> String { @@ -330,11 +357,14 @@ class TranscriptionService { return text } - private func isHallucination(text: String, json: [String: Any]) -> Bool { + func isHallucination(text: String, json: [String: Any]) -> Bool { let normalized = text + .folding(options: .diacriticInsensitive, locale: nil) .lowercased() .trimmingCharacters(in: CharacterSet.punctuationCharacters.union(.whitespacesAndNewlines)) - guard hallucinationPhrases.contains(normalized) else { + let matchesMarker = normalized.count <= hallucinationMarkerMaxLength + && hallucinationMarkers.contains { normalized.contains($0) } + guard hallucinationPhrases.contains(normalized) || matchesMarker else { return false } diff --git a/Tests/TranscriptionServiceTests.swift b/Tests/TranscriptionServiceTests.swift new file mode 100644 index 00000000..c0bff6c4 --- /dev/null +++ b/Tests/TranscriptionServiceTests.swift @@ -0,0 +1,38 @@ +import Foundation + +@main +struct TranscriptionServiceTests { + static func main() throws { + let service = try TranscriptionService(apiKey: "test-key") + + // Silent audio: whisper's subtitle-credit hallucinations get dropped in any language. + for text in [ + "Undertekster av Ai-Media", + "Untertitel von ZDF, 2021", + "Sous-titres réalisés para la communauté d'Amara.org", + "Subtítulos realizados por la comunidad de Amara.org", + "字幕by索兰娅", + "Thank you.", + ] { + assert(service.isHallucination(text: text, json: response(text, noSpeechProb: 0.9)), + "expected '\(text)' to be filtered on silent audio") + } + + // Real speech: same phrases survive when whisper is confident there is speech. + for text in ["Undertekster av Ai-Media", "Thank you."] { + assert(!service.isHallucination(text: text, json: response(text, noSpeechProb: 0.01)), + "expected '\(text)' to survive when no_speech_prob is low") + } + + // A real sentence that merely mentions subtitles is not a credit line. + let sentence = "Can you add subtitles by tomorrow so the team can review the launch video?" + assert(!service.isHallucination(text: sentence, json: response(sentence, noSpeechProb: 0.9)), + "expected a long sentence mentioning subtitles to survive") + + print("TranscriptionServiceTests passed") + } + + private static func response(_ text: String, noSpeechProb: Double) -> [String: Any] { + ["text": text, "segments": [["no_speech_prob": noSpeechProb]]] + } +}