diff --git a/.github/screenshots/whisper-language-selection.jpg b/.github/screenshots/whisper-language-selection.jpg new file mode 100644 index 00000000..7075062d Binary files /dev/null and b/.github/screenshots/whisper-language-selection.jpg differ diff --git a/Fluid.xcodeproj/project.pbxproj b/Fluid.xcodeproj/project.pbxproj index 2b135c8c..aacc8d63 100644 --- a/Fluid.xcodeproj/project.pbxproj +++ b/Fluid.xcodeproj/project.pbxproj @@ -19,6 +19,7 @@ 272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */; }; A62300000000000000000002 /* AudioBufferConverterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A62300000000000000000001 /* AudioBufferConverterTests.swift */; }; C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */; }; + C0DE63700000000000000002 /* WhisperLanguageSelectionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DE63700000000000000001 /* WhisperLanguageSelectionTests.swift */; }; 7CDB0A2F2F3C4D5600FB7CAD /* dictation_fixture.wav in Resources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */; }; 7CDB0A302F3C4D5600FB7CAD /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */; }; 7CE006BD2E80EBE600DDCCD6 /* AppUpdater in Frameworks */ = {isa = PBXBuildFile; productRef = 7CE006BC2E80EBE600DDCCD6 /* AppUpdater */; }; @@ -54,6 +55,7 @@ 980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TemperatureSupportTests.swift; sourceTree = ""; }; A62300000000000000000001 /* AudioBufferConverterTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioBufferConverterTests.swift; sourceTree = ""; }; C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioEngineRetirementDrainTests.swift; sourceTree = ""; }; + C0DE63700000000000000001 /* WhisperLanguageSelectionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WhisperLanguageSelectionTests.swift; sourceTree = ""; }; 7C078D8F2E3B339200FB7CAC /* FluidVoice Debug.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "FluidVoice Debug.app"; sourceTree = BUILT_PRODUCTS_DIR; }; 7C91B0022F42AA0100C0DEF0 /* HotkeyShortcutTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HotkeyShortcutTests.swift; sourceTree = ""; }; 7CDB0A202F3C4D5600FB7CAD /* FluidDictationIntegrationTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = FluidDictationIntegrationTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -132,6 +134,7 @@ 980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */, A62300000000000000000001 /* AudioBufferConverterTests.swift */, C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */, + C0DE63700000000000000001 /* WhisperLanguageSelectionTests.swift */, ); path = FluidDictationIntegrationTests; sourceTree = ""; @@ -291,6 +294,7 @@ 272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */, A62300000000000000000002 /* AudioBufferConverterTests.swift in Sources */, C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */, + C0DE63700000000000000002 /* WhisperLanguageSelectionTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/Sources/Fluid/Persistence/BackupService.swift b/Sources/Fluid/Persistence/BackupService.swift index 03528d67..7bb8896a 100644 --- a/Sources/Fluid/Persistence/BackupService.swift +++ b/Sources/Fluid/Persistence/BackupService.swift @@ -17,6 +17,7 @@ struct SettingsBackupPayload: Codable, Equatable { let privateAIBackendPreference: SettingsStore.PrivateAIBackendPreference? let privateAIContextTokenLimit: Int? let selectedSpeechModel: SettingsStore.SpeechModel + let selectedWhisperLanguageCode: String? let selectedCohereLanguage: SettingsStore.CohereLanguage let selectedNemotronLanguage: SettingsStore.NemotronLanguage? let selectedAppleSpeechLocaleIdentifier: String? diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift index e786893f..d018e2a3 100644 --- a/Sources/Fluid/Persistence/SettingsStore.swift +++ b/Sources/Fluid/Persistence/SettingsStore.swift @@ -12,6 +12,7 @@ import FluidAudio // swiftlint:disable file_length type_body_length final class SettingsStore: ObservableObject { static let shared = SettingsStore() + private static let automaticWhisperLanguageCode = "auto" static let transcriptionPreviewCharLimitRange: ClosedRange = 50...800 static let transcriptionPreviewCharLimitStep = 50 static let defaultTranscriptionPreviewCharLimit = 150 @@ -3007,6 +3008,7 @@ final class SettingsStore: ObservableObject { privateAIBackendPreference: self.privateAIBackendPreference, privateAIContextTokenLimit: self.privateAIContextTokenLimit, selectedSpeechModel: self.selectedSpeechModel, + selectedWhisperLanguageCode: Self.whisperLanguageBackupValue(for: self.selectedWhisperLanguageCode), selectedCohereLanguage: self.selectedCohereLanguage, selectedNemotronLanguage: self.selectedNemotronLanguage, selectedAppleSpeechLocaleIdentifier: self.selectedAppleSpeechLocaleIdentifier, @@ -3115,6 +3117,9 @@ final class SettingsStore: ObservableObject { self.privateAIContextTokenLimit = privateAIContextTokenLimit } self.selectedSpeechModel = payload.selectedSpeechModel + if let selectedWhisperLanguageCode = payload.selectedWhisperLanguageCode { + self.selectedWhisperLanguageCode = Self.whisperLanguageCode(fromBackupValue: selectedWhisperLanguageCode) + } self.selectedCohereLanguage = payload.selectedCohereLanguage if let selectedNemotronLanguage = payload.selectedNemotronLanguage { self.selectedNemotronLanguage = selectedNemotronLanguage @@ -5000,6 +5005,7 @@ private extension SettingsStore { /// Unified Speech Model (replaces above two) static let selectedSpeechModel = "SelectedSpeechModel" + static let selectedWhisperLanguageCode = "SelectedWhisperLanguageCode" static let selectedCohereLanguage = "SelectedCohereLanguage" static let selectedNemotronLanguage = "SelectedNemotronLanguage" static let selectedAppleSpeechLocaleIdentifier = "SelectedAppleSpeechLocaleIdentifier" @@ -5258,6 +5264,31 @@ extension SettingsStore { } } + /// The language Whisper should transcribe, or `nil` to detect it from each recording. + /// Existing installs inherit the language chosen during onboarding until the user + /// explicitly selects Automatic or another language. + var selectedWhisperLanguageCode: String? { + get { + if let stored = self.defaults.string(forKey: Keys.selectedWhisperLanguageCode) { + guard stored != Self.automaticWhisperLanguageCode else { return nil } + return VoiceEngineLanguageCatalog.whisperLanguage(forCode: stored) == nil ? nil : stored + } + return VoiceEngineLanguageCatalog.whisperLanguageCode(for: self.onboardingSelectedLanguageID) + } + set { + objectWillChange.send() + self.defaults.set(newValue ?? Self.automaticWhisperLanguageCode, forKey: Keys.selectedWhisperLanguageCode) + } + } + + static func whisperLanguageBackupValue(for languageCode: String?) -> String { + languageCode ?? self.automaticWhisperLanguageCode + } + + static func whisperLanguageCode(fromBackupValue value: String) -> String? { + value == self.automaticWhisperLanguageCode ? nil : value + } + var selectedCohereLanguage: CohereLanguage { get { if let rawValue = self.defaults.string(forKey: Keys.selectedCohereLanguage), diff --git a/Sources/Fluid/Persistence/VoiceEngineLanguageCatalog.swift b/Sources/Fluid/Persistence/VoiceEngineLanguageCatalog.swift index 7d380539..7d13f4b7 100644 --- a/Sources/Fluid/Persistence/VoiceEngineLanguageCatalog.swift +++ b/Sources/Fluid/Persistence/VoiceEngineLanguageCatalog.swift @@ -93,6 +93,14 @@ enum VoiceEngineLanguageCatalog { self.allLanguages(availableModels: availableModels).first { $0.id == id } } + static var whisperLanguages: [VoiceEngineLanguage] { + self.languageDefinitions.filter { self.whisperLanguageCode(for: $0.id) != nil } + } + + static func whisperLanguage(forCode languageCode: String) -> VoiceEngineLanguage? { + self.whisperLanguages.first { self.whisperLanguageCode(for: $0.id) == languageCode } + } + static func routes( for language: VoiceEngineLanguage, availableModels: [SettingsStore.SpeechModel] = SettingsStore.SpeechModel.availableModels @@ -117,8 +125,10 @@ enum VoiceEngineLanguageCatalog { settings.selectedSpeechModel = route.model switch route.binding { - case .automatic, .whisper: + case .automatic: break + case let .whisper(languageCode): + settings.selectedWhisperLanguageCode = languageCode case let .appleSpeech(localeIdentifier): settings.selectedAppleSpeechLocaleIdentifier = localeIdentifier case let .cohere(language): @@ -182,7 +192,7 @@ enum VoiceEngineLanguageCatalog { self.nemotronLanguageMap[languageID] } - private static func whisperLanguageCode(for languageID: String) -> String? { + static func whisperLanguageCode(for languageID: String) -> String? { self.whisperLanguageCodeMap[languageID] } @@ -351,7 +361,7 @@ enum VoiceEngineLanguageCatalog { private static let whisperLanguageCodeMap: [String: String] = { var languageMap: [String: String] = [:] for language in Self.languageDefinitions { - let whisperCode = language.id == "he" ? "iw" : language.id + let whisperCode = language.id guard Self.whisperSupportedLanguageCodes.contains(whisperCode) else { continue } languageMap[language.id] = whisperCode } @@ -362,7 +372,7 @@ enum VoiceEngineLanguageCatalog { "af", "am", "ar", "as", "az", "ba", "be", "bg", "bn", "bo", "br", "bs", "ca", "cs", "cy", "da", "de", "el", "en", "es", "et", "eu", "fa", "fi", "fo", "fr", "gl", "gu", "ha", "haw", - "hi", "hr", "ht", "hu", "hy", "id", "is", "it", "iw", "ja", + "he", "hi", "hr", "ht", "hu", "hy", "id", "is", "it", "ja", "jw", "ka", "kk", "km", "kn", "ko", "la", "lb", "ln", "lo", "lt", "lv", "mg", "mi", "mk", "ml", "mn", "mr", "ms", "mt", "my", "ne", "nl", "nn", "no", "oc", "pa", "pl", "ps", "pt", diff --git a/Sources/Fluid/Services/WhisperProvider.swift b/Sources/Fluid/Services/WhisperProvider.swift index c2e47a1a..feecfe6d 100644 --- a/Sources/Fluid/Services/WhisperProvider.swift +++ b/Sources/Fluid/Services/WhisperProvider.swift @@ -25,13 +25,20 @@ final class WhisperProvider: TranscriptionProvider { private let overriddenModelDirectory: URL? private let urlSession: URLSession + private let languageCodeOverride: String? var modelOverride: SettingsStore.SpeechModel? - init(modelDirectory: URL? = nil, urlSession: URLSession = .shared, modelOverride: SettingsStore.SpeechModel? = nil) { + init( + modelDirectory: URL? = nil, + urlSession: URLSession = .shared, + modelOverride: SettingsStore.SpeechModel? = nil, + languageCodeOverride: String? = nil + ) { self.overriddenModelDirectory = modelDirectory self.urlSession = urlSession self.modelOverride = modelOverride + self.languageCodeOverride = languageCodeOverride } deinit { @@ -309,14 +316,16 @@ final class WhisperProvider: TranscriptionProvider { ) } - let transcript = try await session.run( - samples, - options: RunOptions(timestamps: .segment) - ) + let languageCode = self.languageCodeOverride ?? SettingsStore.shared.selectedWhisperLanguageCode + let transcript = try await session.run(samples, options: Self.runOptions(languageCode: languageCode)) let fullText = transcript.text.trimmingCharacters(in: .whitespacesAndNewlines) return ASRTranscriptionResult(text: fullText, confidence: 1.0) } + static func runOptions(languageCode: String?) -> RunOptions { + RunOptions(timestamps: .segment, language: languageCode) + } + func modelsExistOnDisk() -> Bool { return self.isModelFileValid(at: self.modelURL, for: self.selectedModel) } diff --git a/Sources/Fluid/UI/AISettings/VoiceEngineSettingsView.swift b/Sources/Fluid/UI/AISettings/VoiceEngineSettingsView.swift index 91851f44..e370eed3 100644 --- a/Sources/Fluid/UI/AISettings/VoiceEngineSettingsView.swift +++ b/Sources/Fluid/UI/AISettings/VoiceEngineSettingsView.swift @@ -5,6 +5,8 @@ struct VoiceEngineSettingsView: View { @ObservedObject var settings: SettingsStore @Environment(\.colorScheme) var colorScheme @State var isShowingNemotronLanguagePicker = false + @State var isShowingWhisperLanguagePicker = false + @State var whisperLanguageSearchText = "" let theme: AppTheme var voiceEngineTitleText: Color { diff --git a/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift b/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift index 131ffc5d..c55b5d2c 100644 --- a/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift +++ b/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift @@ -548,7 +548,9 @@ extension VoiceEngineSettingsView { @ViewBuilder private func speechModelLanguagePicker(for model: SettingsStore.SpeechModel) -> some View { - if model == .cohereTranscribeSixBit { + if model.isWhisperModel { + self.whisperLanguagePickerButton + } else if model == .cohereTranscribeSixBit { Menu { ForEach(SettingsStore.CohereLanguage.allCases) { language in Button { @@ -572,6 +574,112 @@ extension VoiceEngineSettingsView { } } + private var whisperLanguagePickerButton: some View { + Button { + self.whisperLanguageSearchText = "" + self.isShowingWhisperLanguagePicker.toggle() + } label: { + self.languageChipLabel(self.selectedWhisperLanguageName) + } + .buttonStyle(.plain) + .popover(isPresented: self.$isShowingWhisperLanguagePicker, arrowEdge: .bottom) { + self.whisperLanguagePickerPopover + } + } + + private var selectedWhisperLanguageName: String { + guard let languageCode = self.settings.selectedWhisperLanguageCode, + let language = VoiceEngineLanguageCatalog.whisperLanguage(forCode: languageCode) + else { + return "Automatic" + } + return language.displayName + } + + private var filteredWhisperLanguages: [VoiceEngineLanguage] { + let query = self.normalizedWhisperLanguageSearchText + guard !query.isEmpty else { return VoiceEngineLanguageCatalog.whisperLanguages } + return VoiceEngineLanguageCatalog.whisperLanguages.filter { language in + language.displayName.lowercased().contains(query) || + language.id.lowercased().contains(query) || + language.aliases.contains { $0.lowercased().contains(query) } + } + } + + private var normalizedWhisperLanguageSearchText: String { + self.whisperLanguageSearchText.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + } + + private var whisperLanguagePickerPopover: some View { + VStack(spacing: 0) { + HStack(spacing: 8) { + Image(systemName: "magnifyingglass") + .foregroundStyle(self.voiceEngineTertiaryText) + TextField("Search languages", text: self.$whisperLanguageSearchText) + .textFieldStyle(.plain) + } + .padding(.horizontal, 12) + .frame(height: 36) + + Divider() + + ScrollView(.vertical, showsIndicators: true) { + LazyVStack(alignment: .leading, spacing: 0) { + if self.normalizedWhisperLanguageSearchText.isEmpty || + "automatic".contains(self.normalizedWhisperLanguageSearchText) + { + Button { + self.settings.selectedWhisperLanguageCode = nil + self.isShowingWhisperLanguagePicker = false + } label: { + self.whisperLanguagePickerRow( + title: "Automatic", + isSelected: self.settings.selectedWhisperLanguageCode == nil + ) + } + .buttonStyle(.plain) + + Divider() + .padding(.vertical, 4) + } + + ForEach(self.filteredWhisperLanguages) { language in + let languageCode = VoiceEngineLanguageCatalog.whisperLanguageCode(for: language.id) + Button { + self.settings.selectedWhisperLanguageCode = languageCode + self.isShowingWhisperLanguagePicker = false + } label: { + self.whisperLanguagePickerRow( + title: language.displayName, + isSelected: languageCode == self.settings.selectedWhisperLanguageCode + ) + } + .buttonStyle(.plain) + } + } + .padding(.vertical, 6) + } + } + .frame(width: 280, height: 420) + } + + private func whisperLanguagePickerRow(title: String, isSelected: Bool) -> some View { + HStack(spacing: 8) { + Text(title) + .font(self.theme.typography.bodySmall) + .foregroundStyle(.primary) + Spacer(minLength: 12) + if isSelected { + Image(systemName: "checkmark") + .font(self.theme.typography.bodySmall) + .foregroundStyle(self.theme.palette.accent) + } + } + .contentShape(Rectangle()) + .padding(.horizontal, 12) + .frame(height: 28) + } + private func languageChipLabel(_ title: String) -> some View { HStack(spacing: 5) { Image(systemName: "globe") diff --git a/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift b/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift index ba08bc0f..4a2dc446 100644 --- a/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift +++ b/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift @@ -1223,7 +1223,11 @@ final class DictationE2ETests: XCTestCase { let modelDirectory = Self.modelDirectoryForRun() try FileManager.default.createDirectory(at: modelDirectory, withIntermediateDirectories: true) - let provider = WhisperProvider(modelDirectory: modelDirectory, modelOverride: .whisperTiny) + let provider = WhisperProvider( + modelDirectory: modelDirectory, + modelOverride: .whisperTiny, + languageCodeOverride: "en" + ) // Act try await provider.prepare() diff --git a/Tests/FluidDictationIntegrationTests/WhisperLanguageSelectionTests.swift b/Tests/FluidDictationIntegrationTests/WhisperLanguageSelectionTests.swift new file mode 100644 index 00000000..2ed30922 --- /dev/null +++ b/Tests/FluidDictationIntegrationTests/WhisperLanguageSelectionTests.swift @@ -0,0 +1,43 @@ +@testable import FluidVoice_Debug +import XCTest + +final class WhisperLanguageSelectionTests: XCTestCase { + func testHungarianLanguageCodeIsAvailable() { + XCTAssertEqual(VoiceEngineLanguageCatalog.whisperLanguageCode(for: "hu"), "hu") + XCTAssertEqual(VoiceEngineLanguageCatalog.whisperLanguage(forCode: "hu")?.displayName, "Hungarian") + } + + func testHebrewUsesWhisperLanguageCode() { + XCTAssertEqual(VoiceEngineLanguageCatalog.whisperLanguageCode(for: "he"), "he") + XCTAssertEqual(VoiceEngineLanguageCatalog.whisperLanguage(forCode: "he")?.displayName, "Hebrew") + } + + func testWhisperRunOptionsUseSelectedLanguage() { + XCTAssertEqual(WhisperProvider.runOptions(languageCode: "hu").language, "hu") + } + + func testWhisperRunOptionsAllowAutomaticDetection() { + XCTAssertNil(WhisperProvider.runOptions(languageCode: nil).language) + } + + func testAutomaticWhisperLanguageRoundTripsThroughBackupValue() { + let backupValue = SettingsStore.whisperLanguageBackupValue(for: nil) + + XCTAssertEqual(backupValue, "auto") + XCTAssertNil(SettingsStore.whisperLanguageCode(fromBackupValue: backupValue)) + } + + func testForcedWhisperLanguageRoundTripsThroughBackupValue() { + let backupValue = SettingsStore.whisperLanguageBackupValue(for: "hu") + + XCTAssertEqual(backupValue, "hu") + XCTAssertEqual(SettingsStore.whisperLanguageCode(fromBackupValue: backupValue), "hu") + } + + func testWhisperLanguageCodesAreUnique() { + let languageCodes = VoiceEngineLanguageCatalog.whisperLanguages.compactMap { + VoiceEngineLanguageCatalog.whisperLanguageCode(for: $0.id) + } + XCTAssertEqual(languageCodes.count, Set(languageCodes).count) + } +}