Skip to content

Delay dictation overlay until audio capture is ready #701

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
supfeer wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Delay dictation overlay until audio capture is ready #701

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 0 additions & 13 deletions Sources/Fluid/ContentView.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3045,25 +3045,16 @@ struct ContentView: View {
self.advanceOverlayLifecycle()
self.setActiveRecordingMode(.dictate)
let shouldShowDictationOverlay = !self.isRecordingForCommand && !self.isRecordingForRewrite
let shouldPlayStartSound = !self.isRecordingForCommand
&& !self.isRecordingForRewrite
&& self.asr.micStatus == .authorized

// Ensure normal dictation mode is set (command/rewrite modes set their own)
if shouldShowDictationOverlay {
self.menuBarManager.setOverlayMode(.dictation)
}

Task {
if shouldPlayStartSound, !self.asr.isRunning {
TranscriptionSoundPlayer.shared.playStartSound()
}
await self.asr.start(onCaptureStarted: {
self.captureRecordingContext()
self.prewarmPrivateAIDictationIfNeeded(for: .primary)
if shouldShowDictationOverlay {
self.menuBarManager.showRecordingOverlayImmediately()
}
})
if !self.asr.isRunning {
self.menuBarManager.hideRecordingOverlayImmediately(reason: "asr_start_failed")
Expand Down Expand Up @@ -3680,14 +3671,10 @@ extension ContentView {
Task {
let asrStartStartedAt = ProcessInfo.processInfo.systemUptime
DebugLogger.shared.benchmark("APP_BENCH", message: "asr_start_call", source: "AppBenchmark")
if SettingsStore.shared.enableTranscriptionSounds, !self.asr.isRunning {
TranscriptionSoundPlayer.shared.playStartSound()
}
await self.asr.start(onCaptureStarted: {
self.captureRecordingContext()
self.appBench("overlay_mode_request mode=Dictation")
self.menuBarManager.setOverlayMode(.dictation)
self.menuBarManager.showRecordingOverlayImmediately()
self.appBench("overlay_mode_requested mode=Dictation")
self.prewarmPrivateAIDictationIfNeeded(for: slot)
})
Expand Down
33 changes: 27 additions & 6 deletions Sources/Fluid/Services/ASRService.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ final class ASRService: ObservableObject {
}

@Published var isRunning: Bool = false
@Published private(set) var isAudioCaptureReady: Bool = false
@Published var finalText: String = ""
@Published var partialTranscription: String = ""
@Published var wordBoostStatusText: String = "Word boost: off"
Expand All @@ -105,6 +106,7 @@ final class ASRService: ObservableObject {

private(set) var isStarting: Bool = false // Guard against re-entrant start() calls
private var audioCaptureStartWaiters: [CheckedContinuation<Void, Never>] = []
private var pendingCaptureStartedHandler: (@MainActor () -> Void)?
var isRunningOrStarting: Bool { self.isRunning || self.isStarting }
private var hasCompletedFirstTranscription: Bool = false // Track if model has warmed up with first transcription
private var lastBoostHitTerm: String?
Expand Down Expand Up @@ -1068,14 +1070,15 @@ final class ASRService: ObservableObject {
/// from CoreAudio's realtime callback thread.
private lazy var audioCapturePipeline: AudioCapturePipeline = .init(
audioBuffer: self.audioBuffer,
onFirstAudio: { sessionID, sampleCount, frameLength, sampleRate, acquisitionMs, elapsedMs in
DispatchQueue.main.async {
onFirstAudio: { [weak self] sessionID, sampleCount, frameLength, sampleRate, acquisitionMs, elapsedMs in
Task { @MainActor [weak self] in
let bufferMs = Int((Double(frameLength) / sampleRate * 1000).rounded())
DebugLogger.shared.benchmark(
"ASR_BENCH",
message: "session=\(sessionID) first_audio sampleCount=\(sampleCount) frameLength=\(frameLength) sampleRate=\(Int(sampleRate.rounded())) bufferMs=\(bufferMs) acquisitionMs=\(acquisitionMs) elapsedMs=\(elapsedMs)",
source: "ASRBenchmark"
)
self?.markAudioCaptureReady(sessionID: sessionID)
}
},
onDurationMismatch: { [weak self] sessionID, capturedMilliseconds, elapsedMilliseconds in
Expand Down Expand Up @@ -1334,6 +1337,9 @@ final class ASRService: ObservableObject {
return
}
self.isStarting = true
self.isAudioCaptureReady = false
self.pendingCaptureStartedHandler = onCaptureStarted
self.isDictionaryTrainingCaptureActive = forDictionaryTraining
defer { self.finishAudioCaptureStart() }

// Reset media pause state for this session
Expand Down Expand Up @@ -1369,18 +1375,14 @@ final class ASRService: ObservableObject {
self.benchmarkLog("recording_start model=\(dims.model) provider=\(dims.provider) supportsStreaming=\(SettingsStore.shared.selectedSpeechModel.supportsStreaming)")
DebugLogger.shared.debug("✅ Buffers cleared", source: "ASRService")

self.isDictionaryTrainingCaptureActive = false

do {
if SettingsStore.shared.microphoneSelectionMode == .manual {
AppServices.shared.microphonePreferenceCoordinator.enforcePreferredInput(reason: "recording start")
}

try await self.startPreferredAudioCapture()
self.isDictionaryTrainingCaptureActive = forDictionaryTraining
self.isRunning = true
DebugLogger.shared.info("✅ Audio capture running", source: "ASRService")
onCaptureStarted?()

// Pause only after capture is live so media control cannot delay the
// first PCM packet. A quick stop while this await is in flight is
Expand Down Expand Up @@ -1421,6 +1423,8 @@ final class ASRService: ObservableObject {
DebugLogger.shared.info("✅ START() completed successfully", source: "ASRService")
} catch {
self.isDictionaryTrainingCaptureActive = false
self.isAudioCaptureReady = false
self.pendingCaptureStartedHandler = nil
self.audioCapturePipeline.setRecordingEnabled(false)
self.isRunning = false
self.stopActiveAudioCapture()
Expand Down Expand Up @@ -1460,6 +1464,19 @@ final class ASRService: ObservableObject {
}
}

private func markAudioCaptureReady(sessionID: Int) {
guard sessionID == self.benchmarkSessionID,
self.isRunning || self.isStarting,
self.isAudioCaptureReady == false
else { return }

self.isAudioCaptureReady = true
let handler = self.pendingCaptureStartedHandler
self.pendingCaptureStartedHandler = nil
DebugLogger.shared.info("✅ Audio capture accepted its first buffer", source: "ASRService")
handler?()
}

func waitForPendingStart() async {
guard self.isStarting else { return }
await withCheckedContinuation { continuation in
Expand Down Expand Up @@ -1549,6 +1566,8 @@ final class ASRService: ObservableObject {

// Set isRunning to false before teardown so in-flight ASR chunks stop safely.
DebugLogger.shared.debug("🚫 Setting isRunning = false...", source: "ASRService")
self.isAudioCaptureReady = false
self.pendingCaptureStartedHandler = nil
self.isRunning = false
DebugLogger.shared.debug("✅ isRunning disabled", source: "ASRService")

Expand Down Expand Up @@ -1882,6 +1901,8 @@ final class ASRService: ObservableObject {
DebugLogger.shared.info("🛑 Stopping recording - releasing audio devices", source: "ASRService")

// CRITICAL: Set isRunning to false FIRST to signal any in-flight chunks to abort early
self.isAudioCaptureReady = false
self.pendingCaptureStartedHandler = nil
self.isRunning = false
self.audioCapturePipeline.setRecordingEnabled(false)

Expand Down
82 changes: 30 additions & 52 deletions Sources/Fluid/Services/MenuBarManager.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate {
private var pendingShowOperation: DispatchWorkItem?
private var pendingHideOperation: DispatchWorkItem?
private var pendingProcessingShowOperation: DispatchWorkItem?
/// Keep capture live briefly before presenting the recording UI.
private let recordingVisualDelay: DispatchTimeInterval = .milliseconds(500)
/// Show immediately so users see the processing state right away.
private let processingVisualDelay: DispatchTimeInterval = .milliseconds(0)
/// Legacy debounce used by generic processing callers. Successful dictation
Expand All @@ -72,16 +74,22 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate {
func configure(asrService: ASRService) {
self.asrService = asrService

// Subscribe to recording state changes
// Keep menu bar state tied to the capture lifecycle.
asrService.$isRunning
.receive(on: DispatchQueue.main)
.sink { [weak self] isRunning in
self?.isRecording = isRunning
self?.updateMenuBarIcon()
self?.updateMenu()
}
.store(in: &self.cancellables)

// Handle overlay lifecycle (independent of window state)
self?.handleOverlayState(isRunning: isRunning, asrService: asrService)
// The recording overlay represents usable audio capture, not merely a
// backend that has returned successfully from start().
asrService.$isAudioCaptureReady
.receive(on: DispatchQueue.main)
.sink { [weak self] isCaptureReady in
self?.handleOverlayState(isCaptureReady: isCaptureReady, asrService: asrService)
}
.store(in: &self.cancellables)

Expand All @@ -97,8 +105,8 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate {
.store(in: &self.cancellables)
}

private func handleOverlayState(isRunning: Bool, asrService: ASRService) {
self.overlayBench("handle_state isRunning=\(isRunning) overlayVisible=\(self.overlayVisible) processing=\(self.isProcessingActive) mode=\(self.currentOverlayMode.rawValue)")
private func handleOverlayState(isCaptureReady: Bool, asrService: ASRService) {
self.overlayBench("handle_state isCaptureReady=\(isCaptureReady) overlayVisible=\(self.overlayVisible) processing=\(self.isProcessingActive) mode=\(self.currentOverlayMode.rawValue)")

// Dictionary training owns its recording controls, so showing the
// regular dictation notch here would create two competing overlays.
Expand All @@ -115,24 +123,28 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate {
// Don't hide the overlay while AI processing is active.
// Without this, the notch can disappear during the short "Refining..." phase because
// `isRunning` becomes false before post-processing completes.
if !isRunning, self.isProcessingActive {
if !isCaptureReady, self.isProcessingActive {
self.overlayBench("handle_state_return reason=processing_active")
return
}

// Prevent rapid state changes that could cause cycles
guard self.overlayVisible != isRunning else {
guard self.overlayVisible != isCaptureReady else {
self.overlayBench("handle_state_return reason=visibility_unchanged")
return
}

if isRunning {
if isCaptureReady {
if self.currentOverlayMode == .dictation {
AutomaticDictionaryCorrectionTracker.shared.cancel()
}

// Cancel any pending hide operation
self.pendingHideOperation?.cancel()
self.pendingHideOperation = nil

self.overlayVisible = true
self.overlayBench("show_request mode=\(self.currentOverlayMode.rawValue)")
self.overlayBench("show_request mode=\(self.currentOverlayMode.rawValue) delayMs=500")

// If expanded command output is showing, check if we should keep it or close it
if NotchOverlayManager.shared.isCommandOutputExpanded {
Expand Down Expand Up @@ -172,6 +184,11 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate {

// Show notch overlay
self.overlayBench("show_workitem_execute mode=\(self.currentOverlayMode.rawValue)")
if self.currentOverlayMode == .dictation,
SettingsStore.shared.enableTranscriptionSounds
{
TranscriptionSoundPlayer.shared.playStartSound()
}
NotchOverlayManager.shared.show(
audioLevelPublisher: asrService.audioLevelPublisher,
mode: self.currentOverlayMode
Expand All @@ -181,7 +198,10 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate {
self.pendingShowOperation = nil
}
self.pendingShowOperation = showItem
DispatchQueue.main.async(execute: showItem)
DispatchQueue.main.asyncAfter(
deadline: .now() + self.recordingVisualDelay,
execute: showItem
)
} else {
// Cancel any pending show operation
self.pendingShowOperation?.cancel()
Expand Down Expand Up @@ -222,48 +242,6 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate {
}
}

func showRecordingOverlayImmediately() {
AutomaticDictionaryCorrectionTracker.shared.cancel()

guard let asrService else {
self.overlayBench("instant_show_return reason=no_asr_service")
return
}

self.pendingHideOperation?.cancel()
self.pendingHideOperation = nil
self.pendingShowOperation?.cancel()
self.pendingShowOperation = nil

guard !self.overlayVisible else {
self.overlayBench("instant_show_return reason=already_visible")
return
}

self.overlayVisible = true
self.overlayBench("instant_show_request mode=\(self.currentOverlayMode.rawValue)")

if NotchOverlayManager.shared.isCommandOutputExpanded {
if self.currentOverlayMode == .command, NotchOverlayManager.shared.supportsCommandNotchUI {
NotchContentState.shared.setRecordingInExpandedMode(true)
self.expandedModeAudioSubscription = asrService.audioLevelPublisher
.receive(on: DispatchQueue.main)
.sink { level in
NotchContentState.shared.updateExpandedModeAudioLevel(level)
}
return
}
NotchOverlayManager.shared.hideExpandedCommandOutput()
}

self.overlayBench("show_workitem_execute mode=\(self.currentOverlayMode.rawValue)")
NotchOverlayManager.shared.show(
audioLevelPublisher: asrService.audioLevelPublisher,
mode: self.currentOverlayMode
)
self.overlayBench("show_workitem_return mode=\(self.currentOverlayMode.rawValue)")
}

func hideRecordingOverlayImmediately(reason: String) {
self.pendingShowOperation?.cancel()
self.pendingShowOperation = nil
Expand Down
Loading