diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs index d51fb88..ee12be1 100644 --- a/src-tauri/src/main.rs +++ b/src-tauri/src/main.rs @@ -739,8 +739,16 @@ fn transcribe_upload_start( /// `transcribe_start` shape minus the device picker. Audio chunks /// are pushed via `transcribe_feed_remote_audio`; end-of-stream is /// signaled by the same command's `is_final` flag. -#[tauri::command] -fn transcribe_start_remote_session( +/// +/// Host-side model resolution: the requesting peer sends the model +/// *it* would use locally, but this host may not have that exact tag +/// installed. Mirror remote inference ("the peer's resolver picks the +/// actual tag") — honour the requested model when it's present here, +/// otherwise fall back to this machine's resolved transcribe pair. +/// Diarization is likewise the host's call: drop it if the requested +/// composite isn't installed rather than failing the whole session. +#[tauri::command] +async fn transcribe_start_remote_session( session_id: String, runtime: String, model: String, @@ -748,6 +756,21 @@ fn transcribe_start_remote_session( sample_rate: u32, window: tauri::WebviewWindow, ) -> Result<(), String> { + let (runtime, model) = if models::find(&model, models::ModelKind::Asr) + .map(models::is_installed) + .unwrap_or(false) + { + (runtime, model) + } else { + match resolver::resolve_pair("transcribe").await { + Ok((resolved_model, resolved_runtime)) => (resolved_runtime, resolved_model), + Err(e) => return Err(format!("no transcribe model available on host: {e:#}")), + } + }; + let diarize_model = match diarize_model { + Some(d) if models::composite_installed(&d, models::ModelKind::Diarize) => Some(d), + _ => None, + }; transcribe::start_remote_session( session_id, runtime, @@ -759,6 +782,47 @@ fn transcribe_start_remote_session( .map_err(|e| e.to_string()) } +/// Capture the local mic for a *remote* transcribe session: 16 kHz mono +/// i16 PCM is emitted on `myownllm://transcribe-capture/` for +/// the frontend to forward to the host peer, which runs the ASR (see +/// `transcribe_start_remote_session`). Stop with `transcribe_capture_stop`. +#[tauri::command] +fn transcribe_capture_start( + stream_id: String, + device: Option, + window: tauri::WebviewWindow, +) -> Result<(), String> { + transcribe::start_capture(stream_id, device, window).map_err(|e| e.to_string()) +} + +/// Decode a local audio/video file into 16 kHz mono PCM for a remote +/// transcribe session, emitted on the same channel as +/// `transcribe_capture_start` — the file-upload path's sender half. +#[tauri::command] +fn transcribe_decode_file_start( + stream_id: String, + file_path: String, + window: tauri::WebviewWindow, +) -> Result<(), String> { + transcribe::start_decode_file(stream_id, std::path::PathBuf::from(file_path), window) + .map_err(|e| e.to_string()) +} + +/// Stop a sender capture / decode started by `transcribe_capture_start` +/// or `transcribe_decode_file_start`. Flushes the trailing audio, then +/// the worker emits its terminal `is_final` frame. +#[tauri::command] +fn transcribe_capture_stop(stream_id: String) { + transcribe::stop_capture(&stream_id); +} + +/// Pause / resume a sender capture. Mic frames are dropped while paused; +/// the remote ASR session stays open. +#[tauri::command] +fn transcribe_capture_set_paused(stream_id: String, paused: bool) { + transcribe::set_capture_paused(&stream_id, paused); +} + /// Push a single PCM chunk into a running remote session. `bytes_b64` /// is base64-encoded i16 little-endian PCM at the session's sample /// rate (16 kHz mono on the LLM's wire format). Decoded + converted @@ -1500,6 +1564,10 @@ fn main() { transcribe_upload_start, transcribe_start_remote_session, transcribe_feed_remote_audio, + transcribe_capture_start, + transcribe_decode_file_start, + transcribe_capture_stop, + transcribe_capture_set_paused, asr_models_list, asr_model_pull, asr_model_pull_cancel, diff --git a/src-tauri/src/transcribe.rs b/src-tauri/src/transcribe.rs index 7d666a1..bd0d223 100644 --- a/src-tauri/src/transcribe.rs +++ b/src-tauri/src/transcribe.rs @@ -1276,6 +1276,427 @@ pub fn end_remote_audio(stream_id: &str) { } } +// ---------------------------------------------------------------------- +// Sender-side capture for *remote* transcription. +// +// The inverse of `start_remote_session`: when the user pins a peer as +// the transcribe host, the audio is captured *here* (mic or decoded +// file) and shipped to that peer, which runs the ASR and streams +// segments back (`mesh-transcribe.ts`). These functions produce the +// receiver's wire format — 16 kHz mono i16-LE PCM — and emit it to the +// frontend as `myownllm://transcribe-capture/` frames; the +// TS layer forwards each one over the `transcribe_audio/` channel. +// No ASR runs locally: this side is just a microphone (or file +// decoder) with a resampler. +// ---------------------------------------------------------------------- + +/// One PCM frame emitted to the frontend by the sender-side capture / +/// decode loops. `bytes_b64` is base64 i16-LE PCM at [`TARGET_SR`] +/// (16 kHz mono) — empty on the terminal frame. `is_final` marks the +/// last frame of the stream; `error` is set instead of audio when the +/// mic / decoder failed. +#[derive(Debug, Clone, Serialize)] +struct CapturePcm { + index: u64, + bytes_b64: String, + is_final: bool, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +/// Per-stream control for an in-flight sender capture (mic or file). +/// Separate from `sessions()` — these never run ASR, so they don't need +/// the full `Session` shape; just cancel + pause flags. +struct CaptureCtl { + cancel: Arc, + paused: Arc, +} + +fn capture_sessions() -> &'static DashMap { + static MAP: OnceLock> = OnceLock::new(); + MAP.get_or_init(DashMap::new) +} + +/// ~250 ms of audio per emitted frame at 16 kHz — small enough that the +/// peer's ASR starts almost immediately, large enough that the +/// per-frame base64 + IPC overhead stays negligible (~8 KB/frame). +const CAPTURE_FRAME_SAMPLES: usize = TARGET_SR as usize / 4; + +/// f32 mono samples → base64 of i16-LE PCM, the receiver's wire shape. +fn pcm_f32_to_i16_b64(samples: &[f32]) -> String { + let mut bytes = Vec::with_capacity(samples.len() * 2); + for &s in samples { + let v = (s.clamp(-1.0, 1.0) * i16::MAX as f32) as i16; + bytes.extend_from_slice(&v.to_le_bytes()); + } + data_encoding::BASE64.encode(&bytes) +} + +/// Drain `pending` into [`CAPTURE_FRAME_SAMPLES`]-sized PCM frames and +/// emit each to the frontend. In the steady state (`flush == false`) +/// only whole frames go out, leaving a partial behind to coalesce with +/// the next batch; on `flush` everything is sent so the last word isn't +/// dropped. +fn emit_pcm_frames( + pending: &mut Vec, + index: &mut u64, + window: &WebviewWindow, + event: &str, + flush: bool, +) { + use tauri::Emitter; + let min = if flush { 1 } else { CAPTURE_FRAME_SAMPLES }; + while pending.len() >= min { + let take = pending.len().min(CAPTURE_FRAME_SAMPLES); + let chunk: Vec = pending.drain(..take).collect(); + let _ = window.emit( + event, + CapturePcm { + index: *index, + bytes_b64: pcm_f32_to_i16_b64(&chunk), + is_final: false, + error: None, + }, + ); + *index += 1; + } +} + +fn emit_capture_final(window: &WebviewWindow, event: &str, error: Option) { + use tauri::Emitter; + let _ = window.emit( + event, + CapturePcm { + index: u64::MAX, + bytes_b64: String::new(), + is_final: true, + error, + }, + ); +} + +/// Start capturing the local mic and forwarding 16 kHz mono PCM to the +/// frontend for a remote transcribe session. Frames land on +/// `myownllm://transcribe-capture/`. Stop with +/// [`stop_capture`]; pause/resume with [`set_capture_paused`]. +pub fn start_capture( + stream_id: String, + device_name: Option, + window: WebviewWindow, +) -> Result<()> { + if capture_sessions().contains_key(&stream_id) { + return Err(anyhow!("capture {stream_id} is already running")); + } + let cancel = Arc::new(AtomicBool::new(false)); + let paused = Arc::new(AtomicBool::new(false)); + capture_sessions().insert( + stream_id.clone(), + CaptureCtl { + cancel: cancel.clone(), + paused: paused.clone(), + }, + ); + thread::spawn(move || { + let event = format!("myownllm://transcribe-capture/{stream_id}"); + let res = run_capture(&event, device_name.as_deref(), &cancel, &paused, &window); + capture_sessions().remove(&stream_id); + emit_capture_final(&window, &event, res.err().map(|e| format!("{e:#}"))); + }); + Ok(()) +} + +/// Start decoding a local audio/video file into 16 kHz mono PCM for a +/// remote transcribe session — the file-upload analogue of +/// [`start_capture`], emitting on the same channel and stopped the same +/// way. +pub fn start_decode_file( + stream_id: String, + file_path: PathBuf, + window: WebviewWindow, +) -> Result<()> { + if capture_sessions().contains_key(&stream_id) { + return Err(anyhow!("capture {stream_id} is already running")); + } + let cancel = Arc::new(AtomicBool::new(false)); + let paused = Arc::new(AtomicBool::new(false)); + capture_sessions().insert( + stream_id.clone(), + CaptureCtl { + cancel: cancel.clone(), + paused: paused.clone(), + }, + ); + thread::spawn(move || { + let event = format!("myownllm://transcribe-capture/{stream_id}"); + let res = run_decode_file(&event, &file_path, &cancel, &paused, &window); + capture_sessions().remove(&stream_id); + emit_capture_final(&window, &event, res.err().map(|e| format!("{e:#}"))); + }); + Ok(()) +} + +/// Stop a sender capture / decode (mic or file). The loop flushes its +/// pending samples, then the worker emits the terminal frame. Idempotent. +pub fn stop_capture(stream_id: &str) { + if let Some(c) = capture_sessions().get(stream_id) { + c.cancel.store(true, Ordering::SeqCst); + } +} + +/// Pause / resume a sender capture. While paused, mic frames are dropped +/// (the remote ASR session stays open, just starved of audio) and a file +/// decode holds at the current packet. No-op for an unknown id. +pub fn set_capture_paused(stream_id: &str, paused: bool) { + if let Some(c) = capture_sessions().get(stream_id) { + c.paused.store(paused, Ordering::SeqCst); + } +} + +/// Mic-capture forward loop. cpal pushes native-rate mono f32 over a +/// channel; we resample to 16 kHz, slice into [`CAPTURE_FRAME_SAMPLES`] +/// frames, and emit each as base64 i16 PCM. Returns when cancelled +/// (Stop) or on an unrecoverable audio error. The terminal frame is +/// emitted by the caller. +fn run_capture( + event: &str, + device_name: Option<&str>, + cancel: &Arc, + paused: &Arc, + window: &WebviewWindow, +) -> Result<()> { + let host = cpal::default_host(); + let device = match device_name { + Some(name) if !name.is_empty() => host + .input_devices()? + .find(|d| d.name().map(|n| n == name).unwrap_or(false)) + .ok_or_else(|| anyhow!("input device '{name}' not found"))?, + _ => host + .default_input_device() + .ok_or_else(|| anyhow!("no default input device"))?, + }; + let (format, stream_cfg) = probe_input_config(&device)?; + let sr = stream_cfg.sample_rate.0; + let channels = stream_cfg.channels as usize; + + let (tx, rx) = bounded::>(256); + let stream_err: Arc>> = Arc::new(Mutex::new(None)); + // The realtime audio callback does the bare minimum — downmix + + // one non-blocking send — and bails while paused/cancelled. + let stream = match format { + cpal::SampleFormat::F32 => { + let tx = tx.clone(); + let cancel = cancel.clone(); + let paused = paused.clone(); + device.build_input_stream( + &stream_cfg, + move |data: &[f32], _| { + if cancel.load(Ordering::Relaxed) || paused.load(Ordering::Relaxed) { + return; + } + let _ = tx.try_send(downmix_f32(data, channels)); + }, + stream_err_fn(stream_err.clone()), + None, + )? + } + cpal::SampleFormat::I16 => { + let tx = tx.clone(); + let cancel = cancel.clone(); + let paused = paused.clone(); + device.build_input_stream( + &stream_cfg, + move |data: &[i16], _| { + if cancel.load(Ordering::Relaxed) || paused.load(Ordering::Relaxed) { + return; + } + let f: Vec = data.iter().map(|&s| s as f32 / i16::MAX as f32).collect(); + let _ = tx.try_send(downmix_f32(&f, channels)); + }, + stream_err_fn(stream_err.clone()), + None, + )? + } + cpal::SampleFormat::U16 => { + let tx = tx.clone(); + let cancel = cancel.clone(); + let paused = paused.clone(); + device.build_input_stream( + &stream_cfg, + move |data: &[u16], _| { + if cancel.load(Ordering::Relaxed) || paused.load(Ordering::Relaxed) { + return; + } + let f: Vec = data + .iter() + .map(|&s| (s as f32 - 32768.0) / 32768.0) + .collect(); + let _ = tx.try_send(downmix_f32(&f, channels)); + }, + stream_err_fn(stream_err.clone()), + None, + )? + } + other => return Err(anyhow!("unsupported sample format {other:?}")), + }; + stream.play()?; + + let mut pending: Vec = Vec::with_capacity(CAPTURE_FRAME_SAMPLES * 2); + let mut index: u64 = 0; + loop { + if cancel.load(Ordering::SeqCst) { + break; + } + match rx.recv_timeout(Duration::from_millis(100)) { + Ok(mono_native) => { + let mut mono16 = resample_linear(&mono_native, sr, TARGET_SR); + pending.append(&mut mono16); + emit_pcm_frames(&mut pending, &mut index, window, event, false); + } + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => break, + } + if let Some(err) = stream_err.lock().ok().and_then(|mut s| s.take()) { + return Err(anyhow!("microphone error: {err}")); + } + } + emit_pcm_frames(&mut pending, &mut index, window, event, true); + drop(stream); + Ok(()) +} + +/// File-decode forward loop. Mirrors `run_upload`'s decoder stage but +/// ships PCM to the peer instead of running ASR locally: decode → +/// downmix → resample to 16 kHz → emit base64 i16 frames. The terminal +/// frame is emitted by the caller. +fn run_decode_file( + event: &str, + file_path: &Path, + cancel: &Arc, + paused: &Arc, + window: &WebviewWindow, +) -> Result<()> { + use std::fs::File; + use symphonia::core::audio::SampleBuffer; + use symphonia::core::codecs::{DecoderOptions, CODEC_TYPE_NULL}; + use symphonia::core::errors::Error as SymError; + use symphonia::core::formats::FormatOptions; + use symphonia::core::io::MediaSourceStream; + use symphonia::core::meta::MetadataOptions; + use symphonia::core::probe::Hint; + + let file = File::open(file_path).map_err(|e| anyhow!("open audio file: {e}"))?; + let mss = MediaSourceStream::new(Box::new(file), Default::default()); + let mut hint = Hint::new(); + if let Some(ext) = file_path.extension().and_then(|e| e.to_str()) { + hint.with_extension(ext); + } + let probed = symphonia::default::get_probe() + .format( + &hint, + mss, + &FormatOptions::default(), + &MetadataOptions::default(), + ) + .map_err(|e| anyhow!("probe audio: {e}"))?; + let mut format = probed.format; + let track = format + .tracks() + .iter() + .find(|t| t.codec_params.codec != CODEC_TYPE_NULL) + .ok_or_else(|| { + anyhow!( + "no audio track in {} — pick an audio file, or a video that has audio.", + file_path.display() + ) + })?; + let track_id = track.id; + let codec_params = track.codec_params.clone(); + let src_rate = codec_params + .sample_rate + .ok_or_else(|| anyhow!("audio file has no declared sample rate"))?; + let src_channels = codec_params.channels.map(|c| c.count()).unwrap_or(1); + let mut decoder = symphonia::default::get_codecs() + .make(&codec_params, &DecoderOptions::default()) + .map_err(|e| anyhow!("make decoder: {e}"))?; + + // Resample in ~0.5 s source batches: bounds the linear resampler's + // per-call boundary error without holding the whole file in RAM. + let batch_at_src_rate = (src_rate as usize / 2).max(1); + let mut buf: Vec = Vec::with_capacity(batch_at_src_rate * 2); + let mut pending: Vec = Vec::with_capacity(CAPTURE_FRAME_SAMPLES * 2); + let mut sb: Option> = None; + let mut index: u64 = 0; + + loop { + if cancel.load(Ordering::SeqCst) { + break; + } + while paused.load(Ordering::SeqCst) { + if cancel.load(Ordering::SeqCst) { + break; + } + thread::sleep(Duration::from_millis(100)); + } + let packet = match format.next_packet() { + Ok(p) => p, + Err(SymError::IoError(ref e)) if e.kind() == std::io::ErrorKind::UnexpectedEof => break, + Err(e) => return Err(anyhow!("symphonia read packet: {e}")), + }; + if packet.track_id() != track_id { + continue; + } + let decoded = match decoder.decode(&packet) { + Ok(d) => d, + Err(SymError::IoError(_)) => continue, + Err(SymError::DecodeError(_)) => continue, + Err(e) => return Err(anyhow!("symphonia decode: {e}")), + }; + let frames = decoded.frames(); + let spec = *decoded.spec(); + let sb_ref = match sb.as_mut() { + Some(b) => { + if b.capacity() < decoded.capacity() { + sb = Some(SampleBuffer::new(decoded.capacity() as u64, spec)); + sb.as_mut().unwrap() + } else { + b + } + } + None => { + sb = Some(SampleBuffer::new(decoded.capacity() as u64, spec)); + sb.as_mut().unwrap() + } + }; + sb_ref.copy_interleaved_ref(decoded); + let samples = sb_ref.samples(); + if src_channels == 1 { + buf.extend_from_slice(samples); + } else { + for f in 0..frames { + let base = f * src_channels; + let mut sum = 0.0f32; + for c in 0..src_channels { + sum += samples[base + c]; + } + buf.push(sum / src_channels as f32); + } + } + while buf.len() >= batch_at_src_rate { + let chunk: Vec = buf.drain(..batch_at_src_rate).collect(); + let mut r = resample_linear(&chunk, src_rate, TARGET_SR); + pending.append(&mut r); + emit_pcm_frames(&mut pending, &mut index, window, event, false); + } + } + // Flush the decoder remainder + the trailing partial frame. + if !buf.is_empty() { + let mut r = resample_linear(&buf, src_rate, TARGET_SR); + pending.append(&mut r); + } + emit_pcm_frames(&mut pending, &mut index, window, event, true); + Ok(()) +} + /// Mesh-flavour run_session: same backend setup + ingest_loop + /// decode loop as `run_session`, with the cpal capture replaced /// by an externally-fed `Receiver>`. The receiver lives diff --git a/src/ui/TranscribeView.svelte b/src/ui/TranscribeView.svelte index 038652a..de5e540 100644 --- a/src/ui/TranscribeView.svelte +++ b/src/ui/TranscribeView.svelte @@ -16,6 +16,8 @@ transcribeUi, startRecording, startUpload, + startRemoteRecording, + startRemoteUpload, stopRecording, abortRecording, pauseRecording, @@ -568,21 +570,32 @@ return; } if (transcribeViaDevicePubkey) { - // Routing pin is set. Distinguish the two failure modes the - // user might be in: peer-offline (pin valid but host away) vs - // receiver-not-yet-wired (Rust audio-chunk pipeline lands in - // a follow-up). Either way: pause-or-error, never silently - // run locally — the user picked a host and deserves to know - // why their pick isn't being used. - if (transcribePinUnavailable) { + // Routing pin is set: capture the mic locally and stream it to the + // pinned host, which runs the ASR (see `startRemoteRecording`). If + // the host is offline, pause-or-error rather than silently running + // locally — the user picked a host and deserves to know why their + // pick isn't being used. + if (transcribePinUnavailable || !transcribeRoutedPeer) { transcribeError = "Pinned peer is offline. Pick another host or 'this device' " + "in the bar under this pane to record."; - } else { - transcribeError = - "Remote transcribe is wired on the sender but the receiver " + - "pipeline lands in a follow-up. Pick 'this device' in the " + - "bar under this pane to record locally."; + return; + } + const conv = await persist({ force: true }); + try { + await startRemoteRecording({ + targetPeerId: transcribeRoutedPeer.peer_id, + runtime, + model, + // The host resolves its own model + diarize; we send our pick + // as a hint and the composite name when "Identify speakers" is + // on (the host drops it if it lacks the diarize model). + diarizeModel: diarizeEnabled ? defaultDiarizeModel : null, + device: mic.device_name || null, + conversationId: conv?.id ?? null, + }); + } catch (e) { + transcribeError = String(e); } return; } @@ -699,15 +712,26 @@ return; } if (transcribeViaDevicePubkey) { - if (transcribePinUnavailable) { + // Routing pin is set: decode the file locally and stream its audio + // to the pinned host, which runs the ASR (see `startRemoteUpload`). + if (transcribePinUnavailable || !transcribeRoutedPeer) { transcribeError = "Pinned peer is offline. Pick another host or 'this device' " + "in the bar under this pane to upload."; - } else { - transcribeError = - "Remote transcribe is wired on the sender but the receiver " + - "pipeline lands in a follow-up. Pick 'this device' in the " + - "bar under this pane to upload locally."; + return; + } + const conv = await persist({ force: true }); + try { + await startRemoteUpload({ + targetPeerId: transcribeRoutedPeer.peer_id, + runtime, + model, + diarizeModel: diarizeEnabled ? defaultDiarizeModel : null, + filePath, + conversationId: conv?.id ?? null, + }); + } catch (e) { + transcribeError = String(e); } return; } diff --git a/src/ui/transcribe-state.svelte.ts b/src/ui/transcribe-state.svelte.ts index c9d5577..357600e 100644 --- a/src/ui/transcribe-state.svelte.ts +++ b/src/ui/transcribe-state.svelte.ts @@ -1,6 +1,8 @@ import { invoke } from "@tauri-apps/api/core"; import { listen, type UnlistenFn } from "@tauri-apps/api/event"; +import { meshClient } from "../mesh-daemon.svelte"; + /** One unit of decoded speech emitted by the ASR worker. Mirror of * `transcribe::EmittedSegment` in src-tauri. Speaker IDs are * optional — present only when diarization is enabled and the @@ -173,6 +175,24 @@ let elapsedTimer: ReturnType | null = null; * that arrives so a follow-up persist() can't race the last delta. */ let stopResolver: (() => void) | null = null; +// ---- remote (mesh-host) transcription -------------------------------- +// When a peer is pinned as the transcribe host, audio is captured +// locally — Rust `transcribe_capture_start` (mic) / +// `transcribe_decode_file_start` (file), both emitting PCM on +// `myownllm://transcribe-capture/` — and forwarded to the peer via +// `meshClient.sendTranscribeRequest`; the host runs the ASR and streams +// segments back through that RPC's `on_segment` callback. These mirror +// the local session's lifecycle so the StatusBar + TranscribeView don't +// need to know whether the active session is local or remote. +// +// Non-null `remoteCaptureId` is the marker that the active session is +// remote, so the lifecycle controls (stop/abort/pause/resume) route to +// the capture commands instead of the local `transcribe_*` ones. +let remoteCaptureId: string | null = null; +let remoteMeshCancel: (() => void) | null = null; +let unlistenCapture: UnlistenFn | null = null; +let remoteFinished = false; + function clearTimers() { if (elapsedTimer) clearInterval(elapsedTimer); elapsedTimer = null; @@ -383,6 +403,15 @@ export async function startUpload(args: { } export async function pauseRecording(): Promise { + if (remoteCaptureId) { + if (!transcribeUi.active || transcribeUi.paused || transcribeUi.uploadOnly) return; + await invoke("transcribe_capture_set_paused", { + streamId: remoteCaptureId, + paused: true, + }); + transcribeUi.paused = true; + return; + } if (!transcribeUi.active || transcribeUi.paused || transcribeUi.drainOnly) return; if (!transcribeUi.streamId) return; await invoke("transcribe_pause", { streamId: transcribeUi.streamId }); @@ -390,6 +419,16 @@ export async function pauseRecording(): Promise { } export async function resumeRecording(): Promise { + if (remoteCaptureId) { + if (!transcribeUi.active || !transcribeUi.paused) return; + await invoke("transcribe_capture_set_paused", { + streamId: remoteCaptureId, + paused: false, + }); + transcribeUi.paused = false; + transcribeUi.startedAt = Date.now() - transcribeUi.elapsed * 1000; + return; + } if (!transcribeUi.active || !transcribeUi.paused) return; if (!transcribeUi.streamId) return; await invoke("transcribe_resume", { streamId: transcribeUi.streamId }); @@ -402,6 +441,26 @@ export async function resumeRecording(): Promise { * worker has emitted its final frame (after the drain completes), so * callers can safely persist the transcript right after `await`. */ export async function stopRecording(): Promise { + if (remoteCaptureId) { + // Stop capturing; the Rust loop flushes its trailing audio and emits + // a final PCM frame, which we forward to the host as the end-of-audio + // marker. The host drains its ASR backlog, streams the last segments, + // and ends the RPC — `on_done` then resolves this promise via + // `finalizeRemote`. + transcribeUi.draining = true; + const done = new Promise((resolve) => { + stopResolver = resolve; + }); + try { + await invoke("transcribe_capture_stop", { streamId: remoteCaptureId }); + } catch (e) { + console.warn("transcribe_capture_stop failed:", e); + finalizeRemote(null); + return; + } + await done; + return; + } const id = transcribeUi.streamId; if (!id) return; // Capture has stopped; the session is now draining its backlog until the @@ -429,6 +488,18 @@ export async function stopRecording(): Promise { * frame, like `stopRecording`. Safe to call whether or not a graceful * stop is already in flight. */ export async function abortRecording(): Promise { + if (remoteCaptureId) { + // Cut off now: stop the local capture, then tear down (which drops the + // RPC so we don't wait on the host's backlog). Whatever segments + // already streamed back are kept — the caller persists them. + try { + await invoke("transcribe_capture_stop", { streamId: remoteCaptureId }); + } catch (e) { + console.warn("transcribe_capture_stop failed:", e); + } + finalizeRemote(null); + return; + } const id = transcribeUi.streamId; if (!id) return; // If no graceful stop preceded this, make sure the awaited `done` promise @@ -491,6 +562,235 @@ export async function startDrain(args: { transcribeUi.liveDelta = ""; } +/** One transcript segment streamed back from the host peer. Mirror of + * `SegmentPayload` in `mesh-transcribe.ts`. */ +interface RemoteSegment { + text: string; + speaker?: number; + overlap?: boolean; + start_ms?: number; + end_ms?: number; +} + +/** One PCM frame the Rust capture / decode loop emits for us to forward + * to the host. Mirror of `CapturePcm` in `src-tauri/src/transcribe.rs`: + * `bytes_b64` is i16-LE PCM at 16 kHz mono, empty on the terminal frame; + * `error` is set instead of audio when the mic / decoder failed. */ +interface CapturePcm { + index: number; + bytes_b64: string; + is_final: boolean; + error?: string | null; +} + +function bytesFromBase64(b64: string): Uint8Array { + const bin = atob(b64); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; +} + +/** Tear a remote session down exactly once: drop the capture listener, + * surface any error, flip the UI back to idle, and resolve a pending + * stop()/abort(). The local-session equivalent is the `final`-frame + * branch in `attachListener`. */ +function finalizeRemote(error: string | null): void { + if (remoteFinished) return; + remoteFinished = true; + clearTimers(); + unlistenCapture?.(); + unlistenCapture = null; + // Release the RPC. On a clean host finish this stream is already ended + // (cancel is then a harmless no-op); on an abort or a local-capture + // failure it's what stops the host waiting for audio that won't come. + remoteMeshCancel?.(); + remoteMeshCancel = null; + // Stop the local mic / decoder if it's still running. Idempotent: on the + // graceful-stop path the capture already ended; this is the backstop for + // the error / peer-dropped paths so the microphone doesn't stay open. + const id = remoteCaptureId; + remoteCaptureId = null; + if (id) void invoke("transcribe_capture_stop", { streamId: id }).catch(() => {}); + if (error) transcribeUi.error = error; + transcribeUi.active = false; + transcribeUi.draining = false; + const r = stopResolver; + stopResolver = null; + r?.(); +} + +interface RemoteCommon { + targetPeerId: string; + runtime: string; + model: string; + diarizeModel: string | null; + conversationId: string | null; +} + +/** Shared driver for remote Record + remote Upload. Opens the + * `transcribe` RPC on the host (segments stream back via `on_segment`), + * wires the local capture/decode PCM events through to the host, brings + * up the transcribe UI state, then kicks off the local audio source via + * `startCapture`. */ +async function runRemoteSession( + common: RemoteCommon, + uploadOnly: boolean, + startCapture: (captureId: string) => Promise, +): Promise { + if (transcribeUi.active) return; + transcribeUi.error = ""; + remoteFinished = false; + const captureId = crypto.randomUUID(); + + // 1. Open the remote ASR session. Segments arrive on `on_segment`; the + // terminal state on `on_done` (clean) / `on_error`. + let handle: { + id: string; + sendAudioChunk: (pcm: Uint8Array, isFinal: boolean) => void; + cancel: () => void; + }; + try { + handle = await meshClient.sendTranscribeRequest({ + target_peer_id: common.targetPeerId, + runtime: common.runtime, + model: common.model, + diarize_model: common.diarizeModel, + on_segment: (seg: RemoteSegment) => { + if (remoteFinished) return; + const e: EmittedSegment = { + start_ms: seg.start_ms ?? 0, + end_ms: seg.end_ms ?? 0, + text: seg.text, + speaker: seg.speaker, + overlap: seg.overlap, + }; + transcribeUi.liveSegments = [...transcribeUi.liveSegments, e]; + transcribeUi.liveDelta = transcribeUi.liveDelta + e.text + " "; + transcribeUi.framePulse++; + }, + on_done: () => finalizeRemote(null), + on_error: (m: string) => finalizeRemote(m), + }); + } catch (e) { + transcribeUi.error = `Couldn't start remote transcription: ${e}`; + throw e; + } + remoteMeshCancel = handle.cancel; + + // 2. Forward captured PCM → host. Listen before starting the capture so + // the first frame can't be missed. + let sentFinal = false; + const sendFinalOnce = () => { + if (sentFinal) return; + sentFinal = true; + handle.sendAudioChunk(new Uint8Array(0), true); + }; + unlistenCapture = await listen( + `myownllm://transcribe-capture/${captureId}`, + (ev) => { + if (remoteFinished) return; + const f = ev.payload; + if (f.error) { + // Local mic / decoder failed mid-stream. Close out the host + // session, then tear down with the error surfaced inline. + sendFinalOnce(); + finalizeRemote(f.error); + return; + } + if (f.bytes_b64) { + handle.sendAudioChunk(bytesFromBase64(f.bytes_b64), false); + } + // The terminal frame (flush complete) carries no audio — signal + // end-of-stream to the host so it drains and finalizes. + if (f.is_final) sendFinalOnce(); + }, + ); + + // 3. Bring up the UI state, mirroring startRecording / startUpload. + transcribeUi.active = true; + transcribeUi.paused = false; + transcribeUi.drainOnly = false; + transcribeUi.uploadOnly = uploadOnly; + transcribeUi.streamId = captureId; + transcribeUi.runtime = common.runtime; + transcribeUi.model = common.model; + transcribeUi.conversationId = common.conversationId; + transcribeUi.startedAt = Date.now(); + transcribeUi.elapsed = 0; + transcribeUi.pendingChunks = 0; + transcribeUi.liveSegments = []; + transcribeUi.liveDelta = ""; + remoteCaptureId = captureId; + if (uploadOnly) { + transcribeUi.uploadProgress = { total_ms: null, decoded_ms: 0, processed_ms: 0 }; + } else { + elapsedTimer = setInterval(() => { + if (transcribeUi.paused) return; + transcribeUi.elapsed = Math.floor((Date.now() - transcribeUi.startedAt) / 1000); + }, 250); + } + + // 4. Start the local audio source. On failure, tear everything down. + try { + await startCapture(captureId); + } catch (e) { + finalizeRemote(String(e)); + throw e; + } +} + +/** Remote Record: capture the local mic, transcribe on `targetPeerId`. */ +export async function startRemoteRecording(args: { + targetPeerId: string; + runtime: string; + model: string; + diarizeModel: string | null; + device: string | null; + conversationId: string | null; +}): Promise { + await runRemoteSession( + { + targetPeerId: args.targetPeerId, + runtime: args.runtime, + model: args.model, + diarizeModel: args.diarizeModel, + conversationId: args.conversationId, + }, + false, + (captureId) => + invoke("transcribe_capture_start", { + streamId: captureId, + device: args.device, + }), + ); +} + +/** Remote Upload: decode a local file, transcribe it on `targetPeerId`. */ +export async function startRemoteUpload(args: { + targetPeerId: string; + runtime: string; + model: string; + diarizeModel: string | null; + filePath: string; + conversationId: string | null; +}): Promise { + await runRemoteSession( + { + targetPeerId: args.targetPeerId, + runtime: args.runtime, + model: args.model, + diarizeModel: args.diarizeModel, + conversationId: args.conversationId, + }, + true, + (captureId) => + invoke("transcribe_decode_file_start", { + streamId: captureId, + filePath: args.filePath, + }), + ); +} + /** Hand back whatever segments have streamed in since the last flush, * emptying the buffer. Called by `TranscribeView` so it can merge them * into the rendered transcript and persist. */