From 06b02846477cd1838a48adb0637dc164db811988 Mon Sep 17 00:00:00 2001 From: HimanM <67066047+HimanM@users.noreply.github.com> Date: Sun, 21 Jun 2026 19:21:23 +0530 Subject: [PATCH] Add browser voice input and reply toggle --- docs/feature-checklists/04-voice-mode.md | 26 +++-- frontend/src/app/page.tsx | 118 +++++++++++++++++++++++ frontend/src/lib/speech.spec.ts | 11 +++ frontend/src/lib/speech.ts | 60 ++++++++++++ 4 files changed, 205 insertions(+), 10 deletions(-) create mode 100644 frontend/src/lib/speech.spec.ts create mode 100644 frontend/src/lib/speech.ts diff --git a/docs/feature-checklists/04-voice-mode.md b/docs/feature-checklists/04-voice-mode.md index 9a323eb..6baeb27 100644 --- a/docs/feature-checklists/04-voice-mode.md +++ b/docs/feature-checklists/04-voice-mode.md @@ -6,19 +6,25 @@ Add browser-native voice input and optional spoken assistant replies using built ## Checklist -- [ ] Review candidate voice input/output implementation -- [ ] Decide the smallest supported browser behavior and fallback text -- [ ] Add voice input button and listening state -- [ ] Add optional voice output toggle for assistant replies -- [ ] Keep the chat input usable when voice APIs are unavailable -- [ ] Ensure the UI remains clean on desktop and mobile -- [ ] Add at least one lightweight check for prompt/input behavior -- [ ] Run `npm run lint` -- [ ] Run `npm run build` -- [ ] Verify the flow in a live browser session +- [x] Review candidate voice input/output implementation +- [x] Decide the smallest supported browser behavior and fallback text +- [x] Add voice input button and listening state +- [x] Add optional voice output toggle for assistant replies +- [x] Keep the chat input usable when voice APIs are unavailable +- [x] Ensure the UI remains clean on desktop and mobile +- [x] Add at least one lightweight check for prompt/input behavior +- [x] Run `npm run lint` +- [x] Run `npm run build` +- [x] Verify the flow in a live browser session - [ ] Commit, push branch, and open draft PR ## Notes - Candidate reference: `tmp_compare/kavi-kapruka/lib/speech.ts` - Candidate reference: `tmp_compare/kavi-kapruka/components/chat/VoiceButton.tsx` +- The implementation uses browser-native Web Speech APIs only. No backend work and no paid service dependency. +- Lightweight check: `node --test --experimental-strip-types frontend/src/lib/speech.spec.ts` +- Live browser verification used a Playwright fallback because the in-app Browser tool was not callable in this thread. +- Browser checks covered: + - supported-browser mock: mic button renders, enters listening state, and the spoken-reply toggle flips on + - unsupported-browser mock: the normal text input still renders while voice controls stay hidden diff --git a/frontend/src/app/page.tsx b/frontend/src/app/page.tsx index 1585bb5..9767982 100644 --- a/frontend/src/app/page.tsx +++ b/frontend/src/app/page.tsx @@ -7,6 +7,7 @@ import CheckoutDrawer from "@/components/CheckoutDrawer"; import GiftAdvisor from "@/components/GiftAdvisor"; import { useCart } from "@/hooks/useCart"; import { useChat } from "@/hooks/useChat"; +import { createRecognition, isSpeechRecognitionSupported, isSpeechSynthesisSupported, speakText, stopSpeaking, type RecognitionLike } from "@/lib/speech"; import { getBackendMeta, type BackendMeta, updateCheckoutInfo, updateBudget, type CheckoutInfoPayload } from "@/lib/api"; function generateSessionId() { @@ -15,6 +16,7 @@ function generateSessionId() { const SESSION_STORAGE_KEY = "kapruka.chat.session"; const SESSION_TTL_MS = 5 * 60 * 1000; +const VOICE_REPLY_STORAGE_KEY = "kapruka.chat.voiceReplies"; function loadSessionId() { if (typeof window === "undefined") return generateSessionId(); @@ -45,6 +47,15 @@ function saveSessionId(sessionId: string) { } } +function loadVoiceRepliesEnabled() { + if (typeof window === "undefined") return false; + try { + return window.localStorage.getItem(VOICE_REPLY_STORAGE_KEY) === "1"; + } catch { + return false; + } +} + const suggestions = [ { label: "Restock groceries", prompt: "Help me restock weekly groceries on Kapruka" }, { label: "Birthday cake", prompt: "Find birthday cakes on Kapruka under Rs. 10,000" }, @@ -118,10 +129,16 @@ export default function Home() { const [budgetDraft, setBudgetDraft] = useState(""); const [budgetSaving, setBudgetSaving] = useState(false); const [inkPosition, setInkPosition] = useState({ x: "50%", y: "42%" }); + const [voiceInputSupported] = useState(isSpeechRecognitionSupported); + const [voiceRepliesSupported] = useState(isSpeechSynthesisSupported); + const [voiceRepliesEnabled, setVoiceRepliesEnabled] = useState(loadVoiceRepliesEnabled); + const [isListening, setIsListening] = useState(false); const messagesEndRef = useRef(null); const inputRef = useRef(null); const budgetInputRef = useRef(null); const sendButtonRef = useRef(null); + const recognitionRef = useRef(null); + const lastSpokenAssistantRef = useRef(""); const submitCurrentMessage = useCallback(() => { const rawMessage = inputRef.current?.value ?? input; @@ -139,10 +156,36 @@ export default function Home() { saveSessionId(sessionId); }, [sessionId]); + useEffect(() => { + return () => { + recognitionRef.current?.abort(); + stopSpeaking(); + }; + }, []); + + useEffect(() => { + try { + window.localStorage.setItem(VOICE_REPLY_STORAGE_KEY, voiceRepliesEnabled ? "1" : "0"); + } catch { + // ignore + } + if (!voiceRepliesEnabled) stopSpeaking(); + }, [voiceRepliesEnabled]); + useEffect(() => { messagesEndRef.current?.scrollIntoView({ behavior: "smooth" }); }, [messages]); + useEffect(() => { + if (!voiceRepliesEnabled || isStreaming) return; + const latestAssistant = [...messages].reverse().find((message) => message.role === "assistant" && message.content.trim()); + if (!latestAssistant) return; + const signature = `${latestAssistant.id}:${latestAssistant.content}`; + if (signature === lastSpokenAssistantRef.current) return; + lastSpokenAssistantRef.current = signature; + speakText(latestAssistant.content); + }, [messages, isStreaming, voiceRepliesEnabled]); + useEffect(() => { const el = inputRef.current; if (!el) return; @@ -246,6 +289,50 @@ export default function Home() { sendMessage(prompt); }; + const toggleListening = () => { + if (isListening) { + recognitionRef.current?.stop(); + setIsListening(false); + return; + } + + const recognition = createRecognition("en-US"); + if (!recognition) return; + recognitionRef.current = recognition; + + recognition.onresult = (event: { + resultIndex: number; + results: { [key: number]: { 0: { transcript: string }; isFinal: boolean }; length: number }; + }) => { + let interim = ""; + let final = ""; + for (let index = event.resultIndex; index < event.results.length; index += 1) { + const result = event.results[index]; + const transcript = result[0].transcript; + if (result.isFinal) final += transcript; + else interim += transcript; + } + + const nextValue = (final || interim).trimStart(); + if (!nextValue) return; + setInput(nextValue); + if (inputRef.current) inputRef.current.value = nextValue; + if (final.trim()) { + recognition.stop(); + setIsListening(false); + } + }; + recognition.onerror = () => setIsListening(false); + recognition.onend = () => setIsListening(false); + + try { + recognition.start(); + setIsListening(true); + } catch { + setIsListening(false); + } + }; + useEffect(() => { const inputEl = inputRef.current; const buttonEl = sendButtonRef.current; @@ -561,6 +648,22 @@ export default function Home() {
+ {voiceInputSupported ? ( + + ) : null} + {voiceRepliesSupported ? ( + + ) : null}