Iteration 5: add browser voice mode

docs/feature-checklists/04-voice-mode.md

@@ -6,19 +6,25 @@ Add browser-native voice input and optional spoken assistant replies using built
 
 ## Checklist
 
-- [ ] Review candidate voice input/output implementation
-- [ ] Decide the smallest supported browser behavior and fallback text
-- [ ] Add voice input button and listening state
-- [ ] Add optional voice output toggle for assistant replies
-- [ ] Keep the chat input usable when voice APIs are unavailable
-- [ ] Ensure the UI remains clean on desktop and mobile
-- [ ] Add at least one lightweight check for prompt/input behavior
-- [ ] Run `npm run lint`
-- [ ] Run `npm run build`
-- [ ] Verify the flow in a live browser session
+- [x] Review candidate voice input/output implementation
+- [x] Decide the smallest supported browser behavior and fallback text
+- [x] Add voice input button and listening state
+- [x] Add optional voice output toggle for assistant replies
+- [x] Keep the chat input usable when voice APIs are unavailable
+- [x] Ensure the UI remains clean on desktop and mobile
+- [x] Add at least one lightweight check for prompt/input behavior
+- [x] Run `npm run lint`
+- [x] Run `npm run build`
+- [x] Verify the flow in a live browser session
 - [ ] Commit, push branch, and open draft PR
 
 ## Notes
 
 - Candidate reference: `tmp_compare/kavi-kapruka/lib/speech.ts`
 - Candidate reference: `tmp_compare/kavi-kapruka/components/chat/VoiceButton.tsx`
+- The implementation uses browser-native Web Speech APIs only. No backend work and no paid service dependency.
+- Lightweight check: `node --test --experimental-strip-types frontend/src/lib/speech.spec.ts`
+- Live browser verification used a Playwright fallback because the in-app Browser tool was not callable in this thread.
+- Browser checks covered:
+  - supported-browser mock: mic button renders, enters listening state, and the spoken-reply toggle flips on
+  - unsupported-browser mock: the normal text input still renders while voice controls stay hidden

frontend/src/app/page.tsx

@@ -7,6 +7,7 @@ import CheckoutDrawer from "@/components/CheckoutDrawer";
 import GiftAdvisor from "@/components/GiftAdvisor";
 import { useCart } from "@/hooks/useCart";
 import { useChat } from "@/hooks/useChat";
+import { createRecognition, isSpeechRecognitionSupported, isSpeechSynthesisSupported, speakText, stopSpeaking, type RecognitionLike } from "@/lib/speech";
 import { getBackendMeta, type BackendMeta, updateCheckoutInfo, updateBudget, type CheckoutInfoPayload } from "@/lib/api";
 
 function generateSessionId() {
@@ -15,6 +16,7 @@ function generateSessionId() {
 
 const SESSION_STORAGE_KEY = "kapruka.chat.session";
 const SESSION_TTL_MS = 5 * 60 * 1000;
+const VOICE_REPLY_STORAGE_KEY = "kapruka.chat.voiceReplies";
 
 function loadSessionId() {
   if (typeof window === "undefined") return generateSessionId();
@@ -45,6 +47,15 @@ function saveSessionId(sessionId: string) {
   }
 }
 
+function loadVoiceRepliesEnabled() {
+  if (typeof window === "undefined") return false;
+  try {
+    return window.localStorage.getItem(VOICE_REPLY_STORAGE_KEY) === "1";
+  } catch {
+    return false;
+  }
+}
+
 const suggestions = [
   { label: "Restock groceries", prompt: "Help me restock weekly groceries on Kapruka" },
   { label: "Birthday cake", prompt: "Find birthday cakes on Kapruka under Rs. 10,000" },
@@ -118,10 +129,16 @@ export default function Home() {
   const [budgetDraft, setBudgetDraft] = useState("");
   const [budgetSaving, setBudgetSaving] = useState(false);
   const [inkPosition, setInkPosition] = useState({ x: "50%", y: "42%" });
+  const [voiceInputSupported] = useState(isSpeechRecognitionSupported);
+  const [voiceRepliesSupported] = useState(isSpeechSynthesisSupported);
+  const [voiceRepliesEnabled, setVoiceRepliesEnabled] = useState(loadVoiceRepliesEnabled);
+  const [isListening, setIsListening] = useState(false);
   const messagesEndRef = useRef<HTMLDivElement>(null);
   const inputRef = useRef<HTMLInputElement>(null);
   const budgetInputRef = useRef<HTMLInputElement>(null);
   const sendButtonRef = useRef<HTMLButtonElement>(null);
+  const recognitionRef = useRef<RecognitionLike | null>(null);
+  const lastSpokenAssistantRef = useRef<string>("");
 
   const submitCurrentMessage = useCallback(() => {
     const rawMessage = inputRef.current?.value ?? input;
@@ -139,10 +156,36 @@ export default function Home() {
     saveSessionId(sessionId);
   }, [sessionId]);
 
+  useEffect(() => {
+    return () => {
+      recognitionRef.current?.abort();
+      stopSpeaking();
+    };
+  }, []);
+
+  useEffect(() => {
+    try {
+      window.localStorage.setItem(VOICE_REPLY_STORAGE_KEY, voiceRepliesEnabled ? "1" : "0");
+    } catch {
+      // ignore
+    }
+    if (!voiceRepliesEnabled) stopSpeaking();
+  }, [voiceRepliesEnabled]);
+
   useEffect(() => {
     messagesEndRef.current?.scrollIntoView({ behavior: "smooth" });
   }, [messages]);
 
+  useEffect(() => {
+    if (!voiceRepliesEnabled || isStreaming) return;
+    const latestAssistant = [...messages].reverse().find((message) => message.role === "assistant" && message.content.trim());
+    if (!latestAssistant) return;
+    const signature = `${latestAssistant.id}:${latestAssistant.content}`;
+    if (signature === lastSpokenAssistantRef.current) return;
+    lastSpokenAssistantRef.current = signature;
+    speakText(latestAssistant.content);
+  }, [messages, isStreaming, voiceRepliesEnabled]);
+
   useEffect(() => {
     const el = inputRef.current;
     if (!el) return;
@@ -246,6 +289,50 @@ export default function Home() {
     sendMessage(prompt);
   };
 
+  const toggleListening = () => {
+    if (isListening) {
+      recognitionRef.current?.stop();
+      setIsListening(false);
+      return;
+    }
+
+    const recognition = createRecognition("en-US");
+    if (!recognition) return;
+    recognitionRef.current = recognition;
+
+    recognition.onresult = (event: {
+      resultIndex: number;
+      results: { [key: number]: { 0: { transcript: string }; isFinal: boolean }; length: number };
+    }) => {
+      let interim = "";
+      let final = "";
+      for (let index = event.resultIndex; index < event.results.length; index += 1) {
+        const result = event.results[index];
+        const transcript = result[0].transcript;
+        if (result.isFinal) final += transcript;
+        else interim += transcript;
+      }
+
+      const nextValue = (final || interim).trimStart();
+      if (!nextValue) return;
+      setInput(nextValue);
+      if (inputRef.current) inputRef.current.value = nextValue;
+      if (final.trim()) {
+        recognition.stop();
+        setIsListening(false);
+      }
+    };
+    recognition.onerror = () => setIsListening(false);
+    recognition.onend = () => setIsListening(false);
+
+    try {
+      recognition.start();
+      setIsListening(true);
+    } catch {
+      setIsListening(false);
+    }
+  };
+
   useEffect(() => {
     const inputEl = inputRef.current;
     const buttonEl = sendButtonRef.current;
@@ -561,6 +648,22 @@ export default function Home() {
 
           <div className="shrink-0 border-t border-border/80 bg-bg/95 px-5 py-4 backdrop-blur md:px-8 md:py-5">
             <form onSubmit={handleSubmit} className="mx-auto flex w-full max-w-3xl gap-3">
+              {voiceInputSupported ? (
+                <button
+                  type="button"
+                  onClick={toggleListening}
+                  disabled={isStreaming}
+                  className={`grid h-12 w-12 shrink-0 place-items-center rounded-xl border transition ${
+                    isListening
+                      ? "border-accent bg-accent text-white"
+                      : "border-border bg-surface text-ink hover:border-border-hover hover:bg-surface-2"
+                  } disabled:cursor-not-allowed disabled:opacity-40`}
+                  aria-label={isListening ? "Stop voice input" : "Start voice input"}
+                  title={isListening ? "Listening... tap to stop" : "Speak your message"}
+                >
+                  <span className="text-base font-semibold">{isListening ? "■" : "Mic"}</span>
+                </button>
+              ) : null}
               <input
                 ref={inputRef}
                 value={input}
@@ -577,6 +680,21 @@ export default function Home() {
                 disabled={isStreaming}
                 className="h-12 min-w-0 flex-1 rounded-xl border border-border bg-surface-2 px-5 text-sm text-ink outline-none transition placeholder:text-muted focus:border-ink focus:bg-surface focus:shadow-[0_0_0_3px_rgba(37,36,31,0.08)] disabled:opacity-40"
               />
+              {voiceRepliesSupported ? (
+                <button
+                  type="button"
+                  onClick={() => setVoiceRepliesEnabled((current) => !current)}
+                  className={`inline-flex h-12 shrink-0 items-center rounded-xl border px-3 text-sm transition ${
+                    voiceRepliesEnabled
+                      ? "border-accent bg-accent text-white"
+                      : "border-border bg-surface text-ink hover:border-border-hover hover:bg-surface-2"
+                  }`}
+                  aria-label={voiceRepliesEnabled ? "Turn off spoken replies" : "Turn on spoken replies"}
+                  title={voiceRepliesEnabled ? "Spoken replies on" : "Spoken replies off"}
+                >
+                  Voice
+                </button>
+              ) : null}
               <button
                 ref={sendButtonRef}
                 type="button"

frontend/src/lib/speech.spec.ts

@@ -0,0 +1,11 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import { stripForSpeech } from "./speech.ts";
+
+test("stripForSpeech removes markdown and keeps readable text", () => {
+  assert.equal(
+    stripForSpeech("**Order Ready!** [Pay now](https://example.com) 🎉"),
+    "Order Ready! Pay now"
+  );
+});

frontend/src/lib/speech.ts

@@ -0,0 +1,60 @@
+"use client";
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+export interface RecognitionLike {
+  lang: string;
+  continuous: boolean;
+  interimResults: boolean;
+  start: () => void;
+  stop: () => void;
+  abort: () => void;
+  onresult: ((event: any) => void) | null;
+  onend: (() => void) | null;
+  onerror: (() => void) | null;
+}
+
+export function isSpeechRecognitionSupported() {
+  if (typeof window === "undefined") return false;
+  return Boolean((window as any).SpeechRecognition || (window as any).webkitSpeechRecognition);
+}
+
+export function createRecognition(lang = "en-US"): RecognitionLike | null {
+  if (typeof window === "undefined") return null;
+  const Ctor = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
+  if (!Ctor) return null;
+  const recognition: RecognitionLike = new Ctor();
+  recognition.lang = lang;
+  recognition.continuous = false;
+  recognition.interimResults = true;
+  return recognition;
+}
+
+export function isSpeechSynthesisSupported() {
+  return typeof window !== "undefined" && typeof window.speechSynthesis !== "undefined";
+}
+
+export function stripForSpeech(text: string) {
+  return text
+    .replace(/\[(.*?)\]\(.*?\)/g, "$1")
+    .replace(/[*_`#>~]/g, "")
+    .replace(/[^\p{L}\p{N}\p{P}\p{Z}]/gu, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+export function speakText(text: string) {
+  if (!isSpeechSynthesisSupported()) return;
+  const clean = stripForSpeech(text);
+  if (!clean) return;
+  window.speechSynthesis.cancel();
+  const utterance = new SpeechSynthesisUtterance(clean);
+  utterance.rate = 1.02;
+  utterance.pitch = 1;
+  window.speechSynthesis.speak(utterance);
+}
+
+export function stopSpeaking() {
+  if (!isSpeechSynthesisSupported()) return;
+  window.speechSynthesis.cancel();
+}