From 914586d1bbd58458a82ff57fe73030d1c0575582 Mon Sep 17 00:00:00 2001 From: Yash Dewasthale Date: Thu, 6 Aug 2026 17:28:59 +0530 Subject: [PATCH] feat: enhance voice features with Smallest.ai integration and Jarvis wake functionality: - Updated voice capture to exclusively use Smallest.ai for Speech-to-Text (STT) with new environment variables. - Introduced Text-to-Speech (TTS) capabilities using ElevenLabs, with fallback options for macOS. - Added Jarvis wake functionality to launch daily workspace apps via voice commands. - Updated documentation and example environment files to reflect new configurations and usage instructions. --- AGENTS.md | 2 +- apps/supercode-cli/server/.env.example | 43 ++++ apps/supercode-cli/server/README.md | 24 ++- .../server/src/cli/ai/chat/chat.ts | 189 +++++++++++++---- .../src/cli/commands/slashCommands/index.ts | 6 +- .../server/src/voice/__tests__/jarvis.test.ts | 78 +++++++ .../server/src/voice/__tests__/speech.test.ts | 73 ++++--- apps/supercode-cli/server/src/voice/jarvis.ts | 191 ++++++++++++++++++ apps/supercode-cli/server/src/voice/speech.ts | 163 +++++++++++++-- 9 files changed, 666 insertions(+), 103 deletions(-) create mode 100644 apps/supercode-cli/server/src/voice/__tests__/jarvis.test.ts create mode 100644 apps/supercode-cli/server/src/voice/jarvis.ts diff --git a/AGENTS.md b/AGENTS.md index 4ba2d4e..ea4978f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -110,7 +110,7 @@ packages/ - Web app: `apps/web/.env` or `apps/.env.local` - Required for auth: `GITHUB_CLIENT_ID`, `GITHUB_CLIENT_SECRET`, `NEXT_PUBLIC_BETTER_AUTH_URL` - Required for DB: `DATABASE_URL` -- Voice/STT: `ELEVENLABS_API_KEY` (default provider), `STT_PROVIDER` (`elevenlabs`|`groq`), `ELEVENLABS_MODEL`, `STT_LANGUAGE` +- Voice/STT: `SMALLEST_API_KEY` (Smallest.ai Pulse STT — the only STT provider), `SMALLEST_MODEL`, `STT_LANGUAGE`. Voice TTS (spoken reply): `VOICE_REPLY` (`on`|`off`), `ELEVENLABS_VOICE_ID`, `ELEVENLABS_TTS_MODEL` (macOS `say` fallback when no key). Note `ELEVENLABS_API_KEY` doubles as both STT and TTS key historically; STT now always routes to Smallest.ai. - Secrets: `INFISICAL_CLIENT_ID`, `INFISICAL_CLIENT_SECRET` (when Infisical is configured) ### Linting diff --git a/apps/supercode-cli/server/.env.example b/apps/supercode-cli/server/.env.example index 2972666..d6f3f7b 100644 --- a/apps/supercode-cli/server/.env.example +++ b/apps/supercode-cli/server/.env.example @@ -80,6 +80,49 @@ GOOGLE_CSE_ID="" # Firecrawl FIRECRAWL_API_KEY="" +# ─── Voice / Speech-to-Text (STT) ────────────── +# Used by the CLI voice capture and POST /api/voice/transcribe. +# Voice capture uses Smallest.ai (Pulse STT) exclusively. +STT_PROVIDER="smallest" +STT_LANGUAGE="en" + +# Smallest.ai (Pulse STT) — https://app.smallest.ai/dashboard/api-keys +SMALLEST_API_KEY="" +# "pulse-pro" (best English accuracy) or "pulse" (multilingual) by default +SMALLEST_MODEL="pulse-pro" +SMALLEST_LANGUAGE="en" + +# ─── Voice / Text-to-Speech (TTS / spoken reply) ──── +# The CLI speaks the assistant's answer aloud after a voice-triggered turn +# (Clicky-style). ElevenLabs TTS when ELEVENLABS_API_KEY is set, otherwise the +# built-in macOS `say` voice. Requires macOS for audio playback. +# Set VOICE_REPLY="off" to disable spoken replies entirely. +VOICE_REPLY="on" +# Voice used for ElevenLabs TTS +ELEVENLABS_VOICE_ID="21m00Tcm4TlvDq8ikWAM" +# "eleven_turbo_v2_5" (low latency) by default +ELEVENLABS_TTS_MODEL="" + +# ─── Jarvis wake-work agent ───────────────────── +# ⚪ Optional. When voice input (or /jarvis) is triggered, the CLI launches +# these daily apps. WhatsApp opens in the native macOS desktop app; everything +# else opens in the browser named by JARVIS_BROWSER (default "Dia") rather than +# the OS default. Set the Slack/Linear URLs to your real workspace URLs; +# JARVIS_URLS is a comma-separated list of extra apps. +JARVIS_WAKE_PHRASE="jarvis wake up,jarvis wakeup,jarvis" +# Browser used to open the non-WhatsApp targets (macOS app name) +JARVIS_BROWSER="Dia" +# Native macOS app for WhatsApp (falls back to the URL when uninstalled) +JARVIS_WHATSAPP_APP="WhatsApp" +JARVIS_GITHUB_URL="https://github.com/yashdev9274/supercli" +JARVIS_WHATSAPP_URL="https://web.whatsapp.com" +JARVIS_SLACK_URL="https://join.slack.com/t/supercodeai/shared_invite/zt-43enen35h-glS1qR854YB~HUg2AIW0vg" +JARVIS_LINEAR_URL="https://linear.app/supercodeai/projects/all" +JARVIS_TWITTER_URL="https://x.com" +# New Warp terminal tab opened at this folder (uses Warp's new_tab deep link) +JARVIS_WARP_DIR="/Users/yashdewasthale/dev/saas/supercli" +JARVIS_URLS="" + # ─── Dodo Payments ───────────────────────────── # 🟡 Required for paid-tier billing (checkout, webhooks, refunds) DODO_PAYMENTS_API_KEY="" diff --git a/apps/supercode-cli/server/README.md b/apps/supercode-cli/server/README.md index 718040f..645820d 100644 --- a/apps/supercode-cli/server/README.md +++ b/apps/supercode-cli/server/README.md @@ -44,17 +44,27 @@ The AI has access to file reading, searching, web fetching, and code execution t ### Voice Input -Voice capture requires `ffmpeg` and an STT provider API key. +Voice capture requires `ffmpeg` and a Smallest.ai API key. | Env Var | Description | Default | |---|---|---| -| `STT_PROVIDER` | STT provider (`elevenlabs` or `groq`) | `elevenlabs` | -| `ELEVENLABS_API_KEY` | ElevenLabs API key (required for ElevenLabs STT) | — | -| `ELEVENLABS_MODEL` | ElevenLabs model ID | `scribe_v1` | -| `GROQ_API_KEY` | Groq API key (required when `STT_PROVIDER=groq`) | — | -| `STT_LANGUAGE` | Transcription language | `en` | +| `SMALLEST_API_KEY` | Smallest.ai API key (https://app.smallest.ai/dashboard/api-keys) | — | +| `SMALLEST_MODEL` | Smallest.ai STT model | `pulse-pro` | +| `SMALLEST_LANGUAGE` | Transcription language | `en` | -Press **Ctrl+Shift+V** during a chat session to start voice capture. +### Voice Reply (speaks the answer back) + +After a voice-triggered turn, supercode reads the assistant's reply aloud using +ElevenLabs TTS when a key is available, otherwise macOS `say`. macOS only. + +| Env | Description | Default | +|---|---|---| +| `VOICE_REPLY` | Enable spoken replies (`on`/`off`) | `on` | +| `ELEVENLABS_VOICE_ID` | ElevenLabs TTS voice | `21m00Tcm4TlvDq8ikWAM` | +| `ELEVENLABS_TTS_MODEL` | ElevenLabs TTS model | `eleven_turbo_v2_5` | + +Press **Ctrl+V** (or **F2**) during a chat session to start voice capture. The +captured command is run as a normal agent turn, then the reply is spoken back. ## License diff --git a/apps/supercode-cli/server/src/cli/ai/chat/chat.ts b/apps/supercode-cli/server/src/cli/ai/chat/chat.ts index 46c903d..c0097f1 100644 --- a/apps/supercode-cli/server/src/cli/ai/chat/chat.ts +++ b/apps/supercode-cli/server/src/cli/ai/chat/chat.ts @@ -85,7 +85,9 @@ import { voiceCaptureFlow, canVoiceCapture, stopCapture, + speakText, } from "src/voice/speech.ts" +import { isJarvisWake, runJarvisStart } from "src/voice/jarvis.ts" import path from "node:path" import { AtPicker, DragDropTracker } from "./at-picker.ts" import { @@ -112,6 +114,21 @@ async function getUserFromToken() { return result.user } +// Store the current user for feature gating +let currentUser: { id: string; name: string | null; email: string } | null = null + +// Check if the current user is Yash Dewasthale (for feature gating) +function isYashDewasthale(): boolean { + if (!currentUser) return false + const name = currentUser.name?.toLowerCase() ?? "" + const email = currentUser.email?.toLowerCase() ?? "" + return ( + name.includes("yash") && name.includes("dewasthale") || + email === "yashdev.yvd@gmail.com" || + email === "yash@supercode.ai" + ) +} + export async function initConversation(userId: string, conversationId: string | null = null, mode = "chat") { const thinking = createThinking("loading conversation") const conversation = await getOrCreateConversation(conversationId, mode) @@ -967,6 +984,10 @@ let stdinPrevWrapLines = 1 // transcribed text instead of wiping stdinInput to "". let voiceJustCaptured = false +// Set when a voice capture auto-submits (Clicky-style) so the loop can speak +// the reply back once the assistant turn finishes. Consumed at most once. +let voiceAutoSubmitted = false + // Loaded skill context — injected as a system message so the AI uses it // without pasting the full text into the user's input. export let loadedSkillName: string | undefined @@ -1309,48 +1330,7 @@ function stdinKeypress(_str: string, key: any) { return } - const resolve = stdinResolve - stdinResolve = null - // Clear slash list (content below input line) - for (let i = 0; i < slashListLines; i++) { - readline.moveCursor(process.stdout, 0, 1) - } - for (let i = 0; i < slashListLines; i++) { - readline.cursorTo(process.stdout, 0) - readline.clearLine(process.stdout, 0) - if (i < slashListLines - 1) { - readline.moveCursor(process.stdout, 0, -1) - } - } - slashListLines = 0 - // Clear @ picker overlay - atPicker.close() - for (let i = 0; i < atListLines; i++) { - readline.moveCursor(process.stdout, 0, 1) - } - for (let i = 0; i < atListLines; i++) { - readline.cursorTo(process.stdout, 0) - readline.clearLine(process.stdout, 0) - if (i < atListLines - 1) { - readline.moveCursor(process.stdout, 0, -1) - } - } - atListLines = 0 - // Clear drag-drop indicator - for (let i = 0; i < ddListLines; i++) { - readline.moveCursor(process.stdout, 0, 1) - } - for (let i = 0; i < ddListLines; i++) { - readline.cursorTo(process.stdout, 0) - readline.clearLine(process.stdout, 0) - if (i < ddListLines - 1) { - readline.moveCursor(process.stdout, 0, -1) - } - } - ddListLines = 0 - ddTracker.clear() - process.stdout.write("\r\n") - resolve({ input: stdinInput, mode: stdinMode }) + commitInput() return } @@ -1566,6 +1546,11 @@ function stdinKeypress(_str: string, key: any) { } async function startVoiceCapture() { + if (!isYashDewasthale()) { + activeFooter?.setStatusMessage("⛭ Voice features are only available for Yash Dewasthale") + setTimeout(() => activeFooter?.setStatusMessage(""), 4000) + return + } const check = canVoiceCapture() if (!check.ok) { const reason = check.reason ?? "unknown" @@ -1575,24 +1560,117 @@ async function startVoiceCapture() { } const prevMode = voiceCaptureActive voiceCaptureActive = true - activeFooter?.setStatusMessage("🎤 Recording... (voice key or Enter to stop)") + activeFooter?.setStatusMessage("🎤 Recording... (voice key or Enter to stop)") try { const text = await voiceCaptureFlow() if (text) { + if (isJarvisWake(text)) { + // Wake-word caught — don't submit an agent turn, just start the + // workspace, speak a confirmation, and print the open summary. + const opened = await runJarvisAndSpeak() + process.stdout.write( + `\r\n ${chalk.hex(theme.green)("◆")} ${chalk.hex(theme.amber)("Jarvis")} woke — opening: ${opened.join(", ") || "none configured"}\r\n\n`, + ) + if (stdinResolve) renderInput() + else voiceJustCaptured = true + return + } stdinInput = stdinInput.slice(0, stdinCursor) + text + " " + stdinInput.slice(stdinCursor) stdinCursor += text.length + 1 + slashSelected = -1 + historyIndex = -1 + if (stdinResolve) { + // Loop is idle awaiting input — Clicky-style: auto-submit the spoken + // command so the agent actually does the thing and speaks back. + voiceAutoSubmitted = true + commitInput() + } else { + // Agent is busy — just fill the input so the user can review/send later. + voiceJustCaptured = true + } } else { activeFooter?.setStatusMessage("🎤 No speech detected — press voice key to retry") + setTimeout(() => activeFooter?.setStatusMessage(""), 4000) + // Nothing was submitted — restore the input prompt in place. + if (stdinResolve) renderInput() } } catch (err) { activeFooter?.setStatusMessage("⛭ Voice failed: " + (err instanceof Error ? err.message : err)) setTimeout(() => activeFooter?.setStatusMessage(""), 4000) + if (stdinResolve) renderInput() } finally { voiceCaptureActive = prevMode } } +// Wake Jarvis: launch the configured workspace targets in the default browser, +// speak a short confirmation, and report the status line for text. Returns the +// names that opened successfully. +async function runJarvisAndSpeak(): Promise { + if (!isYashDewasthale()) { + return [] + } + const { opened } = runJarvisStart() + const reply = opened.length + ? `Jarvis online. Opening ${opened.slice(0, 3).join(", ")}${opened.length > 3 ? " and more" : ""}.` + : "Jarvis online, but no workspace apps are configured." + activeFooter?.setStatusMessage(`🤖 Jarvis online · ${opened.length} app${opened.length === 1 ? "" : "s"} opening`) + setTimeout(() => activeFooter?.setStatusMessage(""), 6000) + await speakText(reply) + return opened +} + +// Commit whatever stdinInput currently holds as a submitted chat turn. Shared +// by the Enter key and the Clicky-style voice auto-execute path so both clear +// the input overlays (slash list, @ picker, drag-drop) and resolve the +// pending chatInput() promise identically. +function commitInput(): void { + const resolve = stdinResolve + if (!resolve) return + stdinResolve = null + // Clear slash list (content below input line) + for (let i = 0; i < slashListLines; i++) { + readline.moveCursor(process.stdout, 0, 1) + } + for (let i = 0; i < slashListLines; i++) { + readline.cursorTo(process.stdout, 0) + readline.clearLine(process.stdout, 0) + if (i < slashListLines - 1) { + readline.moveCursor(process.stdout, 0, -1) + } + } + slashListLines = 0 + // Clear @ picker overlay + atPicker.close() + for (let i = 0; i < atListLines; i++) { + readline.moveCursor(process.stdout, 0, 1) + } + for (let i = 0; i < atListLines; i++) { + readline.cursorTo(process.stdout, 0) + readline.clearLine(process.stdout, 0) + if (i < atListLines - 1) { + readline.moveCursor(process.stdout, 0, -1) + } + } + atListLines = 0 + // Clear drag-drop indicator + for (let i = 0; i < ddListLines; i++) { + readline.moveCursor(process.stdout, 0, 1) + } + for (let i = 0; i < ddListLines; i++) { + readline.cursorTo(process.stdout, 0) + readline.clearLine(process.stdout, 0) + if (i < ddListLines - 1) { + readline.moveCursor(process.stdout, 0, -1) + } + } + ddListLines = 0 + ddTracker.clear() + process.stdout.write("\r\n") + resolve({ input: stdinInput, mode: stdinMode }) +} + function ensureStdinHandler() { const stdin = process.stdin readline.emitKeypressEvents(stdin) @@ -2048,6 +2126,17 @@ export async function chatLoop( // (via setImmediate) so the text should already be visible. voiceJustCaptured = true process.stdout.write(`\r\n`) + } else if (result?.type === "jarvis") { + if (!isYashDewasthale()) { + process.stdout.write( + `\r\n ${chalk.hex(theme.red)("◆")} ${chalk.hex(theme.red)("Jarvis is only available for Yash Dewasthale")}\r\n\n`, + ) + } else { + const opened = await runJarvisAndSpeak() + process.stdout.write( + `\r\n ${chalk.hex(theme.green)("◆")} ${chalk.hex(theme.amber)("Jarvis")} woke — opening: ${opened.join(", ") || "none configured"}\r\n\n`, + ) + } } else if (result?.type === "skills") { if (result.skillName && result.message) { loadedSkillName = result.skillName @@ -2259,6 +2348,7 @@ export async function chatLoop( if (result.content && result.content !== "(cancelled)") { await addMessage(conversation.id, "assistant", result.content) } + voiceAutoSubmitted = false process.stdout.write(`\r\n ${chalk.hex(theme.amber)("◆")} cancelled\r\n`) continue } @@ -2291,10 +2381,16 @@ export async function chatLoop( footer, ) if (agentResult.aborted) { + voiceAutoSubmitted = false process.stdout.write(`\r\n ${chalk.hex(theme.amber)("◆")} cancelled\r\n`) continue } await addMessage(conversation.id, "assistant", agentResult.content) + if (voiceAutoSubmitted) { + voiceAutoSubmitted = false + process.stdout.write("\r\n") + await speakText(agentResult.content) + } lastUsage = agentResult.usage lastElapsed = agentResult.elapsed await maybeCompactConversation(conversation.id) @@ -2303,6 +2399,11 @@ export async function chatLoop( } await addMessage(conversation.id, "assistant", result.content) + if (voiceAutoSubmitted) { + voiceAutoSubmitted = false + process.stdout.write("\r\n") + await speakText(result.content) + } // Phase 8: in plan mode, persist the assistant's response to scratch // so /plan execute can pick it up. @@ -2326,6 +2427,7 @@ export async function chatLoop( await maybeCompactConversation(conversation.id) } catch (error: any) { const errMsg = error?.message ?? "Unknown error" + voiceAutoSubmitted = false process.stdout.write(`\r\n ${chalk.hex(theme.red)("◆")} ${chalk.hex(theme.red)(errMsg)}\r\n\n`) } finally { clearSkill() @@ -2394,6 +2496,7 @@ export async function startChat( console.log() const user = await getUserFromToken() + currentUser = user const conversation = await initConversation(user.id, conversationId, initialMode) await chatLoop(aiProvider, conversation, workspaceInfo) diff --git a/apps/supercode-cli/server/src/cli/commands/slashCommands/index.ts b/apps/supercode-cli/server/src/cli/commands/slashCommands/index.ts index 893543f..b176de5 100644 --- a/apps/supercode-cli/server/src/cli/commands/slashCommands/index.ts +++ b/apps/supercode-cli/server/src/cli/commands/slashCommands/index.ts @@ -9,7 +9,7 @@ import { theme, heavyDivider } from "src/cli/utils/tui.ts" import type { ModelProvider } from "src/cli/ai/provider.ts" export interface SlashCommandResult { - type: "model_change" | "help" | "unknown" | "exit" | "connect" | "context" | "compact" | "plan" | "scratch" | "skills" | "voice" | "verbose" | "message" | "clear" | "new_conversation" + type: "model_change" | "help" | "unknown" | "exit" | "connect" | "context" | "compact" | "plan" | "scratch" | "skills" | "voice" | "jarvis" | "verbose" | "message" | "clear" | "new_conversation" provider?: ModelProvider model?: string label?: string @@ -29,6 +29,7 @@ export const COMMANDS = [ { cmd: "/plan", desc: "Switch to plan mode (read-only)" }, { cmd: "/scratch", desc: "List/show/delete subagent artifacts in .super/scratch/" }, { cmd: "/voice", desc: "Capture voice input via microphone" }, + { cmd: "/jarvis", desc: "Wake Jarvis — start the daily workspace apps" }, { cmd: "/verbose", desc: "Toggle live tool call debug logs" }, { cmd: "/search", desc: "Search the web via Firecrawl" }, { cmd: "/scrape", desc: "Scrape a URL via Firecrawl" }, @@ -102,6 +103,9 @@ const handlers: Record Promise> = voice: async () => { return { type: "voice" } }, + jarvis: async () => { + return { type: "jarvis" } + }, verbose: async () => { return { type: "verbose" } }, diff --git a/apps/supercode-cli/server/src/voice/__tests__/jarvis.test.ts b/apps/supercode-cli/server/src/voice/__tests__/jarvis.test.ts new file mode 100644 index 0000000..e46676f --- /dev/null +++ b/apps/supercode-cli/server/src/voice/__tests__/jarvis.test.ts @@ -0,0 +1,78 @@ +import { test, expect } from "bun:test" +import { isJarvisWake, getWakePhrases, buildJarvisTargets } from "src/voice/jarvis.ts" + +test("default wake phrases cover bare Jarvis and wake up variants", () => { + const phrases = getWakePhrases() + expect(phrases).toContain("jarvis") + expect(phrases).toContain("jarvis wake up") + expect(phrases).toContain("jarvis wakeup") +}) + +test("isJarvisWake matches exact, suffixed, and punctuation variants", () => { + expect(isJarvisWake("jarvis wake up")).toBe(true) + expect(isJarvisWake("Jarvis wake up please")).toBe(true) + expect(isJarvisWake("Jarvis, wake up!")).toBe(true) + expect(isJarvisWake("JARVIS")).toBe(true) +}) + +test("isJarvisWake rejects ordinary commands", () => { + expect(isJarvisWake("open github")).toBe(false) + expect(isJarvisWake("hello there")).toBe(false) + expect(isJarvisWake("")).toBe(false) +}) + +test("isJarvisWake catches STT mispronunciations of Jarvis", () => { + expect(isJarvisWake("Java, wake up.")).toBe(true) + expect(isJarvisWake("Java wakeup")).toBe(true) + expect(isJarvisWake("jarve wake up please")).toBe(true) + expect(isJarvisWake("hervis wake")).toBe(true) + expect(isJarvisWake("hey jarvis wake up")).toBe(true) +}) + +test("isJarvisWake does not fire on a jarvis-like name without a wake signal", () => { + expect(isJarvisWake("java is a programming language")).toBe(false) + expect(isJarvisWake("java is good")).toBe(false) +}) + +test("buildJarvisTargets uses defaults for the daily workspace set", () => { + const targets = buildJarvisTargets() + const names = targets.map((t) => t.name) + expect(names).toContain("github") + expect(names).toContain("whatsapp") + expect(names).toContain("slack") + expect(names).toContain("linear") + expect(names).toContain("twitter") + const github = targets.find((t) => t.name === "github") + expect(github?.url).toBe("https://github.com/yashdev9274/supercli") + const slack = targets.find((t) => t.name === "slack") + expect(slack?.url).toContain("join.slack.com/t/supercodeai") + const linear = targets.find((t) => t.name === "linear") + expect(linear?.url).toBe("https://linear.app/supercodeai/projects/all") +}) + +test("warp target opens a new tab via the warp deep link", () => { + const warp = buildJarvisTargets().find((t) => t.name === "warp") + expect(warp?.uri).toBe( + "warp://action/new_tab?path=%2FUsers%2Fyashdewasthale%2Fdev%2Fsaas%2Fsupercli", + ) +}) + +test("whatsapp target opens the native macOS desktop app", () => { + const whatsapp = buildJarvisTargets().find((t) => t.name === "whatsapp") + expect(whatsapp?.app).toBe("WhatsApp") + expect(whatsapp?.url).toBe("https://web.whatsapp.com") +}) + +test("JARVIS_WAKE_PHRASE env overrides defaults", () => { + const prev = process.env.JARVIS_WAKE_PHRASE + process.env.JARVIS_WAKE_PHRASE = "ready set go" + try { + expect(getWakePhrases()).toContain("ready set go") + expect(isJarvisWake("ready set go now")).toBe(true) + // A jarvis-like name + wake keyword still fires even with a custom phrase. + expect(isJarvisWake("java wake up")).toBe(true) + } finally { + if (prev === undefined) delete process.env.JARVIS_WAKE_PHRASE + else process.env.JARVIS_WAKE_PHRASE = prev + } +}) diff --git a/apps/supercode-cli/server/src/voice/__tests__/speech.test.ts b/apps/supercode-cli/server/src/voice/__tests__/speech.test.ts index f2e3fb2..8698be2 100644 --- a/apps/supercode-cli/server/src/voice/__tests__/speech.test.ts +++ b/apps/supercode-cli/server/src/voice/__tests__/speech.test.ts @@ -7,6 +7,7 @@ describe("canVoiceCapture", () => { process.env = { PATH: origEnv.PATH, FFMPEG_PATH: origEnv.FFMPEG_PATH } delete process.env.ELEVENLABS_API_KEY delete process.env.GROQ_API_KEY + delete process.env.SMALLEST_API_KEY delete process.env.STT_PROVIDER }) @@ -14,31 +15,15 @@ describe("canVoiceCapture", () => { process.env = { ...origEnv } }) - it("returns ok=false when ELEVENLABS_API_KEY is missing and STT_PROVIDER=elevenlabs", async () => { - process.env.STT_PROVIDER = "elevenlabs" + it("returns ok=false when SMALLEST_API_KEY is missing", async () => { const { canVoiceCapture } = await import("../speech.ts") const result = canVoiceCapture() expect(result.ok).toBe(false) - expect(result.reason).toContain("ELEVENLABS_API_KEY") - }) - - it("returns ok=false when GROQ_API_KEY is missing and STT_PROVIDER=groq", async () => { - process.env.STT_PROVIDER = "groq" - const { canVoiceCapture } = await import("../speech.ts") - const result = canVoiceCapture() - expect(result.ok).toBe(false) - expect(result.reason).toContain("GROQ_API_KEY") - }) - - it("uses elevenlabs by default when STT_PROVIDER is unset", async () => { - const { canVoiceCapture } = await import("../speech.ts") - const result = canVoiceCapture() - expect(result.ok).toBe(false) - expect(result.reason).toContain("ELEVENLABS_API_KEY") + expect(result.reason).toContain("SMALLEST_API_KEY") }) it("returns ok=false with ffmpeg reason when ffmpeg is missing", async () => { - process.env.ELEVENLABS_API_KEY = "sk-test" + process.env.SMALLEST_API_KEY = "sk-test" process.env.FFMPEG_PATH = "/nonexistent/ffmpeg" const { canVoiceCapture } = await import("../speech.ts") const result = canVoiceCapture() @@ -59,26 +44,54 @@ describe("getSttProvider", () => { process.env = { ...origEnv } }) - it('returns "elevenlabs" when STT_PROVIDER is unset', async () => { + it('always returns "smallest" (Smallest.ai is the only STT provider)', async () => { const mod = await import("../speech.ts") - expect((mod as any).getSttProvider()).toBe("elevenlabs") + expect((mod as any).getSttProvider()).toBe("smallest") }) - it('returns "elevenlabs" when STT_PROVIDER is "elevenlabs"', async () => { - process.env.STT_PROVIDER = "elevenlabs" + it('ignores STT_PROVIDER and still returns "smallest"', async () => { + process.env.STT_PROVIDER = "groq" const mod = await import("../speech.ts") - expect((mod as any).getSttProvider()).toBe("elevenlabs") + expect((mod as any).getSttProvider()).toBe("smallest") }) +}) - it('returns "groq" when STT_PROVIDER is "groq"', async () => { - process.env.STT_PROVIDER = "groq" +describe("stripForSpeech", () => { + it("removes code fences and inline code", async () => { + const mod = await import("../speech.ts") + const out = (mod as any).stripForSpeech( + "Here is the fix:\n```ts\nconst x = 1\n```\nUse `y` instead.", + ) + expect(out).not.toContain("```") + expect(out).not.toContain("const x = 1") + expect(out).toContain("Here is the fix") + expect(out).toContain("y instead") + }) + + it("strips markdown symbols, links, and headings", async () => { const mod = await import("../speech.ts") - expect((mod as any).getSttProvider()).toBe("groq") + const out = (mod as any).stripForSpeech( + "# Title\n\n- one\n- two\n\nSee [docs](https://example.com) for **more**.", + ) + expect(out).not.toContain("#") + expect(out).not.toContain("https://example.com") + expect(out).toContain("docs") + expect(out).toContain("more") }) - it('returns "elevenlabs" for unknown STT_PROVIDER values', async () => { - process.env.STT_PROVIDER = "invalid" + it("truncates to SPEECH_MAX_CHARS", async () => { + const mod = await import("../speech.ts") + const long = "word ".repeat(2000) + const out = (mod as any).stripForSpeech(long) + expect(out.length).toBeLessThanOrEqual(3000) + }) +}) + +describe("isTtsAvailable", () => { + it("returns a boolean reflecting the platform (darwin only)", async () => { const mod = await import("../speech.ts") - expect((mod as any).getSttProvider()).toBe("elevenlabs") + const result = (mod as any).isTtsAvailable() + expect(typeof result).toBe("boolean") + expect(result).toBe(process.platform === "darwin") }) }) diff --git a/apps/supercode-cli/server/src/voice/jarvis.ts b/apps/supercode-cli/server/src/voice/jarvis.ts new file mode 100644 index 0000000..0dafc2c --- /dev/null +++ b/apps/supercode-cli/server/src/voice/jarvis.ts @@ -0,0 +1,191 @@ +import { spawn } from "node:child_process" + +const DEFAULT_WAKE_PHRASES = "jarvis wake up,jarvis wakeup,jarvis" + +function normalizeForWake(text: string): string { + return text + .toLowerCase() + .replace(/[^a-z0-9 ]/g, " ") + .replace(/\s+/g, " ") + .trim() +} + +// The phrases that wake Jarvis. Comma-separated, case-insensitive. +// Defaults cover "Jarvis wake up", "Jarvis wakeup" and a bare "Jarvis". +export function getWakePhrases(): string[] { + const raw = process.env.JARVIS_WAKE_PHRASE || DEFAULT_WAKE_PHRASES + return raw + .split(",") + .map((s) => normalizeForWake(s)) + .filter(Boolean) +} + +// Edit distance (Levenshtein) for fuzzy matching — lets us catch the way real +// STT mishears "Jarvis" ("Java, wake up", "jarve wake up", "hervis" …). +function editDistance(a: string, b: string): number { + const m = a.length + const n = b.length + const d: number[][] = Array.from({ length: m + 1 }, () => Array(n + 1).fill(0)) + for (let i = 0; i <= m; i++) d[i]![0] = i + for (let j = 0; j <= n; j++) d[0]![j] = j + for (let i = 1; i <= m; i++) { + for (let j = 1; j <= n; j++) { + const cost = a.charAt(i - 1) === b.charAt(j - 1) ? 0 : 1 + d[i]![j] = Math.min(d[i - 1]![j]! + 1, d[i]![j - 1]! + 1, d[i - 1]![j - 1]! + cost) + } + } + return d[m]![n]! +} + +// Sounded-out aliases the wake name could come back as. "java" is the big one +// (Jarvis → Java), so it's first-class here rather than left to fuzzy math. +const NAME_ALIASES = new Set([ + "java", "javis", "jarve", "jerus", "hervis", "harvis", "jarvee", "jervis", +]) + +function isNameToken(word: string): boolean { + return NAME_ALIASES.has(word) || word.startsWith("jar") || editDistance(word, "jarvis") <= 2 +} + +const WAKE_ALIASES = new Set(["wake", "wakeup", "woke", "awake", "activate"]) + +function isWakeToken(word: string): boolean { + return ( + WAKE_ALIASES.has(word) || + editDistance(word, "wake") <= 1 || + editDistance(word, "wakeup") <= 1 + ) +} + +// True when the transcribed utterance is a wake call. It matches exactly first +// (a wake phrase at the start of the utterance, so a bare "Jarvis" counts), then +// falls back to a fuzzy name+signal check so mispronunciations the STT renders +// as "Java, wake up." or "jarve wakeup" still trigger Jarvis. +export function isJarvisWake(text: string): boolean { + const norm = normalizeForWake(text) + if (!norm) return false + const tokens = norm.split(" ") + + // Strict path — any wake phrase is a prefix of the utterance. + if (getWakePhrases().some((p) => norm === p || norm.startsWith(`${p} `))) return true + + // Fuzzy path: a jarvis-like name near the start, then a wake keyword. + // Tolerate a leading filler word ("hey", "okay", "um") before the name. + const idx = tokens.slice(0, 3).findIndex((w) => isNameToken(w)) + if (idx === -1) return false + const rest = tokens.slice(idx + 1) + // Require the wake keyword reasonably close after the name (within 4 words). + return rest.slice(0, 4).some(isWakeToken) +} + +export interface JarvisTarget { + name: string + url: string + // When set (macOS), open the native app (e.g. WhatsApp desktop) instead of a + // browser tab. Ignored on non-macOS, which falls back to the URL. + app?: string + // When set, open this raw URI directly (custom scheme such as Warp's deep + // link) instead of handing the URL to a browser/app. + uri?: string +} + +function envOr(key: string, fallback: string): string { + const v = process.env[key]?.trim() + return v || fallback +} + +// The daily-startup set: GitHub, WhatsApp, Slack workspace, Linear workspace, +// Twitter, a new Warp terminal tab at the repo root, then any extras in +// JARVIS_URLS. Set JARVIS_SLACK_URL and JARVIS_LINEAR_URL to your real +// workspace URLs. WhatsApp opens in the macOS desktop app (WhatsApp.app) when +// present, so it lands native instead of the browser. Everything else opens in +// the configured browser (JARVIS_BROWSER, default "Dia") rather than the OS +// default. The Warp tab uses Warp's deep link so it opens a fresh tab at +// JARVIS_WARP_DIR instead of a new window. +export function buildJarvisTargets(): JarvisTarget[] { + const targets: JarvisTarget[] = [] + const push = (name: string, url: string, app?: string) => { + if (url) targets.push(app ? { name, url, app } : { name, url }) + } + + push("github", envOr("JARVIS_GITHUB_URL", "https://github.com/yashdev9274/supercli")) + push( + "whatsapp", + envOr("JARVIS_WHATSAPP_URL", "https://web.whatsapp.com"), + envOr("JARVIS_WHATSAPP_APP", "WhatsApp"), + ) + push( + "slack", + envOr( + "JARVIS_SLACK_URL", + "https://join.slack.com/t/supercodeai/shared_invite/zt-43enen35h-glS1qR854YB~HUg2AIW0vg", + ), + ) + push("linear", envOr("JARVIS_LINEAR_URL", "https://linear.app/supercodeai/projects/all")) + push("twitter", envOr("JARVIS_TWITTER_URL", "https://x.com")) + + const warpDir = envOr("JARVIS_WARP_DIR", "/Users/yashdewasthale/dev/saas/supercli") + if (warpDir) { + targets.push({ + name: "warp", + url: warpDir, + uri: `warp://action/new_tab?path=${encodeURIComponent(warpDir)}`, + }) + } + + const extra = process.env.JARVIS_URLS || "" + extra + .split(",") + .map((u) => u.trim()) + .filter(Boolean) + .forEach((url, i) => push(`app ${i + 1}`, url)) + + return targets +} + +function openUrl(url: string, app?: string, uri?: string): boolean { + if (process.platform === "darwin") { + if (uri) { + // Custom scheme deep link — `open warp://…` hands it to the handler. + try { + const proc = spawn("open", [uri], { stdio: "ignore", detached: true }) + proc.unref() + return true + } catch { + return false + } + } + // macOS: `open -a ` opens the URL in a specific browser/app. + const browser = envOr("JARVIS_BROWSER", "Dia") + const args = app ? ["-a", app, url] : ["-a", browser, url] + try { + const proc = spawn("open", args, { stdio: "ignore", detached: true }) + proc.unref() + return true + } catch { + return false + } + } + + const cmd = process.platform === "win32" ? "start" : "xdg-open" + try { + const proc = spawn(cmd, [uri ?? url], { stdio: "ignore", detached: true }) + proc.unref() + return true + } catch { + return false + } +} + +// Launch every configured Jarvis target. WhatsApp opens in its desktop app on +// macOS, Warp opens a new tab at the repo, and the rest open in the Dia +// browser. Returns the names that opened successfully and the ones that failed. +export function runJarvisStart(): { opened: string[]; failed: string[] } { + const opened: string[] = [] + const failed: string[] = [] + for (const target of buildJarvisTargets()) { + if (openUrl(target.url, target.app, target.uri)) opened.push(target.name) + else failed.push(target.name) + } + return { opened, failed } +} \ No newline at end of file diff --git a/apps/supercode-cli/server/src/voice/speech.ts b/apps/supercode-cli/server/src/voice/speech.ts index e7d67ed..3fba497 100644 --- a/apps/supercode-cli/server/src/voice/speech.ts +++ b/apps/supercode-cli/server/src/voice/speech.ts @@ -1,7 +1,7 @@ import { spawnSync, spawn, type ChildProcess } from "child_process" import { tmpdir } from "os" import { join } from "path" -import { unlinkSync, readFileSync } from "fs" +import { unlinkSync, readFileSync, writeFileSync } from "fs" import { randomUUID } from "crypto" import { getStoredToken } from "src/lib/token" @@ -17,16 +17,20 @@ const STT_LANGUAGE = process.env.STT_LANGUAGE || "en" const GROQ_URL = "https://api.groq.com/openai/v1/audio/transcriptions" const GROQ_MODEL = process.env.GROQ_MODEL || "whisper-large-v3-turbo" - +/* smallest.ai provider (Pulse STT) */ +const SMALLEST_URL = "https://api.smallest.ai/waves/v1/stt/" +const SMALLEST_MODEL = process.env.SMALLEST_MODEL || "pulse-pro" +const SMALLEST_LANGUAGE = process.env.SMALLEST_LANGUAGE || STT_LANGUAGE || "en" const DEFAULT_MAX_DURATION_MS = 4_000 let activeFfmpegProcess: ChildProcess | null = null -export function getSttProvider(): "elevenlabs" | "groq" { - const raw = process.env.STT_PROVIDER || "elevenlabs" - if (raw === "groq") return "groq" - return "elevenlabs" +export type SttProvider = "elevenlabs" | "groq" | "smallest" + +// Voice capture uses Smallest.ai (Pulse STT) exclusively. +export function getSttProvider(): SttProvider { + return "smallest" } export function stopCapture(): void { @@ -51,14 +55,8 @@ export function canVoiceCapture(): { } { if (!isFfmpegAvailable()) return { ok: false, reason: `ffmpeg not found at ${getFfmpegPath()}` } - const provider = getSttProvider() - if (provider === "groq") { - if (!process.env.GROQ_API_KEY && !process.env.SUPERCODE_SERVER_URL) - return { ok: false, reason: "GROQ_API_KEY not set and no server proxy configured" } - } else { - if (!process.env.ELEVENLABS_API_KEY && !process.env.SUPERCODE_SERVER_URL) - return { ok: false, reason: "ELEVENLABS_API_KEY not set and no server proxy configured" } - } + if (!process.env.SMALLEST_API_KEY && !process.env.SUPERCODE_SERVER_URL) + return { ok: false, reason: "SMALLEST_API_KEY not set and no server proxy configured" } return { ok: true } } @@ -197,6 +195,48 @@ export async function transcribeGroq(filePath: string): Promise { return data.text ?? "" } +/* smallest.ai provider (Pulse STT) */ +export async function transcribeSmallest(filePath: string): Promise { + const apiKey = process.env.SMALLEST_API_KEY + if (!apiKey) throw new Error("SMALLEST_API_KEY not configured") + + const params = new URLSearchParams({ + model: SMALLEST_MODEL, + language: SMALLEST_LANGUAGE, + }) + + const start = performance.now() + const res = await fetch(`${SMALLEST_URL}?${params.toString()}`, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/octet-stream", + }, + body: Bun.file(filePath), + signal: AbortSignal.timeout(30_000), + }) + + const body = await res.text().catch(() => "") + const elapsed = ((performance.now() - start) / 1000).toFixed(2) + + if (!res.ok) { + throw new Error(`Smallest transcription error ${res.status}: ${body}`) + } + + let data: { transcription?: string } + try { + data = JSON.parse(body) as { transcription?: string } + } catch { + throw new Error(`Smallest transcription invalid JSON: ${body}`) + } + + if (data.transcription === undefined) { + throw new Error(`Smallest transcription missing "transcription": ${body}`) + } + + return data.transcription +} + async function transcribeViaServer(filePath: string): Promise { const serverUrl = process.env.SUPERCODE_SERVER_URL || "https://supercode-8w7e.onrender.com" const token = await getStoredToken() @@ -227,18 +267,99 @@ async function transcribeViaServer(filePath: string): Promise { } export async function transcribeAudio(filePath: string): Promise { - const provider = getSttProvider() + if (process.env.SMALLEST_API_KEY) return transcribeSmallest(filePath) + return transcribeViaServer(filePath) +} + +// ─── Text-to-speech (spoken reply) ────────────────────────────────────────── +const ELEVENLABS_TTS_URL = "https://api.elevenlabs.io/v1/text-to-speech" +const ELEVENLABS_VOICE_ID = process.env.ELEVENLABS_VOICE_ID || "21m00Tcm4TlvDq8ikWAM" +const ELEVENLABS_TTS_MODEL = process.env.ELEVENLABS_TTS_MODEL || "eleven_turbo_v2_5" +const SPEECH_MAX_CHARS = 3000 +const VOICE_REPLY_ENABLED = (process.env.VOICE_REPLY ?? "on").toLowerCase() !== "off" + +export function isTtsAvailable(): boolean { + if (!VOICE_REPLY_ENABLED) return false + // Spoken replies are played back with macOS `afplay` / `say` + return process.platform === "darwin" +} - if (provider === "groq") { - if (process.env.GROQ_API_KEY) return transcribeGroq(filePath) - return transcribeViaServer(filePath) +// Strip markdown so the reply reads naturally instead of reading code verbatim. +export function stripForSpeech(text: string): string { + return text + .replace(/```[\s\S]*?```/g, " ") + .replace(/`([^`]*)`/g, "$1") + .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") + .replace(/^#{1,6}\s+/gm, "") + .replace(/^\s*[-*+]\s+/gm, "") + .replace(/[#*_~>|]/g, "") + .replace(/^\s{2,}/gm, "") + .replace(/\n{3,}/g, "\n\n") + .trim() + .slice(0, SPEECH_MAX_CHARS) +} + +function playAudioFile(filePath: string): Promise { + return new Promise((resolve) => { + const proc = spawn("afplay", [filePath]) + proc.on("exit", () => { + try { unlinkSync(filePath) } catch {} + resolve() + }) + proc.on("error", () => { + try { unlinkSync(filePath) } catch {} + resolve() + }) + }) +} + +function speakWithSay(text: string): Promise { + return new Promise((resolve) => { + const proc = spawn("say", []) + proc.on("exit", () => resolve()) + proc.on("error", () => resolve()) + proc.stdin?.write(text) + proc.stdin?.end() + }) +} + +export async function speakText(text: string): Promise { + if (!isTtsAvailable()) return + const clean = stripForSpeech(text) + if (!clean) return + + const apiKey = process.env.ELEVENLABS_API_KEY + if (!apiKey) { + // Local fallback — no API key needed + await speakWithSay(clean) + return } - if (process.env.ELEVENLABS_API_KEY) return transcribeElevenLabs(filePath) - return transcribeViaServer(filePath) + try { + const res = await fetch(`${ELEVENLABS_TTS_URL}/${ELEVENLABS_VOICE_ID}`, { + method: "POST", + headers: { + "xi-api-key": apiKey, + "Content-Type": "application/json", + }, + body: JSON.stringify({ text: clean, model_id: ELEVENLABS_TTS_MODEL }), + signal: AbortSignal.timeout(30_000), + }) + if (!res.ok) { + const body = await res.text().catch(() => "") + throw new Error(`ElevenLabs TTS error ${res.status}: ${body}`) + } + const bytes = new Uint8Array(await res.arrayBuffer()) + const tmpFile = join(tmpdir(), `tts-${randomUUID()}.mp3`) + writeFileSync(tmpFile, bytes) + await playAudioFile(tmpFile) + } catch { + // Fall back to local `say` on any network/fetch failure + await speakWithSay(clean) + } } -const SOUND_DESCRIPTION_RE = /\([^)]*?(?:noise|clicking|static|background|sound|audio|speaking|unintelligible|laughs?|coughs?|clears?\s+(?:throat|voice)|throat|pause|music|beep|tone|silence|indistinct|foreign|applause|sniffling|sighs?|breathing|rustling|mumbling|chatter|echo)[^)]*?\)/gi +const SOUND_DESCRIPTION_RE = /\([^)]*?(?:noise|clicking|static|static|background|sound|audio|speaking|unintelligible|laughs?|coughs?|clears?\s+(?:throat|voice)|throat|pause|music|beep|tone|silence|indistinct|foreign|applause|sniffling|sighs?|breathing|rustling|mumbling|chatter|echo)[^)]*?\)/gi function sanitizeTranscription(text: string): string { return text.replace(SOUND_DESCRIPTION_RE, "").trim()