diff --git a/apps/desktop/src/features/control-panel/ControlPanelApp.tsx b/apps/desktop/src/features/control-panel/ControlPanelApp.tsx index 11cf0932..eee44cd8 100644 --- a/apps/desktop/src/features/control-panel/ControlPanelApp.tsx +++ b/apps/desktop/src/features/control-panel/ControlPanelApp.tsx @@ -45,6 +45,11 @@ import { type SecurityBudgetDisplaySettings, } from "@/services/securityBudgetDisplayService"; import { loadDesktopRuntimeDefaultsSnapshot, loadHydratedSettings } from "@/services/settingsService"; +import { + resolveVoiceNotificationVoicePreset, + VOICE_NOTIFICATION_VOICE_PRESET_OPTIONS, + type VoiceNotificationVoicePreset, +} from "@/services/voiceNotificationConfig"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { buildDesktopOnboardingPresentation } from "@/features/onboarding/onboardingGeometry"; import { @@ -510,11 +515,13 @@ function ToggleLine({ checked, description, disabled = false, label, onCheckedCh function ChoiceGroup({ className, + disabled = false, options, value, onValueChange, }: { className?: string; + disabled?: boolean; options: readonly ChoiceOption[]; value: T; onValueChange: (value: T) => void; @@ -522,7 +529,7 @@ function ChoiceGroup({ const classes = ["control-panel-shell__choice-group", className].filter(Boolean).join(" "); return ( -
+
{options.map((option) => { const checked = option.value === value; @@ -532,6 +539,7 @@ function ChoiceGroup({ type="button" role="radio" aria-checked={checked} + disabled={disabled} className="control-panel-shell__choice-option" data-state={checked ? "checked" : "unchecked"} onClick={() => onValueChange(option.value)} @@ -1281,7 +1289,7 @@ export function ControlPanelApp() { updateSettings((current) => ({ @@ -1296,19 +1304,20 @@ export function ControlPanelApp() { - + options={VOICE_NOTIFICATION_VOICE_PRESET_OPTIONS} + value={resolveVoiceNotificationVoicePreset(draft.settings.general.voice_type)} + onValueChange={(value: VoiceNotificationVoicePreset) => updateSettings((current) => ({ ...current, settings: { ...current.settings, - general: { ...current.settings.general, voice_type: event.target.value }, + general: { ...current.settings.general, voice_type: value }, }, })) } diff --git a/apps/desktop/src/features/shell-ball/ShellBallApp.tsx b/apps/desktop/src/features/shell-ball/ShellBallApp.tsx index afc88a33..bcde0cc7 100644 --- a/apps/desktop/src/features/shell-ball/ShellBallApp.tsx +++ b/apps/desktop/src/features/shell-ball/ShellBallApp.tsx @@ -13,6 +13,7 @@ import { ShellBallAttachmentTray } from "./components/ShellBallAttachmentTray"; import { ShellBallBubbleZone } from "./components/ShellBallBubbleZone"; import { ShellBallInputBar } from "./components/ShellBallInputBar"; import { ShellBallVoiceHints } from "./components/ShellBallVoiceHints"; +import { areShellBallSelectionSnapshotsEqual } from "./selection/selection.provider"; import type { ShellBallSelectionSnapshot } from "./selection/selection.types"; import { useShellBallInteraction } from "./useShellBallInteraction"; import { getShellBallMotionConfig } from "./shellBall.motion"; @@ -39,6 +40,12 @@ import { setShellBallPressLock, } from "../../platform/shellBallWindow"; import { loadSettings } from "../../services/settingsService"; +import { + speakShellBallClipboardDetectedNotification, + speakShellBallSelectionDetectedNotification, + speakShellBallSelectionClickGreeting, + speakShellBallStartupGreeting, +} from "../../services/voiceNotificationService"; import { openOrFocusDesktopWindow } from "../../platform/windowController"; import { buildDesktopOnboardingPresentation } from "@/features/onboarding/onboardingGeometry"; import { @@ -391,9 +398,12 @@ export function ShellBallApp() { const clipboardPromptClearTimeoutRef = useRef(null); const selectionPromptClearTimeoutRef = useRef(null); const selectionPromptExpiryTimeoutRef = useRef(null); + const lastSpokenSelectionPromptRef = useRef(null); + const lastSpokenClipboardPromptTextRef = useRef(null); const previousVisualStateRef = useRef(visualState); const transitionQueueRef = useRef(Promise.resolve()); const edgeDockRevealHideTimeoutRef = useRef(null); + const startupGreetingPlayedRef = useRef(false); const dragDropHandlersRef = useRef<{ handleDroppedFiles: (paths: string[]) => Promise | void; }>({ @@ -482,6 +492,16 @@ export function ShellBallApp() { window.removeEventListener("storage", syncFloatingBallSizeFromStorage); }; }, []); + + useEffect(() => { + if (startupGreetingPlayedRef.current) { + return; + } + + startupGreetingPlayedRef.current = true; + void speakShellBallStartupGreeting(); + }, []); + const { ballDockSettling, ballDragActive, @@ -903,6 +923,16 @@ export function ShellBallApp() { setTextDragActive(false); }, []); + const clearSelectionPrompt = useCallback(() => { + lastSpokenSelectionPromptRef.current = null; + setSelectionPrompt(null); + }, []); + + const clearClipboardPrompt = useCallback(() => { + lastSpokenClipboardPromptTextRef.current = null; + setClipboardPrompt(null); + }, []); + const handleWindowTextDrag = useCallback((event: DragEvent) => { if (!shouldAcceptShellBallTextDrop(event.dataTransfer)) { clearTextDragState(); @@ -952,9 +982,9 @@ export function ShellBallApp() { window.clearTimeout(selectionPromptClearTimeoutRef.current); selectionPromptClearTimeoutRef.current = null; } - setSelectionPrompt(null); + clearSelectionPrompt(); } - }, [visualState]); + }, [clearSelectionPrompt, visualState]); useEffect(() => { if (selectionPrompt === null) { @@ -967,19 +997,19 @@ export function ShellBallApp() { const updatedAtMs = resolveShellBallSelectionUpdatedAtMs(selectionPrompt.updated_at); if (updatedAtMs === null) { - setSelectionPrompt(null); + clearSelectionPrompt(); return; } const remainingMs = updatedAtMs + SHELL_BALL_SELECTION_PROMPT_WINDOW_MS - Date.now(); if (remainingMs <= 0) { - setSelectionPrompt(null); + clearSelectionPrompt(); return; } selectionPromptExpiryTimeoutRef.current = window.setTimeout(() => { selectionPromptExpiryTimeoutRef.current = null; - setSelectionPrompt(null); + clearSelectionPrompt(); }, remainingMs); return () => { @@ -988,7 +1018,7 @@ export function ShellBallApp() { selectionPromptExpiryTimeoutRef.current = null; } }; - }, [selectionPrompt]); + }, [clearSelectionPrompt, selectionPrompt]); useEffect(() => { if (clipboardPrompt === null) { @@ -1001,13 +1031,13 @@ export function ShellBallApp() { const remainingMs = clipboardPrompt.expiresAt - Date.now(); if (remainingMs <= 0) { - setClipboardPrompt(null); + clearClipboardPrompt(); return; } clipboardPromptClearTimeoutRef.current = window.setTimeout(() => { clipboardPromptClearTimeoutRef.current = null; - setClipboardPrompt(null); + clearClipboardPrompt(); }, remainingMs); return () => { @@ -1016,7 +1046,7 @@ export function ShellBallApp() { clipboardPromptClearTimeoutRef.current = null; } }; - }, [clipboardPrompt]); + }, [clearClipboardPrompt, clipboardPrompt]); useEffect(() => { const currentWindow = getCurrentWindow(); @@ -1036,6 +1066,11 @@ export function ShellBallApp() { selectionPromptClearTimeoutRef.current = null; } + if (!areShellBallSelectionSnapshotsEqual(lastSpokenSelectionPromptRef.current, payload.snapshot)) { + lastSpokenSelectionPromptRef.current = payload.snapshot; + void speakShellBallSelectionDetectedNotification(); + } + setSelectionPrompt(payload.snapshot); return; } @@ -1049,7 +1084,7 @@ export function ShellBallApp() { // a short debounce before retiring the alert opportunity. selectionPromptClearTimeoutRef.current = window.setTimeout(() => { selectionPromptClearTimeoutRef.current = null; - setSelectionPrompt(null); + clearSelectionPrompt(); }, SHELL_BALL_SELECTION_PROMPT_CLEAR_DELAY_MS); }) .then((unlisten) => { @@ -1069,7 +1104,7 @@ export function ShellBallApp() { } cleanup?.(); }; - }, []); + }, [clearSelectionPrompt]); useEffect(() => { const currentWindow = getCurrentWindow(); @@ -1084,10 +1119,15 @@ export function ShellBallApp() { void currentWindow .listen(shellBallWindowSyncEvents.clipboardSnapshot, ({ payload }) => { if (payload.text.trim() === "") { - setClipboardPrompt(null); + clearClipboardPrompt(); return; } + if (lastSpokenClipboardPromptTextRef.current !== payload.text) { + lastSpokenClipboardPromptTextRef.current = payload.text; + void speakShellBallClipboardDetectedNotification(); + } + setClipboardPrompt({ text: payload.text, expiresAt: Date.now() + SHELL_BALL_CLIPBOARD_PROMPT_WINDOW_MS, @@ -1106,7 +1146,7 @@ export function ShellBallApp() { disposed = true; cleanup?.(); }; - }, []); + }, [clearClipboardPrompt]); const handleMascotPrimaryAction = useCallback(() => { if (isShellBallSelectionPromptActive(selectionPrompt)) { @@ -1120,29 +1160,38 @@ export function ShellBallApp() { return; } - setSelectionPrompt(null); + void speakShellBallSelectionClickGreeting(); + clearSelectionPrompt(); void handleCoordinatorSelectedTextPrompt(activeSelectionPrompt); return; } if (selectionPrompt !== null) { - setSelectionPrompt(null); + clearSelectionPrompt(); return; } if (clipboardPrompt !== null) { if (!isShellBallClipboardPromptActive(clipboardPrompt)) { - setClipboardPrompt(null); + clearClipboardPrompt(); return; } - setClipboardPrompt(null); + clearClipboardPrompt(); void handleCoordinatorClipboardPrompt(clipboardPrompt.text); return; } handlePrimaryClick(); - }, [clipboardPrompt, handleCoordinatorClipboardPrompt, handleCoordinatorSelectedTextPrompt, handlePrimaryClick, selectionPrompt]); + }, [ + clearClipboardPrompt, + clearSelectionPrompt, + clipboardPrompt, + handleCoordinatorClipboardPrompt, + handleCoordinatorSelectedTextPrompt, + handlePrimaryClick, + selectionPrompt, + ]); const handleDockAwareRegionEnter = useCallback((event: ReactPointerEvent) => { void event; diff --git a/apps/desktop/src/features/shell-ball/components/ShellBallMascot.tsx b/apps/desktop/src/features/shell-ball/components/ShellBallMascot.tsx index 4d899203..0727784f 100644 --- a/apps/desktop/src/features/shell-ball/components/ShellBallMascot.tsx +++ b/apps/desktop/src/features/shell-ball/components/ShellBallMascot.tsx @@ -73,7 +73,9 @@ export function getShellBallMascotHotspotGestureAction(input: { } if (input.gesture === "single_click") { - return input.selectionIndicatorVisible || input.alertOpportunityAvailable ? "primary_click" : "noop"; + return input.selectionIndicatorVisible || input.alertOpportunityAvailable || input.visualState === "idle" + ? "primary_click" + : "noop"; } if (canTriggerShellBallMascotSecondaryGestures(input.visualState)) { diff --git a/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts b/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts index 245a308b..d8031d7f 100644 --- a/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts +++ b/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts @@ -28,6 +28,14 @@ import { } from "./shellBall.interaction"; import { getShellBallMotionConfig } from "./shellBall.motion"; import { collectShellBallSpeechTranscript, composeShellBallSpeechDraft } from "./shellBall.speech"; +import { + resolveShellBallClipboardDetectedNotificationText, + resolveShellBallSelectionDetectedNotificationText, + resolveDeliveryReadyVoiceNotificationText, + resolveShellBallSelectionClickGreetingText, + resolveShellBallStartupGreetingText, + resolveVoiceNotificationVoice, +} from "../../services/voiceNotificationService"; import { compactPageContext, mapDesktopWindowSnapshotToPageContext, @@ -2532,6 +2540,74 @@ test("shell-ball speech transcript collection merges recognition chunks", () => ); }); +test("voice notification helpers keep startup greeting, detection prompts, selection click greeting, and formal delivery reminders short", () => { + assert.equal(resolveShellBallStartupGreetingText(), "CialloClaw 已启动"); + assert.equal(resolveShellBallSelectionDetectedNotificationText(0), "发现一段高亮文字啦"); + assert.equal(resolveShellBallSelectionDetectedNotificationText(0.2), "我接住新选中的内容啦"); + assert.equal(resolveShellBallSelectionDetectedNotificationText(0.4), "新的选中内容到啦"); + assert.equal(resolveShellBallSelectionDetectedNotificationText(0.6), "这段选中文字交给我吧"); + assert.equal(resolveShellBallSelectionDetectedNotificationText(0.8), "我看到你圈出的这段话啦"); + assert.equal(resolveShellBallSelectionClickGreetingText(), "Ciallo"); + assert.equal(resolveShellBallClipboardDetectedNotificationText(0), "剪贴板刚刚更新啦"); + assert.equal(resolveShellBallClipboardDetectedNotificationText(0.2), "我接住新复制的内容啦"); + assert.equal(resolveShellBallClipboardDetectedNotificationText(0.4), "新的剪贴板内容到啦"); + assert.equal(resolveShellBallClipboardDetectedNotificationText(0.6), "这段复制内容交给我吧"); + assert.equal(resolveShellBallClipboardDetectedNotificationText(0.8), "我看到你刚复制的小纸条啦"); + assert.equal( + resolveDeliveryReadyVoiceNotificationText({ + task_id: "task-bubble-delivery", + delivery_result: { + type: "bubble", + }, + } as any), + null, + ); + assert.equal( + resolveDeliveryReadyVoiceNotificationText({ + task_id: "task-formal-delivery", + delivery_result: { + type: "task_detail", + }, + } as any), + "任务结果已准备好", + ); +}); + +test("voice notification helpers prefer the saved voice type and matching locale", () => { + const resolvedDefaultFemale = resolveVoiceNotificationVoice({ + language: "zh-CN", + voiceType: "default_female", + voices: [ + { lang: "en-US", name: "Alloy" }, + { lang: "ja-JP", name: "Sakura Female" }, + { lang: "zh-CN", name: "Xiaoyi" }, + { lang: "zh-CN", name: "Xiaoxiao" }, + ], + }); + assert.equal(resolvedDefaultFemale?.name, "Xiaoxiao"); + + const resolvedSoftGirl = resolveVoiceNotificationVoice({ + language: "zh-CN", + voiceType: "soft_girl", + voices: [ + { lang: "zh-CN", name: "Tingting" }, + { lang: "zh-CN", name: "Tongtong" }, + { lang: "zh-CN", name: "Xiaohan" }, + ], + }); + assert.equal(resolvedSoftGirl?.name, "Xiaohan"); + + const resolvedCustomVoice = resolveVoiceNotificationVoice({ + language: "zh-CN", + voiceType: "voice_nebula", + voices: [ + { lang: "zh-CN", name: "voice_nebula" }, + { lang: "zh-CN", name: "Xiaoxiao" }, + ], + }); + assert.equal(resolvedCustomVoice?.name, "voice_nebula"); +}); + test("shell-ball voice recognition final state routes final transcript out of the input draft and restores draft on cancel", () => { assert.deepEqual( resolveShellBallVoiceRecognitionFinalState({ @@ -8173,7 +8249,7 @@ test("shell-ball floating size normalization falls back to medium", () => { assert.equal(normalizeShellBallFloatingSize(undefined), "medium"); }); -test("shell-ball mascot hotspot policy only opens primary click for selected-text prompts", () => { +test("shell-ball mascot hotspot policy opens primary click for idle acknowledgement and prompt intake", () => { assert.equal( getShellBallMascotHotspotGestureAction({ visualState: "voice_locked", @@ -8189,7 +8265,7 @@ test("shell-ball mascot hotspot policy only opens primary click for selected-tex gesture: "single_click", suppressed: false, }), - "noop", + "primary_click", ); assert.equal( @@ -8870,6 +8946,60 @@ test("shell-ball routes active resumable text follow-ups through task steer", () assert.match(coordinatorSource, /message: submittedText/); }); +test("shell-ball voice notifications are consumed locally from startup, selection and clipboard prompts, selection acceptance, and formal task notifications", () => { + const appSource = readFileSync(resolve(desktopRoot, "src/features/shell-ball/ShellBallApp.tsx"), "utf8"); + const mascotSource = readFileSync(resolve(desktopRoot, "src/features/shell-ball/components/ShellBallMascot.tsx"), "utf8"); + const coordinatorSource = readFileSync(resolve(desktopRoot, "src/features/shell-ball/useShellBallCoordinator.ts"), "utf8"); + const controlPanelSource = readFileSync(resolve(desktopRoot, "src/features/control-panel/ControlPanelApp.tsx"), "utf8"); + const voiceConfigSource = readFileSync(resolve(desktopRoot, "src/services/voiceNotificationConfig.ts"), "utf8"); + const voiceServiceSource = readFileSync(resolve(desktopRoot, "src/services/voiceNotificationService.ts"), "utf8"); + + assert.match(appSource, /areShellBallSelectionSnapshotsEqual/); + assert.match(appSource, /speakShellBallClipboardDetectedNotification/); + assert.match(appSource, /speakShellBallSelectionDetectedNotification/); + assert.match(appSource, /speakShellBallSelectionClickGreeting/); + assert.match(appSource, /speakShellBallStartupGreeting/); + assert.match(appSource, /const startupGreetingPlayedRef = useRef\(false\);/); + assert.match(appSource, /const lastSpokenSelectionPromptRef = useRef\(null\);/); + assert.match(appSource, /const lastSpokenClipboardPromptTextRef = useRef\(null\);/); + assert.match(appSource, /startupGreetingPlayedRef\.current = true;\s*void speakShellBallStartupGreeting\(\);/); + assert.match(appSource, /void speakShellBallSelectionDetectedNotification\(\);/); + assert.match(appSource, /void speakShellBallSelectionClickGreeting\(\);/); + assert.match(appSource, /void speakShellBallClipboardDetectedNotification\(\);/); + assert.doesNotMatch(appSource, /speakShellBallIdleGreeting/); + assert.match(mascotSource, /input\.selectionIndicatorVisible \|\| input\.alertOpportunityAvailable \|\| input\.visualState === "idle"/); + assert.match(controlPanelSource, /VOICE_NOTIFICATION_VOICE_PRESET_OPTIONS/); + assert.match(controlPanelSource, /resolveVoiceNotificationVoicePreset\(draft\.settings\.general\.voice_type\)/); + assert.match(voiceConfigSource, /label: "小女孩", value: "default_female"/); + assert.match(voiceConfigSource, /label: "软萌", value: "soft_girl"/); + assert.match(voiceConfigSource, /label: "元气", value: "bright_girl"/); + assert.match(voiceConfigSource, /label: "少年", value: "default_male"/); + assert.match(coordinatorSource, /import \{ speakApprovalPendingNotification, speakDeliveryReadyNotification \} from "@\/services\/voiceNotificationService";/); + assert.match(coordinatorSource, /speakApprovalPendingNotification\(\{\s*approval_request: input\.approvalRequest,\s*task_id: input\.taskId,\s*\}\);/); + assert.match(coordinatorSource, /speakDeliveryReadyNotification\(\{\s*delivery_result: input\.deliveryResult,\s*task_id: input\.taskId,\s*\}\);/); + assert.match(voiceServiceSource, /const STARTUP_GREETING_TEXT = "CialloClaw 已启动";/); + assert.match(voiceServiceSource, /const SELECTION_DETECTED_TEXT_1 = "发现一段高亮文字啦";/); + assert.match(voiceServiceSource, /const SELECTION_DETECTED_TEXT_5 = "我看到你圈出的这段话啦";/); + assert.match(voiceServiceSource, /const SELECTION_CLICK_GREETING_TEXT = "Ciallo";/); + assert.match(voiceServiceSource, /const CLIPBOARD_DETECTED_TEXT_1 = "剪贴板刚刚更新啦";/); + assert.match(voiceServiceSource, /const CLIPBOARD_DETECTED_TEXT_5 = "我看到你刚复制的小纸条啦";/); + assert.match(voiceServiceSource, /const APPROVAL_PENDING_TEXT = "有一个操作需要你确认";/); + assert.match(voiceServiceSource, /const DELIVERY_READY_TEXT = "任务结果已准备好";/); + assert.match(voiceServiceSource, /resolveShellBallSelectionDetectedNotificationText/); + assert.match(voiceServiceSource, /resolveShellBallClipboardDetectedNotificationText/); + assert.match(voiceServiceSource, /const VOICE_LIST_READY_TIMEOUT_MS = 600;/); + assert.match(voiceServiceSource, /let latestVoiceNotificationRequestId = 0;/); + assert.match(voiceServiceSource, /const requestId = \+\+latestVoiceNotificationRequestId;/); + assert.match(voiceServiceSource, /if \(requestId !== latestVoiceNotificationRequestId\) \{\s*return false;\s*\}/); + assert.match(voiceServiceSource, /const latestSettings = getVoiceNotificationSettings\(\);/); + assert.match(voiceServiceSource, /if \(!latestSettings\.voice_notification_enabled\) \{\s*return false;\s*\}/); + assert.match(voiceServiceSource, /addEventListener\("voiceschanged", handleVoicesChanged\)/); + assert.match(voiceServiceSource, /return payload\.delivery_result\.type === "bubble" \? null : DELIVERY_READY_TEXT;/); + assert.match(voiceServiceSource, /"xiaoxiao"/); + assert.match(voiceServiceSource, /"soft_girl"/); + assert.match(voiceServiceSource, /"bright_girl"/); +}); + test("shell-ball falls back to regular submit when active steer status races", () => { const coordinatorSource = readFileSync(resolve(desktopRoot, "src/features/shell-ball/useShellBallCoordinator.ts"), "utf8"); diff --git a/apps/desktop/src/features/shell-ball/useShellBallCoordinator.ts b/apps/desktop/src/features/shell-ball/useShellBallCoordinator.ts index d313df54..18f77089 100644 --- a/apps/desktop/src/features/shell-ball/useShellBallCoordinator.ts +++ b/apps/desktop/src/features/shell-ball/useShellBallCoordinator.ts @@ -39,6 +39,7 @@ import { submitShellBallInput } from "./shellBallSubmit"; import { isRpcChannelUnavailable } from "@/rpc/fallback"; import { readClipboardText } from "@/services/clipboardService"; import { startTaskFromSelectedText } from "@/services/taskService"; +import { speakApprovalPendingNotification, speakDeliveryReadyNotification } from "@/services/voiceNotificationService"; import { requestDashboardTaskDetailOpen } from "@/features/dashboard/shared/dashboardTaskDetailNavigation"; import { buildShellBallIntentCorrectionPlaceholder, @@ -1402,6 +1403,10 @@ export function useShellBallCoordinator(input: ShellBallCoordinatorInput) { ]), ); revealBubbleRegionRef.current(); + speakApprovalPendingNotification({ + approval_request: input.approvalRequest, + task_id: input.taskId, + }); }, []); const appendDeliveryReadyBubble = useCallback((input: QueuedDeliveryReadyNotification) => { @@ -1442,6 +1447,10 @@ export function useShellBallCoordinator(input: ShellBallCoordinatorInput) { }); revealBubbleRegionRef.current(); void autoOpenShellBallDeliveryResultRef.current(input.taskId, input.deliveryResult); + speakDeliveryReadyNotification({ + delivery_result: input.deliveryResult, + task_id: input.taskId, + }); }, [allocateBubbleTurnIndex, bindTaskToBubbleTurn, getTaskBubbleTurnIndex]); const appendRuntimeObservationBubble = useCallback((taskId: string, payload: ShellBallRuntimeNotification) => { diff --git a/apps/desktop/src/services/settingsService.ts b/apps/desktop/src/services/settingsService.ts index 703b3f8d..147e4362 100644 --- a/apps/desktop/src/services/settingsService.ts +++ b/apps/desktop/src/services/settingsService.ts @@ -6,6 +6,7 @@ import { } from "../platform/desktopRuntimeDefaults"; import { syncDesktopSettingsSnapshot } from "../platform/desktopSettingsSnapshot"; import { loadStoredValue, saveStoredValue } from "../platform/storage"; +import { DEFAULT_VOICE_NOTIFICATION_VOICE_TYPE } from "./voiceNotificationConfig"; // SETTINGS_KEY is the single storage key for the desktop snapshot. const SETTINGS_KEY = "cialloclaw.settings"; @@ -90,7 +91,7 @@ function createDefaultSettings(): DesktopSettings { auto_launch: true, theme_mode: "follow_system", voice_notification_enabled: true, - voice_type: "default_female", + voice_type: DEFAULT_VOICE_NOTIFICATION_VOICE_TYPE, download: { workspace_path: runtimeDefaults?.workspace_path ?? DEFAULT_WORKSPACE_PLACEHOLDER, ask_before_save_each_file: true, diff --git a/apps/desktop/src/services/voiceNotificationConfig.ts b/apps/desktop/src/services/voiceNotificationConfig.ts new file mode 100644 index 00000000..26bb9048 --- /dev/null +++ b/apps/desktop/src/services/voiceNotificationConfig.ts @@ -0,0 +1,24 @@ +export const DEFAULT_VOICE_NOTIFICATION_VOICE_TYPE = "default_female"; + +export const VOICE_NOTIFICATION_VOICE_PRESET_OPTIONS = [ + { label: "小女孩", value: "default_female" }, + { label: "软萌", value: "soft_girl" }, + { label: "元气", value: "bright_girl" }, + { label: "少年", value: "default_male" }, +] as const; + +export type VoiceNotificationVoicePreset = (typeof VOICE_NOTIFICATION_VOICE_PRESET_OPTIONS)[number]["value"]; + +/** + * Normalizes persisted voice settings into one of the supported preset values + * when the control panel needs to render a radio-style choice group. + * + * @param voiceType Persisted `general.voice_type` value. + * @returns The matching preset or the refined default girl preset. + */ +export function resolveVoiceNotificationVoicePreset(voiceType: string): VoiceNotificationVoicePreset { + const normalizedVoiceType = voiceType.trim().toLowerCase(); + const matchedPreset = VOICE_NOTIFICATION_VOICE_PRESET_OPTIONS.find((option) => option.value === normalizedVoiceType); + + return matchedPreset?.value ?? DEFAULT_VOICE_NOTIFICATION_VOICE_TYPE; +} diff --git a/apps/desktop/src/services/voiceNotificationService.ts b/apps/desktop/src/services/voiceNotificationService.ts new file mode 100644 index 00000000..22891a7e --- /dev/null +++ b/apps/desktop/src/services/voiceNotificationService.ts @@ -0,0 +1,452 @@ +import type { ApprovalPendingNotification, DeliveryReadyNotification, SettingsSnapshot } from "@cialloclaw/protocol"; +import { loadSettings } from "./settingsService"; +import { DEFAULT_VOICE_NOTIFICATION_VOICE_TYPE } from "./voiceNotificationConfig"; + +type VoiceNotificationKind = + | "startup_greeting" + | "selection_detected" + | "selection_click_greeting" + | "clipboard_detected" + | "approval_pending" + | "delivery_ready"; + +type VoiceNotificationSettings = SettingsSnapshot["settings"]["general"]; + +type SpeechVoiceLike = { + lang: string; + name: string; +}; + +type SpeechSynthesisLike = { + addEventListener?: (type: "voiceschanged", listener: () => void) => void; + cancel: () => void; + getVoices: () => SpeechVoiceLike[]; + removeEventListener?: (type: "voiceschanged", listener: () => void) => void; + speak: (utterance: SpeechSynthesisUtterance) => void; +}; + +const STARTUP_GREETING_TEXT = "CialloClaw 已启动"; +const SELECTION_CLICK_GREETING_TEXT = "Ciallo"; +const SELECTION_DETECTED_TEXT_1 = "发现一段高亮文字啦"; +const SELECTION_DETECTED_TEXT_2 = "我接住新选中的内容啦"; +const SELECTION_DETECTED_TEXT_3 = "新的选中内容到啦"; +const SELECTION_DETECTED_TEXT_4 = "这段选中文字交给我吧"; +const SELECTION_DETECTED_TEXT_5 = "我看到你圈出的这段话啦"; +const CLIPBOARD_DETECTED_TEXT_1 = "剪贴板刚刚更新啦"; +const CLIPBOARD_DETECTED_TEXT_2 = "我接住新复制的内容啦"; +const CLIPBOARD_DETECTED_TEXT_3 = "新的剪贴板内容到啦"; +const CLIPBOARD_DETECTED_TEXT_4 = "这段复制内容交给我吧"; +const CLIPBOARD_DETECTED_TEXT_5 = "我看到你刚复制的小纸条啦"; +const APPROVAL_PENDING_TEXT = "有一个操作需要你确认"; +const DELIVERY_READY_TEXT = "任务结果已准备好"; +const VOICE_LIST_READY_TIMEOUT_MS = 600; +let latestVoiceNotificationRequestId = 0; + +function normalizeVoiceType(voiceType: string) { + return voiceType.trim().toLowerCase(); +} + +function normalizeVoiceLabel(value: string) { + return value.trim().toLowerCase(); +} + +function resolveRandomVoiceNotificationText(options: readonly string[], randomValue = Math.random()) { + if (options.length === 0) { + return ""; + } + + const normalizedRandomValue = Number.isFinite(randomValue) ? randomValue : 0; + const boundedRandomValue = Math.min(Math.max(normalizedRandomValue, 0), 0.999_999_999_999); + const resolvedIndex = Math.min(options.length - 1, Math.floor(boundedRandomValue * options.length)); + + return options[resolvedIndex] ?? options[0] ?? ""; +} + +function getVoiceNotificationSettings(): VoiceNotificationSettings { + return loadSettings().settings.general; +} + +function getVoiceNotificationHost() { + if (typeof window === "undefined" || typeof window.speechSynthesis === "undefined" || typeof SpeechSynthesisUtterance === "undefined") { + return null; + } + + return { + synthesizer: window.speechSynthesis as SpeechSynthesisLike, + utteranceConstructor: SpeechSynthesisUtterance, + }; +} + +function getVoiceTypePreferences(voiceType: string, language: string) { + const normalizedVoiceType = normalizeVoiceType(voiceType); + const languagePreference = language.toLowerCase(); + + if (normalizedVoiceType === "default_female") { + return [ + languagePreference, + "xiaoxiao", + "xiaoyi", + "tongtong", + "xiaohan", + "child", + "kid", + "tingting", + "yunxia", + "girl", + "young", + "cute", + "female", + "woman", + ]; + } + + if (normalizedVoiceType === "soft_girl") { + return [ + languagePreference, + "xiaoxiao", + "xiaohan", + "tongtong", + "soft", + "gentle", + "cute", + "child", + "girl", + "female", + ]; + } + + if (normalizedVoiceType === "bright_girl") { + return [ + languagePreference, + "xiaoyi", + "tingting", + "xiaoxuan", + "bright", + "lively", + "young", + "girl", + "female", + ]; + } + + if (normalizedVoiceType === "default_male") { + return [ + languagePreference, + "male", + "man", + "boy", + "yunxi", + "yunyang", + "daichi", + "keita", + ]; + } + + if (normalizedVoiceType.length === 0) { + return getVoiceTypePreferences(DEFAULT_VOICE_NOTIFICATION_VOICE_TYPE, language); + } + + return [normalizedVoiceType]; +} + +/** + * Picks the closest available system voice for the saved desktop preference. + * + * @param input Available voices plus the preferred voice type and language. + * @returns The best matching speech synthesis voice or `null` when none exist. + */ +export function resolveVoiceNotificationVoice(input: { + language: string; + voiceType: string; + voices: SpeechVoiceLike[]; +}) { + if (input.voices.length === 0) { + return null; + } + + const language = input.language.trim().toLowerCase(); + const preferredLanguagePrefix = language.split("-")[0] ?? language; + const normalizedVoiceType = normalizeVoiceType(input.voiceType); + const preferences = getVoiceTypePreferences(input.voiceType, input.language); + + let bestVoice: SpeechVoiceLike | null = null; + let bestScore = Number.NEGATIVE_INFINITY; + + input.voices.forEach((voice) => { + const normalizedName = normalizeVoiceLabel(voice.name); + const normalizedLanguage = normalizeVoiceLabel(voice.lang); + const normalizedLanguagePrefix = normalizedLanguage.split("-")[0] ?? normalizedLanguage; + let score = 0; + + if (normalizedVoiceType.length > 0 && normalizedName === normalizedVoiceType) { + score += 100; + } + + preferences.forEach((preference, index) => { + if (normalizedName.includes(preference) || normalizedLanguage.includes(preference)) { + score += 30 - index; + } + }); + + if (normalizedLanguage === language) { + score += 20; + } else if (normalizedLanguagePrefix === preferredLanguagePrefix) { + score += 10; + } + + if (score > bestScore) { + bestVoice = voice; + bestScore = score; + } + }); + + return bestVoice ?? input.voices[0] ?? null; +} + +/** + * Decides the spoken copy for a formal delivery notification. + * + * @param payload The stable `delivery.ready` notification payload. + * @returns The short spoken line, or `null` when the delivery should stay silent. + */ +export function resolveDeliveryReadyVoiceNotificationText(payload: DeliveryReadyNotification) { + return payload.delivery_result.type === "bubble" ? null : DELIVERY_READY_TEXT; +} + +/** + * Builds the startup greeting text for the floating ball window. + * + * @returns The short greeting that should be spoken after startup. + */ +export function resolveShellBallStartupGreetingText() { + return STARTUP_GREETING_TEXT; +} + +/** + * Builds a playful selection-detected reminder for the floating ball so + * repeated demos feel less robotic without expanding the notification scope. + * + * @param randomValue Optional deterministic random input for contract tests. + * @returns One of five local selection-detected reminder lines. + */ +export function resolveShellBallSelectionDetectedNotificationText(randomValue = Math.random()) { + return resolveRandomVoiceNotificationText([ + SELECTION_DETECTED_TEXT_1, + SELECTION_DETECTED_TEXT_2, + SELECTION_DETECTED_TEXT_3, + SELECTION_DETECTED_TEXT_4, + SELECTION_DETECTED_TEXT_5, + ], randomValue); +} + +/** + * Builds the short `Ciallo` acknowledgement spoken when the user accepts a + * fresh selected-text reminder from the floating ball. + * + * @returns The short acknowledgement used for the selected-text prompt click. + */ +export function resolveShellBallSelectionClickGreetingText() { + return SELECTION_CLICK_GREETING_TEXT; +} + +/** + * Builds a playful clipboard-detected reminder for the floating ball so local + * clipboard prompts feel lively during repeated desktop demos. + * + * @param randomValue Optional deterministic random input for contract tests. + * @returns One of five local clipboard-detected reminder lines. + */ +export function resolveShellBallClipboardDetectedNotificationText(randomValue = Math.random()) { + return resolveRandomVoiceNotificationText([ + CLIPBOARD_DETECTED_TEXT_1, + CLIPBOARD_DETECTED_TEXT_2, + CLIPBOARD_DETECTED_TEXT_3, + CLIPBOARD_DETECTED_TEXT_4, + CLIPBOARD_DETECTED_TEXT_5, + ], randomValue); +} + +function resolveVoiceNotificationText(kind: VoiceNotificationKind, payload?: ApprovalPendingNotification | DeliveryReadyNotification) { + if (kind === "startup_greeting") { + return resolveShellBallStartupGreetingText(); + } + + if (kind === "selection_detected") { + return resolveShellBallSelectionDetectedNotificationText(); + } + + if (kind === "selection_click_greeting") { + return resolveShellBallSelectionClickGreetingText(); + } + + if (kind === "clipboard_detected") { + return resolveShellBallClipboardDetectedNotificationText(); + } + + if (kind === "approval_pending") { + return APPROVAL_PENDING_TEXT; + } + + if (!payload || !("delivery_result" in payload)) { + return null; + } + + return resolveDeliveryReadyVoiceNotificationText(payload); +} + +function resolveVoiceNotificationLanguage() { + return "zh-CN"; +} + +function waitForVoiceNotificationVoices(synthesizer: SpeechSynthesisLike) { + const availableVoices = synthesizer.getVoices(); + if (availableVoices.length > 0 || synthesizer.addEventListener === undefined || synthesizer.removeEventListener === undefined) { + return Promise.resolve(availableVoices); + } + + return new Promise((resolve) => { + let settled = false; + + const finalize = () => { + if (settled) { + return; + } + + settled = true; + window.clearTimeout(timeoutId); + synthesizer.removeEventListener?.("voiceschanged", handleVoicesChanged); + resolve(synthesizer.getVoices()); + }; + + const handleVoicesChanged = () => { + finalize(); + }; + + // Chromium can populate speech voices after the first startup paint, so a + // short bounded wait keeps the saved desktop voice preference effective. + const timeoutId = window.setTimeout(() => { + finalize(); + }, VOICE_LIST_READY_TIMEOUT_MS); + + synthesizer.addEventListener("voiceschanged", handleVoicesChanged); + }); +} + +async function speakVoiceNotification(input: { + kind: VoiceNotificationKind; + payload?: ApprovalPendingNotification | DeliveryReadyNotification; +}) { + const requestId = ++latestVoiceNotificationRequestId; + const host = getVoiceNotificationHost(); + if (host === null) { + return false; + } + + const initialSettings = getVoiceNotificationSettings(); + if (!initialSettings.voice_notification_enabled) { + return false; + } + + const text = resolveVoiceNotificationText(input.kind, input.payload); + if (text === null) { + return false; + } + + const language = resolveVoiceNotificationLanguage(); + const utterance = new host.utteranceConstructor(text); + const availableVoices = await waitForVoiceNotificationVoices(host.synthesizer); + + // Voice discovery can resolve out of order during startup, so only the most + // recent local notification is allowed to claim the shared synthesizer. + if (requestId !== latestVoiceNotificationRequestId) { + return false; + } + + // The control panel can change local voice settings while the browser is + // still resolving system voices, so re-read the latest snapshot right before + // claiming the shared synthesizer. + const latestSettings = getVoiceNotificationSettings(); + if (!latestSettings.voice_notification_enabled) { + return false; + } + + const resolvedVoice = resolveVoiceNotificationVoice({ + language, + voiceType: latestSettings.voice_type, + voices: availableVoices, + }); + + utterance.lang = language; + if (resolvedVoice !== null) { + utterance.voice = resolvedVoice as SpeechSynthesisVoice; + } + + host.synthesizer.cancel(); + host.synthesizer.speak(utterance); + return true; +} + +/** + * Plays the startup greeting once the floating ball becomes available. + * + * @returns Whether the local desktop runtime attempted speech playback. + */ +export function speakShellBallStartupGreeting() { + return speakVoiceNotification({ kind: "startup_greeting" }); +} + +/** + * Plays a short local reminder when the floating ball captures new selected + * text and exposes the click-to-start affordance. + * + * @returns Whether the local desktop runtime attempted speech playback. + */ +export function speakShellBallSelectionDetectedNotification() { + return speakVoiceNotification({ kind: "selection_detected" }); +} + +/** + * Plays a short `Ciallo` acknowledgement when the user accepts a selected-text + * reminder from the floating ball. + * + * @returns Whether the local desktop runtime attempted speech playback. + */ +export function speakShellBallSelectionClickGreeting() { + return speakVoiceNotification({ kind: "selection_click_greeting" }); +} + +/** + * Plays a short local reminder when the floating ball captures new clipboard + * text and exposes the click-to-submit affordance. + * + * @returns Whether the local desktop runtime attempted speech playback. + */ +export function speakShellBallClipboardDetectedNotification() { + return speakVoiceNotification({ kind: "clipboard_detected" }); +} + +/** + * Plays a short approval reminder for the formal `approval.pending` notification. + * + * @param payload The stable task approval notification payload. + * @returns Whether the local desktop runtime attempted speech playback. + */ +export function speakApprovalPendingNotification(payload: ApprovalPendingNotification) { + return speakVoiceNotification({ + kind: "approval_pending", + payload, + }); +} + +/** + * Plays a short delivery reminder for non-bubble formal delivery results. + * + * @param payload The stable task delivery notification payload. + * @returns Whether the local desktop runtime attempted speech playback. + */ +export function speakDeliveryReadyNotification(payload: DeliveryReadyNotification) { + return speakVoiceNotification({ + kind: "delivery_ready", + payload, + }); +} diff --git a/docs/control-panel-settings.md b/docs/control-panel-settings.md index 05e94933..2d912a83 100644 --- a/docs/control-panel-settings.md +++ b/docs/control-panel-settings.md @@ -158,20 +158,25 @@ #### 3.5.2 建议拆分的设置项 1. **声音总开关** - - 控制是否播放任务成功、提醒到达、悬浮球气泡提示等音效或语音提示。 + - 控制是否播放悬浮球启动、选中文本检测、剪贴板检测、选中文本提醒态点击确认、授权提醒、正式结果提醒等本地语音提示。 2. **提示音/语音风格选择** - - 例如：不同声线版本的“Ciallo”。 + - 例如：用按钮选择“小女孩 / 软萌 / 元气 / 少年”等明确预设。 3. **典型触发场景说明** - - 任务成功 - - 便签到期 + - 悬浮球启动完成 + - 检测到新的选中文本 + - 检测到新的剪贴板内容 + - 选中文本后出现提醒态，点击该提醒继续发起协作时播报 `Ciallo` - 授权提醒 - - 高价值信息球弹出 + - 正式结果已准备好 #### 3.5.3 交互规则 - 开启声音时，应支持试听。 - 关闭总开关后，风格选择区域应置灰但保留当前已选项。 +- 控制面板里的声音草稿只有在保存后才会同步到悬浮球等其他桌面窗口；未保存的面板改动不应直接改变当前本地播报行为。 - 声音设置应明确说明是否影响系统通知、仅影响应用内反馈，还是两者都影响。 +- 当前桌面最小消费范围为：悬浮球窗口启动成功后的提示音、选中文本检测、剪贴板检测、选中文本提醒态被点击后的本地确认，以及 `approval.pending`、非 `bubble` 的 `delivery.ready` 本地语音提示。 +- 第一版语音播报只在桌面端本地消费已有入口与正式通知，不新增后端 TTS 服务，也不扩展新的 JSON-RPC 方法或协议字段。 ### 3.6 下载设置