From 570c0f1c2358f5720a587cfe40d63db965809d2e Mon Sep 17 00:00:00 2001 From: xgopilot Date: Wed, 13 May 2026 09:13:08 +0000 Subject: [PATCH 1/9] docs(control-panel): clarify voice notification scope Generated with [codeagent](https://github.com/qbox/codeagent) Co-authored-by: Blackcloudss <161042579+Blackcloudss@users.noreply.github.com> --- docs/control-panel-settings.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/control-panel-settings.md b/docs/control-panel-settings.md index 05e94933..b35e5d0b 100644 --- a/docs/control-panel-settings.md +++ b/docs/control-panel-settings.md @@ -172,6 +172,8 @@ - 开启声音时，应支持试听。 - 关闭总开关后，风格选择区域应置灰但保留当前已选项。 - 声音设置应明确说明是否影响系统通知、仅影响应用内反馈，还是两者都影响。 +- 当前桌面最小正式消费范围为：悬浮球窗口启动成功后的问候语，以及 `approval.pending`、非 `bubble` 的 `delivery.ready` 本地语音提示。 +- 第一版语音播报仅在桌面端本地消费正式通知，不新增后端 TTS 服务，也不扩展新的 JSON-RPC 方法或协议字段。 ### 3.6 下载设置 From b57b269e042e34cbb19a65ea9fcc17b49b7f17ea Mon Sep 17 00:00:00 2001 From: xgopilot Date: Wed, 13 May 2026 09:13:24 +0000 Subject: [PATCH 2/9] feat(desktop): add shell ball voice notifications Generated with [codeagent](https://github.com/qbox/codeagent) Co-authored-by: Blackcloudss <161042579+Blackcloudss@users.noreply.github.com> --- .../src/features/shell-ball/ShellBallApp.tsx | 12 + .../shell-ball/shellBall.contract.test.ts | 67 +++++ .../shell-ball/useShellBallCoordinator.ts | 9 + .../src/services/voiceNotificationService.ts | 244 ++++++++++++++++++ 4 files changed, 332 insertions(+) create mode 100644 apps/desktop/src/services/voiceNotificationService.ts diff --git a/apps/desktop/src/features/shell-ball/ShellBallApp.tsx b/apps/desktop/src/features/shell-ball/ShellBallApp.tsx index afc88a33..1ffae6b6 100644 --- a/apps/desktop/src/features/shell-ball/ShellBallApp.tsx +++ b/apps/desktop/src/features/shell-ball/ShellBallApp.tsx @@ -39,6 +39,7 @@ import { setShellBallPressLock, } from "../../platform/shellBallWindow"; import { loadSettings } from "../../services/settingsService"; +import { speakShellBallStartupGreeting } from "../../services/voiceNotificationService"; import { openOrFocusDesktopWindow } from "../../platform/windowController"; import { buildDesktopOnboardingPresentation } from "@/features/onboarding/onboardingGeometry"; import { @@ -394,6 +395,7 @@ export function ShellBallApp() { const previousVisualStateRef = useRef(visualState); const transitionQueueRef = useRef(Promise.resolve()); const edgeDockRevealHideTimeoutRef = useRef(null); + const startupGreetingPlayedRef = useRef(false); const dragDropHandlersRef = useRef<{ handleDroppedFiles: (paths: string[]) => Promise | void; }>({ @@ -482,6 +484,16 @@ export function ShellBallApp() { window.removeEventListener("storage", syncFloatingBallSizeFromStorage); }; }, []); + + useEffect(() => { + if (startupGreetingPlayedRef.current) { + return; + } + + startupGreetingPlayedRef.current = true; + speakShellBallStartupGreeting(); + }, []); + const { ballDockSettling, ballDragActive, diff --git a/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts b/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts index 245a308b..7a5a5537 100644 --- a/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts +++ b/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts @@ -28,6 +28,11 @@ import { } from "./shellBall.interaction"; import { getShellBallMotionConfig } from "./shellBall.motion"; import { collectShellBallSpeechTranscript, composeShellBallSpeechDraft } from "./shellBall.speech"; +import { + resolveDeliveryReadyVoiceNotificationText, + resolveShellBallStartupGreetingText, + resolveVoiceNotificationVoice, +} from "../../services/voiceNotificationService"; import { compactPageContext, mapDesktopWindowSnapshotToPageContext, @@ -2532,6 +2537,51 @@ test("shell-ball speech transcript collection merges recognition chunks", () => ); }); +test("voice notification helpers keep startup greeting and formal delivery reminders short", () => { + assert.equal(resolveShellBallStartupGreetingText(), "Ciallo!"); + assert.equal( + resolveDeliveryReadyVoiceNotificationText({ + task_id: "task-bubble-delivery", + delivery_result: { + type: "bubble", + }, + } as any), + null, + ); + assert.equal( + resolveDeliveryReadyVoiceNotificationText({ + task_id: "task-formal-delivery", + delivery_result: { + type: "task_detail", + }, + } as any), + "任务结果已准备好", + ); +}); + +test("voice notification helpers prefer the saved voice type and matching locale", () => { + const resolvedDefaultFemale = resolveVoiceNotificationVoice({ + language: "ja-JP", + voiceType: "default_female", + voices: [ + { lang: "en-US", name: "Alloy" }, + { lang: "ja-JP", name: "Sakura Female" }, + { lang: "zh-CN", name: "Xiaoxiao" }, + ], + }); + assert.equal(resolvedDefaultFemale?.name, "Sakura Female"); + + const resolvedCustomVoice = resolveVoiceNotificationVoice({ + language: "zh-CN", + voiceType: "voice_nebula", + voices: [ + { lang: "zh-CN", name: "voice_nebula" }, + { lang: "zh-CN", name: "Xiaoxiao" }, + ], + }); + assert.equal(resolvedCustomVoice?.name, "voice_nebula"); +}); + test("shell-ball voice recognition final state routes final transcript out of the input draft and restores draft on cancel", () => { assert.deepEqual( resolveShellBallVoiceRecognitionFinalState({ @@ -8870,6 +8920,23 @@ test("shell-ball routes active resumable text follow-ups through task steer", () assert.match(coordinatorSource, /message: submittedText/); }); +test("shell-ball voice notifications are consumed locally from startup and formal task notifications", () => { + const appSource = readFileSync(resolve(desktopRoot, "src/features/shell-ball/ShellBallApp.tsx"), "utf8"); + const coordinatorSource = readFileSync(resolve(desktopRoot, "src/features/shell-ball/useShellBallCoordinator.ts"), "utf8"); + const voiceServiceSource = readFileSync(resolve(desktopRoot, "src/services/voiceNotificationService.ts"), "utf8"); + + assert.match(appSource, /import \{ speakShellBallStartupGreeting \} from "..\/..\/services\/voiceNotificationService";/); + assert.match(appSource, /const startupGreetingPlayedRef = useRef\(false\);/); + assert.match(appSource, /startupGreetingPlayedRef\.current = true;\s*speakShellBallStartupGreeting\(\);/); + assert.match(coordinatorSource, /import \{ speakApprovalPendingNotification, speakDeliveryReadyNotification \} from "@\/services\/voiceNotificationService";/); + assert.match(coordinatorSource, /speakApprovalPendingNotification\(\{\s*approval_request: input\.approvalRequest,\s*task_id: input\.taskId,\s*\}\);/); + assert.match(coordinatorSource, /speakDeliveryReadyNotification\(\{\s*delivery_result: input\.deliveryResult,\s*task_id: input\.taskId,\s*\}\);/); + assert.match(voiceServiceSource, /const STARTUP_GREETING_TEXT = "Ciallo!";/); + assert.match(voiceServiceSource, /const APPROVAL_PENDING_TEXT = "有一个操作需要你确认";/); + assert.match(voiceServiceSource, /const DELIVERY_READY_TEXT = "任务结果已准备好";/); + assert.match(voiceServiceSource, /return payload\.delivery_result\.type === "bubble" \? null : DELIVERY_READY_TEXT;/); +}); + test("shell-ball falls back to regular submit when active steer status races", () => { const coordinatorSource = readFileSync(resolve(desktopRoot, "src/features/shell-ball/useShellBallCoordinator.ts"), "utf8"); diff --git a/apps/desktop/src/features/shell-ball/useShellBallCoordinator.ts b/apps/desktop/src/features/shell-ball/useShellBallCoordinator.ts index d313df54..18f77089 100644 --- a/apps/desktop/src/features/shell-ball/useShellBallCoordinator.ts +++ b/apps/desktop/src/features/shell-ball/useShellBallCoordinator.ts @@ -39,6 +39,7 @@ import { submitShellBallInput } from "./shellBallSubmit"; import { isRpcChannelUnavailable } from "@/rpc/fallback"; import { readClipboardText } from "@/services/clipboardService"; import { startTaskFromSelectedText } from "@/services/taskService"; +import { speakApprovalPendingNotification, speakDeliveryReadyNotification } from "@/services/voiceNotificationService"; import { requestDashboardTaskDetailOpen } from "@/features/dashboard/shared/dashboardTaskDetailNavigation"; import { buildShellBallIntentCorrectionPlaceholder, @@ -1402,6 +1403,10 @@ export function useShellBallCoordinator(input: ShellBallCoordinatorInput) { ]), ); revealBubbleRegionRef.current(); + speakApprovalPendingNotification({ + approval_request: input.approvalRequest, + task_id: input.taskId, + }); }, []); const appendDeliveryReadyBubble = useCallback((input: QueuedDeliveryReadyNotification) => { @@ -1442,6 +1447,10 @@ export function useShellBallCoordinator(input: ShellBallCoordinatorInput) { }); revealBubbleRegionRef.current(); void autoOpenShellBallDeliveryResultRef.current(input.taskId, input.deliveryResult); + speakDeliveryReadyNotification({ + delivery_result: input.deliveryResult, + task_id: input.taskId, + }); }, [allocateBubbleTurnIndex, bindTaskToBubbleTurn, getTaskBubbleTurnIndex]); const appendRuntimeObservationBubble = useCallback((taskId: string, payload: ShellBallRuntimeNotification) => { diff --git a/apps/desktop/src/services/voiceNotificationService.ts b/apps/desktop/src/services/voiceNotificationService.ts new file mode 100644 index 00000000..05a17048 --- /dev/null +++ b/apps/desktop/src/services/voiceNotificationService.ts @@ -0,0 +1,244 @@ +import type { ApprovalPendingNotification, DeliveryReadyNotification, SettingsSnapshot } from "@cialloclaw/protocol"; +import { loadSettings } from "./settingsService"; + +type VoiceNotificationKind = "startup_greeting" | "approval_pending" | "delivery_ready"; + +type VoiceNotificationSettings = SettingsSnapshot["settings"]["general"]; + +type SpeechVoiceLike = { + lang: string; + name: string; +}; + +type SpeechSynthesisLike = { + cancel: () => void; + getVoices: () => SpeechVoiceLike[]; + speak: (utterance: SpeechSynthesisUtterance) => void; +}; + +const STARTUP_GREETING_TEXT = "Ciallo!"; +const APPROVAL_PENDING_TEXT = "有一个操作需要你确认"; +const DELIVERY_READY_TEXT = "任务结果已准备好"; + +function normalizeVoiceType(voiceType: string) { + return voiceType.trim().toLowerCase(); +} + +function normalizeVoiceLabel(value: string) { + return value.trim().toLowerCase(); +} + +function getVoiceNotificationSettings(): VoiceNotificationSettings { + return loadSettings().settings.general; +} + +function getVoiceNotificationHost() { + if (typeof window === "undefined" || typeof window.speechSynthesis === "undefined" || typeof SpeechSynthesisUtterance === "undefined") { + return null; + } + + return { + synthesizer: window.speechSynthesis as SpeechSynthesisLike, + utteranceConstructor: SpeechSynthesisUtterance, + }; +} + +function getVoiceTypePreferences(voiceType: string, language: string) { + const normalizedVoiceType = normalizeVoiceType(voiceType); + + if (normalizedVoiceType === "default_female") { + return [ + "ja-jp", + "female", + "woman", + "girl", + "kyoko", + "sakura", + "nanami", + "sayaka", + "xiaoxiao", + "tingting", + ]; + } + + if (normalizedVoiceType === "default_male") { + return [ + language.toLowerCase(), + "male", + "man", + "boy", + "yunxi", + "yunyang", + "daichi", + "keita", + ]; + } + + return normalizedVoiceType.length > 0 ? [normalizedVoiceType] : [language.toLowerCase()]; +} + +/** + * Picks the closest available system voice for the saved desktop preference. + * + * @param input Available voices plus the preferred voice type and language. + * @returns The best matching speech synthesis voice or `null` when none exist. + */ +export function resolveVoiceNotificationVoice(input: { + language: string; + voiceType: string; + voices: SpeechVoiceLike[]; +}) { + if (input.voices.length === 0) { + return null; + } + + const language = input.language.trim().toLowerCase(); + const preferredLanguagePrefix = language.split("-")[0] ?? language; + const normalizedVoiceType = normalizeVoiceType(input.voiceType); + const preferences = getVoiceTypePreferences(input.voiceType, input.language); + + let bestVoice: SpeechVoiceLike | null = null; + let bestScore = Number.NEGATIVE_INFINITY; + + input.voices.forEach((voice) => { + const normalizedName = normalizeVoiceLabel(voice.name); + const normalizedLanguage = normalizeVoiceLabel(voice.lang); + const normalizedLanguagePrefix = normalizedLanguage.split("-")[0] ?? normalizedLanguage; + let score = 0; + + if (normalizedVoiceType.length > 0 && normalizedName === normalizedVoiceType) { + score += 100; + } + + preferences.forEach((preference, index) => { + if (normalizedName.includes(preference) || normalizedLanguage.includes(preference)) { + score += 30 - index; + } + }); + + if (normalizedLanguage === language) { + score += 20; + } else if (normalizedLanguagePrefix === preferredLanguagePrefix) { + score += 10; + } + + if (score > bestScore) { + bestVoice = voice; + bestScore = score; + } + }); + + return bestVoice ?? input.voices[0] ?? null; +} + +/** + * Decides the spoken copy for a formal delivery notification. + * + * @param payload The stable `delivery.ready` notification payload. + * @returns The short spoken line, or `null` when the delivery should stay silent. + */ +export function resolveDeliveryReadyVoiceNotificationText(payload: DeliveryReadyNotification) { + return payload.delivery_result.type === "bubble" ? null : DELIVERY_READY_TEXT; +} + +/** + * Builds the startup greeting text for the floating ball window. + * + * @returns The short greeting that should be spoken after startup. + */ +export function resolveShellBallStartupGreetingText() { + return STARTUP_GREETING_TEXT; +} + +function resolveVoiceNotificationText(kind: VoiceNotificationKind, payload?: ApprovalPendingNotification | DeliveryReadyNotification) { + if (kind === "startup_greeting") { + return resolveShellBallStartupGreetingText(); + } + + if (kind === "approval_pending") { + return APPROVAL_PENDING_TEXT; + } + + if (!payload || !("delivery_result" in payload)) { + return null; + } + + return resolveDeliveryReadyVoiceNotificationText(payload); +} + +function resolveVoiceNotificationLanguage(kind: VoiceNotificationKind) { + return kind === "startup_greeting" ? "ja-JP" : "zh-CN"; +} + +function speakVoiceNotification(input: { + kind: VoiceNotificationKind; + payload?: ApprovalPendingNotification | DeliveryReadyNotification; +}) { + const host = getVoiceNotificationHost(); + if (host === null) { + return false; + } + + const settings = getVoiceNotificationSettings(); + if (!settings.voice_notification_enabled) { + return false; + } + + const text = resolveVoiceNotificationText(input.kind, input.payload); + if (text === null) { + return false; + } + + const language = resolveVoiceNotificationLanguage(input.kind); + const utterance = new host.utteranceConstructor(text); + const availableVoices = host.synthesizer.getVoices(); + const resolvedVoice = resolveVoiceNotificationVoice({ + language, + voiceType: settings.voice_type, + voices: availableVoices, + }); + + utterance.lang = language; + if (resolvedVoice !== null) { + utterance.voice = resolvedVoice as SpeechSynthesisVoice; + } + + host.synthesizer.cancel(); + host.synthesizer.speak(utterance); + return true; +} + +/** + * Plays the startup greeting once the floating ball becomes available. + * + * @returns Whether the local desktop runtime attempted speech playback. + */ +export function speakShellBallStartupGreeting() { + return speakVoiceNotification({ kind: "startup_greeting" }); +} + +/** + * Plays a short approval reminder for the formal `approval.pending` notification. + * + * @param payload The stable task approval notification payload. + * @returns Whether the local desktop runtime attempted speech playback. + */ +export function speakApprovalPendingNotification(payload: ApprovalPendingNotification) { + return speakVoiceNotification({ + kind: "approval_pending", + payload, + }); +} + +/** + * Plays a short delivery reminder for non-bubble formal delivery results. + * + * @param payload The stable task delivery notification payload. + * @returns Whether the local desktop runtime attempted speech playback. + */ +export function speakDeliveryReadyNotification(payload: DeliveryReadyNotification) { + return speakVoiceNotification({ + kind: "delivery_ready", + payload, + }); +} From 91fe3d25bdd77d1980afa9ee493c4edb9c6099c3 Mon Sep 17 00:00:00 2001 From: xgopilot Date: Wed, 13 May 2026 09:44:18 +0000 Subject: [PATCH 3/9] fix(desktop): refine shell ball voice notifications Generated with [codeagent](https://github.com/qbox/codeagent) Co-authored-by: Blackcloudss <161042579+Blackcloudss@users.noreply.github.com> --- .../control-panel/ControlPanelApp.tsx | 4 +- .../src/features/shell-ball/ShellBallApp.tsx | 26 +++++- .../shell-ball/shellBall.contract.test.ts | 22 +++-- .../src/services/voiceNotificationService.ts | 92 ++++++++++++++++--- docs/control-panel-settings.md | 12 +-- 5 files changed, 128 insertions(+), 28 deletions(-) diff --git a/apps/desktop/src/features/control-panel/ControlPanelApp.tsx b/apps/desktop/src/features/control-panel/ControlPanelApp.tsx index 11cf0932..013d8ed7 100644 --- a/apps/desktop/src/features/control-panel/ControlPanelApp.tsx +++ b/apps/desktop/src/features/control-panel/ControlPanelApp.tsx @@ -1281,7 +1281,7 @@ export function ControlPanelApp() { updateSettings((current) => ({ @@ -1296,7 +1296,7 @@ export function ControlPanelApp() { (null); const startupGreetingPlayedRef = useRef(false); + // Local prompt events can replay while helper windows sync state, so keep the + // last spoken keys here to avoid duplicate demo-time voice spam. + const selectionPromptVoiceKeyRef = useRef(""); + const clipboardPromptVoiceTextRef = useRef(""); const dragDropHandlersRef = useRef<{ handleDroppedFiles: (paths: string[]) => Promise | void; }>({ @@ -491,7 +499,7 @@ export function ShellBallApp() { } startupGreetingPlayedRef.current = true; - speakShellBallStartupGreeting(); + void speakShellBallStartupGreeting(); }, []); const { @@ -1048,6 +1056,13 @@ export function ShellBallApp() { selectionPromptClearTimeoutRef.current = null; } + const normalizedSelectionText = payload.snapshot.text.trim(); + const voiceKey = `${payload.snapshot.updated_at}:${normalizedSelectionText}`; + if (normalizedSelectionText.length > 0 && selectionPromptVoiceKeyRef.current !== voiceKey) { + selectionPromptVoiceKeyRef.current = voiceKey; + void speakShellBallSelectionDetectedNotification(); + } + setSelectionPrompt(payload.snapshot); return; } @@ -1096,10 +1111,17 @@ export function ShellBallApp() { void currentWindow .listen(shellBallWindowSyncEvents.clipboardSnapshot, ({ payload }) => { if (payload.text.trim() === "") { + clipboardPromptVoiceTextRef.current = ""; setClipboardPrompt(null); return; } + const normalizedClipboardText = payload.text.trim(); + if (clipboardPromptVoiceTextRef.current !== normalizedClipboardText) { + clipboardPromptVoiceTextRef.current = normalizedClipboardText; + void speakShellBallClipboardDetectedNotification(); + } + setClipboardPrompt({ text: payload.text, expiresAt: Date.now() + SHELL_BALL_CLIPBOARD_PROMPT_WINDOW_MS, diff --git a/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts b/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts index 7a5a5537..d59bfc62 100644 --- a/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts +++ b/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts @@ -2538,7 +2538,7 @@ test("shell-ball speech transcript collection merges recognition chunks", () => }); test("voice notification helpers keep startup greeting and formal delivery reminders short", () => { - assert.equal(resolveShellBallStartupGreetingText(), "Ciallo!"); + assert.equal(resolveShellBallStartupGreetingText(), "CialloClaw 已启动"); assert.equal( resolveDeliveryReadyVoiceNotificationText({ task_id: "task-bubble-delivery", @@ -2561,7 +2561,7 @@ test("voice notification helpers keep startup greeting and formal delivery remin test("voice notification helpers prefer the saved voice type and matching locale", () => { const resolvedDefaultFemale = resolveVoiceNotificationVoice({ - language: "ja-JP", + language: "zh-CN", voiceType: "default_female", voices: [ { lang: "en-US", name: "Alloy" }, @@ -2569,7 +2569,7 @@ test("voice notification helpers prefer the saved voice type and matching locale { lang: "zh-CN", name: "Xiaoxiao" }, ], }); - assert.equal(resolvedDefaultFemale?.name, "Sakura Female"); + assert.equal(resolvedDefaultFemale?.name, "Xiaoxiao"); const resolvedCustomVoice = resolveVoiceNotificationVoice({ language: "zh-CN", @@ -8925,15 +8925,25 @@ test("shell-ball voice notifications are consumed locally from startup and forma const coordinatorSource = readFileSync(resolve(desktopRoot, "src/features/shell-ball/useShellBallCoordinator.ts"), "utf8"); const voiceServiceSource = readFileSync(resolve(desktopRoot, "src/services/voiceNotificationService.ts"), "utf8"); - assert.match(appSource, /import \{ speakShellBallStartupGreeting \} from "..\/..\/services\/voiceNotificationService";/); + assert.match(appSource, /speakShellBallClipboardDetectedNotification/); + assert.match(appSource, /speakShellBallSelectionDetectedNotification/); + assert.match(appSource, /speakShellBallStartupGreeting/); assert.match(appSource, /const startupGreetingPlayedRef = useRef\(false\);/); - assert.match(appSource, /startupGreetingPlayedRef\.current = true;\s*speakShellBallStartupGreeting\(\);/); + assert.match(appSource, /const selectionPromptVoiceKeyRef = useRef\(""\);/); + assert.match(appSource, /const clipboardPromptVoiceTextRef = useRef\(""\);/); + assert.match(appSource, /startupGreetingPlayedRef\.current = true;\s*void speakShellBallStartupGreeting\(\);/); + assert.match(appSource, /void speakShellBallSelectionDetectedNotification\(\);/); + assert.match(appSource, /void speakShellBallClipboardDetectedNotification\(\);/); assert.match(coordinatorSource, /import \{ speakApprovalPendingNotification, speakDeliveryReadyNotification \} from "@\/services\/voiceNotificationService";/); assert.match(coordinatorSource, /speakApprovalPendingNotification\(\{\s*approval_request: input\.approvalRequest,\s*task_id: input\.taskId,\s*\}\);/); assert.match(coordinatorSource, /speakDeliveryReadyNotification\(\{\s*delivery_result: input\.deliveryResult,\s*task_id: input\.taskId,\s*\}\);/); - assert.match(voiceServiceSource, /const STARTUP_GREETING_TEXT = "Ciallo!";/); + assert.match(voiceServiceSource, /const STARTUP_GREETING_TEXT = "CialloClaw 已启动";/); + assert.match(voiceServiceSource, /const SELECTION_DETECTED_TEXT = "检测到选中文本";/); + assert.match(voiceServiceSource, /const CLIPBOARD_DETECTED_TEXT = "检测到剪贴板内容";/); assert.match(voiceServiceSource, /const APPROVAL_PENDING_TEXT = "有一个操作需要你确认";/); assert.match(voiceServiceSource, /const DELIVERY_READY_TEXT = "任务结果已准备好";/); + assert.match(voiceServiceSource, /const VOICE_LIST_READY_TIMEOUT_MS = 600;/); + assert.match(voiceServiceSource, /addEventListener\("voiceschanged", handleVoicesChanged\)/); assert.match(voiceServiceSource, /return payload\.delivery_result\.type === "bubble" \? null : DELIVERY_READY_TEXT;/); }); diff --git a/apps/desktop/src/services/voiceNotificationService.ts b/apps/desktop/src/services/voiceNotificationService.ts index 05a17048..7e7676f8 100644 --- a/apps/desktop/src/services/voiceNotificationService.ts +++ b/apps/desktop/src/services/voiceNotificationService.ts @@ -1,7 +1,12 @@ import type { ApprovalPendingNotification, DeliveryReadyNotification, SettingsSnapshot } from "@cialloclaw/protocol"; import { loadSettings } from "./settingsService"; -type VoiceNotificationKind = "startup_greeting" | "approval_pending" | "delivery_ready"; +type VoiceNotificationKind = + | "startup_greeting" + | "selection_detected" + | "clipboard_detected" + | "approval_pending" + | "delivery_ready"; type VoiceNotificationSettings = SettingsSnapshot["settings"]["general"]; @@ -11,14 +16,19 @@ type SpeechVoiceLike = { }; type SpeechSynthesisLike = { + addEventListener?: (type: "voiceschanged", listener: () => void) => void; cancel: () => void; getVoices: () => SpeechVoiceLike[]; + removeEventListener?: (type: "voiceschanged", listener: () => void) => void; speak: (utterance: SpeechSynthesisUtterance) => void; }; -const STARTUP_GREETING_TEXT = "Ciallo!"; +const STARTUP_GREETING_TEXT = "CialloClaw 已启动"; +const SELECTION_DETECTED_TEXT = "检测到选中文本"; +const CLIPBOARD_DETECTED_TEXT = "检测到剪贴板内容"; const APPROVAL_PENDING_TEXT = "有一个操作需要你确认"; const DELIVERY_READY_TEXT = "任务结果已准备好"; +const VOICE_LIST_READY_TIMEOUT_MS = 600; function normalizeVoiceType(voiceType: string) { return voiceType.trim().toLowerCase(); @@ -48,16 +58,14 @@ function getVoiceTypePreferences(voiceType: string, language: string) { if (normalizedVoiceType === "default_female") { return [ - "ja-jp", + language.toLowerCase(), "female", "woman", "girl", - "kyoko", - "sakura", - "nanami", - "sayaka", "xiaoxiao", + "xiaoyi", "tingting", + "yunxia", ]; } @@ -155,6 +163,14 @@ function resolveVoiceNotificationText(kind: VoiceNotificationKind, payload?: App return resolveShellBallStartupGreetingText(); } + if (kind === "selection_detected") { + return SELECTION_DETECTED_TEXT; + } + + if (kind === "clipboard_detected") { + return CLIPBOARD_DETECTED_TEXT; + } + if (kind === "approval_pending") { return APPROVAL_PENDING_TEXT; } @@ -166,11 +182,45 @@ function resolveVoiceNotificationText(kind: VoiceNotificationKind, payload?: App return resolveDeliveryReadyVoiceNotificationText(payload); } -function resolveVoiceNotificationLanguage(kind: VoiceNotificationKind) { - return kind === "startup_greeting" ? "ja-JP" : "zh-CN"; +function resolveVoiceNotificationLanguage() { + return "zh-CN"; +} + +function waitForVoiceNotificationVoices(synthesizer: SpeechSynthesisLike) { + const availableVoices = synthesizer.getVoices(); + if (availableVoices.length > 0 || synthesizer.addEventListener === undefined || synthesizer.removeEventListener === undefined) { + return Promise.resolve(availableVoices); + } + + return new Promise((resolve) => { + let settled = false; + + const finalize = () => { + if (settled) { + return; + } + + settled = true; + window.clearTimeout(timeoutId); + synthesizer.removeEventListener?.("voiceschanged", handleVoicesChanged); + resolve(synthesizer.getVoices()); + }; + + const handleVoicesChanged = () => { + finalize(); + }; + + // Chromium can populate speech voices after the first startup paint, so a + // short bounded wait keeps the saved desktop voice preference effective. + const timeoutId = window.setTimeout(() => { + finalize(); + }, VOICE_LIST_READY_TIMEOUT_MS); + + synthesizer.addEventListener("voiceschanged", handleVoicesChanged); + }); } -function speakVoiceNotification(input: { +async function speakVoiceNotification(input: { kind: VoiceNotificationKind; payload?: ApprovalPendingNotification | DeliveryReadyNotification; }) { @@ -189,9 +239,9 @@ function speakVoiceNotification(input: { return false; } - const language = resolveVoiceNotificationLanguage(input.kind); + const language = resolveVoiceNotificationLanguage(); const utterance = new host.utteranceConstructor(text); - const availableVoices = host.synthesizer.getVoices(); + const availableVoices = await waitForVoiceNotificationVoices(host.synthesizer); const resolvedVoice = resolveVoiceNotificationVoice({ language, voiceType: settings.voice_type, @@ -217,6 +267,24 @@ export function speakShellBallStartupGreeting() { return speakVoiceNotification({ kind: "startup_greeting" }); } +/** + * Plays a short reminder when the shell-ball detects a fresh text selection. + * + * @returns Whether the local desktop runtime attempted speech playback. + */ +export function speakShellBallSelectionDetectedNotification() { + return speakVoiceNotification({ kind: "selection_detected" }); +} + +/** + * Plays a short reminder when the shell-ball receives a fresh clipboard prompt. + * + * @returns Whether the local desktop runtime attempted speech playback. + */ +export function speakShellBallClipboardDetectedNotification() { + return speakVoiceNotification({ kind: "clipboard_detected" }); +} + /** * Plays a short approval reminder for the formal `approval.pending` notification. * diff --git a/docs/control-panel-settings.md b/docs/control-panel-settings.md index b35e5d0b..2bed9fcf 100644 --- a/docs/control-panel-settings.md +++ b/docs/control-panel-settings.md @@ -158,22 +158,22 @@ #### 3.5.2 建议拆分的设置项 1. **声音总开关** - - 控制是否播放任务成功、提醒到达、悬浮球气泡提示等音效或语音提示。 + - 控制是否播放悬浮球启动、文本承接提醒、授权提醒、正式结果提醒等本地语音提示。 2. **提示音/语音风格选择** - 例如：不同声线版本的“Ciallo”。 3. **典型触发场景说明** - - 任务成功 - - 便签到期 + - 悬浮球启动完成 + - 检测到选中文本或剪贴板内容 - 授权提醒 - - 高价值信息球弹出 + - 正式结果已准备好 #### 3.5.3 交互规则 - 开启声音时，应支持试听。 - 关闭总开关后，风格选择区域应置灰但保留当前已选项。 - 声音设置应明确说明是否影响系统通知、仅影响应用内反馈，还是两者都影响。 -- 当前桌面最小正式消费范围为：悬浮球窗口启动成功后的问候语，以及 `approval.pending`、非 `bubble` 的 `delivery.ready` 本地语音提示。 -- 第一版语音播报仅在桌面端本地消费正式通知，不新增后端 TTS 服务，也不扩展新的 JSON-RPC 方法或协议字段。 +- 当前桌面最小消费范围为：悬浮球窗口启动成功后的提示音、选中文本与剪贴板承接提醒，以及 `approval.pending`、非 `bubble` 的 `delivery.ready` 本地语音提示。 +- 第一版语音播报只在桌面端本地消费已有入口与正式通知，不新增后端 TTS 服务，也不扩展新的 JSON-RPC 方法或协议字段。 ### 3.6 下载设置 From 8913d091c19beae282e198b72fff8a1b40585c39 Mon Sep 17 00:00:00 2001 From: xgopilot Date: Wed, 13 May 2026 11:13:07 +0000 Subject: [PATCH 4/9] fix(desktop): stabilize shell-ball voice prompt dedupe Generated with [codeagent](https://github.com/qbox/codeagent) Co-authored-by: Blackcloudss <161042579+Blackcloudss@users.noreply.github.com> --- .../src/features/shell-ball/ShellBallApp.tsx | 85 +++++++++++++------ .../shell-ball/shellBall.contract.test.ts | 8 +- 2 files changed, 65 insertions(+), 28 deletions(-) diff --git a/apps/desktop/src/features/shell-ball/ShellBallApp.tsx b/apps/desktop/src/features/shell-ball/ShellBallApp.tsx index 973d7294..32522e38 100644 --- a/apps/desktop/src/features/shell-ball/ShellBallApp.tsx +++ b/apps/desktop/src/features/shell-ball/ShellBallApp.tsx @@ -13,6 +13,7 @@ import { ShellBallAttachmentTray } from "./components/ShellBallAttachmentTray"; import { ShellBallBubbleZone } from "./components/ShellBallBubbleZone"; import { ShellBallInputBar } from "./components/ShellBallInputBar"; import { ShellBallVoiceHints } from "./components/ShellBallVoiceHints"; +import { areShellBallSelectionSnapshotsEqual } from "./selection/selection.provider"; import type { ShellBallSelectionSnapshot } from "./selection/selection.types"; import { useShellBallInteraction } from "./useShellBallInteraction"; import { getShellBallMotionConfig } from "./shellBall.motion"; @@ -131,6 +132,11 @@ type ShellBallClipboardPrompt = { expiresAt: number; }; +type ShellBallClipboardVoicePrompt = { + expiresAt: number; + text: string; +}; + async function pickShellBallFiles(): Promise { const result = await invoke("pick_shell_ball_files"); return Array.isArray(result) ? result : []; @@ -402,8 +408,8 @@ export function ShellBallApp() { const startupGreetingPlayedRef = useRef(false); // Local prompt events can replay while helper windows sync state, so keep the // last spoken keys here to avoid duplicate demo-time voice spam. - const selectionPromptVoiceKeyRef = useRef(""); - const clipboardPromptVoiceTextRef = useRef(""); + const selectionPromptVoiceSnapshotRef = useRef(null); + const clipboardPromptVoiceRef = useRef(null); const dragDropHandlersRef = useRef<{ handleDroppedFiles: (paths: string[]) => Promise | void; }>({ @@ -923,6 +929,16 @@ export function ShellBallApp() { setTextDragActive(false); }, []); + const clearSelectionPrompt = useCallback(() => { + selectionPromptVoiceSnapshotRef.current = null; + setSelectionPrompt(null); + }, []); + + const clearClipboardPrompt = useCallback(() => { + clipboardPromptVoiceRef.current = null; + setClipboardPrompt(null); + }, []); + const handleWindowTextDrag = useCallback((event: DragEvent) => { if (!shouldAcceptShellBallTextDrop(event.dataTransfer)) { clearTextDragState(); @@ -972,9 +988,9 @@ export function ShellBallApp() { window.clearTimeout(selectionPromptClearTimeoutRef.current); selectionPromptClearTimeoutRef.current = null; } - setSelectionPrompt(null); + clearSelectionPrompt(); } - }, [visualState]); + }, [clearSelectionPrompt, visualState]); useEffect(() => { if (selectionPrompt === null) { @@ -987,19 +1003,19 @@ export function ShellBallApp() { const updatedAtMs = resolveShellBallSelectionUpdatedAtMs(selectionPrompt.updated_at); if (updatedAtMs === null) { - setSelectionPrompt(null); + clearSelectionPrompt(); return; } const remainingMs = updatedAtMs + SHELL_BALL_SELECTION_PROMPT_WINDOW_MS - Date.now(); if (remainingMs <= 0) { - setSelectionPrompt(null); + clearSelectionPrompt(); return; } selectionPromptExpiryTimeoutRef.current = window.setTimeout(() => { selectionPromptExpiryTimeoutRef.current = null; - setSelectionPrompt(null); + clearSelectionPrompt(); }, remainingMs); return () => { @@ -1008,7 +1024,7 @@ export function ShellBallApp() { selectionPromptExpiryTimeoutRef.current = null; } }; - }, [selectionPrompt]); + }, [clearSelectionPrompt, selectionPrompt]); useEffect(() => { if (clipboardPrompt === null) { @@ -1021,13 +1037,13 @@ export function ShellBallApp() { const remainingMs = clipboardPrompt.expiresAt - Date.now(); if (remainingMs <= 0) { - setClipboardPrompt(null); + clearClipboardPrompt(); return; } clipboardPromptClearTimeoutRef.current = window.setTimeout(() => { clipboardPromptClearTimeoutRef.current = null; - setClipboardPrompt(null); + clearClipboardPrompt(); }, remainingMs); return () => { @@ -1036,7 +1052,7 @@ export function ShellBallApp() { clipboardPromptClearTimeoutRef.current = null; } }; - }, [clipboardPrompt]); + }, [clearClipboardPrompt, clipboardPrompt]); useEffect(() => { const currentWindow = getCurrentWindow(); @@ -1057,9 +1073,11 @@ export function ShellBallApp() { } const normalizedSelectionText = payload.snapshot.text.trim(); - const voiceKey = `${payload.snapshot.updated_at}:${normalizedSelectionText}`; - if (normalizedSelectionText.length > 0 && selectionPromptVoiceKeyRef.current !== voiceKey) { - selectionPromptVoiceKeyRef.current = voiceKey; + if ( + normalizedSelectionText.length > 0 + && !areShellBallSelectionSnapshotsEqual(selectionPromptVoiceSnapshotRef.current, payload.snapshot) + ) { + selectionPromptVoiceSnapshotRef.current = payload.snapshot; void speakShellBallSelectionDetectedNotification(); } @@ -1076,7 +1094,7 @@ export function ShellBallApp() { // a short debounce before retiring the alert opportunity. selectionPromptClearTimeoutRef.current = window.setTimeout(() => { selectionPromptClearTimeoutRef.current = null; - setSelectionPrompt(null); + clearSelectionPrompt(); }, SHELL_BALL_SELECTION_PROMPT_CLEAR_DELAY_MS); }) .then((unlisten) => { @@ -1096,7 +1114,7 @@ export function ShellBallApp() { } cleanup?.(); }; - }, []); + }, [clearSelectionPrompt]); useEffect(() => { const currentWindow = getCurrentWindow(); @@ -1111,14 +1129,21 @@ export function ShellBallApp() { void currentWindow .listen(shellBallWindowSyncEvents.clipboardSnapshot, ({ payload }) => { if (payload.text.trim() === "") { - clipboardPromptVoiceTextRef.current = ""; - setClipboardPrompt(null); + clearClipboardPrompt(); return; } const normalizedClipboardText = payload.text.trim(); - if (clipboardPromptVoiceTextRef.current !== normalizedClipboardText) { - clipboardPromptVoiceTextRef.current = normalizedClipboardText; + const activeVoicePrompt = clipboardPromptVoiceRef.current; + if ( + activeVoicePrompt === null + || activeVoicePrompt.text !== normalizedClipboardText + || activeVoicePrompt.expiresAt <= Date.now() + ) { + clipboardPromptVoiceRef.current = { + text: normalizedClipboardText, + expiresAt: Date.now() + SHELL_BALL_CLIPBOARD_PROMPT_WINDOW_MS, + }; void speakShellBallClipboardDetectedNotification(); } @@ -1140,7 +1165,7 @@ export function ShellBallApp() { disposed = true; cleanup?.(); }; - }, []); + }, [clearClipboardPrompt]); const handleMascotPrimaryAction = useCallback(() => { if (isShellBallSelectionPromptActive(selectionPrompt)) { @@ -1154,29 +1179,37 @@ export function ShellBallApp() { return; } - setSelectionPrompt(null); + clearSelectionPrompt(); void handleCoordinatorSelectedTextPrompt(activeSelectionPrompt); return; } if (selectionPrompt !== null) { - setSelectionPrompt(null); + clearSelectionPrompt(); return; } if (clipboardPrompt !== null) { if (!isShellBallClipboardPromptActive(clipboardPrompt)) { - setClipboardPrompt(null); + clearClipboardPrompt(); return; } - setClipboardPrompt(null); + clearClipboardPrompt(); void handleCoordinatorClipboardPrompt(clipboardPrompt.text); return; } handlePrimaryClick(); - }, [clipboardPrompt, handleCoordinatorClipboardPrompt, handleCoordinatorSelectedTextPrompt, handlePrimaryClick, selectionPrompt]); + }, [ + clearClipboardPrompt, + clearSelectionPrompt, + clipboardPrompt, + handleCoordinatorClipboardPrompt, + handleCoordinatorSelectedTextPrompt, + handlePrimaryClick, + selectionPrompt, + ]); const handleDockAwareRegionEnter = useCallback((event: ReactPointerEvent) => { void event; diff --git a/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts b/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts index d59bfc62..17adb041 100644 --- a/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts +++ b/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts @@ -8929,8 +8929,12 @@ test("shell-ball voice notifications are consumed locally from startup and forma assert.match(appSource, /speakShellBallSelectionDetectedNotification/); assert.match(appSource, /speakShellBallStartupGreeting/); assert.match(appSource, /const startupGreetingPlayedRef = useRef\(false\);/); - assert.match(appSource, /const selectionPromptVoiceKeyRef = useRef\(""\);/); - assert.match(appSource, /const clipboardPromptVoiceTextRef = useRef\(""\);/); + assert.match(appSource, /const selectionPromptVoiceSnapshotRef = useRef\(null\);/); + assert.match(appSource, /const clipboardPromptVoiceRef = useRef\(null\);/); + assert.match(appSource, /!areShellBallSelectionSnapshotsEqual\(selectionPromptVoiceSnapshotRef\.current, payload\.snapshot\)/); + assert.doesNotMatch(appSource, /const voiceKey = `\$\{payload\.snapshot\.updated_at\}:\$\{normalizedSelectionText\}`;/); + assert.match(appSource, /clipboardPromptVoiceRef\.current = null;\s*setClipboardPrompt\(null\);/); + assert.match(appSource, /activeVoicePrompt\.expiresAt <= Date\.now\(\)/); assert.match(appSource, /startupGreetingPlayedRef\.current = true;\s*void speakShellBallStartupGreeting\(\);/); assert.match(appSource, /void speakShellBallSelectionDetectedNotification\(\);/); assert.match(appSource, /void speakShellBallClipboardDetectedNotification\(\);/); From ea9e3060163f1ebd433476a54c6c8a657cfe3ac0 Mon Sep 17 00:00:00 2001 From: xgopilot Date: Wed, 13 May 2026 11:39:20 +0000 Subject: [PATCH 5/9] fix(desktop): guard stale shell-ball voice playback Generated with [codeagent](https://github.com/qbox/codeagent) Co-authored-by: Blackcloudss <161042579+Blackcloudss@users.noreply.github.com> --- .../src/features/shell-ball/shellBall.contract.test.ts | 3 +++ apps/desktop/src/services/voiceNotificationService.ts | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts b/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts index 17adb041..a17559bf 100644 --- a/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts +++ b/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts @@ -8947,6 +8947,9 @@ test("shell-ball voice notifications are consumed locally from startup and forma assert.match(voiceServiceSource, /const APPROVAL_PENDING_TEXT = "有一个操作需要你确认";/); assert.match(voiceServiceSource, /const DELIVERY_READY_TEXT = "任务结果已准备好";/); assert.match(voiceServiceSource, /const VOICE_LIST_READY_TIMEOUT_MS = 600;/); + assert.match(voiceServiceSource, /let latestVoiceNotificationRequestId = 0;/); + assert.match(voiceServiceSource, /const requestId = \+\+latestVoiceNotificationRequestId;/); + assert.match(voiceServiceSource, /if \(requestId !== latestVoiceNotificationRequestId\) \{\s*return false;\s*\}/); assert.match(voiceServiceSource, /addEventListener\("voiceschanged", handleVoicesChanged\)/); assert.match(voiceServiceSource, /return payload\.delivery_result\.type === "bubble" \? null : DELIVERY_READY_TEXT;/); }); diff --git a/apps/desktop/src/services/voiceNotificationService.ts b/apps/desktop/src/services/voiceNotificationService.ts index 7e7676f8..e65f40f3 100644 --- a/apps/desktop/src/services/voiceNotificationService.ts +++ b/apps/desktop/src/services/voiceNotificationService.ts @@ -29,6 +29,7 @@ const CLIPBOARD_DETECTED_TEXT = "检测到剪贴板内容"; const APPROVAL_PENDING_TEXT = "有一个操作需要你确认"; const DELIVERY_READY_TEXT = "任务结果已准备好"; const VOICE_LIST_READY_TIMEOUT_MS = 600; +let latestVoiceNotificationRequestId = 0; function normalizeVoiceType(voiceType: string) { return voiceType.trim().toLowerCase(); @@ -224,6 +225,7 @@ async function speakVoiceNotification(input: { kind: VoiceNotificationKind; payload?: ApprovalPendingNotification | DeliveryReadyNotification; }) { + const requestId = ++latestVoiceNotificationRequestId; const host = getVoiceNotificationHost(); if (host === null) { return false; @@ -248,6 +250,12 @@ async function speakVoiceNotification(input: { voices: availableVoices, }); + // Voice discovery can resolve out of order during startup, so only the most + // recent local notification is allowed to claim the shared synthesizer. + if (requestId !== latestVoiceNotificationRequestId) { + return false; + } + utterance.lang = language; if (resolvedVoice !== null) { utterance.voice = resolvedVoice as SpeechSynthesisVoice; From fe2a68d32185352b1d36d1aed885792e28dffd76 Mon Sep 17 00:00:00 2001 From: xgopilot Date: Wed, 13 May 2026 12:08:02 +0000 Subject: [PATCH 6/9] fix(desktop): tighten shell-ball voice notification gating Generated with [codeagent](https://github.com/qbox/codeagent) Co-authored-by: Blackcloudss <161042579+Blackcloudss@users.noreply.github.com> --- .../control-panel/ControlPanelApp.tsx | 4 +- .../src/features/shell-ball/ShellBallApp.tsx | 78 ++++++++++++++++++- .../shell-ball/components/ShellBallMascot.tsx | 4 +- .../shell-ball/shellBall.contract.test.ts | 68 +++++++++++++++- .../src/services/voiceNotificationService.ts | 56 ++++++++++--- docs/control-panel-settings.md | 6 +- 6 files changed, 196 insertions(+), 20 deletions(-) diff --git a/apps/desktop/src/features/control-panel/ControlPanelApp.tsx b/apps/desktop/src/features/control-panel/ControlPanelApp.tsx index 013d8ed7..76b9253f 100644 --- a/apps/desktop/src/features/control-panel/ControlPanelApp.tsx +++ b/apps/desktop/src/features/control-panel/ControlPanelApp.tsx @@ -1281,7 +1281,7 @@ export function ControlPanelApp() { updateSettings((current) => ({ @@ -1296,7 +1296,7 @@ export function ControlPanelApp() { now; } +/** + * Restricts the idle click greeting to the true resting shell-ball state so + * local acknowledgement speech never competes with task intake or onboarding. + * + * @param input Local shell-ball state that can suppress the idle click voice. + * @returns Whether an idle single click should schedule the local greeting. + */ +export function shouldSpeakShellBallIdleClickGreeting(input: { + clipboardPromptActive: boolean; + hasPendingAgentLoading: boolean; + hasPendingApproval: boolean; + hasPendingFiles: boolean; + hasVisibleBubbleItems: boolean; + hasWrittenInput: boolean; + onboardingActive: boolean; + selectionPromptActive: boolean; + visualState: ShellBallVisualState; +}) { + return ( + input.visualState === "idle" && + !input.selectionPromptActive && + !input.clipboardPromptActive && + !input.hasVisibleBubbleItems && + !input.hasPendingAgentLoading && + !input.hasPendingApproval && + !input.hasWrittenInput && + !input.hasPendingFiles && + !input.onboardingActive + ); +} + export function resolveShellBallInlineInputMode(input: { shouldRenderInlineInput: boolean; snapshotInputBarMode: ShellBallInputBarMode; @@ -405,6 +438,7 @@ export function ShellBallApp() { const previousVisualStateRef = useRef(visualState); const transitionQueueRef = useRef(Promise.resolve()); const edgeDockRevealHideTimeoutRef = useRef(null); + const idleClickGreetingTimeoutRef = useRef(null); const startupGreetingPlayedRef = useRef(false); // Local prompt events can replay while helper windows sync state, so keep the // last spoken keys here to avoid duplicate demo-time voice spam. @@ -482,6 +516,18 @@ export function ShellBallApp() { const hasPendingAgentLoading = visibleBubbleItems.some((item) => item.role === "agent" && item.desktop.presentationHint === "loading"); const hasPendingApproval = snapshot.bubbleItems.some((item) => item.desktop.inlineApproval?.status === "idle"); const hasAlertOpportunity = isShellBallSelectionPromptActive(selectionPrompt) || isShellBallClipboardPromptActive(clipboardPrompt); + const onboardingActive = onboardingSession?.isOpen === true; + const idleClickGreetingEligible = shouldSpeakShellBallIdleClickGreeting({ + clipboardPromptActive: isShellBallClipboardPromptActive(clipboardPrompt), + hasPendingAgentLoading, + hasPendingApproval, + hasPendingFiles: pendingFiles.length > 0, + hasVisibleBubbleItems: visibleBubbleItems.length > 0, + hasWrittenInput: inputValue.trim() !== "", + onboardingActive, + selectionPromptActive: isShellBallSelectionPromptActive(selectionPrompt), + visualState, + }); useEffect(() => { if (typeof window === "undefined") { @@ -541,6 +587,13 @@ export function ShellBallApp() { } }, []); + const cancelIdleClickGreeting = useCallback(() => { + if (idleClickGreetingTimeoutRef.current !== null) { + window.clearTimeout(idleClickGreetingTimeoutRef.current); + idleClickGreetingTimeoutRef.current = null; + } + }, []); + const reportInteractiveRegions = useCallback(async () => { const currentWindow = getCurrentWindow(); @@ -866,12 +919,21 @@ export function ShellBallApp() { // Reset native mascot hotspot state only when the shell-ball host actually // unmounts so ordinary frame updates do not churn IPC requests. return () => { + cancelIdleClickGreeting(); cancelEdgeDockRevealHide(); void setShellBallInteractiveRegions([]); void setShellBallPressLock(false); lastReportedInteractiveRegionsRef.current = ""; }; - }, [cancelEdgeDockRevealHide]); + }, [cancelEdgeDockRevealHide, cancelIdleClickGreeting]); + + useEffect(() => { + if (idleClickGreetingEligible) { + return; + } + + cancelIdleClickGreeting(); + }, [cancelIdleClickGreeting, idleClickGreetingEligible]); useEffect(() => { if (getCurrentWindow().label !== shellBallWindowLabels.ball) { @@ -882,6 +944,8 @@ export function ShellBallApp() { }, [inputFocused, reportInteractiveRegions]); function handleDoubleClick() { + cancelIdleClickGreeting(); + if (!shouldOpenDashboardFromDoubleClick) { return; } @@ -1185,6 +1249,7 @@ export function ShellBallApp() { } if (selectionPrompt !== null) { + cancelIdleClickGreeting(); clearSelectionPrompt(); return; } @@ -1196,18 +1261,29 @@ export function ShellBallApp() { } clearClipboardPrompt(); + cancelIdleClickGreeting(); void handleCoordinatorClipboardPrompt(clipboardPrompt.text); return; } + if (idleClickGreetingEligible) { + cancelIdleClickGreeting(); + idleClickGreetingTimeoutRef.current = window.setTimeout(() => { + idleClickGreetingTimeoutRef.current = null; + void speakShellBallIdleGreeting(); + }, SHELL_BALL_IDLE_CLICK_GREETING_DELAY_MS); + } + handlePrimaryClick(); }, [ + cancelIdleClickGreeting, clearClipboardPrompt, clearSelectionPrompt, clipboardPrompt, handleCoordinatorClipboardPrompt, handleCoordinatorSelectedTextPrompt, handlePrimaryClick, + idleClickGreetingEligible, selectionPrompt, ]); diff --git a/apps/desktop/src/features/shell-ball/components/ShellBallMascot.tsx b/apps/desktop/src/features/shell-ball/components/ShellBallMascot.tsx index 4d899203..0727784f 100644 --- a/apps/desktop/src/features/shell-ball/components/ShellBallMascot.tsx +++ b/apps/desktop/src/features/shell-ball/components/ShellBallMascot.tsx @@ -73,7 +73,9 @@ export function getShellBallMascotHotspotGestureAction(input: { } if (input.gesture === "single_click") { - return input.selectionIndicatorVisible || input.alertOpportunityAvailable ? "primary_click" : "noop"; + return input.selectionIndicatorVisible || input.alertOpportunityAvailable || input.visualState === "idle" + ? "primary_click" + : "noop"; } if (canTriggerShellBallMascotSecondaryGestures(input.visualState)) { diff --git a/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts b/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts index a17559bf..39fa8728 100644 --- a/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts +++ b/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts @@ -30,6 +30,7 @@ import { getShellBallMotionConfig } from "./shellBall.motion"; import { collectShellBallSpeechTranscript, composeShellBallSpeechDraft } from "./shellBall.speech"; import { resolveDeliveryReadyVoiceNotificationText, + resolveShellBallIdleClickGreetingText, resolveShellBallStartupGreetingText, resolveVoiceNotificationVoice, } from "../../services/voiceNotificationService"; @@ -44,6 +45,7 @@ import { normalizeShellBallFloatingSize, resolveShellBallInlineInputMode, ShellBallApp, + shouldSpeakShellBallIdleClickGreeting, shouldRetainShellBallEdgeDockReveal, shouldArmShellBallTextDropTarget, shouldShowShellBallFileDropOverlay, @@ -2539,6 +2541,7 @@ test("shell-ball speech transcript collection merges recognition chunks", () => test("voice notification helpers keep startup greeting and formal delivery reminders short", () => { assert.equal(resolveShellBallStartupGreetingText(), "CialloClaw 已启动"); + assert.equal(resolveShellBallIdleClickGreetingText(), "CialloClaw"); assert.equal( resolveDeliveryReadyVoiceNotificationText({ task_id: "task-bubble-delivery", @@ -2566,10 +2569,11 @@ test("voice notification helpers prefer the saved voice type and matching locale voices: [ { lang: "en-US", name: "Alloy" }, { lang: "ja-JP", name: "Sakura Female" }, + { lang: "zh-CN", name: "Xiaoyi" }, { lang: "zh-CN", name: "Xiaoxiao" }, ], }); - assert.equal(resolvedDefaultFemale?.name, "Xiaoxiao"); + assert.equal(resolvedDefaultFemale?.name, "Xiaoyi"); const resolvedCustomVoice = resolveVoiceNotificationVoice({ language: "zh-CN", @@ -8223,7 +8227,7 @@ test("shell-ball floating size normalization falls back to medium", () => { assert.equal(normalizeShellBallFloatingSize(undefined), "medium"); }); -test("shell-ball mascot hotspot policy only opens primary click for selected-text prompts", () => { +test("shell-ball mascot hotspot policy opens primary click for idle acknowledgement and prompt intake", () => { assert.equal( getShellBallMascotHotspotGestureAction({ visualState: "voice_locked", @@ -8239,7 +8243,7 @@ test("shell-ball mascot hotspot policy only opens primary click for selected-tex gesture: "single_click", suppressed: false, }), - "noop", + "primary_click", ); assert.equal( @@ -8262,6 +8266,53 @@ test("shell-ball mascot hotspot policy only opens primary click for selected-tex ); }); +test("shell-ball idle click greeting only speaks from the true resting state", () => { + assert.equal( + shouldSpeakShellBallIdleClickGreeting({ + clipboardPromptActive: false, + hasPendingAgentLoading: false, + hasPendingApproval: false, + hasPendingFiles: false, + hasVisibleBubbleItems: false, + hasWrittenInput: false, + onboardingActive: false, + selectionPromptActive: false, + visualState: "idle", + }), + true, + ); + + assert.equal( + shouldSpeakShellBallIdleClickGreeting({ + clipboardPromptActive: false, + hasPendingAgentLoading: false, + hasPendingApproval: false, + hasPendingFiles: false, + hasVisibleBubbleItems: true, + hasWrittenInput: false, + onboardingActive: false, + selectionPromptActive: false, + visualState: "idle", + }), + false, + ); + + assert.equal( + shouldSpeakShellBallIdleClickGreeting({ + clipboardPromptActive: false, + hasPendingAgentLoading: false, + hasPendingApproval: false, + hasPendingFiles: false, + hasVisibleBubbleItems: false, + hasWrittenInput: false, + onboardingActive: false, + selectionPromptActive: false, + visualState: "hover_input", + }), + false, + ); +}); + test("shell-ball mascot hotspot policy opens dashboard only from resting double click", () => { assert.equal( getShellBallMascotHotspotGestureAction({ @@ -8922,12 +8973,19 @@ test("shell-ball routes active resumable text follow-ups through task steer", () test("shell-ball voice notifications are consumed locally from startup and formal task notifications", () => { const appSource = readFileSync(resolve(desktopRoot, "src/features/shell-ball/ShellBallApp.tsx"), "utf8"); + const mascotSource = readFileSync(resolve(desktopRoot, "src/features/shell-ball/components/ShellBallMascot.tsx"), "utf8"); const coordinatorSource = readFileSync(resolve(desktopRoot, "src/features/shell-ball/useShellBallCoordinator.ts"), "utf8"); const voiceServiceSource = readFileSync(resolve(desktopRoot, "src/services/voiceNotificationService.ts"), "utf8"); assert.match(appSource, /speakShellBallClipboardDetectedNotification/); + assert.match(appSource, /speakShellBallIdleGreeting/); assert.match(appSource, /speakShellBallSelectionDetectedNotification/); assert.match(appSource, /speakShellBallStartupGreeting/); + assert.match(appSource, /const SHELL_BALL_IDLE_CLICK_GREETING_DELAY_MS = 220;/); + assert.match(appSource, /const idleClickGreetingTimeoutRef = useRef\(null\);/); + assert.match(appSource, /export function shouldSpeakShellBallIdleClickGreeting/); + assert.match(appSource, /void speakShellBallIdleGreeting\(\);/); + assert.match(appSource, /cancelIdleClickGreeting\(\);/); assert.match(appSource, /const startupGreetingPlayedRef = useRef\(false\);/); assert.match(appSource, /const selectionPromptVoiceSnapshotRef = useRef\(null\);/); assert.match(appSource, /const clipboardPromptVoiceRef = useRef\(null\);/); @@ -8938,10 +8996,12 @@ test("shell-ball voice notifications are consumed locally from startup and forma assert.match(appSource, /startupGreetingPlayedRef\.current = true;\s*void speakShellBallStartupGreeting\(\);/); assert.match(appSource, /void speakShellBallSelectionDetectedNotification\(\);/); assert.match(appSource, /void speakShellBallClipboardDetectedNotification\(\);/); + assert.match(mascotSource, /input\.selectionIndicatorVisible \|\| input\.alertOpportunityAvailable \|\| input\.visualState === "idle"/); assert.match(coordinatorSource, /import \{ speakApprovalPendingNotification, speakDeliveryReadyNotification \} from "@\/services\/voiceNotificationService";/); assert.match(coordinatorSource, /speakApprovalPendingNotification\(\{\s*approval_request: input\.approvalRequest,\s*task_id: input\.taskId,\s*\}\);/); assert.match(coordinatorSource, /speakDeliveryReadyNotification\(\{\s*delivery_result: input\.deliveryResult,\s*task_id: input\.taskId,\s*\}\);/); assert.match(voiceServiceSource, /const STARTUP_GREETING_TEXT = "CialloClaw 已启动";/); + assert.match(voiceServiceSource, /const IDLE_CLICK_GREETING_TEXT = "CialloClaw";/); assert.match(voiceServiceSource, /const SELECTION_DETECTED_TEXT = "检测到选中文本";/); assert.match(voiceServiceSource, /const CLIPBOARD_DETECTED_TEXT = "检测到剪贴板内容";/); assert.match(voiceServiceSource, /const APPROVAL_PENDING_TEXT = "有一个操作需要你确认";/); @@ -8950,6 +9010,8 @@ test("shell-ball voice notifications are consumed locally from startup and forma assert.match(voiceServiceSource, /let latestVoiceNotificationRequestId = 0;/); assert.match(voiceServiceSource, /const requestId = \+\+latestVoiceNotificationRequestId;/); assert.match(voiceServiceSource, /if \(requestId !== latestVoiceNotificationRequestId\) \{\s*return false;\s*\}/); + assert.match(voiceServiceSource, /const latestSettings = getVoiceNotificationSettings\(\);/); + assert.match(voiceServiceSource, /if \(!latestSettings\.voice_notification_enabled\) \{\s*return false;\s*\}/); assert.match(voiceServiceSource, /addEventListener\("voiceschanged", handleVoicesChanged\)/); assert.match(voiceServiceSource, /return payload\.delivery_result\.type === "bubble" \? null : DELIVERY_READY_TEXT;/); }); diff --git a/apps/desktop/src/services/voiceNotificationService.ts b/apps/desktop/src/services/voiceNotificationService.ts index e65f40f3..3ce96c17 100644 --- a/apps/desktop/src/services/voiceNotificationService.ts +++ b/apps/desktop/src/services/voiceNotificationService.ts @@ -3,6 +3,7 @@ import { loadSettings } from "./settingsService"; type VoiceNotificationKind = | "startup_greeting" + | "idle_click_greeting" | "selection_detected" | "clipboard_detected" | "approval_pending" @@ -24,6 +25,7 @@ type SpeechSynthesisLike = { }; const STARTUP_GREETING_TEXT = "CialloClaw 已启动"; +const IDLE_CLICK_GREETING_TEXT = "CialloClaw"; const SELECTION_DETECTED_TEXT = "检测到选中文本"; const CLIPBOARD_DETECTED_TEXT = "检测到剪贴板内容"; const APPROVAL_PENDING_TEXT = "有一个操作需要你确认"; @@ -60,13 +62,14 @@ function getVoiceTypePreferences(voiceType: string, language: string) { if (normalizedVoiceType === "default_female") { return [ language.toLowerCase(), - "female", - "woman", - "girl", - "xiaoxiao", "xiaoyi", + "xiaoxiao", "tingting", "yunxia", + "girl", + "young", + "female", + "woman", ]; } @@ -159,11 +162,24 @@ export function resolveShellBallStartupGreetingText() { return STARTUP_GREETING_TEXT; } +/** + * Builds the idle-click acknowledgement spoken from the resting floating ball. + * + * @returns The short idle greeting used for single-click acknowledgement. + */ +export function resolveShellBallIdleClickGreetingText() { + return IDLE_CLICK_GREETING_TEXT; +} + function resolveVoiceNotificationText(kind: VoiceNotificationKind, payload?: ApprovalPendingNotification | DeliveryReadyNotification) { if (kind === "startup_greeting") { return resolveShellBallStartupGreetingText(); } + if (kind === "idle_click_greeting") { + return resolveShellBallIdleClickGreetingText(); + } + if (kind === "selection_detected") { return SELECTION_DETECTED_TEXT; } @@ -231,8 +247,8 @@ async function speakVoiceNotification(input: { return false; } - const settings = getVoiceNotificationSettings(); - if (!settings.voice_notification_enabled) { + const initialSettings = getVoiceNotificationSettings(); + if (!initialSettings.voice_notification_enabled) { return false; } @@ -244,11 +260,6 @@ async function speakVoiceNotification(input: { const language = resolveVoiceNotificationLanguage(); const utterance = new host.utteranceConstructor(text); const availableVoices = await waitForVoiceNotificationVoices(host.synthesizer); - const resolvedVoice = resolveVoiceNotificationVoice({ - language, - voiceType: settings.voice_type, - voices: availableVoices, - }); // Voice discovery can resolve out of order during startup, so only the most // recent local notification is allowed to claim the shared synthesizer. @@ -256,6 +267,20 @@ async function speakVoiceNotification(input: { return false; } + // The control panel can change local voice settings while the browser is + // still resolving system voices, so re-read the latest snapshot right before + // claiming the shared synthesizer. + const latestSettings = getVoiceNotificationSettings(); + if (!latestSettings.voice_notification_enabled) { + return false; + } + + const resolvedVoice = resolveVoiceNotificationVoice({ + language, + voiceType: latestSettings.voice_type, + voices: availableVoices, + }); + utterance.lang = language; if (resolvedVoice !== null) { utterance.voice = resolvedVoice as SpeechSynthesisVoice; @@ -275,6 +300,15 @@ export function speakShellBallStartupGreeting() { return speakVoiceNotification({ kind: "startup_greeting" }); } +/** + * Plays a short idle acknowledgement when the resting shell-ball is clicked. + * + * @returns Whether the local desktop runtime attempted speech playback. + */ +export function speakShellBallIdleGreeting() { + return speakVoiceNotification({ kind: "idle_click_greeting" }); +} + /** * Plays a short reminder when the shell-ball detects a fresh text selection. * diff --git a/docs/control-panel-settings.md b/docs/control-panel-settings.md index 2bed9fcf..dd8e23a4 100644 --- a/docs/control-panel-settings.md +++ b/docs/control-panel-settings.md @@ -158,11 +158,12 @@ #### 3.5.2 建议拆分的设置项 1. **声音总开关** - - 控制是否播放悬浮球启动、文本承接提醒、授权提醒、正式结果提醒等本地语音提示。 + - 控制是否播放悬浮球启动、空闲单击确认、文本承接提醒、授权提醒、正式结果提醒等本地语音提示。 2. **提示音/语音风格选择** - 例如：不同声线版本的“Ciallo”。 3. **典型触发场景说明** - 悬浮球启动完成 + - 空闲单击悬浮球 - 检测到选中文本或剪贴板内容 - 授权提醒 - 正式结果已准备好 @@ -171,8 +172,9 @@ - 开启声音时，应支持试听。 - 关闭总开关后，风格选择区域应置灰但保留当前已选项。 +- 控制面板里的声音草稿只有在保存后才会同步到悬浮球等其他桌面窗口；未保存的面板改动不应直接改变当前本地播报行为。 - 声音设置应明确说明是否影响系统通知、仅影响应用内反馈，还是两者都影响。 -- 当前桌面最小消费范围为：悬浮球窗口启动成功后的提示音、选中文本与剪贴板承接提醒，以及 `approval.pending`、非 `bubble` 的 `delivery.ready` 本地语音提示。 +- 当前桌面最小消费范围为：悬浮球窗口启动成功后的提示音、空闲单击悬浮球时的本地确认、选中文本与剪贴板承接提醒，以及 `approval.pending`、非 `bubble` 的 `delivery.ready` 本地语音提示。 - 第一版语音播报只在桌面端本地消费已有入口与正式通知，不新增后端 TTS 服务，也不扩展新的 JSON-RPC 方法或协议字段。 ### 3.6 下载设置 From 2b6794d4371236e9cc72834a88c1fa5ff5af95cc Mon Sep 17 00:00:00 2001 From: Cirno9 <161042579+Blackcloudss@users.noreply.github.com> Date: Wed, 13 May 2026 20:27:58 +0800 Subject: [PATCH 7/9] Update IDLE_CLICK_GREETING_TEXT in contract tes --- apps/desktop/src/features/shell-ball/shellBall.contract.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts b/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts index 39fa8728..9feb19e0 100644 --- a/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts +++ b/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts @@ -9001,7 +9001,7 @@ test("shell-ball voice notifications are consumed locally from startup and forma assert.match(coordinatorSource, /speakApprovalPendingNotification\(\{\s*approval_request: input\.approvalRequest,\s*task_id: input\.taskId,\s*\}\);/); assert.match(coordinatorSource, /speakDeliveryReadyNotification\(\{\s*delivery_result: input\.deliveryResult,\s*task_id: input\.taskId,\s*\}\);/); assert.match(voiceServiceSource, /const STARTUP_GREETING_TEXT = "CialloClaw 已启动";/); - assert.match(voiceServiceSource, /const IDLE_CLICK_GREETING_TEXT = "CialloClaw";/); + assert.match(voiceServiceSource, /const IDLE_CLICK_GREETING_TEXT = "Ciallo";/); assert.match(voiceServiceSource, /const SELECTION_DETECTED_TEXT = "检测到选中文本";/); assert.match(voiceServiceSource, /const CLIPBOARD_DETECTED_TEXT = "检测到剪贴板内容";/); assert.match(voiceServiceSource, /const APPROVAL_PENDING_TEXT = "有一个操作需要你确认";/); From b3bc87183fd6b3f9c73d1faa909108e3c2be0487 Mon Sep 17 00:00:00 2001 From: xgopilot Date: Wed, 13 May 2026 13:26:22 +0000 Subject: [PATCH 8/9] fix(desktop): refine shell-ball voice triggers Generated with [codeagent](https://github.com/qbox/codeagent) Co-authored-by: Blackcloudss <161042579+Blackcloudss@users.noreply.github.com> --- .../control-panel/ControlPanelApp.tsx | 25 ++-- .../src/features/shell-ball/ShellBallApp.tsx | 117 +----------------- .../shell-ball/shellBall.contract.test.ts | 105 +++++----------- apps/desktop/src/services/settingsService.ts | 3 +- .../src/services/voiceNotificationConfig.ts | 24 ++++ .../src/services/voiceNotificationService.ts | 102 ++++++++------- docs/control-panel-settings.md | 9 +- 7 files changed, 140 insertions(+), 245 deletions(-) create mode 100644 apps/desktop/src/services/voiceNotificationConfig.ts diff --git a/apps/desktop/src/features/control-panel/ControlPanelApp.tsx b/apps/desktop/src/features/control-panel/ControlPanelApp.tsx index 76b9253f..f8c99e37 100644 --- a/apps/desktop/src/features/control-panel/ControlPanelApp.tsx +++ b/apps/desktop/src/features/control-panel/ControlPanelApp.tsx @@ -45,6 +45,11 @@ import { type SecurityBudgetDisplaySettings, } from "@/services/securityBudgetDisplayService"; import { loadDesktopRuntimeDefaultsSnapshot, loadHydratedSettings } from "@/services/settingsService"; +import { + resolveVoiceNotificationVoicePreset, + VOICE_NOTIFICATION_VOICE_PRESET_OPTIONS, + type VoiceNotificationVoicePreset, +} from "@/services/voiceNotificationConfig"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { buildDesktopOnboardingPresentation } from "@/features/onboarding/onboardingGeometry"; import { @@ -510,11 +515,13 @@ function ToggleLine({ checked, description, disabled = false, label, onCheckedCh function ChoiceGroup({ className, + disabled = false, options, value, onValueChange, }: { className?: string; + disabled?: boolean; options: readonly ChoiceOption[]; value: T; onValueChange: (value: T) => void; @@ -522,7 +529,7 @@ function ChoiceGroup({ const classes = ["control-panel-shell__choice-group", className].filter(Boolean).join(" "); return ( -
+
{options.map((option) => { const checked = option.value === value; @@ -532,6 +539,7 @@ function ChoiceGroup({ type="button" role="radio" aria-checked={checked} + disabled={disabled} className="control-panel-shell__choice-option" data-state={checked ? "checked" : "unchecked"} onClick={() => onValueChange(option.value)} @@ -1281,7 +1289,7 @@ export function ControlPanelApp() { updateSettings((current) => ({ @@ -1296,19 +1304,20 @@ export function ControlPanelApp() { - + options={VOICE_NOTIFICATION_VOICE_PRESET_OPTIONS} + value={resolveVoiceNotificationVoicePreset(draft.settings.general.voice_type)} + onValueChange={(value: VoiceNotificationVoicePreset) => updateSettings((current) => ({ ...current, settings: { ...current.settings, - general: { ...current.settings.general, voice_type: event.target.value }, + general: { ...current.settings.general, voice_type: value }, }, })) } diff --git a/apps/desktop/src/features/shell-ball/ShellBallApp.tsx b/apps/desktop/src/features/shell-ball/ShellBallApp.tsx index b2ba4ea9..1923dbe1 100644 --- a/apps/desktop/src/features/shell-ball/ShellBallApp.tsx +++ b/apps/desktop/src/features/shell-ball/ShellBallApp.tsx @@ -13,7 +13,6 @@ import { ShellBallAttachmentTray } from "./components/ShellBallAttachmentTray"; import { ShellBallBubbleZone } from "./components/ShellBallBubbleZone"; import { ShellBallInputBar } from "./components/ShellBallInputBar"; import { ShellBallVoiceHints } from "./components/ShellBallVoiceHints"; -import { areShellBallSelectionSnapshotsEqual } from "./selection/selection.provider"; import type { ShellBallSelectionSnapshot } from "./selection/selection.types"; import { useShellBallInteraction } from "./useShellBallInteraction"; import { getShellBallMotionConfig } from "./shellBall.motion"; @@ -41,9 +40,7 @@ import { } from "../../platform/shellBallWindow"; import { loadSettings } from "../../services/settingsService"; import { - speakShellBallClipboardDetectedNotification, - speakShellBallIdleGreeting, - speakShellBallSelectionDetectedNotification, + speakShellBallSelectionClickGreeting, speakShellBallStartupGreeting, } from "../../services/voiceNotificationService"; import { openOrFocusDesktopWindow } from "../../platform/windowController"; @@ -80,7 +77,6 @@ const SHELL_BALL_SELECTION_PROMPT_WINDOW_MS = 10_000; const SHELL_BALL_CLIPBOARD_PROMPT_WINDOW_MS = 10_000; const SHELL_BALL_EDGE_DOCK_REVEAL_GUARD_PX = 16; const SHELL_BALL_EDGE_DOCK_REVEAL_HIDE_DELAY_MS = 90; -const SHELL_BALL_IDLE_CLICK_GREETING_DELAY_MS = 220; export function normalizeShellBallFloatingSize(size: string | null | undefined): ShellBallFloatingSize { if (size === "small" || size === "medium" || size === "large") { @@ -134,11 +130,6 @@ type ShellBallClipboardPrompt = { expiresAt: number; }; -type ShellBallClipboardVoicePrompt = { - expiresAt: number; - text: string; -}; - async function pickShellBallFiles(): Promise { const result = await invoke("pick_shell_ball_files"); return Array.isArray(result) ? result : []; @@ -240,37 +231,6 @@ export function isShellBallClipboardPromptActive( return prompt !== null && prompt.expiresAt > now; } -/** - * Restricts the idle click greeting to the true resting shell-ball state so - * local acknowledgement speech never competes with task intake or onboarding. - * - * @param input Local shell-ball state that can suppress the idle click voice. - * @returns Whether an idle single click should schedule the local greeting. - */ -export function shouldSpeakShellBallIdleClickGreeting(input: { - clipboardPromptActive: boolean; - hasPendingAgentLoading: boolean; - hasPendingApproval: boolean; - hasPendingFiles: boolean; - hasVisibleBubbleItems: boolean; - hasWrittenInput: boolean; - onboardingActive: boolean; - selectionPromptActive: boolean; - visualState: ShellBallVisualState; -}) { - return ( - input.visualState === "idle" && - !input.selectionPromptActive && - !input.clipboardPromptActive && - !input.hasVisibleBubbleItems && - !input.hasPendingAgentLoading && - !input.hasPendingApproval && - !input.hasWrittenInput && - !input.hasPendingFiles && - !input.onboardingActive - ); -} - export function resolveShellBallInlineInputMode(input: { shouldRenderInlineInput: boolean; snapshotInputBarMode: ShellBallInputBarMode; @@ -438,12 +398,7 @@ export function ShellBallApp() { const previousVisualStateRef = useRef(visualState); const transitionQueueRef = useRef(Promise.resolve()); const edgeDockRevealHideTimeoutRef = useRef(null); - const idleClickGreetingTimeoutRef = useRef(null); const startupGreetingPlayedRef = useRef(false); - // Local prompt events can replay while helper windows sync state, so keep the - // last spoken keys here to avoid duplicate demo-time voice spam. - const selectionPromptVoiceSnapshotRef = useRef(null); - const clipboardPromptVoiceRef = useRef(null); const dragDropHandlersRef = useRef<{ handleDroppedFiles: (paths: string[]) => Promise | void; }>({ @@ -516,18 +471,6 @@ export function ShellBallApp() { const hasPendingAgentLoading = visibleBubbleItems.some((item) => item.role === "agent" && item.desktop.presentationHint === "loading"); const hasPendingApproval = snapshot.bubbleItems.some((item) => item.desktop.inlineApproval?.status === "idle"); const hasAlertOpportunity = isShellBallSelectionPromptActive(selectionPrompt) || isShellBallClipboardPromptActive(clipboardPrompt); - const onboardingActive = onboardingSession?.isOpen === true; - const idleClickGreetingEligible = shouldSpeakShellBallIdleClickGreeting({ - clipboardPromptActive: isShellBallClipboardPromptActive(clipboardPrompt), - hasPendingAgentLoading, - hasPendingApproval, - hasPendingFiles: pendingFiles.length > 0, - hasVisibleBubbleItems: visibleBubbleItems.length > 0, - hasWrittenInput: inputValue.trim() !== "", - onboardingActive, - selectionPromptActive: isShellBallSelectionPromptActive(selectionPrompt), - visualState, - }); useEffect(() => { if (typeof window === "undefined") { @@ -587,13 +530,6 @@ export function ShellBallApp() { } }, []); - const cancelIdleClickGreeting = useCallback(() => { - if (idleClickGreetingTimeoutRef.current !== null) { - window.clearTimeout(idleClickGreetingTimeoutRef.current); - idleClickGreetingTimeoutRef.current = null; - } - }, []); - const reportInteractiveRegions = useCallback(async () => { const currentWindow = getCurrentWindow(); @@ -919,21 +855,12 @@ export function ShellBallApp() { // Reset native mascot hotspot state only when the shell-ball host actually // unmounts so ordinary frame updates do not churn IPC requests. return () => { - cancelIdleClickGreeting(); cancelEdgeDockRevealHide(); void setShellBallInteractiveRegions([]); void setShellBallPressLock(false); lastReportedInteractiveRegionsRef.current = ""; }; - }, [cancelEdgeDockRevealHide, cancelIdleClickGreeting]); - - useEffect(() => { - if (idleClickGreetingEligible) { - return; - } - - cancelIdleClickGreeting(); - }, [cancelIdleClickGreeting, idleClickGreetingEligible]); + }, [cancelEdgeDockRevealHide]); useEffect(() => { if (getCurrentWindow().label !== shellBallWindowLabels.ball) { @@ -944,8 +871,6 @@ export function ShellBallApp() { }, [inputFocused, reportInteractiveRegions]); function handleDoubleClick() { - cancelIdleClickGreeting(); - if (!shouldOpenDashboardFromDoubleClick) { return; } @@ -994,12 +919,10 @@ export function ShellBallApp() { }, []); const clearSelectionPrompt = useCallback(() => { - selectionPromptVoiceSnapshotRef.current = null; setSelectionPrompt(null); }, []); const clearClipboardPrompt = useCallback(() => { - clipboardPromptVoiceRef.current = null; setClipboardPrompt(null); }, []); @@ -1136,15 +1059,6 @@ export function ShellBallApp() { selectionPromptClearTimeoutRef.current = null; } - const normalizedSelectionText = payload.snapshot.text.trim(); - if ( - normalizedSelectionText.length > 0 - && !areShellBallSelectionSnapshotsEqual(selectionPromptVoiceSnapshotRef.current, payload.snapshot) - ) { - selectionPromptVoiceSnapshotRef.current = payload.snapshot; - void speakShellBallSelectionDetectedNotification(); - } - setSelectionPrompt(payload.snapshot); return; } @@ -1197,20 +1111,6 @@ export function ShellBallApp() { return; } - const normalizedClipboardText = payload.text.trim(); - const activeVoicePrompt = clipboardPromptVoiceRef.current; - if ( - activeVoicePrompt === null - || activeVoicePrompt.text !== normalizedClipboardText - || activeVoicePrompt.expiresAt <= Date.now() - ) { - clipboardPromptVoiceRef.current = { - text: normalizedClipboardText, - expiresAt: Date.now() + SHELL_BALL_CLIPBOARD_PROMPT_WINDOW_MS, - }; - void speakShellBallClipboardDetectedNotification(); - } - setClipboardPrompt({ text: payload.text, expiresAt: Date.now() + SHELL_BALL_CLIPBOARD_PROMPT_WINDOW_MS, @@ -1243,13 +1143,13 @@ export function ShellBallApp() { return; } + void speakShellBallSelectionClickGreeting(); clearSelectionPrompt(); void handleCoordinatorSelectedTextPrompt(activeSelectionPrompt); return; } if (selectionPrompt !== null) { - cancelIdleClickGreeting(); clearSelectionPrompt(); return; } @@ -1261,29 +1161,18 @@ export function ShellBallApp() { } clearClipboardPrompt(); - cancelIdleClickGreeting(); void handleCoordinatorClipboardPrompt(clipboardPrompt.text); return; } - if (idleClickGreetingEligible) { - cancelIdleClickGreeting(); - idleClickGreetingTimeoutRef.current = window.setTimeout(() => { - idleClickGreetingTimeoutRef.current = null; - void speakShellBallIdleGreeting(); - }, SHELL_BALL_IDLE_CLICK_GREETING_DELAY_MS); - } - handlePrimaryClick(); }, [ - cancelIdleClickGreeting, clearClipboardPrompt, clearSelectionPrompt, clipboardPrompt, handleCoordinatorClipboardPrompt, handleCoordinatorSelectedTextPrompt, handlePrimaryClick, - idleClickGreetingEligible, selectionPrompt, ]); diff --git a/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts b/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts index 9feb19e0..42c850aa 100644 --- a/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts +++ b/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts @@ -30,7 +30,7 @@ import { getShellBallMotionConfig } from "./shellBall.motion"; import { collectShellBallSpeechTranscript, composeShellBallSpeechDraft } from "./shellBall.speech"; import { resolveDeliveryReadyVoiceNotificationText, - resolveShellBallIdleClickGreetingText, + resolveShellBallSelectionClickGreetingText, resolveShellBallStartupGreetingText, resolveVoiceNotificationVoice, } from "../../services/voiceNotificationService"; @@ -45,7 +45,6 @@ import { normalizeShellBallFloatingSize, resolveShellBallInlineInputMode, ShellBallApp, - shouldSpeakShellBallIdleClickGreeting, shouldRetainShellBallEdgeDockReveal, shouldArmShellBallTextDropTarget, shouldShowShellBallFileDropOverlay, @@ -2539,9 +2538,9 @@ test("shell-ball speech transcript collection merges recognition chunks", () => ); }); -test("voice notification helpers keep startup greeting and formal delivery reminders short", () => { +test("voice notification helpers keep startup greeting, selection click greeting, and formal delivery reminders short", () => { assert.equal(resolveShellBallStartupGreetingText(), "CialloClaw 已启动"); - assert.equal(resolveShellBallIdleClickGreetingText(), "CialloClaw"); + assert.equal(resolveShellBallSelectionClickGreetingText(), "Ciallo"); assert.equal( resolveDeliveryReadyVoiceNotificationText({ task_id: "task-bubble-delivery", @@ -2573,7 +2572,18 @@ test("voice notification helpers prefer the saved voice type and matching locale { lang: "zh-CN", name: "Xiaoxiao" }, ], }); - assert.equal(resolvedDefaultFemale?.name, "Xiaoyi"); + assert.equal(resolvedDefaultFemale?.name, "Xiaoxiao"); + + const resolvedSoftGirl = resolveVoiceNotificationVoice({ + language: "zh-CN", + voiceType: "soft_girl", + voices: [ + { lang: "zh-CN", name: "Tingting" }, + { lang: "zh-CN", name: "Tongtong" }, + { lang: "zh-CN", name: "Xiaohan" }, + ], + }); + assert.equal(resolvedSoftGirl?.name, "Xiaohan"); const resolvedCustomVoice = resolveVoiceNotificationVoice({ language: "zh-CN", @@ -8266,53 +8276,6 @@ test("shell-ball mascot hotspot policy opens primary click for idle acknowledgem ); }); -test("shell-ball idle click greeting only speaks from the true resting state", () => { - assert.equal( - shouldSpeakShellBallIdleClickGreeting({ - clipboardPromptActive: false, - hasPendingAgentLoading: false, - hasPendingApproval: false, - hasPendingFiles: false, - hasVisibleBubbleItems: false, - hasWrittenInput: false, - onboardingActive: false, - selectionPromptActive: false, - visualState: "idle", - }), - true, - ); - - assert.equal( - shouldSpeakShellBallIdleClickGreeting({ - clipboardPromptActive: false, - hasPendingAgentLoading: false, - hasPendingApproval: false, - hasPendingFiles: false, - hasVisibleBubbleItems: true, - hasWrittenInput: false, - onboardingActive: false, - selectionPromptActive: false, - visualState: "idle", - }), - false, - ); - - assert.equal( - shouldSpeakShellBallIdleClickGreeting({ - clipboardPromptActive: false, - hasPendingAgentLoading: false, - hasPendingApproval: false, - hasPendingFiles: false, - hasVisibleBubbleItems: false, - hasWrittenInput: false, - onboardingActive: false, - selectionPromptActive: false, - visualState: "hover_input", - }), - false, - ); -}); - test("shell-ball mascot hotspot policy opens dashboard only from resting double click", () => { assert.equal( getShellBallMascotHotspotGestureAction({ @@ -8971,39 +8934,34 @@ test("shell-ball routes active resumable text follow-ups through task steer", () assert.match(coordinatorSource, /message: submittedText/); }); -test("shell-ball voice notifications are consumed locally from startup and formal task notifications", () => { +test("shell-ball voice notifications are consumed locally from startup, selection acceptance, and formal task notifications", () => { const appSource = readFileSync(resolve(desktopRoot, "src/features/shell-ball/ShellBallApp.tsx"), "utf8"); const mascotSource = readFileSync(resolve(desktopRoot, "src/features/shell-ball/components/ShellBallMascot.tsx"), "utf8"); const coordinatorSource = readFileSync(resolve(desktopRoot, "src/features/shell-ball/useShellBallCoordinator.ts"), "utf8"); + const controlPanelSource = readFileSync(resolve(desktopRoot, "src/features/control-panel/ControlPanelApp.tsx"), "utf8"); + const voiceConfigSource = readFileSync(resolve(desktopRoot, "src/services/voiceNotificationConfig.ts"), "utf8"); const voiceServiceSource = readFileSync(resolve(desktopRoot, "src/services/voiceNotificationService.ts"), "utf8"); - assert.match(appSource, /speakShellBallClipboardDetectedNotification/); - assert.match(appSource, /speakShellBallIdleGreeting/); - assert.match(appSource, /speakShellBallSelectionDetectedNotification/); + assert.match(appSource, /speakShellBallSelectionClickGreeting/); assert.match(appSource, /speakShellBallStartupGreeting/); - assert.match(appSource, /const SHELL_BALL_IDLE_CLICK_GREETING_DELAY_MS = 220;/); - assert.match(appSource, /const idleClickGreetingTimeoutRef = useRef\(null\);/); - assert.match(appSource, /export function shouldSpeakShellBallIdleClickGreeting/); - assert.match(appSource, /void speakShellBallIdleGreeting\(\);/); - assert.match(appSource, /cancelIdleClickGreeting\(\);/); assert.match(appSource, /const startupGreetingPlayedRef = useRef\(false\);/); - assert.match(appSource, /const selectionPromptVoiceSnapshotRef = useRef\(null\);/); - assert.match(appSource, /const clipboardPromptVoiceRef = useRef\(null\);/); - assert.match(appSource, /!areShellBallSelectionSnapshotsEqual\(selectionPromptVoiceSnapshotRef\.current, payload\.snapshot\)/); - assert.doesNotMatch(appSource, /const voiceKey = `\$\{payload\.snapshot\.updated_at\}:\$\{normalizedSelectionText\}`;/); - assert.match(appSource, /clipboardPromptVoiceRef\.current = null;\s*setClipboardPrompt\(null\);/); - assert.match(appSource, /activeVoicePrompt\.expiresAt <= Date\.now\(\)/); assert.match(appSource, /startupGreetingPlayedRef\.current = true;\s*void speakShellBallStartupGreeting\(\);/); - assert.match(appSource, /void speakShellBallSelectionDetectedNotification\(\);/); - assert.match(appSource, /void speakShellBallClipboardDetectedNotification\(\);/); + assert.match(appSource, /void speakShellBallSelectionClickGreeting\(\);/); + assert.doesNotMatch(appSource, /speakShellBallClipboardDetectedNotification/); + assert.doesNotMatch(appSource, /speakShellBallIdleGreeting/); + assert.doesNotMatch(appSource, /speakShellBallSelectionDetectedNotification/); assert.match(mascotSource, /input\.selectionIndicatorVisible \|\| input\.alertOpportunityAvailable \|\| input\.visualState === "idle"/); + assert.match(controlPanelSource, /VOICE_NOTIFICATION_VOICE_PRESET_OPTIONS/); + assert.match(controlPanelSource, /resolveVoiceNotificationVoicePreset\(draft\.settings\.general\.voice_type\)/); + assert.match(voiceConfigSource, /label: "小女孩", value: "default_female"/); + assert.match(voiceConfigSource, /label: "软萌", value: "soft_girl"/); + assert.match(voiceConfigSource, /label: "元气", value: "bright_girl"/); + assert.match(voiceConfigSource, /label: "少年", value: "default_male"/); assert.match(coordinatorSource, /import \{ speakApprovalPendingNotification, speakDeliveryReadyNotification \} from "@\/services\/voiceNotificationService";/); assert.match(coordinatorSource, /speakApprovalPendingNotification\(\{\s*approval_request: input\.approvalRequest,\s*task_id: input\.taskId,\s*\}\);/); assert.match(coordinatorSource, /speakDeliveryReadyNotification\(\{\s*delivery_result: input\.deliveryResult,\s*task_id: input\.taskId,\s*\}\);/); assert.match(voiceServiceSource, /const STARTUP_GREETING_TEXT = "CialloClaw 已启动";/); - assert.match(voiceServiceSource, /const IDLE_CLICK_GREETING_TEXT = "Ciallo";/); - assert.match(voiceServiceSource, /const SELECTION_DETECTED_TEXT = "检测到选中文本";/); - assert.match(voiceServiceSource, /const CLIPBOARD_DETECTED_TEXT = "检测到剪贴板内容";/); + assert.match(voiceServiceSource, /const SELECTION_CLICK_GREETING_TEXT = "Ciallo";/); assert.match(voiceServiceSource, /const APPROVAL_PENDING_TEXT = "有一个操作需要你确认";/); assert.match(voiceServiceSource, /const DELIVERY_READY_TEXT = "任务结果已准备好";/); assert.match(voiceServiceSource, /const VOICE_LIST_READY_TIMEOUT_MS = 600;/); @@ -9014,6 +8972,9 @@ test("shell-ball voice notifications are consumed locally from startup and forma assert.match(voiceServiceSource, /if \(!latestSettings\.voice_notification_enabled\) \{\s*return false;\s*\}/); assert.match(voiceServiceSource, /addEventListener\("voiceschanged", handleVoicesChanged\)/); assert.match(voiceServiceSource, /return payload\.delivery_result\.type === "bubble" \? null : DELIVERY_READY_TEXT;/); + assert.match(voiceServiceSource, /"xiaoxiao"/); + assert.match(voiceServiceSource, /"soft_girl"/); + assert.match(voiceServiceSource, /"bright_girl"/); }); test("shell-ball falls back to regular submit when active steer status races", () => { diff --git a/apps/desktop/src/services/settingsService.ts b/apps/desktop/src/services/settingsService.ts index 703b3f8d..147e4362 100644 --- a/apps/desktop/src/services/settingsService.ts +++ b/apps/desktop/src/services/settingsService.ts @@ -6,6 +6,7 @@ import { } from "../platform/desktopRuntimeDefaults"; import { syncDesktopSettingsSnapshot } from "../platform/desktopSettingsSnapshot"; import { loadStoredValue, saveStoredValue } from "../platform/storage"; +import { DEFAULT_VOICE_NOTIFICATION_VOICE_TYPE } from "./voiceNotificationConfig"; // SETTINGS_KEY is the single storage key for the desktop snapshot. const SETTINGS_KEY = "cialloclaw.settings"; @@ -90,7 +91,7 @@ function createDefaultSettings(): DesktopSettings { auto_launch: true, theme_mode: "follow_system", voice_notification_enabled: true, - voice_type: "default_female", + voice_type: DEFAULT_VOICE_NOTIFICATION_VOICE_TYPE, download: { workspace_path: runtimeDefaults?.workspace_path ?? DEFAULT_WORKSPACE_PLACEHOLDER, ask_before_save_each_file: true, diff --git a/apps/desktop/src/services/voiceNotificationConfig.ts b/apps/desktop/src/services/voiceNotificationConfig.ts new file mode 100644 index 00000000..26bb9048 --- /dev/null +++ b/apps/desktop/src/services/voiceNotificationConfig.ts @@ -0,0 +1,24 @@ +export const DEFAULT_VOICE_NOTIFICATION_VOICE_TYPE = "default_female"; + +export const VOICE_NOTIFICATION_VOICE_PRESET_OPTIONS = [ + { label: "小女孩", value: "default_female" }, + { label: "软萌", value: "soft_girl" }, + { label: "元气", value: "bright_girl" }, + { label: "少年", value: "default_male" }, +] as const; + +export type VoiceNotificationVoicePreset = (typeof VOICE_NOTIFICATION_VOICE_PRESET_OPTIONS)[number]["value"]; + +/** + * Normalizes persisted voice settings into one of the supported preset values + * when the control panel needs to render a radio-style choice group. + * + * @param voiceType Persisted `general.voice_type` value. + * @returns The matching preset or the refined default girl preset. + */ +export function resolveVoiceNotificationVoicePreset(voiceType: string): VoiceNotificationVoicePreset { + const normalizedVoiceType = voiceType.trim().toLowerCase(); + const matchedPreset = VOICE_NOTIFICATION_VOICE_PRESET_OPTIONS.find((option) => option.value === normalizedVoiceType); + + return matchedPreset?.value ?? DEFAULT_VOICE_NOTIFICATION_VOICE_TYPE; +} diff --git a/apps/desktop/src/services/voiceNotificationService.ts b/apps/desktop/src/services/voiceNotificationService.ts index 3ce96c17..91f7c253 100644 --- a/apps/desktop/src/services/voiceNotificationService.ts +++ b/apps/desktop/src/services/voiceNotificationService.ts @@ -1,11 +1,10 @@ import type { ApprovalPendingNotification, DeliveryReadyNotification, SettingsSnapshot } from "@cialloclaw/protocol"; import { loadSettings } from "./settingsService"; +import { DEFAULT_VOICE_NOTIFICATION_VOICE_TYPE } from "./voiceNotificationConfig"; type VoiceNotificationKind = | "startup_greeting" - | "idle_click_greeting" - | "selection_detected" - | "clipboard_detected" + | "selection_click_greeting" | "approval_pending" | "delivery_ready"; @@ -25,9 +24,7 @@ type SpeechSynthesisLike = { }; const STARTUP_GREETING_TEXT = "CialloClaw 已启动"; -const IDLE_CLICK_GREETING_TEXT = "CialloClaw"; -const SELECTION_DETECTED_TEXT = "检测到选中文本"; -const CLIPBOARD_DETECTED_TEXT = "检测到剪贴板内容"; +const SELECTION_CLICK_GREETING_TEXT = "Ciallo"; const APPROVAL_PENDING_TEXT = "有一个操作需要你确认"; const DELIVERY_READY_TEXT = "任务结果已准备好"; const VOICE_LIST_READY_TIMEOUT_MS = 600; @@ -58,24 +55,59 @@ function getVoiceNotificationHost() { function getVoiceTypePreferences(voiceType: string, language: string) { const normalizedVoiceType = normalizeVoiceType(voiceType); + const languagePreference = language.toLowerCase(); if (normalizedVoiceType === "default_female") { return [ - language.toLowerCase(), - "xiaoyi", + languagePreference, "xiaoxiao", + "xiaoyi", + "tongtong", + "xiaohan", + "child", + "kid", "tingting", "yunxia", "girl", "young", + "cute", "female", "woman", ]; } + if (normalizedVoiceType === "soft_girl") { + return [ + languagePreference, + "xiaoxiao", + "xiaohan", + "tongtong", + "soft", + "gentle", + "cute", + "child", + "girl", + "female", + ]; + } + + if (normalizedVoiceType === "bright_girl") { + return [ + languagePreference, + "xiaoyi", + "tingting", + "xiaoxuan", + "bright", + "lively", + "young", + "girl", + "female", + ]; + } + if (normalizedVoiceType === "default_male") { return [ - language.toLowerCase(), + languagePreference, "male", "man", "boy", @@ -86,7 +118,11 @@ function getVoiceTypePreferences(voiceType: string, language: string) { ]; } - return normalizedVoiceType.length > 0 ? [normalizedVoiceType] : [language.toLowerCase()]; + if (normalizedVoiceType.length === 0) { + return getVoiceTypePreferences(DEFAULT_VOICE_NOTIFICATION_VOICE_TYPE, language); + } + + return [normalizedVoiceType]; } /** @@ -163,12 +199,13 @@ export function resolveShellBallStartupGreetingText() { } /** - * Builds the idle-click acknowledgement spoken from the resting floating ball. + * Builds the short `Ciallo` acknowledgement spoken when the user accepts a + * fresh selected-text reminder from the floating ball. * - * @returns The short idle greeting used for single-click acknowledgement. + * @returns The short acknowledgement used for the selected-text prompt click. */ -export function resolveShellBallIdleClickGreetingText() { - return IDLE_CLICK_GREETING_TEXT; +export function resolveShellBallSelectionClickGreetingText() { + return SELECTION_CLICK_GREETING_TEXT; } function resolveVoiceNotificationText(kind: VoiceNotificationKind, payload?: ApprovalPendingNotification | DeliveryReadyNotification) { @@ -176,16 +213,8 @@ function resolveVoiceNotificationText(kind: VoiceNotificationKind, payload?: App return resolveShellBallStartupGreetingText(); } - if (kind === "idle_click_greeting") { - return resolveShellBallIdleClickGreetingText(); - } - - if (kind === "selection_detected") { - return SELECTION_DETECTED_TEXT; - } - - if (kind === "clipboard_detected") { - return CLIPBOARD_DETECTED_TEXT; + if (kind === "selection_click_greeting") { + return resolveShellBallSelectionClickGreetingText(); } if (kind === "approval_pending") { @@ -301,30 +330,13 @@ export function speakShellBallStartupGreeting() { } /** - * Plays a short idle acknowledgement when the resting shell-ball is clicked. - * - * @returns Whether the local desktop runtime attempted speech playback. - */ -export function speakShellBallIdleGreeting() { - return speakVoiceNotification({ kind: "idle_click_greeting" }); -} - -/** - * Plays a short reminder when the shell-ball detects a fresh text selection. - * - * @returns Whether the local desktop runtime attempted speech playback. - */ -export function speakShellBallSelectionDetectedNotification() { - return speakVoiceNotification({ kind: "selection_detected" }); -} - -/** - * Plays a short reminder when the shell-ball receives a fresh clipboard prompt. + * Plays a short `Ciallo` acknowledgement when the user accepts a selected-text + * reminder from the floating ball. * * @returns Whether the local desktop runtime attempted speech playback. */ -export function speakShellBallClipboardDetectedNotification() { - return speakVoiceNotification({ kind: "clipboard_detected" }); +export function speakShellBallSelectionClickGreeting() { + return speakVoiceNotification({ kind: "selection_click_greeting" }); } /** diff --git a/docs/control-panel-settings.md b/docs/control-panel-settings.md index dd8e23a4..7af9625d 100644 --- a/docs/control-panel-settings.md +++ b/docs/control-panel-settings.md @@ -158,13 +158,12 @@ #### 3.5.2 建议拆分的设置项 1. **声音总开关** - - 控制是否播放悬浮球启动、空闲单击确认、文本承接提醒、授权提醒、正式结果提醒等本地语音提示。 + - 控制是否播放悬浮球启动、选中文本提醒态点击确认、授权提醒、正式结果提醒等本地语音提示。 2. **提示音/语音风格选择** - - 例如：不同声线版本的“Ciallo”。 + - 例如：用按钮选择“小女孩 / 软萌 / 元气 / 少年”等明确预设。 3. **典型触发场景说明** - 悬浮球启动完成 - - 空闲单击悬浮球 - - 检测到选中文本或剪贴板内容 + - 选中文本后出现提醒态，点击该提醒继续发起协作时播报 `Ciallo` - 授权提醒 - 正式结果已准备好 @@ -174,7 +173,7 @@ - 关闭总开关后，风格选择区域应置灰但保留当前已选项。 - 控制面板里的声音草稿只有在保存后才会同步到悬浮球等其他桌面窗口；未保存的面板改动不应直接改变当前本地播报行为。 - 声音设置应明确说明是否影响系统通知、仅影响应用内反馈，还是两者都影响。 -- 当前桌面最小消费范围为：悬浮球窗口启动成功后的提示音、空闲单击悬浮球时的本地确认、选中文本与剪贴板承接提醒，以及 `approval.pending`、非 `bubble` 的 `delivery.ready` 本地语音提示。 +- 当前桌面最小消费范围为：悬浮球窗口启动成功后的提示音、选中文本提醒态被点击后的本地确认，以及 `approval.pending`、非 `bubble` 的 `delivery.ready` 本地语音提示。 - 第一版语音播报只在桌面端本地消费已有入口与正式通知，不新增后端 TTS 服务，也不扩展新的 JSON-RPC 方法或协议字段。 ### 3.6 下载设置 From fa87aa2307ac53b836bf170b941cf3366af6ea7d Mon Sep 17 00:00:00 2001 From: xgopilot Date: Wed, 13 May 2026 13:46:11 +0000 Subject: [PATCH 9/9] feat(desktop): randomize shell-ball prompt voice copy Generated with [codeagent](https://github.com/qbox/codeagent) Co-authored-by: Blackcloudss <161042579+Blackcloudss@users.noreply.github.com> --- .../control-panel/ControlPanelApp.tsx | 2 +- .../src/features/shell-ball/ShellBallApp.tsx | 17 ++++ .../shell-ball/shellBall.contract.test.ts | 31 ++++++- .../src/services/voiceNotificationService.ts | 86 +++++++++++++++++++ docs/control-panel-settings.md | 6 +- 5 files changed, 135 insertions(+), 7 deletions(-) diff --git a/apps/desktop/src/features/control-panel/ControlPanelApp.tsx b/apps/desktop/src/features/control-panel/ControlPanelApp.tsx index f8c99e37..eee44cd8 100644 --- a/apps/desktop/src/features/control-panel/ControlPanelApp.tsx +++ b/apps/desktop/src/features/control-panel/ControlPanelApp.tsx @@ -1289,7 +1289,7 @@ export function ControlPanelApp() { updateSettings((current) => ({ diff --git a/apps/desktop/src/features/shell-ball/ShellBallApp.tsx b/apps/desktop/src/features/shell-ball/ShellBallApp.tsx index 1923dbe1..bcde0cc7 100644 --- a/apps/desktop/src/features/shell-ball/ShellBallApp.tsx +++ b/apps/desktop/src/features/shell-ball/ShellBallApp.tsx @@ -13,6 +13,7 @@ import { ShellBallAttachmentTray } from "./components/ShellBallAttachmentTray"; import { ShellBallBubbleZone } from "./components/ShellBallBubbleZone"; import { ShellBallInputBar } from "./components/ShellBallInputBar"; import { ShellBallVoiceHints } from "./components/ShellBallVoiceHints"; +import { areShellBallSelectionSnapshotsEqual } from "./selection/selection.provider"; import type { ShellBallSelectionSnapshot } from "./selection/selection.types"; import { useShellBallInteraction } from "./useShellBallInteraction"; import { getShellBallMotionConfig } from "./shellBall.motion"; @@ -40,6 +41,8 @@ import { } from "../../platform/shellBallWindow"; import { loadSettings } from "../../services/settingsService"; import { + speakShellBallClipboardDetectedNotification, + speakShellBallSelectionDetectedNotification, speakShellBallSelectionClickGreeting, speakShellBallStartupGreeting, } from "../../services/voiceNotificationService"; @@ -395,6 +398,8 @@ export function ShellBallApp() { const clipboardPromptClearTimeoutRef = useRef(null); const selectionPromptClearTimeoutRef = useRef(null); const selectionPromptExpiryTimeoutRef = useRef(null); + const lastSpokenSelectionPromptRef = useRef(null); + const lastSpokenClipboardPromptTextRef = useRef(null); const previousVisualStateRef = useRef(visualState); const transitionQueueRef = useRef(Promise.resolve()); const edgeDockRevealHideTimeoutRef = useRef(null); @@ -919,10 +924,12 @@ export function ShellBallApp() { }, []); const clearSelectionPrompt = useCallback(() => { + lastSpokenSelectionPromptRef.current = null; setSelectionPrompt(null); }, []); const clearClipboardPrompt = useCallback(() => { + lastSpokenClipboardPromptTextRef.current = null; setClipboardPrompt(null); }, []); @@ -1059,6 +1066,11 @@ export function ShellBallApp() { selectionPromptClearTimeoutRef.current = null; } + if (!areShellBallSelectionSnapshotsEqual(lastSpokenSelectionPromptRef.current, payload.snapshot)) { + lastSpokenSelectionPromptRef.current = payload.snapshot; + void speakShellBallSelectionDetectedNotification(); + } + setSelectionPrompt(payload.snapshot); return; } @@ -1111,6 +1123,11 @@ export function ShellBallApp() { return; } + if (lastSpokenClipboardPromptTextRef.current !== payload.text) { + lastSpokenClipboardPromptTextRef.current = payload.text; + void speakShellBallClipboardDetectedNotification(); + } + setClipboardPrompt({ text: payload.text, expiresAt: Date.now() + SHELL_BALL_CLIPBOARD_PROMPT_WINDOW_MS, diff --git a/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts b/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts index 42c850aa..d8031d7f 100644 --- a/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts +++ b/apps/desktop/src/features/shell-ball/shellBall.contract.test.ts @@ -29,6 +29,8 @@ import { import { getShellBallMotionConfig } from "./shellBall.motion"; import { collectShellBallSpeechTranscript, composeShellBallSpeechDraft } from "./shellBall.speech"; import { + resolveShellBallClipboardDetectedNotificationText, + resolveShellBallSelectionDetectedNotificationText, resolveDeliveryReadyVoiceNotificationText, resolveShellBallSelectionClickGreetingText, resolveShellBallStartupGreetingText, @@ -2538,9 +2540,19 @@ test("shell-ball speech transcript collection merges recognition chunks", () => ); }); -test("voice notification helpers keep startup greeting, selection click greeting, and formal delivery reminders short", () => { +test("voice notification helpers keep startup greeting, detection prompts, selection click greeting, and formal delivery reminders short", () => { assert.equal(resolveShellBallStartupGreetingText(), "CialloClaw 已启动"); + assert.equal(resolveShellBallSelectionDetectedNotificationText(0), "发现一段高亮文字啦"); + assert.equal(resolveShellBallSelectionDetectedNotificationText(0.2), "我接住新选中的内容啦"); + assert.equal(resolveShellBallSelectionDetectedNotificationText(0.4), "新的选中内容到啦"); + assert.equal(resolveShellBallSelectionDetectedNotificationText(0.6), "这段选中文字交给我吧"); + assert.equal(resolveShellBallSelectionDetectedNotificationText(0.8), "我看到你圈出的这段话啦"); assert.equal(resolveShellBallSelectionClickGreetingText(), "Ciallo"); + assert.equal(resolveShellBallClipboardDetectedNotificationText(0), "剪贴板刚刚更新啦"); + assert.equal(resolveShellBallClipboardDetectedNotificationText(0.2), "我接住新复制的内容啦"); + assert.equal(resolveShellBallClipboardDetectedNotificationText(0.4), "新的剪贴板内容到啦"); + assert.equal(resolveShellBallClipboardDetectedNotificationText(0.6), "这段复制内容交给我吧"); + assert.equal(resolveShellBallClipboardDetectedNotificationText(0.8), "我看到你刚复制的小纸条啦"); assert.equal( resolveDeliveryReadyVoiceNotificationText({ task_id: "task-bubble-delivery", @@ -8934,7 +8946,7 @@ test("shell-ball routes active resumable text follow-ups through task steer", () assert.match(coordinatorSource, /message: submittedText/); }); -test("shell-ball voice notifications are consumed locally from startup, selection acceptance, and formal task notifications", () => { +test("shell-ball voice notifications are consumed locally from startup, selection and clipboard prompts, selection acceptance, and formal task notifications", () => { const appSource = readFileSync(resolve(desktopRoot, "src/features/shell-ball/ShellBallApp.tsx"), "utf8"); const mascotSource = readFileSync(resolve(desktopRoot, "src/features/shell-ball/components/ShellBallMascot.tsx"), "utf8"); const coordinatorSource = readFileSync(resolve(desktopRoot, "src/features/shell-ball/useShellBallCoordinator.ts"), "utf8"); @@ -8942,14 +8954,19 @@ test("shell-ball voice notifications are consumed locally from startup, selectio const voiceConfigSource = readFileSync(resolve(desktopRoot, "src/services/voiceNotificationConfig.ts"), "utf8"); const voiceServiceSource = readFileSync(resolve(desktopRoot, "src/services/voiceNotificationService.ts"), "utf8"); + assert.match(appSource, /areShellBallSelectionSnapshotsEqual/); + assert.match(appSource, /speakShellBallClipboardDetectedNotification/); + assert.match(appSource, /speakShellBallSelectionDetectedNotification/); assert.match(appSource, /speakShellBallSelectionClickGreeting/); assert.match(appSource, /speakShellBallStartupGreeting/); assert.match(appSource, /const startupGreetingPlayedRef = useRef\(false\);/); + assert.match(appSource, /const lastSpokenSelectionPromptRef = useRef\(null\);/); + assert.match(appSource, /const lastSpokenClipboardPromptTextRef = useRef\(null\);/); assert.match(appSource, /startupGreetingPlayedRef\.current = true;\s*void speakShellBallStartupGreeting\(\);/); + assert.match(appSource, /void speakShellBallSelectionDetectedNotification\(\);/); assert.match(appSource, /void speakShellBallSelectionClickGreeting\(\);/); - assert.doesNotMatch(appSource, /speakShellBallClipboardDetectedNotification/); + assert.match(appSource, /void speakShellBallClipboardDetectedNotification\(\);/); assert.doesNotMatch(appSource, /speakShellBallIdleGreeting/); - assert.doesNotMatch(appSource, /speakShellBallSelectionDetectedNotification/); assert.match(mascotSource, /input\.selectionIndicatorVisible \|\| input\.alertOpportunityAvailable \|\| input\.visualState === "idle"/); assert.match(controlPanelSource, /VOICE_NOTIFICATION_VOICE_PRESET_OPTIONS/); assert.match(controlPanelSource, /resolveVoiceNotificationVoicePreset\(draft\.settings\.general\.voice_type\)/); @@ -8961,9 +8978,15 @@ test("shell-ball voice notifications are consumed locally from startup, selectio assert.match(coordinatorSource, /speakApprovalPendingNotification\(\{\s*approval_request: input\.approvalRequest,\s*task_id: input\.taskId,\s*\}\);/); assert.match(coordinatorSource, /speakDeliveryReadyNotification\(\{\s*delivery_result: input\.deliveryResult,\s*task_id: input\.taskId,\s*\}\);/); assert.match(voiceServiceSource, /const STARTUP_GREETING_TEXT = "CialloClaw 已启动";/); + assert.match(voiceServiceSource, /const SELECTION_DETECTED_TEXT_1 = "发现一段高亮文字啦";/); + assert.match(voiceServiceSource, /const SELECTION_DETECTED_TEXT_5 = "我看到你圈出的这段话啦";/); assert.match(voiceServiceSource, /const SELECTION_CLICK_GREETING_TEXT = "Ciallo";/); + assert.match(voiceServiceSource, /const CLIPBOARD_DETECTED_TEXT_1 = "剪贴板刚刚更新啦";/); + assert.match(voiceServiceSource, /const CLIPBOARD_DETECTED_TEXT_5 = "我看到你刚复制的小纸条啦";/); assert.match(voiceServiceSource, /const APPROVAL_PENDING_TEXT = "有一个操作需要你确认";/); assert.match(voiceServiceSource, /const DELIVERY_READY_TEXT = "任务结果已准备好";/); + assert.match(voiceServiceSource, /resolveShellBallSelectionDetectedNotificationText/); + assert.match(voiceServiceSource, /resolveShellBallClipboardDetectedNotificationText/); assert.match(voiceServiceSource, /const VOICE_LIST_READY_TIMEOUT_MS = 600;/); assert.match(voiceServiceSource, /let latestVoiceNotificationRequestId = 0;/); assert.match(voiceServiceSource, /const requestId = \+\+latestVoiceNotificationRequestId;/); diff --git a/apps/desktop/src/services/voiceNotificationService.ts b/apps/desktop/src/services/voiceNotificationService.ts index 91f7c253..22891a7e 100644 --- a/apps/desktop/src/services/voiceNotificationService.ts +++ b/apps/desktop/src/services/voiceNotificationService.ts @@ -4,7 +4,9 @@ import { DEFAULT_VOICE_NOTIFICATION_VOICE_TYPE } from "./voiceNotificationConfig type VoiceNotificationKind = | "startup_greeting" + | "selection_detected" | "selection_click_greeting" + | "clipboard_detected" | "approval_pending" | "delivery_ready"; @@ -25,6 +27,16 @@ type SpeechSynthesisLike = { const STARTUP_GREETING_TEXT = "CialloClaw 已启动"; const SELECTION_CLICK_GREETING_TEXT = "Ciallo"; +const SELECTION_DETECTED_TEXT_1 = "发现一段高亮文字啦"; +const SELECTION_DETECTED_TEXT_2 = "我接住新选中的内容啦"; +const SELECTION_DETECTED_TEXT_3 = "新的选中内容到啦"; +const SELECTION_DETECTED_TEXT_4 = "这段选中文字交给我吧"; +const SELECTION_DETECTED_TEXT_5 = "我看到你圈出的这段话啦"; +const CLIPBOARD_DETECTED_TEXT_1 = "剪贴板刚刚更新啦"; +const CLIPBOARD_DETECTED_TEXT_2 = "我接住新复制的内容啦"; +const CLIPBOARD_DETECTED_TEXT_3 = "新的剪贴板内容到啦"; +const CLIPBOARD_DETECTED_TEXT_4 = "这段复制内容交给我吧"; +const CLIPBOARD_DETECTED_TEXT_5 = "我看到你刚复制的小纸条啦"; const APPROVAL_PENDING_TEXT = "有一个操作需要你确认"; const DELIVERY_READY_TEXT = "任务结果已准备好"; const VOICE_LIST_READY_TIMEOUT_MS = 600; @@ -38,6 +50,18 @@ function normalizeVoiceLabel(value: string) { return value.trim().toLowerCase(); } +function resolveRandomVoiceNotificationText(options: readonly string[], randomValue = Math.random()) { + if (options.length === 0) { + return ""; + } + + const normalizedRandomValue = Number.isFinite(randomValue) ? randomValue : 0; + const boundedRandomValue = Math.min(Math.max(normalizedRandomValue, 0), 0.999_999_999_999); + const resolvedIndex = Math.min(options.length - 1, Math.floor(boundedRandomValue * options.length)); + + return options[resolvedIndex] ?? options[0] ?? ""; +} + function getVoiceNotificationSettings(): VoiceNotificationSettings { return loadSettings().settings.general; } @@ -198,6 +222,23 @@ export function resolveShellBallStartupGreetingText() { return STARTUP_GREETING_TEXT; } +/** + * Builds a playful selection-detected reminder for the floating ball so + * repeated demos feel less robotic without expanding the notification scope. + * + * @param randomValue Optional deterministic random input for contract tests. + * @returns One of five local selection-detected reminder lines. + */ +export function resolveShellBallSelectionDetectedNotificationText(randomValue = Math.random()) { + return resolveRandomVoiceNotificationText([ + SELECTION_DETECTED_TEXT_1, + SELECTION_DETECTED_TEXT_2, + SELECTION_DETECTED_TEXT_3, + SELECTION_DETECTED_TEXT_4, + SELECTION_DETECTED_TEXT_5, + ], randomValue); +} + /** * Builds the short `Ciallo` acknowledgement spoken when the user accepts a * fresh selected-text reminder from the floating ball. @@ -208,15 +249,40 @@ export function resolveShellBallSelectionClickGreetingText() { return SELECTION_CLICK_GREETING_TEXT; } +/** + * Builds a playful clipboard-detected reminder for the floating ball so local + * clipboard prompts feel lively during repeated desktop demos. + * + * @param randomValue Optional deterministic random input for contract tests. + * @returns One of five local clipboard-detected reminder lines. + */ +export function resolveShellBallClipboardDetectedNotificationText(randomValue = Math.random()) { + return resolveRandomVoiceNotificationText([ + CLIPBOARD_DETECTED_TEXT_1, + CLIPBOARD_DETECTED_TEXT_2, + CLIPBOARD_DETECTED_TEXT_3, + CLIPBOARD_DETECTED_TEXT_4, + CLIPBOARD_DETECTED_TEXT_5, + ], randomValue); +} + function resolveVoiceNotificationText(kind: VoiceNotificationKind, payload?: ApprovalPendingNotification | DeliveryReadyNotification) { if (kind === "startup_greeting") { return resolveShellBallStartupGreetingText(); } + if (kind === "selection_detected") { + return resolveShellBallSelectionDetectedNotificationText(); + } + if (kind === "selection_click_greeting") { return resolveShellBallSelectionClickGreetingText(); } + if (kind === "clipboard_detected") { + return resolveShellBallClipboardDetectedNotificationText(); + } + if (kind === "approval_pending") { return APPROVAL_PENDING_TEXT; } @@ -329,6 +395,16 @@ export function speakShellBallStartupGreeting() { return speakVoiceNotification({ kind: "startup_greeting" }); } +/** + * Plays a short local reminder when the floating ball captures new selected + * text and exposes the click-to-start affordance. + * + * @returns Whether the local desktop runtime attempted speech playback. + */ +export function speakShellBallSelectionDetectedNotification() { + return speakVoiceNotification({ kind: "selection_detected" }); +} + /** * Plays a short `Ciallo` acknowledgement when the user accepts a selected-text * reminder from the floating ball. @@ -339,6 +415,16 @@ export function speakShellBallSelectionClickGreeting() { return speakVoiceNotification({ kind: "selection_click_greeting" }); } +/** + * Plays a short local reminder when the floating ball captures new clipboard + * text and exposes the click-to-submit affordance. + * + * @returns Whether the local desktop runtime attempted speech playback. + */ +export function speakShellBallClipboardDetectedNotification() { + return speakVoiceNotification({ kind: "clipboard_detected" }); +} + /** * Plays a short approval reminder for the formal `approval.pending` notification. * diff --git a/docs/control-panel-settings.md b/docs/control-panel-settings.md index 7af9625d..2d912a83 100644 --- a/docs/control-panel-settings.md +++ b/docs/control-panel-settings.md @@ -158,11 +158,13 @@ #### 3.5.2 建议拆分的设置项 1. **声音总开关** - - 控制是否播放悬浮球启动、选中文本提醒态点击确认、授权提醒、正式结果提醒等本地语音提示。 + - 控制是否播放悬浮球启动、选中文本检测、剪贴板检测、选中文本提醒态点击确认、授权提醒、正式结果提醒等本地语音提示。 2. **提示音/语音风格选择** - 例如：用按钮选择“小女孩 / 软萌 / 元气 / 少年”等明确预设。 3. **典型触发场景说明** - 悬浮球启动完成 + - 检测到新的选中文本 + - 检测到新的剪贴板内容 - 选中文本后出现提醒态，点击该提醒继续发起协作时播报 `Ciallo` - 授权提醒 - 正式结果已准备好 @@ -173,7 +175,7 @@ - 关闭总开关后，风格选择区域应置灰但保留当前已选项。 - 控制面板里的声音草稿只有在保存后才会同步到悬浮球等其他桌面窗口；未保存的面板改动不应直接改变当前本地播报行为。 - 声音设置应明确说明是否影响系统通知、仅影响应用内反馈，还是两者都影响。 -- 当前桌面最小消费范围为：悬浮球窗口启动成功后的提示音、选中文本提醒态被点击后的本地确认，以及 `approval.pending`、非 `bubble` 的 `delivery.ready` 本地语音提示。 +- 当前桌面最小消费范围为：悬浮球窗口启动成功后的提示音、选中文本检测、剪贴板检测、选中文本提醒态被点击后的本地确认，以及 `approval.pending`、非 `bubble` 的 `delivery.ready` 本地语音提示。 - 第一版语音播报只在桌面端本地消费已有入口与正式通知，不新增后端 TTS 服务，也不扩展新的 JSON-RPC 方法或协议字段。 ### 3.6 下载设置