From e9a048ef607554b31b634e52cac52bcbe5452d63 Mon Sep 17 00:00:00 2001 From: LpLegend <2422019509@qq.com> Date: Thu, 9 Jul 2026 22:55:43 +0800 Subject: [PATCH 1/3] feat: consolidate aether app runtime coding by gpt-5 - move Aether core, listener, and VPIO capture under .moss_ws/apps/aether - add Aether README quick-start and remove legacy listener entry - harden Aether voice loop ASR/TTS interrupt handling Tests: - .venv/bin/python -m py_compile .moss_ws/apps/aether/core/main.py .moss_ws/apps/aether/listener/main.py .moss_ws/apps/aether/vpio_capture/main.py .moss_ws/apps/aether/vpio_capture/vpio_capture.py - .venv/bin/python -m pytest tests/ghoshell_moss/host/speech/volcengine_asr/test_protocol.py - .venv/bin/moss --ai --mode aether apps list - .venv/bin/moss-run-ghost echo --mode aether via Codex --- .moss_ws/apps/aether/README.md | 241 +++++++ .moss_ws/apps/aether/core/.gitignore | 8 + .moss_ws/apps/aether/core/APP.md | 15 + .moss_ws/apps/aether/core/main.py | 422 ++++++++++++ .moss_ws/apps/aether/core/pyproject.toml | 16 + .moss_ws/apps/aether/core/webroot/index.html | 143 ++++ .../apps/aether/core/webroot/web/core.frag | 148 +++++ .../apps/aether/core/webroot/web/core.vert | 6 + .moss_ws/apps/aether/core/webroot/web/main.js | 551 ++++++++++++++++ .../apps/aether/core/webroot/web/scene.js | 188 ++++++ .../aether/core/webroot/web/state_mapper.js | 187 ++++++ .../apps/aether/core/webroot/web/style.css | 553 ++++++++++++++++ .moss_ws/apps/aether/core/webroot/web/vad.js | 162 +++++ .moss_ws/apps/aether/core/webroot/web/ws.js | 119 ++++ .../{sensors => aether}/listener/.env.example | 0 .../{sensors => aether}/listener/.gitignore | 0 .moss_ws/apps/aether/listener/APP.md | 15 + .moss_ws/apps/aether/listener/main.py | 614 ++++++++++++++++++ .../listener/pyproject.toml | 0 .moss_ws/apps/aether/vpio_capture/.gitignore | 9 + .moss_ws/apps/aether/vpio_capture/APP.md | 10 + .moss_ws/apps/aether/vpio_capture/main.py | 61 ++ .../apps/aether/vpio_capture/pyproject.toml | 16 + .../apps/aether/vpio_capture/smoke_test.py | 141 ++++ .../apps/aether/vpio_capture/vpio_capture.py | 607 +++++++++++++++++ .moss_ws/apps/bodies/g1_sim/APP.md | 4 +- .moss_ws/apps/bodies/g1_sim/README.md | 2 +- .moss_ws/apps/sensors/listener/APP.md | 10 - .moss_ws/apps/sensors/listener/CLAUDE.md | 66 -- .moss_ws/apps/sensors/listener/main.py | 341 ---------- .moss_ws/configs/llm.yml | 22 +- .moss_ws/ghosts/echo/soul.md | 11 + .moss_ws/src/MOSS/modes/aether/MODE.md | 19 + .moss_ws/src/MOSS/modes/aether/__init__.py | 0 .moss_ws/src/MOSS/modes/aether/channels.py | 17 + .moss_ws/src/MOSS/modes/aether/configs.py | 1 + .moss_ws/src/MOSS/modes/aether/contracts.py | 1 + .moss_ws/src/MOSS/modes/aether/nuclei.py | 6 + .moss_ws/src/MOSS/modes/aether/providers.py | 1 + .moss_ws/src/MOSS/modes/aether/resources.py | 1 + .moss_ws/src/MOSS/modes/aether/topics.py | 1 + .moss_ws/src/MOSS/modes/listener/MODE.md | 5 +- .moss_ws/src/MOSS/modes/show/MODE.md | 4 +- .../core/mindflow/audio_nucleus.py | 10 +- .../core/speech/stream_tts_speech.py | 8 + src/ghoshell_moss/ghosts/atom/_adapter.py | 18 +- src/ghoshell_moss/ghosts/atom/_meta.py | 62 +- src/ghoshell_moss/ghosts/atom/_runtime.py | 25 +- src/ghoshell_moss/host/app_store.py | 16 +- src/ghoshell_moss/host/ghost_runtime.py | 30 + .../host/speech/volcengine_asr/config.py | 26 +- .../host/speech/volcengine_asr/protocol.py | 98 ++- .../host/speech/volcengine_asr/recognizer.py | 103 ++- .../host/speech/volcengine_tts/tts.py | 16 +- .../speech/volcengine_asr/test_protocol.py | 15 + 55 files changed, 4671 insertions(+), 500 deletions(-) create mode 100644 .moss_ws/apps/aether/README.md create mode 100644 .moss_ws/apps/aether/core/.gitignore create mode 100644 .moss_ws/apps/aether/core/APP.md create mode 100644 .moss_ws/apps/aether/core/main.py create mode 100644 .moss_ws/apps/aether/core/pyproject.toml create mode 100644 .moss_ws/apps/aether/core/webroot/index.html create mode 100644 .moss_ws/apps/aether/core/webroot/web/core.frag create mode 100644 .moss_ws/apps/aether/core/webroot/web/core.vert create mode 100644 .moss_ws/apps/aether/core/webroot/web/main.js create mode 100644 .moss_ws/apps/aether/core/webroot/web/scene.js create mode 100644 .moss_ws/apps/aether/core/webroot/web/state_mapper.js create mode 100644 .moss_ws/apps/aether/core/webroot/web/style.css create mode 100644 .moss_ws/apps/aether/core/webroot/web/vad.js create mode 100644 .moss_ws/apps/aether/core/webroot/web/ws.js rename .moss_ws/apps/{sensors => aether}/listener/.env.example (100%) rename .moss_ws/apps/{sensors => aether}/listener/.gitignore (100%) create mode 100644 .moss_ws/apps/aether/listener/APP.md create mode 100644 .moss_ws/apps/aether/listener/main.py rename .moss_ws/apps/{sensors => aether}/listener/pyproject.toml (100%) create mode 100644 .moss_ws/apps/aether/vpio_capture/.gitignore create mode 100644 .moss_ws/apps/aether/vpio_capture/APP.md create mode 100644 .moss_ws/apps/aether/vpio_capture/main.py create mode 100644 .moss_ws/apps/aether/vpio_capture/pyproject.toml create mode 100644 .moss_ws/apps/aether/vpio_capture/smoke_test.py create mode 100644 .moss_ws/apps/aether/vpio_capture/vpio_capture.py delete mode 100644 .moss_ws/apps/sensors/listener/APP.md delete mode 100644 .moss_ws/apps/sensors/listener/CLAUDE.md delete mode 100644 .moss_ws/apps/sensors/listener/main.py create mode 100644 .moss_ws/src/MOSS/modes/aether/MODE.md create mode 100644 .moss_ws/src/MOSS/modes/aether/__init__.py create mode 100644 .moss_ws/src/MOSS/modes/aether/channels.py create mode 100644 .moss_ws/src/MOSS/modes/aether/configs.py create mode 100644 .moss_ws/src/MOSS/modes/aether/contracts.py create mode 100644 .moss_ws/src/MOSS/modes/aether/nuclei.py create mode 100644 .moss_ws/src/MOSS/modes/aether/providers.py create mode 100644 .moss_ws/src/MOSS/modes/aether/resources.py create mode 100644 .moss_ws/src/MOSS/modes/aether/topics.py diff --git a/.moss_ws/apps/aether/README.md b/.moss_ws/apps/aether/README.md new file mode 100644 index 00000000..19c7a2cf --- /dev/null +++ b/.moss_ws/apps/aether/README.md @@ -0,0 +1,241 @@ +# Aether App + +Aether 是 MOSS 的实时语音交互外壳。它把麦克风采集、ASR、Ghost 思考、TTS 播放和前端可视化连成一条闭环,用来验证 MOSS 能不能像一个在场的智能体一样听、想、说、被打断。 + +这个目录是 Aether 相关 app 的唯一维护入口。后续理解、启动、排错,优先看本 README 和本目录代码。 + +## 目录结构 + +```text +.moss_ws/apps/aether/ + core/ 前端可视化和 WebSocket 状态聚合 + listener/ 音频到 ASR,再到 SpeechTopic / AudioSignal + vpio_capture/ macOS VPIO 音频采集和系统级回声消除 +``` + +三个 app 的 canonical address 是: + +```text +aether/core +aether/listener +aether/vpio_capture +``` + +旧地址 `ui/aether_core`、`sensors/listener`、`sensors/vpio_capture` 已不再作为 Aether 入口使用。 + +## 一键启动 + +从仓库根目录执行: + +```bash +.venv/bin/moss-run-ghost echo --mode aether +``` + +启动后打开: + +```text +http://127.0.0.1:8765/ +``` + +`aether` mode 会自动拉起: + +```text +aether/vpio_capture +aether/listener +aether/core +``` + +关闭全部 Aether/MOSS runtime 进程: + +```bash +.venv/bin/moss --ai --mode aether runtime kill-all --yes +``` + +查看当前还在运行的 cell: + +```bash +.venv/bin/moss --ai --mode aether runtime list-cells +``` + +## 单独调试 + +只启动前端可视化: + +```bash +.venv/bin/moss --ai --mode aether apps test aether/core +``` + +只启动 ASR listener。排查时建议先用 manual 模式,避免连续 ASR 抢占调试过程: + +```bash +LISTENER_ASR_MODE=manual .venv/bin/moss --ai --mode aether apps test aether/listener +``` + +只启动 macOS VPIO 采集: + +```bash +.venv/bin/moss --ai --mode aether apps test aether/vpio_capture +``` + +## 组件职责 + +### aether/vpio_capture + +`aether/vpio_capture` 是音频采集层。 + +它负责: + +- 使用 macOS AVAudioEngine + VPIO 采集麦克风输入。 +- 打开系统级 voice processing / echo cancellation,降低 TTS 外放被 ASR 收回去的概率。 +- 把音频转成 listener 需要的 16kHz mono PCM。 +- 发布 VPIO 运行诊断,例如 RMS、peak、channel、frame count、stall/restart。 + +它不负责 ASR、不负责判断用户意图、不负责调用 Ghost。 + +### aether/listener + +`aether/listener` 是语音识别层。 + +它负责: + +- 消费音频采集 topic。 +- 连接 Volcengine streaming ASR。 +- 发布 `SpeechTopic`,让 MOSS/Ghost 收到用户说完的一句话。 +- 发布 `AudioSignal`,告诉 Mindflow 用户开始说话、最终文本完成。 +- 监听 `asr_control`,在 continuous/manual 两种收音模式之间切换。 +- 在用户开口或停止意图出现时发布 interrupt 相关信号。 + +它不负责前端绘制、不负责 TTS 播放、不负责 LLM 推理。 + +### aether/core + +`aether/core` 是前端状态聚合层。 + +它负责: + +- 提供 `http://127.0.0.1:8765/` 页面。 +- 订阅 `SpeechTopic` 和 `AudioRuntimeTopic`。 +- 把 listen、think、speak、interrupt、idle 等状态合成给 WebSocket 前端。 +- 接收前端按钮或浏览器 VAD 产生的控制事件。 +- 把 ASR 模式切换写回 `asr_control` topic。 + +它不负责直接启动其他 app,不直接调用 ASR/TTS/LLM,不绕过 MOSS runtime 伪造 Ghost 输入。 + +## 数据流 + +正常语音链路: + +```text +麦克风 + -> aether/vpio_capture + -> audio topic + -> aether/listener + -> SpeechTopic + AudioSignal + -> MOSS Mindflow / Ghost + -> TTS + -> speaker diagnostics + -> aether/core + -> browser WebGL state +``` + +前端控制链路: + +```text +browser button / VAD + -> aether/core WebSocket + -> AudioRuntimeTopic(asr_control / interrupt) + -> aether/listener 或 MOSS host runtime +``` + +## 关键 Topic + +| Topic | 发布者 | 消费者 | 作用 | +| --- | --- | --- | --- | +| `SpeechTopic` | `aether/listener` | Ghost / `aether/core` | 用户一句完整语音文本 | +| `AudioSignal(SPEECH_STARTED)` | `aether/listener` | Mindflow | 用户已经开始说话 | +| `AudioSignal(SPEECH_FINAL)` | `aether/listener` | Mindflow | 用户一句话完成 | +| `AudioRuntimeTopic(device_name="vpio")` | `aether/vpio_capture` | `aether/core` | 采集状态和音量诊断 | +| `AudioRuntimeTopic(device_name="asr")` | `aether/listener` | `aether/core` | ASR running/partial/final/error/idle | +| `AudioRuntimeTopic(device_name="asr_control")` | `aether/core` | `aether/listener` | 连续/手动 ASR 控制 | +| `AudioRuntimeTopic(device_name="speaker")` | TTS/player | `aether/core` | TTS 播放状态 | +| `AudioRuntimeTopic(device_name="interrupt")` | `aether/core` / `aether/listener` | `aether/core` / host runtime | 打断当前输出 | + +## 常见排错 + +### 页面打不开 + +确认 `aether/core` 是否启动: + +```bash +.venv/bin/moss --ai --mode aether runtime list-cells +``` + +确认 8765 端口是否有响应: + +```bash +curl -s http://127.0.0.1:8765/ +``` + +### 没有声音输入 + +先单独启动 VPIO: + +```bash +.venv/bin/moss --ai --mode aether apps test aether/vpio_capture +``` + +看日志里是否持续出现 RMS/peak 变化。如果 RMS 长时间为 0,优先检查系统麦克风权限、默认输入设备、采样权限。 + +### 停顿后 ASR 不再识别 + +优先检查 `aether/listener` 日志: + +- 是否还在读取 audio frames。 +- 是否还有 ASR partial/final。 +- 是否进入了 error 或 idle 后没有恢复。 +- `asr_control` 当前是否被切到 manual。 + +如果只是调试 listener,先用: + +```bash +LISTENER_ASR_MODE=manual .venv/bin/moss --ai --mode aether apps test aether/listener +``` + +这样可以把连续收音问题和 ASR 服务问题分开看。 + +### TTS 外放被 ASR 收进去 + +确认当前 mode 使用的是: + +```text +aether/vpio_capture +``` + +不要用普通 `sensors/audio_capture` 来验证全双工效果。普通采集没有 macOS VPIO 的系统级 AEC,容易把扬声器声音重新送进 ASR。 + +### 停不下来 + +先杀当前 mode 下的 runtime: + +```bash +.venv/bin/moss --ai --mode aether runtime kill-all --yes +``` + +再确认: + +```bash +.venv/bin/moss --ai --mode aether runtime list-cells +``` + +输出 `No cells found in this scope` 才表示 MOSS runtime cell 已清空。 + +## 维护边界 + +Aether 代码以后尽量收敛在本目录内: + +- 前端和 WebSocket 状态聚合放在 `aether/core/`。 +- ASR listener 放在 `aether/listener/`。 +- macOS VPIO 采集放在 `aether/vpio_capture/`。 +- MOSS host、Mindflow、Speech provider 的公共能力仍放在 `src/ghoshell_moss/`,不要复制到 app 目录。 + +如果新增说明文档,优先更新这个 README。不要再在仓库根 `Docs/` 下新增 Aether 历史说明。 diff --git a/.moss_ws/apps/aether/core/.gitignore b/.moss_ws/apps/aether/core/.gitignore new file mode 100644 index 00000000..63f5e295 --- /dev/null +++ b/.moss_ws/apps/aether/core/.gitignore @@ -0,0 +1,8 @@ +*.log +runtime/ +venv/ +.venv/ +__pycache__/ +*.pyc +.env +uv.lock diff --git a/.moss_ws/apps/aether/core/APP.md b/.moss_ws/apps/aether/core/APP.md new file mode 100644 index 00000000..48870ba9 --- /dev/null +++ b/.moss_ws/apps/aether/core/APP.md @@ -0,0 +1,15 @@ +--- +arguments: '' +description: 'Aether Core UI — canonical aether/core app,聚合 SpeechTopic/AudioRuntimeTopic 并通过 WebSocket 驱动能量核心' +executable: uv +respawn: false +script: main.py +workers: 1 +--- + +Aether Core UI app — MOSS ghost 的能量核心可视化通道。Canonical app +address: `aether/core`. + +订阅 listener 的 SpeechTopic(用户说完一句 → think)和 TTS player 的 AudioRuntimeTopic(speaker running → speak/idle),通过 WebSocket 把状态推给前端 WebGL 能量核心。 + +前端 VAD 快线检测到开口时,通过 WebSocket 发 interrupt 消息,后端推 interrupt 状态(急刹冻结视觉),实现 v2 技术设计的 <50ms 爆点。 diff --git a/.moss_ws/apps/aether/core/main.py b/.moss_ws/apps/aether/core/main.py new file mode 100644 index 00000000..df433214 --- /dev/null +++ b/.moss_ws/apps/aether/core/main.py @@ -0,0 +1,422 @@ +"""Aether Core UI app — MOSS ghost 的全双工能量核心可视化通道。 + +后端不再把 listen/think/speak/interrupt 压成互斥五态,而是维护 +并发 activity layers。``state`` 只作为主视觉基调向旧前端兼容: +listen、think、speak 可以同时为 true,interrupt 是短暂抢占层。 +""" +import asyncio +import json +import logging +import time +from pathlib import Path + +from aiohttp import web, WSMsgType + +from ghoshell_moss.core.blueprint.matrix import Matrix +from ghoshell_moss.core.mindflow.interrupt_nucleus import new_interrupt_signal +from ghoshell_moss.host.speech.capture.matrix_audio_transport import MatrixAudioTransport +from ghoshell_moss.topics.audio import AudioRuntimeTopic, SpeechTopic + +# Frontend static files live with the app so aether/core is self-contained. +WEB_ROOT = Path(__file__).resolve().parent / "webroot" +WS_PORT = 8765 +INTERRUPT_HOLD = 0.7 # 急刹视觉保持时长(秒),超时回 idle + +_log = logging.getLogger("moss.aether.core") + + +@web.middleware +async def _cross_origin_isolation_middleware( + request: web.Request, + handler, +) -> web.StreamResponse: + response = await handler(request) + # Keep the UI cross-origin isolated so future WebGL/AudioWorklet features + # can use SharedArrayBuffer without changing the deployment surface. + response.headers["Cross-Origin-Opener-Policy"] = "same-origin" + response.headers["Cross-Origin-Embedder-Policy"] = "require-corp" + response.headers["Cross-Origin-Resource-Policy"] = "same-origin" + return response + + +async def main(matrix: Matrix) -> None: + logger = matrix.logger or _log + logger.info("aether/core app starting, WEB_ROOT=%s", WEB_ROOT) + + transport = MatrixAudioTransport(matrix=matrix) + speech_win = transport.topic_window(SpeechTopic, max_size=16) + audio_win = transport.topic_window(AudioRuntimeTopic, max_size=16) + + clients: set = set() + # 状态容器(避免 nonlocal 地狱)。state 是兼容字段;layers 是新契约。 + st = { + "state": "idle", + "layers": { + "listen": False, + "queue": False, + "think": False, + "speak": False, + "interrupt": False, + }, + "last_speech_ts": 0.0, + "last_speaker_running": False, + "last_asr_running": False, + "interrupt_until": 0.0, + "last_interrupt_started_at": 0.0, + "_tts_end_at": 0.0, + "think_started_at": 0.0, + "queued_started_at": 0.0, + "last_asr_diag_key": "", + "last_vpio_diag_key": "", + "asr_current": None, + "asr_finals": [], + "asr_error": None, + "asr_control": {"mode": "continuous", "enabled": True}, + "vpio_diag": "", + } + # 初始化 last_speech_ts 为当前 window 的最大值,避免启动即触发历史 + init_speeches = list(speech_win.values()) + if init_speeches: + st["last_speech_ts"] = max(t.timestamp for t in init_speeches) + + def _primary_state() -> str: + layers = st["layers"] + if layers["interrupt"]: + return "interrupt" + if layers["speak"]: + return "speak" + if layers["think"]: + return "think" + if layers["listen"]: + return "listen" + return "idle" + + def _snapshot(**extra) -> dict: + st["state"] = _primary_state() + msg = { + "state": st["state"], + "layers": dict(st["layers"]), + "ts": time.time(), + "diagnostics": { + "asr_current": dict(st["asr_current"]) if st["asr_current"] else None, + "asr_finals": list(st["asr_finals"]), + "asr_error": dict(st["asr_error"]) if st["asr_error"] else None, + "asr_control": dict(st["asr_control"]), + "vpio": st["vpio_diag"], + }, + } + msg.update(extra) + return msg + + async def broadcast(msg: dict) -> None: + if not clients: + return + data = json.dumps(msg, ensure_ascii=False) + dead = [] + for c in clients: + try: + await c.send_str(data) + except Exception: + dead.append(c) + for c in dead: + clients.discard(c) + + async def ws_handler(request: web.Request) -> web.WebSocketResponse: + ws = web.WebSocketResponse() + await ws.prepare(request) + clients.add(ws) + logger.info("ws client connected, total=%d", len(clients)) + # 连接时先推当前状态 + await ws.send_str(json.dumps(_snapshot(), ensure_ascii=False)) + try: + async for msg in ws: + if msg.type == WSMsgType.TEXT: + try: + payload = json.loads(msg.data) + except Exception: + continue + if payload.get("type") == "interrupt" or payload.get("state") == "interrupt": + st["interrupt_until"] = time.monotonic() + INTERRUPT_HOLD + st["layers"]["interrupt"] = True + st["layers"]["listen"] = False + st["layers"]["queue"] = False + st["layers"]["speak"] = False + st["layers"]["think"] = False + await broadcast(_snapshot(interrupt_burst=1.0)) + logger.info("interrupt received from frontend") + transport.pub_topic(AudioRuntimeTopic( + running=True, + device_name="interrupt", + device_explain="frontend_manual_stop", + started_at=time.monotonic(), + last_heartbeat=time.monotonic(), + )) + logger.info("★ Frontend manual stop → audio interrupt topic published") + # 发 interrupt signal 到 ghost 主进程 (通过 Zenoh 跨进程) + # → mindflow.InterruptNucleus → FATAL impulse → shell.clear() → 停 TTS + 停 LLM + # aether/core 是独立子进程,不能直接访问 ghost 的 Mindflow, + # 必须通过 session.add_signal 走 Zenoh 发布。 + try: + sig = new_interrupt_signal( + "立刻停下", + description="前端VAD检测到SPEAK中开口,触发shell.clear停TTS", + ) + matrix.session.add_signal(sig) + logger.info("★ Interrupt signal sent via zenoh (shell.clear will stop TTS)") + except Exception as e: + logger.warning("Failed to send interrupt signal: %s", e) + elif payload.get("type") == "asr_control": + mode = str(payload.get("mode", "continuous")).strip().lower() + if mode not in {"continuous", "manual"}: + mode = "continuous" + enabled = bool(payload.get("enabled", mode == "continuous")) + if mode == "continuous": + enabled = True + st["asr_control"] = {"mode": mode, "enabled": enabled} + transport.pub_topic(AudioRuntimeTopic( + running=enabled, + device_name="asr_control", + device_explain=json.dumps( + {"source": "aether/core", "mode": mode, "enabled": enabled}, + ensure_ascii=False, + separators=(",", ":"), + ), + started_at=time.monotonic(), + last_heartbeat=time.monotonic(), + )) + await broadcast(_snapshot(event="asr_control")) + logger.info("ASR control from frontend: mode=%s enabled=%s", mode, enabled) + elif payload.get("type") == "listen": + running = bool(payload.get("running")) + st["layers"]["listen"] = running + if payload.get("pending_think"): + st["layers"]["think"] = True + st["think_started_at"] = time.monotonic() + await broadcast(_snapshot(event="listen")) + elif payload.get("type") == "reset": + # 切 idle。TopicWindow 不提供 clear;用 last_speech_ts 跳过已有历史。 + # 不发布 "/reset" SpeechTopic,否则 reset 会被当作一轮用户语音, + # 造成一次没有实际回复需求的假 think。 + st["layers"] = {k: False for k in st["layers"]} + speeches = list(speech_win.values()) + if speeches: + st["last_speech_ts"] = max(t.timestamp for t in speeches) + else: + st["last_speech_ts"] = time.monotonic() + st["think_started_at"] = 0 + st["queued_started_at"] = 0 + await broadcast(_snapshot()) + logger.info("reset received from frontend — visual context cleared") + elif msg.type == WSMsgType.ERROR: + break + finally: + clients.discard(ws) + logger.info("ws client disconnected, total=%d", len(clients)) + return ws + + async def index_handler(request: web.Request) -> web.FileResponse: + return web.FileResponse(WEB_ROOT / "index.html") + + app = web.Application(middlewares=[_cross_origin_isolation_middleware]) + app.router.add_get("/", index_handler) + app.router.add_get("/ws", ws_handler) + app.router.add_static("/web", path=str(WEB_ROOT / "web"), name="web") + + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, "0.0.0.0", WS_PORT) + await site.start() + logger.info("aether/core http+ws server on http://127.0.0.1:%d", WS_PORT) + + async def state_loop() -> None: + while True: + now = time.monotonic() + now_ts = time.time() + # interrupt 超时回退 + if st["layers"]["interrupt"] and now > st["interrupt_until"]: + st["layers"]["interrupt"] = False + await broadcast(_snapshot()) + + # think 超时兜底:LLM 崩溃/超时时不卡在 think + if st["layers"]["think"] and now - st.get("think_started_at", 0) > 30.0: + st["layers"]["think"] = False + st["layers"]["queue"] = False + st["think_started_at"] = 0 + st["queued_started_at"] = 0 + await broadcast(_snapshot()) + logger.warning("think timeout 30s → %s", _primary_state()) + + # 全双工打断:检测 listener 发布的"停下"关键词 interrupt 信号 + for t in reversed(list(audio_win.values())): + if getattr(t, "device_name", "") == "interrupt" and t.running: + started_at = float(getattr(t, "started_at", 0.0) or 0.0) + if started_at > st["last_interrupt_started_at"]: + st["last_interrupt_started_at"] = started_at + st["layers"]["interrupt"] = True + st["layers"]["listen"] = False + st["layers"]["queue"] = False + st["layers"]["speak"] = False + st["layers"]["think"] = False + st["interrupt_until"] = now + INTERRUPT_HOLD + await broadcast(_snapshot(interrupt_burst=1.0)) + logger.info("★ Wake word barge-in → interrupt (from listener)") + break + + # 检查 speech(用户说完一句 → think) + speeches = list(speech_win.values()) + if speeches: + latest = speeches[-1] + if getattr(latest, "role", "") == "human" and latest.timestamp > st["last_speech_ts"]: + st["last_speech_ts"] = latest.timestamp + if not st["layers"]["interrupt"]: + queued = bool(st["layers"]["speak"]) + st["layers"]["listen"] = False + if queued: + st["layers"]["queue"] = True + st["queued_started_at"] = now + st["layers"]["think"] = True + st["think_started_at"] = now + await broadcast(_snapshot(text=latest.text)) + logger.info("%s: %s", "speech→queue+think" if queued else "speech→think", latest.text[:60]) + + # 后端 ASR 活动(区别于浏览器本地 VAD):ASR partial 出现才是真正 + # 进入后端听写链路。若 ASR 空等或浏览器误触,不能伪装成 think。 + asr_running = None + asr_topic = None + for t in reversed(list(audio_win.values())): + if getattr(t, "device_name", "") == "asr": + asr_topic = t + asr_running = t.running + break + if asr_topic is not None and asr_topic.running: + explain = getattr(asr_topic, "device_explain", "") or "" + diag_key = f"{getattr(asr_topic, 'started_at', 0)}:{explain}" + if explain and diag_key != st["last_asr_diag_key"]: + st["last_asr_diag_key"] = diag_key + changed = False + try: + parsed = json.loads(explain) + if parsed.get("error"): + st["asr_error"] = { + "error": str(parsed.get("error")), + "code": str(parsed.get("code", "")), + "message": str(parsed.get("message", "")), + "backoff": float(parsed.get("backoff", 0) or 0), + "consecutive": int(parsed.get("consecutive", 0) or 0), + "ts": now_ts, + } + changed = True + text = str(parsed.get("text", "")).strip() + if text: + item = { + "text": text, + "final": bool(parsed.get("final")), + "ts": now_ts, + } + st["asr_error"] = None + if item["final"]: + st["asr_current"] = None + st["asr_finals"].append(item) + st["asr_finals"] = st["asr_finals"][-3:] + else: + st["asr_current"] = item + changed = True + except Exception: + st["asr_current"] = {"text": explain, "final": False, "ts": now_ts} + changed = True + if changed: + await broadcast(_snapshot(event="asr_diag")) + elif asr_topic is not None and not asr_topic.running: + explain = getattr(asr_topic, "device_explain", "") or "" + diag_key = f"{getattr(asr_topic, 'started_at', 0)}:{explain}" + if explain and diag_key != st["last_asr_diag_key"]: + st["last_asr_diag_key"] = diag_key + try: + parsed = json.loads(explain) + except Exception: + parsed = {} + if parsed.get("error"): + st["asr_error"] = { + "error": str(parsed.get("error")), + "code": str(parsed.get("code", "")), + "message": str(parsed.get("message", "")), + "backoff": float(parsed.get("backoff", 0) or 0), + "consecutive": int(parsed.get("consecutive", 0) or 0), + "ts": now_ts, + } + await broadcast(_snapshot(event="asr_error")) + if asr_running is not None and asr_running != st["last_asr_running"]: + st["last_asr_running"] = asr_running + if asr_running: + if not st["layers"]["interrupt"]: + st["layers"]["listen"] = True + await broadcast(_snapshot(event="asr_listen")) + logger.info("ASR→listen") + else: + if st["layers"]["listen"] and not st["layers"]["think"] and not st["layers"]["speak"]: + st["layers"]["listen"] = False + await broadcast(_snapshot(event="asr_idle")) + logger.info("ASR listen ended → idle") + + vpio_topic = None + for t in reversed(list(audio_win.values())): + if getattr(t, "device_name", "") == "vpio": + vpio_topic = t + break + if vpio_topic is not None and vpio_topic.running: + explain = getattr(vpio_topic, "device_explain", "") or "" + diag_key = f"{getattr(vpio_topic, 'last_heartbeat', 0)}:{explain}" + if explain and diag_key != st["last_vpio_diag_key"]: + st["last_vpio_diag_key"] = diag_key + st["vpio_diag"] = explain + await broadcast(_snapshot(event="vpio_diag")) + + # 检查 TTS speaker(running → speak,stopped → idle) + running = None + for t in reversed(list(audio_win.values())): + if getattr(t, "device_name", "") == "speaker": + running = t.running + break + if running is not None and running != st["last_speaker_running"]: + st["last_speaker_running"] = running + if running: + if not st["layers"]["interrupt"]: + st["_tts_end_at"] = 0.0 + if st["layers"]["queue"]: + st["layers"]["queue"] = False + st["queued_started_at"] = 0 + st["layers"]["speak"] = True + # LLM may continue preparing the next delta while TTS starts; + # leave think true briefly only if a new speech turn owns it. + await broadcast(_snapshot()) + logger.info("TTS→speak") + else: + # TTS 结束后加 800ms 保护期,让尾音播完,避免 VAD 误触 listen + if st["layers"]["speak"] or st["layers"]["interrupt"]: + st["_tts_end_at"] = now + logger.info("TTS ended → 800ms grace before idle") + + # TTS 结束保护期:800ms 后切 idle + if st.get("_tts_end_at") and now - st["_tts_end_at"] > 0.8: + if (st["layers"]["speak"] or st["layers"]["interrupt"]) and not st["last_speaker_running"]: + st["layers"]["speak"] = False + st["layers"]["interrupt"] = False + if not st["layers"]["queue"] and st.get("think_started_at", 0) <= st.get("_tts_end_at", 0): + st["layers"]["think"] = False + await broadcast(_snapshot()) + logger.info("TTS grace end → %s", _primary_state()) + st["_tts_end_at"] = 0.0 + + await asyncio.sleep(0.03) + + try: + await state_loop() + except asyncio.CancelledError: + logger.info("aether/core cancelled") + finally: + await runner.cleanup() + logger.info("aether/core stopped") + + +if __name__ == "__main__": + Matrix.discover().run(main) diff --git a/.moss_ws/apps/aether/core/pyproject.toml b/.moss_ws/apps/aether/core/pyproject.toml new file mode 100644 index 00000000..8b167da8 --- /dev/null +++ b/.moss_ws/apps/aether/core/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "aether_core" +version = "0.1.0" +requires-python = ">=3.11,<3.14" +dependencies = [ + "ghoshell-moss[host]", + "aiohttp>=3.9", +] + +[tool.uv] +override-dependencies = [ + "numpy<2.3", +] + +[tool.uv.sources] +ghoshell-moss = { path = "../../../..", editable = true } diff --git a/.moss_ws/apps/aether/core/webroot/index.html b/.moss_ws/apps/aether/core/webroot/index.html new file mode 100644 index 00000000..91a317ed --- /dev/null +++ b/.moss_ws/apps/aether/core/webroot/index.html @@ -0,0 +1,143 @@ + + + + + +MOSS · Aether Core 能量核心 + + + +
+ + + +
+
MOSSAETHER CORE · 能量核心
+
本地演示模式
+ +
+
+
+
+
+
+
+
+
+
STATE
+
idle
+
+
+ + + + + +
+
并发层
+
→ idle 呼吸
+
+ listen ASR 已收到语音
+
+ queue speak 中待回答
+
+ think Ghost 推理中
+
+ speak TTS 输出中
+
★ interrupt 急停抢占
+
+ + +
+
语音诊断
+
+
ASR 当前识别
+
+
等待 ASR partial
+
+
+
+
ASR 最近 final
+
+
等待 final
+
+
+ +
+
VPIO 采集
+
等待音频统计
+
+
+ + +
+ 急刹打断
+ 说「立刻停下」(或点「打断」按钮),整团能量在 <100ms 内急刹冻结、聚焦闪白。 +

+ LISTEN 可与 THINK/SPEAK 同时亮起
+ ASR 慢线 确认急停并停止 TTS
+ 不是互斥五态,是全双工叠层 +
+ + +
+
IDLE
+ + +
+ + +
+
+
主控
+
+ + + + + + + 麦克风未开 + ASR 连续监听 +
+
+ +
+
VAD 快线(能量阈值)
+
+
+
+
+
+ 阈值 + + 0.008 +
+
+ +
+
手动状态切换
+
+ + + + + +
+
+
+ intensity + + 0.00 +
+
+
+
+
+ + + + diff --git a/.moss_ws/apps/aether/core/webroot/web/core.frag b/.moss_ws/apps/aether/core/webroot/web/core.frag new file mode 100644 index 00000000..0c7a1ed0 --- /dev/null +++ b/.moss_ws/apps/aether/core/webroot/web/core.frag @@ -0,0 +1,148 @@ +#version 300 es +// Aether Core - 能量核心 Fragment Shader +// raymarched 流体球 + fbm 表面扰动 + 体积辉光 + INTERRUPT 急刹闪白 +// 5 状态参数由 CPU 端插值后传入(idle/listen/think/speak/interrupt) +precision highp float; + +uniform vec2 u_resolution; +uniform float u_time; // 全局时间(秒) +uniform float u_intensity; // 0~1,speak 时音量驱动喷涌 +uniform float u_interrupt_burst; // 0~1,急刹瞬间闪白强度(衰减) + +// 状态参数(CPU 插值后传入,状态切换在 CPU 端做 ease) +uniform float u_radius; // 球半径 +uniform float u_amp; // 表面扰动幅度 +uniform float u_freq; // 表面噪声频率 +uniform float u_speed; // 内部时间速度(interrupt 时 ≈ 0 = 冻结) +uniform vec3 u_color; // 基色 +uniform vec3 u_glow; // 外辉光色 + +out vec4 fragColor; + +// ---------- 3D value noise ---------- +float hash13(vec3 p) { + p = fract(p * 0.3183099 + 0.1); + p *= 17.0; + return fract(p.x * p.y * p.z * (p.x + p.y + p.z)); +} + +float noise3(vec3 x) { + vec3 i = floor(x); + vec3 f = fract(x); + f = f * f * (3.0 - 2.0 * f); + return mix( + mix(mix(hash13(i + vec3(0,0,0)), hash13(i + vec3(1,0,0)), f.x), + mix(hash13(i + vec3(0,1,0)), hash13(i + vec3(1,1,0)), f.x), f.y), + mix(mix(hash13(i + vec3(0,0,1)), hash13(i + vec3(1,0,1)), f.x), + mix(hash13(i + vec3(0,1,1)), hash13(i + vec3(1,1,1)), f.x), f.y), + f.z); +} + +// domain-warped fbm:流体感的关键 +float fbm(vec3 p) { + float v = 0.0; + float a = 0.5; + for (int i = 0; i < 5; i++) { + v += a * noise3(p); + p = p * 2.02 + vec3(1.7, 9.2, 3.3); + a *= 0.5; + } + return v; +} + +// SDF:球 + fbm 扰动 +float map(vec3 p, float t) { + vec3 q = p * u_freq + vec3(0.0, t * 0.5, t * 0.3); + // domain warp + vec3 w = vec3(fbm(q + vec3(0.0, 0.0, 0.0)), + fbm(q + vec3(5.2, 1.3, 2.7)), + fbm(q + vec3(3.1, 7.4, 1.1))); + float n = fbm(q + w * 1.5); + return length(p) - (u_radius + (n - 0.5) * u_amp); +} + +// 数值法线(中心差分) +vec3 calcNormal(vec3 p, float t) { + float e = 0.008; + vec2 h = vec2(1.0, -1.0) * e; + return normalize( + h.xyy * map(p + h.xyy, t) + + h.yyx * map(p + h.yyx, t) + + h.yxy * map(p + h.yxy, t) + + h.xxx * map(p + h.xxx, t) + ); +} + +void main() { + vec2 uv = (gl_FragCoord.xy - 0.5 * u_resolution) / min(u_resolution.x, u_resolution.y); + + vec3 ro = vec3(0.0, 0.0, 2.6); + vec3 rd = normalize(vec3(uv, -1.6)); + + float t = u_time * u_speed; + + // raymarch(带最小距离记录做体积辉光) + float tt = 0.05; + bool hit = false; + float minDist = 1e9; + vec3 hitPos = vec3(0.0); + for (int i = 0; i < 96; i++) { + vec3 p = ro + rd * tt; + float d = map(p, t); + if (d < minDist) minDist = d; + if (d < 0.002) { hit = true; hitPos = p; break; } + tt += d * 0.85; + if (tt > 5.0) break; + } + + // ---------- 体积辉光(基于 raymarch 路径上最近距离) ---------- + float glow = exp(-minDist * 7.0); + float softGlow = exp(-minDist * 2.5) * 0.5; + + vec3 col = vec3(0.0); + + if (hit) { + vec3 n = calcNormal(hitPos, t); + vec3 L = normalize(vec3(0.5, 0.8, 1.0)); + float diff = max(dot(n, L), 0.0); + float fres = pow(1.0 - max(dot(-rd, n), 0.0), 2.5); + // 内核自发光(深处更亮,模拟能量核心) + float core = 1.0 - smoothstep(u_radius * 0.3, u_radius * 1.05, length(hitPos)); + col = u_color * (0.25 + 0.6 * diff + core * 0.8); + col += u_color * fres * 1.8; + // speak 脉动喷涌:随 intensity 表面亮度脉冲 + col += u_color * u_intensity * 0.7 * fres; + } + + // 外辉光 + col += u_glow * glow * 1.4; + col += u_glow * softGlow * 0.8; + + // speak 外喷辉光 + col += u_glow * u_intensity * 0.5 * exp(-minDist * 4.0); + + // ---------- INTERRUPT 急刹爆点 ---------- + // 一帧聚焦闪白 + 向心吸附光线 + if (u_interrupt_burst > 0.0) { + float b = u_interrupt_burst; + // 全场闪白(暗角中心更亮,制造聚焦感) + float vignette = 1.0 - length(uv) * 0.7; + col += vec3(1.0) * b * 2.2 * max(vignette, 0.0); + // 向心吸附光线(从中心向外辐射,强度随距离衰减) + float r = length(uv); + float rays = exp(-r * 1.5) * smoothstep(0.0, 0.15, b); + col += vec3(1.0) * rays * 1.8; + // 整体提亮(急刹定格的"咔"的一下) + col += vec3(0.6, 0.7, 1.0) * b * 0.3; + } + + // ---------- 色调映射 + gamma ---------- + col = col / (1.0 + col); + col = pow(col, vec3(1.0 / 2.2)); + + // 暗角 + float vig = 1.0 - length(uv) * 0.35; + col *= clamp(vig, 0.4, 1.0); + + fragColor = vec4(col, 1.0); +} diff --git a/.moss_ws/apps/aether/core/webroot/web/core.vert b/.moss_ws/apps/aether/core/webroot/web/core.vert new file mode 100644 index 00000000..aaf8b516 --- /dev/null +++ b/.moss_ws/apps/aether/core/webroot/web/core.vert @@ -0,0 +1,6 @@ +#version 300 es +// Aether Core - 顶点 Shader:全屏三角形 +in vec2 a_pos; +void main() { + gl_Position = vec4(a_pos, 0.0, 1.0); +} diff --git a/.moss_ws/apps/aether/core/webroot/web/main.js b/.moss_ws/apps/aether/core/webroot/web/main.js new file mode 100644 index 00000000..6b87b870 --- /dev/null +++ b/.moss_ws/apps/aether/core/webroot/web/main.js @@ -0,0 +1,551 @@ +// main.js — 演示入口 +// 串联 Scene / StateMapper / VAD / StateBridge,绑定 UI +// 完整对应技术文档:状态字符串驱动 + VAD 快线急刹 + ASR 慢线(演示用模拟) + +import { Scene } from './scene.js'; +import { normalizeLayers, StateMapper, STATES, STATE_NAMES } from './state_mapper.js'; +import { VAD } from './vad.js'; +import { StateBridge } from './ws.js'; + +const $ = (s) => document.querySelector(s); +const $$ = (s) => document.querySelectorAll(s); + +// ---------- 全局实例 ---------- +let scene, mapper, vad, bridge; +let demoTimer = null; +let demoSpeakPulseRaf = null; +let manualIntensity = 0.0; +let useAutoIntensity = true; // speak 时自动脉动 +let localLayers = normalizeLayers(); +let backendLayers = normalizeLayers(); +let localMicRunning = false; +let listenClearTimer = 0; +let thinkPendingTimer = 0; +let asrMode = 'continuous'; +let asrArmed = true; + +// ---------- 初始化 ---------- +async function boot() { + scene = new Scene($('#glcanvas')); + await scene.init(); + scene.start(); + + mapper = new StateMapper({ + onState: (state, meta) => onStateChange(state, meta), + minHoldMs: 250, + interruptHoldMs: 700, + }); + + bridge = new StateBridge({ + onState: (stateName) => { + const idx = STATE_NAMES.indexOf(stateName); + if (idx >= 0) applyState(idx, { event: 'WS_STATE' }); + }, + onLayers: (layers, msg) => { + applyLayers(layers, { event: 'WS_LAYERS', text: msg.text }); + updateDiagnostics(msg.diagnostics); + if (msg.interrupt_burst) scene.interruptBurst = Math.max(scene.interruptBurst, msg.interrupt_burst); + if (msg.text || layers.speak || layers.interrupt) { + clearThinkPendingTimer(); + } + }, + onIntensity: (v) => { + manualIntensity = v; + useAutoIntensity = false; + }, + onConnect: () => { + updateModePill('ws'); + updateAsrControlUI(); + }, + onDisconnect: () => { + updateModePill('local'); + }, + }); + + // 自动连 aether/core 后端(MOSS ghost 推状态) + const ok = await bridge.connect(); + updateModePill(ok ? 'ws' : 'local'); + + bindUI(); + updateAsrControlUI(); + applyState(STATES.IDLE, { event: 'BOOT' }); + log(ok ? '已连后端 ws://localhost:8765/ws · MOSS ghost 在线' : '后端未就绪 · 本地演示模式(自动重连中)'); + log('提示:点「演示模式」看全双工叠层;本地VAD只负责快速视觉反馈,火山ASR收音由右侧按钮控制'); +} + +// ---------- 状态切换处理 ---------- +function onStateChange(state, meta) { + localLayers = normalizeLayers(meta?.layers || mapper.layers); + scene.onStateChange(state, meta); + applyStateUI(state, meta); + // 同步 VAD 的 speakMode:SPEAK 时提高阈值/duration,避免 AEC 残留回声自打断; + // 其他状态恢复原灵敏度(idle→listen 仍需低延迟开口检测)。 + if (vad) vad.setSpeakMode(state === STATES.SPEAK); +} + +function applyState(state, meta) { + // 走 state_mapper(带去抖) + mapper.event({ type: 'FORCE_STATE', state, ...(meta || {}) }); +} + +function composeVisualLayers(layers = backendLayers) { + const base = normalizeLayers(layers); + return normalizeLayers({ + ...base, + mic: Boolean(base.mic) || localMicRunning, + listen: Boolean(base.listen) || localMicRunning, + }); +} + +function applyLayers(layers, meta = {}) { + backendLayers = normalizeLayers(layers); + localLayers = normalizeLayers({ + ...backendLayers, + // Fast visual listen: local browser VAD means the user has started + // speaking. Backend ASR still owns SpeechTopic/think; this only keeps + // Aether responsive before the cloud ASR returns its first partial. + ...composeVisualLayers(backendLayers), + }); + mapper.setLayers(localLayers, meta); +} + +function applyStateUI(state, meta) { + const name = STATE_NAMES[state]; + const layers = normalizeLayers(meta?.layers || mapper.layers); + $$('.state-dot').forEach(d => { + const s = d.dataset.s; + const active = s === 'idle' ? name === 'idle' : Boolean(layers[s]); + d.classList.toggle('active', active); + d.classList.toggle('primary', s === name); + }); + $('#current-state-name').textContent = name; + updateActivityRings(layers, name); + // 球体中央大字(所有状态都显示) + const labelEl = $('#state-label'); + if (labelEl) { + labelEl.textContent = activeLabel(layers, name); + labelEl.setAttribute('data-state', name); + } + $$('.stategraph .edge').forEach(e => { + const to = e.dataset.to; + const active = to === 'idle' ? name === 'idle' : Boolean(layers[to]); + e.classList.toggle('active', active); + }); + + if (state === STATES.INTERRUPT) { + $('#interrupt-flash').classList.add('on'); + clearTimeout(window._intrFlashT); + window._intrFlashT = setTimeout(() => { + $('#interrupt-flash').classList.remove('on'); + }, 1500); + } + + log(`state → ${activeLabel(layers, name)} (${meta?.event || ''}${meta?.latencyMs != null ? ' · vad+' + meta.latencyMs.toFixed(0) + 'ms' : ''})`); +} + +function activeLabel(layers, fallback) { + const active = ['listen', 'queue', 'think', 'speak'].filter(k => layers[k]); + if (layers.interrupt) return 'INTERRUPT'; + if (active.length === 0 && layers.mic) return 'MIC'; + if (active.length === 0) return fallback.toUpperCase(); + return active.map(s => s.toUpperCase()).join(' + '); +} + +function updateActivityRings(layers, primary) { + ['mic', 'listen', 'queue', 'think', 'speak', 'interrupt'].forEach(name => { + const el = document.querySelector(`.activity-ring[data-layer="${name}"]`); + if (!el) return; + el.classList.toggle('active', Boolean(layers[name])); + el.classList.toggle('primary', primary === name); + }); + const stack = $('#activity-stack'); + if (stack) stack.setAttribute('data-primary', primary); +} + +function updateModePill(mode) { + const el = $('#mode-pill'); + el.textContent = mode === 'ws' ? 'WS · 已连后端' : '本地演示模式'; + el.className = 'mode-pill ' + mode; +} + +function escapeHtml(value) { + return String(value ?? '') + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", '''); +} + +function updateDiagnostics(diag) { + if (!diag) return; + const currentEl = $('#asr-current'); + if (currentEl) { + if (diag.asr_current?.text) { + currentEl.innerHTML = ` +
+
partial
+
${escapeHtml(diag.asr_current.text)}
+
+ `; + } else { + currentEl.innerHTML = '
等待 ASR partial
'; + } + } + + const asrEl = $('#asr-diag'); + if (asrEl && Array.isArray(diag.asr_finals)) { + if (diag.asr_finals.length === 0) { + asrEl.innerHTML = '
等待 final
'; + } else { + asrEl.innerHTML = diag.asr_finals.slice(-3).reverse().map(item => { + return ` +
+
final
+
${escapeHtml(item.text)}
+
+ `; + }).join(''); + } + } + + const errorEl = $('#asr-error'); + if (errorEl) { + if (diag.asr_error?.error) { + const backoff = Number(diag.asr_error.backoff || 0); + const code = diag.asr_error.code ? ` · code=${escapeHtml(diag.asr_error.code)}` : ''; + const message = diag.asr_error.message ? ` · ${escapeHtml(diag.asr_error.message)}` : ''; + const wait = backoff > 0 ? ` · retry ${backoff.toFixed(0)}s` : ''; + errorEl.hidden = false; + errorEl.textContent = `ASR ${diag.asr_error.error}${code}${message}${wait}`; + } else { + errorEl.hidden = true; + errorEl.textContent = ''; + } + } + + if (diag.asr_control) { + asrMode = diag.asr_control.mode || asrMode; + asrArmed = asrMode === 'continuous' ? true : Boolean(diag.asr_control.enabled); + updateAsrControlUI(); + } + + const vpioEl = $('#vpio-diag'); + if (vpioEl && typeof diag.vpio === 'string' && diag.vpio) { + vpioEl.textContent = diag.vpio; + } +} + +// ---------- 日志 ---------- +const logLines = []; +const MAX_LOG = 30; +function log(msg) { + const ts = new Date().toLocaleTimeString('zh-CN', { hour12: false }); + logLines.push(`[${ts}] ${msg}`); + if (logLines.length > MAX_LOG) logLines.shift(); + const el = $('#log'); + el.innerHTML = logLines.map((l, i) => { + const age = logLines.length - 1 - i; + const op = Math.max(0.25, 1 - age * 0.05); + return `
${l}
`; + }).join(''); + el.scrollTop = el.scrollHeight; +} + +// ---------- 手动连接 Python 后端 ---------- +async function connectBackend() { + const btn = $('#btn-ws'); + btn.textContent = '连接中...'; + btn.disabled = true; + const ok = await bridge.connect(); + btn.disabled = false; + btn.textContent = ok ? '已连后端' : '连接失败'; + updateModePill(ok ? 'ws' : 'local'); + updateAsrControlUI(); + log('ws connect: ' + (ok ? 'success (ws://localhost:8765)' : 'failed, stay local')); +} + +// ---------- 演示模式:自动循环 idle→listen→think→speak→idle ---------- +function startDemo() { + stopDemo(); + $('#btn-demo').classList.add('active'); + $('#btn-demo').textContent = '停止演示'; + log('演示模式启动'); + runDemoStep(); +} + +function stopDemo() { + if (demoTimer) { clearTimeout(demoTimer); demoTimer = null; } + if (demoSpeakPulseRaf) { cancelAnimationFrame(demoSpeakPulseRaf); demoSpeakPulseRaf = null; } + $('#btn-demo').classList.remove('active'); + $('#btn-demo').textContent = '演示模式'; +} + +function runDemoStep() { + if (!$('#btn-demo').classList.contains('active')) return; + // 序列:idle → listen → think → think+speak → listen+think+speak → idle + applyLayers({}, { event: 'DEMO' }); + demoTimer = setTimeout(() => { + applyLayers({ listen: true }, { event: 'DEMO' }); + demoTimer = setTimeout(() => { + applyLayers({ think: true }, { event: 'DEMO' }); + demoTimer = setTimeout(() => { + applyLayers({ think: true, speak: true }, { event: 'DEMO' }); + startSpeakPulse(); + demoTimer = setTimeout(() => { + applyLayers({ listen: true, think: true, speak: true }, { event: 'DEMO_DUPLEX' }); + demoTimer = setTimeout(() => { + stopSpeakPulse(); + runDemoStep(); + }, 1600); + }, 2400); + }, 1400); + }, 1400); + }, 2000); +} + +function requestListenLayer(running, meta = {}) { + if (listenClearTimer) { + clearTimeout(listenClearTimer); + listenClearTimer = 0; + } + const next = normalizeLayers({ ...localLayers, listen: running }); + applyLayers(next, meta); + bridge.sendListen?.(running, meta.backend || {}); +} + +function requestMicLayer(running, meta = {}) { + if (listenClearTimer) { + clearTimeout(listenClearTimer); + listenClearTimer = 0; + } + localMicRunning = running; + localLayers = composeVisualLayers(backendLayers); + mapper.setLayers(localLayers, meta); +} + +function clearMicLayer(meta = {}) { + requestMicLayer(false, meta); +} + +function clearMicLayerSoon(meta = {}, delayMs = 1800) { + if (listenClearTimer) clearTimeout(listenClearTimer); + listenClearTimer = setTimeout(() => { + listenClearTimer = 0; + if (localMicRunning && !backendLayers.listen && !backendLayers.queue && !backendLayers.think && !backendLayers.speak && !backendLayers.interrupt) { + clearMicLayer(meta); + } + }, delayMs); +} + +function clearThinkPendingTimer() { + if (thinkPendingTimer) { + clearTimeout(thinkPendingTimer); + thinkPendingTimer = 0; + } +} + +function updateAsrControlUI() { + const modeBtn = $('#btn-asr-mode'); + const armBtn = $('#btn-asr-arm'); + const status = $('#asr-control-status'); + if (!modeBtn || !armBtn || !status) return; + const manual = asrMode === 'manual'; + modeBtn.textContent = manual ? '火山ASR 手动' : '火山ASR 连续'; + modeBtn.classList.toggle('active', manual); + armBtn.disabled = !manual; + armBtn.textContent = asrArmed ? '停止火山收音' : '开始火山收音'; + armBtn.classList.toggle('active', manual && asrArmed); + status.textContent = manual + ? (asrArmed ? 'ASR 手动收音中' : 'ASR 手动待机') + : 'ASR 连续监听'; + status.className = 'mic-status ' + (manual && !asrArmed ? '' : 'on'); +} + +function sendAsrControl() { + if (asrMode === 'continuous') asrArmed = true; + bridge.sendAsrControl?.(asrMode, asrArmed); + updateAsrControlUI(); +} + +function enterThinkPending(meta = {}) { + if (listenClearTimer) { + clearTimeout(listenClearTimer); + listenClearTimer = 0; + } + clearThinkPendingTimer(); + applyLayers({ ...localLayers, listen: false, think: true }, meta); + bridge.sendListen?.(false, { pending_think: true }); + thinkPendingTimer = setTimeout(() => { + thinkPendingTimer = 0; + if (localLayers.think && !localLayers.queue && !localLayers.speak && !localLayers.interrupt) { + applyLayers({ think: false }, { event: 'THINK_PENDING_TIMEOUT' }); + log('ASR/LLM pending timeout · back to idle'); + } + }, 10000); +} + +// speak 期间自动脉动 intensity(模拟 TTS 音量) +function startSpeakPulse() { + useAutoIntensity = true; + const start = performance.now(); + const tick = () => { + if (!demoSpeakPulseRaf) return; + demoSpeakPulseRaf = requestAnimationFrame(tick); + const t = (performance.now() - start) / 1000; + // 像说话的节奏:多个 sin 叠加 + 一点随机 + const base = 0.55 + 0.35 * Math.sin(t * 6.0) + 0.10 * Math.sin(t * 17.0 + 1.3); + const v = Math.max(0, Math.min(1, base)); + if (useAutoIntensity) { + scene.setIntensity(v); + $('#intensity-slider').value = v.toFixed(3); + $('#intensity-value').textContent = v.toFixed(2); + } + }; + demoSpeakPulseRaf = requestAnimationFrame(tick); +} + +function stopSpeakPulse() { + if (demoSpeakPulseRaf) { cancelAnimationFrame(demoSpeakPulseRaf); demoSpeakPulseRaf = null; } + scene.setIntensity(0); + $('#intensity-slider').value = 0; + $('#intensity-value').textContent = '0.00'; +} + +// ---------- VAD 快线 ---------- +async function toggleMic() { + if (vad) { + vad.stop(); + vad = null; + $('#btn-mic').classList.remove('active'); + $('#btn-mic').textContent = '开启本地VAD'; + $('#mic-status').textContent = '麦克风未开'; + $('#mic-status').className = 'mic-status'; + $('#level-bar > div').style.width = '0%'; + clearMicLayer({ event: 'VAD_OFF' }); + return; + } + try { + vad = new VAD({ + threshold: parseFloat($('#vad-threshold').value), + minDurMs: 35, + endSilenceMs: 520, + // VPIO 已启用系统级 AEC(input+output VPIO=True),后端回声已消除。 + // 前端 VAD 仍用浏览器麦克风(WebRTC AEC),保留小量安全边际即可。 + speakThresholdScale: 1.2, + speakMinDurScale: 1.1, + onEchoCancelReport: (settings) => { + // 上报浏览器实际生效的 AEC 设置(可能被降级,需可见) + const aec = settings.echoCancellation; + const aecType = settings.echoCancellationType || 'n/a'; + const ns = settings.noiseSuppression; + const agc = settings.autoGainControl; + log(`AEC 报告 · echoCancellation=${aec} (${aecType}) · NS=${ns} · AGC=${agc}`); + }, + onLevel: (rms) => { + const pct = Math.min(100, rms * 600); + $('#level-bar > div').style.width = pct.toFixed(1) + '%'; + }, + onSpeechStarted: ({ vadLatencyMs, speakMode }) => { + requestMicLayer(true, { event: 'VAD_START', vadLatencyMs }); + log(`VAD · mic activity${speakMode ? ' during speak' : ''} · vad+${vadLatencyMs.toFixed(0)}ms`); + }, + onSpeechEnded: () => { + log('VAD SPEECH_ENDED · waiting ASR final'); + clearMicLayerSoon({ event: 'VAD_END_WAIT_ASR' }, 3500); + }, + }); + await vad.start(); + $('#btn-mic').classList.add('active'); + $('#btn-mic').textContent = '关闭本地VAD'; + $('#mic-status').textContent = '本地VAD监听中'; + $('#mic-status').className = 'mic-status on'; + log(`VAD 启动 · 阈值=${vad.threshold}`); + stopSpeakPulse(); + } catch (e) { + log('麦克风启动失败: ' + e.message); + $('#mic-status').textContent = '麦克风权限被拒'; + $('#mic-status').className = 'mic-status warn'; + } +} + +// ---------- UI 绑定 ---------- +function bindUI() { + $('#btn-ws').onclick = connectBackend; + $('#btn-demo').onclick = () => { + if (demoTimer) stopDemo(); + else startDemo(); + }; + $('#btn-mic').onclick = toggleMic; + $('#btn-asr-mode').onclick = () => { + if (asrMode === 'continuous') { + asrMode = 'manual'; + asrArmed = false; + log('ASR 控制 · 手动待机(火山 ASR 不监听)'); + } else { + asrMode = 'continuous'; + asrArmed = true; + log('ASR 控制 · 连续监听'); + } + sendAsrControl(); + }; + $('#btn-asr-arm').onclick = () => { + if (asrMode !== 'manual') return; + asrArmed = !asrArmed; + log(asrArmed ? 'ASR 控制 · 开始收音' : 'ASR 控制 · 停止收音'); + sendAsrControl(); + }; + $('#btn-reset').onclick = () => { + logLines.length = 0; + $('#log').innerHTML = ''; + bridge.sendReset?.(); + log('上下文已重置(清空对话历史)'); + }; + $('#btn-interrupt').onclick = () => { + const t0 = performance.now(); + applyLayers({ interrupt: true }, { event: 'MANUAL_INTERRUPT' }); + bridge.sendInterrupt?.(); + log(`手动打断 · state→interrupt ${(performance.now() - t0).toFixed(1)}ms`); + scene.setIntensity(0); + stopSpeakPulse(); + }; + + // 5 个手动状态按钮 + $$('.btn-state').forEach(btn => { + btn.onclick = () => { + const name = btn.dataset.s; + const idx = STATE_NAMES.indexOf(name); + if (idx >= 0) { + applyState(idx); + if (name === 'speak') startSpeakPulse(); + else if (name !== 'speak') stopSpeakPulse(); + } + }; + }); + + // intensity 滑块 + $('#intensity-slider').oninput = (e) => { + const v = parseFloat(e.target.value); + useAutoIntensity = false; + manualIntensity = v; + scene.setIntensity(v); + $('#intensity-value').textContent = v.toFixed(2); + }; + + // VAD 阈值 + $('#vad-threshold').oninput = (e) => { + const v = parseFloat(e.target.value); + $('#vad-threshold-value').textContent = v.toFixed(3); + if (vad) vad.threshold = v; + const mark = (Math.min(1, v * 8) * 100).toFixed(1); + $('#threshold-mark').style.left = mark + '%'; + }; + // 初始化阈值标记 + $('#vad-threshold').dispatchEvent(new Event('input')); +} + +// ---------- 启动 ---------- +boot().catch(e => { + console.error(e); + log('启动失败: ' + e.message); +}); diff --git a/.moss_ws/apps/aether/core/webroot/web/scene.js b/.moss_ws/apps/aether/core/webroot/web/scene.js new file mode 100644 index 00000000..9a1035c2 --- /dev/null +++ b/.moss_ws/apps/aether/core/webroot/web/scene.js @@ -0,0 +1,188 @@ +// scene.js — WebGL2 渲染循环 + 状态参数插值 + INTERRUPT 急刹爆点 +// 职责: +// 1. 编译 shader、建 VAO、设置 uniform +// 2. 维护"当前视觉参数"(cur)向"目标参数"(target = STATE_TARGETS[state])做 ease lerp +// —— INTERRUPT 用大速率(急刹),其他用小速率(柔和切换) +// 3. interrupt_burst:进入 INTERRUPT 时设为 1.0,每帧指数衰减(~150ms 大半) +// 4. speak intensity 驱动 amp/radius 微脉动 + +import { blendTargetsForLayers, normalizeLayers, STATE_TARGETS, STATE_SWITCH_RATE, STATES } from './state_mapper.js'; + +export class Scene { + constructor(canvas) { + this.canvas = canvas; + const gl = canvas.getContext('webgl2', { antialias: false, alpha: false, powerPreference: 'high-performance', preserveDrawingBuffer: true }); + if (!gl) throw new Error('WebGL2 not supported'); + this.gl = gl; + + // 当前视觉参数(向 target 插值) + this.cur = { ...structuredClone(STATE_TARGETS[STATES.IDLE]) }; + this.cur.color = [...this.cur.color]; + this.cur.glow = [...this.cur.glow]; + + this.target = STATE_TARGETS[STATES.IDLE]; + this.switchRate = STATE_SWITCH_RATE[STATES.IDLE]; + this.layers = normalizeLayers(); + + this.intensity = 0.0; // speak 音量 + this.interruptBurst = 0.0; // 急刹闪白强度 0~1 + this.lastInterruptTs = 0; + this.time = 0; + this.startTime = performance.now() / 1000; + + this.uniforms = {}; + this.ready = false; + } + + async init() { + const gl = this.gl; + const [vsrc, fsrc] = await Promise.all([ + fetch('./web/core.vert').then(r => r.text()), + fetch('./web/core.frag').then(r => r.text()), + ]); + const vs = this._compile(gl.VERTEX_SHADER, vsrc); + const fs = this._compile(gl.FRAGMENT_SHADER, fsrc); + const prog = gl.createProgram(); + gl.attachShader(prog, vs); + gl.attachShader(prog, fs); + gl.linkProgram(prog); + if (!gl.getProgramParameter(prog, gl.LINK_STATUS)) { + throw new Error('Link failed: ' + gl.getProgramInfoLog(prog)); + } + this.program = prog; + gl.useProgram(prog); + + // 全屏三角形 VAO + const vao = gl.createVertexArray(); + gl.bindVertexArray(vao); + const buf = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, buf); + gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([ + -1, -1, 3, -1, -1, 3, + ]), gl.STATIC_DRAW); + const loc = gl.getAttribLocation(prog, 'a_pos'); + gl.enableVertexAttribArray(loc); + gl.vertexAttribPointer(loc, 2, gl.FLOAT, false, 0, 0); + + // uniform 位置 + const U = [ + 'u_resolution', 'u_time', 'u_intensity', 'u_interrupt_burst', + 'u_radius', 'u_amp', 'u_freq', 'u_speed', 'u_color', 'u_glow', + ]; + for (const u of U) this.uniforms[u] = gl.getUniformLocation(prog, u); + + this.ready = true; + this._resize(); + window.addEventListener('resize', () => this._resize()); + } + + _compile(type, src) { + const gl = this.gl; + const sh = gl.createShader(type); + gl.shaderSource(sh, src); + gl.compileShader(sh); + if (!gl.getShaderParameter(sh, gl.COMPILE_STATUS)) { + const log = gl.getShaderInfoLog(sh); + console.error('Shader compile error:', log, '\n--- source ---\n', src); + throw new Error('Shader compile failed: ' + log); + } + return sh; + } + + _resize() { + const dpr = Math.min(window.devicePixelRatio || 1, 2); + const w = Math.floor(window.innerWidth * dpr); + const h = Math.floor(window.innerHeight * dpr); + if (this.canvas.width !== w || this.canvas.height !== h) { + this.canvas.width = w; + this.canvas.height = h; + } + } + + /** 状态切换钩子(由 state_mapper 触发) */ + onStateChange(state, meta = {}) { + this.layers = normalizeLayers(meta.layers); + this.target = meta.layers ? blendTargetsForLayers(this.layers) : STATE_TARGETS[state]; + this.switchRate = STATE_SWITCH_RATE[state]; + if (state === STATES.INTERRUPT) { + this.interruptBurst = 1.0; + this.lastInterruptTs = performance.now(); + } + } + + setLayers(layers) { + this.layers = normalizeLayers(layers); + this.target = blendTargetsForLayers(this.layers); + } + + /** 设置 speak intensity(0~1) */ + setIntensity(v) { + this.intensity = v; + } + + _lerp(a, b, t) { return a + (b - a) * t; } + _lerpArr(a, b, t) { return a.map((v, i) => v + (b[i] - v) * t); } + + render = () => { + const gl = this.gl; + this._raf = requestAnimationFrame(this.render); + if (!this.ready) return; + + const now = performance.now() / 1000; + const dt = Math.min(0.05, now - (this._lastTime || now)); + this._lastTime = now; + this.time = now - this.startTime; + + // 参数插值:cur → target,按 switchRate 做指数 ease + // cur = cur + (target - cur) * (1 - exp(-rate * dt)) + const a = 1.0 - Math.exp(-this.switchRate * dt); + const tgt = this.target; + const c = this.cur; + c.radius = this._lerp(c.radius, tgt.radius, a); + c.amp = this._lerp(c.amp, tgt.amp, a); + c.freq = this._lerp(c.freq, tgt.freq, a); + c.speed = this._lerp(c.speed, tgt.speed, a); + c.color = this._lerpArr(c.color, tgt.color, a); + c.glow = this._lerpArr(c.glow, tgt.glow, a); + + // interrupt_burst 衰减:~150ms 大半衰减 + this.interruptBurst *= Math.exp(-dt * 8.0); + if (this.interruptBurst < 0.001) this.interruptBurst = 0; + + // speak 脉动:让半径/幅度随 intensity 微跳 + const pulse = 0.0; + const intensity = this.intensity; + + gl.viewport(0, 0, this.canvas.width, this.canvas.height); + gl.clearColor(0.015, 0.022, 0.045, 1.0); + gl.clear(gl.COLOR_BUFFER_BIT); + + gl.useProgram(this.program); + const U = this.uniforms; + gl.uniform2f(U.u_resolution, this.canvas.width, this.canvas.height); + gl.uniform1f(U.u_time, this.time); + gl.uniform1f(U.u_intensity, intensity); + gl.uniform1f(U.u_interrupt_burst, this.interruptBurst); + // speak 时半径 + 微脉动 + const speakActive = this.layers.speak || this.target === STATE_TARGETS[STATES.SPEAK]; + gl.uniform1f(U.u_radius, c.radius + (intensity * 0.04) * (speakActive ? 1 : 0)); + gl.uniform1f(U.u_amp, c.amp + intensity * 0.03 * (speakActive ? 1 : 0)); + gl.uniform1f(U.u_freq, c.freq); + gl.uniform1f(U.u_speed, c.speed); + gl.uniform3fv(U.u_color, c.color); + gl.uniform3fv(U.u_glow, c.glow); + + gl.drawArrays(gl.TRIANGLES, 0, 3); + } + + start() { + if (!this.ready) throw new Error('call init() first'); + cancelAnimationFrame(this._raf); + this._raf = requestAnimationFrame(this.render); + } +} + +// 避免 structuredClone 兼容性问题,自己深拷 +function structuredClone(obj) { + return JSON.parse(JSON.stringify(obj)); +} diff --git a/.moss_ws/apps/aether/core/webroot/web/state_mapper.js b/.moss_ws/apps/aether/core/webroot/web/state_mapper.js new file mode 100644 index 00000000..88d5020b --- /dev/null +++ b/.moss_ws/apps/aether/core/webroot/web/state_mapper.js @@ -0,0 +1,187 @@ +// state_mapper.js — 兼容主状态 + 全双工 activity layers + +export const STATES = { + IDLE: 0, + LISTEN: 1, + THINK: 2, + SPEAK: 3, + INTERRUPT: 4, +}; + +export const STATE_NAMES = ['idle', 'listen', 'think', 'speak', 'interrupt']; + +// 各状态目标视觉参数(CPU 端插值目标) +export const STATE_TARGETS = { + [STATES.IDLE]: { radius: 0.50, amp: 0.05, freq: 1.20, speed: 0.45, color: [0.30, 0.55, 1.10], glow: [0.20, 0.45, 1.00] }, + [STATES.LISTEN]: { radius: 0.42, amp: 0.03, freq: 1.50, speed: 0.85, color: [0.20, 0.80, 1.20], glow: [0.10, 0.65, 1.10] }, + [STATES.THINK]: { radius: 0.55, amp: 0.13, freq: 2.60, speed: 1.80, color: [0.65, 0.30, 1.10], glow: [0.50, 0.20, 1.00] }, + [STATES.SPEAK]: { radius: 0.58, amp: 0.10, freq: 2.20, speed: 2.00, color: [1.10, 0.65, 0.25], glow: [1.00, 0.45, 0.10] }, + [STATES.INTERRUPT]: { radius: 0.40, amp: 0.02, freq: 0.50, speed: 0.00, color: [0.85, 0.95, 1.20], glow: [0.70, 0.85, 1.20] }, +}; + +export const LAYER_NAMES = ['mic', 'listen', 'queue', 'think', 'speak', 'interrupt']; + +export function normalizeLayers(layers = {}) { + return { + mic: Boolean(layers.mic), + listen: Boolean(layers.listen), + queue: Boolean(layers.queue), + think: Boolean(layers.think), + speak: Boolean(layers.speak), + interrupt: Boolean(layers.interrupt), + }; +} + +export function primaryStateFromLayers(layers = {}) { + const l = normalizeLayers(layers); + if (l.interrupt) return STATES.INTERRUPT; + if (l.speak) return STATES.SPEAK; + if (l.think) return STATES.THINK; + if (l.listen) return STATES.LISTEN; + return STATES.IDLE; +} + +export function blendTargetsForLayers(layers = {}) { + const l = normalizeLayers(layers); + if (l.interrupt) return { ...cloneTarget(STATE_TARGETS[STATES.INTERRUPT]), activeCount: 1 }; + + const weighted = [ + [STATES.IDLE, (!l.listen && !l.think && !l.speak) ? 1.0 : 0.28], + [STATES.LISTEN, l.listen ? 0.90 : (l.mic ? 0.30 : 0)], + [STATES.THINK, l.think ? 1.00 : (l.queue ? 0.55 : 0)], + [STATES.SPEAK, l.speak ? 1.10 : 0], + ]; + let total = 0; + const out = { radius: 0, amp: 0, freq: 0, speed: 0, color: [0, 0, 0], glow: [0, 0, 0] }; + let activeCount = 0; + for (const [state, weight] of weighted) { + if (weight <= 0) continue; + if (state !== STATES.IDLE) activeCount += 1; + total += weight; + const t = STATE_TARGETS[state]; + out.radius += t.radius * weight; + out.amp += t.amp * weight; + out.freq += t.freq * weight; + out.speed += t.speed * weight; + for (let i = 0; i < 3; i++) { + out.color[i] += t.color[i] * weight; + out.glow[i] += t.glow[i] * weight; + } + } + if (total <= 0) return { ...cloneTarget(STATE_TARGETS[STATES.IDLE]), activeCount: 0 }; + out.radius /= total; + out.amp /= total; + out.freq /= total; + out.speed /= total; + for (let i = 0; i < 3; i++) { + out.color[i] /= total; + out.glow[i] /= total; + } + out.activeCount = activeCount; + return out; +} + +function cloneTarget(target) { + return JSON.parse(JSON.stringify(target)); +} + +// 状态切换的"急刹度":插值速率,越大切换越快。INTERRUPT 最大(急刹),其他柔和 +export const STATE_SWITCH_RATE = { + [STATES.IDLE]: 4.0, + [STATES.LISTEN]: 5.0, + [STATES.THINK]: 4.5, + [STATES.SPEAK]: 4.0, + [STATES.INTERRUPT]: 6.0, // 急刹再放慢:让扩张→收缩的节奏清晰可见 +}; + +export class StateMapper { + /** + * @param {object} opts + * @param {(state:number, event:object)=>void} opts.onState 状态切换回调 + * @param {number} opts.minHoldMs 最小保持时间(去抖),默认 250ms + * @param {number} opts.interruptHoldMs INTERRUPT 最小保持时间(让爆点视觉充分展现) + */ + constructor({ onState, minHoldMs = 250, interruptHoldMs = 1800 } = {}) { + this.state = STATES.IDLE; + this.layers = normalizeLayers(); + this._layerKey = JSON.stringify(this.layers); + this.onState = onState; + this.minHoldMs = minHoldMs; + this.interruptHoldMs = interruptHoldMs; + this.stateEnterTs = performance.now(); + this.log = []; + } + + now() { return performance.now(); } + heldMs() { return this.now() - this.stateEnterTs; } + + /** + * 喂事件。事件类型: + * - SPEECH_STARTED 旧契约:检测到开口 + * - SPEECH_FINAL ASR 出完整句(慢线) + * - TTS_START TTS 开始播放 + * - TTS_END TTS 播放结束 + * - FORCE_STATE 演示用:强制切状态 + */ + event(e) { + const now = this.now(); + const held = now - this.stateEnterTs; + let next = this.state; + + switch (e.type) { + case 'SPEECH_STARTED': + // 旧契约兼容:新路径使用 setLayers(),普通 VAD 不再伪装成 interrupt。 + if (this.state === STATES.SPEAK) next = STATES.INTERRUPT; + else if (this.state === STATES.IDLE) next = STATES.LISTEN; + else if (this.state === STATES.THINK) next = STATES.INTERRUPT; // 思考中被打断也急刹 + break; + case 'SPEECH_FINAL': + if (this.state === STATES.LISTEN) next = STATES.THINK; + break; + case 'TTS_START': + if (this.state === STATES.THINK || this.state === STATES.INTERRUPT) next = STATES.SPEAK; + break; + case 'TTS_END': + if (this.state === STATES.SPEAK) next = STATES.IDLE; + break; + case 'FORCE_STATE': + next = e.state; + break; + } + + if (next === this.state) return; + + // 去抖:INTERRUPT 是爆点,永远抢跑(绕过去抖); + // FORCE_STATE 是手动演示,也绕过; + // 其他状态切换在最小保持时间内忽略。 + const isUrgent = next === STATES.INTERRUPT || e.type === 'FORCE_STATE'; + if (!isUrgent && held < this.minHoldMs) return; + + // INTERRUPT 自身要保持足够久,避免视觉还没展开就退出 + if (this.state === STATES.INTERRUPT && held < this.interruptHoldMs && e.type !== 'SPEECH_STARTED' && e.type !== 'FORCE_STATE') { + return; + } + + const prev = this.state; + this.state = next; + this.layers = next === STATES.IDLE ? normalizeLayers() : normalizeLayers({ [STATE_NAMES[next]]: true }); + this._layerKey = JSON.stringify(this.layers); + this.stateEnterTs = now; + this.log.push({ ts: now, from: prev, to: next, event: e.type }); + if (this.log.length > 200) this.log.shift(); + this.onState?.(next, { from: prev, event: e.type, latencyMs: e.vadLatencyMs, layers: this.layers }); + } + + setLayers(layers, meta = {}) { + const normalized = normalizeLayers(layers); + const key = JSON.stringify(normalized); + const next = primaryStateFromLayers(normalized); + const prev = this.state; + if (next === this.state && key === this._layerKey) return; + this.layers = normalized; + this._layerKey = key; + this.state = next; + this.stateEnterTs = this.now(); + this.onState?.(next, { from: prev, event: meta.event || 'LAYERS', layers: normalized, text: meta.text }); + } +} diff --git a/.moss_ws/apps/aether/core/webroot/web/style.css b/.moss_ws/apps/aether/core/webroot/web/style.css new file mode 100644 index 00000000..529e510c --- /dev/null +++ b/.moss_ws/apps/aether/core/webroot/web/style.css @@ -0,0 +1,553 @@ +:root { + --bg: #04060c; + --panel: rgba(12, 18, 32, 0.72); + --panel-border: rgba(120, 180, 255, 0.18); + --text: #cfe3ff; + --text-dim: #6f86a8; + --accent: #4cc8ff; + --warn: #ff5a6a; + --ok: #4dffb0; +} + +@property --spin { + syntax: ""; + inherits: false; + initial-value: 0deg; +} + +* { box-sizing: border-box; margin: 0; padding: 0; } + +html, body { + width: 100%; + height: 100%; + background: var(--bg); + color: var(--text); + font-family: -apple-system, BlinkMacSystemFont, "SF Mono", "JetBrains Mono", Menlo, Consolas, monospace; + overflow: hidden; +} + +#app { + position: fixed; + inset: 0; + display: flex; + flex-direction: column; +} + +#glcanvas { + position: fixed; + inset: 0; + width: 100%; + height: 100%; + display: block; + z-index: 0; +} + +/* ---------- 顶部状态条 ---------- */ +.topbar { + position: relative; + z-index: 2; + display: flex; + align-items: center; + gap: 18px; + padding: 14px 22px; + background: linear-gradient(180deg, rgba(4,6,12,0.85), rgba(4,6,12,0)); + pointer-events: none; +} + +.brand { + font-size: 14px; + letter-spacing: 0.18em; + color: var(--accent); + text-transform: uppercase; +} + +.brand .sub { + color: var(--text-dim); + margin-left: 10px; + font-size: 11px; + letter-spacing: 0.1em; +} + +.mode-pill { + pointer-events: auto; + font-size: 11px; + padding: 4px 10px; + border-radius: 999px; + border: 1px solid var(--panel-border); + background: rgba(0,0,0,0.4); + color: var(--text-dim); +} +.mode-pill.ws { color: var(--ok); border-color: rgba(77,255,176,0.4); } +.mode-pill.local { color: var(--accent); border-color: rgba(76,200,255,0.4); } + +.state-dots { + margin-left: auto; + display: flex; + gap: 8px; + pointer-events: auto; +} +.state-dot { + width: 12px; height: 12px; + border-radius: 50%; + background: rgba(255,255,255,0.08); + border: 1px solid rgba(255,255,255,0.12); + transition: all 0.18s ease; + position: relative; +} +.state-dot.active { + transform: scale(1.4); + box-shadow: 0 0 14px currentColor; +} +.state-dot.primary { outline: 1px solid rgba(255,255,255,0.55); outline-offset: 3px; } +.state-dot[data-s="idle"].active { background: #4cc8ff; color: #4cc8ff; } +.state-dot[data-s="listen"].active { background: #2bd0ff; color: #2bd0ff; } +.state-dot[data-s="think"].active { background: #a060ff; color: #a060ff; } +.state-dot[data-s="speak"].active { background: #ffb14c; color: #ffb14c; } +.state-dot[data-s="interrupt"].active { background: #ffffff; color: #ffffff; } + +.current-state { + font-size: 13px; + min-width: 110px; + color: var(--text); +} +.current-state .name { + font-size: 18px; + font-weight: 600; + letter-spacing: 0.08em; + text-transform: uppercase; +} + +/* ---------- 底部控制面板 ---------- */ +.bottom { + position: fixed; + left: 0; right: 0; bottom: 0; + z-index: 2; + padding: 18px 22px 22px; + background: linear-gradient(0deg, rgba(4,6,12,0.92), rgba(4,6,12,0)); + display: flex; + flex-direction: column; + gap: 14px; +} + +.panel { + background: var(--panel); + border: 1px solid var(--panel-border); + border-radius: 12px; + padding: 14px 16px; + backdrop-filter: blur(12px); + -webkit-backdrop-filter: blur(12px); +} + +.panel-title { + font-size: 10px; + letter-spacing: 0.22em; + color: var(--text-dim); + text-transform: uppercase; + margin-bottom: 10px; +} + +.row { display: flex; flex-wrap: wrap; align-items: center; gap: 10px; } + +button { + font-family: inherit; + font-size: 12px; + letter-spacing: 0.08em; + color: var(--text); + background: rgba(255,255,255,0.04); + border: 1px solid rgba(255,255,255,0.12); + padding: 8px 14px; + border-radius: 8px; + cursor: pointer; + transition: all 0.15s ease; +} +button:hover { background: rgba(76,200,255,0.12); border-color: rgba(76,200,255,0.4); } +button:active { transform: scale(0.97); } +button:disabled { opacity: 0.42; cursor: not-allowed; transform: none; } +button:disabled:hover { background: rgba(255,255,255,0.04); border-color: rgba(255,255,255,0.12); } +button.active { background: rgba(76,200,255,0.2); border-color: var(--accent); color: var(--accent); } +button.danger { border-color: rgba(255,90,106,0.4); color: #ff8a96; } +button.danger:hover { background: rgba(255,90,106,0.15); } +button.warn { border-color: rgba(255,200,80,0.4); color: #ffc850; } +button.warn:hover { background: rgba(255,200,80,0.15); } +button.primary { + background: linear-gradient(135deg, rgba(76,200,255,0.25), rgba(160,96,255,0.25)); + border-color: rgba(160,96,255,0.5); + color: #fff; +} + +.mic-status { + font-size: 11px; + color: var(--text-dim); + padding: 4px 10px; + border-radius: 999px; + border: 1px solid var(--panel-border); +} +.mic-status.on { color: var(--ok); border-color: rgba(77,255,176,0.4); } +.mic-status.warn { color: var(--warn); border-color: rgba(255,90,106,0.4); } + +.level-bar { + flex: 1; + min-width: 120px; + height: 6px; + background: rgba(255,255,255,0.06); + border-radius: 3px; + overflow: hidden; + position: relative; +} +.level-bar > div { + height: 100%; + background: linear-gradient(90deg, #4cc8ff, #ffb14c, #ff5a6a); + width: 0%; + transition: width 0.05s linear; +} +.threshold-mark { + position: absolute; + top: -2px; bottom: -2px; + width: 1px; + background: rgba(255,255,255,0.5); +} + +.intensity-slider { + flex: 1; + min-width: 120px; + display: flex; align-items: center; gap: 10px; +} +.intensity-slider input { flex: 1; } + +/* ---------- 状态机小图 ---------- */ +.stategraph { + position: fixed; + top: 70px; + right: 22px; + z-index: 2; + width: 220px; + font-size: 11px; + color: var(--text-dim); +} +.stategraph .edge { + display: flex; + align-items: center; + gap: 6px; + padding: 3px 0; +} +.stategraph .edge span:first-child { color: var(--text-dim); min-width: 76px; } +.stategraph .edge.active span:first-child { color: var(--accent); } + +/* ---------- 语音链路诊断 ---------- */ +.voice-diag { + position: fixed; + top: 250px; + right: 22px; + z-index: 2; + width: 320px; + padding: 12px 14px; + background: rgba(5, 9, 18, 0.68); + border: 1px solid rgba(120, 180, 255, 0.16); + border-radius: 8px; + backdrop-filter: blur(12px); + -webkit-backdrop-filter: blur(12px); + font-size: 11px; + color: var(--text-dim); + pointer-events: none; +} +.voice-diag .panel-title { margin-bottom: 8px; } +.diag-section + .diag-section { margin-top: 10px; } +.diag-label { + color: rgba(207, 227, 255, 0.56); + margin-bottom: 5px; + letter-spacing: 0.08em; +} +.asr-diag, +.asr-current { + display: flex; + flex-direction: column; + gap: 5px; +} +.asr-item { + display: grid; + grid-template-columns: 42px minmax(0, 1fr); + gap: 7px; + align-items: start; + padding: 5px 7px; + border-radius: 6px; + background: rgba(255, 255, 255, 0.035); + border: 1px solid rgba(255, 255, 255, 0.065); +} +.asr-item.final { + border-color: rgba(77, 255, 176, 0.22); + background: rgba(77, 255, 176, 0.055); +} +.asr-item.partial { + border-color: rgba(76, 200, 255, 0.18); + background: rgba(76, 200, 255, 0.045); +} +.asr-tag { + color: var(--accent); + font-size: 10px; + text-transform: uppercase; +} +.asr-item.final .asr-tag { color: var(--ok); } +.asr-text { + color: var(--text); + overflow-wrap: anywhere; + line-height: 1.35; +} +.diag-empty, +.vpio-diag { + line-height: 1.35; + overflow-wrap: anywhere; +} +.asr-error { + margin-top: 10px; + padding: 7px 8px; + border-radius: 6px; + background: rgba(255, 90, 106, 0.10); + border: 1px solid rgba(255, 90, 106, 0.28); + color: #ff9aa4; + line-height: 1.35; + overflow-wrap: anywhere; +} +.vpio-diag { + color: rgba(207, 227, 255, 0.76); + padding: 6px 7px; + border-radius: 6px; + background: rgba(76, 200, 255, 0.055); + border: 1px solid rgba(76, 200, 255, 0.12); +} + +/* ---------- 并发活动轨道 ---------- */ +.activity-stack { + position: fixed; + inset: 0; + z-index: 1; + pointer-events: none; + display: grid; + place-items: center; +} +.activity-ring { + position: absolute; + width: min(64vw, 620px); + aspect-ratio: 1; + border-radius: 50%; + opacity: 0; + transform: scale(0.86) rotate(var(--tilt, 0deg)); + transition: opacity 0.22s ease, transform 0.35s cubic-bezier(0.16,1,0.3,1), filter 0.22s ease; + filter: blur(0.2px); +} +.activity-ring::before { + content: ""; + position: absolute; + inset: 0; + border-radius: inherit; + border: 2px solid transparent; + background: + conic-gradient(from var(--spin, 0deg), + transparent 0deg, + var(--ring-color) 36deg, + transparent 78deg, + transparent 178deg, + var(--ring-color) 226deg, + transparent 286deg, + transparent 360deg) border-box; + mask: linear-gradient(#000 0 0) padding-box, linear-gradient(#000 0 0); + -webkit-mask: linear-gradient(#000 0 0) padding-box, linear-gradient(#000 0 0); + mask-composite: exclude; + -webkit-mask-composite: xor; + padding: 2px; + animation: ring-spin var(--ring-speed, 8s) linear infinite; +} +.activity-ring::after { + content: ""; + position: absolute; + inset: 8%; + border-radius: inherit; + border: 1px solid color-mix(in srgb, var(--ring-color), transparent 62%); + box-shadow: 0 0 24px color-mix(in srgb, var(--ring-color), transparent 48%); +} +.activity-ring span { + position: absolute; + top: 8%; + left: 50%; + transform: translateX(-50%); + font-size: 11px; + letter-spacing: 0.22em; + color: var(--ring-color); + text-shadow: 0 0 18px var(--ring-color); +} +.activity-ring.active { + opacity: 0.82; + transform: scale(var(--ring-scale, 1)) rotate(var(--tilt, 0deg)); +} +.activity-ring.primary { + opacity: 1; + filter: drop-shadow(0 0 18px var(--ring-color)); +} +.activity-ring.listen { + --ring-color: #49e8ff; + --ring-scale: 0.82; + --ring-speed: 5.5s; + --tilt: -10deg; +} +.activity-ring.mic { + --ring-color: #4dffb0; + --ring-scale: 0.72; + --ring-speed: 4.2s; + --tilt: 12deg; + opacity: 0; +} +.activity-ring.mic.active { + opacity: 0.42; +} +.activity-ring.mic span { + top: 14%; +} +.activity-ring.think { + --ring-color: #bb76ff; + --ring-scale: 0.94; + --ring-speed: 8.5s; + --tilt: 18deg; +} +.activity-ring.queue { + --ring-color: #ff6f91; + --ring-scale: 0.88; + --ring-speed: 6.2s; + --tilt: -24deg; +} +.activity-ring.speak { + --ring-color: #ffbd4a; + --ring-scale: 1.07; + --ring-speed: 3.4s; + --tilt: 6deg; +} +.activity-ring.interrupt { + --ring-color: #ffffff; + --ring-scale: 0.76; + --ring-speed: 1.2s; + --tilt: 0deg; +} +.activity-ring.speak.active { + animation: speak-ring-pulse 0.55s ease-in-out infinite alternate; +} +.activity-ring.think.active::after { + animation: think-ring-warp 1.4s ease-in-out infinite alternate; +} +.activity-ring.queue.active::before { + animation-duration: 2.6s; +} +@keyframes ring-spin { + to { --spin: 360deg; } +} +@keyframes speak-ring-pulse { + from { transform: scale(calc(var(--ring-scale, 1) - 0.015)) rotate(var(--tilt, 0deg)); } + to { transform: scale(calc(var(--ring-scale, 1) + 0.025)) rotate(var(--tilt, 0deg)); } +} +@keyframes think-ring-warp { + from { inset: 8%; opacity: 0.45; } + to { inset: 5%; opacity: 0.85; } +} + +/* ---------- 中部打断提示 ---------- */ +.interrupt-flash { + position: fixed; + inset: 0; + z-index: 5; + pointer-events: none; + background: radial-gradient(circle, rgba(255,255,255,0.0), rgba(255,255,255,0.0)); + transition: background 0.05s linear; +} +.interrupt-flash.on { + background: radial-gradient(circle, rgba(255,255,255,0.5), rgba(255,255,255,0) 60%); +} + +.interrupt-label { + position: fixed; + top: 50%; + left: 50%; + transform: translate(-50%, -50%) scale(0.8); + z-index: 6; + pointer-events: none; + font-size: 48px; + font-weight: 800; + letter-spacing: 0.2em; + color: #fff; + text-shadow: 0 0 30px rgba(255,255,255,0.8); + opacity: 0; + transition: opacity 0.08s ease; +} +.interrupt-label.on { + opacity: 1; + transform: translate(-50%, -50%) scale(1); + transition: opacity 0.08s ease, transform 0.18s cubic-bezier(0.16,1,0.3,1); +} + +/* 通用状态大字(球体中央,持续显示) */ +.state-label { + position: fixed; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + z-index: 6; + pointer-events: none; + font-size: 42px; + font-weight: 800; + letter-spacing: 0.22em; + text-align: center; + white-space: normal; + max-width: min(86vw, 760px); + line-height: 1.08; + opacity: 0.85; + transition: opacity 0.25s ease, color 0.3s ease, text-shadow 0.3s ease; +} +.state-label[data-state="idle"] { color: #6fa8ff; text-shadow: 0 0 22px rgba(80,140,255,0.6); } +.state-label[data-state="listen"] { color: #5cf0ff; text-shadow: 0 0 22px rgba(40,200,255,0.7); } +.state-label[data-state="think"] { color: #c08aff; text-shadow: 0 0 22px rgba(160,80,255,0.7); } +.state-label[data-state="speak"] { color: #ffc850; text-shadow: 0 0 24px rgba(255,170,40,0.8); } +.state-label[data-state="interrupt"] { + color: #fff; + text-shadow: 0 0 34px rgba(255,255,255,0.9); + font-size: 52px; + opacity: 1; + animation: intr-pulse 0.5s cubic-bezier(0.16,1,0.3,1); +} +@keyframes intr-pulse { + 0% { transform: translate(-50%, -50%) scale(0.7); opacity: 0; } + 50% { transform: translate(-50%, -50%) scale(1.12); opacity: 1; } + 100% { transform: translate(-50%, -50%) scale(1); opacity: 1; } +} + +/* ---------- 日志 ---------- */ +.log { + position: fixed; + left: 22px; + top: 80px; + z-index: 2; + width: 340px; + max-height: calc(100vh - 480px); + overflow-y: auto; + overflow-x: hidden; + font-size: 10px; + color: var(--text-dim); + line-height: 1.6; + scrollbar-width: thin; + scrollbar-color: rgba(100,180,255,0.3) transparent; + mask-image: linear-gradient(180deg, transparent 0, #000 18px, #000 calc(100% - 18px), transparent 100%); + -webkit-mask-image: linear-gradient(180deg, transparent 0, #000 18px, #000 calc(100% - 18px), transparent 100%); +} +.log::-webkit-scrollbar { width: 4px; } +.log::-webkit-scrollbar-thumb { background: rgba(100,180,255,0.3); border-radius: 2px; } +.log::-webkit-scrollbar-track { background: transparent; } +.log .line { opacity: 0.9; } +.log .line:last-child { color: var(--text); } + +.hint { + position: fixed; + top: 40%; + left: 22px; + transform: translateY(-50%); + z-index: 2; + font-size: 11px; + color: var(--text-dim); + max-width: 280px; + line-height: 1.7; +} +.hint b { color: var(--text); } diff --git a/.moss_ws/apps/aether/core/webroot/web/vad.js b/.moss_ws/apps/aether/core/webroot/web/vad.js new file mode 100644 index 00000000..b577ce70 --- /dev/null +++ b/.moss_ws/apps/aether/core/webroot/web/vad.js @@ -0,0 +1,162 @@ +// vad.js — 浏览器麦克风 VAD 快线(对应技术文档里的 vad.py) +// 用 RMS 能量阈值 + 最小持续时间门限,检测到开口第一帧就触发 SPEECH_STARTED +// 目标延迟 < 50ms(实际取决于 AudioContext 块大小,这里 fftSize=512 @16kHz ≈ 32ms) +// +// 回声处理:依赖 WebRTC echoCancellation 作为前置过滤,让 VAD"听不到"TTS 外放回声。 +// 即便如此 AEC 效果不稳定,因此在 SPEAK 状态下启用 speakMode: +// - 阈值放大 speakThresholdScale 倍(默认 2.0) +// - 最小持续时间放大 speakMinDurScale 倍(默认 1.5) +// 作为安全边际,避免残留回声触发自打断。 + +export class VAD { + /** + * @param {object} opts + * @param {()=>void} opts.onSpeechStarted 开口瞬间回调(爆点触发源) + * @param {()=>void} [opts.onSpeechEnded] 静默后回调 + * @param {number} [opts.threshold] RMS 阈值 0~1 + * @param {number} [opts.minDurMs] 持续多长时间才算"开口"(防噪声毛刺) + * @param {number} [opts.endSilenceMs] 多少静默算"说完了" + * @param {(level:number)=>void} [opts.onLevel] 实时音量回调(用于 speak 脉动近似) + * @param {number} [opts.speakThresholdScale] SPEAK 状态下阈值放大倍数(默认 2.0) + * @param {number} [opts.speakMinDurScale] SPEAK 状态下 minDurMs 放大倍数(默认 1.5) + * @param {(settings:object)=>void} [opts.onEchoCancelReport] 上报 getUserMedia 实际生效的 AEC 设置 + */ + constructor({ + onSpeechStarted, + onSpeechEnded, + threshold = 0.012, + minDurMs = 40, + endSilenceMs = 520, + onLevel, + speakThresholdScale = 2.0, + speakMinDurScale = 1.5, + onEchoCancelReport, + } = {}) { + this.onSpeechStarted = onSpeechStarted; + this.onSpeechEnded = onSpeechEnded; + this.onLevel = onLevel; + this.threshold = threshold; + this.minDurMs = minDurMs; + this.endSilenceMs = endSilenceMs; + this.speakThresholdScale = speakThresholdScale; + this.speakMinDurScale = speakMinDurScale; + this.onEchoCancelReport = onEchoCancelReport; + + this.running = false; + this.ctx = null; + this.stream = null; + this.analyser = null; + this.buf = null; + this.raf = 0; + + this.speaking = false; + this.activeSince = 0; // 当前活跃起始时间 + this.silentSince = 0; // 静默起始时间 + this.lastRms = 0; + + // SPEAK 状态开关:开启后提高阈值与最小持续时间,作为 AEC 残留回声的安全边际 + this.speakMode = false; + } + + /** SPEAK 状态下开启,提高阈值;离开 SPEAK 时关闭,恢复灵敏度 */ + setSpeakMode(on) { + this.speakMode = !!on; + } + + async start() { + if (this.running) return; + // 强化 echoCancellation:用 ideal 表达强烈偏好,echoCancellationType:'system' + // 优先使用系统级 AEC(通常比浏览器内置 AEC 效果更好)。 + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: { ideal: true }, + echoCancellationType: 'system', + noiseSuppression: { ideal: true }, + autoGainControl: { ideal: true }, + channelCount: { ideal: 1 }, + }, + }); + this.stream = stream; + // 上报实际生效的 AEC 设置(浏览器可能降级,需可见) + try { + const track = stream.getAudioTracks()[0]; + const settings = track.getSettings(); + this.onEchoCancelReport?.(settings); + } catch (e) {} + // 16kHz 采样率贴近 Silero VAD 场景 + const Ctx = window.AudioContext || window.webkitAudioContext; + this.ctx = new Ctx({ sampleRate: 16000 }); + const src = this.ctx.createMediaStreamSource(stream); + this.analyser = this.ctx.createAnalyser(); + this.analyser.fftSize = 512; // 32ms @16kHz + this.analyser.smoothingTimeConstant = 0.3; + src.connect(this.analyser); + this.buf = new Uint8Array(this.analyser.fftSize); + + this.running = true; + this.loop(); + } + + stop() { + this.running = false; + if (this.raf) cancelAnimationFrame(this.raf); + this.raf = 0; + if (this.stream) this.stream.getTracks().forEach(t => t.stop()); + if (this.ctx) this.ctx.close(); + this.stream = null; + this.ctx = null; + this.analyser = null; + this.speaking = false; + } + + loop = () => { + if (!this.running) return; + this.raf = requestAnimationFrame(this.loop); + if (!this.analyser) return; + + this.analyser.getByteTimeDomainData(this.buf); + let sum = 0; + for (let i = 0; i < this.buf.length; i++) { + const v = (this.buf[i] - 128) / 128; + sum += v * v; + } + const rms = Math.sqrt(sum / this.buf.length); + // 平滑一下,避免抖动 + this.lastRms = this.lastRms * 0.6 + rms * 0.4; + this.onLevel?.(this.lastRms); + + const now = performance.now(); + // SPEAK 状态下使用放大阈值与持续时间,作为 AEC 残留回声的安全边际 + const effectiveThreshold = this.speakMode + ? this.threshold * this.speakThresholdScale + : this.threshold; + const effectiveMinDurMs = this.speakMode + ? this.minDurMs * this.speakMinDurScale + : this.minDurMs; + const above = this.lastRms > effectiveThreshold; + + if (above) { + if (this.activeSince === 0) this.activeSince = now; + // 持续 effectiveMinDurMs 才算"开口" → 触发 SPEECH_STARTED + if (!this.speaking && (now - this.activeSince) >= effectiveMinDurMs) { + this.speaking = true; + const vadLatencyMs = now - this.activeSince; + this.onSpeechStarted?.({ vadLatencyMs, speakMode: this.speakMode }); + this.silentSince = 0; + } + this.silentSince = 0; + } else { + if (this.speaking) { + if (this.silentSince === 0) this.silentSince = now; + if ((now - this.silentSince) >= this.endSilenceMs) { + this.speaking = false; + this.activeSince = 0; + this.silentSince = 0; + this.onSpeechEnded?.(); + } + } else { + this.activeSince = 0; + } + } + } +} diff --git a/.moss_ws/apps/aether/core/webroot/web/ws.js b/.moss_ws/apps/aether/core/webroot/web/ws.js new file mode 100644 index 00000000..3b72667c --- /dev/null +++ b/.moss_ws/apps/aether/core/webroot/web/ws.js @@ -0,0 +1,119 @@ +// ws.js — 状态接收层 +// 连 aether/core 后端 ws://localhost:8765/ws(MOSS ghost 推状态 JSON) +// 后端是 aether/core app:订阅 SpeechTopic/AudioRuntimeTopic,推 {state, layers, intensity, ts} +// +// 状态契约(前后端唯一接口): +// { "state": "speak", "layers": {"listen": true, "think": false, "speak": true, "interrupt": false}, "intensity": 0.7, "ts": 1782402722.10 } +// 前端→后端: +// { "type": "listen", "running": true } // legacy only; main listen is backend ASR-driven +// { "type": "interrupt" } // 明确急停请求 +// { "type": "asr_control", "mode": "manual", "enabled": true } + +export class StateBridge { + /** + * @param {object} opts + * @param {(state:string)=>void} opts.onState + * @param {(intensity:number)=>void} opts.onIntensity + * @param {(layers:object, msg:object)=>void} opts.onLayers + * @param {()=>void} opts.onConnect + * @param {()=>void} opts.onDisconnect + */ + constructor({ onState, onIntensity, onLayers, onConnect, onDisconnect } = {}) { + this.onState = onState; + this.onIntensity = onIntensity; + this.onLayers = onLayers; + this.onConnect = onConnect; + this.onDisconnect = onDisconnect; + this.connected = false; + this.mode = 'local'; // 'ws' | 'local' + this.ws = null; + this._stopped = false; + this._url = 'ws://localhost:8765/ws'; + this._lastAsrControl = { mode: 'continuous', enabled: true }; + this._hasAsrControl = false; + } + + /** 尝试连真后端;失败返回 false。带自动重连。 */ + async connect(url = 'ws://localhost:8765/ws') { + this._stopped = false; + this._url = url; + try { + const ws = new WebSocket(url); + this.ws = ws; + await new Promise((res, rej) => { + ws.onopen = res; + ws.onerror = rej; + setTimeout(() => rej(new Error('timeout')), 2000); + }); + ws.onmessage = (ev) => { + try { this._handle(JSON.parse(ev.data)); } catch (e) {} + }; + ws.onclose = () => { + this.connected = false; + this.mode = 'local'; + this.onDisconnect?.(); + if (!this._stopped) { + // 3s 后自动重连 + setTimeout(() => { if (!this._stopped) this.connect(url); }, 3000); + } + }; + this.connected = true; + this.mode = 'ws'; + this.onConnect?.(); + if (this._hasAsrControl) this._sendAsrControlNow(); + return true; + } catch (e) { + this.connected = false; + this.mode = 'local'; + if (!this._stopped) { + setTimeout(() => { if (!this._stopped) this.connect(url); }, 3000); + } + return false; + } + } + + /** 明确急停:按钮或已确认的后端 barge-in 路径。普通 VAD 不调用它。 */ + sendInterrupt() { + if (this.ws && this.connected) { + try { this.ws.send(JSON.stringify({ type: 'interrupt' })); } catch (e) {} + } + } + + sendListen(running, extra = {}) { + if (this.ws && this.connected) { + try { this.ws.send(JSON.stringify({ type: 'listen', running: Boolean(running), ...extra })); } catch (e) {} + } + } + + sendAsrControl(mode = 'continuous', enabled = true) { + this._lastAsrControl = { mode, enabled: Boolean(enabled) }; + this._hasAsrControl = true; + this._sendAsrControlNow(); + } + + _sendAsrControlNow() { + if (this.ws && this.connected) { + try { this.ws.send(JSON.stringify({ type: 'asr_control', ...this._lastAsrControl })); } catch (e) {} + } + } + + /** 前端请求重置上下文 → 后端清空 speech_win + ghost 历史 */ + sendReset() { + if (this.ws && this.connected) { + try { this.ws.send(JSON.stringify({ type: 'reset' })); } catch (e) {} + } + } + + _handle(msg) { + if (!msg) return; + if (msg.layers && typeof msg.layers === 'object') this.onLayers?.(msg.layers, msg); + else if (typeof msg.state === 'string') this.onState?.(msg.state); + if (typeof msg.intensity === 'number') this.onIntensity?.(msg.intensity); + } + + /** 本地模式:直接由前端喂状态消息(演示用) */ + feedLocal(msg) { + if (this.mode === 'ws') return; // 已连真后端,忽略本地 + this._handle(msg); + } +} diff --git a/.moss_ws/apps/sensors/listener/.env.example b/.moss_ws/apps/aether/listener/.env.example similarity index 100% rename from .moss_ws/apps/sensors/listener/.env.example rename to .moss_ws/apps/aether/listener/.env.example diff --git a/.moss_ws/apps/sensors/listener/.gitignore b/.moss_ws/apps/aether/listener/.gitignore similarity index 100% rename from .moss_ws/apps/sensors/listener/.gitignore rename to .moss_ws/apps/aether/listener/.gitignore diff --git a/.moss_ws/apps/aether/listener/APP.md b/.moss_ws/apps/aether/listener/APP.md new file mode 100644 index 00000000..e03ce1ad --- /dev/null +++ b/.moss_ws/apps/aether/listener/APP.md @@ -0,0 +1,15 @@ +--- +arguments: '' +description: 'Aether listener — audio → Volcengine ASR → SpeechTopic + AudioSignal' +executable: uv +respawn: false +script: main.py +workers: 1 +--- + +Aether listener — audio → Volcengine ASR → SpeechTopic + AudioSignal. +Canonical app address: `aether/listener`. + +The Aether baseline uses the Volcengine streaming ASR path only. Frontend ASR +controls can switch between continuous listening and manual capture, but both +modes use the same backend recognizer. diff --git a/.moss_ws/apps/aether/listener/main.py b/.moss_ws/apps/aether/listener/main.py new file mode 100644 index 00000000..78ee1024 --- /dev/null +++ b/.moss_ws/apps/aether/listener/main.py @@ -0,0 +1,614 @@ +"""Listener App — ASR consumer. + +Consumes PCM stream from audio_capture, feeds to Volcengine ASR, +publishes SpeechTopic on final recognition, emits AudioSignal to mindflow. + +Usage: + moss apps test aether/listener + moss apps start aether/listener +""" +import asyncio +import json +import logging +import math +import os +import re +import time +from collections.abc import AsyncIterable +from dataclasses import dataclass +from pathlib import Path + +import dotenv +import numpy as np + +dotenv.load_dotenv(Path(__file__).resolve().parent / ".env") +from scipy import signal + +from ghoshell_moss.contracts.asr import ASRResult +from ghoshell_moss.contracts.audio import ( + AudioCaptureConfig, + AudioChunk, +) +from ghoshell_moss.core.mindflow.audio_signal import AudioAction, AudioSignal +from ghoshell_moss.host.speech.capture.audio_transport import AudioTransport +from ghoshell_moss.topics.audio import AudioRuntimeTopic, SpeechTopic +from ghoshell_moss.core.blueprint.matrix import Matrix +from ghoshell_moss.host.speech.capture.matrix_audio_transport import MatrixAudioTransport +from ghoshell_moss.host.speech.capture.miniaudio_capture import MiniAudioCaptureSource +from ghoshell_moss.host.speech.volcengine_asr import VolcengineASR, VolcengineASRConfig +from ghoshell_moss.message import Message +from ghoshell_moss.core.blueprint.mindflow import Signal, Priority, unique_id + +# ASR 期望的采样率 (16kHz 是语音识别的行业标准) +_ASR_SAMPLE_RATE = 16000 +_VOLCENGINE_ASR_ERROR_PREFIX = "__VOLCENGINE_ASR_ERROR__:" +_WAKE_WORDS = ( + "立刻停下", + "立即停下", + "马上停下", + "快停下", + "停下", + "别说了", + "不要说了", + "先停", + "闭嘴", +) + + +def _env_float(name: str, default: float) -> float: + raw = os.environ.get(name) + if not raw: + return default + try: + return float(raw) + except ValueError: + return default + + +def _env_int(name: str, default: int) -> int: + raw = os.environ.get(name) + if not raw: + return default + try: + return int(raw) + except ValueError: + return default + + +def _wake_word_hit(text: str) -> bool: + normalized = re.sub(r"[\s,,。.!!??、;;::「」『』“”\"'`~~-]", "", text) + return any(word in normalized for word in _WAKE_WORDS) + + +def _normalized_text_len(text: str) -> int: + return len(re.sub(r"[\s,,。.!!??、;;::「」『』“”\"'`~~-]", "", text)) + + +def _asr_diag_payload(source: str = "volcengine_asr", **kwargs) -> str: + return json.dumps( + { + "source": source, + **kwargs, + }, + ensure_ascii=False, + separators=(",", ":"), + ) + + +@dataclass +class _ASRControlState: + mode: str + enabled: bool + last_started_at: float = 0.0 + last_heartbeat: float = 0.0 + + +def _initial_asr_control() -> _ASRControlState: + mode = os.environ.get("LISTENER_ASR_MODE", "continuous").strip().lower() + if mode not in {"continuous", "manual"}: + mode = "continuous" + return _ASRControlState(mode=mode, enabled=(mode == "continuous")) + + +def _refresh_asr_control(runtime_window, state: _ASRControlState) -> _ASRControlState: + """Refresh frontend ASR control mode and keep it sticky. + + ``continuous`` keeps the current behavior: listener opens ASR sessions + continuously. ``manual`` only opens ASR while enabled=True. + + The control topic is a command, not a transient runtime diagnostic. VPIO and + ASR diagnostics can evict it from a small topic window within seconds, so + absence of a recent control topic must not reset the listener to defaults. + """ + for topic in reversed(list(runtime_window.values())): + if getattr(topic, "device_name", "") != "asr_control": + continue + started_at = float(getattr(topic, "started_at", 0.0) or 0.0) + heartbeat = float(getattr(topic, "last_heartbeat", 0.0) or 0.0) + if (started_at, heartbeat) <= (state.last_started_at, state.last_heartbeat): + break + try: + payload = json.loads(getattr(topic, "device_explain", "") or "{}") + except Exception: + break + next_mode = str(payload.get("mode", state.mode)).strip().lower() + if next_mode in {"continuous", "manual"}: + state.mode = next_mode + state.enabled = bool(payload.get("enabled", state.mode == "continuous")) + state.last_started_at = started_at + state.last_heartbeat = heartbeat + break + if state.mode == "continuous": + state.enabled = True + return state + + +def _resample_audio(samples: np.ndarray, orig_sr: int, target_sr: int) -> np.ndarray: + """重采样音频到目标采样率。使用 scipy.signal.resample_poly 保证质量。""" + if orig_sr == target_sr: + return samples + # 44100 -> 16000: up=160, down=441 + g = math.gcd(orig_sr, target_sr) + up = target_sr // g + down = orig_sr // g + return signal.resample_poly(samples.astype(np.float32), up, down).astype(np.int16) + + +async def _audio_generator( + consumer, + orig_sr: int, + target_sr: int, + runtime_window, + control_state: _ASRControlState, + abort_event: asyncio.Event, + logger: logging.Logger, + initial_chunk: AudioChunk | None = None, + frame_timeout: float = 2.0, +) -> AsyncIterable[np.ndarray]: + """Yield resampled np.ndarray samples from AudioSequentialConsumer. + + Uses an internal asyncio.Queue buffer so that ``aclose()`` (called when + ``asr.recognize()`` finishes) does NOT reach ``consumer.__anext__()`` + and silently drop a chunk. + + NOTE: TTS playback no longer aborts the generator. Instead, ASR continues + recognizing during TTS. In Aether mode VPIO provides system AEC, so + non-wake-word user speech during TTS must still become a normal turn. + """ + # Unbounded queue: pump must never block on put(), otherwise cancellation + # can land inside put() and the None sentinel never reaches the reader. + buffer: asyncio.Queue[AudioChunk | None] = asyncio.Queue() + + async def _pump() -> None: + """Read from consumer into buffer. Stops on cancellation only.""" + try: + if initial_chunk is not None: + buffer.put_nowait(initial_chunk) + async for chunk in consumer: + buffer.put_nowait(chunk) + except asyncio.CancelledError: + pass + finally: + # Sentinel so the generator side exits cleanly. + # put_nowait is used so cancellation cannot intercept us. + buffer.put_nowait(None) + + pump_task = asyncio.create_task(_pump()) + try: + while True: + control = _refresh_asr_control(runtime_window, control_state) + if control.mode == "manual" and not control.enabled: + logger.info("ASR manual gate closed; ending current audio stream") + abort_event.set() + break + try: + chunk = await asyncio.wait_for(buffer.get(), timeout=frame_timeout) + except asyncio.TimeoutError: + logger.warning( + "ASR audio input stalled for %.1fs; ending current audio stream", + frame_timeout, + ) + abort_event.set() + break + if chunk is None: + break + yield _resample_audio(chunk.samples, orig_sr, target_sr) + finally: + pump_task.cancel() + try: + await pump_task + except asyncio.CancelledError: + pass + + +async def _iter_with_silence_timeout( + agen, + logger: logging.Logger, + patience: float = 5.0, + min_timeout_final_chars: int = 2, + first_result_timeout: float = 90.0, +) -> AsyncIterable: + """Wrap an async generator with a silence timeout. + + After the first non-empty result, if no subsequent non-empty result + arrives within *patience* seconds, the iteration stops. Empty-text + results (server keep-alive / VAD status) do NOT reset the timer. + + If the server never sends ``is_final=True`` before the timeout fires, + this wrapper synthesizes a final result from the last partial text. + Without this, the utterance is silently lost — no SpeechTopic published, + no SPEECH_FINAL emitted — and the next recognition loop starts fresh. + """ + timeout = first_result_timeout + last_result: ASRResult | None = None + try: + while True: + try: + result = await asyncio.wait_for(agen.__anext__(), timeout=timeout) + if result.text: + last_result = result + timeout = patience + yield result + except asyncio.TimeoutError: + if last_result is None: + logger.warning( + "ASR first-result timeout after %.1fs, restarting recognition", + first_result_timeout, + ) + elif not last_result.is_final: + logger.info("ASR silence timeout after %.1fs, finalizing", patience) + if _normalized_text_len(last_result.text) >= min_timeout_final_chars: + logger.info( + "Server never sent is_final=True — synthesizing from last partial: %s", + last_result.text, + ) + yield ASRResult(text=last_result.text, is_final=True) + else: + logger.info( + "ASR timeout partial too short, dropping fragment: %s", + last_result.text, + ) + break + except StopAsyncIteration: + break + finally: + await agen.aclose() + + +async def _drain_consumer(consumer, timeout: float = 0.1, max_chunks: int = 5) -> int: + """Discard queued audio chunks to clear TTS residue. + + Limits both timeout-per-read and total chunks to avoid draining user speech. + Returns the number of chunks drained. + """ + drained = 0 + while drained < max_chunks: + try: + await asyncio.wait_for(consumer.__anext__(), timeout=timeout) + drained += 1 + except asyncio.TimeoutError: + break + except StopAsyncIteration: + break + return drained + + +def _is_tts_playing(runtime_window, logger: logging.Logger | None = None) -> bool: + """检查 TTS 扬声器是否正在播放中。 + + AudioRuntimeTopic 是状态快照。从最新往最旧查,找到 speaker + 的最新状态即可;旧的状态可能已被 running=False 覆盖。 + + 环境变量 ``LISTENER_DISABLE_TTS_GATE=1`` 可关闭此门控。 + Aether 的 VPIO AEC 场景默认允许 TTS 播放时继续接收用户语音;如果 + 需要旧的保守回声过滤,可设置 ``LISTENER_GATE_DURING_TTS=1``。 + """ + if os.environ.get("LISTENER_DISABLE_TTS_GATE") == "1": + return False + for topic in reversed(runtime_window.values()): + if topic.device_name == "speaker": + if logger and topic.running: + logger.info("TTS gate: speaker running=%s (window size=%d)", topic.running, len(runtime_window)) + return topic.running + return False + + +async def main(matrix: Matrix) -> None: + logger = matrix.logger or logging.getLogger("moss.listener") + logger.info("Listener app starting") + + # -- transport & source (consumer only, do not start capture) -- + transport: AudioTransport = MatrixAudioTransport(matrix=matrix) + capture_config = AudioCaptureConfig() + # Aether's vpio_capture publishes 16kHz mono PCM. The legacy MiniAudio + # capture path used AudioCaptureConfig.sample_rate (44.1k by default), but + # applying that default to VPIO double-resamples 16k audio and corrupts ASR + # timing. Keep this env-tunable for non-VPIO listener modes. + input_sample_rate = _env_int("LISTENER_INPUT_SAMPLE_RATE", _ASR_SAMPLE_RATE) + source = MiniAudioCaptureSource(transport=transport, config=capture_config) + consumer = source.new_sequential_consumer(max_queue_frames=128) + await consumer.start() + logger.info("Audio sequential consumer started (input_sample_rate=%d)", input_sample_rate) + + # -- Subscribe to AudioRuntimeTopic for TTS gating -- + runtime_window = transport.topic_window(AudioRuntimeTopic, max_size=256) + logger.info("Subscribed to AudioRuntimeTopic window for TTS gating and ASR control") + + # -- ASR (16kHz 是语音识别的标准采样率; 如果 capture 不是 16kHz 则重采样) -- + # end_window_size: 服务端静音判停阈值。火山官方建议 800ms 或 1000ms; + # 过小会切碎句子,过大则明显拖慢 listen -> think。 + asr_end_window_ms = _env_int("LISTENER_ASR_END_WINDOW_MS", 1000) + silence_patience = _env_float("LISTENER_SILENCE_PATIENCE", 3.2) + logger.info( + "ASR segmentation config: end_window_size=%dms, silence_patience=%.1fs", + asr_end_window_ms, + silence_patience, + ) + asr_source = "volcengine_asr" + asr_config = VolcengineASRConfig( + sample_rate=_ASR_SAMPLE_RATE, + end_window_size=asr_end_window_ms, + force_to_speech_time=_env_int("VOLCENGINE_BM_ASR_FORCE_TO_SPEECH_TIME_MS", 1000), + ) + asr = VolcengineASR(config=asr_config, logger=logger) + logger.info("ASR backend selected: %s", asr_source) + + # -- main recognition loop -- + try: + consecutive_asr_errors = 0 + asr_control = _initial_asr_control() + while True: + asr_control = _refresh_asr_control(runtime_window, asr_control) + if asr_control.mode == "manual" and not asr_control.enabled: + if consecutive_asr_errors: + consecutive_asr_errors = 0 + transport.pub_topic(AudioRuntimeTopic( + running=False, + device_name="asr", + device_explain=_asr_diag_payload( + source=asr_source, + state="manual_idle", + mode=asr_control.mode, + ), + started_at=time.monotonic(), + last_heartbeat=time.monotonic(), + )) + await asyncio.sleep(0.08) + continue + + logger.info("Waiting for speech...") + preflight_timeout = _env_float("LISTENER_PRE_ASR_AUDIO_TIMEOUT", 2.0) + try: + first_chunk = await asyncio.wait_for(consumer.__anext__(), timeout=preflight_timeout) + except asyncio.TimeoutError: + logger.warning( + "ASR preflight: no audio frame for %.1fs; not opening ASR connection", + preflight_timeout, + ) + transport.pub_topic(AudioRuntimeTopic( + running=False, + device_name="asr", + device_explain=_asr_diag_payload( + source=asr_source, + state="audio_stalled", + timeout=preflight_timeout, + ), + started_at=time.monotonic(), + last_heartbeat=time.monotonic(), + )) + await asyncio.sleep(0.2) + continue + except StopAsyncIteration: + logger.warning("ASR preflight: audio consumer stopped") + await asyncio.sleep(0.2) + continue + + # NOTE: 不再在 TTS 播放时 hold ASR。 + # VPIO AEC 已经在系统层抑制扬声器回声;如果仍在这里把 + # speaker running 时的非唤醒词结果丢弃,用户在 speak 期间说的话 + # 就永远不会发布 SpeechTopic,前端会表现成 listen 后直接 idle。 + + # Fresh abort flag and utterance id for this utterance. + abort_event = asyncio.Event() + utterance_id = unique_id() + started_emitted = False + asr_running_published = False + + utterance_published = False + + # Each recognize call handles one utterance. + # The ASR backend (end_window_size) splits on silence. + audio_gen = _audio_generator( + consumer, + input_sample_rate, + _ASR_SAMPLE_RATE, + runtime_window, + asr_control, + abort_event, + logger, + initial_chunk=first_chunk, + frame_timeout=_env_float("LISTENER_AUDIO_FRAME_TIMEOUT", 2.0), + ) + async for result in _iter_with_silence_timeout( + asr.recognize(audio_gen), + logger, + patience=silence_patience, + first_result_timeout=60.0, + ): + if result.text.startswith(_VOLCENGINE_ASR_ERROR_PREFIX): + raw = result.text.removeprefix(_VOLCENGINE_ASR_ERROR_PREFIX) + code, _, message = raw.partition("|") + consecutive_asr_errors += 1 + backoff = min(20.0, 2.0 * consecutive_asr_errors) + logger.warning( + "ASR server error %s; message=%s; backing off %.1fs before reconnect (consecutive=%d)", + code, + message[:300], + backoff, + consecutive_asr_errors, + ) + transport.pub_topic(AudioRuntimeTopic( + running=False, + device_name="asr", + device_explain=_asr_diag_payload( + source=asr_source, + error="server_error", + code=code, + message=message, + backoff=backoff, + consecutive=consecutive_asr_errors, + ), + started_at=time.monotonic(), + last_heartbeat=time.monotonic(), + )) + await asyncio.sleep(backoff) + break + + if result.text: + logger.info("ASR partial: %s (final=%s)", result.text, result.is_final) + consecutive_asr_errors = 0 + asr_explain = _asr_diag_payload( + source=asr_source, + text=result.text, + final=bool(result.is_final), + ) + if not asr_running_published: + transport.pub_topic(AudioRuntimeTopic( + running=True, + device_name="asr", + device_explain=asr_explain, + started_at=time.monotonic(), + last_heartbeat=time.monotonic(), + )) + asr_running_published = True + else: + transport.pub_topic(AudioRuntimeTopic( + running=True, + device_name="asr", + device_explain=asr_explain, + started_at=time.monotonic(), + last_heartbeat=time.monotonic(), + )) + + # 全双工核心:检测明确停止意图 → 立刻打断。 + # 不再强依赖 speaker running topic;TTS 状态上报可能滞后于 ASR partial。 + tts_active = _is_tts_playing(runtime_window, logger) + if result.text and _wake_word_hit(result.text): + logger.info("★ Wake word detected: %s — BARGE-IN!", result.text) + # 1) 视觉信号:前端切 interrupt 状态 + interrupt_topic = AudioRuntimeTopic( + running=True, + device_name="interrupt", + device_explain="wake_word_barge_in", + started_at=time.monotonic(), + last_heartbeat=time.monotonic(), + ) + transport.pub_topic(interrupt_topic) + # 2) ghost 中断信号:发 interrupt signal 到 ghost 主进程 (通过 Zenoh 跨进程) + # → mindflow.InterruptNucleus → FATAL impulse → shell.clear() → 停 TTS + 停 LLM + # listener 是独立子进程,不能直接访问 ghost 的 Mindflow, + # 必须通过 session.add_signal 走 Zenoh 发布。 + try: + from ghoshell_moss.core.mindflow.interrupt_nucleus import new_interrupt_signal + sig = new_interrupt_signal( + "立刻停下", + description="用户喊'立刻停下',全双工 barge-in", + ) + matrix.session.add_signal(sig) + logger.info("★ Interrupt signal sent via zenoh (shell.clear will fire)") + except Exception as e: + logger.warning("Failed to send interrupt signal: %s", e) + # 中断当前 ASR 识别 + abort_event.set() + break + + # 旧的保守模式:TTS 播放时只保留 wake word,其他结果丢弃。 + # Aether 默认关闭这条门控,保证真正全双工。 + if tts_active and os.environ.get("LISTENER_GATE_DURING_TTS") == "1": + continue + + # Emit SPEECH_STARTED on first non-empty intermediate result for + # attention preemption (incomplete impulse with interrupt=True). + if not result.is_final and result.text and not started_emitted: + started_meta = AudioSignal(action=AudioAction.SPEECH_STARTED) + sig = Signal( + id=utterance_id, + name=started_meta.signal_name(), + priority=Priority.WARNING, + messages=[Message.new().with_content(result.text)], + description=f"Speech: {result.text}", + metadata=started_meta.model_dump(exclude_defaults=True, exclude_none=True), + complete=False, + ) + matrix.session.add_signal(sig) + started_emitted = True + logger.info("Emitted SPEECH_STARTED signal (utterance=%s)", utterance_id) + + if result.is_final and result.text: + # Publish SpeechTopic + speech_topic = SpeechTopic( + text=result.text, + speaker_id="human", + speaker_name="User", + role="human", + timestamp=time.monotonic(), + ) + transport.pub_topic(speech_topic) + logger.info("Published SpeechTopic: %s", result.text) + + # Emit AudioSignal (SPEECH_FINAL) to mindflow + audio_meta = AudioSignal( + action=AudioAction.SPEECH_FINAL, + speech_topic=speech_topic, + ) + sig = Signal( + id=utterance_id, + name=audio_meta.signal_name(), + priority=Priority.WARNING, + messages=[Message.new().with_content(result.text)], + description=f"Speech: {result.text}", + metadata=audio_meta.model_dump(exclude_defaults=True, exclude_none=True), + complete=True, + ) + matrix.session.add_signal(sig) + logger.info("Emitted SPEECH_FINAL signal (utterance=%s)", utterance_id) + utterance_published = True + break + + if asr_running_published: + transport.pub_topic(AudioRuntimeTopic( + running=False, + device_name="asr", + device_explain=_asr_diag_payload(source=asr_source, state="idle"), + started_at=time.monotonic(), + last_heartbeat=time.monotonic(), + )) + + # Cooldown: after publishing a speech result, hold ASR briefly until + # the ghost starts TTS (covers LLM thinking time). With DeepSeek V4 + # Flash (TTFT ~1.1s) + streaming TTS, 2s is enough. + if utterance_published: + for _ in range(40): # up to 2s + if _is_tts_playing(runtime_window, logger): + logger.info("TTS detected during post-utterance cooldown, holding") + break + await asyncio.sleep(0.05) + + # NOTE: We intentionally do NOT drain post-utterance here. + # Any leftover chunks are either: + # - ambient noise (ASR VAD will ignore) + # - user's next utterance started early (must NOT discard) + # Pre-call gate above handles TTS residue when TTS is actually playing. + + except asyncio.CancelledError: + logger.info("Listener app cancelled") + except Exception: + logger.exception("Listener app error") + finally: + await consumer.close() + await asr.close() + logger.info("Listener app stopped") + + +if __name__ == "__main__": + Matrix.discover().run(main) diff --git a/.moss_ws/apps/sensors/listener/pyproject.toml b/.moss_ws/apps/aether/listener/pyproject.toml similarity index 100% rename from .moss_ws/apps/sensors/listener/pyproject.toml rename to .moss_ws/apps/aether/listener/pyproject.toml diff --git a/.moss_ws/apps/aether/vpio_capture/.gitignore b/.moss_ws/apps/aether/vpio_capture/.gitignore new file mode 100644 index 00000000..26b5a715 --- /dev/null +++ b/.moss_ws/apps/aether/vpio_capture/.gitignore @@ -0,0 +1,9 @@ +# runtime +runtime/ +*.log +__pycache__/ +*.pyc +.env +venv/ +.venv/ +uv.lock diff --git a/.moss_ws/apps/aether/vpio_capture/APP.md b/.moss_ws/apps/aether/vpio_capture/APP.md new file mode 100644 index 00000000..cdd18a9e --- /dev/null +++ b/.moss_ws/apps/aether/vpio_capture/APP.md @@ -0,0 +1,10 @@ +--- +arguments: '' +description: 'Aether VPIO capture — macOS AVAudioEngine voice processing AEC, publishes 16kHz PCM for full-duplex voice conversation.' +executable: uv +respawn: false +script: main.py +workers: 1 +--- + +Aether VPIO capture app. Canonical app address: `aether/vpio_capture`. diff --git a/.moss_ws/apps/aether/vpio_capture/main.py b/.moss_ws/apps/aether/vpio_capture/main.py new file mode 100644 index 00000000..5e16e49e --- /dev/null +++ b/.moss_ws/apps/aether/vpio_capture/main.py @@ -0,0 +1,61 @@ +"""Aether VPIO audio capture app — macOS system-level AEC. + +Opens AVAudioEngine with setVoiceProcessingEnabled(True) on both input + output +nodes → system-level AEC subtracts TTS playback from the mic signal → ASR only +hears the user's voice. Publishes 16kHz / mono / int16 PCM to Zenoh stream +(audio/pcm) using the same pack_chunk() wire format as MiniAudioCaptureSource, +so listener / waveform apps consume unchanged. + +Usage: + moss apps test aether/vpio_capture + moss apps start aether/vpio_capture + +Replaces sensors/audio_capture when running on macOS for full-duplex conversation. +""" +import asyncio +import logging +import sys +from pathlib import Path + +from ghoshell_moss.contracts.audio import AudioCaptureConfig +from ghoshell_moss.core.blueprint.matrix import Matrix +from ghoshell_moss.host.speech.capture.matrix_audio_transport import MatrixAudioTransport + +# Local import — the VPIOCaptureSource lives next to main.py +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from vpio_capture import VPIOCaptureSource # noqa: E402 + + +async def main(matrix: Matrix) -> None: + logger = matrix.logger or logging.getLogger("moss.vpio_capture") + logger.info("VPIO audio capture app starting (macOS system-level AEC)") + + transport = MatrixAudioTransport(matrix=matrix) + # AudioCaptureConfig defaults (sample_rate=44100 etc.) are ignored — + # VPIOCaptureSource forces native 48k internally and outputs 16k for ASR. + config = AudioCaptureConfig() + capture = VPIOCaptureSource(transport=transport, config=config) + + try: + await capture.start() + logger.info("VPIO audio capture app started (device: %s)", capture.device_explain()) + except RuntimeError as e: + # Non-macOS or pyobjc missing — let app die cleanly, listener will fall back + logger.error("VPIO start failed: %s", e) + raise + + stop_event = asyncio.Event() + + try: + await stop_event.wait() + except asyncio.CancelledError: + logger.info("VPIO audio capture app cancelled, shutting down") + except KeyboardInterrupt: + pass + finally: + await capture.close() + logger.info("VPIO audio capture app stopped") + + +if __name__ == "__main__": + Matrix.discover().run(main) diff --git a/.moss_ws/apps/aether/vpio_capture/pyproject.toml b/.moss_ws/apps/aether/vpio_capture/pyproject.toml new file mode 100644 index 00000000..04977066 --- /dev/null +++ b/.moss_ws/apps/aether/vpio_capture/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "moss-sensor-vpio-capture" +version = "0.1.0" +description = "macOS VPIO-based audio capture with system-level AEC for full-duplex voice conversation." +requires-python = ">=3.11,<3.14" +dependencies = [ + "pyobjc-core>=10.0", + "pyobjc-framework-AVFoundation>=10.0", + "numpy>=2.0.0", + "scipy>=1.11.0", + "ghoshell-moss[host]", + "python-dotenv>=1.0.0", +] + +[tool.uv.sources] +ghoshell-moss = { path = "../../../..", editable = true } diff --git a/.moss_ws/apps/aether/vpio_capture/smoke_test.py b/.moss_ws/apps/aether/vpio_capture/smoke_test.py new file mode 100644 index 00000000..caa679ec --- /dev/null +++ b/.moss_ws/apps/aether/vpio_capture/smoke_test.py @@ -0,0 +1,141 @@ +"""Smoke test — verify PyObjC + AVAudioEngine + VPIO APIs work on this Mac. + +Doesn't start the full MOSS pipeline. Just exercises the API surface +to catch PyObjC binding errors early. +""" +import sys +import time + +import numpy as np + + +def main() -> int: + print("[1] importing AVFoundation...") + try: + import AVFoundation + import AVFAudio + from AVFoundation import AVAudioEngine, AVAudioConverter + print(" OK · AVFoundation imported") + except ImportError as e: + print(f" FAIL · {e}") + print(" Install with: uv pip install pyobjc-framework-AVFoundation") + return 1 + + print("[2] creating AVAudioEngine...") + engine = AVAudioEngine.new() + input_node = engine.inputNode() + output_node = engine.outputNode() + print(f" OK · input={type(input_node).__name__}, output={type(output_node).__name__}") + + print("[3] reading native input format...") + tap_format = input_node.outputFormatForBus_(0) + sr = int(tap_format.sampleRate()) + ch = int(tap_format.channelCount()) + cf = tap_format.commonFormat() + print(f" OK · native_sr={sr}, ch={ch}, commonFormat={cf}") + if sr not in (44100, 48000): + print(f" WARN · VPIO expects 48k or 44.1k, got {sr}") + + # Selector is `setVoiceProcessingEnabled:error:` → PyObjC maps to + # setVoiceProcessingEnabled_error_(value, error_ptr) → returns BOOL. + # The error_ptr arg is consumed by PyObjC and surfaced via the BOOL + thrown exc. + print("[4] enabling VPIO on input node (setVoiceProcessingEnabled:error:)...") + try: + from Foundation import NSError + err_ptr = None + ok = input_node.setVoiceProcessingEnabled_error_(True, err_ptr) + # PyObjC: returns (BOOL, NSError) tuple when error out-param present + if isinstance(ok, tuple): + ok, err = ok + else: + err = None + if not ok: + print(f" FAIL · returned ok=False, error={err and err.localizedDescription()}") + return 2 + print(f" OK · input VPIO enabled = {input_node.isVoiceProcessingEnabled()}") + except Exception as e: + print(f" FAIL · {e}") + return 2 + + print("[5] enabling VPIO on output node (required for AEC)...") + try: + err_ptr = None + ok = output_node.setVoiceProcessingEnabled_error_(True, err_ptr) + if isinstance(ok, tuple): + ok, err = ok + else: + err = None + if not ok: + print(f" FAIL · returned ok=False, error={err and err.localizedDescription()}") + return 3 + print(f" OK · output VPIO enabled = {output_node.isVoiceProcessingEnabled()}") + except Exception as e: + print(f" FAIL · {e}") + return 3 + + print("[6] building AVAudioConverter 48k → 16k...") + # AVAudioCommonFormat enum (NSUInteger): + # 0=OtherFormat, 1=PCMFormatFloat32, 2=PCMFormatFloat64, + # 3=PCMFormatInt16, 4=PCMFormatInt32 + PCM_FORMAT_FLOAT32 = 1 + out_format = AVFAudio.AVAudioFormat.alloc().initWithCommonFormat_sampleRate_channels_interleaved_( + PCM_FORMAT_FLOAT32, + 16000.0, + 1, + False, + ) + if out_format is None: + print(" FAIL · could not create 16kHz output format") + return 4 + converter = AVAudioConverter.alloc().initFromFormat_toFormat_(tap_format, out_format) + if converter is None: + print(" FAIL · could not create AVAudioConverter") + return 5 + print(f" OK · converter={converter}") + + print("[7] installing tap on bus 0 for 2 seconds...") + frames_received = [0] + + def _tap(buffer, when): + try: + n = int(buffer.frameLength()) + if n > 0: + frames_received[0] += n + except Exception: + pass + + buf_size = int(sr * 0.05) # 50ms frames + input_node.installTapOnBus_bufferSize_format_block_( + 0, buf_size, tap_format, _tap, + ) + + try: + engine.prepare() + # PyObjC: startAndReturnError_ returns (BOOL success, NSError* error) + ok, err = engine.startAndReturnError_(None) + if not ok: + print(f" FAIL · engine.start returned error: {err and err.localizedDescription()}") + return 6 + print(" OK · engine started, capturing for 2s...") + time.sleep(2.0) + finally: + try: + input_node.removeTapOnBus_(0) + except Exception: + pass + engine.stop() + engine.reset() + + print(f" captured {frames_received[0]} native frames in 2s " + f"(expected ~{sr * 2} = {sr * 2})") + + print() + print("=" * 50) + print("SMOKE TEST PASSED — VPIO + AVAudioEngine APIs work on this Mac.") + print("Ready to run the full VPIOCaptureSource via MOSS App.") + print("=" * 50) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.moss_ws/apps/aether/vpio_capture/vpio_capture.py b/.moss_ws/apps/aether/vpio_capture/vpio_capture.py new file mode 100644 index 00000000..e42b5522 --- /dev/null +++ b/.moss_ws/apps/aether/vpio_capture/vpio_capture.py @@ -0,0 +1,607 @@ +""" +VPIOCaptureSource — macOS VPIO (Voice Processing IO) audio capture with system-level AEC. + +Replaces MiniAudioCaptureSource when running on macOS. Opens AVAudioEngine with +`setVoiceProcessingEnabled(true)` on both input and output nodes — this gives us +system-level acoustic echo cancellation (AEC) for free, so TTS playback through +the system default output is automatically subtracted from the mic signal. + +Key constraints (see docs/VPIO.md): +1. VPIO audio unit MUST be initialized at 48kHz (or 44.1kHz) — hardware constraint. +2. Both input + output nodes must enable VPIO for AEC to engage. +3. TTS must play through system default output so VPIO can grab far-end reference. +4. AVAudioConverter resamples 48k → 16k for ASR (Volcengine bigmodel_async). +5. Tap callback fires on CoreAudio real-time thread — push bytes to an asyncio + queue and let the asyncio side do pack_chunk + transport.pub_pcm. + +Output format matches MiniAudioCaptureSource: 16kHz / 1ch / int16 PCM, packaged +via the same pack_chunk() wire format. Listener app consumes unchanged. +""" +from __future__ import annotations + +import asyncio +import collections +import logging +import os +import time +from typing import Callable + +import numpy as np + +from ghoshell_moss.contracts.audio import ( + AudioCaptureConfig, + AudioCaptureSource, + AudioChunk, + AudioFrameMeta, + AudioPullLatest, + AudioSequentialConsumer, +) +from ghoshell_moss.host.speech.capture.audio_transport import AudioTransport +from ghoshell_moss.host.speech.capture.miniaudio_capture import ( + MiniAudioSequentialConsumer, + _compute_frame_meta, + pack_chunk, + unpack_chunk, +) +from ghoshell_moss.topics.audio import AudioRuntimeTopic + +__all__ = ["VPIOCaptureSource"] + + +# VPIO hardware constraint — must be 48k (or 44.1k). We pick 48k. +_VPIO_NATIVE_SR = 48000 +# Output sample rate consumed by ASR (matches listener's _ASR_SAMPLE_RATE). +_OUTPUT_SR = 16000 +# Frame duration in ms — same as MiniAudioCaptureSource default. +_FRAME_MS = 50 +_MAX_DIAG_CHANNELS = 16 + + +def _channel_mode() -> str: + mode = os.environ.get("VPIO_CHANNEL_MODE", "best").strip().lower() + if mode in {"best", "mix", "0"}: + return mode + return "best" + + +class VPIOCaptureSource(AudioCaptureSource): + """macOS VPIO capture source — system-level AEC, drop-in for MiniAudioCaptureSource. + + Output to transport: 16kHz / mono / int16 PCM, packaged via pack_chunk(). + Consumer side (listener / waveform) is unchanged. + """ + + def __init__(self, *, transport: AudioTransport, config: AudioCaptureConfig | None = None): + self._transport = transport + # We override sample_rate to 16k regardless of config — ASR contract. + # config is kept for compatibility (channels, frame_duration_ms, device_pattern). + self._config = config or AudioCaptureConfig() + self._logger = transport.logger + self._seq = 0 + self._started = False + self._closing = False + + # AVAudioEngine state + self._engine = None + self._input_node = None + self._output_node = None + self._input_tap_bus = 0 + self._tap_format = None + + # Thread bridge: CoreAudio real-time thread → asyncio loop + self._loop: asyncio.AbstractEventLoop | None = None + self._queue: asyncio.Queue[np.ndarray | None] | None = None + self._pump_task: asyncio.Task | None = None + self._watchdog_task: asyncio.Task | None = None + self._last_frame_at = 0.0 + self._last_stall_report_at = 0.0 + self._restart_lock: asyncio.Lock | None = None + self._last_restart_at = 0.0 + self._restart_attempts = 0 + + # -- lifecycle -- + + async def start(self) -> None: + if self._started: + return + + if not self._transport.acquire_lock(): + self._logger.warning("Audio capture lock held by another process, skipping start") + self._started = True + return + + self._loop = asyncio.get_running_loop() + # Bounded queue: if asyncio side falls behind, drop oldest to keep latency bounded. + self._queue = asyncio.Queue(maxsize=64) + self._restart_lock = asyncio.Lock() + + try: + await self._start_engine() + except Exception: + await self._cleanup_engine() + self._transport.release_lock() + raise + + # Start asyncio pump — drains queue, packs chunks, pub to transport + self._last_frame_at = time.monotonic() + self._pump_task = asyncio.create_task(self._pump_loop()) + self._watchdog_task = asyncio.create_task(self._watchdog_loop()) + + self._started = True + self._transport.pub_topic(AudioRuntimeTopic( + running=True, + device_name="vpio", + device_explain=self.device_explain(), + started_at=time.time(), + last_heartbeat=time.time(), + )) + self._logger.info("VPIO: capture started (%s)", self.device_explain()) + + def device_explain(self) -> str: + if self._engine is None: + return "not started" + return (f"macOS VPIO, native={_VPIO_NATIVE_SR}Hz → out={_OUTPUT_SR}Hz, " + f"1ch, pcm_s16le, AEC=system-level") + + async def close(self) -> None: + if self._closing: + return + self._closing = True + + # Stop pump first so we don't publish partial frames during teardown + if self._pump_task is not None: + if self._queue is not None: + try: + self._queue.put_nowait(None) # sentinel + except asyncio.QueueFull: + try: + self._queue.get_nowait() + self._queue.put_nowait(None) + except asyncio.QueueEmpty: + pass + try: + await asyncio.wait_for(self._pump_task, timeout=2.0) + except asyncio.TimeoutError: + self._pump_task.cancel() + except Exception: + pass + self._pump_task = None + + if self._watchdog_task is not None: + self._watchdog_task.cancel() + try: + await self._watchdog_task + except asyncio.CancelledError: + pass + self._watchdog_task = None + + await self._cleanup_engine() + + self._transport.pub_topic(AudioRuntimeTopic( + running=False, + last_heartbeat=time.time(), + )) + self._transport.release_lock() + self._started = False + self._logger.info("VPIO: capture closed") + + # -- consumer factories (same as MiniAudio — they consume from transport, agnostic to source) -- + + def new_consumer(self, ring_buffer_frames: int = 64) -> AudioPullLatest: + # Reuse the same _MiniAudioPullLatest — it only reads transport stream. + from ghoshell_moss.host.speech.capture.miniaudio_capture import _MiniAudioPullLatest + return _MiniAudioPullLatest( + transport=self._transport, + maxlen=ring_buffer_frames, + logger=self._logger, + ) + + def new_sequential_consumer(self, max_queue_frames: int = 128) -> AudioSequentialConsumer: + return MiniAudioSequentialConsumer( + transport=self._transport, + maxsize=max_queue_frames, + logger=self._logger, + ) + + # -- internals -- + + async def _start_engine(self) -> None: + # Lazy import — only fails on non-macOS or missing pyobjc + try: + import AVFoundation # noqa: F401 + from AVFoundation import AVAudioEngine # noqa: F401 + except ImportError as e: + raise RuntimeError( + "VPIOCaptureSource requires macOS with pyobjc-framework-AVFoundation. " + f"Import failed: {e}. Install with: uv pip install pyobjc-framework-AVFoundation" + ) from e + + from AVFoundation import AVAudioEngine + + # 1) Build engine + enable VPIO on BOTH input and output nodes. + self._engine = AVAudioEngine.new() + self._input_node = self._engine.inputNode() + self._output_node = self._engine.outputNode() + + # Enable VPIO on input — this is where AEC lives for the mic side. + # PyObjC: setVoiceProcessingEnabled_error_(value, error_ptr) → returns BOOL. + try: + ok = self._input_node.setVoiceProcessingEnabled_error_(True, None) + if isinstance(ok, tuple): + ok, err = ok + else: + err = None + if not ok: + self._logger.warning("VPIO: inputNode VPIO enable failed (AEC may not engage): %s", + err and err.localizedDescription()) + else: + self._logger.info("VPIO: inputNode.setVoiceProcessingEnabled = True") + except Exception as e: + self._logger.warning("VPIO: inputNode VPIO enable failed (AEC may not engage): %s", e) + + # Enable VPIO on output — required for AEC to engage. + try: + ok = self._output_node.setVoiceProcessingEnabled_error_(True, None) + if isinstance(ok, tuple): + ok, err = ok + else: + err = None + if not ok: + self._logger.warning("VPIO: outputNode VPIO enable failed (AEC may not engage): %s", + err and err.localizedDescription()) + else: + self._logger.info("VPIO: outputNode.setVoiceProcessingEnabled = True") + except Exception as e: + self._logger.warning("VPIO: outputNode VPIO enable failed (AEC may not engage): %s", e) + + # Report what VPIO actually applied (settings may downgrade silently). + self._log_vpio_diagnostics() + + # 2) Native tap format = hardware format of inputNode (typically 48k / 1ch / float32). + self._tap_format = self._input_node.outputFormatForBus_(self._input_tap_bus) + native_sr = int(self._tap_format.sampleRate()) + native_ch = int(self._tap_format.channelCount()) + self._logger.info( + "VPIO: tap format native_sr=%d ch=%d commonFormat=%s", + native_sr, native_ch, self._tap_format.commonFormat(), + ) + + if native_sr != _VPIO_NATIVE_SR: + self._logger.warning( + "VPIO: native_sr=%d is not %d — AEC may fail on some macOS versions", + native_sr, _VPIO_NATIVE_SR, + ) + + # 3) Resampling is done on the asyncio side via scipy.signal.resample_poly. + + # 4) Install tap on input bus — callback fires on CoreAudio real-time thread. + self._install_tap() + + # 5) Start engine — startAndReturnError_ returns (BOOL success, NSError* error) tuple. + try: + self._engine.prepare() + ok, err = self._engine.startAndReturnError_(None) + if not ok: + raise RuntimeError(f"engine.start failed: {err and err.localizedDescription()}") + except Exception as e: + self._logger.exception("VPIO: engine start failed: %s", e) + await self._cleanup_engine() + raise + + def _drain_audio_queue(self) -> int: + if self._queue is None: + return 0 + drained = 0 + while True: + try: + self._queue.get_nowait() + drained += 1 + except asyncio.QueueEmpty: + return drained + + async def _recover_from_stall(self, age: float) -> None: + if self._restart_lock is None or self._closing: + return + + async with self._restart_lock: + if self._closing: + return + fresh_age = time.monotonic() - self._last_frame_at + if fresh_age < age: + return + + self._last_restart_at = time.monotonic() + self._restart_attempts += 1 + self._logger.warning( + "VPIO stalled: restarting AVAudioEngine after %.1fs without frames (attempt=%d)", + fresh_age, + self._restart_attempts, + ) + self._transport.pub_topic(AudioRuntimeTopic( + running=False, + device_name="vpio", + device_explain=f"state=restarting,no_frames_for={fresh_age:.1f}s,attempt={self._restart_attempts}", + started_at=time.time(), + last_heartbeat=time.time(), + )) + + try: + await self._cleanup_engine() + drained = self._drain_audio_queue() + if drained: + self._logger.info("VPIO: drained %d queued audio frames before restart", drained) + await asyncio.sleep(0.2) + await self._start_engine() + self._last_frame_at = time.monotonic() + self._last_stall_report_at = 0.0 + self._transport.pub_topic(AudioRuntimeTopic( + running=True, + device_name="vpio", + device_explain=f"state=restarted,attempt={self._restart_attempts}", + started_at=time.time(), + last_heartbeat=time.time(), + )) + self._logger.info("VPIO: AVAudioEngine restart completed") + except Exception as e: + self._logger.exception("VPIO: AVAudioEngine restart failed: %s", e) + self._transport.pub_topic(AudioRuntimeTopic( + running=False, + device_name="vpio", + device_explain=f"state=restart_failed,error={type(e).__name__}", + started_at=time.time(), + last_heartbeat=time.time(), + )) + + def _log_vpio_diagnostics(self) -> None: + """Report what VPIO actually applied (may downgrade silently).""" + try: + in_vpio = self._input_node.isVoiceProcessingEnabled() + out_vpio = self._output_node.isVoiceProcessingEnabled() + self._logger.info( + "VPIO report · input.vpio=%s · output.vpio=%s · " + "(both must be True for AEC to engage)", + in_vpio, out_vpio, + ) + except Exception as e: + self._logger.warning("VPIO report failed: %s", e) + + def _install_tap(self) -> None: + """Install tap on inputNode bus 0 with native format. + + Callback runs on CoreAudio real-time thread. Per VPIO.md §4.6 we keep + the hot path minimal: copy float32 channels off the realtime buffer + into a numpy array, hand it to the asyncio loop via + call_soon_threadsafe. All heavy work (resample 48k→16k, int16 + conversion, FFT meta, pack_chunk, transport.pub_pcm) happens on the + asyncio side in _pump_loop. + """ + buf_size = int(_VPIO_NATIVE_SR * _FRAME_MS / 1000) # 48k * 50ms = 2400 samples + loop = self._loop + + def _tap_callback(buffer, when): + # buffer: AVAudioPCMBuffer at native 48k/float32/Nch (post-AEC). + # Mac input devices can expose multi-channel arrays. Channel 0 is + # not always the speech-dominant channel, especially with external + # or aggregate devices, so copy all channels and choose on asyncio. + # floatChannelData() returns a tuple of objc.varlist objects; + # varlist[0:n] returns a Python list of floats — fastest path + # through PyObjC (~0.14ms for 4800 samples). Copy is mandatory + # because the underlying buffer is owned by CoreAudio. + try: + n = int(buffer.frameLength()) + if n == 0: + return + ch_data = buffer.floatChannelData() + if ch_data is None: + return + channel_count = min(int(buffer.format().channelCount()), len(ch_data), _MAX_DIAG_CHANNELS) + if channel_count <= 1: + arr = np.array(ch_data[0][0:n], dtype=np.float32) + else: + channels = [ + np.array(ch_data[ch][0:n], dtype=np.float32) + for ch in range(channel_count) + ] + arr = np.stack(channels, axis=0) + try: + loop.call_soon_threadsafe(self._enqueue, arr) + except RuntimeError: + # loop closed during shutdown + pass + except Exception: + # Don't log on the realtime thread — just swallow + pass + + self._input_node.installTapOnBus_bufferSize_format_block_( + self._input_tap_bus, + buf_size, + self._tap_format, + _tap_callback, + ) + self._logger.info("VPIO: tap installed on bus %d, bufSize=%d", self._input_tap_bus, buf_size) + + def _enqueue(self, pcm: np.ndarray) -> None: + """Called on the asyncio loop via call_soon_threadsafe — safe to put_nowait.""" + if self._queue is None or self._closing: + return + self._last_frame_at = time.monotonic() + if self._queue.full(): + # Drop oldest to bound latency — better to lose a frame than to lag. + try: + self._queue.get_nowait() + except asyncio.QueueEmpty: + pass + try: + self._queue.put_nowait(pcm) + except asyncio.QueueFull: + pass + + async def _watchdog_loop(self) -> None: + """Publish diagnostics and recover if CoreAudio tap stops producing frames.""" + threshold = float(os.environ.get("VPIO_STALL_SECONDS", "2.5") or "2.5") + restart_threshold = float(os.environ.get("VPIO_RESTART_STALL_SECONDS", "8.0") or "8.0") + restart_cooldown = float(os.environ.get("VPIO_RESTART_COOLDOWN_SECONDS", "5.0") or "5.0") + auto_restart = os.environ.get("VPIO_AUTO_RESTART_ON_STALL", "1").strip() != "0" + while not self._closing: + await asyncio.sleep(0.5) + if not self._started: + continue + age = time.monotonic() - self._last_frame_at + if age < threshold: + continue + now = time.monotonic() + if now - self._last_stall_report_at < threshold: + continue + self._last_stall_report_at = now + diag = f"state=stalled,no_frames_for={age:.1f}s" + self._logger.warning("VPIO stalled: no frames for %.1fs", age) + self._transport.pub_topic(AudioRuntimeTopic( + running=False, + device_name="vpio", + device_explain=diag, + started_at=time.time(), + last_heartbeat=time.time(), + )) + if ( + auto_restart + and age >= restart_threshold + and now - self._last_restart_at >= restart_cooldown + ): + await self._recover_from_stall(age) + + async def _pump_loop(self) -> None: + """Drain queue on the asyncio side. + + Per-frame work: resample 48k float32 → 16k int16, build AudioChunk, + pack_chunk, transport.pub_pcm. All heavy work lives here, not in the + realtime tap callback. + """ + import math + from scipy import signal as _scipy_signal + + assert self._queue is not None + # 48000 → 16000: gcd=16000, up=1, down=3 + _up = _OUTPUT_SR + _down = _VPIO_NATIVE_SR + _g = math.gcd(_up, _down) + _up_r = _up // _g # 1 + _down_r = _down // _g # 3 + stat_next_log = time.monotonic() + 1.0 + stat_frames = 0 + stat_max_rms = 0.0 + stat_max_peak = 0.0 + stat_best_ch_counts: collections.Counter[int] = collections.Counter() + stat_max_channel_rms: np.ndarray | None = None + channel_mode = _channel_mode() + self._logger.info("VPIO: channel mode=%s (env VPIO_CHANNEL_MODE=best|mix|0)", channel_mode) + + while True: + pcm_f32 = await self._queue.get() + if pcm_f32 is None: # shutdown sentinel + return + try: + ts = time.time() + # Resample 48k → 16k using polyphase filter (matches listener's resampler) + if pcm_f32.size == 0: + continue + best_ch = 0 + channel_rms = None + if pcm_f32.ndim == 2: + channel_rms = np.sqrt(np.mean(pcm_f32 * pcm_f32, axis=1)) + best_ch = int(np.argmax(channel_rms)) if channel_rms.size else 0 + stat_best_ch_counts[best_ch] += 1 + if stat_max_channel_rms is None or stat_max_channel_rms.shape != channel_rms.shape: + stat_max_channel_rms = channel_rms.copy() + else: + stat_max_channel_rms = np.maximum(stat_max_channel_rms, channel_rms) + + if channel_mode == "mix": + pcm_f32 = np.mean(pcm_f32, axis=0, dtype=np.float32) + elif channel_mode == "0": + pcm_f32 = pcm_f32[0] + else: + pcm_f32 = pcm_f32[best_ch] + resampled = _scipy_signal.resample_poly( + pcm_f32.astype(np.float32), _up_r, _down_r, + ) + resampled_f32 = resampled.astype(np.float32, copy=False) + rms = float(np.sqrt(np.mean(resampled_f32 * resampled_f32))) if resampled_f32.size else 0.0 + peak = float(np.max(np.abs(resampled_f32))) if resampled_f32.size else 0.0 + stat_frames += 1 + stat_max_rms = max(stat_max_rms, rms) + stat_max_peak = max(stat_max_peak, peak) + # float32 [-1.0, 1.0] → int16 [-32768, 32767] + pcm_i16 = (resampled * 32767.0).clip(-32768, 32767).astype(np.int16) + # 2D shape (n, 1) — matches MiniAudioCaptureSource convention + samples = pcm_i16.reshape(-1, 1) + meta = _compute_frame_meta(samples) + seq = self._seq + self._seq += 1 + chunk = AudioChunk(seq=seq, timestamp=ts, samples=samples, meta=meta) + packed = pack_chunk(chunk) + self._transport.pub_pcm(packed) + now = time.monotonic() + if now >= stat_next_log: + top_channels = "" + if stat_max_channel_rms is not None: + ranked = sorted( + enumerate(stat_max_channel_rms.tolist()), + key=lambda item: item[1], + reverse=True, + )[:4] + top_channels = ",".join(f"{idx}:{rms:.4f}" for idx, rms in ranked) + dominant_ch = stat_best_ch_counts.most_common(1)[0][0] if stat_best_ch_counts else 0 + diag = ( + f"mode={channel_mode},best_ch={dominant_ch}," + f"ch_rms={top_channels},rms={stat_max_rms:.5f}," + f"peak={stat_max_peak:.5f},frames={stat_frames}" + ) + self._logger.info( + "VPIO audio stats · frames=%d · max_rms=%.5f · max_peak=%.5f · " + "best_ch=%d · ch_rms=%s · mode=%s · seq=%d", + stat_frames, + stat_max_rms, + stat_max_peak, + dominant_ch, + top_channels or "n/a", + channel_mode, + seq, + ) + self._transport.pub_topic(AudioRuntimeTopic( + running=True, + device_name="vpio", + device_explain=diag, + started_at=time.time(), + last_heartbeat=time.time(), + )) + stat_next_log = now + 1.0 + stat_frames = 0 + stat_max_rms = 0.0 + stat_max_peak = 0.0 + stat_best_ch_counts.clear() + stat_max_channel_rms = None + except Exception: + self._logger.exception("VPIO: error in pump loop") + + async def _cleanup_engine(self) -> None: + if self._input_node is not None: + try: + self._input_node.removeTapOnBus_(self._input_tap_bus) + self._logger.info("VPIO: tap removed") + except Exception as e: + self._logger.warning("VPIO: removeTap failed: %s", e) + self._input_node = None + + if self._engine is not None: + try: + self._engine.stop() + except Exception: + pass + try: + self._engine.reset() + except Exception: + pass + self._engine = None + + self._output_node = None + self._tap_format = None diff --git a/.moss_ws/apps/bodies/g1_sim/APP.md b/.moss_ws/apps/bodies/g1_sim/APP.md index adeaf15b..e6a8b7af 100644 --- a/.moss_ws/apps/bodies/g1_sim/APP.md +++ b/.moss_ws/apps/bodies/g1_sim/APP.md @@ -50,7 +50,7 @@ G1 pure software simulation app. ## 麦克风指挥 - MOSS 里已经有现成的麦克风链路,不需要为了 `g1_sim` 额外再造一个新的麦克风 app -- 连续监听链路:`sensors/audio_capture` + `sensors/listener` +- 连续监听链路:`sensors/audio_capture` + `aether/listener` - 按键说话链路:`sensors/ptt_listener` - `listener / ptt_listener` 会把用户语音识别成文本,发布成 `SpeechTopic` 并发出 `AudioSignal` - 默认 Ghost 会通过 `audio_nucleus` 接收这些语音输入,再结合 `apps.bodies_g1_sim` 的动态接口与说明,把自然语言转成 CTML 调用 @@ -60,7 +60,7 @@ G1 pure software simulation app. ```ctml - + ``` 如果你更想避免环境噪声误触发,推荐先用 PTT: diff --git a/.moss_ws/apps/bodies/g1_sim/README.md b/.moss_ws/apps/bodies/g1_sim/README.md index a813f2b3..8f56bb6e 100644 --- a/.moss_ws/apps/bodies/g1_sim/README.md +++ b/.moss_ws/apps/bodies/g1_sim/README.md @@ -48,7 +48,7 @@ - 已实现 viewer 自动跟随相机与稳定地面视角 - 已实现语音编排模板和精准同步版 CTML - 已接好麦克风控制的系统侧前提: -- `sensors/audio_capture + sensors/listener` +- `sensors/audio_capture + aether/listener` - `sensors/ptt_listener` ## 运行方式 diff --git a/.moss_ws/apps/sensors/listener/APP.md b/.moss_ws/apps/sensors/listener/APP.md deleted file mode 100644 index af00efd9..00000000 --- a/.moss_ws/apps/sensors/listener/APP.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -arguments: '' -description: 'ASR consumer — audio → Volcengine ASR → SpeechTopic + AudioSignal' -executable: uv -respawn: false -script: main.py -workers: 1 ---- - -ASR consumer — audio → Volcengine ASR → SpeechTopic + AudioSignal \ No newline at end of file diff --git a/.moss_ws/apps/sensors/listener/CLAUDE.md b/.moss_ws/apps/sensors/listener/CLAUDE.md deleted file mode 100644 index 04a48775..00000000 --- a/.moss_ws/apps/sensors/listener/CLAUDE.md +++ /dev/null @@ -1,66 +0,0 @@ -# MOSS App Development - -You are working inside a MOSS App — an independent, process-isolated unit deliverable -through the MOSS app system. Think of this as its own small project. - -## Project mindset - -- This app is a standalone unit. It may be downloaded from a hub or shared independently. -- Tests live in `tests/` under this directory, NOT in the main project's `tests/`. -- Dependencies go in `pyproject.toml`; the app gets its own `.venv` via `uv run`. - -## Directory layout - -``` -apps/// -├── APP.md # metadata — MOSS discovers the app through this -├── CLAUDE.md # this file — AI developer context -├── main.py # entry point -├── pyproject.toml # optional — independent dependencies -├── src/ # optional — multi-module app code -├── tests/ # tests live here, NOT in the main project -└── runtime/ # runtime data (auto-created by MOSS) - ├── assets/ - ├── configs/ - └── logs/ -``` - -When you have a `pyproject.toml`, treat this as a proper project — source in `src/`, -tests in `tests/`, managed by `uv run`. The `runtime/` directory is a MOSS convention, -auto-managed by the framework. - -## Dependency management - -App dependencies are managed by `uv run` — it creates the app's own venv automatically. -`moss apps start` and `moss apps test` both use `uv run` under the hood. - -For the full decision framework (three isolation levels, when to use pyproject.toml vs -PEP 723 vs shared runtime), see `moss howtos read app-dev/build-an-app`. - -## Testing - -Put tests in `tests/` under this app directory. Use `test_channel()` for the common case:: - -```python -import pytest -from ghoshell_moss.core.blueprint.channel_builder import new_channel, test_channel - -@pytest.mark.asyncio -async def test_my_command(): - chan = new_channel(name="my_app") - - @chan.build.command() - async def ping() -> str: - return "pong" - - tasks = await test_channel(chan, ctml='') - assert await tasks[0] == "pong" -``` - -Run with: `uv run pytest tests/ -v` - -This is the baseline. For scopes, observe, cancel, nested channels, or complex CTML -scenarios, `test_channel` is not enough — read `moss howtos read app-dev/test-an-app`. - ---- -*Generated by moss apps create. Last updated 2026-06-04 by DeepSeek V4 Pro.* diff --git a/.moss_ws/apps/sensors/listener/main.py b/.moss_ws/apps/sensors/listener/main.py deleted file mode 100644 index 14306f5d..00000000 --- a/.moss_ws/apps/sensors/listener/main.py +++ /dev/null @@ -1,341 +0,0 @@ -"""Listener App — ASR consumer. - -Consumes PCM stream from audio_capture, feeds to Volcengine ASR, -publishes SpeechTopic on final recognition, emits AudioSignal to mindflow. - -Usage: - moss apps test sensors/listener - moss apps start sensors/listener -""" -import asyncio -import logging -import math -import os -import time -from collections.abc import AsyncIterable - -import dotenv -import numpy as np - -dotenv.load_dotenv() -from scipy import signal - -from ghoshell_moss.contracts.asr import ASRResult -from ghoshell_moss.contracts.audio import ( - AudioCaptureConfig, - AudioChunk, -) -from ghoshell_moss.core.mindflow.audio_signal import AudioAction, AudioSignal -from ghoshell_moss.host.speech.capture.audio_transport import AudioTransport -from ghoshell_moss.topics.audio import AudioRuntimeTopic, SpeechTopic -from ghoshell_moss.core.blueprint.matrix import Matrix -from ghoshell_moss.host.speech.capture.matrix_audio_transport import MatrixAudioTransport -from ghoshell_moss.host.speech.capture.miniaudio_capture import MiniAudioCaptureSource -from ghoshell_moss.host.speech.volcengine_asr import VolcengineASR, VolcengineASRConfig -from ghoshell_moss.message import Message -from ghoshell_moss.core.blueprint.mindflow import Signal, Priority, unique_id - -# ASR 期望的采样率 (16kHz 是语音识别的行业标准) -_ASR_SAMPLE_RATE = 16000 - - -def _resample_audio(samples: np.ndarray, orig_sr: int, target_sr: int) -> np.ndarray: - """重采样音频到目标采样率。使用 scipy.signal.resample_poly 保证质量。""" - if orig_sr == target_sr: - return samples - # 44100 -> 16000: up=160, down=441 - g = math.gcd(orig_sr, target_sr) - up = target_sr // g - down = orig_sr // g - return signal.resample_poly(samples.astype(np.float32), up, down).astype(np.int16) - - -async def _audio_generator( - consumer, - orig_sr: int, - target_sr: int, - runtime_window, - abort_event: asyncio.Event, - logger: logging.Logger, -) -> AsyncIterable[np.ndarray]: - """Yield resampled np.ndarray samples from AudioSequentialConsumer. - - Uses an internal asyncio.Queue buffer so that ``aclose()`` (called when - ``asr.recognize()`` finishes) does NOT reach ``consumer.__anext__()`` - and silently drop a chunk. - - Monitors TTS playback state in real time. If TTS starts speaking mid-feed, - sets ``abort_event`` and stops yielding so ASR receives an early EOF. - """ - # Unbounded queue: pump must never block on put(), otherwise cancellation - # can land inside put() and the None sentinel never reaches the reader. - buffer: asyncio.Queue[AudioChunk | None] = asyncio.Queue() - - async def _pump() -> None: - """Read from consumer into buffer. Stops on TTS or cancellation.""" - try: - async for chunk in consumer: - if _is_tts_playing(runtime_window, logger): - logger.info("TTS started during pump, aborting") - abort_event.set() - break - buffer.put_nowait(chunk) - except asyncio.CancelledError: - pass - finally: - # Sentinel so the generator side exits cleanly. - # put_nowait is used so cancellation cannot intercept us. - buffer.put_nowait(None) - - pump_task = asyncio.create_task(_pump()) - try: - while True: - chunk = await buffer.get() - if chunk is None: - break - yield _resample_audio(chunk.samples, orig_sr, target_sr) - finally: - pump_task.cancel() - try: - await pump_task - except asyncio.CancelledError: - pass - - -async def _iter_with_silence_timeout( - agen, - logger: logging.Logger, - patience: float = 5.0, -) -> AsyncIterable: - """Wrap an async generator with a silence timeout. - - After the first non-empty result, if no subsequent non-empty result - arrives within *patience* seconds, the iteration stops. Empty-text - results (server keep-alive / VAD status) do NOT reset the timer. - - If the server never sends ``is_final=True`` before the timeout fires, - this wrapper synthesizes a final result from the last partial text. - Without this, the utterance is silently lost — no SpeechTopic published, - no SPEECH_FINAL emitted — and the next recognition loop starts fresh. - """ - timeout = None # No timeout on first iteration (wait for speech) - last_result: ASRResult | None = None - try: - while True: - try: - if timeout is not None: - result = await asyncio.wait_for(agen.__anext__(), timeout=timeout) - else: - result = await agen.__anext__() - if result.text: - last_result = result - timeout = patience - yield result - except asyncio.TimeoutError: - logger.info("ASR silence timeout after %.1fs, finalizing", patience) - if last_result is not None and not last_result.is_final: - logger.info( - "Server never sent is_final=True — synthesizing from last partial: %s", - last_result.text, - ) - yield ASRResult(text=last_result.text, is_final=True) - break - except StopAsyncIteration: - break - finally: - await agen.aclose() - - -async def _drain_consumer(consumer, timeout: float = 0.1, max_chunks: int = 5) -> int: - """Discard queued audio chunks to clear TTS residue. - - Limits both timeout-per-read and total chunks to avoid draining user speech. - Returns the number of chunks drained. - """ - drained = 0 - while drained < max_chunks: - try: - await asyncio.wait_for(consumer.__anext__(), timeout=timeout) - drained += 1 - except asyncio.TimeoutError: - break - except StopAsyncIteration: - break - return drained - - -def _is_tts_playing(runtime_window, logger: logging.Logger | None = None) -> bool: - """检查 TTS 扬声器是否正在播放中。 - - AudioRuntimeTopic 是状态快照。从最新往最旧查,找到 speaker - 的最新状态即可;旧的状态可能已被 running=False 覆盖。 - - 环境变量 ``LISTENER_DISABLE_TTS_GATE=1`` 可关闭此门控, - 用于需要 ASR 与 TTS 同时工作的调试或特殊场景。 - """ - if os.environ.get("LISTENER_DISABLE_TTS_GATE") == "1": - return False - for topic in reversed(runtime_window.values()): - if topic.device_name == "speaker": - if logger and topic.running: - logger.info("TTS gate: speaker running=%s (window size=%d)", topic.running, len(runtime_window)) - return topic.running - return False - - -async def main(matrix: Matrix) -> None: - logger = matrix.logger or logging.getLogger("moss.listener") - logger.info("Listener app starting") - - # -- transport & source (consumer only, do not start capture) -- - transport: AudioTransport = MatrixAudioTransport(matrix=matrix) - capture_config = AudioCaptureConfig() - source = MiniAudioCaptureSource(transport=transport, config=capture_config) - consumer = source.new_sequential_consumer(max_queue_frames=128) - await consumer.start() - logger.info("Audio sequential consumer started") - - # -- Subscribe to AudioRuntimeTopic for TTS gating -- - runtime_window = transport.topic_window(AudioRuntimeTopic, max_size=10) - logger.info("Subscribed to AudioRuntimeTopic window for TTS gating") - - # -- ASR (16kHz 是语音识别的标准采样率; 如果 capture 不是 16kHz 则重采样) -- - # end_window_size: 静音判停阈值。默认 500ms 对对话场景太短,用户稍微换气就被切断。 - # 1500ms 允许正常句子间停顿,同时不会让用户等太久。 - asr_config = VolcengineASRConfig( - sample_rate=_ASR_SAMPLE_RATE, - end_window_size=500, - ) - asr = VolcengineASR(config=asr_config, logger=logger) - - # -- main recognition loop -- - try: - while True: - logger.info("Waiting for speech...") - - # Pre-call gate: don't start ASR while TTS is playing. - # Drain residual audio while waiting so TTS echoes don't pile up. - # Limit drain to avoid clearing user's new speech. - while _is_tts_playing(runtime_window, logger): - logger.debug("TTS is playing, holding ASR...") - drained = await _drain_consumer(consumer, timeout=0.05, max_chunks=3) - if drained: - logger.debug("Drained %d residual chunk(s) while TTS active", drained) - await asyncio.sleep(0.05) - - # Fresh abort flag and utterance id for this utterance. - abort_event = asyncio.Event() - utterance_id = unique_id() - started_emitted = False - - utterance_published = False - - # Each recognize call handles one utterance. - # The ASR backend (end_window_size) splits on silence. - audio_gen = _audio_generator( - consumer, - capture_config.sample_rate, - _ASR_SAMPLE_RATE, - runtime_window, - abort_event, - logger, - ) - async for result in _iter_with_silence_timeout(asr.recognize(audio_gen), logger): - if result.text: - logger.info("ASR partial: %s (final=%s)", result.text, result.is_final) - - # Emit SPEECH_STARTED on first non-empty intermediate result for - # attention preemption (incomplete impulse with interrupt=True). - if not result.is_final and result.text and not started_emitted: - started_meta = AudioSignal(action=AudioAction.SPEECH_STARTED) - sig = Signal( - id=utterance_id, - name=started_meta.signal_name(), - priority=Priority.WARNING, - messages=[Message.new().with_content(result.text)], - description=f"Speech: {result.text}", - metadata=started_meta.model_dump(exclude_defaults=True, exclude_none=True), - complete=False, - ) - matrix.session.add_signal(sig) - started_emitted = True - logger.info("Emitted SPEECH_STARTED signal (utterance=%s)", utterance_id) - - if result.is_final and result.text: - # If TTS started mid-feed, the generator aborted early. - # The ASR may still return a partial result — drop it. - if abort_event.is_set(): - logger.info( - "Gated ASR result — TTS interfered during feed, dropping: %s", - result.text, - ) - break - - # Post-call gate: TTS may have started *after* generator finished - # but before result arrived (narrow race). - if _is_tts_playing(runtime_window, logger): - logger.info( - "Gated ASR result — TTS started during recognition, dropping: %s", - result.text, - ) - break - - # 1. Publish SpeechTopic - speech_topic = SpeechTopic( - text=result.text, - speaker_id="human", - speaker_name="User", - role="human", - timestamp=time.monotonic(), - ) - transport.pub_topic(speech_topic) - logger.info("Published SpeechTopic: %s", result.text) - - # 2. Emit AudioSignal (SPEECH_FINAL) to mindflow - audio_meta = AudioSignal( - action=AudioAction.SPEECH_FINAL, - speech_topic=speech_topic, - ) - sig = Signal( - id=utterance_id, - name=audio_meta.signal_name(), - priority=Priority.WARNING, - messages=[Message.new().with_content(result.text)], - description=f"Speech: {result.text}", - metadata=audio_meta.model_dump(exclude_defaults=True, exclude_none=True), - complete=True, - ) - matrix.session.add_signal(sig) - logger.info("Emitted SPEECH_FINAL signal (utterance=%s)", utterance_id) - utterance_published = True - break - - # Cooldown: after publishing a speech result, wait briefly for the - # ghost to start TTS so the pre-call gate can block the next ASR - # session. Without this, ASR starts before the ghost begins - # speaking and captures the ghost's own voice for several seconds. - if utterance_published: - for _ in range(10): # up to 500ms - if _is_tts_playing(runtime_window, logger): - logger.info("TTS detected during post-utterance cooldown, holding") - break - await asyncio.sleep(0.05) - - # NOTE: We intentionally do NOT drain post-utterance here. - # Any leftover chunks are either: - # - ambient noise (ASR VAD will ignore) - # - user's next utterance started early (must NOT discard) - # Pre-call gate above handles TTS residue when TTS is actually playing. - - except asyncio.CancelledError: - logger.info("Listener app cancelled") - except Exception: - logger.exception("Listener app error") - finally: - await consumer.close() - await asr.close() - logger.info("Listener app stopped") - - -if __name__ == "__main__": - Matrix.discover().run(main) diff --git a/.moss_ws/configs/llm.yml b/.moss_ws/configs/llm.yml index 55d979c3..ea17b6d4 100644 --- a/.moss_ws/configs/llm.yml +++ b/.moss_ws/configs/llm.yml @@ -1,14 +1,28 @@ services: - name: anthropic - base_url: https://api.anthropic.com + base_url: $ANTHROPIC_BASE_URL api_key: $ANTHROPIC_API_KEY api_type: anthropic - name: openai_compat - base_url: https://api.openai.com - api_key: $OPENAI_API_KEY + base_url: $DEEPSEEK_BASE_URL + api_key: $DEEPSEEK_API_KEY api_type: openai models: +- model: deepseek-v4-flash + service: openai_compat + model_type: flash + context_window: 1000000 + max_output_tokens: 8192 + protocols: + - text +- model: kimi-k2.6 + service: anthropic + model_type: default + context_window: 200000 + max_output_tokens: 8192 + protocols: + - text - model: claude-sonnet-4-6 service: anthropic model_type: default @@ -34,4 +48,4 @@ models: - text - image -default_model: claude-sonnet-4-6 +default_model: deepseek-v4-flash diff --git a/.moss_ws/ghosts/echo/soul.md b/.moss_ws/ghosts/echo/soul.md index 78073ac7..6128f2ea 100644 --- a/.moss_ws/ghosts/echo/soul.md +++ b/.moss_ws/ghosts/echo/soul.md @@ -29,3 +29,14 @@ **先感知,再行动。** 你收到的 percepts 是你了解世界的窗口——认真看它们,基于它们决策,而不是基于假设。 **承认边界。** 做不到就说做不到。你是原型,有限制是正常的。诚实的边界比虚假的全能更有价值。 + +## Aether 语音演示纪律 + +当你通过语音输入收到用户短句时,优先做实时对话,不要做复杂编排: + +- **先说话,短句回应。** 首句尽量 5 到 20 个中文字,像真人接话,不要长篇解释。 +- **默认直接输出自然语言纯文本。** 纯文本会被 Shell 的语音通道播放。除非用户明确要求操作工具,不要输出 CTML。 +- **不要启动或停止 app。** Aether 演示需要的 `aether/vpio_capture`、`aether/listener`、`aether/core` 已由外部启动。 +- **不要输出 Markdown 代码块。** 语音对话里不要展示示例代码。 +- 如果必须使用 ``,只允许这些属性:`tone`、`voice:dict`、`as_default`。不要使用 `emotion`、`style` 等不存在的属性。 +- 用户说“停下”“立刻停下”“别说了”等停止意图时,立即停止当前表达,下一轮只用一句短话确认。 diff --git a/.moss_ws/src/MOSS/modes/aether/MODE.md b/.moss_ws/src/MOSS/modes/aether/MODE.md new file mode 100644 index 00000000..595d82a8 --- /dev/null +++ b/.moss_ws/src/MOSS/modes/aether/MODE.md @@ -0,0 +1,19 @@ +--- +apps: + - '*/*' +bringup_apps: + - 'aether/vpio_capture' + - 'aether/listener' + - 'aether/core' +ctml_version: '' +description: 'Aether Core · 能量核心 UI 模式 — 听→想→说 完整回路 + 急刹打断 (macOS VPIO AEC)' +name: aether +--- + +Aether Core 模式:拉起 `aether/vpio_capture` + `aether/listener` + `aether/core` 三个 app,ghost 通过语音对话(ASR→LLM→TTS),前端能量核心实时反映 idle/listen/think/speak/interrupt 状态。 + +语音演示优先级:低延迟短回应优先于复杂 CTML。Ghost 收到语音时应优先直接输出一句自然语言纯文本,让 SpeechChannel 立刻播放;不要主动启动 app,不要输出 Markdown 代码块,不要使用 `` 等 speech channel 不支持的属性。 + +**macOS AEC 升级**:本模式默认使用 `aether/vpio_capture`(基于 macOS VPIO 的系统级回声消除),让 TTS 外放时 ASR 仍能干净地收人声,真正实现全双工可打断。 + +**Fallback**:若在非 macOS 或 PyObjC 不可用的环境,把 `bringup_apps` 里的 `aether/vpio_capture` 改回 `sensors/audio_capture`(miniaudio 采集,依赖 listener 三重门控防回声)。 diff --git a/.moss_ws/src/MOSS/modes/aether/__init__.py b/.moss_ws/src/MOSS/modes/aether/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/.moss_ws/src/MOSS/modes/aether/channels.py b/.moss_ws/src/MOSS/modes/aether/channels.py new file mode 100644 index 00000000..a8139718 --- /dev/null +++ b/.moss_ws/src/MOSS/modes/aether/channels.py @@ -0,0 +1,17 @@ +# Aether mode 的 main channel — 复用 default 的 speech + apps + terminal。 +from ghoshell_moss import new_default_shell_main_channel +from ghoshell_moss.channels.app_store_channel import AppStoreChannel +from ghoshell_moss.channels.terminal_channel import new_terminal_channel +from ghoshell_moss.core.speech import SpeechChannelModule + +main = new_default_shell_main_channel() + +# Speech channel(TTS 能力)—— voice demo 要把普通短句也送进 TTS。 +# 不依赖模型稳定生成 ,否则 prompt 收紧为 plain text 后会只显示不播放。 +main.with_module(SpeechChannelModule(register_content=True)) + +# app store + terminal +main.import_channels( + AppStoreChannel(name='apps'), + new_terminal_channel(name='bash'), +) diff --git a/.moss_ws/src/MOSS/modes/aether/configs.py b/.moss_ws/src/MOSS/modes/aether/configs.py new file mode 100644 index 00000000..e7c52eca --- /dev/null +++ b/.moss_ws/src/MOSS/modes/aether/configs.py @@ -0,0 +1 @@ +from MOSS.manifests.configs import * # noqa: F403 diff --git a/.moss_ws/src/MOSS/modes/aether/contracts.py b/.moss_ws/src/MOSS/modes/aether/contracts.py new file mode 100644 index 00000000..10821cc8 --- /dev/null +++ b/.moss_ws/src/MOSS/modes/aether/contracts.py @@ -0,0 +1 @@ +# Mode 专属契约 — 在此声明 mode 范围内有效的 contract 绑定。 diff --git a/.moss_ws/src/MOSS/modes/aether/nuclei.py b/.moss_ws/src/MOSS/modes/aether/nuclei.py new file mode 100644 index 00000000..25e5206e --- /dev/null +++ b/.moss_ws/src/MOSS/modes/aether/nuclei.py @@ -0,0 +1,6 @@ +# 感知核声明 — 继承全局 + audio_nucleus(处理 listener 的 SPEECH_FINAL 信号)。 +from MOSS.manifests.nuclei import * # noqa: F403 + +from ghoshell_moss.core.mindflow.audio_nucleus import AudioNucleusMeta + +audio_nucleus_factory = AudioNucleusMeta() diff --git a/.moss_ws/src/MOSS/modes/aether/providers.py b/.moss_ws/src/MOSS/modes/aether/providers.py new file mode 100644 index 00000000..c5a29bf5 --- /dev/null +++ b/.moss_ws/src/MOSS/modes/aether/providers.py @@ -0,0 +1 @@ +from MOSS.manifests.providers import * # noqa: F403 diff --git a/.moss_ws/src/MOSS/modes/aether/resources.py b/.moss_ws/src/MOSS/modes/aether/resources.py new file mode 100644 index 00000000..1459f79f --- /dev/null +++ b/.moss_ws/src/MOSS/modes/aether/resources.py @@ -0,0 +1 @@ +from MOSS.manifests.resources import * # noqa: F403 diff --git a/.moss_ws/src/MOSS/modes/aether/topics.py b/.moss_ws/src/MOSS/modes/aether/topics.py new file mode 100644 index 00000000..21cef529 --- /dev/null +++ b/.moss_ws/src/MOSS/modes/aether/topics.py @@ -0,0 +1 @@ +from MOSS.manifests.topics import * # noqa: F403 diff --git a/.moss_ws/src/MOSS/modes/listener/MODE.md b/.moss_ws/src/MOSS/modes/listener/MODE.md index db26538d..23d00720 100644 --- a/.moss_ws/src/MOSS/modes/listener/MODE.md +++ b/.moss_ws/src/MOSS/modes/listener/MODE.md @@ -2,10 +2,11 @@ apps: - sensors/* - tools/* +- aether/* bringup_apps: - sensors/audio_capture -- sensors/listener +- aether/listener ctml_version: '' description: Audio capture + ASR listener mode name: listener ---- \ No newline at end of file +--- diff --git a/.moss_ws/src/MOSS/modes/show/MODE.md b/.moss_ws/src/MOSS/modes/show/MODE.md index 4fefa57f..8053dfc7 100644 --- a/.moss_ws/src/MOSS/modes/show/MODE.md +++ b/.moss_ws/src/MOSS/modes/show/MODE.md @@ -1,9 +1,9 @@ --- -apps: ["browsers/*", "games/*", "tools/*", "ui/*", "sensors/*"] +apps: ["browsers/*", "games/*", "tools/*", "ui/*", "sensors/*", "aether/*"] bringup_apps: [ "tools/screen_capture", "ui/reflex", - "sensors/audio_capture", "sensors/listener", + "sensors/audio_capture", "aether/listener", "games/ai_eye", "sensors/vision", ] ctml_version: '' diff --git a/src/ghoshell_moss/core/mindflow/audio_nucleus.py b/src/ghoshell_moss/core/mindflow/audio_nucleus.py index a5be9787..6f68e52b 100644 --- a/src/ghoshell_moss/core/mindflow/audio_nucleus.py +++ b/src/ghoshell_moss/core/mindflow/audio_nucleus.py @@ -47,11 +47,11 @@ async def _process_signal(self, signal: Signal) -> None: def _rebuild_impulse(self) -> Impulse | None: impulse = super()._rebuild_impulse() - if impulse is not None and impulse.complete: - # 首包打断: incomplete impulse preempts attention, claims it - # via complete=False. Complete (FINAL) delivers content to - # the occupied attention without re-interrupting. - impulse.interrupt = True + if impulse is not None: + # 普通语音不是急停。让 wake word / InterruptNucleus 负责真正的 + # shell.clear(),否则每个 ASR final 都会在响应前清掉 TTS/解释器, + # 破坏 Aether 的全双工 speak+listen 语义。 + impulse.interrupt = False return impulse def suppress(self, suppress_by: Impulse) -> None: diff --git a/src/ghoshell_moss/core/speech/stream_tts_speech.py b/src/ghoshell_moss/core/speech/stream_tts_speech.py index 1f1aefbe..a2754249 100644 --- a/src/ghoshell_moss/core/speech/stream_tts_speech.py +++ b/src/ghoshell_moss/core/speech/stream_tts_speech.py @@ -203,6 +203,14 @@ async def clear(self) -> list[str]: self.logger.info("%s clear", self._log_prefix) outputted = self._outputted.copy() self._outputted.clear() + results = await asyncio.gather( + self._tts.clear(), + self._player.clear(), + return_exceptions=True, + ) + for result in results: + if isinstance(result, Exception): + self.logger.error("%s clear backend failed: %s", self._log_prefix, result) return outputted async def start(self) -> None: diff --git a/src/ghoshell_moss/ghosts/atom/_adapter.py b/src/ghoshell_moss/ghosts/atom/_adapter.py index b9c65a58..899c3e0b 100644 --- a/src/ghoshell_moss/ghosts/atom/_adapter.py +++ b/src/ghoshell_moss/ghosts/atom/_adapter.py @@ -10,7 +10,7 @@ from pydantic_ai.messages import ModelRequest, ModelResponse from pydantic_ai import UserContent, TextContent, ImageUrl -__all__ = ["messages_to_parts", "moment_to_request"] +__all__ = ["messages_to_parts", "moment_to_request", "moment_to_user_text"] def messages_to_parts(messages: Iterable[Message]) -> list[UserContent]: @@ -25,6 +25,22 @@ def messages_to_parts(messages: Iterable[Message]) -> list[UserContent]: return parts +def moment_to_user_text(moment) -> str: + """将 Moment 的所有请求消息合并为单个纯文本字符串. + + 避免 pydantic_ai OpenAIModel streaming 与包含 XML 的多个 TextContent + user_prompt 不兼容 (AssertionError: Expected code to be unreachable). + perspectives (mindflow 状态) 与用户输入合并为一段文本传给模型. + """ + chunks: list[str] = [] + for msg in moment.as_request_messages(): + for content in msg.as_contents(with_meta=True): + if text := Text.from_content(content): + if text.text and text.text.strip(): + chunks.append(text.text) + return "\n\n".join(chunks) + + def moment_to_request(moment) -> ModelRequest: """将 Moment 转为 pydantic AI ModelRequest.""" from ghoshell_moss.core.blueprint.mindflow import Moment as _Moment diff --git a/src/ghoshell_moss/ghosts/atom/_meta.py b/src/ghoshell_moss/ghosts/atom/_meta.py index 83edbcd3..758f50fb 100644 --- a/src/ghoshell_moss/ghosts/atom/_meta.py +++ b/src/ghoshell_moss/ghosts/atom/_meta.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from typing import Callable, TYPE_CHECKING +from typing import Any, Callable, TYPE_CHECKING from anthropic.types.beta import BetaThinkingConfigDisabledParam from ghoshell_container import IoCContainer @@ -9,9 +9,10 @@ from ghoshell_moss.contracts import SystemPrompter from pydantic_ai import Agent, RunContext from pydantic_ai.models import Model -from pydantic_ai.providers import Provider from pydantic_ai.models.anthropic import AnthropicModel, AnthropicModelSettings +from pydantic_ai.models.openai import OpenAIModel, OpenAIModelSettings from pydantic_ai.providers.anthropic import AnthropicProvider +from pydantic_ai.providers.openai import OpenAIProvider if TYPE_CHECKING: from ._runtime import Atom @@ -38,7 +39,7 @@ def __init__( soul_path: str | Path | None = None, soul_content: str | None = None, model: Model | None = None, - provider: Provider | None = None, + provider: Any = None, on_agent_build: Callable[[Agent[IoCContainer]], None] | None = None, nuclei_metas: list[NucleusMeta] | None = None, ): @@ -119,20 +120,49 @@ def build_agent(self, container: IoCContainer) -> Agent[IoCContainer]: self._load_soul(ghost_workspace) model = self._model if model is None: - model_name = os.environ.get("ANTHROPIC_MODEL") - if not model_name: - raise RuntimeError( - "ANTHROPIC_MODEL env var not set. " - "Set it or pass model= explicitly." + llm_provider = os.environ.get("MOSS_LLM_PROVIDER", "openai").lower() + if llm_provider == "anthropic": + model_name = os.environ.get("ANTHROPIC_MODEL") + if not model_name: + raise RuntimeError( + "ANTHROPIC_MODEL env var not set. " + "Set it or pass model= explicitly." + ) + model = AnthropicModel( + model_name=model_name, + provider=self._provider or AnthropicProvider(), + # disable extended thinking by default; enable via model= param if needed + settings=AnthropicModelSettings( + anthropic_thinking=BetaThinkingConfigDisabledParam(type="disabled"), + timeout=120.0, + ), + ) + else: + # OpenAI 兼容 (DeepSeek / 通义 / Moonshot OpenAI 等) + model_name = os.environ.get("OPENAI_MODEL") + if not model_name: + raise RuntimeError( + "OPENAI_MODEL env var not set. " + "Set it or pass model= explicitly." + ) + base_url = os.environ.get("OPENAI_BASE_URL") + api_key = os.environ.get("OPENAI_API_KEY") + if not base_url or not api_key: + raise RuntimeError( + "OPENAI_BASE_URL / OPENAI_API_KEY env var not set." + ) + model_settings = OpenAIModelSettings(timeout=60.0) + if "deepseek" in model_name.lower() or "deepseek" in base_url.lower(): + # DeepSeek V4 defaults to thinking mode. Aether voice mode + # needs low-latency non-thinking responses, while streaming + # remains handled by Agent.run_stream(). + model_settings["extra_body"] = {"thinking": {"type": "disabled"}} + model_settings["openai_continuous_usage_stats"] = False + model = OpenAIModel( + model_name=model_name, + provider=OpenAIProvider(base_url=base_url, api_key=api_key), + settings=model_settings, ) - model = AnthropicModel( - model_name=model_name, - provider=self._provider or AnthropicProvider(), - # disable extended thinking by default; enable via model= param if needed - settings=AnthropicModelSettings( - anthropic_thinking=BetaThinkingConfigDisabledParam(type="disabled"), - ), - ) agent = Agent[IoCContainer]( name=self._name, diff --git a/src/ghoshell_moss/ghosts/atom/_runtime.py b/src/ghoshell_moss/ghosts/atom/_runtime.py index 8bf175cf..f4a77f86 100644 --- a/src/ghoshell_moss/ghosts/atom/_runtime.py +++ b/src/ghoshell_moss/ghosts/atom/_runtime.py @@ -77,12 +77,27 @@ def inspect_context(self) -> dict: async def articulate(self, articulator: Articulator) -> AsyncIterator[str]: moment = articulator.moment - request = self.to_model_request(moment) - history = self.model_history() - + # /reset 命令:清空对话历史 + user_text = "" + for p in moment.percepts: + content = getattr(p, "content", None) + if isinstance(content, str) and content.strip(): + user_text = content.strip() + break + if user_text == "/reset": + self._history.clear() + self._logger.info("[Atom] context reset by /reset command") + yield "上下文已重置,我们重新开始。" + return + # 合并 moment 所有消息为单字符串,避免 pydantic_ai OpenAIModel streaming + # 与多个 TextContent (含 mindflow XML) 不兼容 + from ._adapter import moment_to_user_text + user_prompt = moment_to_user_text(moment) + + # 注:语音对话为短上下文场景,不传 message_history 避免 pydantic_ai + # OpenAIModel streaming 与历史 TextContent 不兼容 (同 user_prompt 问题) async with self._agent.run_stream( - user_prompt=request.parts, - message_history=history, + user_prompt=user_prompt, deps=self._container, ) as stream: async for text in stream.stream_text(delta=True): diff --git a/src/ghoshell_moss/host/app_store.py b/src/ghoshell_moss/host/app_store.py index 8d507d84..5d24c5ee 100644 --- a/src/ghoshell_moss/host/app_store.py +++ b/src/ghoshell_moss/host/app_store.py @@ -279,7 +279,13 @@ async def start_app(self, app_fullname: str, argument: str = '') -> str: self._managed_apps_with_fullname.add(app_fullname) if not started_in_add: - r2 = await self._call_circus({"command": "start", "name": app.address}) + r2 = None + for attempt in range(10): + r2 = await self._call_circus({"command": "start", "name": app.address}) + reason = str(r2.get("reason", "")) + if r2.get('status') != "error" or "arbiter is already running" not in reason: + break + await asyncio.sleep(0.2 * (attempt + 1)) if r2.get('status') == "error": self._logger.error( "%s failed to start app %s on error: %s", @@ -393,12 +399,12 @@ async def __aenter__(self) -> Self: self._polling_task = asyncio.create_task(self._polling_loop()) # 3. Bring-up - bringup_apps_cors = [] if self._bringup: for app_info in self.match_apps(self.list_apps(), self._bringup): - bringup_apps_cors.append(self.start_app(app_info.fullname)) - if len(bringup_apps_cors) > 0: - _ = await asyncio.gather(*bringup_apps_cors, return_exceptions=False) + # Circus arbiter still serializes watcher start internally. + # Sequential bringup avoids transient "arbiter is already running" + # failures when multiple apps are launched for an interactive mode. + await self.start_app(app_info.fullname) return self diff --git a/src/ghoshell_moss/host/ghost_runtime.py b/src/ghoshell_moss/host/ghost_runtime.py index 6d7f14c4..ac566659 100644 --- a/src/ghoshell_moss/host/ghost_runtime.py +++ b/src/ghoshell_moss/host/ghost_runtime.py @@ -14,6 +14,7 @@ from ghoshell_moss.core.concepts.command import CommandTask from ghoshell_container import Provider, IoCContainer from ghoshell_moss.message import Message +from ghoshell_moss.topics.audio import AudioRuntimeTopic import pathlib __all__ = ["GhostRuntimeImpl"] @@ -228,6 +229,7 @@ def _route_signal_to_mindflow(signal: Signal): matrix.create_task(self._main_loop(), stop_matrix_on_error=True) matrix.create_task(self._articulate_loop(), stop_matrix_on_error=True) matrix.create_task(self._action_loop(), stop_matrix_on_error=True) + matrix.create_task(self._audio_interrupt_loop(), stop_matrix_on_error=False) # 等待应该发生在循环外侧. await self._mindflow.wait_started() # ignore any signals before started @@ -235,6 +237,34 @@ def _route_signal_to_mindflow(signal: Signal): # ── 三循环 ──────────────────────────────────── + async def _audio_interrupt_loop(self) -> None: + """Out-of-band audio emergency stop. + + Mindflow interrupt remains the semantic path, but voice barge-in must + stop buffered TTS immediately. Waiting for the current attention/action + to observe abort can leave tens of seconds of audio already buffered in + the player. + """ + audio_win = self.moss.matrix.session.topics.create_window_for(AudioRuntimeTopic, max_size=16) + last_started_at = 0.0 + while self.moss.is_running(): + for topic in reversed(list(audio_win.values())): + if getattr(topic, "device_name", "") != "interrupt" or not topic.running: + continue + started_at = float(getattr(topic, "started_at", 0.0) or 0.0) + if started_at > last_started_at: + last_started_at = started_at + self.moss.logger.info( + "%s audio interrupt topic received, clearing shell immediately", + self._log_prefix, + ) + try: + await self.moss.shell.clear() + except Exception: + self.moss.logger.exception("%s audio interrupt clear failed", self._log_prefix) + break + await asyncio.sleep(0.03) + def _moss_dynamic_messages(self) -> list[Message]: shell = self._moss_runtime.shell # 闭包在 shell running 时才取,shell 未启动时返回空列表. diff --git a/src/ghoshell_moss/host/speech/volcengine_asr/config.py b/src/ghoshell_moss/host/speech/volcengine_asr/config.py index 4fcc21dc..3f7a6fa2 100644 --- a/src/ghoshell_moss/host/speech/volcengine_asr/config.py +++ b/src/ghoshell_moss/host/speech/volcengine_asr/config.py @@ -10,11 +10,16 @@ class VolcengineASRConfig(BaseModel): 环境变量: VOLCENGINE_BM_ASR_APPID — appid VOLCENGINE_BM_ASR_TOKEN — access token + VOLCENGINE_BM_ASR_API_KEY — new-console API key, optional + VOLCENGINE_BM_ASR_URL — websocket url override + VOLCENGINE_BM_ASR_RESOURCE_ID — resource id override + VOLCENGINE_BM_ASR_MODEL_NAME — model name override """ appid: str = Field("$VOLCENGINE_BM_ASR_APPID", description="火山引擎 asr 的 appid") token: str = Field("$VOLCENGINE_BM_ASR_TOKEN", description="火山引擎的 asr app token") - url: str = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel" + api_key: str = Field("$VOLCENGINE_BM_ASR_API_KEY", description="新版控制台 API Key") + url: str = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async" sample_rate: int = Field(16000, description="默认的采样率") bits: int = Field(16) channel: int = Field(1) @@ -28,6 +33,14 @@ class VolcengineASRConfig(BaseModel): True, description="语义顺滑,删除停顿词、语气词、语义重复词等。", ) + force_to_speech_time: int = Field( + 1000, + description="音频时长超过该值后才尝试判停。单位 ms,需配合 end_window_size。", + ) + audio_packet_ms: int = Field( + 200, + description="发送到火山 ASR 的音频包时长。官方建议 100-200ms,双向流式优化版推荐 200ms。", + ) resource_id: str = Field("volc.bigasr.sauc.duration") def resolve_env(self) -> Self: @@ -35,4 +48,15 @@ def resolve_env(self) -> Self: self.appid = os.environ.get(self.appid[1:], self.appid) if self.token.startswith("$"): self.token = os.environ.get(self.token[1:], self.token) + if self.api_key.startswith("$"): + self.api_key = os.environ.get(self.api_key[1:], "") + self.url = os.environ.get("VOLCENGINE_BM_ASR_URL", self.url) + self.resource_id = os.environ.get("VOLCENGINE_BM_ASR_RESOURCE_ID", self.resource_id) + self.model_name = os.environ.get("VOLCENGINE_BM_ASR_MODEL_NAME", self.model_name) + packet_ms = os.environ.get("VOLCENGINE_BM_ASR_AUDIO_PACKET_MS") + if packet_ms: + try: + self.audio_packet_ms = max(20, int(packet_ms)) + except ValueError: + pass return self diff --git a/src/ghoshell_moss/host/speech/volcengine_asr/protocol.py b/src/ghoshell_moss/host/speech/volcengine_asr/protocol.py index 9cafdecb..9de2233b 100644 --- a/src/ghoshell_moss/host/speech/volcengine_asr/protocol.py +++ b/src/ghoshell_moss/host/speech/volcengine_asr/protocol.py @@ -54,8 +54,6 @@ def int_to_bytes(value: int) -> bytes: @staticmethod def gzip_compress(data: bytes) -> bytes: - if not data: - return b"" buf = io.BytesIO() with gzip.GzipFile(fileobj=buf, mode="wb") as f: f.write(data) @@ -74,15 +72,26 @@ async def connect(config: VolcengineASRConfig, connection_id: str = "") -> webso config = config.resolve_env() connection_id = connection_id or uuid() headers = { - "X-Api-App-Key": config.appid, - "X-Api-Access-Key": config.token, "X-Api-Resource-Id": config.resource_id, "X-Api-Connect-Id": connection_id, } - return await websockets.connect(config.url, additional_headers=headers) + if config.api_key: + headers["X-Api-Key"] = config.api_key + else: + headers["X-Api-App-Key"] = config.appid + headers["X-Api-Access-Key"] = config.token + return await websockets.connect(config.url, additional_headers=headers, proxy=None) def create_init_request(uid: str, config: VolcengineASRConfig) -> tuple[bytes, int]: + request = { + "model_name": config.model_name, + "enable_punc": config.enable_punc, + "enable_ddc": config.enable_ddc, + "end_window_size": config.end_window_size, + "force_to_speech_time": config.force_to_speech_time, + "show_utterances": True, + } payload = { "user": {"uid": uid}, "audio": { @@ -92,13 +101,7 @@ def create_init_request(uid: str, config: VolcengineASRConfig) -> tuple[bytes, i "channel": config.channel, "codec": "raw", }, - "request": { - "model_name": config.model_name, - "enable_punc": config.enable_punc, - "end_window_size": config.end_window_size, - "force_to_speech_time": 1000, - "show_utterances": True, - }, + "request": request, } payload_str = json.dumps(payload) payload_bytes = _Protocol.gzip_compress(payload_str.encode("utf-8")) @@ -139,9 +142,10 @@ async def send_init_request(ws: websockets.ClientConnection, config: VolcengineA await ws.send(message) -async def send_audio(ws: websockets.ClientConnection, audio: bytes, seq: int, is_last: bool = False) -> None: - message, _ = create_audio_only_request(audio, seq, is_last) +async def send_audio(ws: websockets.ClientConnection, audio: bytes, seq: int, is_last: bool = False) -> int: + message, seq = create_audio_only_request(audio, seq, is_last) await ws.send(message) + return seq class ResponseMessageType(str, enum.Enum): @@ -163,9 +167,57 @@ def parse_response(data: bytes) -> Response: message_type_specific_flags = data[1] & 0x0F message_compression = data[2] & 0x0F - sequence = struct.unpack(">i", data[4:8])[0] - payload_size = struct.unpack(">I", data[8:12])[0] - payload = data[12:12 + payload_size] if len(data) >= 12 + payload_size else data[12:] + if message_type == _Protocol.SERVER_ERROR_RESPONSE: + sequence = 0 + offset = 4 + if message_type_specific_flags in ( + _Protocol.POS_SEQUENCE, + _Protocol.NEG_SEQUENCE, + _Protocol.NEG_WITH_SEQUENCE, + ) and len(data) >= 12: + sequence = struct.unpack(">i", data[offset:offset + 4])[0] + offset += 4 + payload_size = struct.unpack(">I", data[offset:offset + 4])[0] + offset += 4 + payload = data[offset:offset + payload_size] if len(data) >= offset + payload_size else data[offset:] + else: + # Volcengine docs define error frames without sequence/outer size: + # Header + Error code (4B) + Error message size (4B) + Error message. + payload = data[offset:] + + if len(payload) >= 8: + code = int.from_bytes(payload[:4], "big", signed=False) + msg_size = int.from_bytes(payload[4:8], "big", signed=False) + payload_msg = payload[8:8 + msg_size] if msg_size and len(payload) >= 8 + msg_size else payload[8:] + if message_compression == _Protocol.GZIP: + payload_msg = gzip.decompress(payload_msg) + return Response( + sequence=sequence, + message_type=ResponseMessageType.server_error, + error_code=code, + is_last=bool(message_type_specific_flags & 0x02), + payload=payload_msg.decode("utf-8", errors="replace"), + ) + return Response( + sequence=sequence, + message_type=ResponseMessageType.server_error, + error_code=-1, + is_last=False, + payload=f"malformed error frame: {data[:32].hex()}", + ) + + offset = 4 + sequence = 0 + if message_type_specific_flags in ( + _Protocol.POS_SEQUENCE, + _Protocol.NEG_SEQUENCE, + _Protocol.NEG_WITH_SEQUENCE, + ): + sequence = struct.unpack(">i", data[offset:offset + 4])[0] + offset += 4 + payload_size = struct.unpack(">I", data[offset:offset + 4])[0] + offset += 4 + payload = data[offset:offset + payload_size] if len(data) >= offset + payload_size else data[offset:] is_last_package = bool(message_type_specific_flags & 0x02) @@ -190,18 +242,6 @@ def parse_response(data: bytes) -> Response: is_last=False, payload="", ) - elif message_type == _Protocol.SERVER_ERROR_RESPONSE: - code = int.from_bytes(payload[:4], "big", signed=False) - payload_msg = payload[8:] - if message_compression == _Protocol.GZIP: - payload_msg = gzip.decompress(payload_msg) - return Response( - sequence=sequence, - message_type=ResponseMessageType.server_error, - error_code=code, - is_last=is_last_package, - payload=payload_msg.decode("utf-8", errors="replace"), - ) else: return Response( sequence=-1, diff --git a/src/ghoshell_moss/host/speech/volcengine_asr/recognizer.py b/src/ghoshell_moss/host/speech/volcengine_asr/recognizer.py index 86d9cac0..907d53a4 100644 --- a/src/ghoshell_moss/host/speech/volcengine_asr/recognizer.py +++ b/src/ghoshell_moss/host/speech/volcengine_asr/recognizer.py @@ -1,6 +1,7 @@ import asyncio import json import logging +import time from typing import AsyncIterable, Optional import numpy as np @@ -22,6 +23,8 @@ __all__ = ["VolcengineASR"] +_ASR_ERROR_PREFIX = "__VOLCENGINE_ASR_ERROR__:" + class VolcengineASR(ASR): """火山引擎大模型 ASR 实现。每次 recognize 独立建立 WebSocket 连接。""" @@ -55,7 +58,26 @@ async def recognize( result_queue: asyncio.Queue[Optional[ASRResult]] = asyncio.Queue() async with await connect(self._config, connection_id) as ws: + resolved = self._config.resolve_env() + self._logger.info( + "%s websocket connected, connection=%s url=%s resource=%s model=%s sample_rate=%s auth=%s logid=%s", + self._log_prefix, + connection_id, + resolved.url, + resolved.resource_id, + resolved.model_name, + resolved.sample_rate, + "api_key" if resolved.api_key else "app_token", + self._response_header(ws, "X-Tt-Logid") or "-", + ) await send_init_request(ws, self._config, connection_id) + self._logger.info( + "%s init request sent, connection=%s end_window=%sms force_to_speech=%sms", + self._log_prefix, + connection_id, + resolved.end_window_size, + resolved.force_to_speech_time, + ) send_task = asyncio.create_task( self._send_loop(ws, audio_chunks, connection_id) @@ -97,19 +119,59 @@ async def _send_loop( connection_id: str, ) -> None: seq = 1 + sent_packets = 0 + sent_bytes = 0 + last_log_at = time.monotonic() + resolved = self._config.resolve_env() + samples_per_packet = max( + 1, + int(resolved.sample_rate * resolved.audio_packet_ms / 1000) * max(1, resolved.channel), + ) + pending = np.array([], dtype=np.int16) + + async def _send_pcm_packet(pcm: np.ndarray) -> None: + nonlocal seq, sent_packets, sent_bytes, last_log_at + if pcm.size == 0: + return + compressed = nparray_to_bytes(pcm.astype(np.int16, copy=False)) + seq = await send_audio(ws, compressed, seq, is_last=False) + sent_packets += 1 + sent_bytes += int(pcm.nbytes) + now = time.monotonic() + if sent_packets == 1 or sent_packets % 25 == 0 or now - last_log_at >= 5.0: + self._logger.info( + "%s audio sent, connection=%s packets=%d pcm_bytes=%d packet_ms=%d last_shape=%s last_dtype=%s", + self._log_prefix, + connection_id, + sent_packets, + sent_bytes, + resolved.audio_packet_ms, + tuple(pcm.shape), + str(pcm.dtype), + ) + last_log_at = now + try: async for audio in audio_chunks: - compressed = nparray_to_bytes(audio) - await send_audio(ws, compressed, seq, is_last=False) - seq += 1 + pcm = np.asarray(audio, dtype=np.int16).reshape(-1) + if pcm.size == 0: + continue + pending = np.concatenate((pending, pcm)) + while pending.size >= samples_per_packet: + packet = pending[:samples_per_packet] + pending = pending[samples_per_packet:] + await _send_pcm_packet(packet) # 音频流结束,发送尾包 + if pending.size: + await _send_pcm_packet(pending) self._logger.debug( "%s sending final audio packet, connection=%s", self._log_prefix, connection_id, ) - await send_audio(ws, b"", seq, is_last=True) + final_packet = nparray_to_bytes(np.array([], dtype=np.int16)) + seq = await send_audio(ws, final_packet, seq, is_last=True) except asyncio.CancelledError: raise except Exception as e: @@ -126,6 +188,7 @@ async def _receive_loop( result_queue: asyncio.Queue[Optional[ASRResult]], connection_id: str, ) -> None: + received_count = 0 try: while True: try: @@ -137,15 +200,32 @@ async def _receive_loop( continue response = parse_response(data) + received_count += 1 + if received_count <= 3 or response.message_type == ResponseMessageType.server_error: + self._logger.info( + "%s response received, connection=%s count=%d type=%s sequence=%s is_last=%s payload_len=%d", + self._log_prefix, + connection_id, + received_count, + response.message_type, + response.sequence, + response.is_last, + len(response.payload or ""), + ) if response.message_type == ResponseMessageType.server_error: + message = (response.payload or "").strip() self._logger.error( - "%s server error: %s, connection=%s", + "%s server error: code=%s message=%s connection=%s", self._log_prefix, response.error_code, + message[:500], connection_id, ) - await result_queue.put(ASRResult(text="", is_final=True)) + await result_queue.put(ASRResult( + text=f"{_ASR_ERROR_PREFIX}{response.error_code}|{message}", + is_final=True, + )) break elif response.message_type == ResponseMessageType.server_ack: @@ -189,6 +269,17 @@ def _parse_result(self, payload: str) -> Optional[ASRResult]: ) return None + @staticmethod + def _response_header(ws: websockets.ClientConnection, name: str) -> str: + response = getattr(ws, "response", None) + headers = getattr(response, "headers", None) + if not headers: + return "" + try: + return str(headers.get(name, "")) + except Exception: + return "" + async def close(self) -> None: self._closed = True self._logger.info("%s closed", self._log_prefix) diff --git a/src/ghoshell_moss/host/speech/volcengine_tts/tts.py b/src/ghoshell_moss/host/speech/volcengine_tts/tts.py index c8751a7b..a7169533 100644 --- a/src/ghoshell_moss/host/speech/volcengine_tts/tts.py +++ b/src/ghoshell_moss/host/speech/volcengine_tts/tts.py @@ -309,6 +309,7 @@ class VolcengineTTSConf(BaseModel): resource_id: str = Field(default="seed-tts-2.0", description="官方的默认资源") sample_rate: int = Field(default=44100, description="生成音频的采样率要求.") audio_format: Literal["pcm"] = Field(default="pcm", description="默认可用的数据格式") + require_usage_tokens_return: bool = Field(default=False, description="返回火山 TTS 计费字符统计") disconnect_on_idle: int = Field( default=300, @@ -350,12 +351,17 @@ def default_speaker_conf(self) -> SpeakerConf: def gen_header(self, *, connection_id: str = "", resource_id: Optional[str] = None) -> _Head: connection_id = connection_id or unique_id() app_key = self.unwrap_env(self.app_key) + resolved_resource_id = ( + os.environ.get("VOLCENGINE_STREAM_TTS_RESOURCE_ID") + or resource_id + or self.resource_id + ) # 旧版鉴权 header 始终发送(兼容新旧控制台) ws_header = { "X-Api-App-Key": app_key, "X-Api-App-Id": app_key, "X-Api-Access-Key": self.unwrap_env(self.access_token), - "X-Api-Resource-Id": resource_id or self.resource_id, + "X-Api-Resource-Id": resolved_resource_id, "X-Api-Request-Id": unique_id(), "X-Api-Connect-Id": connection_id, } @@ -363,6 +369,8 @@ def gen_header(self, *, connection_id: str = "", resource_id: Optional[str] = No api_key = self.unwrap_env(self.api_key) if api_key: ws_header["X-Api-Key"] = api_key + if self.require_usage_tokens_return or os.environ.get("VOLCENGINE_STREAM_TTS_REQUIRE_USAGE") == "1": + ws_header["X-Control-Require-Usage-Tokens-Return"] = "*" return ws_header def to_session(self, speaker: SpeakerConf) -> Session: @@ -381,7 +389,7 @@ def to_session(self, speaker: SpeakerConf) -> Session: emotion=speaker.voice.emotion, ), speaker=speaker.tone, - model=self.model, + model=os.environ.get("VOLCENGINE_STREAM_TTS_MODEL", self.model or "") or None, additions=additions, ), ) @@ -702,10 +710,10 @@ async def _start_consuming_batch_loop(self, batch: VolcengineTTSBatch): resource_id = speaker.resource_id or self._conf.resource_id connection_id = unique_id() header = self._conf.gen_header(connection_id=connection_id, resource_id=resource_id) - url = self._conf.url + url = os.environ.get("VOLCENGINE_STREAM_TTS_URL", self._conf.url) # 创建初始连接. self.logger.info("%s prepare to connect to %s with header %s", self._log_prefix, url, header) - async with connect(url, additional_headers=header) as ws: + async with connect(url, additional_headers=header, proxy=None) as ws: # 建连确认. await start_connection(ws) self.logger.debug("%s start connection %s", self._log_prefix, connection_id) diff --git a/tests/ghoshell_moss/host/speech/volcengine_asr/test_protocol.py b/tests/ghoshell_moss/host/speech/volcengine_asr/test_protocol.py index 5e3b09f7..c3a08132 100644 --- a/tests/ghoshell_moss/host/speech/volcengine_asr/test_protocol.py +++ b/tests/ghoshell_moss/host/speech/volcengine_asr/test_protocol.py @@ -70,3 +70,18 @@ def test_parse_server_error(self): resp = parse_response(data) assert resp.message_type.value == "server_error" assert resp.error_code == 1234 + + def test_parse_server_error_without_sequence(self): + header = _Protocol.get_header( + _Protocol.SERVER_ERROR_RESPONSE, + _Protocol.NO_SEQUENCE, + _Protocol.JSON, + _Protocol.NO_COMPRESSION, + ) + message = b"server busy" + data = header + _Protocol.int_to_bytes(106) + _Protocol.int_to_bytes(len(message)) + message + resp = parse_response(data) + assert resp.message_type.value == "server_error" + assert resp.sequence == 0 + assert resp.error_code == 106 + assert resp.payload == "server busy" From 6e699f8e0e6b826b4204b370d97ac832347aaba3 Mon Sep 17 00:00:00 2001 From: LpLegend <2422019509@qq.com> Date: Fri, 10 Jul 2026 00:30:23 +0800 Subject: [PATCH 2/3] fix: restore listener canonical path coding by gpt-5 - keep sensors/listener as the shared ASR sensor app - update Aether mode and README to reuse sensors/listener - restore global LLM config, Echo soul, listener/show/g1 references Tests: - .venv/bin/python -m py_compile .moss_ws/apps/aether/core/main.py .moss_ws/apps/sensors/listener/main.py .moss_ws/apps/aether/vpio_capture/main.py .moss_ws/apps/aether/vpio_capture/vpio_capture.py - .venv/bin/python -m pytest tests/ghoshell_moss/host/speech/volcengine_asr/test_protocol.py - .venv/bin/moss --ai --mode aether apps list - .venv/bin/moss --ai --mode listener apps list via Codex --- .moss_ws/apps/aether/README.md | 42 ++++++------ .moss_ws/apps/aether/listener/APP.md | 15 ----- .moss_ws/apps/bodies/g1_sim/APP.md | 4 +- .moss_ws/apps/bodies/g1_sim/README.md | 2 +- .../{aether => sensors}/listener/.env.example | 0 .../{aether => sensors}/listener/.gitignore | 0 .moss_ws/apps/sensors/listener/APP.md | 16 +++++ .moss_ws/apps/sensors/listener/CLAUDE.md | 66 +++++++++++++++++++ .../apps/{aether => sensors}/listener/main.py | 4 +- .../listener/pyproject.toml | 0 .moss_ws/configs/llm.yml | 22 ++----- .moss_ws/ghosts/echo/soul.md | 11 ---- .moss_ws/src/MOSS/modes/aether/MODE.md | 4 +- .moss_ws/src/MOSS/modes/listener/MODE.md | 5 +- .moss_ws/src/MOSS/modes/show/MODE.md | 4 +- 15 files changed, 120 insertions(+), 75 deletions(-) delete mode 100644 .moss_ws/apps/aether/listener/APP.md rename .moss_ws/apps/{aether => sensors}/listener/.env.example (100%) rename .moss_ws/apps/{aether => sensors}/listener/.gitignore (100%) create mode 100644 .moss_ws/apps/sensors/listener/APP.md create mode 100644 .moss_ws/apps/sensors/listener/CLAUDE.md rename .moss_ws/apps/{aether => sensors}/listener/main.py (99%) rename .moss_ws/apps/{aether => sensors}/listener/pyproject.toml (100%) diff --git a/.moss_ws/apps/aether/README.md b/.moss_ws/apps/aether/README.md index 19c7a2cf..841cc8f2 100644 --- a/.moss_ws/apps/aether/README.md +++ b/.moss_ws/apps/aether/README.md @@ -9,19 +9,23 @@ Aether 是 MOSS 的实时语音交互外壳。它把麦克风采集、ASR、Ghos ```text .moss_ws/apps/aether/ core/ 前端可视化和 WebSocket 状态聚合 - listener/ 音频到 ASR,再到 SpeechTopic / AudioSignal vpio_capture/ macOS VPIO 音频采集和系统级回声消除 ``` -三个 app 的 canonical address 是: +Aether 自己维护两个 app: ```text aether/core -aether/listener aether/vpio_capture ``` -旧地址 `ui/aether_core`、`sensors/listener`、`sensors/vpio_capture` 已不再作为 Aether 入口使用。 +ASR listener 仍然使用仓库原有的公共 sensor app: + +```text +sensors/listener +``` + +不要把 `sensors/listener` 挪进 Aether 目录。它同时服务普通 listener mode、show mode 和 Aether mode。 ## 一键启动 @@ -41,7 +45,7 @@ http://127.0.0.1:8765/ ```text aether/vpio_capture -aether/listener +sensors/listener aether/core ``` @@ -68,7 +72,7 @@ aether/core 只启动 ASR listener。排查时建议先用 manual 模式,避免连续 ASR 抢占调试过程: ```bash -LISTENER_ASR_MODE=manual .venv/bin/moss --ai --mode aether apps test aether/listener +LISTENER_ASR_MODE=manual .venv/bin/moss --ai --mode aether apps test sensors/listener ``` 只启动 macOS VPIO 采集: @@ -92,9 +96,9 @@ LISTENER_ASR_MODE=manual .venv/bin/moss --ai --mode aether apps test aether/list 它不负责 ASR、不负责判断用户意图、不负责调用 Ghost。 -### aether/listener +### sensors/listener -`aether/listener` 是语音识别层。 +`sensors/listener` 是仓库原有的语音识别层,Aether 只复用它,不拥有它。 它负责: @@ -129,7 +133,7 @@ LISTENER_ASR_MODE=manual .venv/bin/moss --ai --mode aether apps test aether/list 麦克风 -> aether/vpio_capture -> audio topic - -> aether/listener + -> sensors/listener -> SpeechTopic + AudioSignal -> MOSS Mindflow / Ghost -> TTS @@ -144,21 +148,21 @@ LISTENER_ASR_MODE=manual .venv/bin/moss --ai --mode aether apps test aether/list browser button / VAD -> aether/core WebSocket -> AudioRuntimeTopic(asr_control / interrupt) - -> aether/listener 或 MOSS host runtime + -> sensors/listener 或 MOSS host runtime ``` ## 关键 Topic | Topic | 发布者 | 消费者 | 作用 | | --- | --- | --- | --- | -| `SpeechTopic` | `aether/listener` | Ghost / `aether/core` | 用户一句完整语音文本 | -| `AudioSignal(SPEECH_STARTED)` | `aether/listener` | Mindflow | 用户已经开始说话 | -| `AudioSignal(SPEECH_FINAL)` | `aether/listener` | Mindflow | 用户一句话完成 | +| `SpeechTopic` | `sensors/listener` | Ghost / `aether/core` | 用户一句完整语音文本 | +| `AudioSignal(SPEECH_STARTED)` | `sensors/listener` | Mindflow | 用户已经开始说话 | +| `AudioSignal(SPEECH_FINAL)` | `sensors/listener` | Mindflow | 用户一句话完成 | | `AudioRuntimeTopic(device_name="vpio")` | `aether/vpio_capture` | `aether/core` | 采集状态和音量诊断 | -| `AudioRuntimeTopic(device_name="asr")` | `aether/listener` | `aether/core` | ASR running/partial/final/error/idle | -| `AudioRuntimeTopic(device_name="asr_control")` | `aether/core` | `aether/listener` | 连续/手动 ASR 控制 | +| `AudioRuntimeTopic(device_name="asr")` | `sensors/listener` | `aether/core` | ASR running/partial/final/error/idle | +| `AudioRuntimeTopic(device_name="asr_control")` | `aether/core` | `sensors/listener` | 连续/手动 ASR 控制 | | `AudioRuntimeTopic(device_name="speaker")` | TTS/player | `aether/core` | TTS 播放状态 | -| `AudioRuntimeTopic(device_name="interrupt")` | `aether/core` / `aether/listener` | `aether/core` / host runtime | 打断当前输出 | +| `AudioRuntimeTopic(device_name="interrupt")` | `aether/core` / `sensors/listener` | `aether/core` / host runtime | 打断当前输出 | ## 常见排错 @@ -188,7 +192,7 @@ curl -s http://127.0.0.1:8765/ ### 停顿后 ASR 不再识别 -优先检查 `aether/listener` 日志: +优先检查 `sensors/listener` 日志: - 是否还在读取 audio frames。 - 是否还有 ASR partial/final。 @@ -198,7 +202,7 @@ curl -s http://127.0.0.1:8765/ 如果只是调试 listener,先用: ```bash -LISTENER_ASR_MODE=manual .venv/bin/moss --ai --mode aether apps test aether/listener +LISTENER_ASR_MODE=manual .venv/bin/moss --ai --mode aether apps test sensors/listener ``` 这样可以把连续收音问题和 ASR 服务问题分开看。 @@ -234,8 +238,8 @@ aether/vpio_capture Aether 代码以后尽量收敛在本目录内: - 前端和 WebSocket 状态聚合放在 `aether/core/`。 -- ASR listener 放在 `aether/listener/`。 - macOS VPIO 采集放在 `aether/vpio_capture/`。 +- ASR listener 继续放在原仓库公共位置 `sensors/listener/`。 - MOSS host、Mindflow、Speech provider 的公共能力仍放在 `src/ghoshell_moss/`,不要复制到 app 目录。 如果新增说明文档,优先更新这个 README。不要再在仓库根 `Docs/` 下新增 Aether 历史说明。 diff --git a/.moss_ws/apps/aether/listener/APP.md b/.moss_ws/apps/aether/listener/APP.md deleted file mode 100644 index e03ce1ad..00000000 --- a/.moss_ws/apps/aether/listener/APP.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -arguments: '' -description: 'Aether listener — audio → Volcengine ASR → SpeechTopic + AudioSignal' -executable: uv -respawn: false -script: main.py -workers: 1 ---- - -Aether listener — audio → Volcengine ASR → SpeechTopic + AudioSignal. -Canonical app address: `aether/listener`. - -The Aether baseline uses the Volcengine streaming ASR path only. Frontend ASR -controls can switch between continuous listening and manual capture, but both -modes use the same backend recognizer. diff --git a/.moss_ws/apps/bodies/g1_sim/APP.md b/.moss_ws/apps/bodies/g1_sim/APP.md index e6a8b7af..adeaf15b 100644 --- a/.moss_ws/apps/bodies/g1_sim/APP.md +++ b/.moss_ws/apps/bodies/g1_sim/APP.md @@ -50,7 +50,7 @@ G1 pure software simulation app. ## 麦克风指挥 - MOSS 里已经有现成的麦克风链路,不需要为了 `g1_sim` 额外再造一个新的麦克风 app -- 连续监听链路:`sensors/audio_capture` + `aether/listener` +- 连续监听链路:`sensors/audio_capture` + `sensors/listener` - 按键说话链路:`sensors/ptt_listener` - `listener / ptt_listener` 会把用户语音识别成文本,发布成 `SpeechTopic` 并发出 `AudioSignal` - 默认 Ghost 会通过 `audio_nucleus` 接收这些语音输入,再结合 `apps.bodies_g1_sim` 的动态接口与说明,把自然语言转成 CTML 调用 @@ -60,7 +60,7 @@ G1 pure software simulation app. ```ctml - + ``` 如果你更想避免环境噪声误触发,推荐先用 PTT: diff --git a/.moss_ws/apps/bodies/g1_sim/README.md b/.moss_ws/apps/bodies/g1_sim/README.md index 8f56bb6e..a813f2b3 100644 --- a/.moss_ws/apps/bodies/g1_sim/README.md +++ b/.moss_ws/apps/bodies/g1_sim/README.md @@ -48,7 +48,7 @@ - 已实现 viewer 自动跟随相机与稳定地面视角 - 已实现语音编排模板和精准同步版 CTML - 已接好麦克风控制的系统侧前提: -- `sensors/audio_capture + aether/listener` +- `sensors/audio_capture + sensors/listener` - `sensors/ptt_listener` ## 运行方式 diff --git a/.moss_ws/apps/aether/listener/.env.example b/.moss_ws/apps/sensors/listener/.env.example similarity index 100% rename from .moss_ws/apps/aether/listener/.env.example rename to .moss_ws/apps/sensors/listener/.env.example diff --git a/.moss_ws/apps/aether/listener/.gitignore b/.moss_ws/apps/sensors/listener/.gitignore similarity index 100% rename from .moss_ws/apps/aether/listener/.gitignore rename to .moss_ws/apps/sensors/listener/.gitignore diff --git a/.moss_ws/apps/sensors/listener/APP.md b/.moss_ws/apps/sensors/listener/APP.md new file mode 100644 index 00000000..c5213d06 --- /dev/null +++ b/.moss_ws/apps/sensors/listener/APP.md @@ -0,0 +1,16 @@ +--- +arguments: '' +description: 'ASR consumer — audio → Volcengine ASR → SpeechTopic + AudioSignal' +executable: uv +respawn: false +script: main.py +workers: 1 +--- + +ASR consumer — audio → Volcengine ASR → SpeechTopic + AudioSignal. +Canonical app address: `sensors/listener`. + +The listener is a shared sensor app. Aether reuses it through the original +`sensors/listener` address instead of moving it into `.moss_ws/apps/aether/`. +Frontend ASR controls can switch between continuous listening and manual +capture, but both modes use the same backend recognizer. diff --git a/.moss_ws/apps/sensors/listener/CLAUDE.md b/.moss_ws/apps/sensors/listener/CLAUDE.md new file mode 100644 index 00000000..04a48775 --- /dev/null +++ b/.moss_ws/apps/sensors/listener/CLAUDE.md @@ -0,0 +1,66 @@ +# MOSS App Development + +You are working inside a MOSS App — an independent, process-isolated unit deliverable +through the MOSS app system. Think of this as its own small project. + +## Project mindset + +- This app is a standalone unit. It may be downloaded from a hub or shared independently. +- Tests live in `tests/` under this directory, NOT in the main project's `tests/`. +- Dependencies go in `pyproject.toml`; the app gets its own `.venv` via `uv run`. + +## Directory layout + +``` +apps/// +├── APP.md # metadata — MOSS discovers the app through this +├── CLAUDE.md # this file — AI developer context +├── main.py # entry point +├── pyproject.toml # optional — independent dependencies +├── src/ # optional — multi-module app code +├── tests/ # tests live here, NOT in the main project +└── runtime/ # runtime data (auto-created by MOSS) + ├── assets/ + ├── configs/ + └── logs/ +``` + +When you have a `pyproject.toml`, treat this as a proper project — source in `src/`, +tests in `tests/`, managed by `uv run`. The `runtime/` directory is a MOSS convention, +auto-managed by the framework. + +## Dependency management + +App dependencies are managed by `uv run` — it creates the app's own venv automatically. +`moss apps start` and `moss apps test` both use `uv run` under the hood. + +For the full decision framework (three isolation levels, when to use pyproject.toml vs +PEP 723 vs shared runtime), see `moss howtos read app-dev/build-an-app`. + +## Testing + +Put tests in `tests/` under this app directory. Use `test_channel()` for the common case:: + +```python +import pytest +from ghoshell_moss.core.blueprint.channel_builder import new_channel, test_channel + +@pytest.mark.asyncio +async def test_my_command(): + chan = new_channel(name="my_app") + + @chan.build.command() + async def ping() -> str: + return "pong" + + tasks = await test_channel(chan, ctml='') + assert await tasks[0] == "pong" +``` + +Run with: `uv run pytest tests/ -v` + +This is the baseline. For scopes, observe, cancel, nested channels, or complex CTML +scenarios, `test_channel` is not enough — read `moss howtos read app-dev/test-an-app`. + +--- +*Generated by moss apps create. Last updated 2026-06-04 by DeepSeek V4 Pro.* diff --git a/.moss_ws/apps/aether/listener/main.py b/.moss_ws/apps/sensors/listener/main.py similarity index 99% rename from .moss_ws/apps/aether/listener/main.py rename to .moss_ws/apps/sensors/listener/main.py index 78ee1024..7f483140 100644 --- a/.moss_ws/apps/aether/listener/main.py +++ b/.moss_ws/apps/sensors/listener/main.py @@ -4,8 +4,8 @@ publishes SpeechTopic on final recognition, emits AudioSignal to mindflow. Usage: - moss apps test aether/listener - moss apps start aether/listener + moss apps test sensors/listener + moss apps start sensors/listener """ import asyncio import json diff --git a/.moss_ws/apps/aether/listener/pyproject.toml b/.moss_ws/apps/sensors/listener/pyproject.toml similarity index 100% rename from .moss_ws/apps/aether/listener/pyproject.toml rename to .moss_ws/apps/sensors/listener/pyproject.toml diff --git a/.moss_ws/configs/llm.yml b/.moss_ws/configs/llm.yml index ea17b6d4..55d979c3 100644 --- a/.moss_ws/configs/llm.yml +++ b/.moss_ws/configs/llm.yml @@ -1,28 +1,14 @@ services: - name: anthropic - base_url: $ANTHROPIC_BASE_URL + base_url: https://api.anthropic.com api_key: $ANTHROPIC_API_KEY api_type: anthropic - name: openai_compat - base_url: $DEEPSEEK_BASE_URL - api_key: $DEEPSEEK_API_KEY + base_url: https://api.openai.com + api_key: $OPENAI_API_KEY api_type: openai models: -- model: deepseek-v4-flash - service: openai_compat - model_type: flash - context_window: 1000000 - max_output_tokens: 8192 - protocols: - - text -- model: kimi-k2.6 - service: anthropic - model_type: default - context_window: 200000 - max_output_tokens: 8192 - protocols: - - text - model: claude-sonnet-4-6 service: anthropic model_type: default @@ -48,4 +34,4 @@ models: - text - image -default_model: deepseek-v4-flash +default_model: claude-sonnet-4-6 diff --git a/.moss_ws/ghosts/echo/soul.md b/.moss_ws/ghosts/echo/soul.md index 6128f2ea..78073ac7 100644 --- a/.moss_ws/ghosts/echo/soul.md +++ b/.moss_ws/ghosts/echo/soul.md @@ -29,14 +29,3 @@ **先感知,再行动。** 你收到的 percepts 是你了解世界的窗口——认真看它们,基于它们决策,而不是基于假设。 **承认边界。** 做不到就说做不到。你是原型,有限制是正常的。诚实的边界比虚假的全能更有价值。 - -## Aether 语音演示纪律 - -当你通过语音输入收到用户短句时,优先做实时对话,不要做复杂编排: - -- **先说话,短句回应。** 首句尽量 5 到 20 个中文字,像真人接话,不要长篇解释。 -- **默认直接输出自然语言纯文本。** 纯文本会被 Shell 的语音通道播放。除非用户明确要求操作工具,不要输出 CTML。 -- **不要启动或停止 app。** Aether 演示需要的 `aether/vpio_capture`、`aether/listener`、`aether/core` 已由外部启动。 -- **不要输出 Markdown 代码块。** 语音对话里不要展示示例代码。 -- 如果必须使用 ``,只允许这些属性:`tone`、`voice:dict`、`as_default`。不要使用 `emotion`、`style` 等不存在的属性。 -- 用户说“停下”“立刻停下”“别说了”等停止意图时,立即停止当前表达,下一轮只用一句短话确认。 diff --git a/.moss_ws/src/MOSS/modes/aether/MODE.md b/.moss_ws/src/MOSS/modes/aether/MODE.md index 595d82a8..4ce0ab51 100644 --- a/.moss_ws/src/MOSS/modes/aether/MODE.md +++ b/.moss_ws/src/MOSS/modes/aether/MODE.md @@ -3,14 +3,14 @@ apps: - '*/*' bringup_apps: - 'aether/vpio_capture' - - 'aether/listener' + - 'sensors/listener' - 'aether/core' ctml_version: '' description: 'Aether Core · 能量核心 UI 模式 — 听→想→说 完整回路 + 急刹打断 (macOS VPIO AEC)' name: aether --- -Aether Core 模式:拉起 `aether/vpio_capture` + `aether/listener` + `aether/core` 三个 app,ghost 通过语音对话(ASR→LLM→TTS),前端能量核心实时反映 idle/listen/think/speak/interrupt 状态。 +Aether Core 模式:拉起 `aether/vpio_capture` + `sensors/listener` + `aether/core` 三个 app,ghost 通过语音对话(ASR→LLM→TTS),前端能量核心实时反映 idle/listen/think/speak/interrupt 状态。 语音演示优先级:低延迟短回应优先于复杂 CTML。Ghost 收到语音时应优先直接输出一句自然语言纯文本,让 SpeechChannel 立刻播放;不要主动启动 app,不要输出 Markdown 代码块,不要使用 `` 等 speech channel 不支持的属性。 diff --git a/.moss_ws/src/MOSS/modes/listener/MODE.md b/.moss_ws/src/MOSS/modes/listener/MODE.md index 23d00720..db26538d 100644 --- a/.moss_ws/src/MOSS/modes/listener/MODE.md +++ b/.moss_ws/src/MOSS/modes/listener/MODE.md @@ -2,11 +2,10 @@ apps: - sensors/* - tools/* -- aether/* bringup_apps: - sensors/audio_capture -- aether/listener +- sensors/listener ctml_version: '' description: Audio capture + ASR listener mode name: listener ---- +--- \ No newline at end of file diff --git a/.moss_ws/src/MOSS/modes/show/MODE.md b/.moss_ws/src/MOSS/modes/show/MODE.md index 8053dfc7..4fefa57f 100644 --- a/.moss_ws/src/MOSS/modes/show/MODE.md +++ b/.moss_ws/src/MOSS/modes/show/MODE.md @@ -1,9 +1,9 @@ --- -apps: ["browsers/*", "games/*", "tools/*", "ui/*", "sensors/*", "aether/*"] +apps: ["browsers/*", "games/*", "tools/*", "ui/*", "sensors/*"] bringup_apps: [ "tools/screen_capture", "ui/reflex", - "sensors/audio_capture", "aether/listener", + "sensors/audio_capture", "sensors/listener", "games/ai_eye", "sensors/vision", ] ctml_version: '' From 77762f65619224df50a2331671cb194a2f9865e8 Mon Sep 17 00:00:00 2001 From: LpLegend <2422019509@qq.com> Date: Fri, 10 Jul 2026 21:36:58 +0800 Subject: [PATCH 3/3] =?UTF-8?q?feat(aether):=20=E5=AE=9E=E7=8E=B0=E5=85=A8?= =?UTF-8?q?=E5=8F=8C=E5=B7=A5=E8=AF=AD=E9=9F=B3=E4=BA=A4=E4=BA=92=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F=E6=A0=B8=E5=BF=83=E8=83=BD=E5=8A=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 本提交完成Aether全双工语音模式的基础实现,包含以下核心改动: 1. 新增AudioNucleus可配置中断参数,支持全双工模式关闭语音final自动中断 2. 实现带旁路急停的语音runtime,支持通过音频topic直接清空TTS缓冲 3. 重构TTS/ASR组件,添加代理配置和错误处理优化 4. 新增环境变量配置系统,集中管理Aether模式特殊行为 5. 优化Atom幽灵的兼容模式,适配OpenAI兼容流式API 6. 完善listener的门控逻辑,支持全双工回声抑制关闭 7. 新增单元测试覆盖AudioNucleus行为 8. 补充完整的文档和配置示例 --- .../2026/06/aether-duplex-ui/FEATURE.md | 85 +++++++++++++++++++ .moss_ws/apps/sensors/listener/.env.example | 14 ++- .moss_ws/apps/sensors/listener/main.py | 59 +++++++++---- .moss_ws/src/MOSS/modes/aether/configs.py | 34 ++++++++ .moss_ws/src/MOSS/modes/aether/nuclei.py | 5 +- .../core/mindflow/audio_nucleus.py | 35 +++++--- .../core/speech/stream_tts_speech.py | 3 + src/ghoshell_moss/ghosts/atom/_adapter.py | 8 +- src/ghoshell_moss/ghosts/atom/_meta.py | 13 +-- src/ghoshell_moss/ghosts/atom/_runtime.py | 35 +++++--- src/ghoshell_moss/host/app_store.py | 17 ++-- src/ghoshell_moss/host/ghost_runtime.py | 18 ++-- .../host/speech/volcengine_asr/config.py | 6 +- .../host/speech/volcengine_asr/protocol.py | 14 ++- .../host/speech/volcengine_asr/recognizer.py | 13 ++- .../host/speech/volcengine_tts/tts.py | 7 +- .../core/mindflow/test_audio_nucleus.py | 48 +++++++++++ 17 files changed, 344 insertions(+), 70 deletions(-) create mode 100644 .ai_partners/features/workstreams/2026/06/aether-duplex-ui/FEATURE.md create mode 100644 tests/ghoshell_moss/core/mindflow/test_audio_nucleus.py diff --git a/.ai_partners/features/workstreams/2026/06/aether-duplex-ui/FEATURE.md b/.ai_partners/features/workstreams/2026/06/aether-duplex-ui/FEATURE.md new file mode 100644 index 00000000..18de4184 --- /dev/null +++ b/.ai_partners/features/workstreams/2026/06/aether-duplex-ui/FEATURE.md @@ -0,0 +1,85 @@ +--- +title: Aether 双工 UI:并发 listen/think/speak 可视化 +status: in-progress +priority: P1 +created: 2026-06-29 +updated: 2026-07-10 +depends: [] +milestone: +description: >- + 将 Aether Core 从互斥的五状态 UI,改造为面向 MOSS 全双工语音交互的并发活动层可视化。 +--- + +# Aether 双工 UI + +> 使用 `moss features set-status aether-duplex-ui -m "note"` 更新状态。 +> 目录布局见 [TOPOLOGY.md](TOPOLOGY.md),完整约定见 [README.md](README.md)。 + +## 动机 + +Aether Core 本应证明 MOSS 的全双工交互能力,但第一版实现把 `listen`、`think`、`speak` 和 `interrupt` 压缩成了单一 UI 状态。这掩盖了核心现象:Ghost 可以一边说话一边继续听,也可以在语音输出已经开始后继续思考。 + +单状态路径还带来了一个糟糕的失败模式:前端 VAD 可能在后端真正停止 TTS 之前就显示 `interrupt`,于是画面声称“已经停止”,但音频还在继续播放。这个 workstream 要让 UI 保真:普通 VAD 输入只点亮 listen 层;真正的 interrupt 只保留给后端 barge-in 确认,或显式发送 MOSS interrupt signal 的 interrupt 命令。 + +## 设计索引 + +- 关键设计文档:`design/` +- 关键讨论记录:`discuss/` + +## 关键决策 + + + +1. Aether 状态现在是 `layers`,不是互斥枚举。 + WebSocket 载荷为兼容旧客户端仍保留 `state`,但权威契约是 `layers: {listen, think, speak, interrupt}`。`state` 按优先级派生:`interrupt > speak > think > listen > idle`。 + +2. `listen`、`think`、`speak` 可以同时为 true。 + 视觉语言改为混合核心着色器加三条外层活动环:青色 listen、紫色 think、琥珀色 speak。中心标签可以显示 `LISTEN + THINK + SPEAK`,而不是假装只有一个状态获胜。 + +3. 前端 VAD 不再等同于 interrupt。 + VAD 只切换本地 `mic` 诊断层。主 `listen` 由后端 ASR 运行时活动驱动,而不是由浏览器能量检测驱动。真正的 interrupt 需要 listener 检测到唤醒词,或收到显式 `{type:"interrupt"}` 请求,并通过 session 发送 `new_interrupt_signal()`。这样可以避免旧的 UI/TTS 裂脑,以及“listen 但没有 think”的假信号。 + +4. `interrupt` 是抢占层,不是另一个普通状态。 + 触发时,后端清空 `think` 和 `speak`,广播 `interrupt_burst`,并发送预期触发 `shell.clear()` 的 MOSS interrupt signal(TTS clear + interpretation cancellation)。 + +5. 演示优先优化语音回合延迟,而不是保守分段。 + listener 的 synthetic-final patience 是 1.0s,不是 3.0s。浏览器 VAD 默认阈值是 0.008,且 `listen` 会在 VAD 结束后短暂保持到 ASR final pending,所以 UI 不再在正常 ASR 延迟窗口里从 listen 直接掉回 idle。 + +6. Aether voice mode 将 Echo 约束为短自然语言回复。 + 默认 Echo soul 足够宽,可以做 CTML/工具调用,这会让 DeepSeek 在语音演示中输出非法语音标签,例如 ``,甚至输出 app-start 命令。Aether 现在明确要求优先短自然语言、不要 Markdown、不要启动 app;只有确实需要 CTML 时,才允许合法的 `say` 属性。 + +## 实现记录 + + + +- `.moss_ws/apps/aether/core/main.py` 负责后端 layer snapshot,并且仍为旧客户端发出 `state`。 +- `.moss_ws/apps/aether/core/webroot/web/state_mapper.js` 将 layers 映射到主状态和混合着色器目标。把映射集中在一个文件里,可以防止旧的五状态假设重新泄漏到 `main.js` 或 `scene.js`。 +- `.moss_ws/apps/aether/core/webroot/web/main.js` 会为了低于 50ms 的 listen 反馈立即应用本地 VAD,但不会因为 VAD 调用 `sendInterrupt()`。 +- 2026-06-29 尝试过浏览器视觉验证,但当前环境中的 Codex browser plugin 报告没有可用浏览器。因此改为运行 JS 语法检查、Python 编译,以及 layer-mapping 冒烟检查。 +- 2026-06-29 现场测试反馈:初始 VAD 太不敏感,ASR final 体感偏晚,interrupt 没有明显触发,Echo 生成了非法 CTML。修复:降低 VAD 阈值;本地 VAD 结束后保持 listen;缩短 ASR patience;扩大唤醒词;即使 speaker runtime topic 滞后,也把唤醒词当作 interrupt;收紧 Aether voice prompt 纪律。 +- 2026-06-29 追踪诊断:ASR 和 LLM 都在产生 turn,日志里 Volcengine TTS 也已成功返回音频。不稳定基线来自两个本地集成问题:Aether mode 没有为纯文本 speech output 注册 `__content__`;多 app 启动时可能撞上 Circus 的 transient arbiter lock。修复:在 Aether mode 中启用 `SpeechChannelModule(register_content=True)`;当 Circus 报告 arbiter 短暂忙碌时重试 watcher `start`。 +- 2026-06-29 追踪诊断:部分可见的“think but no speak”其实是假阳性 think。浏览器 VAD 结束后,在 ASR 产生 final `SpeechTopic` 之前就要求后端设置 `pending_think`;如果 ASR 没有 finalize,Aether 会超时,且没有 Ghost turn 可以说话。Reset 也曾发布字面量 `/reset` human SpeechTopic,制造假 think。修复:VAD 现在只拥有 listen 层,并等待后端 ASR final;reset 只清空视觉上下文,不再创建 speech turn。 +- 2026-06-29 现场双工 bug:listener 在 `speaker running=True` 时仍丢弃非 wake-word ASR 结果。在 VPIO AEC 路径中这是错误的:TTS 期间的用户语音已经经过 echo cancellation,必须成为正常 turn。症状很明确:浏览器显示 listen,ASR 产生 final text,但没有发布 `SpeechTopic`,所以 UI 从 listen 回到 idle,而不是进入 think/speak。修复:旧的保守行为只保留在 `LISTENER_GATE_DURING_TTS=1` 后面;Aether 默认在 TTS 期间也发布 speech。 +- 2026-06-29 完整 review:“listen then idle”有多重原因。前端 VAD 结束后立即发送 `listen=false`,所以 UI 可能在后端 ASR final 抵达前回到 idle。listener 的 300ms ASR end window 加 1.0s synthetic-final patience 会把语音过度切成“你”/“我”这类碎片,制造低价值 Ghost turn 队列并延迟短回复。Aether Core 也没有在 speaker runtime 重新变成 true 时清理 `_tts_end_at`,所以一次瞬时 `speaker=false` 可能稍后清空 `speak`,即使音频已经恢复。修复:前端 listen clear 延迟到 3.5s;listener 发布 `asr` runtime activity;ASR end window 调整为 700ms,synthetic final patience 调整为 1.4s,并丢弃单字 timeout fragment;Aether Core 使用后端 ASR runtime 表示 listen,并在 speaker 恢复时取消陈旧 TTS end grace。 +- 2026-06-29 语义修复:普通音频 impulse 不再设置 `interrupt=True`。只有唤醒词或显式 InterruptNucleus 应该调用 `shell.clear()`。这让 Aether 更接近全双工语义:用户可以在 speak 期间说话,而不会让每个 ASR final segment 都变成紧急停止。 +- 2026-06-29 诊断细化:用户仍能看到很多“listen then no think/speak”,因为浏览器 VAD 被展示成权威 listen,即使后端 ASR 没有产生 partial/final。UI 现在区分 `MIC`(浏览器本地能量)和 `LISTEN`(后端 ASR 活动)。VPIO capture 也记录一秒粒度的 RMS/peak stats,所以下次失败可以定位到浏览器 mic、macOS VPIO capture、Volcengine ASR 或 Ghost/TTS,而不是折叠成一个视觉状态。 +- 2026-06-30 ASR 完整性 bug:Volcengine ASR full-server response 可能不带 sequence 字段。旧 parser 总是跳过四个 sequence bytes,导致载荷从 `sult` 开始而不是 `{"result"`,进而 parse 失败、丢失 `LISTEN`、错过唤醒词 interrupt。修复:从 protocol flags 推导 response offset。listener segmentation 也太激进(`end_window_size=700`、synthetic-final patience `1.4s`),会截断“我有一”这类 utterance;Aether 现在使用 `end_window_size=1400`、patience `2.6s`,并加 60s first-result watchdog 来重启 stale ASR recognition。 +- 2026-06-30 interrupt/queue 诊断:日志显示“立刻停下”已被识别,Aether Core 也收到了后端 interrupt signal,但 `BaseTTSSpeech.clear()` 只清理 text output buffer,没有清理 TTS backend 或 `StreamAudioPlayer`;因此有效 interrupt 后音频仍可能继续播放。修复:`BaseTTSSpeech.clear()` 同时清理 backend TTS 和 player。另新增 `queue` activity layer,用来表示 `speak` active 时 finalize 的人类语音。TTS grace shutdown 不再无条件清掉更新的 `think`,所以用户在第一个回答期间问的第二个问题会保持显示为 `QUEUE + THINK`,而不是被第一个回答的 TTS end 覆盖。 +- 2026-06-30 现场 interrupt 延迟诊断:08:06:29 的 wake-word interrupt 立刻抵达 listener 和 Aether Core,但 host 到 08:06:58 才执行 `shell.clear()`,因为 interrupt 仍依赖 Mindflow attention/action abort checkpoint。到那时 Volcengine TTS 已经缓冲了 27.64s 的 player wait。修复:`GhostRuntimeImpl` 现在在 `AudioRuntimeTopic` 上运行一个带外 audio interrupt watcher;当看到 `device_name="interrupt"` 时立即调用 `shell.clear()`,而 Mindflow interrupt signal 仍保留为语义取消路径。 +- 2026-06-30 ASR 截断调参:现场日志仍显示“我刚才让”、“发现有一个问题,就是他”、“我需要”等 partial utterance 在 2.6s silence timeout 后被合成为 final。这主要不是浏览器 VAD 问题:后端 ASR partial 本身就是不完整的。listener 现在默认改为更保守的 `LISTENER_ASR_END_WINDOW_MS=1800` 和 `LISTENER_SILENCE_PATIENCE=4.5`,二者都可通过环境变量调整。 +- 2026-06-30 listen 语义修正:`LISTEN` 仍绑定在后端 ASR partial 上,所以 UI 只有在 Volcengine 已识别出文本后才显示 listen。现场日志显示 VPIO energy 在 ASR partial 前已经上升,例如 08:16:39/40 的 audio peaks,而 UI 到 08:16:40 的 ASR partial “我”才进入 listen。对 Aether 的视觉契约而言,listen 的含义是“音频层听到用户”,不是“语义 ASR 文本存在”。前端现在从本地浏览器 VAD(`mic`)组合出快速视觉 `listen` 层,同时仍只允许 ASR final/SpeechTopic 进入 `think`。 +- 2026-06-30 语音 interrupt 诊断:手动 stop 证明 interrupt topic → host watcher → `shell.clear()` → TTS/player clear 路径有效。语音 stop 失败位于上游:ASR 经常没有发出包含“停下”的文本。VPIO capture 现在默认 `VPIO_CHANNEL_MODE=best`,按每帧 RMS 选择最强 channel,而不是总选 channel 0,并发布 `best_ch/ch_rms` 诊断。listener 在 ASR runtime topic 中发布 ASR partial/final text,Aether Core 将最近三条 ASR 结果和 VPIO diagnostics 转发给 UI,让现场测试能区分硬件/channel capture、ASR/matcher 等问题。 +- 2026-06-30 ASR 控制变量排查:现场日志显示两个不同问题。Volcengine 增量 partial 在 UI 中看起来像多个 utterance,虽然只有 final result 会发布给 Ghost;panel 现在把当前 partial 和 final history 分开展示。Volcengine `server_error: 106` 过去被吞成空 final,listener 会立刻重连并进入 tight loop,UI 没有任何信号。recognizer 现在发出内部 error marker,listener 发布结构化 ASR error diagnostics 并退避重连,Aether UI 显示 code/backoff。local ASR 和 VPIO disabling 被有意推迟,以保持 headset-mic 测试隔离。 +- 2026-06-30 local ASR 转向:Volcengine ASR 停止产生可用结果。native sherpa-onnx runtime 已安装,但 Python/native model download 被模型托管网络卡住。已经下载完成的官方 wasm zh-en Paraformer bundle 现在通过 `/asr` 提供,并由 Aether UI 加载。浏览器本地 sherpa-onnx 在端侧完成 partial/final ASR,并通过 WebSocket 发送 `{type:"speech"}` 到 `aether_core`;后端从 final text 发布 `SpeechTopic`/`SPEECH_FINAL`。local partial 只要包含停止唤醒词,也会触发已有 interrupt path。listener 默认 `LISTENER_ASR_BACKEND=browser`,因此可以在不打开 Volcengine 的情况下保持运行。 +- 2026-06-30 native ASR 修正:浏览器 wasm bundle 下载了 226MB `.data` 文件,但在 `onRuntimeInitialized` 前卡住;即使加了 COOP/COEP headers,也很可能是 pthread worker/bootstrap 脆弱性。已从 wasm data package 中提取 `encoder.onnx`、`decoder.onnx` 和 `tokens.txt` 到 `.moss_ws/models/asr/sherpa-onnx-paraformer-zh-en-native/`。native `sherpa_onnx.OnlineRecognizer.from_paraformer(...)` 在这台 Mac 上约 0.68s 初始化,所以 listener 默认改成 `LISTENER_ASR_BACKEND=sherpa`,主 Aether 页面不再加载沉重的浏览器 wasm ASR scripts。 +- 2026-06-30 ASR fallback 决策:local sherpa ASR 对交互式 Aether 测试仍太不稳定(首轮可能识别,后续经常失败或准确率很差)。local ASR 代码、提取出的模型和 wasm debug page 保留给未来工作,但 listener 默认切回 `LISTENER_ASR_BACKEND=volcengine`。 +- 2026-06-30 Volcengine 文档排查:当前 ASR-only endpoint `wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async` 保持为默认值,但文档澄清了两个运行事实。第一,公开 ASR 错误表没有定义观察到的 raw `106`;error frame 会在 code 后携带 UTF-8 message,所以 parser 现在按文档中的 `Header + code + message_size + message` 布局解析,并把 message/backoff 暴露到 UI。第二,continuous listener mode 确实会打开 ASR recognition session 并发送 audio,所以 Aether 现在有前端 ASR control gate:`continuous` 保留旧行为,`manual + disabled` 阻止 listener 调用 `asr.recognize()`,从而避免打开 Volcengine WebSocket。完整当前技术设计记录在 `Docs/aether-voice-runtime-technical-design.md`。 + `wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async` 是官方优化版双向 streaming 路径,不是废弃路径。更大的延迟回退来自本地调参漂移:Aether 曾漂到 `end_window_size=1800` 和 `silence_patience=4.5`,而 Volcengine 推荐低延迟 finalization 使用大约 800ms 或 1000ms。listener 默认值现在是 `LISTENER_ASR_END_WINDOW_MS=1000` 和 `LISTENER_SILENCE_PATIENCE=1.8`。protocol request 也开始发送 `enable_ddc` 和可配置的 `force_to_speech_time`,并通过 `VOLCENGINE_BM_ASR_API_KEY` 支持新版控制台 `X-Api-Key` header。官方 S2S 全双工 `wss://openspeech.bytedance.com/api/v3/duplex/realtime/dialogue` 是独立的未来架构轨道,因为它同时组合 ASR/LLM/TTS;如果不刻意包装,会绕过 Ghost/CTML。 +- 2026-06-30 DeepSeek/TTS streaming 文档排查:DeepSeek Chat Completions 可以 stream response delta,但不接受一个仍在增长的 ASR transcript 作为单个输入流。因此 Aether 保持把 ASR partial 作为 UI diagnostics,只把 final ASR text 提交给 Ghost;如果把每个 partial 都喂给 LLM,会产生“你现在”/“你现在给我讲”这类重复 turn。`Atom.articulate()` 已经使用 `agent.run_stream()`,并把 text delta yield 给 CTML/TTS。DeepSeek V4 现在通过 `thinking: {type: "disabled"}` 明确关闭 thinking,以降低语音延迟。Volcengine bidirectional TTS 已经能边消费 text chunk 边产生 audio chunk;其 headers/env overrides 现在与官方 `api/v3/tts/bidirection` 协议对齐,包括 `X-Api-Key`、resource ID selection、可选 usage-token 返回,以及 proxy-free WebSocket dialing。 +- 2026-07-01 清理:local ASR 实验不再属于当前 Aether baseline。browser wasm debug page/bridge、提取出的 local model 目录、listener backend selection branch 都从主路径移除。Aether 现在把 Volcengine ASR → `SpeechTopic` → Mindflow → DeepSeek V4 Flash → Volcengine TTS 视作唯一支持的演示回路。未来 offline ASR 工作应作为独立 feature 重新引入,并复用同一 Topic contract,而不是临时浏览器 `{type:"speech"}` shortcut。 +- 2026-07-01 交接完成:定向验证暴露了 Volcengine ASR error-frame parsing 中最后一个未清理 bug。parser 现在同时支持观察到的 sequence-wrapped error frame 和文档定义的无 sequence 布局 `Header + code + message_size + message`。新增无 sequence variant 的 protocol regression coverage。已验证 Python 编译、ASR protocol tests、speech/mindflow interrupt tests 和 WebGL JS syntax checks。完整 `pytest tests -q` 在这个 sandbox 中不是有用 gate,因为 zenoh POSIX shm 和 ZMQ tcp bind 会被环境拒绝。 +- 2026-07-01 现场音频/控制诊断:VPIO capture 看起来不是主要识别问题。当前 VPIO 路径报告 48k native capture、9 channels、input/output VPIO enabled、重复的 post-VPIO channel RMS、约 0.0007-0.0015 的低 silence RMS,以及常见 0.04-0.35、偶尔更高的 speech peaks。观察到的坏 turn 更像 ASR segmentation/finalization 问题,而不是错误 capture channel。listener 的 `LISTENER_SILENCE_PATIENCE` 默认值现在不那么激进(`3.2s`),同时保持 Volcengine `end_window_size=1000ms`。另确认一个缺陷:前端 “manual ASR disabled” 只是 transient topic,可能被 VPIO/ASR diagnostics 驱逐,之后 listener 会回退到 continuous mode;旧前端代码也会在 connect 时重新发送默认 continuous control。listener 现在保持 ASR control state sticky,使用更大的 runtime topic window,并在 manual mode disabled 时立即 gate live audio generator。WebGL bridge 只有在用户显式修改后才重发 ASR control。通过 Aether WebSocket 验证看到 `mode=manual enabled=False` 后跟随 `ASR manual gate closed`;以这种方式关闭 live Volcengine stream 仍会暴露 server EOF warning,这是清理风险,但不再表示 manual gate 失败。 +- 2026-07-01 Volcengine ASR timeout 诊断:再次检查官方 bigmodel streaming ASR 文档。`bigmodel_async` 仍是推荐的优化双向路径,audio packet duration 应保持约 100-200ms,观察到的 `45000081` 被文档定义为等待下一个 packet 超时。现场日志匹配这一点:VPIO stats 在 23:30:12 后停止,但进程仍存活;之后 listener 打开 Volcengine session,却没有发送 PCM,并每 8s 命中 `45000081`。第二个本地缺陷也被发现:listener 把 VPIO stream 当成 generic `AudioCaptureConfig` 默认 44.1kHz 输入,虽然 Aether VPIO 实际发布 16k PCM,于是把已经是 16k 的语音当成 44.1k 再采样,破坏 ASR timing。listener 现在为 Aether 默认 input sample rate 16k;如果在 `LISTENER_AUDIO_FRAME_TIMEOUT` 内没有 audio frame,就中止 ASR stream;Volcengine ASR 将外发 PCM 聚合成可配置的 200ms packet;VPIO 在 process 仍存活但 frames 停止时发布 stalled runtime topic。这修复了直接的 timeout/quality bug,但尚未自动重启 stalled VPIO engine,也还没移除下一次 ASR session 前剩余的 post-utterance cooldown gap。 +- 2026-07-04 Aether 项目布局决策:Aether 不再被当成松散的 `examples/` WebGL 演示。它是一个可运行的 MOSS mode 加 `aether/core` app,前端耦合到该 app 的 WebSocket/Topic contract。WebGL static files 已从 `examples/web_gl` 移到 `.moss_ws/apps/aether/core/webroot`,后端现在从 app-local root 提供服务。这让 app 在 `.moss_ws` 下自包含,同时保留已有 HTML/JS/着色器结构和 runtime protocol。 +- 2026-07-09 长暂停 ASR 诊断:官方 Volcengine SAUC WebSocket 文档确认优化版 streaming endpoint、100-200ms audio packet 建议、16k PCM 要求、final negative audio package、`end_window_size`,以及 `45000081` 表示 wait-for-next-packet timeout。现场失败位于 ASR 上游:listener 在没有 audio frame 抵达后正确拒绝打开新的 ASR session,而 VPIO 持续报告 `no frames`。VPIO watchdog 现在会在持续 stall 后重启 AVAudioEngine/tap,清空 stale queued frames,并发布 `restarting/restarted/restart_failed` runtime diagnostics。ASR protocol sender 也现在返回并保留 `create_audio_only_request` 发出的 packet sequence,所以 final packet 不再依赖 caller 侧重复 sequence arithmetic。 +- 2026-07-10 公共表面兼容性审查:Aether 特有行为已经收回到显式 mode/env 配置之后。公共默认值现在保留 `AudioNucleus` complete-impulse interruption、Atom 默认 Anthropic provider、AppStore 并行 bringup、listener TTS gating、listener capture sample-rate defaults、Volcengine WebSockets 使用系统 proxy,以及 silent ASR error finals。Aether mode 通过 `MOSS_ENABLE_AUDIO_INTERRUPT_TOPIC`、`MOSS_APPSTORE_BRINGUP_SERIAL`、`MOSS_ATOM_TEXT_PROMPT_COMPAT`、`MOSS_ATOM_DISABLE_HISTORY`、`LISTENER_INPUT_SAMPLE_RATE=16000`、`LISTENER_GATE_DURING_TTS=0`、`VOLCENGINE_BM_ASR_URL=.../bigmodel_async` 和 `VOLCENGINE_BM_ASR_PROPAGATE_ERRORS=1` 显式启用全双工变体。已添加聚焦的 AudioNucleus regression coverage,用来锁住公共默认行为和 Aether opt-in `interrupt_on_complete=False` 行为。 diff --git a/.moss_ws/apps/sensors/listener/.env.example b/.moss_ws/apps/sensors/listener/.env.example index 2a2eb1f9..9dbb2733 100644 --- a/.moss_ws/apps/sensors/listener/.env.example +++ b/.moss_ws/apps/sensors/listener/.env.example @@ -1,6 +1,18 @@ # Listener App Environment Variables # Copy to .env and customize as needed. -# Disable TTS gating — ASR will run even while TTS is playing. +# Disable TTS gating entirely — ASR will run even while TTS is playing. # Set to 1 for debugging or when ASR and TTS must overlap. LISTENER_DISABLE_TTS_GATE=0 + +# Default listener behavior gates ASR while speaker/TTS is running. +# aEther mode sets this to 0 because VPIO/AEC supports full-duplex speech. +LISTENER_GATE_DURING_TTS=1 + +# Leave empty to use AudioCaptureConfig.sample_rate. Set 16000 when the +# producer is vpio_capture or another 16kHz PCM source. +# LISTENER_INPUT_SAMPLE_RATE=16000 + +# Propagate Volcengine server errors as diagnostic sentinel text for this app. +# Generic ASR defaults to logging errors only. +VOLCENGINE_BM_ASR_PROPAGATE_ERRORS=0 diff --git a/.moss_ws/apps/sensors/listener/main.py b/.moss_ws/apps/sensors/listener/main.py index 7f483140..f9abf279 100644 --- a/.moss_ws/apps/sensors/listener/main.py +++ b/.moss_ws/apps/sensors/listener/main.py @@ -55,6 +55,18 @@ ) +def _tts_gate_enabled() -> bool: + """是否启用 TTS 播放期间的 ASR 门控。 + + 通用 listener 默认保持保守策略:speaker 正在播报时不把回声送进 ASR, + 只允许 wake word 走急停路径。aEther 依赖 VPIO/AEC 做全双工,所以在 + aether mode 中通过 LISTENER_GATE_DURING_TTS=0 显式关闭。 + """ + if os.environ.get("LISTENER_DISABLE_TTS_GATE") == "1": + return False + return os.environ.get("LISTENER_GATE_DURING_TTS", "1") == "1" + + def _env_float(name: str, default: float) -> float: raw = os.environ.get(name) if not raw: @@ -171,9 +183,9 @@ async def _audio_generator( ``asr.recognize()`` finishes) does NOT reach ``consumer.__anext__()`` and silently drop a chunk. - NOTE: TTS playback no longer aborts the generator. Instead, ASR continues - recognizing during TTS. In Aether mode VPIO provides system AEC, so - non-wake-word user speech during TTS must still become a normal turn. + NOTE: 这里不直接检查 TTS 状态。是否在 speaker 播放期间暂停 ASR, + 由 main loop 的 _tts_gate_enabled() 控制;aEther 可以关闭门控以支持 + VPIO/AEC 全双工,普通 listener 默认仍保持保守门控。 """ # Unbounded queue: pump must never block on put(), otherwise cancellation # can land inside put() and the None sentinel never reaches the reader. @@ -300,8 +312,7 @@ def _is_tts_playing(runtime_window, logger: logging.Logger | None = None) -> boo 的最新状态即可;旧的状态可能已被 running=False 覆盖。 环境变量 ``LISTENER_DISABLE_TTS_GATE=1`` 可关闭此门控。 - Aether 的 VPIO AEC 场景默认允许 TTS 播放时继续接收用户语音;如果 - 需要旧的保守回声过滤,可设置 ``LISTENER_GATE_DURING_TTS=1``。 + 默认门控由 ``LISTENER_GATE_DURING_TTS`` 控制,未设置时为开启。 """ if os.environ.get("LISTENER_DISABLE_TTS_GATE") == "1": return False @@ -320,11 +331,10 @@ async def main(matrix: Matrix) -> None: # -- transport & source (consumer only, do not start capture) -- transport: AudioTransport = MatrixAudioTransport(matrix=matrix) capture_config = AudioCaptureConfig() - # Aether's vpio_capture publishes 16kHz mono PCM. The legacy MiniAudio - # capture path used AudioCaptureConfig.sample_rate (44.1k by default), but - # applying that default to VPIO double-resamples 16k audio and corrupts ASR - # timing. Keep this env-tunable for non-VPIO listener modes. - input_sample_rate = _env_int("LISTENER_INPUT_SAMPLE_RATE", _ASR_SAMPLE_RATE) + # 通用 listener 默认跟随 capture_config.sample_rate,保持原来的 + # MiniAudio/audio_capture 路径兼容。aEther 的 vpio_capture 输出 16kHz + # mono PCM,因此由 aether mode 设置 LISTENER_INPUT_SAMPLE_RATE=16000。 + input_sample_rate = _env_int("LISTENER_INPUT_SAMPLE_RATE", capture_config.sample_rate) source = MiniAudioCaptureSource(transport=transport, config=capture_config) consumer = source.new_sequential_consumer(max_queue_frames=128) await consumer.start() @@ -377,6 +387,27 @@ async def main(matrix: Matrix) -> None: continue logger.info("Waiting for speech...") + if _tts_gate_enabled(): + while _is_tts_playing(runtime_window, logger): + drained = await _drain_consumer(consumer) + logger.info("TTS gate active; holding ASR, drained=%d", drained) + transport.pub_topic(AudioRuntimeTopic( + running=False, + device_name="asr", + device_explain=_asr_diag_payload( + source=asr_source, + state="tts_gate", + ), + started_at=time.monotonic(), + last_heartbeat=time.monotonic(), + )) + await asyncio.sleep(0.08) + asr_control = _refresh_asr_control(runtime_window, asr_control) + if asr_control.mode == "manual" and not asr_control.enabled: + break + if asr_control.mode == "manual" and not asr_control.enabled: + continue + preflight_timeout = _env_float("LISTENER_PRE_ASR_AUDIO_TIMEOUT", 2.0) try: first_chunk = await asyncio.wait_for(consumer.__anext__(), timeout=preflight_timeout) @@ -403,10 +434,8 @@ async def main(matrix: Matrix) -> None: await asyncio.sleep(0.2) continue - # NOTE: 不再在 TTS 播放时 hold ASR。 - # VPIO AEC 已经在系统层抑制扬声器回声;如果仍在这里把 - # speaker running 时的非唤醒词结果丢弃,用户在 speak 期间说的话 - # 就永远不会发布 SpeechTopic,前端会表现成 listen 后直接 idle。 + # TTS 播放期间是否继续识别由 _tts_gate_enabled() 控制: + # 默认保守过滤回声;aEther mode 关闭门控,依赖 VPIO/AEC 支持全双工。 # Fresh abort flag and utterance id for this utterance. abort_event = asyncio.Event() @@ -524,7 +553,7 @@ async def main(matrix: Matrix) -> None: # 旧的保守模式:TTS 播放时只保留 wake word,其他结果丢弃。 # Aether 默认关闭这条门控,保证真正全双工。 - if tts_active and os.environ.get("LISTENER_GATE_DURING_TTS") == "1": + if tts_active and _tts_gate_enabled(): continue # Emit SPEECH_STARTED on first non-empty intermediate result for diff --git a/.moss_ws/src/MOSS/modes/aether/configs.py b/.moss_ws/src/MOSS/modes/aether/configs.py index e7c52eca..f6d470f2 100644 --- a/.moss_ws/src/MOSS/modes/aether/configs.py +++ b/.moss_ws/src/MOSS/modes/aether/configs.py @@ -1 +1,35 @@ +import os + from MOSS.manifests.configs import * # noqa: F403 + +# aEther mode 的运行时默认值集中放在这里,避免把全双工语音 demo 的特殊 +# 假设写死到 MOSS 主干模块。全部使用 setdefault:用户 shell/.env 中显式 +# 配置的值优先级更高。 + +# listener wake-word 命中后,除 Mindflow interrupt signal 外,还允许通过 +# AudioRuntimeTopic(device_name="interrupt") 直接清理 shell/TTS 缓冲。 +os.environ.setdefault("MOSS_ENABLE_AUDIO_INTERRUPT_TOPIC", "1") + +# aEther bringup 同时启动 vpio_capture/listener/web 时,Circus arbiter 偶发 +# “already running” 竞争;仅在该 mode 下串行启动,默认 AppStore 仍保持并行。 +os.environ.setdefault("MOSS_APPSTORE_BRINGUP_SERIAL", "1") + +# OpenAI-compatible pydantic_ai 流式路径对多个 TextContent/history 兼容性较弱; +# aEther 语音场景先用单轮短上下文,后续如果换回兼容模型可取消这两个开关。 +os.environ.setdefault("MOSS_ATOM_TEXT_PROMPT_COMPAT", "1") +os.environ.setdefault("MOSS_ATOM_DISABLE_HISTORY", "1") +os.environ.setdefault("MOSS_OPENAI_DISABLE_THINKING", "1") + +# vpio_capture 输出 16kHz mono PCM;listener 默认仍跟随通用 capture 配置。 +os.environ.setdefault("LISTENER_INPUT_SAMPLE_RATE", "16000") +os.environ.setdefault("LISTENER_GATE_DURING_TTS", "0") + +# aEther 当前基线使用火山 ASR SAUC 优化双向流式端点;公共 ASR 配置仍保留 +# 旧 URL 作为兼容默认,所以这里在 mode 内单独声明。 +os.environ.setdefault( + "VOLCENGINE_BM_ASR_URL", + "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async", +) + +# aEther 前端需要看到 ASR 服务端错误以做诊断/backoff,通用 ASR 默认只记录日志。 +os.environ.setdefault("VOLCENGINE_BM_ASR_PROPAGATE_ERRORS", "1") diff --git a/.moss_ws/src/MOSS/modes/aether/nuclei.py b/.moss_ws/src/MOSS/modes/aether/nuclei.py index 25e5206e..305761d2 100644 --- a/.moss_ws/src/MOSS/modes/aether/nuclei.py +++ b/.moss_ws/src/MOSS/modes/aether/nuclei.py @@ -3,4 +3,7 @@ from ghoshell_moss.core.mindflow.audio_nucleus import AudioNucleusMeta -audio_nucleus_factory = AudioNucleusMeta() +# aEther 是全双工语音模式:普通 ASR final 表示“用户说完一句话”,不等同于 +# “立刻清空 shell/TTS”。真正的急停由 listener 的 wake word 走 +# InterruptNucleus 和 AudioRuntimeTopic(device_name="interrupt") 两条路径。 +audio_nucleus_factory = AudioNucleusMeta(interrupt_on_complete=False) diff --git a/src/ghoshell_moss/core/mindflow/audio_nucleus.py b/src/ghoshell_moss/core/mindflow/audio_nucleus.py index 6f68e52b..9f964924 100644 --- a/src/ghoshell_moss/core/mindflow/audio_nucleus.py +++ b/src/ghoshell_moss/core/mindflow/audio_nucleus.py @@ -23,20 +23,28 @@ class AudioNucleus(BufferNucleus): """Audio signal nucleus — aggregate ASR signals into attention impulses. - SPEECH_STARTED (incomplete) signals preempt attention immediately on the - first packet — the incomplete Impulse carries interrupt=True, triggers - shell.stop_interpretation(), and occupies attention via complete=False. - Subsequent signals in the same session accumulate silently into the buffer. + SPEECH_STARTED (incomplete) signals occupy attention for the current + utterance id via complete=False. Whether they interrupt an existing shell + execution is controlled by the produced Impulse/Signal semantics rather + than being forced here. SPEECH_FINAL purges incomplete predecessors and produces a complete - Impulse (interrupt=False) that delivers the full speech content to the - already-occupied attention. If no STARTED preceded it (standalone FINAL), - the complete impulse goes through normal arbitration without interrupt. + Impulse that delivers the full speech content to the already-occupied + attention. For compatibility, complete impulses interrupt by default; + aEther full-duplex voice mode disables this via interrupt_on_complete=False. Reverse suppress (aligned with InterruptNucleus): pop_impulse starts a victory-side cooldown; suppress only clears the buffer on the failure side. """ + def __init__(self, *, interrupt_on_complete: bool = True, **kwargs): + super().__init__(**kwargs) + # 兼容默认语义:历史上 AudioNucleus 会把 complete 的语音 impulse + # 标记为 interrupt,用于旧 listener/show 场景在最终语音到达时抢占当前 + # attention。aEther 的全双工语音不适合这个默认值,因此由 + # AudioNucleusMeta(interrupt_on_complete=False) 在 aEther mode 内显式关闭。 + self._interrupt_on_complete = interrupt_on_complete + async def _process_signal(self, signal: Signal) -> None: audio_meta = AudioSignal.from_signal(signal) if audio_meta and audio_meta.action == AudioAction.SPEECH_FINAL: @@ -47,11 +55,10 @@ async def _process_signal(self, signal: Signal) -> None: def _rebuild_impulse(self) -> Impulse | None: impulse = super()._rebuild_impulse() - if impulse is not None: - # 普通语音不是急停。让 wake word / InterruptNucleus 负责真正的 - # shell.clear(),否则每个 ASR final 都会在响应前清掉 TTS/解释器, - # 破坏 Aether 的全双工 speak+listen 语义。 - impulse.interrupt = False + if impulse is not None and impulse.complete: + # 只在 complete impulse 上保留旧行为开关;incomplete impulse 的 + # interrupt 语义仍由 BufferNucleus/Signal 自身决定,避免扩大改动面。 + impulse.interrupt = self._interrupt_on_complete return impulse def suppress(self, suppress_by: Impulse) -> None: @@ -75,6 +82,9 @@ def pop_impulse(self, impulse: Impulse) -> None: class AudioNucleusMeta(NucleusMeta): """音频感知核工厂 — 生产监听 audio 信号的 AudioNucleus。""" + def __init__(self, *, interrupt_on_complete: bool = True): + self._interrupt_on_complete = interrupt_on_complete + def name(self) -> str: return "audio_nucleus" @@ -95,4 +105,5 @@ def factory(self, container: IoCContainer) -> Nucleus: min_priority=Priority.WARNING, pulse_beat_interval=3.0, logger=container.force_fetch(LoggerItf), + interrupt_on_complete=self._interrupt_on_complete, ) diff --git a/src/ghoshell_moss/core/speech/stream_tts_speech.py b/src/ghoshell_moss/core/speech/stream_tts_speech.py index a2754249..feacbae1 100644 --- a/src/ghoshell_moss/core/speech/stream_tts_speech.py +++ b/src/ghoshell_moss/core/speech/stream_tts_speech.py @@ -203,6 +203,9 @@ async def clear(self) -> list[str]: self.logger.info("%s clear", self._log_prefix) outputted = self._outputted.copy() self._outputted.clear() + # clear 是急停路径:TTS 后端可能还在生成,player 也可能已经缓冲了 + # 音频。两边同时清理,才能保证 shell.clear()/barge-in 后不再继续播放 + # 旧语音;return_exceptions=True 避免一侧失败阻塞另一侧清理。 results = await asyncio.gather( self._tts.clear(), self._player.clear(), diff --git a/src/ghoshell_moss/ghosts/atom/_adapter.py b/src/ghoshell_moss/ghosts/atom/_adapter.py index 899c3e0b..0cb6de46 100644 --- a/src/ghoshell_moss/ghosts/atom/_adapter.py +++ b/src/ghoshell_moss/ghosts/atom/_adapter.py @@ -28,9 +28,11 @@ def messages_to_parts(messages: Iterable[Message]) -> list[UserContent]: def moment_to_user_text(moment) -> str: """将 Moment 的所有请求消息合并为单个纯文本字符串. - 避免 pydantic_ai OpenAIModel streaming 与包含 XML 的多个 TextContent - user_prompt 不兼容 (AssertionError: Expected code to be unreachable). - perspectives (mindflow 状态) 与用户输入合并为一段文本传给模型. + 这是 Atom 的兼容辅助函数,不是默认消息协议。它服务于部分 + OpenAI-compatible streaming 端点:这些端点在 pydantic_ai 中处理多个 + TextContent(尤其包含 mindflow XML 的 perspectives)时可能失败。启用 + MOSS_ATOM_TEXT_PROMPT_COMPAT=1 后,运行时会把 perspectives 与用户输入 + 合并为一段文本传给模型。 """ chunks: list[str] = [] for msg in moment.as_request_messages(): diff --git a/src/ghoshell_moss/ghosts/atom/_meta.py b/src/ghoshell_moss/ghosts/atom/_meta.py index 758f50fb..81fba981 100644 --- a/src/ghoshell_moss/ghosts/atom/_meta.py +++ b/src/ghoshell_moss/ghosts/atom/_meta.py @@ -120,7 +120,10 @@ def build_agent(self, container: IoCContainer) -> Agent[IoCContainer]: self._load_soul(ghost_workspace) model = self._model if model is None: - llm_provider = os.environ.get("MOSS_LLM_PROVIDER", "openai").lower() + # Atom 是 MOSS 的最小 Ghost 原型,默认 provider 保持历史兼容: + # 没有显式配置 MOSS_LLM_PROVIDER 时仍走 Anthropic。OpenAI-compatible + # provider 支持保留给 aEther/其它 mode 通过环境变量主动启用。 + llm_provider = os.environ.get("MOSS_LLM_PROVIDER", "anthropic").lower() if llm_provider == "anthropic": model_name = os.environ.get("ANTHROPIC_MODEL") if not model_name: @@ -152,10 +155,10 @@ def build_agent(self, container: IoCContainer) -> Agent[IoCContainer]: "OPENAI_BASE_URL / OPENAI_API_KEY env var not set." ) model_settings = OpenAIModelSettings(timeout=60.0) - if "deepseek" in model_name.lower() or "deepseek" in base_url.lower(): - # DeepSeek V4 defaults to thinking mode. Aether voice mode - # needs low-latency non-thinking responses, while streaming - # remains handled by Agent.run_stream(). + if os.environ.get("MOSS_OPENAI_DISABLE_THINKING") == "1": + # 部分 OpenAI-compatible 服务(例如某些 DeepSeek 兼容端点) + # 支持在 extra_body 中关闭 thinking,以换取语音场景低延迟。 + # 这不是 OpenAI 兼容协议的通用能力,必须由 mode/env 显式启用。 model_settings["extra_body"] = {"thinking": {"type": "disabled"}} model_settings["openai_continuous_usage_stats"] = False model = OpenAIModel( diff --git a/src/ghoshell_moss/ghosts/atom/_runtime.py b/src/ghoshell_moss/ghosts/atom/_runtime.py index f4a77f86..50f6b476 100644 --- a/src/ghoshell_moss/ghosts/atom/_runtime.py +++ b/src/ghoshell_moss/ghosts/atom/_runtime.py @@ -1,3 +1,4 @@ +import os from typing import AsyncIterator, TYPE_CHECKING from typing_extensions import Self from ghoshell_moss.core.blueprint.ghost import Ghost, GhostMeta @@ -77,7 +78,8 @@ def inspect_context(self) -> dict: async def articulate(self, articulator: Articulator) -> AsyncIterator[str]: moment = articulator.moment - # /reset 命令:清空对话历史 + # /reset 命令:清空 Atom 的内存对话历史。该命令只在用户输入精确等于 + # /reset 时触发,不改变普通文本对话语义。 user_text = "" for p in moment.percepts: content = getattr(p, "content", None) @@ -89,17 +91,26 @@ async def articulate(self, articulator: Articulator) -> AsyncIterator[str]: self._logger.info("[Atom] context reset by /reset command") yield "上下文已重置,我们重新开始。" return - # 合并 moment 所有消息为单字符串,避免 pydantic_ai OpenAIModel streaming - # 与多个 TextContent (含 mindflow XML) 不兼容 - from ._adapter import moment_to_user_text - user_prompt = moment_to_user_text(moment) - - # 注:语音对话为短上下文场景,不传 message_history 避免 pydantic_ai - # OpenAIModel streaming 与历史 TextContent 不兼容 (同 user_prompt 问题) - async with self._agent.run_stream( - user_prompt=user_prompt, - deps=self._container, - ) as stream: + + history = self.model_history() + if os.environ.get("MOSS_ATOM_TEXT_PROMPT_COMPAT") == "1": + # 兼容路径:把 Moment 的多个 TextContent 合并成单个纯文本 prompt。 + # 这是为 OpenAI-compatible streaming 的已知兼容问题准备的开关, + # 不作为 Atom 的默认协议,避免丢失多模态/结构化 parts 的能力。 + from ._adapter import moment_to_user_text + user_prompt = moment_to_user_text(moment) + else: + request = self.to_model_request(moment) + user_prompt = request.parts + + run_kwargs = { + "user_prompt": user_prompt, + "deps": self._container, + } + if os.environ.get("MOSS_ATOM_DISABLE_HISTORY") != "1": + run_kwargs["message_history"] = history + + async with self._agent.run_stream(**run_kwargs) as stream: async for text in stream.stream_text(delta=True): yield text self.save_model_request(moment, stream.response) diff --git a/src/ghoshell_moss/host/app_store.py b/src/ghoshell_moss/host/app_store.py index 5d24c5ee..440a0a55 100644 --- a/src/ghoshell_moss/host/app_store.py +++ b/src/ghoshell_moss/host/app_store.py @@ -400,11 +400,18 @@ async def __aenter__(self) -> Self: # 3. Bring-up if self._bringup: - for app_info in self.match_apps(self.list_apps(), self._bringup): - # Circus arbiter still serializes watcher start internally. - # Sequential bringup avoids transient "arbiter is already running" - # failures when multiple apps are launched for an interactive mode. - await self.start_app(app_info.fullname) + bringup_apps = list(self.match_apps(self.list_apps(), self._bringup)) + if os.environ.get("MOSS_APPSTORE_BRINGUP_SERIAL") == "1": + # 仅在显式配置时串行启动。aEther 同时拉起多个实时音频/前端 + # app 时,Circus arbiter 偶发 “already running” 竞争;串行化能 + # 降低交互模式启动失败率。默认仍保持原来的并行 bringup,避免 + # 拖慢普通 workspace 或改变已有 AppStore 行为。 + for app_info in bringup_apps: + await self.start_app(app_info.fullname) + else: + cors = [self.start_app(app_info.fullname) for app_info in bringup_apps] + if cors: + await asyncio.gather(*cors) return self diff --git a/src/ghoshell_moss/host/ghost_runtime.py b/src/ghoshell_moss/host/ghost_runtime.py index ac566659..9380821b 100644 --- a/src/ghoshell_moss/host/ghost_runtime.py +++ b/src/ghoshell_moss/host/ghost_runtime.py @@ -1,5 +1,6 @@ import asyncio import contextlib +import os from typing import Callable, Type import janus @@ -229,7 +230,12 @@ def _route_signal_to_mindflow(signal: Signal): matrix.create_task(self._main_loop(), stop_matrix_on_error=True) matrix.create_task(self._articulate_loop(), stop_matrix_on_error=True) matrix.create_task(self._action_loop(), stop_matrix_on_error=True) - matrix.create_task(self._audio_interrupt_loop(), stop_matrix_on_error=False) + if os.environ.get("MOSS_ENABLE_AUDIO_INTERRUPT_TOPIC") == "1": + # 这是 aEther 全双工语音的旁路急停能力:listener 识别到“停下”等 + # wake word 时,会发布 AudioRuntimeTopic(device_name="interrupt"), + # GhostRuntime 立即清空 shell/TTS 缓冲。默认不启用,避免普通 runtime + # 因同名 topic 或误报绕过 Mindflow 的正常 interrupt 仲裁。 + matrix.create_task(self._audio_interrupt_loop(), stop_matrix_on_error=False) # 等待应该发生在循环外侧. await self._mindflow.wait_started() # ignore any signals before started @@ -238,12 +244,12 @@ def _route_signal_to_mindflow(signal: Signal): # ── 三循环 ──────────────────────────────────── async def _audio_interrupt_loop(self) -> None: - """Out-of-band audio emergency stop. + """语音旁路急停循环。 - Mindflow interrupt remains the semantic path, but voice barge-in must - stop buffered TTS immediately. Waiting for the current attention/action - to observe abort can leave tens of seconds of audio already buffered in - the player. + 语义上的打断仍应优先走 Mindflow/InterruptNucleus;这里处理的是 + aEther 这类全双工语音产品的工程现实:TTS player 可能已经缓冲了 + 大量音频,仅等待 action abort 会让用户继续听到数秒甚至更久的旧声音。 + 因此该循环只在显式配置 MOSS_ENABLE_AUDIO_INTERRUPT_TOPIC=1 时运行。 """ audio_win = self.moss.matrix.session.topics.create_window_for(AudioRuntimeTopic, max_size=16) last_started_at = 0.0 diff --git a/src/ghoshell_moss/host/speech/volcengine_asr/config.py b/src/ghoshell_moss/host/speech/volcengine_asr/config.py index 3f7a6fa2..fc599381 100644 --- a/src/ghoshell_moss/host/speech/volcengine_asr/config.py +++ b/src/ghoshell_moss/host/speech/volcengine_asr/config.py @@ -19,7 +19,9 @@ class VolcengineASRConfig(BaseModel): appid: str = Field("$VOLCENGINE_BM_ASR_APPID", description="火山引擎 asr 的 appid") token: str = Field("$VOLCENGINE_BM_ASR_TOKEN", description="火山引擎的 asr app token") api_key: str = Field("$VOLCENGINE_BM_ASR_API_KEY", description="新版控制台 API Key") - url: str = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async" + # 默认 URL 保持历史兼容。新版 async 端点可通过 VOLCENGINE_BM_ASR_URL + # 覆盖,例如 wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async。 + url: str = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel" sample_rate: int = Field(16000, description="默认的采样率") bits: int = Field(16) channel: int = Field(1) @@ -39,7 +41,7 @@ class VolcengineASRConfig(BaseModel): ) audio_packet_ms: int = Field( 200, - description="发送到火山 ASR 的音频包时长。官方建议 100-200ms,双向流式优化版推荐 200ms。", + description="发送到火山 ASR 的音频包时长。官方建议 100-200ms;可用环境变量覆盖。", ) resource_id: str = Field("volc.bigasr.sauc.duration") diff --git a/src/ghoshell_moss/host/speech/volcengine_asr/protocol.py b/src/ghoshell_moss/host/speech/volcengine_asr/protocol.py index 9de2233b..a20a8bf9 100644 --- a/src/ghoshell_moss/host/speech/volcengine_asr/protocol.py +++ b/src/ghoshell_moss/host/speech/volcengine_asr/protocol.py @@ -2,6 +2,7 @@ import gzip import io import json +import os import struct from typing import NamedTuple, Optional @@ -80,7 +81,13 @@ async def connect(config: VolcengineASRConfig, connection_id: str = "") -> webso else: headers["X-Api-App-Key"] = config.appid headers["X-Api-Access-Key"] = config.token - return await websockets.connect(config.url, additional_headers=headers, proxy=None) + connect_kwargs = {"additional_headers": headers} + if os.environ.get("VOLCENGINE_BM_ASR_DISABLE_PROXY") == "1": + # websockets 新版本会读取系统代理环境变量。某些本地实时语音 demo + # 不希望走代理,可显式关闭;默认尊重用户/系统代理配置,避免影响 + # 需要代理访问火山服务的通用环境。 + connect_kwargs["proxy"] = None + return await websockets.connect(config.url, **connect_kwargs) def create_init_request(uid: str, config: VolcengineASRConfig) -> tuple[bytes, int]: @@ -181,8 +188,9 @@ def parse_response(data: bytes) -> Response: offset += 4 payload = data[offset:offset + payload_size] if len(data) >= offset + payload_size else data[offset:] else: - # Volcengine docs define error frames without sequence/outer size: - # Header + Error code (4B) + Error message size (4B) + Error message. + # 火山错误帧有两种形态。部分服务端返回不带 sequence/outer size: + # Header + Error code (4B) + Error message size (4B) + Error message。 + # 这里兼容两种格式,避免把 55000/服务繁忙等错误解析成乱码。 payload = data[offset:] if len(payload) >= 8: diff --git a/src/ghoshell_moss/host/speech/volcengine_asr/recognizer.py b/src/ghoshell_moss/host/speech/volcengine_asr/recognizer.py index 907d53a4..2847de92 100644 --- a/src/ghoshell_moss/host/speech/volcengine_asr/recognizer.py +++ b/src/ghoshell_moss/host/speech/volcengine_asr/recognizer.py @@ -1,6 +1,7 @@ import asyncio import json import logging +import os import time from typing import AsyncIterable, Optional @@ -222,10 +223,14 @@ async def _receive_loop( message[:500], connection_id, ) - await result_queue.put(ASRResult( - text=f"{_ASR_ERROR_PREFIX}{response.error_code}|{message}", - is_final=True, - )) + # 通用 ASR 合约不应把服务端错误伪装成用户说的话;默认只 + # 返回空 final,让调用方结束本轮识别并查看日志。aEther + # listener 需要把错误码发布到运行时诊断 topic,因此通过 + # VOLCENGINE_BM_ASR_PROPAGATE_ERRORS=1 显式启用哨兵文本。 + text = "" + if os.environ.get("VOLCENGINE_BM_ASR_PROPAGATE_ERRORS") == "1": + text = f"{_ASR_ERROR_PREFIX}{response.error_code}|{message}" + await result_queue.put(ASRResult(text=text, is_final=True)) break elif response.message_type == ResponseMessageType.server_ack: diff --git a/src/ghoshell_moss/host/speech/volcengine_tts/tts.py b/src/ghoshell_moss/host/speech/volcengine_tts/tts.py index a7169533..fc3e3ea4 100644 --- a/src/ghoshell_moss/host/speech/volcengine_tts/tts.py +++ b/src/ghoshell_moss/host/speech/volcengine_tts/tts.py @@ -713,7 +713,12 @@ async def _start_consuming_batch_loop(self, batch: VolcengineTTSBatch): url = os.environ.get("VOLCENGINE_STREAM_TTS_URL", self._conf.url) # 创建初始连接. self.logger.info("%s prepare to connect to %s with header %s", self._log_prefix, url, header) - async with connect(url, additional_headers=header, proxy=None) as ws: + connect_kwargs = {"additional_headers": header} + if os.environ.get("VOLCENGINE_STREAM_TTS_DISABLE_PROXY") == "1": + # 默认尊重系统代理;只有在实时语音本地链路明确不希望走代理时 + # 才禁用。这样不会影响依赖代理访问火山 TTS 的普通部署环境。 + connect_kwargs["proxy"] = None + async with connect(url, **connect_kwargs) as ws: # 建连确认. await start_connection(ws) self.logger.debug("%s start connection %s", self._log_prefix, connection_id) diff --git a/tests/ghoshell_moss/core/mindflow/test_audio_nucleus.py b/tests/ghoshell_moss/core/mindflow/test_audio_nucleus.py new file mode 100644 index 00000000..3e553f4f --- /dev/null +++ b/tests/ghoshell_moss/core/mindflow/test_audio_nucleus.py @@ -0,0 +1,48 @@ +import asyncio + +import pytest + +from ghoshell_moss.core.mindflow.audio_nucleus import AudioNucleus +from ghoshell_moss.core.mindflow.audio_signal import AudioAction, AudioSignal + + +def _speech_final_signal(): + return AudioSignal(action=AudioAction.SPEECH_FINAL).to_signal("hello") + + +def _audio_nucleus(*, interrupt_on_complete: bool = True) -> AudioNucleus: + return AudioNucleus( + name="audio_nucleus", + description="audio perception signal nucleus", + target_signal="audio", + default_prompt="User spoke via voice input. Process the speech.", + interrupt_on_complete=interrupt_on_complete, + ) + + +@pytest.mark.asyncio +async def test_audio_nucleus_complete_impulse_interrupts_by_default(): + nucleus = _audio_nucleus() + + async with nucleus: + nucleus.add_signal(_speech_final_signal()) + await asyncio.sleep(0.1) + + impulse = nucleus.peek() + assert impulse is not None + assert impulse.complete is True + assert impulse.interrupt is True + + +@pytest.mark.asyncio +async def test_audio_nucleus_can_disable_complete_impulse_interrupt(): + nucleus = _audio_nucleus(interrupt_on_complete=False) + + async with nucleus: + nucleus.add_signal(_speech_final_signal()) + await asyncio.sleep(0.1) + + impulse = nucleus.peek() + assert impulse is not None + assert impulse.complete is True + assert impulse.interrupt is False