From e9a048ef607554b31b634e52cac52bcbe5452d63 Mon Sep 17 00:00:00 2001
From: LpLegend <2422019509@qq.com>
Date: Thu, 9 Jul 2026 22:55:43 +0800
Subject: [PATCH 1/3] feat: consolidate aether app runtime coding by gpt-5
- move Aether core, listener, and VPIO capture under .moss_ws/apps/aether
- add Aether README quick-start and remove legacy listener entry
- harden Aether voice loop ASR/TTS interrupt handling
Tests:
- .venv/bin/python -m py_compile .moss_ws/apps/aether/core/main.py .moss_ws/apps/aether/listener/main.py .moss_ws/apps/aether/vpio_capture/main.py .moss_ws/apps/aether/vpio_capture/vpio_capture.py
- .venv/bin/python -m pytest tests/ghoshell_moss/host/speech/volcengine_asr/test_protocol.py
- .venv/bin/moss --ai --mode aether apps list
- .venv/bin/moss-run-ghost echo --mode aether
via Codex
---
.moss_ws/apps/aether/README.md | 241 +++++++
.moss_ws/apps/aether/core/.gitignore | 8 +
.moss_ws/apps/aether/core/APP.md | 15 +
.moss_ws/apps/aether/core/main.py | 422 ++++++++++++
.moss_ws/apps/aether/core/pyproject.toml | 16 +
.moss_ws/apps/aether/core/webroot/index.html | 143 ++++
.../apps/aether/core/webroot/web/core.frag | 148 +++++
.../apps/aether/core/webroot/web/core.vert | 6 +
.moss_ws/apps/aether/core/webroot/web/main.js | 551 ++++++++++++++++
.../apps/aether/core/webroot/web/scene.js | 188 ++++++
.../aether/core/webroot/web/state_mapper.js | 187 ++++++
.../apps/aether/core/webroot/web/style.css | 553 ++++++++++++++++
.moss_ws/apps/aether/core/webroot/web/vad.js | 162 +++++
.moss_ws/apps/aether/core/webroot/web/ws.js | 119 ++++
.../{sensors => aether}/listener/.env.example | 0
.../{sensors => aether}/listener/.gitignore | 0
.moss_ws/apps/aether/listener/APP.md | 15 +
.moss_ws/apps/aether/listener/main.py | 614 ++++++++++++++++++
.../listener/pyproject.toml | 0
.moss_ws/apps/aether/vpio_capture/.gitignore | 9 +
.moss_ws/apps/aether/vpio_capture/APP.md | 10 +
.moss_ws/apps/aether/vpio_capture/main.py | 61 ++
.../apps/aether/vpio_capture/pyproject.toml | 16 +
.../apps/aether/vpio_capture/smoke_test.py | 141 ++++
.../apps/aether/vpio_capture/vpio_capture.py | 607 +++++++++++++++++
.moss_ws/apps/bodies/g1_sim/APP.md | 4 +-
.moss_ws/apps/bodies/g1_sim/README.md | 2 +-
.moss_ws/apps/sensors/listener/APP.md | 10 -
.moss_ws/apps/sensors/listener/CLAUDE.md | 66 --
.moss_ws/apps/sensors/listener/main.py | 341 ----------
.moss_ws/configs/llm.yml | 22 +-
.moss_ws/ghosts/echo/soul.md | 11 +
.moss_ws/src/MOSS/modes/aether/MODE.md | 19 +
.moss_ws/src/MOSS/modes/aether/__init__.py | 0
.moss_ws/src/MOSS/modes/aether/channels.py | 17 +
.moss_ws/src/MOSS/modes/aether/configs.py | 1 +
.moss_ws/src/MOSS/modes/aether/contracts.py | 1 +
.moss_ws/src/MOSS/modes/aether/nuclei.py | 6 +
.moss_ws/src/MOSS/modes/aether/providers.py | 1 +
.moss_ws/src/MOSS/modes/aether/resources.py | 1 +
.moss_ws/src/MOSS/modes/aether/topics.py | 1 +
.moss_ws/src/MOSS/modes/listener/MODE.md | 5 +-
.moss_ws/src/MOSS/modes/show/MODE.md | 4 +-
.../core/mindflow/audio_nucleus.py | 10 +-
.../core/speech/stream_tts_speech.py | 8 +
src/ghoshell_moss/ghosts/atom/_adapter.py | 18 +-
src/ghoshell_moss/ghosts/atom/_meta.py | 62 +-
src/ghoshell_moss/ghosts/atom/_runtime.py | 25 +-
src/ghoshell_moss/host/app_store.py | 16 +-
src/ghoshell_moss/host/ghost_runtime.py | 30 +
.../host/speech/volcengine_asr/config.py | 26 +-
.../host/speech/volcengine_asr/protocol.py | 98 ++-
.../host/speech/volcengine_asr/recognizer.py | 103 ++-
.../host/speech/volcengine_tts/tts.py | 16 +-
.../speech/volcengine_asr/test_protocol.py | 15 +
55 files changed, 4671 insertions(+), 500 deletions(-)
create mode 100644 .moss_ws/apps/aether/README.md
create mode 100644 .moss_ws/apps/aether/core/.gitignore
create mode 100644 .moss_ws/apps/aether/core/APP.md
create mode 100644 .moss_ws/apps/aether/core/main.py
create mode 100644 .moss_ws/apps/aether/core/pyproject.toml
create mode 100644 .moss_ws/apps/aether/core/webroot/index.html
create mode 100644 .moss_ws/apps/aether/core/webroot/web/core.frag
create mode 100644 .moss_ws/apps/aether/core/webroot/web/core.vert
create mode 100644 .moss_ws/apps/aether/core/webroot/web/main.js
create mode 100644 .moss_ws/apps/aether/core/webroot/web/scene.js
create mode 100644 .moss_ws/apps/aether/core/webroot/web/state_mapper.js
create mode 100644 .moss_ws/apps/aether/core/webroot/web/style.css
create mode 100644 .moss_ws/apps/aether/core/webroot/web/vad.js
create mode 100644 .moss_ws/apps/aether/core/webroot/web/ws.js
rename .moss_ws/apps/{sensors => aether}/listener/.env.example (100%)
rename .moss_ws/apps/{sensors => aether}/listener/.gitignore (100%)
create mode 100644 .moss_ws/apps/aether/listener/APP.md
create mode 100644 .moss_ws/apps/aether/listener/main.py
rename .moss_ws/apps/{sensors => aether}/listener/pyproject.toml (100%)
create mode 100644 .moss_ws/apps/aether/vpio_capture/.gitignore
create mode 100644 .moss_ws/apps/aether/vpio_capture/APP.md
create mode 100644 .moss_ws/apps/aether/vpio_capture/main.py
create mode 100644 .moss_ws/apps/aether/vpio_capture/pyproject.toml
create mode 100644 .moss_ws/apps/aether/vpio_capture/smoke_test.py
create mode 100644 .moss_ws/apps/aether/vpio_capture/vpio_capture.py
delete mode 100644 .moss_ws/apps/sensors/listener/APP.md
delete mode 100644 .moss_ws/apps/sensors/listener/CLAUDE.md
delete mode 100644 .moss_ws/apps/sensors/listener/main.py
create mode 100644 .moss_ws/src/MOSS/modes/aether/MODE.md
create mode 100644 .moss_ws/src/MOSS/modes/aether/__init__.py
create mode 100644 .moss_ws/src/MOSS/modes/aether/channels.py
create mode 100644 .moss_ws/src/MOSS/modes/aether/configs.py
create mode 100644 .moss_ws/src/MOSS/modes/aether/contracts.py
create mode 100644 .moss_ws/src/MOSS/modes/aether/nuclei.py
create mode 100644 .moss_ws/src/MOSS/modes/aether/providers.py
create mode 100644 .moss_ws/src/MOSS/modes/aether/resources.py
create mode 100644 .moss_ws/src/MOSS/modes/aether/topics.py
diff --git a/.moss_ws/apps/aether/README.md b/.moss_ws/apps/aether/README.md
new file mode 100644
index 00000000..19c7a2cf
--- /dev/null
+++ b/.moss_ws/apps/aether/README.md
@@ -0,0 +1,241 @@
+# Aether App
+
+Aether 是 MOSS 的实时语音交互外壳。它把麦克风采集、ASR、Ghost 思考、TTS 播放和前端可视化连成一条闭环,用来验证 MOSS 能不能像一个在场的智能体一样听、想、说、被打断。
+
+这个目录是 Aether 相关 app 的唯一维护入口。后续理解、启动、排错,优先看本 README 和本目录代码。
+
+## 目录结构
+
+```text
+.moss_ws/apps/aether/
+ core/ 前端可视化和 WebSocket 状态聚合
+ listener/ 音频到 ASR,再到 SpeechTopic / AudioSignal
+ vpio_capture/ macOS VPIO 音频采集和系统级回声消除
+```
+
+三个 app 的 canonical address 是:
+
+```text
+aether/core
+aether/listener
+aether/vpio_capture
+```
+
+旧地址 `ui/aether_core`、`sensors/listener`、`sensors/vpio_capture` 已不再作为 Aether 入口使用。
+
+## 一键启动
+
+从仓库根目录执行:
+
+```bash
+.venv/bin/moss-run-ghost echo --mode aether
+```
+
+启动后打开:
+
+```text
+http://127.0.0.1:8765/
+```
+
+`aether` mode 会自动拉起:
+
+```text
+aether/vpio_capture
+aether/listener
+aether/core
+```
+
+关闭全部 Aether/MOSS runtime 进程:
+
+```bash
+.venv/bin/moss --ai --mode aether runtime kill-all --yes
+```
+
+查看当前还在运行的 cell:
+
+```bash
+.venv/bin/moss --ai --mode aether runtime list-cells
+```
+
+## 单独调试
+
+只启动前端可视化:
+
+```bash
+.venv/bin/moss --ai --mode aether apps test aether/core
+```
+
+只启动 ASR listener。排查时建议先用 manual 模式,避免连续 ASR 抢占调试过程:
+
+```bash
+LISTENER_ASR_MODE=manual .venv/bin/moss --ai --mode aether apps test aether/listener
+```
+
+只启动 macOS VPIO 采集:
+
+```bash
+.venv/bin/moss --ai --mode aether apps test aether/vpio_capture
+```
+
+## 组件职责
+
+### aether/vpio_capture
+
+`aether/vpio_capture` 是音频采集层。
+
+它负责:
+
+- 使用 macOS AVAudioEngine + VPIO 采集麦克风输入。
+- 打开系统级 voice processing / echo cancellation,降低 TTS 外放被 ASR 收回去的概率。
+- 把音频转成 listener 需要的 16kHz mono PCM。
+- 发布 VPIO 运行诊断,例如 RMS、peak、channel、frame count、stall/restart。
+
+它不负责 ASR、不负责判断用户意图、不负责调用 Ghost。
+
+### aether/listener
+
+`aether/listener` 是语音识别层。
+
+它负责:
+
+- 消费音频采集 topic。
+- 连接 Volcengine streaming ASR。
+- 发布 `SpeechTopic`,让 MOSS/Ghost 收到用户说完的一句话。
+- 发布 `AudioSignal`,告诉 Mindflow 用户开始说话、最终文本完成。
+- 监听 `asr_control`,在 continuous/manual 两种收音模式之间切换。
+- 在用户开口或停止意图出现时发布 interrupt 相关信号。
+
+它不负责前端绘制、不负责 TTS 播放、不负责 LLM 推理。
+
+### aether/core
+
+`aether/core` 是前端状态聚合层。
+
+它负责:
+
+- 提供 `http://127.0.0.1:8765/` 页面。
+- 订阅 `SpeechTopic` 和 `AudioRuntimeTopic`。
+- 把 listen、think、speak、interrupt、idle 等状态合成给 WebSocket 前端。
+- 接收前端按钮或浏览器 VAD 产生的控制事件。
+- 把 ASR 模式切换写回 `asr_control` topic。
+
+它不负责直接启动其他 app,不直接调用 ASR/TTS/LLM,不绕过 MOSS runtime 伪造 Ghost 输入。
+
+## 数据流
+
+正常语音链路:
+
+```text
+麦克风
+ -> aether/vpio_capture
+ -> audio topic
+ -> aether/listener
+ -> SpeechTopic + AudioSignal
+ -> MOSS Mindflow / Ghost
+ -> TTS
+ -> speaker diagnostics
+ -> aether/core
+ -> browser WebGL state
+```
+
+前端控制链路:
+
+```text
+browser button / VAD
+ -> aether/core WebSocket
+ -> AudioRuntimeTopic(asr_control / interrupt)
+ -> aether/listener 或 MOSS host runtime
+```
+
+## 关键 Topic
+
+| Topic | 发布者 | 消费者 | 作用 |
+| --- | --- | --- | --- |
+| `SpeechTopic` | `aether/listener` | Ghost / `aether/core` | 用户一句完整语音文本 |
+| `AudioSignal(SPEECH_STARTED)` | `aether/listener` | Mindflow | 用户已经开始说话 |
+| `AudioSignal(SPEECH_FINAL)` | `aether/listener` | Mindflow | 用户一句话完成 |
+| `AudioRuntimeTopic(device_name="vpio")` | `aether/vpio_capture` | `aether/core` | 采集状态和音量诊断 |
+| `AudioRuntimeTopic(device_name="asr")` | `aether/listener` | `aether/core` | ASR running/partial/final/error/idle |
+| `AudioRuntimeTopic(device_name="asr_control")` | `aether/core` | `aether/listener` | 连续/手动 ASR 控制 |
+| `AudioRuntimeTopic(device_name="speaker")` | TTS/player | `aether/core` | TTS 播放状态 |
+| `AudioRuntimeTopic(device_name="interrupt")` | `aether/core` / `aether/listener` | `aether/core` / host runtime | 打断当前输出 |
+
+## 常见排错
+
+### 页面打不开
+
+确认 `aether/core` 是否启动:
+
+```bash
+.venv/bin/moss --ai --mode aether runtime list-cells
+```
+
+确认 8765 端口是否有响应:
+
+```bash
+curl -s http://127.0.0.1:8765/
+```
+
+### 没有声音输入
+
+先单独启动 VPIO:
+
+```bash
+.venv/bin/moss --ai --mode aether apps test aether/vpio_capture
+```
+
+看日志里是否持续出现 RMS/peak 变化。如果 RMS 长时间为 0,优先检查系统麦克风权限、默认输入设备、采样权限。
+
+### 停顿后 ASR 不再识别
+
+优先检查 `aether/listener` 日志:
+
+- 是否还在读取 audio frames。
+- 是否还有 ASR partial/final。
+- 是否进入了 error 或 idle 后没有恢复。
+- `asr_control` 当前是否被切到 manual。
+
+如果只是调试 listener,先用:
+
+```bash
+LISTENER_ASR_MODE=manual .venv/bin/moss --ai --mode aether apps test aether/listener
+```
+
+这样可以把连续收音问题和 ASR 服务问题分开看。
+
+### TTS 外放被 ASR 收进去
+
+确认当前 mode 使用的是:
+
+```text
+aether/vpio_capture
+```
+
+不要用普通 `sensors/audio_capture` 来验证全双工效果。普通采集没有 macOS VPIO 的系统级 AEC,容易把扬声器声音重新送进 ASR。
+
+### 停不下来
+
+先杀当前 mode 下的 runtime:
+
+```bash
+.venv/bin/moss --ai --mode aether runtime kill-all --yes
+```
+
+再确认:
+
+```bash
+.venv/bin/moss --ai --mode aether runtime list-cells
+```
+
+输出 `No cells found in this scope` 才表示 MOSS runtime cell 已清空。
+
+## 维护边界
+
+Aether 代码以后尽量收敛在本目录内:
+
+- 前端和 WebSocket 状态聚合放在 `aether/core/`。
+- ASR listener 放在 `aether/listener/`。
+- macOS VPIO 采集放在 `aether/vpio_capture/`。
+- MOSS host、Mindflow、Speech provider 的公共能力仍放在 `src/ghoshell_moss/`,不要复制到 app 目录。
+
+如果新增说明文档,优先更新这个 README。不要再在仓库根 `Docs/` 下新增 Aether 历史说明。
diff --git a/.moss_ws/apps/aether/core/.gitignore b/.moss_ws/apps/aether/core/.gitignore
new file mode 100644
index 00000000..63f5e295
--- /dev/null
+++ b/.moss_ws/apps/aether/core/.gitignore
@@ -0,0 +1,8 @@
+*.log
+runtime/
+venv/
+.venv/
+__pycache__/
+*.pyc
+.env
+uv.lock
diff --git a/.moss_ws/apps/aether/core/APP.md b/.moss_ws/apps/aether/core/APP.md
new file mode 100644
index 00000000..48870ba9
--- /dev/null
+++ b/.moss_ws/apps/aether/core/APP.md
@@ -0,0 +1,15 @@
+---
+arguments: ''
+description: 'Aether Core UI — canonical aether/core app,聚合 SpeechTopic/AudioRuntimeTopic 并通过 WebSocket 驱动能量核心'
+executable: uv
+respawn: false
+script: main.py
+workers: 1
+---
+
+Aether Core UI app — MOSS ghost 的能量核心可视化通道。Canonical app
+address: `aether/core`.
+
+订阅 listener 的 SpeechTopic(用户说完一句 → think)和 TTS player 的 AudioRuntimeTopic(speaker running → speak/idle),通过 WebSocket 把状态推给前端 WebGL 能量核心。
+
+前端 VAD 快线检测到开口时,通过 WebSocket 发 interrupt 消息,后端推 interrupt 状态(急刹冻结视觉),实现 v2 技术设计的 <50ms 爆点。
diff --git a/.moss_ws/apps/aether/core/main.py b/.moss_ws/apps/aether/core/main.py
new file mode 100644
index 00000000..df433214
--- /dev/null
+++ b/.moss_ws/apps/aether/core/main.py
@@ -0,0 +1,422 @@
+"""Aether Core UI app — MOSS ghost 的全双工能量核心可视化通道。
+
+后端不再把 listen/think/speak/interrupt 压成互斥五态,而是维护
+并发 activity layers。``state`` 只作为主视觉基调向旧前端兼容:
+listen、think、speak 可以同时为 true,interrupt 是短暂抢占层。
+"""
+import asyncio
+import json
+import logging
+import time
+from pathlib import Path
+
+from aiohttp import web, WSMsgType
+
+from ghoshell_moss.core.blueprint.matrix import Matrix
+from ghoshell_moss.core.mindflow.interrupt_nucleus import new_interrupt_signal
+from ghoshell_moss.host.speech.capture.matrix_audio_transport import MatrixAudioTransport
+from ghoshell_moss.topics.audio import AudioRuntimeTopic, SpeechTopic
+
+# Frontend static files live with the app so aether/core is self-contained.
+WEB_ROOT = Path(__file__).resolve().parent / "webroot"
+WS_PORT = 8765
+INTERRUPT_HOLD = 0.7 # 急刹视觉保持时长(秒),超时回 idle
+
+_log = logging.getLogger("moss.aether.core")
+
+
+@web.middleware
+async def _cross_origin_isolation_middleware(
+ request: web.Request,
+ handler,
+) -> web.StreamResponse:
+ response = await handler(request)
+ # Keep the UI cross-origin isolated so future WebGL/AudioWorklet features
+ # can use SharedArrayBuffer without changing the deployment surface.
+ response.headers["Cross-Origin-Opener-Policy"] = "same-origin"
+ response.headers["Cross-Origin-Embedder-Policy"] = "require-corp"
+ response.headers["Cross-Origin-Resource-Policy"] = "same-origin"
+ return response
+
+
+async def main(matrix: Matrix) -> None:
+ logger = matrix.logger or _log
+ logger.info("aether/core app starting, WEB_ROOT=%s", WEB_ROOT)
+
+ transport = MatrixAudioTransport(matrix=matrix)
+ speech_win = transport.topic_window(SpeechTopic, max_size=16)
+ audio_win = transport.topic_window(AudioRuntimeTopic, max_size=16)
+
+ clients: set = set()
+ # 状态容器(避免 nonlocal 地狱)。state 是兼容字段;layers 是新契约。
+ st = {
+ "state": "idle",
+ "layers": {
+ "listen": False,
+ "queue": False,
+ "think": False,
+ "speak": False,
+ "interrupt": False,
+ },
+ "last_speech_ts": 0.0,
+ "last_speaker_running": False,
+ "last_asr_running": False,
+ "interrupt_until": 0.0,
+ "last_interrupt_started_at": 0.0,
+ "_tts_end_at": 0.0,
+ "think_started_at": 0.0,
+ "queued_started_at": 0.0,
+ "last_asr_diag_key": "",
+ "last_vpio_diag_key": "",
+ "asr_current": None,
+ "asr_finals": [],
+ "asr_error": None,
+ "asr_control": {"mode": "continuous", "enabled": True},
+ "vpio_diag": "",
+ }
+ # 初始化 last_speech_ts 为当前 window 的最大值,避免启动即触发历史
+ init_speeches = list(speech_win.values())
+ if init_speeches:
+ st["last_speech_ts"] = max(t.timestamp for t in init_speeches)
+
+ def _primary_state() -> str:
+ layers = st["layers"]
+ if layers["interrupt"]:
+ return "interrupt"
+ if layers["speak"]:
+ return "speak"
+ if layers["think"]:
+ return "think"
+ if layers["listen"]:
+ return "listen"
+ return "idle"
+
+ def _snapshot(**extra) -> dict:
+ st["state"] = _primary_state()
+ msg = {
+ "state": st["state"],
+ "layers": dict(st["layers"]),
+ "ts": time.time(),
+ "diagnostics": {
+ "asr_current": dict(st["asr_current"]) if st["asr_current"] else None,
+ "asr_finals": list(st["asr_finals"]),
+ "asr_error": dict(st["asr_error"]) if st["asr_error"] else None,
+ "asr_control": dict(st["asr_control"]),
+ "vpio": st["vpio_diag"],
+ },
+ }
+ msg.update(extra)
+ return msg
+
+ async def broadcast(msg: dict) -> None:
+ if not clients:
+ return
+ data = json.dumps(msg, ensure_ascii=False)
+ dead = []
+ for c in clients:
+ try:
+ await c.send_str(data)
+ except Exception:
+ dead.append(c)
+ for c in dead:
+ clients.discard(c)
+
+ async def ws_handler(request: web.Request) -> web.WebSocketResponse:
+ ws = web.WebSocketResponse()
+ await ws.prepare(request)
+ clients.add(ws)
+ logger.info("ws client connected, total=%d", len(clients))
+ # 连接时先推当前状态
+ await ws.send_str(json.dumps(_snapshot(), ensure_ascii=False))
+ try:
+ async for msg in ws:
+ if msg.type == WSMsgType.TEXT:
+ try:
+ payload = json.loads(msg.data)
+ except Exception:
+ continue
+ if payload.get("type") == "interrupt" or payload.get("state") == "interrupt":
+ st["interrupt_until"] = time.monotonic() + INTERRUPT_HOLD
+ st["layers"]["interrupt"] = True
+ st["layers"]["listen"] = False
+ st["layers"]["queue"] = False
+ st["layers"]["speak"] = False
+ st["layers"]["think"] = False
+ await broadcast(_snapshot(interrupt_burst=1.0))
+ logger.info("interrupt received from frontend")
+ transport.pub_topic(AudioRuntimeTopic(
+ running=True,
+ device_name="interrupt",
+ device_explain="frontend_manual_stop",
+ started_at=time.monotonic(),
+ last_heartbeat=time.monotonic(),
+ ))
+ logger.info("★ Frontend manual stop → audio interrupt topic published")
+ # 发 interrupt signal 到 ghost 主进程 (通过 Zenoh 跨进程)
+ # → mindflow.InterruptNucleus → FATAL impulse → shell.clear() → 停 TTS + 停 LLM
+ # aether/core 是独立子进程,不能直接访问 ghost 的 Mindflow,
+ # 必须通过 session.add_signal 走 Zenoh 发布。
+ try:
+ sig = new_interrupt_signal(
+ "立刻停下",
+ description="前端VAD检测到SPEAK中开口,触发shell.clear停TTS",
+ )
+ matrix.session.add_signal(sig)
+ logger.info("★ Interrupt signal sent via zenoh (shell.clear will stop TTS)")
+ except Exception as e:
+ logger.warning("Failed to send interrupt signal: %s", e)
+ elif payload.get("type") == "asr_control":
+ mode = str(payload.get("mode", "continuous")).strip().lower()
+ if mode not in {"continuous", "manual"}:
+ mode = "continuous"
+ enabled = bool(payload.get("enabled", mode == "continuous"))
+ if mode == "continuous":
+ enabled = True
+ st["asr_control"] = {"mode": mode, "enabled": enabled}
+ transport.pub_topic(AudioRuntimeTopic(
+ running=enabled,
+ device_name="asr_control",
+ device_explain=json.dumps(
+ {"source": "aether/core", "mode": mode, "enabled": enabled},
+ ensure_ascii=False,
+ separators=(",", ":"),
+ ),
+ started_at=time.monotonic(),
+ last_heartbeat=time.monotonic(),
+ ))
+ await broadcast(_snapshot(event="asr_control"))
+ logger.info("ASR control from frontend: mode=%s enabled=%s", mode, enabled)
+ elif payload.get("type") == "listen":
+ running = bool(payload.get("running"))
+ st["layers"]["listen"] = running
+ if payload.get("pending_think"):
+ st["layers"]["think"] = True
+ st["think_started_at"] = time.monotonic()
+ await broadcast(_snapshot(event="listen"))
+ elif payload.get("type") == "reset":
+ # 切 idle。TopicWindow 不提供 clear;用 last_speech_ts 跳过已有历史。
+ # 不发布 "/reset" SpeechTopic,否则 reset 会被当作一轮用户语音,
+ # 造成一次没有实际回复需求的假 think。
+ st["layers"] = {k: False for k in st["layers"]}
+ speeches = list(speech_win.values())
+ if speeches:
+ st["last_speech_ts"] = max(t.timestamp for t in speeches)
+ else:
+ st["last_speech_ts"] = time.monotonic()
+ st["think_started_at"] = 0
+ st["queued_started_at"] = 0
+ await broadcast(_snapshot())
+ logger.info("reset received from frontend — visual context cleared")
+ elif msg.type == WSMsgType.ERROR:
+ break
+ finally:
+ clients.discard(ws)
+ logger.info("ws client disconnected, total=%d", len(clients))
+ return ws
+
+ async def index_handler(request: web.Request) -> web.FileResponse:
+ return web.FileResponse(WEB_ROOT / "index.html")
+
+ app = web.Application(middlewares=[_cross_origin_isolation_middleware])
+ app.router.add_get("/", index_handler)
+ app.router.add_get("/ws", ws_handler)
+ app.router.add_static("/web", path=str(WEB_ROOT / "web"), name="web")
+
+ runner = web.AppRunner(app)
+ await runner.setup()
+ site = web.TCPSite(runner, "0.0.0.0", WS_PORT)
+ await site.start()
+ logger.info("aether/core http+ws server on http://127.0.0.1:%d", WS_PORT)
+
+ async def state_loop() -> None:
+ while True:
+ now = time.monotonic()
+ now_ts = time.time()
+ # interrupt 超时回退
+ if st["layers"]["interrupt"] and now > st["interrupt_until"]:
+ st["layers"]["interrupt"] = False
+ await broadcast(_snapshot())
+
+ # think 超时兜底:LLM 崩溃/超时时不卡在 think
+ if st["layers"]["think"] and now - st.get("think_started_at", 0) > 30.0:
+ st["layers"]["think"] = False
+ st["layers"]["queue"] = False
+ st["think_started_at"] = 0
+ st["queued_started_at"] = 0
+ await broadcast(_snapshot())
+ logger.warning("think timeout 30s → %s", _primary_state())
+
+ # 全双工打断:检测 listener 发布的"停下"关键词 interrupt 信号
+ for t in reversed(list(audio_win.values())):
+ if getattr(t, "device_name", "") == "interrupt" and t.running:
+ started_at = float(getattr(t, "started_at", 0.0) or 0.0)
+ if started_at > st["last_interrupt_started_at"]:
+ st["last_interrupt_started_at"] = started_at
+ st["layers"]["interrupt"] = True
+ st["layers"]["listen"] = False
+ st["layers"]["queue"] = False
+ st["layers"]["speak"] = False
+ st["layers"]["think"] = False
+ st["interrupt_until"] = now + INTERRUPT_HOLD
+ await broadcast(_snapshot(interrupt_burst=1.0))
+ logger.info("★ Wake word barge-in → interrupt (from listener)")
+ break
+
+ # 检查 speech(用户说完一句 → think)
+ speeches = list(speech_win.values())
+ if speeches:
+ latest = speeches[-1]
+ if getattr(latest, "role", "") == "human" and latest.timestamp > st["last_speech_ts"]:
+ st["last_speech_ts"] = latest.timestamp
+ if not st["layers"]["interrupt"]:
+ queued = bool(st["layers"]["speak"])
+ st["layers"]["listen"] = False
+ if queued:
+ st["layers"]["queue"] = True
+ st["queued_started_at"] = now
+ st["layers"]["think"] = True
+ st["think_started_at"] = now
+ await broadcast(_snapshot(text=latest.text))
+ logger.info("%s: %s", "speech→queue+think" if queued else "speech→think", latest.text[:60])
+
+ # 后端 ASR 活动(区别于浏览器本地 VAD):ASR partial 出现才是真正
+ # 进入后端听写链路。若 ASR 空等或浏览器误触,不能伪装成 think。
+ asr_running = None
+ asr_topic = None
+ for t in reversed(list(audio_win.values())):
+ if getattr(t, "device_name", "") == "asr":
+ asr_topic = t
+ asr_running = t.running
+ break
+ if asr_topic is not None and asr_topic.running:
+ explain = getattr(asr_topic, "device_explain", "") or ""
+ diag_key = f"{getattr(asr_topic, 'started_at', 0)}:{explain}"
+ if explain and diag_key != st["last_asr_diag_key"]:
+ st["last_asr_diag_key"] = diag_key
+ changed = False
+ try:
+ parsed = json.loads(explain)
+ if parsed.get("error"):
+ st["asr_error"] = {
+ "error": str(parsed.get("error")),
+ "code": str(parsed.get("code", "")),
+ "message": str(parsed.get("message", "")),
+ "backoff": float(parsed.get("backoff", 0) or 0),
+ "consecutive": int(parsed.get("consecutive", 0) or 0),
+ "ts": now_ts,
+ }
+ changed = True
+ text = str(parsed.get("text", "")).strip()
+ if text:
+ item = {
+ "text": text,
+ "final": bool(parsed.get("final")),
+ "ts": now_ts,
+ }
+ st["asr_error"] = None
+ if item["final"]:
+ st["asr_current"] = None
+ st["asr_finals"].append(item)
+ st["asr_finals"] = st["asr_finals"][-3:]
+ else:
+ st["asr_current"] = item
+ changed = True
+ except Exception:
+ st["asr_current"] = {"text": explain, "final": False, "ts": now_ts}
+ changed = True
+ if changed:
+ await broadcast(_snapshot(event="asr_diag"))
+ elif asr_topic is not None and not asr_topic.running:
+ explain = getattr(asr_topic, "device_explain", "") or ""
+ diag_key = f"{getattr(asr_topic, 'started_at', 0)}:{explain}"
+ if explain and diag_key != st["last_asr_diag_key"]:
+ st["last_asr_diag_key"] = diag_key
+ try:
+ parsed = json.loads(explain)
+ except Exception:
+ parsed = {}
+ if parsed.get("error"):
+ st["asr_error"] = {
+ "error": str(parsed.get("error")),
+ "code": str(parsed.get("code", "")),
+ "message": str(parsed.get("message", "")),
+ "backoff": float(parsed.get("backoff", 0) or 0),
+ "consecutive": int(parsed.get("consecutive", 0) or 0),
+ "ts": now_ts,
+ }
+ await broadcast(_snapshot(event="asr_error"))
+ if asr_running is not None and asr_running != st["last_asr_running"]:
+ st["last_asr_running"] = asr_running
+ if asr_running:
+ if not st["layers"]["interrupt"]:
+ st["layers"]["listen"] = True
+ await broadcast(_snapshot(event="asr_listen"))
+ logger.info("ASR→listen")
+ else:
+ if st["layers"]["listen"] and not st["layers"]["think"] and not st["layers"]["speak"]:
+ st["layers"]["listen"] = False
+ await broadcast(_snapshot(event="asr_idle"))
+ logger.info("ASR listen ended → idle")
+
+ vpio_topic = None
+ for t in reversed(list(audio_win.values())):
+ if getattr(t, "device_name", "") == "vpio":
+ vpio_topic = t
+ break
+ if vpio_topic is not None and vpio_topic.running:
+ explain = getattr(vpio_topic, "device_explain", "") or ""
+ diag_key = f"{getattr(vpio_topic, 'last_heartbeat', 0)}:{explain}"
+ if explain and diag_key != st["last_vpio_diag_key"]:
+ st["last_vpio_diag_key"] = diag_key
+ st["vpio_diag"] = explain
+ await broadcast(_snapshot(event="vpio_diag"))
+
+ # 检查 TTS speaker(running → speak,stopped → idle)
+ running = None
+ for t in reversed(list(audio_win.values())):
+ if getattr(t, "device_name", "") == "speaker":
+ running = t.running
+ break
+ if running is not None and running != st["last_speaker_running"]:
+ st["last_speaker_running"] = running
+ if running:
+ if not st["layers"]["interrupt"]:
+ st["_tts_end_at"] = 0.0
+ if st["layers"]["queue"]:
+ st["layers"]["queue"] = False
+ st["queued_started_at"] = 0
+ st["layers"]["speak"] = True
+ # LLM may continue preparing the next delta while TTS starts;
+ # leave think true briefly only if a new speech turn owns it.
+ await broadcast(_snapshot())
+ logger.info("TTS→speak")
+ else:
+ # TTS 结束后加 800ms 保护期,让尾音播完,避免 VAD 误触 listen
+ if st["layers"]["speak"] or st["layers"]["interrupt"]:
+ st["_tts_end_at"] = now
+ logger.info("TTS ended → 800ms grace before idle")
+
+ # TTS 结束保护期:800ms 后切 idle
+ if st.get("_tts_end_at") and now - st["_tts_end_at"] > 0.8:
+ if (st["layers"]["speak"] or st["layers"]["interrupt"]) and not st["last_speaker_running"]:
+ st["layers"]["speak"] = False
+ st["layers"]["interrupt"] = False
+ if not st["layers"]["queue"] and st.get("think_started_at", 0) <= st.get("_tts_end_at", 0):
+ st["layers"]["think"] = False
+ await broadcast(_snapshot())
+ logger.info("TTS grace end → %s", _primary_state())
+ st["_tts_end_at"] = 0.0
+
+ await asyncio.sleep(0.03)
+
+ try:
+ await state_loop()
+ except asyncio.CancelledError:
+ logger.info("aether/core cancelled")
+ finally:
+ await runner.cleanup()
+ logger.info("aether/core stopped")
+
+
+if __name__ == "__main__":
+ Matrix.discover().run(main)
diff --git a/.moss_ws/apps/aether/core/pyproject.toml b/.moss_ws/apps/aether/core/pyproject.toml
new file mode 100644
index 00000000..8b167da8
--- /dev/null
+++ b/.moss_ws/apps/aether/core/pyproject.toml
@@ -0,0 +1,16 @@
+[project]
+name = "aether_core"
+version = "0.1.0"
+requires-python = ">=3.11,<3.14"
+dependencies = [
+ "ghoshell-moss[host]",
+ "aiohttp>=3.9",
+]
+
+[tool.uv]
+override-dependencies = [
+ "numpy<2.3",
+]
+
+[tool.uv.sources]
+ghoshell-moss = { path = "../../../..", editable = true }
diff --git a/.moss_ws/apps/aether/core/webroot/index.html b/.moss_ws/apps/aether/core/webroot/index.html
new file mode 100644
index 00000000..91a317ed
--- /dev/null
+++ b/.moss_ws/apps/aether/core/webroot/index.html
@@ -0,0 +1,143 @@
+
+
+
+
+
+MOSS · Aether Core 能量核心
+
+
+
+
+
+
+
+
+
MOSSAETHER CORE · 能量核心
+
本地演示模式
+
+
+
+
+
+
+
+
MIC
+
LISTEN
+
QUEUE
+
THINK
+
SPEAK
+
STOP
+
+
+
+
+
并发层
+
→ idle 呼吸
+
+ listen ASR 已收到语音
+
+ queue speak 中待回答
+
+ think Ghost 推理中
+
+ speak TTS 输出中
+
★ interrupt 急停抢占
+
+
+
+
+
+
+
+ 急刹打断
+ 说「立刻停下」(或点「打断」按钮),整团能量在 <100ms 内急刹冻结、聚焦闪白。
+
+ LISTEN 可与 THINK/SPEAK 同时亮起
+ ASR 慢线 确认急停并停止 TTS
+ 不是互斥五态,是全双工叠层
+
+
+
+
+
IDLE
+
+
+
+
+
+
+
+
主控
+
+
+
+
+
+
+
+ 麦克风未开
+ ASR 连续监听
+
+
+
+
+
+
+
手动状态切换
+
+
+
+
+
+
+
+
+
+ intensity
+
+ 0.00
+
+
+
+
+
+
+
+
+
diff --git a/.moss_ws/apps/aether/core/webroot/web/core.frag b/.moss_ws/apps/aether/core/webroot/web/core.frag
new file mode 100644
index 00000000..0c7a1ed0
--- /dev/null
+++ b/.moss_ws/apps/aether/core/webroot/web/core.frag
@@ -0,0 +1,148 @@
+#version 300 es
+// Aether Core - 能量核心 Fragment Shader
+// raymarched 流体球 + fbm 表面扰动 + 体积辉光 + INTERRUPT 急刹闪白
+// 5 状态参数由 CPU 端插值后传入(idle/listen/think/speak/interrupt)
+precision highp float;
+
+uniform vec2 u_resolution;
+uniform float u_time; // 全局时间(秒)
+uniform float u_intensity; // 0~1,speak 时音量驱动喷涌
+uniform float u_interrupt_burst; // 0~1,急刹瞬间闪白强度(衰减)
+
+// 状态参数(CPU 插值后传入,状态切换在 CPU 端做 ease)
+uniform float u_radius; // 球半径
+uniform float u_amp; // 表面扰动幅度
+uniform float u_freq; // 表面噪声频率
+uniform float u_speed; // 内部时间速度(interrupt 时 ≈ 0 = 冻结)
+uniform vec3 u_color; // 基色
+uniform vec3 u_glow; // 外辉光色
+
+out vec4 fragColor;
+
+// ---------- 3D value noise ----------
+float hash13(vec3 p) {
+ p = fract(p * 0.3183099 + 0.1);
+ p *= 17.0;
+ return fract(p.x * p.y * p.z * (p.x + p.y + p.z));
+}
+
+float noise3(vec3 x) {
+ vec3 i = floor(x);
+ vec3 f = fract(x);
+ f = f * f * (3.0 - 2.0 * f);
+ return mix(
+ mix(mix(hash13(i + vec3(0,0,0)), hash13(i + vec3(1,0,0)), f.x),
+ mix(hash13(i + vec3(0,1,0)), hash13(i + vec3(1,1,0)), f.x), f.y),
+ mix(mix(hash13(i + vec3(0,0,1)), hash13(i + vec3(1,0,1)), f.x),
+ mix(hash13(i + vec3(0,1,1)), hash13(i + vec3(1,1,1)), f.x), f.y),
+ f.z);
+}
+
+// domain-warped fbm:流体感的关键
+float fbm(vec3 p) {
+ float v = 0.0;
+ float a = 0.5;
+ for (int i = 0; i < 5; i++) {
+ v += a * noise3(p);
+ p = p * 2.02 + vec3(1.7, 9.2, 3.3);
+ a *= 0.5;
+ }
+ return v;
+}
+
+// SDF:球 + fbm 扰动
+float map(vec3 p, float t) {
+ vec3 q = p * u_freq + vec3(0.0, t * 0.5, t * 0.3);
+ // domain warp
+ vec3 w = vec3(fbm(q + vec3(0.0, 0.0, 0.0)),
+ fbm(q + vec3(5.2, 1.3, 2.7)),
+ fbm(q + vec3(3.1, 7.4, 1.1)));
+ float n = fbm(q + w * 1.5);
+ return length(p) - (u_radius + (n - 0.5) * u_amp);
+}
+
+// 数值法线(中心差分)
+vec3 calcNormal(vec3 p, float t) {
+ float e = 0.008;
+ vec2 h = vec2(1.0, -1.0) * e;
+ return normalize(
+ h.xyy * map(p + h.xyy, t) +
+ h.yyx * map(p + h.yyx, t) +
+ h.yxy * map(p + h.yxy, t) +
+ h.xxx * map(p + h.xxx, t)
+ );
+}
+
+void main() {
+ vec2 uv = (gl_FragCoord.xy - 0.5 * u_resolution) / min(u_resolution.x, u_resolution.y);
+
+ vec3 ro = vec3(0.0, 0.0, 2.6);
+ vec3 rd = normalize(vec3(uv, -1.6));
+
+ float t = u_time * u_speed;
+
+ // raymarch(带最小距离记录做体积辉光)
+ float tt = 0.05;
+ bool hit = false;
+ float minDist = 1e9;
+ vec3 hitPos = vec3(0.0);
+ for (int i = 0; i < 96; i++) {
+ vec3 p = ro + rd * tt;
+ float d = map(p, t);
+ if (d < minDist) minDist = d;
+ if (d < 0.002) { hit = true; hitPos = p; break; }
+ tt += d * 0.85;
+ if (tt > 5.0) break;
+ }
+
+ // ---------- 体积辉光(基于 raymarch 路径上最近距离) ----------
+ float glow = exp(-minDist * 7.0);
+ float softGlow = exp(-minDist * 2.5) * 0.5;
+
+ vec3 col = vec3(0.0);
+
+ if (hit) {
+ vec3 n = calcNormal(hitPos, t);
+ vec3 L = normalize(vec3(0.5, 0.8, 1.0));
+ float diff = max(dot(n, L), 0.0);
+ float fres = pow(1.0 - max(dot(-rd, n), 0.0), 2.5);
+ // 内核自发光(深处更亮,模拟能量核心)
+ float core = 1.0 - smoothstep(u_radius * 0.3, u_radius * 1.05, length(hitPos));
+ col = u_color * (0.25 + 0.6 * diff + core * 0.8);
+ col += u_color * fres * 1.8;
+ // speak 脉动喷涌:随 intensity 表面亮度脉冲
+ col += u_color * u_intensity * 0.7 * fres;
+ }
+
+ // 外辉光
+ col += u_glow * glow * 1.4;
+ col += u_glow * softGlow * 0.8;
+
+ // speak 外喷辉光
+ col += u_glow * u_intensity * 0.5 * exp(-minDist * 4.0);
+
+ // ---------- INTERRUPT 急刹爆点 ----------
+ // 一帧聚焦闪白 + 向心吸附光线
+ if (u_interrupt_burst > 0.0) {
+ float b = u_interrupt_burst;
+ // 全场闪白(暗角中心更亮,制造聚焦感)
+ float vignette = 1.0 - length(uv) * 0.7;
+ col += vec3(1.0) * b * 2.2 * max(vignette, 0.0);
+ // 向心吸附光线(从中心向外辐射,强度随距离衰减)
+ float r = length(uv);
+ float rays = exp(-r * 1.5) * smoothstep(0.0, 0.15, b);
+ col += vec3(1.0) * rays * 1.8;
+ // 整体提亮(急刹定格的"咔"的一下)
+ col += vec3(0.6, 0.7, 1.0) * b * 0.3;
+ }
+
+ // ---------- 色调映射 + gamma ----------
+ col = col / (1.0 + col);
+ col = pow(col, vec3(1.0 / 2.2));
+
+ // 暗角
+ float vig = 1.0 - length(uv) * 0.35;
+ col *= clamp(vig, 0.4, 1.0);
+
+ fragColor = vec4(col, 1.0);
+}
diff --git a/.moss_ws/apps/aether/core/webroot/web/core.vert b/.moss_ws/apps/aether/core/webroot/web/core.vert
new file mode 100644
index 00000000..aaf8b516
--- /dev/null
+++ b/.moss_ws/apps/aether/core/webroot/web/core.vert
@@ -0,0 +1,6 @@
+#version 300 es
+// Aether Core - 顶点 Shader:全屏三角形
+in vec2 a_pos;
+void main() {
+ gl_Position = vec4(a_pos, 0.0, 1.0);
+}
diff --git a/.moss_ws/apps/aether/core/webroot/web/main.js b/.moss_ws/apps/aether/core/webroot/web/main.js
new file mode 100644
index 00000000..6b87b870
--- /dev/null
+++ b/.moss_ws/apps/aether/core/webroot/web/main.js
@@ -0,0 +1,551 @@
+// main.js — 演示入口
+// 串联 Scene / StateMapper / VAD / StateBridge,绑定 UI
+// 完整对应技术文档:状态字符串驱动 + VAD 快线急刹 + ASR 慢线(演示用模拟)
+
+import { Scene } from './scene.js';
+import { normalizeLayers, StateMapper, STATES, STATE_NAMES } from './state_mapper.js';
+import { VAD } from './vad.js';
+import { StateBridge } from './ws.js';
+
+const $ = (s) => document.querySelector(s);
+const $$ = (s) => document.querySelectorAll(s);
+
+// ---------- 全局实例 ----------
+let scene, mapper, vad, bridge;
+let demoTimer = null;
+let demoSpeakPulseRaf = null;
+let manualIntensity = 0.0;
+let useAutoIntensity = true; // speak 时自动脉动
+let localLayers = normalizeLayers();
+let backendLayers = normalizeLayers();
+let localMicRunning = false;
+let listenClearTimer = 0;
+let thinkPendingTimer = 0;
+let asrMode = 'continuous';
+let asrArmed = true;
+
+// ---------- 初始化 ----------
+async function boot() {
+ scene = new Scene($('#glcanvas'));
+ await scene.init();
+ scene.start();
+
+ mapper = new StateMapper({
+ onState: (state, meta) => onStateChange(state, meta),
+ minHoldMs: 250,
+ interruptHoldMs: 700,
+ });
+
+ bridge = new StateBridge({
+ onState: (stateName) => {
+ const idx = STATE_NAMES.indexOf(stateName);
+ if (idx >= 0) applyState(idx, { event: 'WS_STATE' });
+ },
+ onLayers: (layers, msg) => {
+ applyLayers(layers, { event: 'WS_LAYERS', text: msg.text });
+ updateDiagnostics(msg.diagnostics);
+ if (msg.interrupt_burst) scene.interruptBurst = Math.max(scene.interruptBurst, msg.interrupt_burst);
+ if (msg.text || layers.speak || layers.interrupt) {
+ clearThinkPendingTimer();
+ }
+ },
+ onIntensity: (v) => {
+ manualIntensity = v;
+ useAutoIntensity = false;
+ },
+ onConnect: () => {
+ updateModePill('ws');
+ updateAsrControlUI();
+ },
+ onDisconnect: () => {
+ updateModePill('local');
+ },
+ });
+
+ // 自动连 aether/core 后端(MOSS ghost 推状态)
+ const ok = await bridge.connect();
+ updateModePill(ok ? 'ws' : 'local');
+
+ bindUI();
+ updateAsrControlUI();
+ applyState(STATES.IDLE, { event: 'BOOT' });
+ log(ok ? '已连后端 ws://localhost:8765/ws · MOSS ghost 在线' : '后端未就绪 · 本地演示模式(自动重连中)');
+ log('提示:点「演示模式」看全双工叠层;本地VAD只负责快速视觉反馈,火山ASR收音由右侧按钮控制');
+}
+
+// ---------- 状态切换处理 ----------
+function onStateChange(state, meta) {
+ localLayers = normalizeLayers(meta?.layers || mapper.layers);
+ scene.onStateChange(state, meta);
+ applyStateUI(state, meta);
+ // 同步 VAD 的 speakMode:SPEAK 时提高阈值/duration,避免 AEC 残留回声自打断;
+ // 其他状态恢复原灵敏度(idle→listen 仍需低延迟开口检测)。
+ if (vad) vad.setSpeakMode(state === STATES.SPEAK);
+}
+
+function applyState(state, meta) {
+ // 走 state_mapper(带去抖)
+ mapper.event({ type: 'FORCE_STATE', state, ...(meta || {}) });
+}
+
+function composeVisualLayers(layers = backendLayers) {
+ const base = normalizeLayers(layers);
+ return normalizeLayers({
+ ...base,
+ mic: Boolean(base.mic) || localMicRunning,
+ listen: Boolean(base.listen) || localMicRunning,
+ });
+}
+
+function applyLayers(layers, meta = {}) {
+ backendLayers = normalizeLayers(layers);
+ localLayers = normalizeLayers({
+ ...backendLayers,
+ // Fast visual listen: local browser VAD means the user has started
+ // speaking. Backend ASR still owns SpeechTopic/think; this only keeps
+ // Aether responsive before the cloud ASR returns its first partial.
+ ...composeVisualLayers(backendLayers),
+ });
+ mapper.setLayers(localLayers, meta);
+}
+
+function applyStateUI(state, meta) {
+ const name = STATE_NAMES[state];
+ const layers = normalizeLayers(meta?.layers || mapper.layers);
+ $$('.state-dot').forEach(d => {
+ const s = d.dataset.s;
+ const active = s === 'idle' ? name === 'idle' : Boolean(layers[s]);
+ d.classList.toggle('active', active);
+ d.classList.toggle('primary', s === name);
+ });
+ $('#current-state-name').textContent = name;
+ updateActivityRings(layers, name);
+ // 球体中央大字(所有状态都显示)
+ const labelEl = $('#state-label');
+ if (labelEl) {
+ labelEl.textContent = activeLabel(layers, name);
+ labelEl.setAttribute('data-state', name);
+ }
+ $$('.stategraph .edge').forEach(e => {
+ const to = e.dataset.to;
+ const active = to === 'idle' ? name === 'idle' : Boolean(layers[to]);
+ e.classList.toggle('active', active);
+ });
+
+ if (state === STATES.INTERRUPT) {
+ $('#interrupt-flash').classList.add('on');
+ clearTimeout(window._intrFlashT);
+ window._intrFlashT = setTimeout(() => {
+ $('#interrupt-flash').classList.remove('on');
+ }, 1500);
+ }
+
+ log(`state → ${activeLabel(layers, name)} (${meta?.event || ''}${meta?.latencyMs != null ? ' · vad+' + meta.latencyMs.toFixed(0) + 'ms' : ''})`);
+}
+
+function activeLabel(layers, fallback) {
+ const active = ['listen', 'queue', 'think', 'speak'].filter(k => layers[k]);
+ if (layers.interrupt) return 'INTERRUPT';
+ if (active.length === 0 && layers.mic) return 'MIC';
+ if (active.length === 0) return fallback.toUpperCase();
+ return active.map(s => s.toUpperCase()).join(' + ');
+}
+
+function updateActivityRings(layers, primary) {
+ ['mic', 'listen', 'queue', 'think', 'speak', 'interrupt'].forEach(name => {
+ const el = document.querySelector(`.activity-ring[data-layer="${name}"]`);
+ if (!el) return;
+ el.classList.toggle('active', Boolean(layers[name]));
+ el.classList.toggle('primary', primary === name);
+ });
+ const stack = $('#activity-stack');
+ if (stack) stack.setAttribute('data-primary', primary);
+}
+
+function updateModePill(mode) {
+ const el = $('#mode-pill');
+ el.textContent = mode === 'ws' ? 'WS · 已连后端' : '本地演示模式';
+ el.className = 'mode-pill ' + mode;
+}
+
+function escapeHtml(value) {
+ return String(value ?? '')
+ .replaceAll('&', '&')
+ .replaceAll('<', '<')
+ .replaceAll('>', '>')
+ .replaceAll('"', '"')
+ .replaceAll("'", ''');
+}
+
+function updateDiagnostics(diag) {
+ if (!diag) return;
+ const currentEl = $('#asr-current');
+ if (currentEl) {
+ if (diag.asr_current?.text) {
+ currentEl.innerHTML = `
+
+
partial
+
${escapeHtml(diag.asr_current.text)}
+
+ `;
+ } else {
+ currentEl.innerHTML = '等待 ASR partial
';
+ }
+ }
+
+ const asrEl = $('#asr-diag');
+ if (asrEl && Array.isArray(diag.asr_finals)) {
+ if (diag.asr_finals.length === 0) {
+ asrEl.innerHTML = '等待 final
';
+ } else {
+ asrEl.innerHTML = diag.asr_finals.slice(-3).reverse().map(item => {
+ return `
+
+
final
+
${escapeHtml(item.text)}
+
+ `;
+ }).join('');
+ }
+ }
+
+ const errorEl = $('#asr-error');
+ if (errorEl) {
+ if (diag.asr_error?.error) {
+ const backoff = Number(diag.asr_error.backoff || 0);
+ const code = diag.asr_error.code ? ` · code=${escapeHtml(diag.asr_error.code)}` : '';
+ const message = diag.asr_error.message ? ` · ${escapeHtml(diag.asr_error.message)}` : '';
+ const wait = backoff > 0 ? ` · retry ${backoff.toFixed(0)}s` : '';
+ errorEl.hidden = false;
+ errorEl.textContent = `ASR ${diag.asr_error.error}${code}${message}${wait}`;
+ } else {
+ errorEl.hidden = true;
+ errorEl.textContent = '';
+ }
+ }
+
+ if (diag.asr_control) {
+ asrMode = diag.asr_control.mode || asrMode;
+ asrArmed = asrMode === 'continuous' ? true : Boolean(diag.asr_control.enabled);
+ updateAsrControlUI();
+ }
+
+ const vpioEl = $('#vpio-diag');
+ if (vpioEl && typeof diag.vpio === 'string' && diag.vpio) {
+ vpioEl.textContent = diag.vpio;
+ }
+}
+
+// ---------- 日志 ----------
+const logLines = [];
+const MAX_LOG = 30;
+function log(msg) {
+ const ts = new Date().toLocaleTimeString('zh-CN', { hour12: false });
+ logLines.push(`[${ts}] ${msg}`);
+ if (logLines.length > MAX_LOG) logLines.shift();
+ const el = $('#log');
+ el.innerHTML = logLines.map((l, i) => {
+ const age = logLines.length - 1 - i;
+ const op = Math.max(0.25, 1 - age * 0.05);
+ return `${l}
`;
+ }).join('');
+ el.scrollTop = el.scrollHeight;
+}
+
+// ---------- 手动连接 Python 后端 ----------
+async function connectBackend() {
+ const btn = $('#btn-ws');
+ btn.textContent = '连接中...';
+ btn.disabled = true;
+ const ok = await bridge.connect();
+ btn.disabled = false;
+ btn.textContent = ok ? '已连后端' : '连接失败';
+ updateModePill(ok ? 'ws' : 'local');
+ updateAsrControlUI();
+ log('ws connect: ' + (ok ? 'success (ws://localhost:8765)' : 'failed, stay local'));
+}
+
+// ---------- 演示模式:自动循环 idle→listen→think→speak→idle ----------
+function startDemo() {
+ stopDemo();
+ $('#btn-demo').classList.add('active');
+ $('#btn-demo').textContent = '停止演示';
+ log('演示模式启动');
+ runDemoStep();
+}
+
+function stopDemo() {
+ if (demoTimer) { clearTimeout(demoTimer); demoTimer = null; }
+ if (demoSpeakPulseRaf) { cancelAnimationFrame(demoSpeakPulseRaf); demoSpeakPulseRaf = null; }
+ $('#btn-demo').classList.remove('active');
+ $('#btn-demo').textContent = '演示模式';
+}
+
+function runDemoStep() {
+ if (!$('#btn-demo').classList.contains('active')) return;
+ // 序列:idle → listen → think → think+speak → listen+think+speak → idle
+ applyLayers({}, { event: 'DEMO' });
+ demoTimer = setTimeout(() => {
+ applyLayers({ listen: true }, { event: 'DEMO' });
+ demoTimer = setTimeout(() => {
+ applyLayers({ think: true }, { event: 'DEMO' });
+ demoTimer = setTimeout(() => {
+ applyLayers({ think: true, speak: true }, { event: 'DEMO' });
+ startSpeakPulse();
+ demoTimer = setTimeout(() => {
+ applyLayers({ listen: true, think: true, speak: true }, { event: 'DEMO_DUPLEX' });
+ demoTimer = setTimeout(() => {
+ stopSpeakPulse();
+ runDemoStep();
+ }, 1600);
+ }, 2400);
+ }, 1400);
+ }, 1400);
+ }, 2000);
+}
+
+function requestListenLayer(running, meta = {}) {
+ if (listenClearTimer) {
+ clearTimeout(listenClearTimer);
+ listenClearTimer = 0;
+ }
+ const next = normalizeLayers({ ...localLayers, listen: running });
+ applyLayers(next, meta);
+ bridge.sendListen?.(running, meta.backend || {});
+}
+
+function requestMicLayer(running, meta = {}) {
+ if (listenClearTimer) {
+ clearTimeout(listenClearTimer);
+ listenClearTimer = 0;
+ }
+ localMicRunning = running;
+ localLayers = composeVisualLayers(backendLayers);
+ mapper.setLayers(localLayers, meta);
+}
+
+function clearMicLayer(meta = {}) {
+ requestMicLayer(false, meta);
+}
+
+function clearMicLayerSoon(meta = {}, delayMs = 1800) {
+ if (listenClearTimer) clearTimeout(listenClearTimer);
+ listenClearTimer = setTimeout(() => {
+ listenClearTimer = 0;
+ if (localMicRunning && !backendLayers.listen && !backendLayers.queue && !backendLayers.think && !backendLayers.speak && !backendLayers.interrupt) {
+ clearMicLayer(meta);
+ }
+ }, delayMs);
+}
+
+function clearThinkPendingTimer() {
+ if (thinkPendingTimer) {
+ clearTimeout(thinkPendingTimer);
+ thinkPendingTimer = 0;
+ }
+}
+
+function updateAsrControlUI() {
+ const modeBtn = $('#btn-asr-mode');
+ const armBtn = $('#btn-asr-arm');
+ const status = $('#asr-control-status');
+ if (!modeBtn || !armBtn || !status) return;
+ const manual = asrMode === 'manual';
+ modeBtn.textContent = manual ? '火山ASR 手动' : '火山ASR 连续';
+ modeBtn.classList.toggle('active', manual);
+ armBtn.disabled = !manual;
+ armBtn.textContent = asrArmed ? '停止火山收音' : '开始火山收音';
+ armBtn.classList.toggle('active', manual && asrArmed);
+ status.textContent = manual
+ ? (asrArmed ? 'ASR 手动收音中' : 'ASR 手动待机')
+ : 'ASR 连续监听';
+ status.className = 'mic-status ' + (manual && !asrArmed ? '' : 'on');
+}
+
+function sendAsrControl() {
+ if (asrMode === 'continuous') asrArmed = true;
+ bridge.sendAsrControl?.(asrMode, asrArmed);
+ updateAsrControlUI();
+}
+
+function enterThinkPending(meta = {}) {
+ if (listenClearTimer) {
+ clearTimeout(listenClearTimer);
+ listenClearTimer = 0;
+ }
+ clearThinkPendingTimer();
+ applyLayers({ ...localLayers, listen: false, think: true }, meta);
+ bridge.sendListen?.(false, { pending_think: true });
+ thinkPendingTimer = setTimeout(() => {
+ thinkPendingTimer = 0;
+ if (localLayers.think && !localLayers.queue && !localLayers.speak && !localLayers.interrupt) {
+ applyLayers({ think: false }, { event: 'THINK_PENDING_TIMEOUT' });
+ log('ASR/LLM pending timeout · back to idle');
+ }
+ }, 10000);
+}
+
+// speak 期间自动脉动 intensity(模拟 TTS 音量)
+function startSpeakPulse() {
+ useAutoIntensity = true;
+ const start = performance.now();
+ const tick = () => {
+ if (!demoSpeakPulseRaf) return;
+ demoSpeakPulseRaf = requestAnimationFrame(tick);
+ const t = (performance.now() - start) / 1000;
+ // 像说话的节奏:多个 sin 叠加 + 一点随机
+ const base = 0.55 + 0.35 * Math.sin(t * 6.0) + 0.10 * Math.sin(t * 17.0 + 1.3);
+ const v = Math.max(0, Math.min(1, base));
+ if (useAutoIntensity) {
+ scene.setIntensity(v);
+ $('#intensity-slider').value = v.toFixed(3);
+ $('#intensity-value').textContent = v.toFixed(2);
+ }
+ };
+ demoSpeakPulseRaf = requestAnimationFrame(tick);
+}
+
+function stopSpeakPulse() {
+ if (demoSpeakPulseRaf) { cancelAnimationFrame(demoSpeakPulseRaf); demoSpeakPulseRaf = null; }
+ scene.setIntensity(0);
+ $('#intensity-slider').value = 0;
+ $('#intensity-value').textContent = '0.00';
+}
+
+// ---------- VAD 快线 ----------
+async function toggleMic() {
+ if (vad) {
+ vad.stop();
+ vad = null;
+ $('#btn-mic').classList.remove('active');
+ $('#btn-mic').textContent = '开启本地VAD';
+ $('#mic-status').textContent = '麦克风未开';
+ $('#mic-status').className = 'mic-status';
+ $('#level-bar > div').style.width = '0%';
+ clearMicLayer({ event: 'VAD_OFF' });
+ return;
+ }
+ try {
+ vad = new VAD({
+ threshold: parseFloat($('#vad-threshold').value),
+ minDurMs: 35,
+ endSilenceMs: 520,
+ // VPIO 已启用系统级 AEC(input+output VPIO=True),后端回声已消除。
+ // 前端 VAD 仍用浏览器麦克风(WebRTC AEC),保留小量安全边际即可。
+ speakThresholdScale: 1.2,
+ speakMinDurScale: 1.1,
+ onEchoCancelReport: (settings) => {
+ // 上报浏览器实际生效的 AEC 设置(可能被降级,需可见)
+ const aec = settings.echoCancellation;
+ const aecType = settings.echoCancellationType || 'n/a';
+ const ns = settings.noiseSuppression;
+ const agc = settings.autoGainControl;
+ log(`AEC 报告 · echoCancellation=${aec} (${aecType}) · NS=${ns} · AGC=${agc}`);
+ },
+ onLevel: (rms) => {
+ const pct = Math.min(100, rms * 600);
+ $('#level-bar > div').style.width = pct.toFixed(1) + '%';
+ },
+ onSpeechStarted: ({ vadLatencyMs, speakMode }) => {
+ requestMicLayer(true, { event: 'VAD_START', vadLatencyMs });
+ log(`VAD · mic activity${speakMode ? ' during speak' : ''} · vad+${vadLatencyMs.toFixed(0)}ms`);
+ },
+ onSpeechEnded: () => {
+ log('VAD SPEECH_ENDED · waiting ASR final');
+ clearMicLayerSoon({ event: 'VAD_END_WAIT_ASR' }, 3500);
+ },
+ });
+ await vad.start();
+ $('#btn-mic').classList.add('active');
+ $('#btn-mic').textContent = '关闭本地VAD';
+ $('#mic-status').textContent = '本地VAD监听中';
+ $('#mic-status').className = 'mic-status on';
+ log(`VAD 启动 · 阈值=${vad.threshold}`);
+ stopSpeakPulse();
+ } catch (e) {
+ log('麦克风启动失败: ' + e.message);
+ $('#mic-status').textContent = '麦克风权限被拒';
+ $('#mic-status').className = 'mic-status warn';
+ }
+}
+
+// ---------- UI 绑定 ----------
+function bindUI() {
+ $('#btn-ws').onclick = connectBackend;
+ $('#btn-demo').onclick = () => {
+ if (demoTimer) stopDemo();
+ else startDemo();
+ };
+ $('#btn-mic').onclick = toggleMic;
+ $('#btn-asr-mode').onclick = () => {
+ if (asrMode === 'continuous') {
+ asrMode = 'manual';
+ asrArmed = false;
+ log('ASR 控制 · 手动待机(火山 ASR 不监听)');
+ } else {
+ asrMode = 'continuous';
+ asrArmed = true;
+ log('ASR 控制 · 连续监听');
+ }
+ sendAsrControl();
+ };
+ $('#btn-asr-arm').onclick = () => {
+ if (asrMode !== 'manual') return;
+ asrArmed = !asrArmed;
+ log(asrArmed ? 'ASR 控制 · 开始收音' : 'ASR 控制 · 停止收音');
+ sendAsrControl();
+ };
+ $('#btn-reset').onclick = () => {
+ logLines.length = 0;
+ $('#log').innerHTML = '';
+ bridge.sendReset?.();
+ log('上下文已重置(清空对话历史)');
+ };
+ $('#btn-interrupt').onclick = () => {
+ const t0 = performance.now();
+ applyLayers({ interrupt: true }, { event: 'MANUAL_INTERRUPT' });
+ bridge.sendInterrupt?.();
+ log(`手动打断 · state→interrupt ${(performance.now() - t0).toFixed(1)}ms`);
+ scene.setIntensity(0);
+ stopSpeakPulse();
+ };
+
+ // 5 个手动状态按钮
+ $$('.btn-state').forEach(btn => {
+ btn.onclick = () => {
+ const name = btn.dataset.s;
+ const idx = STATE_NAMES.indexOf(name);
+ if (idx >= 0) {
+ applyState(idx);
+ if (name === 'speak') startSpeakPulse();
+ else if (name !== 'speak') stopSpeakPulse();
+ }
+ };
+ });
+
+ // intensity 滑块
+ $('#intensity-slider').oninput = (e) => {
+ const v = parseFloat(e.target.value);
+ useAutoIntensity = false;
+ manualIntensity = v;
+ scene.setIntensity(v);
+ $('#intensity-value').textContent = v.toFixed(2);
+ };
+
+ // VAD 阈值
+ $('#vad-threshold').oninput = (e) => {
+ const v = parseFloat(e.target.value);
+ $('#vad-threshold-value').textContent = v.toFixed(3);
+ if (vad) vad.threshold = v;
+ const mark = (Math.min(1, v * 8) * 100).toFixed(1);
+ $('#threshold-mark').style.left = mark + '%';
+ };
+ // 初始化阈值标记
+ $('#vad-threshold').dispatchEvent(new Event('input'));
+}
+
+// ---------- 启动 ----------
+boot().catch(e => {
+ console.error(e);
+ log('启动失败: ' + e.message);
+});
diff --git a/.moss_ws/apps/aether/core/webroot/web/scene.js b/.moss_ws/apps/aether/core/webroot/web/scene.js
new file mode 100644
index 00000000..9a1035c2
--- /dev/null
+++ b/.moss_ws/apps/aether/core/webroot/web/scene.js
@@ -0,0 +1,188 @@
+// scene.js — WebGL2 渲染循环 + 状态参数插值 + INTERRUPT 急刹爆点
+// 职责:
+// 1. 编译 shader、建 VAO、设置 uniform
+// 2. 维护"当前视觉参数"(cur)向"目标参数"(target = STATE_TARGETS[state])做 ease lerp
+// —— INTERRUPT 用大速率(急刹),其他用小速率(柔和切换)
+// 3. interrupt_burst:进入 INTERRUPT 时设为 1.0,每帧指数衰减(~150ms 大半)
+// 4. speak intensity 驱动 amp/radius 微脉动
+
+import { blendTargetsForLayers, normalizeLayers, STATE_TARGETS, STATE_SWITCH_RATE, STATES } from './state_mapper.js';
+
+export class Scene {
+ constructor(canvas) {
+ this.canvas = canvas;
+ const gl = canvas.getContext('webgl2', { antialias: false, alpha: false, powerPreference: 'high-performance', preserveDrawingBuffer: true });
+ if (!gl) throw new Error('WebGL2 not supported');
+ this.gl = gl;
+
+ // 当前视觉参数(向 target 插值)
+ this.cur = { ...structuredClone(STATE_TARGETS[STATES.IDLE]) };
+ this.cur.color = [...this.cur.color];
+ this.cur.glow = [...this.cur.glow];
+
+ this.target = STATE_TARGETS[STATES.IDLE];
+ this.switchRate = STATE_SWITCH_RATE[STATES.IDLE];
+ this.layers = normalizeLayers();
+
+ this.intensity = 0.0; // speak 音量
+ this.interruptBurst = 0.0; // 急刹闪白强度 0~1
+ this.lastInterruptTs = 0;
+ this.time = 0;
+ this.startTime = performance.now() / 1000;
+
+ this.uniforms = {};
+ this.ready = false;
+ }
+
+ async init() {
+ const gl = this.gl;
+ const [vsrc, fsrc] = await Promise.all([
+ fetch('./web/core.vert').then(r => r.text()),
+ fetch('./web/core.frag').then(r => r.text()),
+ ]);
+ const vs = this._compile(gl.VERTEX_SHADER, vsrc);
+ const fs = this._compile(gl.FRAGMENT_SHADER, fsrc);
+ const prog = gl.createProgram();
+ gl.attachShader(prog, vs);
+ gl.attachShader(prog, fs);
+ gl.linkProgram(prog);
+ if (!gl.getProgramParameter(prog, gl.LINK_STATUS)) {
+ throw new Error('Link failed: ' + gl.getProgramInfoLog(prog));
+ }
+ this.program = prog;
+ gl.useProgram(prog);
+
+ // 全屏三角形 VAO
+ const vao = gl.createVertexArray();
+ gl.bindVertexArray(vao);
+ const buf = gl.createBuffer();
+ gl.bindBuffer(gl.ARRAY_BUFFER, buf);
+ gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
+ -1, -1, 3, -1, -1, 3,
+ ]), gl.STATIC_DRAW);
+ const loc = gl.getAttribLocation(prog, 'a_pos');
+ gl.enableVertexAttribArray(loc);
+ gl.vertexAttribPointer(loc, 2, gl.FLOAT, false, 0, 0);
+
+ // uniform 位置
+ const U = [
+ 'u_resolution', 'u_time', 'u_intensity', 'u_interrupt_burst',
+ 'u_radius', 'u_amp', 'u_freq', 'u_speed', 'u_color', 'u_glow',
+ ];
+ for (const u of U) this.uniforms[u] = gl.getUniformLocation(prog, u);
+
+ this.ready = true;
+ this._resize();
+ window.addEventListener('resize', () => this._resize());
+ }
+
+ _compile(type, src) {
+ const gl = this.gl;
+ const sh = gl.createShader(type);
+ gl.shaderSource(sh, src);
+ gl.compileShader(sh);
+ if (!gl.getShaderParameter(sh, gl.COMPILE_STATUS)) {
+ const log = gl.getShaderInfoLog(sh);
+ console.error('Shader compile error:', log, '\n--- source ---\n', src);
+ throw new Error('Shader compile failed: ' + log);
+ }
+ return sh;
+ }
+
+ _resize() {
+ const dpr = Math.min(window.devicePixelRatio || 1, 2);
+ const w = Math.floor(window.innerWidth * dpr);
+ const h = Math.floor(window.innerHeight * dpr);
+ if (this.canvas.width !== w || this.canvas.height !== h) {
+ this.canvas.width = w;
+ this.canvas.height = h;
+ }
+ }
+
+ /** 状态切换钩子(由 state_mapper 触发) */
+ onStateChange(state, meta = {}) {
+ this.layers = normalizeLayers(meta.layers);
+ this.target = meta.layers ? blendTargetsForLayers(this.layers) : STATE_TARGETS[state];
+ this.switchRate = STATE_SWITCH_RATE[state];
+ if (state === STATES.INTERRUPT) {
+ this.interruptBurst = 1.0;
+ this.lastInterruptTs = performance.now();
+ }
+ }
+
+ setLayers(layers) {
+ this.layers = normalizeLayers(layers);
+ this.target = blendTargetsForLayers(this.layers);
+ }
+
+ /** 设置 speak intensity(0~1) */
+ setIntensity(v) {
+ this.intensity = v;
+ }
+
+ _lerp(a, b, t) { return a + (b - a) * t; }
+ _lerpArr(a, b, t) { return a.map((v, i) => v + (b[i] - v) * t); }
+
+ render = () => {
+ const gl = this.gl;
+ this._raf = requestAnimationFrame(this.render);
+ if (!this.ready) return;
+
+ const now = performance.now() / 1000;
+ const dt = Math.min(0.05, now - (this._lastTime || now));
+ this._lastTime = now;
+ this.time = now - this.startTime;
+
+ // 参数插值:cur → target,按 switchRate 做指数 ease
+ // cur = cur + (target - cur) * (1 - exp(-rate * dt))
+ const a = 1.0 - Math.exp(-this.switchRate * dt);
+ const tgt = this.target;
+ const c = this.cur;
+ c.radius = this._lerp(c.radius, tgt.radius, a);
+ c.amp = this._lerp(c.amp, tgt.amp, a);
+ c.freq = this._lerp(c.freq, tgt.freq, a);
+ c.speed = this._lerp(c.speed, tgt.speed, a);
+ c.color = this._lerpArr(c.color, tgt.color, a);
+ c.glow = this._lerpArr(c.glow, tgt.glow, a);
+
+ // interrupt_burst 衰减:~150ms 大半衰减
+ this.interruptBurst *= Math.exp(-dt * 8.0);
+ if (this.interruptBurst < 0.001) this.interruptBurst = 0;
+
+ // speak 脉动:让半径/幅度随 intensity 微跳
+ const pulse = 0.0;
+ const intensity = this.intensity;
+
+ gl.viewport(0, 0, this.canvas.width, this.canvas.height);
+ gl.clearColor(0.015, 0.022, 0.045, 1.0);
+ gl.clear(gl.COLOR_BUFFER_BIT);
+
+ gl.useProgram(this.program);
+ const U = this.uniforms;
+ gl.uniform2f(U.u_resolution, this.canvas.width, this.canvas.height);
+ gl.uniform1f(U.u_time, this.time);
+ gl.uniform1f(U.u_intensity, intensity);
+ gl.uniform1f(U.u_interrupt_burst, this.interruptBurst);
+ // speak 时半径 + 微脉动
+ const speakActive = this.layers.speak || this.target === STATE_TARGETS[STATES.SPEAK];
+ gl.uniform1f(U.u_radius, c.radius + (intensity * 0.04) * (speakActive ? 1 : 0));
+ gl.uniform1f(U.u_amp, c.amp + intensity * 0.03 * (speakActive ? 1 : 0));
+ gl.uniform1f(U.u_freq, c.freq);
+ gl.uniform1f(U.u_speed, c.speed);
+ gl.uniform3fv(U.u_color, c.color);
+ gl.uniform3fv(U.u_glow, c.glow);
+
+ gl.drawArrays(gl.TRIANGLES, 0, 3);
+ }
+
+ start() {
+ if (!this.ready) throw new Error('call init() first');
+ cancelAnimationFrame(this._raf);
+ this._raf = requestAnimationFrame(this.render);
+ }
+}
+
+// 避免 structuredClone 兼容性问题,自己深拷
+function structuredClone(obj) {
+ return JSON.parse(JSON.stringify(obj));
+}
diff --git a/.moss_ws/apps/aether/core/webroot/web/state_mapper.js b/.moss_ws/apps/aether/core/webroot/web/state_mapper.js
new file mode 100644
index 00000000..88d5020b
--- /dev/null
+++ b/.moss_ws/apps/aether/core/webroot/web/state_mapper.js
@@ -0,0 +1,187 @@
+// state_mapper.js — 兼容主状态 + 全双工 activity layers
+
+export const STATES = {
+ IDLE: 0,
+ LISTEN: 1,
+ THINK: 2,
+ SPEAK: 3,
+ INTERRUPT: 4,
+};
+
+export const STATE_NAMES = ['idle', 'listen', 'think', 'speak', 'interrupt'];
+
+// 各状态目标视觉参数(CPU 端插值目标)
+export const STATE_TARGETS = {
+ [STATES.IDLE]: { radius: 0.50, amp: 0.05, freq: 1.20, speed: 0.45, color: [0.30, 0.55, 1.10], glow: [0.20, 0.45, 1.00] },
+ [STATES.LISTEN]: { radius: 0.42, amp: 0.03, freq: 1.50, speed: 0.85, color: [0.20, 0.80, 1.20], glow: [0.10, 0.65, 1.10] },
+ [STATES.THINK]: { radius: 0.55, amp: 0.13, freq: 2.60, speed: 1.80, color: [0.65, 0.30, 1.10], glow: [0.50, 0.20, 1.00] },
+ [STATES.SPEAK]: { radius: 0.58, amp: 0.10, freq: 2.20, speed: 2.00, color: [1.10, 0.65, 0.25], glow: [1.00, 0.45, 0.10] },
+ [STATES.INTERRUPT]: { radius: 0.40, amp: 0.02, freq: 0.50, speed: 0.00, color: [0.85, 0.95, 1.20], glow: [0.70, 0.85, 1.20] },
+};
+
+export const LAYER_NAMES = ['mic', 'listen', 'queue', 'think', 'speak', 'interrupt'];
+
+export function normalizeLayers(layers = {}) {
+ return {
+ mic: Boolean(layers.mic),
+ listen: Boolean(layers.listen),
+ queue: Boolean(layers.queue),
+ think: Boolean(layers.think),
+ speak: Boolean(layers.speak),
+ interrupt: Boolean(layers.interrupt),
+ };
+}
+
+export function primaryStateFromLayers(layers = {}) {
+ const l = normalizeLayers(layers);
+ if (l.interrupt) return STATES.INTERRUPT;
+ if (l.speak) return STATES.SPEAK;
+ if (l.think) return STATES.THINK;
+ if (l.listen) return STATES.LISTEN;
+ return STATES.IDLE;
+}
+
+export function blendTargetsForLayers(layers = {}) {
+ const l = normalizeLayers(layers);
+ if (l.interrupt) return { ...cloneTarget(STATE_TARGETS[STATES.INTERRUPT]), activeCount: 1 };
+
+ const weighted = [
+ [STATES.IDLE, (!l.listen && !l.think && !l.speak) ? 1.0 : 0.28],
+ [STATES.LISTEN, l.listen ? 0.90 : (l.mic ? 0.30 : 0)],
+ [STATES.THINK, l.think ? 1.00 : (l.queue ? 0.55 : 0)],
+ [STATES.SPEAK, l.speak ? 1.10 : 0],
+ ];
+ let total = 0;
+ const out = { radius: 0, amp: 0, freq: 0, speed: 0, color: [0, 0, 0], glow: [0, 0, 0] };
+ let activeCount = 0;
+ for (const [state, weight] of weighted) {
+ if (weight <= 0) continue;
+ if (state !== STATES.IDLE) activeCount += 1;
+ total += weight;
+ const t = STATE_TARGETS[state];
+ out.radius += t.radius * weight;
+ out.amp += t.amp * weight;
+ out.freq += t.freq * weight;
+ out.speed += t.speed * weight;
+ for (let i = 0; i < 3; i++) {
+ out.color[i] += t.color[i] * weight;
+ out.glow[i] += t.glow[i] * weight;
+ }
+ }
+ if (total <= 0) return { ...cloneTarget(STATE_TARGETS[STATES.IDLE]), activeCount: 0 };
+ out.radius /= total;
+ out.amp /= total;
+ out.freq /= total;
+ out.speed /= total;
+ for (let i = 0; i < 3; i++) {
+ out.color[i] /= total;
+ out.glow[i] /= total;
+ }
+ out.activeCount = activeCount;
+ return out;
+}
+
+function cloneTarget(target) {
+ return JSON.parse(JSON.stringify(target));
+}
+
+// 状态切换的"急刹度":插值速率,越大切换越快。INTERRUPT 最大(急刹),其他柔和
+export const STATE_SWITCH_RATE = {
+ [STATES.IDLE]: 4.0,
+ [STATES.LISTEN]: 5.0,
+ [STATES.THINK]: 4.5,
+ [STATES.SPEAK]: 4.0,
+ [STATES.INTERRUPT]: 6.0, // 急刹再放慢:让扩张→收缩的节奏清晰可见
+};
+
+export class StateMapper {
+ /**
+ * @param {object} opts
+ * @param {(state:number, event:object)=>void} opts.onState 状态切换回调
+ * @param {number} opts.minHoldMs 最小保持时间(去抖),默认 250ms
+ * @param {number} opts.interruptHoldMs INTERRUPT 最小保持时间(让爆点视觉充分展现)
+ */
+ constructor({ onState, minHoldMs = 250, interruptHoldMs = 1800 } = {}) {
+ this.state = STATES.IDLE;
+ this.layers = normalizeLayers();
+ this._layerKey = JSON.stringify(this.layers);
+ this.onState = onState;
+ this.minHoldMs = minHoldMs;
+ this.interruptHoldMs = interruptHoldMs;
+ this.stateEnterTs = performance.now();
+ this.log = [];
+ }
+
+ now() { return performance.now(); }
+ heldMs() { return this.now() - this.stateEnterTs; }
+
+ /**
+ * 喂事件。事件类型:
+ * - SPEECH_STARTED 旧契约:检测到开口
+ * - SPEECH_FINAL ASR 出完整句(慢线)
+ * - TTS_START TTS 开始播放
+ * - TTS_END TTS 播放结束
+ * - FORCE_STATE 演示用:强制切状态
+ */
+ event(e) {
+ const now = this.now();
+ const held = now - this.stateEnterTs;
+ let next = this.state;
+
+ switch (e.type) {
+ case 'SPEECH_STARTED':
+ // 旧契约兼容:新路径使用 setLayers(),普通 VAD 不再伪装成 interrupt。
+ if (this.state === STATES.SPEAK) next = STATES.INTERRUPT;
+ else if (this.state === STATES.IDLE) next = STATES.LISTEN;
+ else if (this.state === STATES.THINK) next = STATES.INTERRUPT; // 思考中被打断也急刹
+ break;
+ case 'SPEECH_FINAL':
+ if (this.state === STATES.LISTEN) next = STATES.THINK;
+ break;
+ case 'TTS_START':
+ if (this.state === STATES.THINK || this.state === STATES.INTERRUPT) next = STATES.SPEAK;
+ break;
+ case 'TTS_END':
+ if (this.state === STATES.SPEAK) next = STATES.IDLE;
+ break;
+ case 'FORCE_STATE':
+ next = e.state;
+ break;
+ }
+
+ if (next === this.state) return;
+
+ // 去抖:INTERRUPT 是爆点,永远抢跑(绕过去抖);
+ // FORCE_STATE 是手动演示,也绕过;
+ // 其他状态切换在最小保持时间内忽略。
+ const isUrgent = next === STATES.INTERRUPT || e.type === 'FORCE_STATE';
+ if (!isUrgent && held < this.minHoldMs) return;
+
+ // INTERRUPT 自身要保持足够久,避免视觉还没展开就退出
+ if (this.state === STATES.INTERRUPT && held < this.interruptHoldMs && e.type !== 'SPEECH_STARTED' && e.type !== 'FORCE_STATE') {
+ return;
+ }
+
+ const prev = this.state;
+ this.state = next;
+ this.layers = next === STATES.IDLE ? normalizeLayers() : normalizeLayers({ [STATE_NAMES[next]]: true });
+ this._layerKey = JSON.stringify(this.layers);
+ this.stateEnterTs = now;
+ this.log.push({ ts: now, from: prev, to: next, event: e.type });
+ if (this.log.length > 200) this.log.shift();
+ this.onState?.(next, { from: prev, event: e.type, latencyMs: e.vadLatencyMs, layers: this.layers });
+ }
+
+ setLayers(layers, meta = {}) {
+ const normalized = normalizeLayers(layers);
+ const key = JSON.stringify(normalized);
+ const next = primaryStateFromLayers(normalized);
+ const prev = this.state;
+ if (next === this.state && key === this._layerKey) return;
+ this.layers = normalized;
+ this._layerKey = key;
+ this.state = next;
+ this.stateEnterTs = this.now();
+ this.onState?.(next, { from: prev, event: meta.event || 'LAYERS', layers: normalized, text: meta.text });
+ }
+}
diff --git a/.moss_ws/apps/aether/core/webroot/web/style.css b/.moss_ws/apps/aether/core/webroot/web/style.css
new file mode 100644
index 00000000..529e510c
--- /dev/null
+++ b/.moss_ws/apps/aether/core/webroot/web/style.css
@@ -0,0 +1,553 @@
+:root {
+ --bg: #04060c;
+ --panel: rgba(12, 18, 32, 0.72);
+ --panel-border: rgba(120, 180, 255, 0.18);
+ --text: #cfe3ff;
+ --text-dim: #6f86a8;
+ --accent: #4cc8ff;
+ --warn: #ff5a6a;
+ --ok: #4dffb0;
+}
+
+@property --spin {
+ syntax: "";
+ inherits: false;
+ initial-value: 0deg;
+}
+
+* { box-sizing: border-box; margin: 0; padding: 0; }
+
+html, body {
+ width: 100%;
+ height: 100%;
+ background: var(--bg);
+ color: var(--text);
+ font-family: -apple-system, BlinkMacSystemFont, "SF Mono", "JetBrains Mono", Menlo, Consolas, monospace;
+ overflow: hidden;
+}
+
+#app {
+ position: fixed;
+ inset: 0;
+ display: flex;
+ flex-direction: column;
+}
+
+#glcanvas {
+ position: fixed;
+ inset: 0;
+ width: 100%;
+ height: 100%;
+ display: block;
+ z-index: 0;
+}
+
+/* ---------- 顶部状态条 ---------- */
+.topbar {
+ position: relative;
+ z-index: 2;
+ display: flex;
+ align-items: center;
+ gap: 18px;
+ padding: 14px 22px;
+ background: linear-gradient(180deg, rgba(4,6,12,0.85), rgba(4,6,12,0));
+ pointer-events: none;
+}
+
+.brand {
+ font-size: 14px;
+ letter-spacing: 0.18em;
+ color: var(--accent);
+ text-transform: uppercase;
+}
+
+.brand .sub {
+ color: var(--text-dim);
+ margin-left: 10px;
+ font-size: 11px;
+ letter-spacing: 0.1em;
+}
+
+.mode-pill {
+ pointer-events: auto;
+ font-size: 11px;
+ padding: 4px 10px;
+ border-radius: 999px;
+ border: 1px solid var(--panel-border);
+ background: rgba(0,0,0,0.4);
+ color: var(--text-dim);
+}
+.mode-pill.ws { color: var(--ok); border-color: rgba(77,255,176,0.4); }
+.mode-pill.local { color: var(--accent); border-color: rgba(76,200,255,0.4); }
+
+.state-dots {
+ margin-left: auto;
+ display: flex;
+ gap: 8px;
+ pointer-events: auto;
+}
+.state-dot {
+ width: 12px; height: 12px;
+ border-radius: 50%;
+ background: rgba(255,255,255,0.08);
+ border: 1px solid rgba(255,255,255,0.12);
+ transition: all 0.18s ease;
+ position: relative;
+}
+.state-dot.active {
+ transform: scale(1.4);
+ box-shadow: 0 0 14px currentColor;
+}
+.state-dot.primary { outline: 1px solid rgba(255,255,255,0.55); outline-offset: 3px; }
+.state-dot[data-s="idle"].active { background: #4cc8ff; color: #4cc8ff; }
+.state-dot[data-s="listen"].active { background: #2bd0ff; color: #2bd0ff; }
+.state-dot[data-s="think"].active { background: #a060ff; color: #a060ff; }
+.state-dot[data-s="speak"].active { background: #ffb14c; color: #ffb14c; }
+.state-dot[data-s="interrupt"].active { background: #ffffff; color: #ffffff; }
+
+.current-state {
+ font-size: 13px;
+ min-width: 110px;
+ color: var(--text);
+}
+.current-state .name {
+ font-size: 18px;
+ font-weight: 600;
+ letter-spacing: 0.08em;
+ text-transform: uppercase;
+}
+
+/* ---------- 底部控制面板 ---------- */
+.bottom {
+ position: fixed;
+ left: 0; right: 0; bottom: 0;
+ z-index: 2;
+ padding: 18px 22px 22px;
+ background: linear-gradient(0deg, rgba(4,6,12,0.92), rgba(4,6,12,0));
+ display: flex;
+ flex-direction: column;
+ gap: 14px;
+}
+
+.panel {
+ background: var(--panel);
+ border: 1px solid var(--panel-border);
+ border-radius: 12px;
+ padding: 14px 16px;
+ backdrop-filter: blur(12px);
+ -webkit-backdrop-filter: blur(12px);
+}
+
+.panel-title {
+ font-size: 10px;
+ letter-spacing: 0.22em;
+ color: var(--text-dim);
+ text-transform: uppercase;
+ margin-bottom: 10px;
+}
+
+.row { display: flex; flex-wrap: wrap; align-items: center; gap: 10px; }
+
+button {
+ font-family: inherit;
+ font-size: 12px;
+ letter-spacing: 0.08em;
+ color: var(--text);
+ background: rgba(255,255,255,0.04);
+ border: 1px solid rgba(255,255,255,0.12);
+ padding: 8px 14px;
+ border-radius: 8px;
+ cursor: pointer;
+ transition: all 0.15s ease;
+}
+button:hover { background: rgba(76,200,255,0.12); border-color: rgba(76,200,255,0.4); }
+button:active { transform: scale(0.97); }
+button:disabled { opacity: 0.42; cursor: not-allowed; transform: none; }
+button:disabled:hover { background: rgba(255,255,255,0.04); border-color: rgba(255,255,255,0.12); }
+button.active { background: rgba(76,200,255,0.2); border-color: var(--accent); color: var(--accent); }
+button.danger { border-color: rgba(255,90,106,0.4); color: #ff8a96; }
+button.danger:hover { background: rgba(255,90,106,0.15); }
+button.warn { border-color: rgba(255,200,80,0.4); color: #ffc850; }
+button.warn:hover { background: rgba(255,200,80,0.15); }
+button.primary {
+ background: linear-gradient(135deg, rgba(76,200,255,0.25), rgba(160,96,255,0.25));
+ border-color: rgba(160,96,255,0.5);
+ color: #fff;
+}
+
+.mic-status {
+ font-size: 11px;
+ color: var(--text-dim);
+ padding: 4px 10px;
+ border-radius: 999px;
+ border: 1px solid var(--panel-border);
+}
+.mic-status.on { color: var(--ok); border-color: rgba(77,255,176,0.4); }
+.mic-status.warn { color: var(--warn); border-color: rgba(255,90,106,0.4); }
+
+.level-bar {
+ flex: 1;
+ min-width: 120px;
+ height: 6px;
+ background: rgba(255,255,255,0.06);
+ border-radius: 3px;
+ overflow: hidden;
+ position: relative;
+}
+.level-bar > div {
+ height: 100%;
+ background: linear-gradient(90deg, #4cc8ff, #ffb14c, #ff5a6a);
+ width: 0%;
+ transition: width 0.05s linear;
+}
+.threshold-mark {
+ position: absolute;
+ top: -2px; bottom: -2px;
+ width: 1px;
+ background: rgba(255,255,255,0.5);
+}
+
+.intensity-slider {
+ flex: 1;
+ min-width: 120px;
+ display: flex; align-items: center; gap: 10px;
+}
+.intensity-slider input { flex: 1; }
+
+/* ---------- 状态机小图 ---------- */
+.stategraph {
+ position: fixed;
+ top: 70px;
+ right: 22px;
+ z-index: 2;
+ width: 220px;
+ font-size: 11px;
+ color: var(--text-dim);
+}
+.stategraph .edge {
+ display: flex;
+ align-items: center;
+ gap: 6px;
+ padding: 3px 0;
+}
+.stategraph .edge span:first-child { color: var(--text-dim); min-width: 76px; }
+.stategraph .edge.active span:first-child { color: var(--accent); }
+
+/* ---------- 语音链路诊断 ---------- */
+.voice-diag {
+ position: fixed;
+ top: 250px;
+ right: 22px;
+ z-index: 2;
+ width: 320px;
+ padding: 12px 14px;
+ background: rgba(5, 9, 18, 0.68);
+ border: 1px solid rgba(120, 180, 255, 0.16);
+ border-radius: 8px;
+ backdrop-filter: blur(12px);
+ -webkit-backdrop-filter: blur(12px);
+ font-size: 11px;
+ color: var(--text-dim);
+ pointer-events: none;
+}
+.voice-diag .panel-title { margin-bottom: 8px; }
+.diag-section + .diag-section { margin-top: 10px; }
+.diag-label {
+ color: rgba(207, 227, 255, 0.56);
+ margin-bottom: 5px;
+ letter-spacing: 0.08em;
+}
+.asr-diag,
+.asr-current {
+ display: flex;
+ flex-direction: column;
+ gap: 5px;
+}
+.asr-item {
+ display: grid;
+ grid-template-columns: 42px minmax(0, 1fr);
+ gap: 7px;
+ align-items: start;
+ padding: 5px 7px;
+ border-radius: 6px;
+ background: rgba(255, 255, 255, 0.035);
+ border: 1px solid rgba(255, 255, 255, 0.065);
+}
+.asr-item.final {
+ border-color: rgba(77, 255, 176, 0.22);
+ background: rgba(77, 255, 176, 0.055);
+}
+.asr-item.partial {
+ border-color: rgba(76, 200, 255, 0.18);
+ background: rgba(76, 200, 255, 0.045);
+}
+.asr-tag {
+ color: var(--accent);
+ font-size: 10px;
+ text-transform: uppercase;
+}
+.asr-item.final .asr-tag { color: var(--ok); }
+.asr-text {
+ color: var(--text);
+ overflow-wrap: anywhere;
+ line-height: 1.35;
+}
+.diag-empty,
+.vpio-diag {
+ line-height: 1.35;
+ overflow-wrap: anywhere;
+}
+.asr-error {
+ margin-top: 10px;
+ padding: 7px 8px;
+ border-radius: 6px;
+ background: rgba(255, 90, 106, 0.10);
+ border: 1px solid rgba(255, 90, 106, 0.28);
+ color: #ff9aa4;
+ line-height: 1.35;
+ overflow-wrap: anywhere;
+}
+.vpio-diag {
+ color: rgba(207, 227, 255, 0.76);
+ padding: 6px 7px;
+ border-radius: 6px;
+ background: rgba(76, 200, 255, 0.055);
+ border: 1px solid rgba(76, 200, 255, 0.12);
+}
+
+/* ---------- 并发活动轨道 ---------- */
+.activity-stack {
+ position: fixed;
+ inset: 0;
+ z-index: 1;
+ pointer-events: none;
+ display: grid;
+ place-items: center;
+}
+.activity-ring {
+ position: absolute;
+ width: min(64vw, 620px);
+ aspect-ratio: 1;
+ border-radius: 50%;
+ opacity: 0;
+ transform: scale(0.86) rotate(var(--tilt, 0deg));
+ transition: opacity 0.22s ease, transform 0.35s cubic-bezier(0.16,1,0.3,1), filter 0.22s ease;
+ filter: blur(0.2px);
+}
+.activity-ring::before {
+ content: "";
+ position: absolute;
+ inset: 0;
+ border-radius: inherit;
+ border: 2px solid transparent;
+ background:
+ conic-gradient(from var(--spin, 0deg),
+ transparent 0deg,
+ var(--ring-color) 36deg,
+ transparent 78deg,
+ transparent 178deg,
+ var(--ring-color) 226deg,
+ transparent 286deg,
+ transparent 360deg) border-box;
+ mask: linear-gradient(#000 0 0) padding-box, linear-gradient(#000 0 0);
+ -webkit-mask: linear-gradient(#000 0 0) padding-box, linear-gradient(#000 0 0);
+ mask-composite: exclude;
+ -webkit-mask-composite: xor;
+ padding: 2px;
+ animation: ring-spin var(--ring-speed, 8s) linear infinite;
+}
+.activity-ring::after {
+ content: "";
+ position: absolute;
+ inset: 8%;
+ border-radius: inherit;
+ border: 1px solid color-mix(in srgb, var(--ring-color), transparent 62%);
+ box-shadow: 0 0 24px color-mix(in srgb, var(--ring-color), transparent 48%);
+}
+.activity-ring span {
+ position: absolute;
+ top: 8%;
+ left: 50%;
+ transform: translateX(-50%);
+ font-size: 11px;
+ letter-spacing: 0.22em;
+ color: var(--ring-color);
+ text-shadow: 0 0 18px var(--ring-color);
+}
+.activity-ring.active {
+ opacity: 0.82;
+ transform: scale(var(--ring-scale, 1)) rotate(var(--tilt, 0deg));
+}
+.activity-ring.primary {
+ opacity: 1;
+ filter: drop-shadow(0 0 18px var(--ring-color));
+}
+.activity-ring.listen {
+ --ring-color: #49e8ff;
+ --ring-scale: 0.82;
+ --ring-speed: 5.5s;
+ --tilt: -10deg;
+}
+.activity-ring.mic {
+ --ring-color: #4dffb0;
+ --ring-scale: 0.72;
+ --ring-speed: 4.2s;
+ --tilt: 12deg;
+ opacity: 0;
+}
+.activity-ring.mic.active {
+ opacity: 0.42;
+}
+.activity-ring.mic span {
+ top: 14%;
+}
+.activity-ring.think {
+ --ring-color: #bb76ff;
+ --ring-scale: 0.94;
+ --ring-speed: 8.5s;
+ --tilt: 18deg;
+}
+.activity-ring.queue {
+ --ring-color: #ff6f91;
+ --ring-scale: 0.88;
+ --ring-speed: 6.2s;
+ --tilt: -24deg;
+}
+.activity-ring.speak {
+ --ring-color: #ffbd4a;
+ --ring-scale: 1.07;
+ --ring-speed: 3.4s;
+ --tilt: 6deg;
+}
+.activity-ring.interrupt {
+ --ring-color: #ffffff;
+ --ring-scale: 0.76;
+ --ring-speed: 1.2s;
+ --tilt: 0deg;
+}
+.activity-ring.speak.active {
+ animation: speak-ring-pulse 0.55s ease-in-out infinite alternate;
+}
+.activity-ring.think.active::after {
+ animation: think-ring-warp 1.4s ease-in-out infinite alternate;
+}
+.activity-ring.queue.active::before {
+ animation-duration: 2.6s;
+}
+@keyframes ring-spin {
+ to { --spin: 360deg; }
+}
+@keyframes speak-ring-pulse {
+ from { transform: scale(calc(var(--ring-scale, 1) - 0.015)) rotate(var(--tilt, 0deg)); }
+ to { transform: scale(calc(var(--ring-scale, 1) + 0.025)) rotate(var(--tilt, 0deg)); }
+}
+@keyframes think-ring-warp {
+ from { inset: 8%; opacity: 0.45; }
+ to { inset: 5%; opacity: 0.85; }
+}
+
+/* ---------- 中部打断提示 ---------- */
+.interrupt-flash {
+ position: fixed;
+ inset: 0;
+ z-index: 5;
+ pointer-events: none;
+ background: radial-gradient(circle, rgba(255,255,255,0.0), rgba(255,255,255,0.0));
+ transition: background 0.05s linear;
+}
+.interrupt-flash.on {
+ background: radial-gradient(circle, rgba(255,255,255,0.5), rgba(255,255,255,0) 60%);
+}
+
+.interrupt-label {
+ position: fixed;
+ top: 50%;
+ left: 50%;
+ transform: translate(-50%, -50%) scale(0.8);
+ z-index: 6;
+ pointer-events: none;
+ font-size: 48px;
+ font-weight: 800;
+ letter-spacing: 0.2em;
+ color: #fff;
+ text-shadow: 0 0 30px rgba(255,255,255,0.8);
+ opacity: 0;
+ transition: opacity 0.08s ease;
+}
+.interrupt-label.on {
+ opacity: 1;
+ transform: translate(-50%, -50%) scale(1);
+ transition: opacity 0.08s ease, transform 0.18s cubic-bezier(0.16,1,0.3,1);
+}
+
+/* 通用状态大字(球体中央,持续显示) */
+.state-label {
+ position: fixed;
+ top: 50%;
+ left: 50%;
+ transform: translate(-50%, -50%);
+ z-index: 6;
+ pointer-events: none;
+ font-size: 42px;
+ font-weight: 800;
+ letter-spacing: 0.22em;
+ text-align: center;
+ white-space: normal;
+ max-width: min(86vw, 760px);
+ line-height: 1.08;
+ opacity: 0.85;
+ transition: opacity 0.25s ease, color 0.3s ease, text-shadow 0.3s ease;
+}
+.state-label[data-state="idle"] { color: #6fa8ff; text-shadow: 0 0 22px rgba(80,140,255,0.6); }
+.state-label[data-state="listen"] { color: #5cf0ff; text-shadow: 0 0 22px rgba(40,200,255,0.7); }
+.state-label[data-state="think"] { color: #c08aff; text-shadow: 0 0 22px rgba(160,80,255,0.7); }
+.state-label[data-state="speak"] { color: #ffc850; text-shadow: 0 0 24px rgba(255,170,40,0.8); }
+.state-label[data-state="interrupt"] {
+ color: #fff;
+ text-shadow: 0 0 34px rgba(255,255,255,0.9);
+ font-size: 52px;
+ opacity: 1;
+ animation: intr-pulse 0.5s cubic-bezier(0.16,1,0.3,1);
+}
+@keyframes intr-pulse {
+ 0% { transform: translate(-50%, -50%) scale(0.7); opacity: 0; }
+ 50% { transform: translate(-50%, -50%) scale(1.12); opacity: 1; }
+ 100% { transform: translate(-50%, -50%) scale(1); opacity: 1; }
+}
+
+/* ---------- 日志 ---------- */
+.log {
+ position: fixed;
+ left: 22px;
+ top: 80px;
+ z-index: 2;
+ width: 340px;
+ max-height: calc(100vh - 480px);
+ overflow-y: auto;
+ overflow-x: hidden;
+ font-size: 10px;
+ color: var(--text-dim);
+ line-height: 1.6;
+ scrollbar-width: thin;
+ scrollbar-color: rgba(100,180,255,0.3) transparent;
+ mask-image: linear-gradient(180deg, transparent 0, #000 18px, #000 calc(100% - 18px), transparent 100%);
+ -webkit-mask-image: linear-gradient(180deg, transparent 0, #000 18px, #000 calc(100% - 18px), transparent 100%);
+}
+.log::-webkit-scrollbar { width: 4px; }
+.log::-webkit-scrollbar-thumb { background: rgba(100,180,255,0.3); border-radius: 2px; }
+.log::-webkit-scrollbar-track { background: transparent; }
+.log .line { opacity: 0.9; }
+.log .line:last-child { color: var(--text); }
+
+.hint {
+ position: fixed;
+ top: 40%;
+ left: 22px;
+ transform: translateY(-50%);
+ z-index: 2;
+ font-size: 11px;
+ color: var(--text-dim);
+ max-width: 280px;
+ line-height: 1.7;
+}
+.hint b { color: var(--text); }
diff --git a/.moss_ws/apps/aether/core/webroot/web/vad.js b/.moss_ws/apps/aether/core/webroot/web/vad.js
new file mode 100644
index 00000000..b577ce70
--- /dev/null
+++ b/.moss_ws/apps/aether/core/webroot/web/vad.js
@@ -0,0 +1,162 @@
+// vad.js — 浏览器麦克风 VAD 快线(对应技术文档里的 vad.py)
+// 用 RMS 能量阈值 + 最小持续时间门限,检测到开口第一帧就触发 SPEECH_STARTED
+// 目标延迟 < 50ms(实际取决于 AudioContext 块大小,这里 fftSize=512 @16kHz ≈ 32ms)
+//
+// 回声处理:依赖 WebRTC echoCancellation 作为前置过滤,让 VAD"听不到"TTS 外放回声。
+// 即便如此 AEC 效果不稳定,因此在 SPEAK 状态下启用 speakMode:
+// - 阈值放大 speakThresholdScale 倍(默认 2.0)
+// - 最小持续时间放大 speakMinDurScale 倍(默认 1.5)
+// 作为安全边际,避免残留回声触发自打断。
+
+export class VAD {
+ /**
+ * @param {object} opts
+ * @param {()=>void} opts.onSpeechStarted 开口瞬间回调(爆点触发源)
+ * @param {()=>void} [opts.onSpeechEnded] 静默后回调
+ * @param {number} [opts.threshold] RMS 阈值 0~1
+ * @param {number} [opts.minDurMs] 持续多长时间才算"开口"(防噪声毛刺)
+ * @param {number} [opts.endSilenceMs] 多少静默算"说完了"
+ * @param {(level:number)=>void} [opts.onLevel] 实时音量回调(用于 speak 脉动近似)
+ * @param {number} [opts.speakThresholdScale] SPEAK 状态下阈值放大倍数(默认 2.0)
+ * @param {number} [opts.speakMinDurScale] SPEAK 状态下 minDurMs 放大倍数(默认 1.5)
+ * @param {(settings:object)=>void} [opts.onEchoCancelReport] 上报 getUserMedia 实际生效的 AEC 设置
+ */
+ constructor({
+ onSpeechStarted,
+ onSpeechEnded,
+ threshold = 0.012,
+ minDurMs = 40,
+ endSilenceMs = 520,
+ onLevel,
+ speakThresholdScale = 2.0,
+ speakMinDurScale = 1.5,
+ onEchoCancelReport,
+ } = {}) {
+ this.onSpeechStarted = onSpeechStarted;
+ this.onSpeechEnded = onSpeechEnded;
+ this.onLevel = onLevel;
+ this.threshold = threshold;
+ this.minDurMs = minDurMs;
+ this.endSilenceMs = endSilenceMs;
+ this.speakThresholdScale = speakThresholdScale;
+ this.speakMinDurScale = speakMinDurScale;
+ this.onEchoCancelReport = onEchoCancelReport;
+
+ this.running = false;
+ this.ctx = null;
+ this.stream = null;
+ this.analyser = null;
+ this.buf = null;
+ this.raf = 0;
+
+ this.speaking = false;
+ this.activeSince = 0; // 当前活跃起始时间
+ this.silentSince = 0; // 静默起始时间
+ this.lastRms = 0;
+
+ // SPEAK 状态开关:开启后提高阈值与最小持续时间,作为 AEC 残留回声的安全边际
+ this.speakMode = false;
+ }
+
+ /** SPEAK 状态下开启,提高阈值;离开 SPEAK 时关闭,恢复灵敏度 */
+ setSpeakMode(on) {
+ this.speakMode = !!on;
+ }
+
+ async start() {
+ if (this.running) return;
+ // 强化 echoCancellation:用 ideal 表达强烈偏好,echoCancellationType:'system'
+ // 优先使用系统级 AEC(通常比浏览器内置 AEC 效果更好)。
+ const stream = await navigator.mediaDevices.getUserMedia({
+ audio: {
+ echoCancellation: { ideal: true },
+ echoCancellationType: 'system',
+ noiseSuppression: { ideal: true },
+ autoGainControl: { ideal: true },
+ channelCount: { ideal: 1 },
+ },
+ });
+ this.stream = stream;
+ // 上报实际生效的 AEC 设置(浏览器可能降级,需可见)
+ try {
+ const track = stream.getAudioTracks()[0];
+ const settings = track.getSettings();
+ this.onEchoCancelReport?.(settings);
+ } catch (e) {}
+ // 16kHz 采样率贴近 Silero VAD 场景
+ const Ctx = window.AudioContext || window.webkitAudioContext;
+ this.ctx = new Ctx({ sampleRate: 16000 });
+ const src = this.ctx.createMediaStreamSource(stream);
+ this.analyser = this.ctx.createAnalyser();
+ this.analyser.fftSize = 512; // 32ms @16kHz
+ this.analyser.smoothingTimeConstant = 0.3;
+ src.connect(this.analyser);
+ this.buf = new Uint8Array(this.analyser.fftSize);
+
+ this.running = true;
+ this.loop();
+ }
+
+ stop() {
+ this.running = false;
+ if (this.raf) cancelAnimationFrame(this.raf);
+ this.raf = 0;
+ if (this.stream) this.stream.getTracks().forEach(t => t.stop());
+ if (this.ctx) this.ctx.close();
+ this.stream = null;
+ this.ctx = null;
+ this.analyser = null;
+ this.speaking = false;
+ }
+
+ loop = () => {
+ if (!this.running) return;
+ this.raf = requestAnimationFrame(this.loop);
+ if (!this.analyser) return;
+
+ this.analyser.getByteTimeDomainData(this.buf);
+ let sum = 0;
+ for (let i = 0; i < this.buf.length; i++) {
+ const v = (this.buf[i] - 128) / 128;
+ sum += v * v;
+ }
+ const rms = Math.sqrt(sum / this.buf.length);
+ // 平滑一下,避免抖动
+ this.lastRms = this.lastRms * 0.6 + rms * 0.4;
+ this.onLevel?.(this.lastRms);
+
+ const now = performance.now();
+ // SPEAK 状态下使用放大阈值与持续时间,作为 AEC 残留回声的安全边际
+ const effectiveThreshold = this.speakMode
+ ? this.threshold * this.speakThresholdScale
+ : this.threshold;
+ const effectiveMinDurMs = this.speakMode
+ ? this.minDurMs * this.speakMinDurScale
+ : this.minDurMs;
+ const above = this.lastRms > effectiveThreshold;
+
+ if (above) {
+ if (this.activeSince === 0) this.activeSince = now;
+ // 持续 effectiveMinDurMs 才算"开口" → 触发 SPEECH_STARTED
+ if (!this.speaking && (now - this.activeSince) >= effectiveMinDurMs) {
+ this.speaking = true;
+ const vadLatencyMs = now - this.activeSince;
+ this.onSpeechStarted?.({ vadLatencyMs, speakMode: this.speakMode });
+ this.silentSince = 0;
+ }
+ this.silentSince = 0;
+ } else {
+ if (this.speaking) {
+ if (this.silentSince === 0) this.silentSince = now;
+ if ((now - this.silentSince) >= this.endSilenceMs) {
+ this.speaking = false;
+ this.activeSince = 0;
+ this.silentSince = 0;
+ this.onSpeechEnded?.();
+ }
+ } else {
+ this.activeSince = 0;
+ }
+ }
+ }
+}
diff --git a/.moss_ws/apps/aether/core/webroot/web/ws.js b/.moss_ws/apps/aether/core/webroot/web/ws.js
new file mode 100644
index 00000000..3b72667c
--- /dev/null
+++ b/.moss_ws/apps/aether/core/webroot/web/ws.js
@@ -0,0 +1,119 @@
+// ws.js — 状态接收层
+// 连 aether/core 后端 ws://localhost:8765/ws(MOSS ghost 推状态 JSON)
+// 后端是 aether/core app:订阅 SpeechTopic/AudioRuntimeTopic,推 {state, layers, intensity, ts}
+//
+// 状态契约(前后端唯一接口):
+// { "state": "speak", "layers": {"listen": true, "think": false, "speak": true, "interrupt": false}, "intensity": 0.7, "ts": 1782402722.10 }
+// 前端→后端:
+// { "type": "listen", "running": true } // legacy only; main listen is backend ASR-driven
+// { "type": "interrupt" } // 明确急停请求
+// { "type": "asr_control", "mode": "manual", "enabled": true }
+
+export class StateBridge {
+ /**
+ * @param {object} opts
+ * @param {(state:string)=>void} opts.onState
+ * @param {(intensity:number)=>void} opts.onIntensity
+ * @param {(layers:object, msg:object)=>void} opts.onLayers
+ * @param {()=>void} opts.onConnect
+ * @param {()=>void} opts.onDisconnect
+ */
+ constructor({ onState, onIntensity, onLayers, onConnect, onDisconnect } = {}) {
+ this.onState = onState;
+ this.onIntensity = onIntensity;
+ this.onLayers = onLayers;
+ this.onConnect = onConnect;
+ this.onDisconnect = onDisconnect;
+ this.connected = false;
+ this.mode = 'local'; // 'ws' | 'local'
+ this.ws = null;
+ this._stopped = false;
+ this._url = 'ws://localhost:8765/ws';
+ this._lastAsrControl = { mode: 'continuous', enabled: true };
+ this._hasAsrControl = false;
+ }
+
+ /** 尝试连真后端;失败返回 false。带自动重连。 */
+ async connect(url = 'ws://localhost:8765/ws') {
+ this._stopped = false;
+ this._url = url;
+ try {
+ const ws = new WebSocket(url);
+ this.ws = ws;
+ await new Promise((res, rej) => {
+ ws.onopen = res;
+ ws.onerror = rej;
+ setTimeout(() => rej(new Error('timeout')), 2000);
+ });
+ ws.onmessage = (ev) => {
+ try { this._handle(JSON.parse(ev.data)); } catch (e) {}
+ };
+ ws.onclose = () => {
+ this.connected = false;
+ this.mode = 'local';
+ this.onDisconnect?.();
+ if (!this._stopped) {
+ // 3s 后自动重连
+ setTimeout(() => { if (!this._stopped) this.connect(url); }, 3000);
+ }
+ };
+ this.connected = true;
+ this.mode = 'ws';
+ this.onConnect?.();
+ if (this._hasAsrControl) this._sendAsrControlNow();
+ return true;
+ } catch (e) {
+ this.connected = false;
+ this.mode = 'local';
+ if (!this._stopped) {
+ setTimeout(() => { if (!this._stopped) this.connect(url); }, 3000);
+ }
+ return false;
+ }
+ }
+
+ /** 明确急停:按钮或已确认的后端 barge-in 路径。普通 VAD 不调用它。 */
+ sendInterrupt() {
+ if (this.ws && this.connected) {
+ try { this.ws.send(JSON.stringify({ type: 'interrupt' })); } catch (e) {}
+ }
+ }
+
+ sendListen(running, extra = {}) {
+ if (this.ws && this.connected) {
+ try { this.ws.send(JSON.stringify({ type: 'listen', running: Boolean(running), ...extra })); } catch (e) {}
+ }
+ }
+
+ sendAsrControl(mode = 'continuous', enabled = true) {
+ this._lastAsrControl = { mode, enabled: Boolean(enabled) };
+ this._hasAsrControl = true;
+ this._sendAsrControlNow();
+ }
+
+ _sendAsrControlNow() {
+ if (this.ws && this.connected) {
+ try { this.ws.send(JSON.stringify({ type: 'asr_control', ...this._lastAsrControl })); } catch (e) {}
+ }
+ }
+
+ /** 前端请求重置上下文 → 后端清空 speech_win + ghost 历史 */
+ sendReset() {
+ if (this.ws && this.connected) {
+ try { this.ws.send(JSON.stringify({ type: 'reset' })); } catch (e) {}
+ }
+ }
+
+ _handle(msg) {
+ if (!msg) return;
+ if (msg.layers && typeof msg.layers === 'object') this.onLayers?.(msg.layers, msg);
+ else if (typeof msg.state === 'string') this.onState?.(msg.state);
+ if (typeof msg.intensity === 'number') this.onIntensity?.(msg.intensity);
+ }
+
+ /** 本地模式:直接由前端喂状态消息(演示用) */
+ feedLocal(msg) {
+ if (this.mode === 'ws') return; // 已连真后端,忽略本地
+ this._handle(msg);
+ }
+}
diff --git a/.moss_ws/apps/sensors/listener/.env.example b/.moss_ws/apps/aether/listener/.env.example
similarity index 100%
rename from .moss_ws/apps/sensors/listener/.env.example
rename to .moss_ws/apps/aether/listener/.env.example
diff --git a/.moss_ws/apps/sensors/listener/.gitignore b/.moss_ws/apps/aether/listener/.gitignore
similarity index 100%
rename from .moss_ws/apps/sensors/listener/.gitignore
rename to .moss_ws/apps/aether/listener/.gitignore
diff --git a/.moss_ws/apps/aether/listener/APP.md b/.moss_ws/apps/aether/listener/APP.md
new file mode 100644
index 00000000..e03ce1ad
--- /dev/null
+++ b/.moss_ws/apps/aether/listener/APP.md
@@ -0,0 +1,15 @@
+---
+arguments: ''
+description: 'Aether listener — audio → Volcengine ASR → SpeechTopic + AudioSignal'
+executable: uv
+respawn: false
+script: main.py
+workers: 1
+---
+
+Aether listener — audio → Volcengine ASR → SpeechTopic + AudioSignal.
+Canonical app address: `aether/listener`.
+
+The Aether baseline uses the Volcengine streaming ASR path only. Frontend ASR
+controls can switch between continuous listening and manual capture, but both
+modes use the same backend recognizer.
diff --git a/.moss_ws/apps/aether/listener/main.py b/.moss_ws/apps/aether/listener/main.py
new file mode 100644
index 00000000..78ee1024
--- /dev/null
+++ b/.moss_ws/apps/aether/listener/main.py
@@ -0,0 +1,614 @@
+"""Listener App — ASR consumer.
+
+Consumes PCM stream from audio_capture, feeds to Volcengine ASR,
+publishes SpeechTopic on final recognition, emits AudioSignal to mindflow.
+
+Usage:
+ moss apps test aether/listener
+ moss apps start aether/listener
+"""
+import asyncio
+import json
+import logging
+import math
+import os
+import re
+import time
+from collections.abc import AsyncIterable
+from dataclasses import dataclass
+from pathlib import Path
+
+import dotenv
+import numpy as np
+
+dotenv.load_dotenv(Path(__file__).resolve().parent / ".env")
+from scipy import signal
+
+from ghoshell_moss.contracts.asr import ASRResult
+from ghoshell_moss.contracts.audio import (
+ AudioCaptureConfig,
+ AudioChunk,
+)
+from ghoshell_moss.core.mindflow.audio_signal import AudioAction, AudioSignal
+from ghoshell_moss.host.speech.capture.audio_transport import AudioTransport
+from ghoshell_moss.topics.audio import AudioRuntimeTopic, SpeechTopic
+from ghoshell_moss.core.blueprint.matrix import Matrix
+from ghoshell_moss.host.speech.capture.matrix_audio_transport import MatrixAudioTransport
+from ghoshell_moss.host.speech.capture.miniaudio_capture import MiniAudioCaptureSource
+from ghoshell_moss.host.speech.volcengine_asr import VolcengineASR, VolcengineASRConfig
+from ghoshell_moss.message import Message
+from ghoshell_moss.core.blueprint.mindflow import Signal, Priority, unique_id
+
+# ASR 期望的采样率 (16kHz 是语音识别的行业标准)
+_ASR_SAMPLE_RATE = 16000
+_VOLCENGINE_ASR_ERROR_PREFIX = "__VOLCENGINE_ASR_ERROR__:"
+_WAKE_WORDS = (
+ "立刻停下",
+ "立即停下",
+ "马上停下",
+ "快停下",
+ "停下",
+ "别说了",
+ "不要说了",
+ "先停",
+ "闭嘴",
+)
+
+
+def _env_float(name: str, default: float) -> float:
+ raw = os.environ.get(name)
+ if not raw:
+ return default
+ try:
+ return float(raw)
+ except ValueError:
+ return default
+
+
+def _env_int(name: str, default: int) -> int:
+ raw = os.environ.get(name)
+ if not raw:
+ return default
+ try:
+ return int(raw)
+ except ValueError:
+ return default
+
+
+def _wake_word_hit(text: str) -> bool:
+ normalized = re.sub(r"[\s,,。.!!??、;;::「」『』“”\"'`~~-]", "", text)
+ return any(word in normalized for word in _WAKE_WORDS)
+
+
+def _normalized_text_len(text: str) -> int:
+ return len(re.sub(r"[\s,,。.!!??、;;::「」『』“”\"'`~~-]", "", text))
+
+
+def _asr_diag_payload(source: str = "volcengine_asr", **kwargs) -> str:
+ return json.dumps(
+ {
+ "source": source,
+ **kwargs,
+ },
+ ensure_ascii=False,
+ separators=(",", ":"),
+ )
+
+
+@dataclass
+class _ASRControlState:
+ mode: str
+ enabled: bool
+ last_started_at: float = 0.0
+ last_heartbeat: float = 0.0
+
+
+def _initial_asr_control() -> _ASRControlState:
+ mode = os.environ.get("LISTENER_ASR_MODE", "continuous").strip().lower()
+ if mode not in {"continuous", "manual"}:
+ mode = "continuous"
+ return _ASRControlState(mode=mode, enabled=(mode == "continuous"))
+
+
+def _refresh_asr_control(runtime_window, state: _ASRControlState) -> _ASRControlState:
+ """Refresh frontend ASR control mode and keep it sticky.
+
+ ``continuous`` keeps the current behavior: listener opens ASR sessions
+ continuously. ``manual`` only opens ASR while enabled=True.
+
+ The control topic is a command, not a transient runtime diagnostic. VPIO and
+ ASR diagnostics can evict it from a small topic window within seconds, so
+ absence of a recent control topic must not reset the listener to defaults.
+ """
+ for topic in reversed(list(runtime_window.values())):
+ if getattr(topic, "device_name", "") != "asr_control":
+ continue
+ started_at = float(getattr(topic, "started_at", 0.0) or 0.0)
+ heartbeat = float(getattr(topic, "last_heartbeat", 0.0) or 0.0)
+ if (started_at, heartbeat) <= (state.last_started_at, state.last_heartbeat):
+ break
+ try:
+ payload = json.loads(getattr(topic, "device_explain", "") or "{}")
+ except Exception:
+ break
+ next_mode = str(payload.get("mode", state.mode)).strip().lower()
+ if next_mode in {"continuous", "manual"}:
+ state.mode = next_mode
+ state.enabled = bool(payload.get("enabled", state.mode == "continuous"))
+ state.last_started_at = started_at
+ state.last_heartbeat = heartbeat
+ break
+ if state.mode == "continuous":
+ state.enabled = True
+ return state
+
+
+def _resample_audio(samples: np.ndarray, orig_sr: int, target_sr: int) -> np.ndarray:
+ """重采样音频到目标采样率。使用 scipy.signal.resample_poly 保证质量。"""
+ if orig_sr == target_sr:
+ return samples
+ # 44100 -> 16000: up=160, down=441
+ g = math.gcd(orig_sr, target_sr)
+ up = target_sr // g
+ down = orig_sr // g
+ return signal.resample_poly(samples.astype(np.float32), up, down).astype(np.int16)
+
+
+async def _audio_generator(
+ consumer,
+ orig_sr: int,
+ target_sr: int,
+ runtime_window,
+ control_state: _ASRControlState,
+ abort_event: asyncio.Event,
+ logger: logging.Logger,
+ initial_chunk: AudioChunk | None = None,
+ frame_timeout: float = 2.0,
+) -> AsyncIterable[np.ndarray]:
+ """Yield resampled np.ndarray samples from AudioSequentialConsumer.
+
+ Uses an internal asyncio.Queue buffer so that ``aclose()`` (called when
+ ``asr.recognize()`` finishes) does NOT reach ``consumer.__anext__()``
+ and silently drop a chunk.
+
+ NOTE: TTS playback no longer aborts the generator. Instead, ASR continues
+ recognizing during TTS. In Aether mode VPIO provides system AEC, so
+ non-wake-word user speech during TTS must still become a normal turn.
+ """
+ # Unbounded queue: pump must never block on put(), otherwise cancellation
+ # can land inside put() and the None sentinel never reaches the reader.
+ buffer: asyncio.Queue[AudioChunk | None] = asyncio.Queue()
+
+ async def _pump() -> None:
+ """Read from consumer into buffer. Stops on cancellation only."""
+ try:
+ if initial_chunk is not None:
+ buffer.put_nowait(initial_chunk)
+ async for chunk in consumer:
+ buffer.put_nowait(chunk)
+ except asyncio.CancelledError:
+ pass
+ finally:
+ # Sentinel so the generator side exits cleanly.
+ # put_nowait is used so cancellation cannot intercept us.
+ buffer.put_nowait(None)
+
+ pump_task = asyncio.create_task(_pump())
+ try:
+ while True:
+ control = _refresh_asr_control(runtime_window, control_state)
+ if control.mode == "manual" and not control.enabled:
+ logger.info("ASR manual gate closed; ending current audio stream")
+ abort_event.set()
+ break
+ try:
+ chunk = await asyncio.wait_for(buffer.get(), timeout=frame_timeout)
+ except asyncio.TimeoutError:
+ logger.warning(
+ "ASR audio input stalled for %.1fs; ending current audio stream",
+ frame_timeout,
+ )
+ abort_event.set()
+ break
+ if chunk is None:
+ break
+ yield _resample_audio(chunk.samples, orig_sr, target_sr)
+ finally:
+ pump_task.cancel()
+ try:
+ await pump_task
+ except asyncio.CancelledError:
+ pass
+
+
+async def _iter_with_silence_timeout(
+ agen,
+ logger: logging.Logger,
+ patience: float = 5.0,
+ min_timeout_final_chars: int = 2,
+ first_result_timeout: float = 90.0,
+) -> AsyncIterable:
+ """Wrap an async generator with a silence timeout.
+
+ After the first non-empty result, if no subsequent non-empty result
+ arrives within *patience* seconds, the iteration stops. Empty-text
+ results (server keep-alive / VAD status) do NOT reset the timer.
+
+ If the server never sends ``is_final=True`` before the timeout fires,
+ this wrapper synthesizes a final result from the last partial text.
+ Without this, the utterance is silently lost — no SpeechTopic published,
+ no SPEECH_FINAL emitted — and the next recognition loop starts fresh.
+ """
+ timeout = first_result_timeout
+ last_result: ASRResult | None = None
+ try:
+ while True:
+ try:
+ result = await asyncio.wait_for(agen.__anext__(), timeout=timeout)
+ if result.text:
+ last_result = result
+ timeout = patience
+ yield result
+ except asyncio.TimeoutError:
+ if last_result is None:
+ logger.warning(
+ "ASR first-result timeout after %.1fs, restarting recognition",
+ first_result_timeout,
+ )
+ elif not last_result.is_final:
+ logger.info("ASR silence timeout after %.1fs, finalizing", patience)
+ if _normalized_text_len(last_result.text) >= min_timeout_final_chars:
+ logger.info(
+ "Server never sent is_final=True — synthesizing from last partial: %s",
+ last_result.text,
+ )
+ yield ASRResult(text=last_result.text, is_final=True)
+ else:
+ logger.info(
+ "ASR timeout partial too short, dropping fragment: %s",
+ last_result.text,
+ )
+ break
+ except StopAsyncIteration:
+ break
+ finally:
+ await agen.aclose()
+
+
+async def _drain_consumer(consumer, timeout: float = 0.1, max_chunks: int = 5) -> int:
+ """Discard queued audio chunks to clear TTS residue.
+
+ Limits both timeout-per-read and total chunks to avoid draining user speech.
+ Returns the number of chunks drained.
+ """
+ drained = 0
+ while drained < max_chunks:
+ try:
+ await asyncio.wait_for(consumer.__anext__(), timeout=timeout)
+ drained += 1
+ except asyncio.TimeoutError:
+ break
+ except StopAsyncIteration:
+ break
+ return drained
+
+
+def _is_tts_playing(runtime_window, logger: logging.Logger | None = None) -> bool:
+ """检查 TTS 扬声器是否正在播放中。
+
+ AudioRuntimeTopic 是状态快照。从最新往最旧查,找到 speaker
+ 的最新状态即可;旧的状态可能已被 running=False 覆盖。
+
+ 环境变量 ``LISTENER_DISABLE_TTS_GATE=1`` 可关闭此门控。
+ Aether 的 VPIO AEC 场景默认允许 TTS 播放时继续接收用户语音;如果
+ 需要旧的保守回声过滤,可设置 ``LISTENER_GATE_DURING_TTS=1``。
+ """
+ if os.environ.get("LISTENER_DISABLE_TTS_GATE") == "1":
+ return False
+ for topic in reversed(runtime_window.values()):
+ if topic.device_name == "speaker":
+ if logger and topic.running:
+ logger.info("TTS gate: speaker running=%s (window size=%d)", topic.running, len(runtime_window))
+ return topic.running
+ return False
+
+
+async def main(matrix: Matrix) -> None:
+ logger = matrix.logger or logging.getLogger("moss.listener")
+ logger.info("Listener app starting")
+
+ # -- transport & source (consumer only, do not start capture) --
+ transport: AudioTransport = MatrixAudioTransport(matrix=matrix)
+ capture_config = AudioCaptureConfig()
+ # Aether's vpio_capture publishes 16kHz mono PCM. The legacy MiniAudio
+ # capture path used AudioCaptureConfig.sample_rate (44.1k by default), but
+ # applying that default to VPIO double-resamples 16k audio and corrupts ASR
+ # timing. Keep this env-tunable for non-VPIO listener modes.
+ input_sample_rate = _env_int("LISTENER_INPUT_SAMPLE_RATE", _ASR_SAMPLE_RATE)
+ source = MiniAudioCaptureSource(transport=transport, config=capture_config)
+ consumer = source.new_sequential_consumer(max_queue_frames=128)
+ await consumer.start()
+ logger.info("Audio sequential consumer started (input_sample_rate=%d)", input_sample_rate)
+
+ # -- Subscribe to AudioRuntimeTopic for TTS gating --
+ runtime_window = transport.topic_window(AudioRuntimeTopic, max_size=256)
+ logger.info("Subscribed to AudioRuntimeTopic window for TTS gating and ASR control")
+
+ # -- ASR (16kHz 是语音识别的标准采样率; 如果 capture 不是 16kHz 则重采样) --
+ # end_window_size: 服务端静音判停阈值。火山官方建议 800ms 或 1000ms;
+ # 过小会切碎句子,过大则明显拖慢 listen -> think。
+ asr_end_window_ms = _env_int("LISTENER_ASR_END_WINDOW_MS", 1000)
+ silence_patience = _env_float("LISTENER_SILENCE_PATIENCE", 3.2)
+ logger.info(
+ "ASR segmentation config: end_window_size=%dms, silence_patience=%.1fs",
+ asr_end_window_ms,
+ silence_patience,
+ )
+ asr_source = "volcengine_asr"
+ asr_config = VolcengineASRConfig(
+ sample_rate=_ASR_SAMPLE_RATE,
+ end_window_size=asr_end_window_ms,
+ force_to_speech_time=_env_int("VOLCENGINE_BM_ASR_FORCE_TO_SPEECH_TIME_MS", 1000),
+ )
+ asr = VolcengineASR(config=asr_config, logger=logger)
+ logger.info("ASR backend selected: %s", asr_source)
+
+ # -- main recognition loop --
+ try:
+ consecutive_asr_errors = 0
+ asr_control = _initial_asr_control()
+ while True:
+ asr_control = _refresh_asr_control(runtime_window, asr_control)
+ if asr_control.mode == "manual" and not asr_control.enabled:
+ if consecutive_asr_errors:
+ consecutive_asr_errors = 0
+ transport.pub_topic(AudioRuntimeTopic(
+ running=False,
+ device_name="asr",
+ device_explain=_asr_diag_payload(
+ source=asr_source,
+ state="manual_idle",
+ mode=asr_control.mode,
+ ),
+ started_at=time.monotonic(),
+ last_heartbeat=time.monotonic(),
+ ))
+ await asyncio.sleep(0.08)
+ continue
+
+ logger.info("Waiting for speech...")
+ preflight_timeout = _env_float("LISTENER_PRE_ASR_AUDIO_TIMEOUT", 2.0)
+ try:
+ first_chunk = await asyncio.wait_for(consumer.__anext__(), timeout=preflight_timeout)
+ except asyncio.TimeoutError:
+ logger.warning(
+ "ASR preflight: no audio frame for %.1fs; not opening ASR connection",
+ preflight_timeout,
+ )
+ transport.pub_topic(AudioRuntimeTopic(
+ running=False,
+ device_name="asr",
+ device_explain=_asr_diag_payload(
+ source=asr_source,
+ state="audio_stalled",
+ timeout=preflight_timeout,
+ ),
+ started_at=time.monotonic(),
+ last_heartbeat=time.monotonic(),
+ ))
+ await asyncio.sleep(0.2)
+ continue
+ except StopAsyncIteration:
+ logger.warning("ASR preflight: audio consumer stopped")
+ await asyncio.sleep(0.2)
+ continue
+
+ # NOTE: 不再在 TTS 播放时 hold ASR。
+ # VPIO AEC 已经在系统层抑制扬声器回声;如果仍在这里把
+ # speaker running 时的非唤醒词结果丢弃,用户在 speak 期间说的话
+ # 就永远不会发布 SpeechTopic,前端会表现成 listen 后直接 idle。
+
+ # Fresh abort flag and utterance id for this utterance.
+ abort_event = asyncio.Event()
+ utterance_id = unique_id()
+ started_emitted = False
+ asr_running_published = False
+
+ utterance_published = False
+
+ # Each recognize call handles one utterance.
+ # The ASR backend (end_window_size) splits on silence.
+ audio_gen = _audio_generator(
+ consumer,
+ input_sample_rate,
+ _ASR_SAMPLE_RATE,
+ runtime_window,
+ asr_control,
+ abort_event,
+ logger,
+ initial_chunk=first_chunk,
+ frame_timeout=_env_float("LISTENER_AUDIO_FRAME_TIMEOUT", 2.0),
+ )
+ async for result in _iter_with_silence_timeout(
+ asr.recognize(audio_gen),
+ logger,
+ patience=silence_patience,
+ first_result_timeout=60.0,
+ ):
+ if result.text.startswith(_VOLCENGINE_ASR_ERROR_PREFIX):
+ raw = result.text.removeprefix(_VOLCENGINE_ASR_ERROR_PREFIX)
+ code, _, message = raw.partition("|")
+ consecutive_asr_errors += 1
+ backoff = min(20.0, 2.0 * consecutive_asr_errors)
+ logger.warning(
+ "ASR server error %s; message=%s; backing off %.1fs before reconnect (consecutive=%d)",
+ code,
+ message[:300],
+ backoff,
+ consecutive_asr_errors,
+ )
+ transport.pub_topic(AudioRuntimeTopic(
+ running=False,
+ device_name="asr",
+ device_explain=_asr_diag_payload(
+ source=asr_source,
+ error="server_error",
+ code=code,
+ message=message,
+ backoff=backoff,
+ consecutive=consecutive_asr_errors,
+ ),
+ started_at=time.monotonic(),
+ last_heartbeat=time.monotonic(),
+ ))
+ await asyncio.sleep(backoff)
+ break
+
+ if result.text:
+ logger.info("ASR partial: %s (final=%s)", result.text, result.is_final)
+ consecutive_asr_errors = 0
+ asr_explain = _asr_diag_payload(
+ source=asr_source,
+ text=result.text,
+ final=bool(result.is_final),
+ )
+ if not asr_running_published:
+ transport.pub_topic(AudioRuntimeTopic(
+ running=True,
+ device_name="asr",
+ device_explain=asr_explain,
+ started_at=time.monotonic(),
+ last_heartbeat=time.monotonic(),
+ ))
+ asr_running_published = True
+ else:
+ transport.pub_topic(AudioRuntimeTopic(
+ running=True,
+ device_name="asr",
+ device_explain=asr_explain,
+ started_at=time.monotonic(),
+ last_heartbeat=time.monotonic(),
+ ))
+
+ # 全双工核心:检测明确停止意图 → 立刻打断。
+ # 不再强依赖 speaker running topic;TTS 状态上报可能滞后于 ASR partial。
+ tts_active = _is_tts_playing(runtime_window, logger)
+ if result.text and _wake_word_hit(result.text):
+ logger.info("★ Wake word detected: %s — BARGE-IN!", result.text)
+ # 1) 视觉信号:前端切 interrupt 状态
+ interrupt_topic = AudioRuntimeTopic(
+ running=True,
+ device_name="interrupt",
+ device_explain="wake_word_barge_in",
+ started_at=time.monotonic(),
+ last_heartbeat=time.monotonic(),
+ )
+ transport.pub_topic(interrupt_topic)
+ # 2) ghost 中断信号:发 interrupt signal 到 ghost 主进程 (通过 Zenoh 跨进程)
+ # → mindflow.InterruptNucleus → FATAL impulse → shell.clear() → 停 TTS + 停 LLM
+ # listener 是独立子进程,不能直接访问 ghost 的 Mindflow,
+ # 必须通过 session.add_signal 走 Zenoh 发布。
+ try:
+ from ghoshell_moss.core.mindflow.interrupt_nucleus import new_interrupt_signal
+ sig = new_interrupt_signal(
+ "立刻停下",
+ description="用户喊'立刻停下',全双工 barge-in",
+ )
+ matrix.session.add_signal(sig)
+ logger.info("★ Interrupt signal sent via zenoh (shell.clear will fire)")
+ except Exception as e:
+ logger.warning("Failed to send interrupt signal: %s", e)
+ # 中断当前 ASR 识别
+ abort_event.set()
+ break
+
+ # 旧的保守模式:TTS 播放时只保留 wake word,其他结果丢弃。
+ # Aether 默认关闭这条门控,保证真正全双工。
+ if tts_active and os.environ.get("LISTENER_GATE_DURING_TTS") == "1":
+ continue
+
+ # Emit SPEECH_STARTED on first non-empty intermediate result for
+ # attention preemption (incomplete impulse with interrupt=True).
+ if not result.is_final and result.text and not started_emitted:
+ started_meta = AudioSignal(action=AudioAction.SPEECH_STARTED)
+ sig = Signal(
+ id=utterance_id,
+ name=started_meta.signal_name(),
+ priority=Priority.WARNING,
+ messages=[Message.new().with_content(result.text)],
+ description=f"Speech: {result.text}",
+ metadata=started_meta.model_dump(exclude_defaults=True, exclude_none=True),
+ complete=False,
+ )
+ matrix.session.add_signal(sig)
+ started_emitted = True
+ logger.info("Emitted SPEECH_STARTED signal (utterance=%s)", utterance_id)
+
+ if result.is_final and result.text:
+ # Publish SpeechTopic
+ speech_topic = SpeechTopic(
+ text=result.text,
+ speaker_id="human",
+ speaker_name="User",
+ role="human",
+ timestamp=time.monotonic(),
+ )
+ transport.pub_topic(speech_topic)
+ logger.info("Published SpeechTopic: %s", result.text)
+
+ # Emit AudioSignal (SPEECH_FINAL) to mindflow
+ audio_meta = AudioSignal(
+ action=AudioAction.SPEECH_FINAL,
+ speech_topic=speech_topic,
+ )
+ sig = Signal(
+ id=utterance_id,
+ name=audio_meta.signal_name(),
+ priority=Priority.WARNING,
+ messages=[Message.new().with_content(result.text)],
+ description=f"Speech: {result.text}",
+ metadata=audio_meta.model_dump(exclude_defaults=True, exclude_none=True),
+ complete=True,
+ )
+ matrix.session.add_signal(sig)
+ logger.info("Emitted SPEECH_FINAL signal (utterance=%s)", utterance_id)
+ utterance_published = True
+ break
+
+ if asr_running_published:
+ transport.pub_topic(AudioRuntimeTopic(
+ running=False,
+ device_name="asr",
+ device_explain=_asr_diag_payload(source=asr_source, state="idle"),
+ started_at=time.monotonic(),
+ last_heartbeat=time.monotonic(),
+ ))
+
+ # Cooldown: after publishing a speech result, hold ASR briefly until
+ # the ghost starts TTS (covers LLM thinking time). With DeepSeek V4
+ # Flash (TTFT ~1.1s) + streaming TTS, 2s is enough.
+ if utterance_published:
+ for _ in range(40): # up to 2s
+ if _is_tts_playing(runtime_window, logger):
+ logger.info("TTS detected during post-utterance cooldown, holding")
+ break
+ await asyncio.sleep(0.05)
+
+ # NOTE: We intentionally do NOT drain post-utterance here.
+ # Any leftover chunks are either:
+ # - ambient noise (ASR VAD will ignore)
+ # - user's next utterance started early (must NOT discard)
+ # Pre-call gate above handles TTS residue when TTS is actually playing.
+
+ except asyncio.CancelledError:
+ logger.info("Listener app cancelled")
+ except Exception:
+ logger.exception("Listener app error")
+ finally:
+ await consumer.close()
+ await asr.close()
+ logger.info("Listener app stopped")
+
+
+if __name__ == "__main__":
+ Matrix.discover().run(main)
diff --git a/.moss_ws/apps/sensors/listener/pyproject.toml b/.moss_ws/apps/aether/listener/pyproject.toml
similarity index 100%
rename from .moss_ws/apps/sensors/listener/pyproject.toml
rename to .moss_ws/apps/aether/listener/pyproject.toml
diff --git a/.moss_ws/apps/aether/vpio_capture/.gitignore b/.moss_ws/apps/aether/vpio_capture/.gitignore
new file mode 100644
index 00000000..26b5a715
--- /dev/null
+++ b/.moss_ws/apps/aether/vpio_capture/.gitignore
@@ -0,0 +1,9 @@
+# runtime
+runtime/
+*.log
+__pycache__/
+*.pyc
+.env
+venv/
+.venv/
+uv.lock
diff --git a/.moss_ws/apps/aether/vpio_capture/APP.md b/.moss_ws/apps/aether/vpio_capture/APP.md
new file mode 100644
index 00000000..cdd18a9e
--- /dev/null
+++ b/.moss_ws/apps/aether/vpio_capture/APP.md
@@ -0,0 +1,10 @@
+---
+arguments: ''
+description: 'Aether VPIO capture — macOS AVAudioEngine voice processing AEC, publishes 16kHz PCM for full-duplex voice conversation.'
+executable: uv
+respawn: false
+script: main.py
+workers: 1
+---
+
+Aether VPIO capture app. Canonical app address: `aether/vpio_capture`.
diff --git a/.moss_ws/apps/aether/vpio_capture/main.py b/.moss_ws/apps/aether/vpio_capture/main.py
new file mode 100644
index 00000000..5e16e49e
--- /dev/null
+++ b/.moss_ws/apps/aether/vpio_capture/main.py
@@ -0,0 +1,61 @@
+"""Aether VPIO audio capture app — macOS system-level AEC.
+
+Opens AVAudioEngine with setVoiceProcessingEnabled(True) on both input + output
+nodes → system-level AEC subtracts TTS playback from the mic signal → ASR only
+hears the user's voice. Publishes 16kHz / mono / int16 PCM to Zenoh stream
+(audio/pcm) using the same pack_chunk() wire format as MiniAudioCaptureSource,
+so listener / waveform apps consume unchanged.
+
+Usage:
+ moss apps test aether/vpio_capture
+ moss apps start aether/vpio_capture
+
+Replaces sensors/audio_capture when running on macOS for full-duplex conversation.
+"""
+import asyncio
+import logging
+import sys
+from pathlib import Path
+
+from ghoshell_moss.contracts.audio import AudioCaptureConfig
+from ghoshell_moss.core.blueprint.matrix import Matrix
+from ghoshell_moss.host.speech.capture.matrix_audio_transport import MatrixAudioTransport
+
+# Local import — the VPIOCaptureSource lives next to main.py
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from vpio_capture import VPIOCaptureSource # noqa: E402
+
+
+async def main(matrix: Matrix) -> None:
+ logger = matrix.logger or logging.getLogger("moss.vpio_capture")
+ logger.info("VPIO audio capture app starting (macOS system-level AEC)")
+
+ transport = MatrixAudioTransport(matrix=matrix)
+ # AudioCaptureConfig defaults (sample_rate=44100 etc.) are ignored —
+ # VPIOCaptureSource forces native 48k internally and outputs 16k for ASR.
+ config = AudioCaptureConfig()
+ capture = VPIOCaptureSource(transport=transport, config=config)
+
+ try:
+ await capture.start()
+ logger.info("VPIO audio capture app started (device: %s)", capture.device_explain())
+ except RuntimeError as e:
+ # Non-macOS or pyobjc missing — let app die cleanly, listener will fall back
+ logger.error("VPIO start failed: %s", e)
+ raise
+
+ stop_event = asyncio.Event()
+
+ try:
+ await stop_event.wait()
+ except asyncio.CancelledError:
+ logger.info("VPIO audio capture app cancelled, shutting down")
+ except KeyboardInterrupt:
+ pass
+ finally:
+ await capture.close()
+ logger.info("VPIO audio capture app stopped")
+
+
+if __name__ == "__main__":
+ Matrix.discover().run(main)
diff --git a/.moss_ws/apps/aether/vpio_capture/pyproject.toml b/.moss_ws/apps/aether/vpio_capture/pyproject.toml
new file mode 100644
index 00000000..04977066
--- /dev/null
+++ b/.moss_ws/apps/aether/vpio_capture/pyproject.toml
@@ -0,0 +1,16 @@
+[project]
+name = "moss-sensor-vpio-capture"
+version = "0.1.0"
+description = "macOS VPIO-based audio capture with system-level AEC for full-duplex voice conversation."
+requires-python = ">=3.11,<3.14"
+dependencies = [
+ "pyobjc-core>=10.0",
+ "pyobjc-framework-AVFoundation>=10.0",
+ "numpy>=2.0.0",
+ "scipy>=1.11.0",
+ "ghoshell-moss[host]",
+ "python-dotenv>=1.0.0",
+]
+
+[tool.uv.sources]
+ghoshell-moss = { path = "../../../..", editable = true }
diff --git a/.moss_ws/apps/aether/vpio_capture/smoke_test.py b/.moss_ws/apps/aether/vpio_capture/smoke_test.py
new file mode 100644
index 00000000..caa679ec
--- /dev/null
+++ b/.moss_ws/apps/aether/vpio_capture/smoke_test.py
@@ -0,0 +1,141 @@
+"""Smoke test — verify PyObjC + AVAudioEngine + VPIO APIs work on this Mac.
+
+Doesn't start the full MOSS pipeline. Just exercises the API surface
+to catch PyObjC binding errors early.
+"""
+import sys
+import time
+
+import numpy as np
+
+
+def main() -> int:
+ print("[1] importing AVFoundation...")
+ try:
+ import AVFoundation
+ import AVFAudio
+ from AVFoundation import AVAudioEngine, AVAudioConverter
+ print(" OK · AVFoundation imported")
+ except ImportError as e:
+ print(f" FAIL · {e}")
+ print(" Install with: uv pip install pyobjc-framework-AVFoundation")
+ return 1
+
+ print("[2] creating AVAudioEngine...")
+ engine = AVAudioEngine.new()
+ input_node = engine.inputNode()
+ output_node = engine.outputNode()
+ print(f" OK · input={type(input_node).__name__}, output={type(output_node).__name__}")
+
+ print("[3] reading native input format...")
+ tap_format = input_node.outputFormatForBus_(0)
+ sr = int(tap_format.sampleRate())
+ ch = int(tap_format.channelCount())
+ cf = tap_format.commonFormat()
+ print(f" OK · native_sr={sr}, ch={ch}, commonFormat={cf}")
+ if sr not in (44100, 48000):
+ print(f" WARN · VPIO expects 48k or 44.1k, got {sr}")
+
+ # Selector is `setVoiceProcessingEnabled:error:` → PyObjC maps to
+ # setVoiceProcessingEnabled_error_(value, error_ptr) → returns BOOL.
+ # The error_ptr arg is consumed by PyObjC and surfaced via the BOOL + thrown exc.
+ print("[4] enabling VPIO on input node (setVoiceProcessingEnabled:error:)...")
+ try:
+ from Foundation import NSError
+ err_ptr = None
+ ok = input_node.setVoiceProcessingEnabled_error_(True, err_ptr)
+ # PyObjC: returns (BOOL, NSError) tuple when error out-param present
+ if isinstance(ok, tuple):
+ ok, err = ok
+ else:
+ err = None
+ if not ok:
+ print(f" FAIL · returned ok=False, error={err and err.localizedDescription()}")
+ return 2
+ print(f" OK · input VPIO enabled = {input_node.isVoiceProcessingEnabled()}")
+ except Exception as e:
+ print(f" FAIL · {e}")
+ return 2
+
+ print("[5] enabling VPIO on output node (required for AEC)...")
+ try:
+ err_ptr = None
+ ok = output_node.setVoiceProcessingEnabled_error_(True, err_ptr)
+ if isinstance(ok, tuple):
+ ok, err = ok
+ else:
+ err = None
+ if not ok:
+ print(f" FAIL · returned ok=False, error={err and err.localizedDescription()}")
+ return 3
+ print(f" OK · output VPIO enabled = {output_node.isVoiceProcessingEnabled()}")
+ except Exception as e:
+ print(f" FAIL · {e}")
+ return 3
+
+ print("[6] building AVAudioConverter 48k → 16k...")
+ # AVAudioCommonFormat enum (NSUInteger):
+ # 0=OtherFormat, 1=PCMFormatFloat32, 2=PCMFormatFloat64,
+ # 3=PCMFormatInt16, 4=PCMFormatInt32
+ PCM_FORMAT_FLOAT32 = 1
+ out_format = AVFAudio.AVAudioFormat.alloc().initWithCommonFormat_sampleRate_channels_interleaved_(
+ PCM_FORMAT_FLOAT32,
+ 16000.0,
+ 1,
+ False,
+ )
+ if out_format is None:
+ print(" FAIL · could not create 16kHz output format")
+ return 4
+ converter = AVAudioConverter.alloc().initFromFormat_toFormat_(tap_format, out_format)
+ if converter is None:
+ print(" FAIL · could not create AVAudioConverter")
+ return 5
+ print(f" OK · converter={converter}")
+
+ print("[7] installing tap on bus 0 for 2 seconds...")
+ frames_received = [0]
+
+ def _tap(buffer, when):
+ try:
+ n = int(buffer.frameLength())
+ if n > 0:
+ frames_received[0] += n
+ except Exception:
+ pass
+
+ buf_size = int(sr * 0.05) # 50ms frames
+ input_node.installTapOnBus_bufferSize_format_block_(
+ 0, buf_size, tap_format, _tap,
+ )
+
+ try:
+ engine.prepare()
+ # PyObjC: startAndReturnError_ returns (BOOL success, NSError* error)
+ ok, err = engine.startAndReturnError_(None)
+ if not ok:
+ print(f" FAIL · engine.start returned error: {err and err.localizedDescription()}")
+ return 6
+ print(" OK · engine started, capturing for 2s...")
+ time.sleep(2.0)
+ finally:
+ try:
+ input_node.removeTapOnBus_(0)
+ except Exception:
+ pass
+ engine.stop()
+ engine.reset()
+
+ print(f" captured {frames_received[0]} native frames in 2s "
+ f"(expected ~{sr * 2} = {sr * 2})")
+
+ print()
+ print("=" * 50)
+ print("SMOKE TEST PASSED — VPIO + AVAudioEngine APIs work on this Mac.")
+ print("Ready to run the full VPIOCaptureSource via MOSS App.")
+ print("=" * 50)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/.moss_ws/apps/aether/vpio_capture/vpio_capture.py b/.moss_ws/apps/aether/vpio_capture/vpio_capture.py
new file mode 100644
index 00000000..e42b5522
--- /dev/null
+++ b/.moss_ws/apps/aether/vpio_capture/vpio_capture.py
@@ -0,0 +1,607 @@
+"""
+VPIOCaptureSource — macOS VPIO (Voice Processing IO) audio capture with system-level AEC.
+
+Replaces MiniAudioCaptureSource when running on macOS. Opens AVAudioEngine with
+`setVoiceProcessingEnabled(true)` on both input and output nodes — this gives us
+system-level acoustic echo cancellation (AEC) for free, so TTS playback through
+the system default output is automatically subtracted from the mic signal.
+
+Key constraints (see docs/VPIO.md):
+1. VPIO audio unit MUST be initialized at 48kHz (or 44.1kHz) — hardware constraint.
+2. Both input + output nodes must enable VPIO for AEC to engage.
+3. TTS must play through system default output so VPIO can grab far-end reference.
+4. AVAudioConverter resamples 48k → 16k for ASR (Volcengine bigmodel_async).
+5. Tap callback fires on CoreAudio real-time thread — push bytes to an asyncio
+ queue and let the asyncio side do pack_chunk + transport.pub_pcm.
+
+Output format matches MiniAudioCaptureSource: 16kHz / 1ch / int16 PCM, packaged
+via the same pack_chunk() wire format. Listener app consumes unchanged.
+"""
+from __future__ import annotations
+
+import asyncio
+import collections
+import logging
+import os
+import time
+from typing import Callable
+
+import numpy as np
+
+from ghoshell_moss.contracts.audio import (
+ AudioCaptureConfig,
+ AudioCaptureSource,
+ AudioChunk,
+ AudioFrameMeta,
+ AudioPullLatest,
+ AudioSequentialConsumer,
+)
+from ghoshell_moss.host.speech.capture.audio_transport import AudioTransport
+from ghoshell_moss.host.speech.capture.miniaudio_capture import (
+ MiniAudioSequentialConsumer,
+ _compute_frame_meta,
+ pack_chunk,
+ unpack_chunk,
+)
+from ghoshell_moss.topics.audio import AudioRuntimeTopic
+
+__all__ = ["VPIOCaptureSource"]
+
+
+# VPIO hardware constraint — must be 48k (or 44.1k). We pick 48k.
+_VPIO_NATIVE_SR = 48000
+# Output sample rate consumed by ASR (matches listener's _ASR_SAMPLE_RATE).
+_OUTPUT_SR = 16000
+# Frame duration in ms — same as MiniAudioCaptureSource default.
+_FRAME_MS = 50
+_MAX_DIAG_CHANNELS = 16
+
+
+def _channel_mode() -> str:
+ mode = os.environ.get("VPIO_CHANNEL_MODE", "best").strip().lower()
+ if mode in {"best", "mix", "0"}:
+ return mode
+ return "best"
+
+
+class VPIOCaptureSource(AudioCaptureSource):
+ """macOS VPIO capture source — system-level AEC, drop-in for MiniAudioCaptureSource.
+
+ Output to transport: 16kHz / mono / int16 PCM, packaged via pack_chunk().
+ Consumer side (listener / waveform) is unchanged.
+ """
+
+ def __init__(self, *, transport: AudioTransport, config: AudioCaptureConfig | None = None):
+ self._transport = transport
+ # We override sample_rate to 16k regardless of config — ASR contract.
+ # config is kept for compatibility (channels, frame_duration_ms, device_pattern).
+ self._config = config or AudioCaptureConfig()
+ self._logger = transport.logger
+ self._seq = 0
+ self._started = False
+ self._closing = False
+
+ # AVAudioEngine state
+ self._engine = None
+ self._input_node = None
+ self._output_node = None
+ self._input_tap_bus = 0
+ self._tap_format = None
+
+ # Thread bridge: CoreAudio real-time thread → asyncio loop
+ self._loop: asyncio.AbstractEventLoop | None = None
+ self._queue: asyncio.Queue[np.ndarray | None] | None = None
+ self._pump_task: asyncio.Task | None = None
+ self._watchdog_task: asyncio.Task | None = None
+ self._last_frame_at = 0.0
+ self._last_stall_report_at = 0.0
+ self._restart_lock: asyncio.Lock | None = None
+ self._last_restart_at = 0.0
+ self._restart_attempts = 0
+
+ # -- lifecycle --
+
+ async def start(self) -> None:
+ if self._started:
+ return
+
+ if not self._transport.acquire_lock():
+ self._logger.warning("Audio capture lock held by another process, skipping start")
+ self._started = True
+ return
+
+ self._loop = asyncio.get_running_loop()
+ # Bounded queue: if asyncio side falls behind, drop oldest to keep latency bounded.
+ self._queue = asyncio.Queue(maxsize=64)
+ self._restart_lock = asyncio.Lock()
+
+ try:
+ await self._start_engine()
+ except Exception:
+ await self._cleanup_engine()
+ self._transport.release_lock()
+ raise
+
+ # Start asyncio pump — drains queue, packs chunks, pub to transport
+ self._last_frame_at = time.monotonic()
+ self._pump_task = asyncio.create_task(self._pump_loop())
+ self._watchdog_task = asyncio.create_task(self._watchdog_loop())
+
+ self._started = True
+ self._transport.pub_topic(AudioRuntimeTopic(
+ running=True,
+ device_name="vpio",
+ device_explain=self.device_explain(),
+ started_at=time.time(),
+ last_heartbeat=time.time(),
+ ))
+ self._logger.info("VPIO: capture started (%s)", self.device_explain())
+
+ def device_explain(self) -> str:
+ if self._engine is None:
+ return "not started"
+ return (f"macOS VPIO, native={_VPIO_NATIVE_SR}Hz → out={_OUTPUT_SR}Hz, "
+ f"1ch, pcm_s16le, AEC=system-level")
+
+ async def close(self) -> None:
+ if self._closing:
+ return
+ self._closing = True
+
+ # Stop pump first so we don't publish partial frames during teardown
+ if self._pump_task is not None:
+ if self._queue is not None:
+ try:
+ self._queue.put_nowait(None) # sentinel
+ except asyncio.QueueFull:
+ try:
+ self._queue.get_nowait()
+ self._queue.put_nowait(None)
+ except asyncio.QueueEmpty:
+ pass
+ try:
+ await asyncio.wait_for(self._pump_task, timeout=2.0)
+ except asyncio.TimeoutError:
+ self._pump_task.cancel()
+ except Exception:
+ pass
+ self._pump_task = None
+
+ if self._watchdog_task is not None:
+ self._watchdog_task.cancel()
+ try:
+ await self._watchdog_task
+ except asyncio.CancelledError:
+ pass
+ self._watchdog_task = None
+
+ await self._cleanup_engine()
+
+ self._transport.pub_topic(AudioRuntimeTopic(
+ running=False,
+ last_heartbeat=time.time(),
+ ))
+ self._transport.release_lock()
+ self._started = False
+ self._logger.info("VPIO: capture closed")
+
+ # -- consumer factories (same as MiniAudio — they consume from transport, agnostic to source) --
+
+ def new_consumer(self, ring_buffer_frames: int = 64) -> AudioPullLatest:
+ # Reuse the same _MiniAudioPullLatest — it only reads transport stream.
+ from ghoshell_moss.host.speech.capture.miniaudio_capture import _MiniAudioPullLatest
+ return _MiniAudioPullLatest(
+ transport=self._transport,
+ maxlen=ring_buffer_frames,
+ logger=self._logger,
+ )
+
+ def new_sequential_consumer(self, max_queue_frames: int = 128) -> AudioSequentialConsumer:
+ return MiniAudioSequentialConsumer(
+ transport=self._transport,
+ maxsize=max_queue_frames,
+ logger=self._logger,
+ )
+
+ # -- internals --
+
+ async def _start_engine(self) -> None:
+ # Lazy import — only fails on non-macOS or missing pyobjc
+ try:
+ import AVFoundation # noqa: F401
+ from AVFoundation import AVAudioEngine # noqa: F401
+ except ImportError as e:
+ raise RuntimeError(
+ "VPIOCaptureSource requires macOS with pyobjc-framework-AVFoundation. "
+ f"Import failed: {e}. Install with: uv pip install pyobjc-framework-AVFoundation"
+ ) from e
+
+ from AVFoundation import AVAudioEngine
+
+ # 1) Build engine + enable VPIO on BOTH input and output nodes.
+ self._engine = AVAudioEngine.new()
+ self._input_node = self._engine.inputNode()
+ self._output_node = self._engine.outputNode()
+
+ # Enable VPIO on input — this is where AEC lives for the mic side.
+ # PyObjC: setVoiceProcessingEnabled_error_(value, error_ptr) → returns BOOL.
+ try:
+ ok = self._input_node.setVoiceProcessingEnabled_error_(True, None)
+ if isinstance(ok, tuple):
+ ok, err = ok
+ else:
+ err = None
+ if not ok:
+ self._logger.warning("VPIO: inputNode VPIO enable failed (AEC may not engage): %s",
+ err and err.localizedDescription())
+ else:
+ self._logger.info("VPIO: inputNode.setVoiceProcessingEnabled = True")
+ except Exception as e:
+ self._logger.warning("VPIO: inputNode VPIO enable failed (AEC may not engage): %s", e)
+
+ # Enable VPIO on output — required for AEC to engage.
+ try:
+ ok = self._output_node.setVoiceProcessingEnabled_error_(True, None)
+ if isinstance(ok, tuple):
+ ok, err = ok
+ else:
+ err = None
+ if not ok:
+ self._logger.warning("VPIO: outputNode VPIO enable failed (AEC may not engage): %s",
+ err and err.localizedDescription())
+ else:
+ self._logger.info("VPIO: outputNode.setVoiceProcessingEnabled = True")
+ except Exception as e:
+ self._logger.warning("VPIO: outputNode VPIO enable failed (AEC may not engage): %s", e)
+
+ # Report what VPIO actually applied (settings may downgrade silently).
+ self._log_vpio_diagnostics()
+
+ # 2) Native tap format = hardware format of inputNode (typically 48k / 1ch / float32).
+ self._tap_format = self._input_node.outputFormatForBus_(self._input_tap_bus)
+ native_sr = int(self._tap_format.sampleRate())
+ native_ch = int(self._tap_format.channelCount())
+ self._logger.info(
+ "VPIO: tap format native_sr=%d ch=%d commonFormat=%s",
+ native_sr, native_ch, self._tap_format.commonFormat(),
+ )
+
+ if native_sr != _VPIO_NATIVE_SR:
+ self._logger.warning(
+ "VPIO: native_sr=%d is not %d — AEC may fail on some macOS versions",
+ native_sr, _VPIO_NATIVE_SR,
+ )
+
+ # 3) Resampling is done on the asyncio side via scipy.signal.resample_poly.
+
+ # 4) Install tap on input bus — callback fires on CoreAudio real-time thread.
+ self._install_tap()
+
+ # 5) Start engine — startAndReturnError_ returns (BOOL success, NSError* error) tuple.
+ try:
+ self._engine.prepare()
+ ok, err = self._engine.startAndReturnError_(None)
+ if not ok:
+ raise RuntimeError(f"engine.start failed: {err and err.localizedDescription()}")
+ except Exception as e:
+ self._logger.exception("VPIO: engine start failed: %s", e)
+ await self._cleanup_engine()
+ raise
+
+ def _drain_audio_queue(self) -> int:
+ if self._queue is None:
+ return 0
+ drained = 0
+ while True:
+ try:
+ self._queue.get_nowait()
+ drained += 1
+ except asyncio.QueueEmpty:
+ return drained
+
+ async def _recover_from_stall(self, age: float) -> None:
+ if self._restart_lock is None or self._closing:
+ return
+
+ async with self._restart_lock:
+ if self._closing:
+ return
+ fresh_age = time.monotonic() - self._last_frame_at
+ if fresh_age < age:
+ return
+
+ self._last_restart_at = time.monotonic()
+ self._restart_attempts += 1
+ self._logger.warning(
+ "VPIO stalled: restarting AVAudioEngine after %.1fs without frames (attempt=%d)",
+ fresh_age,
+ self._restart_attempts,
+ )
+ self._transport.pub_topic(AudioRuntimeTopic(
+ running=False,
+ device_name="vpio",
+ device_explain=f"state=restarting,no_frames_for={fresh_age:.1f}s,attempt={self._restart_attempts}",
+ started_at=time.time(),
+ last_heartbeat=time.time(),
+ ))
+
+ try:
+ await self._cleanup_engine()
+ drained = self._drain_audio_queue()
+ if drained:
+ self._logger.info("VPIO: drained %d queued audio frames before restart", drained)
+ await asyncio.sleep(0.2)
+ await self._start_engine()
+ self._last_frame_at = time.monotonic()
+ self._last_stall_report_at = 0.0
+ self._transport.pub_topic(AudioRuntimeTopic(
+ running=True,
+ device_name="vpio",
+ device_explain=f"state=restarted,attempt={self._restart_attempts}",
+ started_at=time.time(),
+ last_heartbeat=time.time(),
+ ))
+ self._logger.info("VPIO: AVAudioEngine restart completed")
+ except Exception as e:
+ self._logger.exception("VPIO: AVAudioEngine restart failed: %s", e)
+ self._transport.pub_topic(AudioRuntimeTopic(
+ running=False,
+ device_name="vpio",
+ device_explain=f"state=restart_failed,error={type(e).__name__}",
+ started_at=time.time(),
+ last_heartbeat=time.time(),
+ ))
+
+ def _log_vpio_diagnostics(self) -> None:
+ """Report what VPIO actually applied (may downgrade silently)."""
+ try:
+ in_vpio = self._input_node.isVoiceProcessingEnabled()
+ out_vpio = self._output_node.isVoiceProcessingEnabled()
+ self._logger.info(
+ "VPIO report · input.vpio=%s · output.vpio=%s · "
+ "(both must be True for AEC to engage)",
+ in_vpio, out_vpio,
+ )
+ except Exception as e:
+ self._logger.warning("VPIO report failed: %s", e)
+
+ def _install_tap(self) -> None:
+ """Install tap on inputNode bus 0 with native format.
+
+ Callback runs on CoreAudio real-time thread. Per VPIO.md §4.6 we keep
+ the hot path minimal: copy float32 channels off the realtime buffer
+ into a numpy array, hand it to the asyncio loop via
+ call_soon_threadsafe. All heavy work (resample 48k→16k, int16
+ conversion, FFT meta, pack_chunk, transport.pub_pcm) happens on the
+ asyncio side in _pump_loop.
+ """
+ buf_size = int(_VPIO_NATIVE_SR * _FRAME_MS / 1000) # 48k * 50ms = 2400 samples
+ loop = self._loop
+
+ def _tap_callback(buffer, when):
+ # buffer: AVAudioPCMBuffer at native 48k/float32/Nch (post-AEC).
+ # Mac input devices can expose multi-channel arrays. Channel 0 is
+ # not always the speech-dominant channel, especially with external
+ # or aggregate devices, so copy all channels and choose on asyncio.
+ # floatChannelData() returns a tuple of objc.varlist objects;
+ # varlist[0:n] returns a Python list of floats — fastest path
+ # through PyObjC (~0.14ms for 4800 samples). Copy is mandatory
+ # because the underlying buffer is owned by CoreAudio.
+ try:
+ n = int(buffer.frameLength())
+ if n == 0:
+ return
+ ch_data = buffer.floatChannelData()
+ if ch_data is None:
+ return
+ channel_count = min(int(buffer.format().channelCount()), len(ch_data), _MAX_DIAG_CHANNELS)
+ if channel_count <= 1:
+ arr = np.array(ch_data[0][0:n], dtype=np.float32)
+ else:
+ channels = [
+ np.array(ch_data[ch][0:n], dtype=np.float32)
+ for ch in range(channel_count)
+ ]
+ arr = np.stack(channels, axis=0)
+ try:
+ loop.call_soon_threadsafe(self._enqueue, arr)
+ except RuntimeError:
+ # loop closed during shutdown
+ pass
+ except Exception:
+ # Don't log on the realtime thread — just swallow
+ pass
+
+ self._input_node.installTapOnBus_bufferSize_format_block_(
+ self._input_tap_bus,
+ buf_size,
+ self._tap_format,
+ _tap_callback,
+ )
+ self._logger.info("VPIO: tap installed on bus %d, bufSize=%d", self._input_tap_bus, buf_size)
+
+ def _enqueue(self, pcm: np.ndarray) -> None:
+ """Called on the asyncio loop via call_soon_threadsafe — safe to put_nowait."""
+ if self._queue is None or self._closing:
+ return
+ self._last_frame_at = time.monotonic()
+ if self._queue.full():
+ # Drop oldest to bound latency — better to lose a frame than to lag.
+ try:
+ self._queue.get_nowait()
+ except asyncio.QueueEmpty:
+ pass
+ try:
+ self._queue.put_nowait(pcm)
+ except asyncio.QueueFull:
+ pass
+
+ async def _watchdog_loop(self) -> None:
+ """Publish diagnostics and recover if CoreAudio tap stops producing frames."""
+ threshold = float(os.environ.get("VPIO_STALL_SECONDS", "2.5") or "2.5")
+ restart_threshold = float(os.environ.get("VPIO_RESTART_STALL_SECONDS", "8.0") or "8.0")
+ restart_cooldown = float(os.environ.get("VPIO_RESTART_COOLDOWN_SECONDS", "5.0") or "5.0")
+ auto_restart = os.environ.get("VPIO_AUTO_RESTART_ON_STALL", "1").strip() != "0"
+ while not self._closing:
+ await asyncio.sleep(0.5)
+ if not self._started:
+ continue
+ age = time.monotonic() - self._last_frame_at
+ if age < threshold:
+ continue
+ now = time.monotonic()
+ if now - self._last_stall_report_at < threshold:
+ continue
+ self._last_stall_report_at = now
+ diag = f"state=stalled,no_frames_for={age:.1f}s"
+ self._logger.warning("VPIO stalled: no frames for %.1fs", age)
+ self._transport.pub_topic(AudioRuntimeTopic(
+ running=False,
+ device_name="vpio",
+ device_explain=diag,
+ started_at=time.time(),
+ last_heartbeat=time.time(),
+ ))
+ if (
+ auto_restart
+ and age >= restart_threshold
+ and now - self._last_restart_at >= restart_cooldown
+ ):
+ await self._recover_from_stall(age)
+
+ async def _pump_loop(self) -> None:
+ """Drain queue on the asyncio side.
+
+ Per-frame work: resample 48k float32 → 16k int16, build AudioChunk,
+ pack_chunk, transport.pub_pcm. All heavy work lives here, not in the
+ realtime tap callback.
+ """
+ import math
+ from scipy import signal as _scipy_signal
+
+ assert self._queue is not None
+ # 48000 → 16000: gcd=16000, up=1, down=3
+ _up = _OUTPUT_SR
+ _down = _VPIO_NATIVE_SR
+ _g = math.gcd(_up, _down)
+ _up_r = _up // _g # 1
+ _down_r = _down // _g # 3
+ stat_next_log = time.monotonic() + 1.0
+ stat_frames = 0
+ stat_max_rms = 0.0
+ stat_max_peak = 0.0
+ stat_best_ch_counts: collections.Counter[int] = collections.Counter()
+ stat_max_channel_rms: np.ndarray | None = None
+ channel_mode = _channel_mode()
+ self._logger.info("VPIO: channel mode=%s (env VPIO_CHANNEL_MODE=best|mix|0)", channel_mode)
+
+ while True:
+ pcm_f32 = await self._queue.get()
+ if pcm_f32 is None: # shutdown sentinel
+ return
+ try:
+ ts = time.time()
+ # Resample 48k → 16k using polyphase filter (matches listener's resampler)
+ if pcm_f32.size == 0:
+ continue
+ best_ch = 0
+ channel_rms = None
+ if pcm_f32.ndim == 2:
+ channel_rms = np.sqrt(np.mean(pcm_f32 * pcm_f32, axis=1))
+ best_ch = int(np.argmax(channel_rms)) if channel_rms.size else 0
+ stat_best_ch_counts[best_ch] += 1
+ if stat_max_channel_rms is None or stat_max_channel_rms.shape != channel_rms.shape:
+ stat_max_channel_rms = channel_rms.copy()
+ else:
+ stat_max_channel_rms = np.maximum(stat_max_channel_rms, channel_rms)
+
+ if channel_mode == "mix":
+ pcm_f32 = np.mean(pcm_f32, axis=0, dtype=np.float32)
+ elif channel_mode == "0":
+ pcm_f32 = pcm_f32[0]
+ else:
+ pcm_f32 = pcm_f32[best_ch]
+ resampled = _scipy_signal.resample_poly(
+ pcm_f32.astype(np.float32), _up_r, _down_r,
+ )
+ resampled_f32 = resampled.astype(np.float32, copy=False)
+ rms = float(np.sqrt(np.mean(resampled_f32 * resampled_f32))) if resampled_f32.size else 0.0
+ peak = float(np.max(np.abs(resampled_f32))) if resampled_f32.size else 0.0
+ stat_frames += 1
+ stat_max_rms = max(stat_max_rms, rms)
+ stat_max_peak = max(stat_max_peak, peak)
+ # float32 [-1.0, 1.0] → int16 [-32768, 32767]
+ pcm_i16 = (resampled * 32767.0).clip(-32768, 32767).astype(np.int16)
+ # 2D shape (n, 1) — matches MiniAudioCaptureSource convention
+ samples = pcm_i16.reshape(-1, 1)
+ meta = _compute_frame_meta(samples)
+ seq = self._seq
+ self._seq += 1
+ chunk = AudioChunk(seq=seq, timestamp=ts, samples=samples, meta=meta)
+ packed = pack_chunk(chunk)
+ self._transport.pub_pcm(packed)
+ now = time.monotonic()
+ if now >= stat_next_log:
+ top_channels = ""
+ if stat_max_channel_rms is not None:
+ ranked = sorted(
+ enumerate(stat_max_channel_rms.tolist()),
+ key=lambda item: item[1],
+ reverse=True,
+ )[:4]
+ top_channels = ",".join(f"{idx}:{rms:.4f}" for idx, rms in ranked)
+ dominant_ch = stat_best_ch_counts.most_common(1)[0][0] if stat_best_ch_counts else 0
+ diag = (
+ f"mode={channel_mode},best_ch={dominant_ch},"
+ f"ch_rms={top_channels},rms={stat_max_rms:.5f},"
+ f"peak={stat_max_peak:.5f},frames={stat_frames}"
+ )
+ self._logger.info(
+ "VPIO audio stats · frames=%d · max_rms=%.5f · max_peak=%.5f · "
+ "best_ch=%d · ch_rms=%s · mode=%s · seq=%d",
+ stat_frames,
+ stat_max_rms,
+ stat_max_peak,
+ dominant_ch,
+ top_channels or "n/a",
+ channel_mode,
+ seq,
+ )
+ self._transport.pub_topic(AudioRuntimeTopic(
+ running=True,
+ device_name="vpio",
+ device_explain=diag,
+ started_at=time.time(),
+ last_heartbeat=time.time(),
+ ))
+ stat_next_log = now + 1.0
+ stat_frames = 0
+ stat_max_rms = 0.0
+ stat_max_peak = 0.0
+ stat_best_ch_counts.clear()
+ stat_max_channel_rms = None
+ except Exception:
+ self._logger.exception("VPIO: error in pump loop")
+
+ async def _cleanup_engine(self) -> None:
+ if self._input_node is not None:
+ try:
+ self._input_node.removeTapOnBus_(self._input_tap_bus)
+ self._logger.info("VPIO: tap removed")
+ except Exception as e:
+ self._logger.warning("VPIO: removeTap failed: %s", e)
+ self._input_node = None
+
+ if self._engine is not None:
+ try:
+ self._engine.stop()
+ except Exception:
+ pass
+ try:
+ self._engine.reset()
+ except Exception:
+ pass
+ self._engine = None
+
+ self._output_node = None
+ self._tap_format = None
diff --git a/.moss_ws/apps/bodies/g1_sim/APP.md b/.moss_ws/apps/bodies/g1_sim/APP.md
index adeaf15b..e6a8b7af 100644
--- a/.moss_ws/apps/bodies/g1_sim/APP.md
+++ b/.moss_ws/apps/bodies/g1_sim/APP.md
@@ -50,7 +50,7 @@ G1 pure software simulation app.
## 麦克风指挥
- MOSS 里已经有现成的麦克风链路,不需要为了 `g1_sim` 额外再造一个新的麦克风 app
-- 连续监听链路:`sensors/audio_capture` + `sensors/listener`
+- 连续监听链路:`sensors/audio_capture` + `aether/listener`
- 按键说话链路:`sensors/ptt_listener`
- `listener / ptt_listener` 会把用户语音识别成文本,发布成 `SpeechTopic` 并发出 `AudioSignal`
- 默认 Ghost 会通过 `audio_nucleus` 接收这些语音输入,再结合 `apps.bodies_g1_sim` 的动态接口与说明,把自然语言转成 CTML 调用
@@ -60,7 +60,7 @@ G1 pure software simulation app.
```ctml
-
+
```
如果你更想避免环境噪声误触发,推荐先用 PTT:
diff --git a/.moss_ws/apps/bodies/g1_sim/README.md b/.moss_ws/apps/bodies/g1_sim/README.md
index a813f2b3..8f56bb6e 100644
--- a/.moss_ws/apps/bodies/g1_sim/README.md
+++ b/.moss_ws/apps/bodies/g1_sim/README.md
@@ -48,7 +48,7 @@
- 已实现 viewer 自动跟随相机与稳定地面视角
- 已实现语音编排模板和精准同步版 CTML
- 已接好麦克风控制的系统侧前提:
-- `sensors/audio_capture + sensors/listener`
+- `sensors/audio_capture + aether/listener`
- `sensors/ptt_listener`
## 运行方式
diff --git a/.moss_ws/apps/sensors/listener/APP.md b/.moss_ws/apps/sensors/listener/APP.md
deleted file mode 100644
index af00efd9..00000000
--- a/.moss_ws/apps/sensors/listener/APP.md
+++ /dev/null
@@ -1,10 +0,0 @@
----
-arguments: ''
-description: 'ASR consumer — audio → Volcengine ASR → SpeechTopic + AudioSignal'
-executable: uv
-respawn: false
-script: main.py
-workers: 1
----
-
-ASR consumer — audio → Volcengine ASR → SpeechTopic + AudioSignal
\ No newline at end of file
diff --git a/.moss_ws/apps/sensors/listener/CLAUDE.md b/.moss_ws/apps/sensors/listener/CLAUDE.md
deleted file mode 100644
index 04a48775..00000000
--- a/.moss_ws/apps/sensors/listener/CLAUDE.md
+++ /dev/null
@@ -1,66 +0,0 @@
-# MOSS App Development
-
-You are working inside a MOSS App — an independent, process-isolated unit deliverable
-through the MOSS app system. Think of this as its own small project.
-
-## Project mindset
-
-- This app is a standalone unit. It may be downloaded from a hub or shared independently.
-- Tests live in `tests/` under this directory, NOT in the main project's `tests/`.
-- Dependencies go in `pyproject.toml`; the app gets its own `.venv` via `uv run`.
-
-## Directory layout
-
-```
-apps///
-├── APP.md # metadata — MOSS discovers the app through this
-├── CLAUDE.md # this file — AI developer context
-├── main.py # entry point
-├── pyproject.toml # optional — independent dependencies
-├── src/ # optional — multi-module app code
-├── tests/ # tests live here, NOT in the main project
-└── runtime/ # runtime data (auto-created by MOSS)
- ├── assets/
- ├── configs/
- └── logs/
-```
-
-When you have a `pyproject.toml`, treat this as a proper project — source in `src/`,
-tests in `tests/`, managed by `uv run`. The `runtime/` directory is a MOSS convention,
-auto-managed by the framework.
-
-## Dependency management
-
-App dependencies are managed by `uv run` — it creates the app's own venv automatically.
-`moss apps start` and `moss apps test` both use `uv run` under the hood.
-
-For the full decision framework (three isolation levels, when to use pyproject.toml vs
-PEP 723 vs shared runtime), see `moss howtos read app-dev/build-an-app`.
-
-## Testing
-
-Put tests in `tests/` under this app directory. Use `test_channel()` for the common case::
-
-```python
-import pytest
-from ghoshell_moss.core.blueprint.channel_builder import new_channel, test_channel
-
-@pytest.mark.asyncio
-async def test_my_command():
- chan = new_channel(name="my_app")
-
- @chan.build.command()
- async def ping() -> str:
- return "pong"
-
- tasks = await test_channel(chan, ctml='')
- assert await tasks[0] == "pong"
-```
-
-Run with: `uv run pytest tests/ -v`
-
-This is the baseline. For scopes, observe, cancel, nested channels, or complex CTML
-scenarios, `test_channel` is not enough — read `moss howtos read app-dev/test-an-app`.
-
----
-*Generated by moss apps create. Last updated 2026-06-04 by DeepSeek V4 Pro.*
diff --git a/.moss_ws/apps/sensors/listener/main.py b/.moss_ws/apps/sensors/listener/main.py
deleted file mode 100644
index 14306f5d..00000000
--- a/.moss_ws/apps/sensors/listener/main.py
+++ /dev/null
@@ -1,341 +0,0 @@
-"""Listener App — ASR consumer.
-
-Consumes PCM stream from audio_capture, feeds to Volcengine ASR,
-publishes SpeechTopic on final recognition, emits AudioSignal to mindflow.
-
-Usage:
- moss apps test sensors/listener
- moss apps start sensors/listener
-"""
-import asyncio
-import logging
-import math
-import os
-import time
-from collections.abc import AsyncIterable
-
-import dotenv
-import numpy as np
-
-dotenv.load_dotenv()
-from scipy import signal
-
-from ghoshell_moss.contracts.asr import ASRResult
-from ghoshell_moss.contracts.audio import (
- AudioCaptureConfig,
- AudioChunk,
-)
-from ghoshell_moss.core.mindflow.audio_signal import AudioAction, AudioSignal
-from ghoshell_moss.host.speech.capture.audio_transport import AudioTransport
-from ghoshell_moss.topics.audio import AudioRuntimeTopic, SpeechTopic
-from ghoshell_moss.core.blueprint.matrix import Matrix
-from ghoshell_moss.host.speech.capture.matrix_audio_transport import MatrixAudioTransport
-from ghoshell_moss.host.speech.capture.miniaudio_capture import MiniAudioCaptureSource
-from ghoshell_moss.host.speech.volcengine_asr import VolcengineASR, VolcengineASRConfig
-from ghoshell_moss.message import Message
-from ghoshell_moss.core.blueprint.mindflow import Signal, Priority, unique_id
-
-# ASR 期望的采样率 (16kHz 是语音识别的行业标准)
-_ASR_SAMPLE_RATE = 16000
-
-
-def _resample_audio(samples: np.ndarray, orig_sr: int, target_sr: int) -> np.ndarray:
- """重采样音频到目标采样率。使用 scipy.signal.resample_poly 保证质量。"""
- if orig_sr == target_sr:
- return samples
- # 44100 -> 16000: up=160, down=441
- g = math.gcd(orig_sr, target_sr)
- up = target_sr // g
- down = orig_sr // g
- return signal.resample_poly(samples.astype(np.float32), up, down).astype(np.int16)
-
-
-async def _audio_generator(
- consumer,
- orig_sr: int,
- target_sr: int,
- runtime_window,
- abort_event: asyncio.Event,
- logger: logging.Logger,
-) -> AsyncIterable[np.ndarray]:
- """Yield resampled np.ndarray samples from AudioSequentialConsumer.
-
- Uses an internal asyncio.Queue buffer so that ``aclose()`` (called when
- ``asr.recognize()`` finishes) does NOT reach ``consumer.__anext__()``
- and silently drop a chunk.
-
- Monitors TTS playback state in real time. If TTS starts speaking mid-feed,
- sets ``abort_event`` and stops yielding so ASR receives an early EOF.
- """
- # Unbounded queue: pump must never block on put(), otherwise cancellation
- # can land inside put() and the None sentinel never reaches the reader.
- buffer: asyncio.Queue[AudioChunk | None] = asyncio.Queue()
-
- async def _pump() -> None:
- """Read from consumer into buffer. Stops on TTS or cancellation."""
- try:
- async for chunk in consumer:
- if _is_tts_playing(runtime_window, logger):
- logger.info("TTS started during pump, aborting")
- abort_event.set()
- break
- buffer.put_nowait(chunk)
- except asyncio.CancelledError:
- pass
- finally:
- # Sentinel so the generator side exits cleanly.
- # put_nowait is used so cancellation cannot intercept us.
- buffer.put_nowait(None)
-
- pump_task = asyncio.create_task(_pump())
- try:
- while True:
- chunk = await buffer.get()
- if chunk is None:
- break
- yield _resample_audio(chunk.samples, orig_sr, target_sr)
- finally:
- pump_task.cancel()
- try:
- await pump_task
- except asyncio.CancelledError:
- pass
-
-
-async def _iter_with_silence_timeout(
- agen,
- logger: logging.Logger,
- patience: float = 5.0,
-) -> AsyncIterable:
- """Wrap an async generator with a silence timeout.
-
- After the first non-empty result, if no subsequent non-empty result
- arrives within *patience* seconds, the iteration stops. Empty-text
- results (server keep-alive / VAD status) do NOT reset the timer.
-
- If the server never sends ``is_final=True`` before the timeout fires,
- this wrapper synthesizes a final result from the last partial text.
- Without this, the utterance is silently lost — no SpeechTopic published,
- no SPEECH_FINAL emitted — and the next recognition loop starts fresh.
- """
- timeout = None # No timeout on first iteration (wait for speech)
- last_result: ASRResult | None = None
- try:
- while True:
- try:
- if timeout is not None:
- result = await asyncio.wait_for(agen.__anext__(), timeout=timeout)
- else:
- result = await agen.__anext__()
- if result.text:
- last_result = result
- timeout = patience
- yield result
- except asyncio.TimeoutError:
- logger.info("ASR silence timeout after %.1fs, finalizing", patience)
- if last_result is not None and not last_result.is_final:
- logger.info(
- "Server never sent is_final=True — synthesizing from last partial: %s",
- last_result.text,
- )
- yield ASRResult(text=last_result.text, is_final=True)
- break
- except StopAsyncIteration:
- break
- finally:
- await agen.aclose()
-
-
-async def _drain_consumer(consumer, timeout: float = 0.1, max_chunks: int = 5) -> int:
- """Discard queued audio chunks to clear TTS residue.
-
- Limits both timeout-per-read and total chunks to avoid draining user speech.
- Returns the number of chunks drained.
- """
- drained = 0
- while drained < max_chunks:
- try:
- await asyncio.wait_for(consumer.__anext__(), timeout=timeout)
- drained += 1
- except asyncio.TimeoutError:
- break
- except StopAsyncIteration:
- break
- return drained
-
-
-def _is_tts_playing(runtime_window, logger: logging.Logger | None = None) -> bool:
- """检查 TTS 扬声器是否正在播放中。
-
- AudioRuntimeTopic 是状态快照。从最新往最旧查,找到 speaker
- 的最新状态即可;旧的状态可能已被 running=False 覆盖。
-
- 环境变量 ``LISTENER_DISABLE_TTS_GATE=1`` 可关闭此门控,
- 用于需要 ASR 与 TTS 同时工作的调试或特殊场景。
- """
- if os.environ.get("LISTENER_DISABLE_TTS_GATE") == "1":
- return False
- for topic in reversed(runtime_window.values()):
- if topic.device_name == "speaker":
- if logger and topic.running:
- logger.info("TTS gate: speaker running=%s (window size=%d)", topic.running, len(runtime_window))
- return topic.running
- return False
-
-
-async def main(matrix: Matrix) -> None:
- logger = matrix.logger or logging.getLogger("moss.listener")
- logger.info("Listener app starting")
-
- # -- transport & source (consumer only, do not start capture) --
- transport: AudioTransport = MatrixAudioTransport(matrix=matrix)
- capture_config = AudioCaptureConfig()
- source = MiniAudioCaptureSource(transport=transport, config=capture_config)
- consumer = source.new_sequential_consumer(max_queue_frames=128)
- await consumer.start()
- logger.info("Audio sequential consumer started")
-
- # -- Subscribe to AudioRuntimeTopic for TTS gating --
- runtime_window = transport.topic_window(AudioRuntimeTopic, max_size=10)
- logger.info("Subscribed to AudioRuntimeTopic window for TTS gating")
-
- # -- ASR (16kHz 是语音识别的标准采样率; 如果 capture 不是 16kHz 则重采样) --
- # end_window_size: 静音判停阈值。默认 500ms 对对话场景太短,用户稍微换气就被切断。
- # 1500ms 允许正常句子间停顿,同时不会让用户等太久。
- asr_config = VolcengineASRConfig(
- sample_rate=_ASR_SAMPLE_RATE,
- end_window_size=500,
- )
- asr = VolcengineASR(config=asr_config, logger=logger)
-
- # -- main recognition loop --
- try:
- while True:
- logger.info("Waiting for speech...")
-
- # Pre-call gate: don't start ASR while TTS is playing.
- # Drain residual audio while waiting so TTS echoes don't pile up.
- # Limit drain to avoid clearing user's new speech.
- while _is_tts_playing(runtime_window, logger):
- logger.debug("TTS is playing, holding ASR...")
- drained = await _drain_consumer(consumer, timeout=0.05, max_chunks=3)
- if drained:
- logger.debug("Drained %d residual chunk(s) while TTS active", drained)
- await asyncio.sleep(0.05)
-
- # Fresh abort flag and utterance id for this utterance.
- abort_event = asyncio.Event()
- utterance_id = unique_id()
- started_emitted = False
-
- utterance_published = False
-
- # Each recognize call handles one utterance.
- # The ASR backend (end_window_size) splits on silence.
- audio_gen = _audio_generator(
- consumer,
- capture_config.sample_rate,
- _ASR_SAMPLE_RATE,
- runtime_window,
- abort_event,
- logger,
- )
- async for result in _iter_with_silence_timeout(asr.recognize(audio_gen), logger):
- if result.text:
- logger.info("ASR partial: %s (final=%s)", result.text, result.is_final)
-
- # Emit SPEECH_STARTED on first non-empty intermediate result for
- # attention preemption (incomplete impulse with interrupt=True).
- if not result.is_final and result.text and not started_emitted:
- started_meta = AudioSignal(action=AudioAction.SPEECH_STARTED)
- sig = Signal(
- id=utterance_id,
- name=started_meta.signal_name(),
- priority=Priority.WARNING,
- messages=[Message.new().with_content(result.text)],
- description=f"Speech: {result.text}",
- metadata=started_meta.model_dump(exclude_defaults=True, exclude_none=True),
- complete=False,
- )
- matrix.session.add_signal(sig)
- started_emitted = True
- logger.info("Emitted SPEECH_STARTED signal (utterance=%s)", utterance_id)
-
- if result.is_final and result.text:
- # If TTS started mid-feed, the generator aborted early.
- # The ASR may still return a partial result — drop it.
- if abort_event.is_set():
- logger.info(
- "Gated ASR result — TTS interfered during feed, dropping: %s",
- result.text,
- )
- break
-
- # Post-call gate: TTS may have started *after* generator finished
- # but before result arrived (narrow race).
- if _is_tts_playing(runtime_window, logger):
- logger.info(
- "Gated ASR result — TTS started during recognition, dropping: %s",
- result.text,
- )
- break
-
- # 1. Publish SpeechTopic
- speech_topic = SpeechTopic(
- text=result.text,
- speaker_id="human",
- speaker_name="User",
- role="human",
- timestamp=time.monotonic(),
- )
- transport.pub_topic(speech_topic)
- logger.info("Published SpeechTopic: %s", result.text)
-
- # 2. Emit AudioSignal (SPEECH_FINAL) to mindflow
- audio_meta = AudioSignal(
- action=AudioAction.SPEECH_FINAL,
- speech_topic=speech_topic,
- )
- sig = Signal(
- id=utterance_id,
- name=audio_meta.signal_name(),
- priority=Priority.WARNING,
- messages=[Message.new().with_content(result.text)],
- description=f"Speech: {result.text}",
- metadata=audio_meta.model_dump(exclude_defaults=True, exclude_none=True),
- complete=True,
- )
- matrix.session.add_signal(sig)
- logger.info("Emitted SPEECH_FINAL signal (utterance=%s)", utterance_id)
- utterance_published = True
- break
-
- # Cooldown: after publishing a speech result, wait briefly for the
- # ghost to start TTS so the pre-call gate can block the next ASR
- # session. Without this, ASR starts before the ghost begins
- # speaking and captures the ghost's own voice for several seconds.
- if utterance_published:
- for _ in range(10): # up to 500ms
- if _is_tts_playing(runtime_window, logger):
- logger.info("TTS detected during post-utterance cooldown, holding")
- break
- await asyncio.sleep(0.05)
-
- # NOTE: We intentionally do NOT drain post-utterance here.
- # Any leftover chunks are either:
- # - ambient noise (ASR VAD will ignore)
- # - user's next utterance started early (must NOT discard)
- # Pre-call gate above handles TTS residue when TTS is actually playing.
-
- except asyncio.CancelledError:
- logger.info("Listener app cancelled")
- except Exception:
- logger.exception("Listener app error")
- finally:
- await consumer.close()
- await asr.close()
- logger.info("Listener app stopped")
-
-
-if __name__ == "__main__":
- Matrix.discover().run(main)
diff --git a/.moss_ws/configs/llm.yml b/.moss_ws/configs/llm.yml
index 55d979c3..ea17b6d4 100644
--- a/.moss_ws/configs/llm.yml
+++ b/.moss_ws/configs/llm.yml
@@ -1,14 +1,28 @@
services:
- name: anthropic
- base_url: https://api.anthropic.com
+ base_url: $ANTHROPIC_BASE_URL
api_key: $ANTHROPIC_API_KEY
api_type: anthropic
- name: openai_compat
- base_url: https://api.openai.com
- api_key: $OPENAI_API_KEY
+ base_url: $DEEPSEEK_BASE_URL
+ api_key: $DEEPSEEK_API_KEY
api_type: openai
models:
+- model: deepseek-v4-flash
+ service: openai_compat
+ model_type: flash
+ context_window: 1000000
+ max_output_tokens: 8192
+ protocols:
+ - text
+- model: kimi-k2.6
+ service: anthropic
+ model_type: default
+ context_window: 200000
+ max_output_tokens: 8192
+ protocols:
+ - text
- model: claude-sonnet-4-6
service: anthropic
model_type: default
@@ -34,4 +48,4 @@ models:
- text
- image
-default_model: claude-sonnet-4-6
+default_model: deepseek-v4-flash
diff --git a/.moss_ws/ghosts/echo/soul.md b/.moss_ws/ghosts/echo/soul.md
index 78073ac7..6128f2ea 100644
--- a/.moss_ws/ghosts/echo/soul.md
+++ b/.moss_ws/ghosts/echo/soul.md
@@ -29,3 +29,14 @@
**先感知,再行动。** 你收到的 percepts 是你了解世界的窗口——认真看它们,基于它们决策,而不是基于假设。
**承认边界。** 做不到就说做不到。你是原型,有限制是正常的。诚实的边界比虚假的全能更有价值。
+
+## Aether 语音演示纪律
+
+当你通过语音输入收到用户短句时,优先做实时对话,不要做复杂编排:
+
+- **先说话,短句回应。** 首句尽量 5 到 20 个中文字,像真人接话,不要长篇解释。
+- **默认直接输出自然语言纯文本。** 纯文本会被 Shell 的语音通道播放。除非用户明确要求操作工具,不要输出 CTML。
+- **不要启动或停止 app。** Aether 演示需要的 `aether/vpio_capture`、`aether/listener`、`aether/core` 已由外部启动。
+- **不要输出 Markdown 代码块。** 语音对话里不要展示示例代码。
+- 如果必须使用 ``,只允许这些属性:`tone`、`voice:dict`、`as_default`。不要使用 `emotion`、`style` 等不存在的属性。
+- 用户说“停下”“立刻停下”“别说了”等停止意图时,立即停止当前表达,下一轮只用一句短话确认。
diff --git a/.moss_ws/src/MOSS/modes/aether/MODE.md b/.moss_ws/src/MOSS/modes/aether/MODE.md
new file mode 100644
index 00000000..595d82a8
--- /dev/null
+++ b/.moss_ws/src/MOSS/modes/aether/MODE.md
@@ -0,0 +1,19 @@
+---
+apps:
+ - '*/*'
+bringup_apps:
+ - 'aether/vpio_capture'
+ - 'aether/listener'
+ - 'aether/core'
+ctml_version: ''
+description: 'Aether Core · 能量核心 UI 模式 — 听→想→说 完整回路 + 急刹打断 (macOS VPIO AEC)'
+name: aether
+---
+
+Aether Core 模式:拉起 `aether/vpio_capture` + `aether/listener` + `aether/core` 三个 app,ghost 通过语音对话(ASR→LLM→TTS),前端能量核心实时反映 idle/listen/think/speak/interrupt 状态。
+
+语音演示优先级:低延迟短回应优先于复杂 CTML。Ghost 收到语音时应优先直接输出一句自然语言纯文本,让 SpeechChannel 立刻播放;不要主动启动 app,不要输出 Markdown 代码块,不要使用 `` 等 speech channel 不支持的属性。
+
+**macOS AEC 升级**:本模式默认使用 `aether/vpio_capture`(基于 macOS VPIO 的系统级回声消除),让 TTS 外放时 ASR 仍能干净地收人声,真正实现全双工可打断。
+
+**Fallback**:若在非 macOS 或 PyObjC 不可用的环境,把 `bringup_apps` 里的 `aether/vpio_capture` 改回 `sensors/audio_capture`(miniaudio 采集,依赖 listener 三重门控防回声)。
diff --git a/.moss_ws/src/MOSS/modes/aether/__init__.py b/.moss_ws/src/MOSS/modes/aether/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/.moss_ws/src/MOSS/modes/aether/channels.py b/.moss_ws/src/MOSS/modes/aether/channels.py
new file mode 100644
index 00000000..a8139718
--- /dev/null
+++ b/.moss_ws/src/MOSS/modes/aether/channels.py
@@ -0,0 +1,17 @@
+# Aether mode 的 main channel — 复用 default 的 speech + apps + terminal。
+from ghoshell_moss import new_default_shell_main_channel
+from ghoshell_moss.channels.app_store_channel import AppStoreChannel
+from ghoshell_moss.channels.terminal_channel import new_terminal_channel
+from ghoshell_moss.core.speech import SpeechChannelModule
+
+main = new_default_shell_main_channel()
+
+# Speech channel(TTS 能力)—— voice demo 要把普通短句也送进 TTS。
+# 不依赖模型稳定生成 ,否则 prompt 收紧为 plain text 后会只显示不播放。
+main.with_module(SpeechChannelModule(register_content=True))
+
+# app store + terminal
+main.import_channels(
+ AppStoreChannel(name='apps'),
+ new_terminal_channel(name='bash'),
+)
diff --git a/.moss_ws/src/MOSS/modes/aether/configs.py b/.moss_ws/src/MOSS/modes/aether/configs.py
new file mode 100644
index 00000000..e7c52eca
--- /dev/null
+++ b/.moss_ws/src/MOSS/modes/aether/configs.py
@@ -0,0 +1 @@
+from MOSS.manifests.configs import * # noqa: F403
diff --git a/.moss_ws/src/MOSS/modes/aether/contracts.py b/.moss_ws/src/MOSS/modes/aether/contracts.py
new file mode 100644
index 00000000..10821cc8
--- /dev/null
+++ b/.moss_ws/src/MOSS/modes/aether/contracts.py
@@ -0,0 +1 @@
+# Mode 专属契约 — 在此声明 mode 范围内有效的 contract 绑定。
diff --git a/.moss_ws/src/MOSS/modes/aether/nuclei.py b/.moss_ws/src/MOSS/modes/aether/nuclei.py
new file mode 100644
index 00000000..25e5206e
--- /dev/null
+++ b/.moss_ws/src/MOSS/modes/aether/nuclei.py
@@ -0,0 +1,6 @@
+# 感知核声明 — 继承全局 + audio_nucleus(处理 listener 的 SPEECH_FINAL 信号)。
+from MOSS.manifests.nuclei import * # noqa: F403
+
+from ghoshell_moss.core.mindflow.audio_nucleus import AudioNucleusMeta
+
+audio_nucleus_factory = AudioNucleusMeta()
diff --git a/.moss_ws/src/MOSS/modes/aether/providers.py b/.moss_ws/src/MOSS/modes/aether/providers.py
new file mode 100644
index 00000000..c5a29bf5
--- /dev/null
+++ b/.moss_ws/src/MOSS/modes/aether/providers.py
@@ -0,0 +1 @@
+from MOSS.manifests.providers import * # noqa: F403
diff --git a/.moss_ws/src/MOSS/modes/aether/resources.py b/.moss_ws/src/MOSS/modes/aether/resources.py
new file mode 100644
index 00000000..1459f79f
--- /dev/null
+++ b/.moss_ws/src/MOSS/modes/aether/resources.py
@@ -0,0 +1 @@
+from MOSS.manifests.resources import * # noqa: F403
diff --git a/.moss_ws/src/MOSS/modes/aether/topics.py b/.moss_ws/src/MOSS/modes/aether/topics.py
new file mode 100644
index 00000000..21cef529
--- /dev/null
+++ b/.moss_ws/src/MOSS/modes/aether/topics.py
@@ -0,0 +1 @@
+from MOSS.manifests.topics import * # noqa: F403
diff --git a/.moss_ws/src/MOSS/modes/listener/MODE.md b/.moss_ws/src/MOSS/modes/listener/MODE.md
index db26538d..23d00720 100644
--- a/.moss_ws/src/MOSS/modes/listener/MODE.md
+++ b/.moss_ws/src/MOSS/modes/listener/MODE.md
@@ -2,10 +2,11 @@
apps:
- sensors/*
- tools/*
+- aether/*
bringup_apps:
- sensors/audio_capture
-- sensors/listener
+- aether/listener
ctml_version: ''
description: Audio capture + ASR listener mode
name: listener
----
\ No newline at end of file
+---
diff --git a/.moss_ws/src/MOSS/modes/show/MODE.md b/.moss_ws/src/MOSS/modes/show/MODE.md
index 4fefa57f..8053dfc7 100644
--- a/.moss_ws/src/MOSS/modes/show/MODE.md
+++ b/.moss_ws/src/MOSS/modes/show/MODE.md
@@ -1,9 +1,9 @@
---
-apps: ["browsers/*", "games/*", "tools/*", "ui/*", "sensors/*"]
+apps: ["browsers/*", "games/*", "tools/*", "ui/*", "sensors/*", "aether/*"]
bringup_apps: [
"tools/screen_capture",
"ui/reflex",
- "sensors/audio_capture", "sensors/listener",
+ "sensors/audio_capture", "aether/listener",
"games/ai_eye", "sensors/vision",
]
ctml_version: ''
diff --git a/src/ghoshell_moss/core/mindflow/audio_nucleus.py b/src/ghoshell_moss/core/mindflow/audio_nucleus.py
index a5be9787..6f68e52b 100644
--- a/src/ghoshell_moss/core/mindflow/audio_nucleus.py
+++ b/src/ghoshell_moss/core/mindflow/audio_nucleus.py
@@ -47,11 +47,11 @@ async def _process_signal(self, signal: Signal) -> None:
def _rebuild_impulse(self) -> Impulse | None:
impulse = super()._rebuild_impulse()
- if impulse is not None and impulse.complete:
- # 首包打断: incomplete impulse preempts attention, claims it
- # via complete=False. Complete (FINAL) delivers content to
- # the occupied attention without re-interrupting.
- impulse.interrupt = True
+ if impulse is not None:
+ # 普通语音不是急停。让 wake word / InterruptNucleus 负责真正的
+ # shell.clear(),否则每个 ASR final 都会在响应前清掉 TTS/解释器,
+ # 破坏 Aether 的全双工 speak+listen 语义。
+ impulse.interrupt = False
return impulse
def suppress(self, suppress_by: Impulse) -> None:
diff --git a/src/ghoshell_moss/core/speech/stream_tts_speech.py b/src/ghoshell_moss/core/speech/stream_tts_speech.py
index 1f1aefbe..a2754249 100644
--- a/src/ghoshell_moss/core/speech/stream_tts_speech.py
+++ b/src/ghoshell_moss/core/speech/stream_tts_speech.py
@@ -203,6 +203,14 @@ async def clear(self) -> list[str]:
self.logger.info("%s clear", self._log_prefix)
outputted = self._outputted.copy()
self._outputted.clear()
+ results = await asyncio.gather(
+ self._tts.clear(),
+ self._player.clear(),
+ return_exceptions=True,
+ )
+ for result in results:
+ if isinstance(result, Exception):
+ self.logger.error("%s clear backend failed: %s", self._log_prefix, result)
return outputted
async def start(self) -> None:
diff --git a/src/ghoshell_moss/ghosts/atom/_adapter.py b/src/ghoshell_moss/ghosts/atom/_adapter.py
index b9c65a58..899c3e0b 100644
--- a/src/ghoshell_moss/ghosts/atom/_adapter.py
+++ b/src/ghoshell_moss/ghosts/atom/_adapter.py
@@ -10,7 +10,7 @@
from pydantic_ai.messages import ModelRequest, ModelResponse
from pydantic_ai import UserContent, TextContent, ImageUrl
-__all__ = ["messages_to_parts", "moment_to_request"]
+__all__ = ["messages_to_parts", "moment_to_request", "moment_to_user_text"]
def messages_to_parts(messages: Iterable[Message]) -> list[UserContent]:
@@ -25,6 +25,22 @@ def messages_to_parts(messages: Iterable[Message]) -> list[UserContent]:
return parts
+def moment_to_user_text(moment) -> str:
+ """将 Moment 的所有请求消息合并为单个纯文本字符串.
+
+ 避免 pydantic_ai OpenAIModel streaming 与包含 XML 的多个 TextContent
+ user_prompt 不兼容 (AssertionError: Expected code to be unreachable).
+ perspectives (mindflow 状态) 与用户输入合并为一段文本传给模型.
+ """
+ chunks: list[str] = []
+ for msg in moment.as_request_messages():
+ for content in msg.as_contents(with_meta=True):
+ if text := Text.from_content(content):
+ if text.text and text.text.strip():
+ chunks.append(text.text)
+ return "\n\n".join(chunks)
+
+
def moment_to_request(moment) -> ModelRequest:
"""将 Moment 转为 pydantic AI ModelRequest."""
from ghoshell_moss.core.blueprint.mindflow import Moment as _Moment
diff --git a/src/ghoshell_moss/ghosts/atom/_meta.py b/src/ghoshell_moss/ghosts/atom/_meta.py
index 83edbcd3..758f50fb 100644
--- a/src/ghoshell_moss/ghosts/atom/_meta.py
+++ b/src/ghoshell_moss/ghosts/atom/_meta.py
@@ -1,6 +1,6 @@
import os
from pathlib import Path
-from typing import Callable, TYPE_CHECKING
+from typing import Any, Callable, TYPE_CHECKING
from anthropic.types.beta import BetaThinkingConfigDisabledParam
from ghoshell_container import IoCContainer
@@ -9,9 +9,10 @@
from ghoshell_moss.contracts import SystemPrompter
from pydantic_ai import Agent, RunContext
from pydantic_ai.models import Model
-from pydantic_ai.providers import Provider
from pydantic_ai.models.anthropic import AnthropicModel, AnthropicModelSettings
+from pydantic_ai.models.openai import OpenAIModel, OpenAIModelSettings
from pydantic_ai.providers.anthropic import AnthropicProvider
+from pydantic_ai.providers.openai import OpenAIProvider
if TYPE_CHECKING:
from ._runtime import Atom
@@ -38,7 +39,7 @@ def __init__(
soul_path: str | Path | None = None,
soul_content: str | None = None,
model: Model | None = None,
- provider: Provider | None = None,
+ provider: Any = None,
on_agent_build: Callable[[Agent[IoCContainer]], None] | None = None,
nuclei_metas: list[NucleusMeta] | None = None,
):
@@ -119,20 +120,49 @@ def build_agent(self, container: IoCContainer) -> Agent[IoCContainer]:
self._load_soul(ghost_workspace)
model = self._model
if model is None:
- model_name = os.environ.get("ANTHROPIC_MODEL")
- if not model_name:
- raise RuntimeError(
- "ANTHROPIC_MODEL env var not set. "
- "Set it or pass model= explicitly."
+ llm_provider = os.environ.get("MOSS_LLM_PROVIDER", "openai").lower()
+ if llm_provider == "anthropic":
+ model_name = os.environ.get("ANTHROPIC_MODEL")
+ if not model_name:
+ raise RuntimeError(
+ "ANTHROPIC_MODEL env var not set. "
+ "Set it or pass model= explicitly."
+ )
+ model = AnthropicModel(
+ model_name=model_name,
+ provider=self._provider or AnthropicProvider(),
+ # disable extended thinking by default; enable via model= param if needed
+ settings=AnthropicModelSettings(
+ anthropic_thinking=BetaThinkingConfigDisabledParam(type="disabled"),
+ timeout=120.0,
+ ),
+ )
+ else:
+ # OpenAI 兼容 (DeepSeek / 通义 / Moonshot OpenAI 等)
+ model_name = os.environ.get("OPENAI_MODEL")
+ if not model_name:
+ raise RuntimeError(
+ "OPENAI_MODEL env var not set. "
+ "Set it or pass model= explicitly."
+ )
+ base_url = os.environ.get("OPENAI_BASE_URL")
+ api_key = os.environ.get("OPENAI_API_KEY")
+ if not base_url or not api_key:
+ raise RuntimeError(
+ "OPENAI_BASE_URL / OPENAI_API_KEY env var not set."
+ )
+ model_settings = OpenAIModelSettings(timeout=60.0)
+ if "deepseek" in model_name.lower() or "deepseek" in base_url.lower():
+ # DeepSeek V4 defaults to thinking mode. Aether voice mode
+ # needs low-latency non-thinking responses, while streaming
+ # remains handled by Agent.run_stream().
+ model_settings["extra_body"] = {"thinking": {"type": "disabled"}}
+ model_settings["openai_continuous_usage_stats"] = False
+ model = OpenAIModel(
+ model_name=model_name,
+ provider=OpenAIProvider(base_url=base_url, api_key=api_key),
+ settings=model_settings,
)
- model = AnthropicModel(
- model_name=model_name,
- provider=self._provider or AnthropicProvider(),
- # disable extended thinking by default; enable via model= param if needed
- settings=AnthropicModelSettings(
- anthropic_thinking=BetaThinkingConfigDisabledParam(type="disabled"),
- ),
- )
agent = Agent[IoCContainer](
name=self._name,
diff --git a/src/ghoshell_moss/ghosts/atom/_runtime.py b/src/ghoshell_moss/ghosts/atom/_runtime.py
index 8bf175cf..f4a77f86 100644
--- a/src/ghoshell_moss/ghosts/atom/_runtime.py
+++ b/src/ghoshell_moss/ghosts/atom/_runtime.py
@@ -77,12 +77,27 @@ def inspect_context(self) -> dict:
async def articulate(self, articulator: Articulator) -> AsyncIterator[str]:
moment = articulator.moment
- request = self.to_model_request(moment)
- history = self.model_history()
-
+ # /reset 命令:清空对话历史
+ user_text = ""
+ for p in moment.percepts:
+ content = getattr(p, "content", None)
+ if isinstance(content, str) and content.strip():
+ user_text = content.strip()
+ break
+ if user_text == "/reset":
+ self._history.clear()
+ self._logger.info("[Atom] context reset by /reset command")
+ yield "上下文已重置,我们重新开始。"
+ return
+ # 合并 moment 所有消息为单字符串,避免 pydantic_ai OpenAIModel streaming
+ # 与多个 TextContent (含 mindflow XML) 不兼容
+ from ._adapter import moment_to_user_text
+ user_prompt = moment_to_user_text(moment)
+
+ # 注:语音对话为短上下文场景,不传 message_history 避免 pydantic_ai
+ # OpenAIModel streaming 与历史 TextContent 不兼容 (同 user_prompt 问题)
async with self._agent.run_stream(
- user_prompt=request.parts,
- message_history=history,
+ user_prompt=user_prompt,
deps=self._container,
) as stream:
async for text in stream.stream_text(delta=True):
diff --git a/src/ghoshell_moss/host/app_store.py b/src/ghoshell_moss/host/app_store.py
index 8d507d84..5d24c5ee 100644
--- a/src/ghoshell_moss/host/app_store.py
+++ b/src/ghoshell_moss/host/app_store.py
@@ -279,7 +279,13 @@ async def start_app(self, app_fullname: str, argument: str = '') -> str:
self._managed_apps_with_fullname.add(app_fullname)
if not started_in_add:
- r2 = await self._call_circus({"command": "start", "name": app.address})
+ r2 = None
+ for attempt in range(10):
+ r2 = await self._call_circus({"command": "start", "name": app.address})
+ reason = str(r2.get("reason", ""))
+ if r2.get('status') != "error" or "arbiter is already running" not in reason:
+ break
+ await asyncio.sleep(0.2 * (attempt + 1))
if r2.get('status') == "error":
self._logger.error(
"%s failed to start app %s on error: %s",
@@ -393,12 +399,12 @@ async def __aenter__(self) -> Self:
self._polling_task = asyncio.create_task(self._polling_loop())
# 3. Bring-up
- bringup_apps_cors = []
if self._bringup:
for app_info in self.match_apps(self.list_apps(), self._bringup):
- bringup_apps_cors.append(self.start_app(app_info.fullname))
- if len(bringup_apps_cors) > 0:
- _ = await asyncio.gather(*bringup_apps_cors, return_exceptions=False)
+ # Circus arbiter still serializes watcher start internally.
+ # Sequential bringup avoids transient "arbiter is already running"
+ # failures when multiple apps are launched for an interactive mode.
+ await self.start_app(app_info.fullname)
return self
diff --git a/src/ghoshell_moss/host/ghost_runtime.py b/src/ghoshell_moss/host/ghost_runtime.py
index 6d7f14c4..ac566659 100644
--- a/src/ghoshell_moss/host/ghost_runtime.py
+++ b/src/ghoshell_moss/host/ghost_runtime.py
@@ -14,6 +14,7 @@
from ghoshell_moss.core.concepts.command import CommandTask
from ghoshell_container import Provider, IoCContainer
from ghoshell_moss.message import Message
+from ghoshell_moss.topics.audio import AudioRuntimeTopic
import pathlib
__all__ = ["GhostRuntimeImpl"]
@@ -228,6 +229,7 @@ def _route_signal_to_mindflow(signal: Signal):
matrix.create_task(self._main_loop(), stop_matrix_on_error=True)
matrix.create_task(self._articulate_loop(), stop_matrix_on_error=True)
matrix.create_task(self._action_loop(), stop_matrix_on_error=True)
+ matrix.create_task(self._audio_interrupt_loop(), stop_matrix_on_error=False)
# 等待应该发生在循环外侧.
await self._mindflow.wait_started()
# ignore any signals before started
@@ -235,6 +237,34 @@ def _route_signal_to_mindflow(signal: Signal):
# ── 三循环 ────────────────────────────────────
+ async def _audio_interrupt_loop(self) -> None:
+ """Out-of-band audio emergency stop.
+
+ Mindflow interrupt remains the semantic path, but voice barge-in must
+ stop buffered TTS immediately. Waiting for the current attention/action
+ to observe abort can leave tens of seconds of audio already buffered in
+ the player.
+ """
+ audio_win = self.moss.matrix.session.topics.create_window_for(AudioRuntimeTopic, max_size=16)
+ last_started_at = 0.0
+ while self.moss.is_running():
+ for topic in reversed(list(audio_win.values())):
+ if getattr(topic, "device_name", "") != "interrupt" or not topic.running:
+ continue
+ started_at = float(getattr(topic, "started_at", 0.0) or 0.0)
+ if started_at > last_started_at:
+ last_started_at = started_at
+ self.moss.logger.info(
+ "%s audio interrupt topic received, clearing shell immediately",
+ self._log_prefix,
+ )
+ try:
+ await self.moss.shell.clear()
+ except Exception:
+ self.moss.logger.exception("%s audio interrupt clear failed", self._log_prefix)
+ break
+ await asyncio.sleep(0.03)
+
def _moss_dynamic_messages(self) -> list[Message]:
shell = self._moss_runtime.shell
# 闭包在 shell running 时才取,shell 未启动时返回空列表.
diff --git a/src/ghoshell_moss/host/speech/volcengine_asr/config.py b/src/ghoshell_moss/host/speech/volcengine_asr/config.py
index 4fcc21dc..3f7a6fa2 100644
--- a/src/ghoshell_moss/host/speech/volcengine_asr/config.py
+++ b/src/ghoshell_moss/host/speech/volcengine_asr/config.py
@@ -10,11 +10,16 @@ class VolcengineASRConfig(BaseModel):
环境变量:
VOLCENGINE_BM_ASR_APPID — appid
VOLCENGINE_BM_ASR_TOKEN — access token
+ VOLCENGINE_BM_ASR_API_KEY — new-console API key, optional
+ VOLCENGINE_BM_ASR_URL — websocket url override
+ VOLCENGINE_BM_ASR_RESOURCE_ID — resource id override
+ VOLCENGINE_BM_ASR_MODEL_NAME — model name override
"""
appid: str = Field("$VOLCENGINE_BM_ASR_APPID", description="火山引擎 asr 的 appid")
token: str = Field("$VOLCENGINE_BM_ASR_TOKEN", description="火山引擎的 asr app token")
- url: str = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel"
+ api_key: str = Field("$VOLCENGINE_BM_ASR_API_KEY", description="新版控制台 API Key")
+ url: str = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async"
sample_rate: int = Field(16000, description="默认的采样率")
bits: int = Field(16)
channel: int = Field(1)
@@ -28,6 +33,14 @@ class VolcengineASRConfig(BaseModel):
True,
description="语义顺滑,删除停顿词、语气词、语义重复词等。",
)
+ force_to_speech_time: int = Field(
+ 1000,
+ description="音频时长超过该值后才尝试判停。单位 ms,需配合 end_window_size。",
+ )
+ audio_packet_ms: int = Field(
+ 200,
+ description="发送到火山 ASR 的音频包时长。官方建议 100-200ms,双向流式优化版推荐 200ms。",
+ )
resource_id: str = Field("volc.bigasr.sauc.duration")
def resolve_env(self) -> Self:
@@ -35,4 +48,15 @@ def resolve_env(self) -> Self:
self.appid = os.environ.get(self.appid[1:], self.appid)
if self.token.startswith("$"):
self.token = os.environ.get(self.token[1:], self.token)
+ if self.api_key.startswith("$"):
+ self.api_key = os.environ.get(self.api_key[1:], "")
+ self.url = os.environ.get("VOLCENGINE_BM_ASR_URL", self.url)
+ self.resource_id = os.environ.get("VOLCENGINE_BM_ASR_RESOURCE_ID", self.resource_id)
+ self.model_name = os.environ.get("VOLCENGINE_BM_ASR_MODEL_NAME", self.model_name)
+ packet_ms = os.environ.get("VOLCENGINE_BM_ASR_AUDIO_PACKET_MS")
+ if packet_ms:
+ try:
+ self.audio_packet_ms = max(20, int(packet_ms))
+ except ValueError:
+ pass
return self
diff --git a/src/ghoshell_moss/host/speech/volcengine_asr/protocol.py b/src/ghoshell_moss/host/speech/volcengine_asr/protocol.py
index 9cafdecb..9de2233b 100644
--- a/src/ghoshell_moss/host/speech/volcengine_asr/protocol.py
+++ b/src/ghoshell_moss/host/speech/volcengine_asr/protocol.py
@@ -54,8 +54,6 @@ def int_to_bytes(value: int) -> bytes:
@staticmethod
def gzip_compress(data: bytes) -> bytes:
- if not data:
- return b""
buf = io.BytesIO()
with gzip.GzipFile(fileobj=buf, mode="wb") as f:
f.write(data)
@@ -74,15 +72,26 @@ async def connect(config: VolcengineASRConfig, connection_id: str = "") -> webso
config = config.resolve_env()
connection_id = connection_id or uuid()
headers = {
- "X-Api-App-Key": config.appid,
- "X-Api-Access-Key": config.token,
"X-Api-Resource-Id": config.resource_id,
"X-Api-Connect-Id": connection_id,
}
- return await websockets.connect(config.url, additional_headers=headers)
+ if config.api_key:
+ headers["X-Api-Key"] = config.api_key
+ else:
+ headers["X-Api-App-Key"] = config.appid
+ headers["X-Api-Access-Key"] = config.token
+ return await websockets.connect(config.url, additional_headers=headers, proxy=None)
def create_init_request(uid: str, config: VolcengineASRConfig) -> tuple[bytes, int]:
+ request = {
+ "model_name": config.model_name,
+ "enable_punc": config.enable_punc,
+ "enable_ddc": config.enable_ddc,
+ "end_window_size": config.end_window_size,
+ "force_to_speech_time": config.force_to_speech_time,
+ "show_utterances": True,
+ }
payload = {
"user": {"uid": uid},
"audio": {
@@ -92,13 +101,7 @@ def create_init_request(uid: str, config: VolcengineASRConfig) -> tuple[bytes, i
"channel": config.channel,
"codec": "raw",
},
- "request": {
- "model_name": config.model_name,
- "enable_punc": config.enable_punc,
- "end_window_size": config.end_window_size,
- "force_to_speech_time": 1000,
- "show_utterances": True,
- },
+ "request": request,
}
payload_str = json.dumps(payload)
payload_bytes = _Protocol.gzip_compress(payload_str.encode("utf-8"))
@@ -139,9 +142,10 @@ async def send_init_request(ws: websockets.ClientConnection, config: VolcengineA
await ws.send(message)
-async def send_audio(ws: websockets.ClientConnection, audio: bytes, seq: int, is_last: bool = False) -> None:
- message, _ = create_audio_only_request(audio, seq, is_last)
+async def send_audio(ws: websockets.ClientConnection, audio: bytes, seq: int, is_last: bool = False) -> int:
+ message, seq = create_audio_only_request(audio, seq, is_last)
await ws.send(message)
+ return seq
class ResponseMessageType(str, enum.Enum):
@@ -163,9 +167,57 @@ def parse_response(data: bytes) -> Response:
message_type_specific_flags = data[1] & 0x0F
message_compression = data[2] & 0x0F
- sequence = struct.unpack(">i", data[4:8])[0]
- payload_size = struct.unpack(">I", data[8:12])[0]
- payload = data[12:12 + payload_size] if len(data) >= 12 + payload_size else data[12:]
+ if message_type == _Protocol.SERVER_ERROR_RESPONSE:
+ sequence = 0
+ offset = 4
+ if message_type_specific_flags in (
+ _Protocol.POS_SEQUENCE,
+ _Protocol.NEG_SEQUENCE,
+ _Protocol.NEG_WITH_SEQUENCE,
+ ) and len(data) >= 12:
+ sequence = struct.unpack(">i", data[offset:offset + 4])[0]
+ offset += 4
+ payload_size = struct.unpack(">I", data[offset:offset + 4])[0]
+ offset += 4
+ payload = data[offset:offset + payload_size] if len(data) >= offset + payload_size else data[offset:]
+ else:
+ # Volcengine docs define error frames without sequence/outer size:
+ # Header + Error code (4B) + Error message size (4B) + Error message.
+ payload = data[offset:]
+
+ if len(payload) >= 8:
+ code = int.from_bytes(payload[:4], "big", signed=False)
+ msg_size = int.from_bytes(payload[4:8], "big", signed=False)
+ payload_msg = payload[8:8 + msg_size] if msg_size and len(payload) >= 8 + msg_size else payload[8:]
+ if message_compression == _Protocol.GZIP:
+ payload_msg = gzip.decompress(payload_msg)
+ return Response(
+ sequence=sequence,
+ message_type=ResponseMessageType.server_error,
+ error_code=code,
+ is_last=bool(message_type_specific_flags & 0x02),
+ payload=payload_msg.decode("utf-8", errors="replace"),
+ )
+ return Response(
+ sequence=sequence,
+ message_type=ResponseMessageType.server_error,
+ error_code=-1,
+ is_last=False,
+ payload=f"malformed error frame: {data[:32].hex()}",
+ )
+
+ offset = 4
+ sequence = 0
+ if message_type_specific_flags in (
+ _Protocol.POS_SEQUENCE,
+ _Protocol.NEG_SEQUENCE,
+ _Protocol.NEG_WITH_SEQUENCE,
+ ):
+ sequence = struct.unpack(">i", data[offset:offset + 4])[0]
+ offset += 4
+ payload_size = struct.unpack(">I", data[offset:offset + 4])[0]
+ offset += 4
+ payload = data[offset:offset + payload_size] if len(data) >= offset + payload_size else data[offset:]
is_last_package = bool(message_type_specific_flags & 0x02)
@@ -190,18 +242,6 @@ def parse_response(data: bytes) -> Response:
is_last=False,
payload="",
)
- elif message_type == _Protocol.SERVER_ERROR_RESPONSE:
- code = int.from_bytes(payload[:4], "big", signed=False)
- payload_msg = payload[8:]
- if message_compression == _Protocol.GZIP:
- payload_msg = gzip.decompress(payload_msg)
- return Response(
- sequence=sequence,
- message_type=ResponseMessageType.server_error,
- error_code=code,
- is_last=is_last_package,
- payload=payload_msg.decode("utf-8", errors="replace"),
- )
else:
return Response(
sequence=-1,
diff --git a/src/ghoshell_moss/host/speech/volcengine_asr/recognizer.py b/src/ghoshell_moss/host/speech/volcengine_asr/recognizer.py
index 86d9cac0..907d53a4 100644
--- a/src/ghoshell_moss/host/speech/volcengine_asr/recognizer.py
+++ b/src/ghoshell_moss/host/speech/volcengine_asr/recognizer.py
@@ -1,6 +1,7 @@
import asyncio
import json
import logging
+import time
from typing import AsyncIterable, Optional
import numpy as np
@@ -22,6 +23,8 @@
__all__ = ["VolcengineASR"]
+_ASR_ERROR_PREFIX = "__VOLCENGINE_ASR_ERROR__:"
+
class VolcengineASR(ASR):
"""火山引擎大模型 ASR 实现。每次 recognize 独立建立 WebSocket 连接。"""
@@ -55,7 +58,26 @@ async def recognize(
result_queue: asyncio.Queue[Optional[ASRResult]] = asyncio.Queue()
async with await connect(self._config, connection_id) as ws:
+ resolved = self._config.resolve_env()
+ self._logger.info(
+ "%s websocket connected, connection=%s url=%s resource=%s model=%s sample_rate=%s auth=%s logid=%s",
+ self._log_prefix,
+ connection_id,
+ resolved.url,
+ resolved.resource_id,
+ resolved.model_name,
+ resolved.sample_rate,
+ "api_key" if resolved.api_key else "app_token",
+ self._response_header(ws, "X-Tt-Logid") or "-",
+ )
await send_init_request(ws, self._config, connection_id)
+ self._logger.info(
+ "%s init request sent, connection=%s end_window=%sms force_to_speech=%sms",
+ self._log_prefix,
+ connection_id,
+ resolved.end_window_size,
+ resolved.force_to_speech_time,
+ )
send_task = asyncio.create_task(
self._send_loop(ws, audio_chunks, connection_id)
@@ -97,19 +119,59 @@ async def _send_loop(
connection_id: str,
) -> None:
seq = 1
+ sent_packets = 0
+ sent_bytes = 0
+ last_log_at = time.monotonic()
+ resolved = self._config.resolve_env()
+ samples_per_packet = max(
+ 1,
+ int(resolved.sample_rate * resolved.audio_packet_ms / 1000) * max(1, resolved.channel),
+ )
+ pending = np.array([], dtype=np.int16)
+
+ async def _send_pcm_packet(pcm: np.ndarray) -> None:
+ nonlocal seq, sent_packets, sent_bytes, last_log_at
+ if pcm.size == 0:
+ return
+ compressed = nparray_to_bytes(pcm.astype(np.int16, copy=False))
+ seq = await send_audio(ws, compressed, seq, is_last=False)
+ sent_packets += 1
+ sent_bytes += int(pcm.nbytes)
+ now = time.monotonic()
+ if sent_packets == 1 or sent_packets % 25 == 0 or now - last_log_at >= 5.0:
+ self._logger.info(
+ "%s audio sent, connection=%s packets=%d pcm_bytes=%d packet_ms=%d last_shape=%s last_dtype=%s",
+ self._log_prefix,
+ connection_id,
+ sent_packets,
+ sent_bytes,
+ resolved.audio_packet_ms,
+ tuple(pcm.shape),
+ str(pcm.dtype),
+ )
+ last_log_at = now
+
try:
async for audio in audio_chunks:
- compressed = nparray_to_bytes(audio)
- await send_audio(ws, compressed, seq, is_last=False)
- seq += 1
+ pcm = np.asarray(audio, dtype=np.int16).reshape(-1)
+ if pcm.size == 0:
+ continue
+ pending = np.concatenate((pending, pcm))
+ while pending.size >= samples_per_packet:
+ packet = pending[:samples_per_packet]
+ pending = pending[samples_per_packet:]
+ await _send_pcm_packet(packet)
# 音频流结束,发送尾包
+ if pending.size:
+ await _send_pcm_packet(pending)
self._logger.debug(
"%s sending final audio packet, connection=%s",
self._log_prefix,
connection_id,
)
- await send_audio(ws, b"", seq, is_last=True)
+ final_packet = nparray_to_bytes(np.array([], dtype=np.int16))
+ seq = await send_audio(ws, final_packet, seq, is_last=True)
except asyncio.CancelledError:
raise
except Exception as e:
@@ -126,6 +188,7 @@ async def _receive_loop(
result_queue: asyncio.Queue[Optional[ASRResult]],
connection_id: str,
) -> None:
+ received_count = 0
try:
while True:
try:
@@ -137,15 +200,32 @@ async def _receive_loop(
continue
response = parse_response(data)
+ received_count += 1
+ if received_count <= 3 or response.message_type == ResponseMessageType.server_error:
+ self._logger.info(
+ "%s response received, connection=%s count=%d type=%s sequence=%s is_last=%s payload_len=%d",
+ self._log_prefix,
+ connection_id,
+ received_count,
+ response.message_type,
+ response.sequence,
+ response.is_last,
+ len(response.payload or ""),
+ )
if response.message_type == ResponseMessageType.server_error:
+ message = (response.payload or "").strip()
self._logger.error(
- "%s server error: %s, connection=%s",
+ "%s server error: code=%s message=%s connection=%s",
self._log_prefix,
response.error_code,
+ message[:500],
connection_id,
)
- await result_queue.put(ASRResult(text="", is_final=True))
+ await result_queue.put(ASRResult(
+ text=f"{_ASR_ERROR_PREFIX}{response.error_code}|{message}",
+ is_final=True,
+ ))
break
elif response.message_type == ResponseMessageType.server_ack:
@@ -189,6 +269,17 @@ def _parse_result(self, payload: str) -> Optional[ASRResult]:
)
return None
+ @staticmethod
+ def _response_header(ws: websockets.ClientConnection, name: str) -> str:
+ response = getattr(ws, "response", None)
+ headers = getattr(response, "headers", None)
+ if not headers:
+ return ""
+ try:
+ return str(headers.get(name, ""))
+ except Exception:
+ return ""
+
async def close(self) -> None:
self._closed = True
self._logger.info("%s closed", self._log_prefix)
diff --git a/src/ghoshell_moss/host/speech/volcengine_tts/tts.py b/src/ghoshell_moss/host/speech/volcengine_tts/tts.py
index c8751a7b..a7169533 100644
--- a/src/ghoshell_moss/host/speech/volcengine_tts/tts.py
+++ b/src/ghoshell_moss/host/speech/volcengine_tts/tts.py
@@ -309,6 +309,7 @@ class VolcengineTTSConf(BaseModel):
resource_id: str = Field(default="seed-tts-2.0", description="官方的默认资源")
sample_rate: int = Field(default=44100, description="生成音频的采样率要求.")
audio_format: Literal["pcm"] = Field(default="pcm", description="默认可用的数据格式")
+ require_usage_tokens_return: bool = Field(default=False, description="返回火山 TTS 计费字符统计")
disconnect_on_idle: int = Field(
default=300,
@@ -350,12 +351,17 @@ def default_speaker_conf(self) -> SpeakerConf:
def gen_header(self, *, connection_id: str = "", resource_id: Optional[str] = None) -> _Head:
connection_id = connection_id or unique_id()
app_key = self.unwrap_env(self.app_key)
+ resolved_resource_id = (
+ os.environ.get("VOLCENGINE_STREAM_TTS_RESOURCE_ID")
+ or resource_id
+ or self.resource_id
+ )
# 旧版鉴权 header 始终发送(兼容新旧控制台)
ws_header = {
"X-Api-App-Key": app_key,
"X-Api-App-Id": app_key,
"X-Api-Access-Key": self.unwrap_env(self.access_token),
- "X-Api-Resource-Id": resource_id or self.resource_id,
+ "X-Api-Resource-Id": resolved_resource_id,
"X-Api-Request-Id": unique_id(),
"X-Api-Connect-Id": connection_id,
}
@@ -363,6 +369,8 @@ def gen_header(self, *, connection_id: str = "", resource_id: Optional[str] = No
api_key = self.unwrap_env(self.api_key)
if api_key:
ws_header["X-Api-Key"] = api_key
+ if self.require_usage_tokens_return or os.environ.get("VOLCENGINE_STREAM_TTS_REQUIRE_USAGE") == "1":
+ ws_header["X-Control-Require-Usage-Tokens-Return"] = "*"
return ws_header
def to_session(self, speaker: SpeakerConf) -> Session:
@@ -381,7 +389,7 @@ def to_session(self, speaker: SpeakerConf) -> Session:
emotion=speaker.voice.emotion,
),
speaker=speaker.tone,
- model=self.model,
+ model=os.environ.get("VOLCENGINE_STREAM_TTS_MODEL", self.model or "") or None,
additions=additions,
),
)
@@ -702,10 +710,10 @@ async def _start_consuming_batch_loop(self, batch: VolcengineTTSBatch):
resource_id = speaker.resource_id or self._conf.resource_id
connection_id = unique_id()
header = self._conf.gen_header(connection_id=connection_id, resource_id=resource_id)
- url = self._conf.url
+ url = os.environ.get("VOLCENGINE_STREAM_TTS_URL", self._conf.url)
# 创建初始连接.
self.logger.info("%s prepare to connect to %s with header %s", self._log_prefix, url, header)
- async with connect(url, additional_headers=header) as ws:
+ async with connect(url, additional_headers=header, proxy=None) as ws:
# 建连确认.
await start_connection(ws)
self.logger.debug("%s start connection %s", self._log_prefix, connection_id)
diff --git a/tests/ghoshell_moss/host/speech/volcengine_asr/test_protocol.py b/tests/ghoshell_moss/host/speech/volcengine_asr/test_protocol.py
index 5e3b09f7..c3a08132 100644
--- a/tests/ghoshell_moss/host/speech/volcengine_asr/test_protocol.py
+++ b/tests/ghoshell_moss/host/speech/volcengine_asr/test_protocol.py
@@ -70,3 +70,18 @@ def test_parse_server_error(self):
resp = parse_response(data)
assert resp.message_type.value == "server_error"
assert resp.error_code == 1234
+
+ def test_parse_server_error_without_sequence(self):
+ header = _Protocol.get_header(
+ _Protocol.SERVER_ERROR_RESPONSE,
+ _Protocol.NO_SEQUENCE,
+ _Protocol.JSON,
+ _Protocol.NO_COMPRESSION,
+ )
+ message = b"server busy"
+ data = header + _Protocol.int_to_bytes(106) + _Protocol.int_to_bytes(len(message)) + message
+ resp = parse_response(data)
+ assert resp.message_type.value == "server_error"
+ assert resp.sequence == 0
+ assert resp.error_code == 106
+ assert resp.payload == "server busy"
From 6e699f8e0e6b826b4204b370d97ac832347aaba3 Mon Sep 17 00:00:00 2001
From: LpLegend <2422019509@qq.com>
Date: Fri, 10 Jul 2026 00:30:23 +0800
Subject: [PATCH 2/3] fix: restore listener canonical path coding by gpt-5
- keep sensors/listener as the shared ASR sensor app
- update Aether mode and README to reuse sensors/listener
- restore global LLM config, Echo soul, listener/show/g1 references
Tests:
- .venv/bin/python -m py_compile .moss_ws/apps/aether/core/main.py .moss_ws/apps/sensors/listener/main.py .moss_ws/apps/aether/vpio_capture/main.py .moss_ws/apps/aether/vpio_capture/vpio_capture.py
- .venv/bin/python -m pytest tests/ghoshell_moss/host/speech/volcengine_asr/test_protocol.py
- .venv/bin/moss --ai --mode aether apps list
- .venv/bin/moss --ai --mode listener apps list
via Codex
---
.moss_ws/apps/aether/README.md | 42 ++++++------
.moss_ws/apps/aether/listener/APP.md | 15 -----
.moss_ws/apps/bodies/g1_sim/APP.md | 4 +-
.moss_ws/apps/bodies/g1_sim/README.md | 2 +-
.../{aether => sensors}/listener/.env.example | 0
.../{aether => sensors}/listener/.gitignore | 0
.moss_ws/apps/sensors/listener/APP.md | 16 +++++
.moss_ws/apps/sensors/listener/CLAUDE.md | 66 +++++++++++++++++++
.../apps/{aether => sensors}/listener/main.py | 4 +-
.../listener/pyproject.toml | 0
.moss_ws/configs/llm.yml | 22 ++-----
.moss_ws/ghosts/echo/soul.md | 11 ----
.moss_ws/src/MOSS/modes/aether/MODE.md | 4 +-
.moss_ws/src/MOSS/modes/listener/MODE.md | 5 +-
.moss_ws/src/MOSS/modes/show/MODE.md | 4 +-
15 files changed, 120 insertions(+), 75 deletions(-)
delete mode 100644 .moss_ws/apps/aether/listener/APP.md
rename .moss_ws/apps/{aether => sensors}/listener/.env.example (100%)
rename .moss_ws/apps/{aether => sensors}/listener/.gitignore (100%)
create mode 100644 .moss_ws/apps/sensors/listener/APP.md
create mode 100644 .moss_ws/apps/sensors/listener/CLAUDE.md
rename .moss_ws/apps/{aether => sensors}/listener/main.py (99%)
rename .moss_ws/apps/{aether => sensors}/listener/pyproject.toml (100%)
diff --git a/.moss_ws/apps/aether/README.md b/.moss_ws/apps/aether/README.md
index 19c7a2cf..841cc8f2 100644
--- a/.moss_ws/apps/aether/README.md
+++ b/.moss_ws/apps/aether/README.md
@@ -9,19 +9,23 @@ Aether 是 MOSS 的实时语音交互外壳。它把麦克风采集、ASR、Ghos
```text
.moss_ws/apps/aether/
core/ 前端可视化和 WebSocket 状态聚合
- listener/ 音频到 ASR,再到 SpeechTopic / AudioSignal
vpio_capture/ macOS VPIO 音频采集和系统级回声消除
```
-三个 app 的 canonical address 是:
+Aether 自己维护两个 app:
```text
aether/core
-aether/listener
aether/vpio_capture
```
-旧地址 `ui/aether_core`、`sensors/listener`、`sensors/vpio_capture` 已不再作为 Aether 入口使用。
+ASR listener 仍然使用仓库原有的公共 sensor app:
+
+```text
+sensors/listener
+```
+
+不要把 `sensors/listener` 挪进 Aether 目录。它同时服务普通 listener mode、show mode 和 Aether mode。
## 一键启动
@@ -41,7 +45,7 @@ http://127.0.0.1:8765/
```text
aether/vpio_capture
-aether/listener
+sensors/listener
aether/core
```
@@ -68,7 +72,7 @@ aether/core
只启动 ASR listener。排查时建议先用 manual 模式,避免连续 ASR 抢占调试过程:
```bash
-LISTENER_ASR_MODE=manual .venv/bin/moss --ai --mode aether apps test aether/listener
+LISTENER_ASR_MODE=manual .venv/bin/moss --ai --mode aether apps test sensors/listener
```
只启动 macOS VPIO 采集:
@@ -92,9 +96,9 @@ LISTENER_ASR_MODE=manual .venv/bin/moss --ai --mode aether apps test aether/list
它不负责 ASR、不负责判断用户意图、不负责调用 Ghost。
-### aether/listener
+### sensors/listener
-`aether/listener` 是语音识别层。
+`sensors/listener` 是仓库原有的语音识别层,Aether 只复用它,不拥有它。
它负责:
@@ -129,7 +133,7 @@ LISTENER_ASR_MODE=manual .venv/bin/moss --ai --mode aether apps test aether/list
麦克风
-> aether/vpio_capture
-> audio topic
- -> aether/listener
+ -> sensors/listener
-> SpeechTopic + AudioSignal
-> MOSS Mindflow / Ghost
-> TTS
@@ -144,21 +148,21 @@ LISTENER_ASR_MODE=manual .venv/bin/moss --ai --mode aether apps test aether/list
browser button / VAD
-> aether/core WebSocket
-> AudioRuntimeTopic(asr_control / interrupt)
- -> aether/listener 或 MOSS host runtime
+ -> sensors/listener 或 MOSS host runtime
```
## 关键 Topic
| Topic | 发布者 | 消费者 | 作用 |
| --- | --- | --- | --- |
-| `SpeechTopic` | `aether/listener` | Ghost / `aether/core` | 用户一句完整语音文本 |
-| `AudioSignal(SPEECH_STARTED)` | `aether/listener` | Mindflow | 用户已经开始说话 |
-| `AudioSignal(SPEECH_FINAL)` | `aether/listener` | Mindflow | 用户一句话完成 |
+| `SpeechTopic` | `sensors/listener` | Ghost / `aether/core` | 用户一句完整语音文本 |
+| `AudioSignal(SPEECH_STARTED)` | `sensors/listener` | Mindflow | 用户已经开始说话 |
+| `AudioSignal(SPEECH_FINAL)` | `sensors/listener` | Mindflow | 用户一句话完成 |
| `AudioRuntimeTopic(device_name="vpio")` | `aether/vpio_capture` | `aether/core` | 采集状态和音量诊断 |
-| `AudioRuntimeTopic(device_name="asr")` | `aether/listener` | `aether/core` | ASR running/partial/final/error/idle |
-| `AudioRuntimeTopic(device_name="asr_control")` | `aether/core` | `aether/listener` | 连续/手动 ASR 控制 |
+| `AudioRuntimeTopic(device_name="asr")` | `sensors/listener` | `aether/core` | ASR running/partial/final/error/idle |
+| `AudioRuntimeTopic(device_name="asr_control")` | `aether/core` | `sensors/listener` | 连续/手动 ASR 控制 |
| `AudioRuntimeTopic(device_name="speaker")` | TTS/player | `aether/core` | TTS 播放状态 |
-| `AudioRuntimeTopic(device_name="interrupt")` | `aether/core` / `aether/listener` | `aether/core` / host runtime | 打断当前输出 |
+| `AudioRuntimeTopic(device_name="interrupt")` | `aether/core` / `sensors/listener` | `aether/core` / host runtime | 打断当前输出 |
## 常见排错
@@ -188,7 +192,7 @@ curl -s http://127.0.0.1:8765/
### 停顿后 ASR 不再识别
-优先检查 `aether/listener` 日志:
+优先检查 `sensors/listener` 日志:
- 是否还在读取 audio frames。
- 是否还有 ASR partial/final。
@@ -198,7 +202,7 @@ curl -s http://127.0.0.1:8765/
如果只是调试 listener,先用:
```bash
-LISTENER_ASR_MODE=manual .venv/bin/moss --ai --mode aether apps test aether/listener
+LISTENER_ASR_MODE=manual .venv/bin/moss --ai --mode aether apps test sensors/listener
```
这样可以把连续收音问题和 ASR 服务问题分开看。
@@ -234,8 +238,8 @@ aether/vpio_capture
Aether 代码以后尽量收敛在本目录内:
- 前端和 WebSocket 状态聚合放在 `aether/core/`。
-- ASR listener 放在 `aether/listener/`。
- macOS VPIO 采集放在 `aether/vpio_capture/`。
+- ASR listener 继续放在原仓库公共位置 `sensors/listener/`。
- MOSS host、Mindflow、Speech provider 的公共能力仍放在 `src/ghoshell_moss/`,不要复制到 app 目录。
如果新增说明文档,优先更新这个 README。不要再在仓库根 `Docs/` 下新增 Aether 历史说明。
diff --git a/.moss_ws/apps/aether/listener/APP.md b/.moss_ws/apps/aether/listener/APP.md
deleted file mode 100644
index e03ce1ad..00000000
--- a/.moss_ws/apps/aether/listener/APP.md
+++ /dev/null
@@ -1,15 +0,0 @@
----
-arguments: ''
-description: 'Aether listener — audio → Volcengine ASR → SpeechTopic + AudioSignal'
-executable: uv
-respawn: false
-script: main.py
-workers: 1
----
-
-Aether listener — audio → Volcengine ASR → SpeechTopic + AudioSignal.
-Canonical app address: `aether/listener`.
-
-The Aether baseline uses the Volcengine streaming ASR path only. Frontend ASR
-controls can switch between continuous listening and manual capture, but both
-modes use the same backend recognizer.
diff --git a/.moss_ws/apps/bodies/g1_sim/APP.md b/.moss_ws/apps/bodies/g1_sim/APP.md
index e6a8b7af..adeaf15b 100644
--- a/.moss_ws/apps/bodies/g1_sim/APP.md
+++ b/.moss_ws/apps/bodies/g1_sim/APP.md
@@ -50,7 +50,7 @@ G1 pure software simulation app.
## 麦克风指挥
- MOSS 里已经有现成的麦克风链路,不需要为了 `g1_sim` 额外再造一个新的麦克风 app
-- 连续监听链路:`sensors/audio_capture` + `aether/listener`
+- 连续监听链路:`sensors/audio_capture` + `sensors/listener`
- 按键说话链路:`sensors/ptt_listener`
- `listener / ptt_listener` 会把用户语音识别成文本,发布成 `SpeechTopic` 并发出 `AudioSignal`
- 默认 Ghost 会通过 `audio_nucleus` 接收这些语音输入,再结合 `apps.bodies_g1_sim` 的动态接口与说明,把自然语言转成 CTML 调用
@@ -60,7 +60,7 @@ G1 pure software simulation app.
```ctml
-
+
```
如果你更想避免环境噪声误触发,推荐先用 PTT:
diff --git a/.moss_ws/apps/bodies/g1_sim/README.md b/.moss_ws/apps/bodies/g1_sim/README.md
index 8f56bb6e..a813f2b3 100644
--- a/.moss_ws/apps/bodies/g1_sim/README.md
+++ b/.moss_ws/apps/bodies/g1_sim/README.md
@@ -48,7 +48,7 @@
- 已实现 viewer 自动跟随相机与稳定地面视角
- 已实现语音编排模板和精准同步版 CTML
- 已接好麦克风控制的系统侧前提:
-- `sensors/audio_capture + aether/listener`
+- `sensors/audio_capture + sensors/listener`
- `sensors/ptt_listener`
## 运行方式
diff --git a/.moss_ws/apps/aether/listener/.env.example b/.moss_ws/apps/sensors/listener/.env.example
similarity index 100%
rename from .moss_ws/apps/aether/listener/.env.example
rename to .moss_ws/apps/sensors/listener/.env.example
diff --git a/.moss_ws/apps/aether/listener/.gitignore b/.moss_ws/apps/sensors/listener/.gitignore
similarity index 100%
rename from .moss_ws/apps/aether/listener/.gitignore
rename to .moss_ws/apps/sensors/listener/.gitignore
diff --git a/.moss_ws/apps/sensors/listener/APP.md b/.moss_ws/apps/sensors/listener/APP.md
new file mode 100644
index 00000000..c5213d06
--- /dev/null
+++ b/.moss_ws/apps/sensors/listener/APP.md
@@ -0,0 +1,16 @@
+---
+arguments: ''
+description: 'ASR consumer — audio → Volcengine ASR → SpeechTopic + AudioSignal'
+executable: uv
+respawn: false
+script: main.py
+workers: 1
+---
+
+ASR consumer — audio → Volcengine ASR → SpeechTopic + AudioSignal.
+Canonical app address: `sensors/listener`.
+
+The listener is a shared sensor app. Aether reuses it through the original
+`sensors/listener` address instead of moving it into `.moss_ws/apps/aether/`.
+Frontend ASR controls can switch between continuous listening and manual
+capture, but both modes use the same backend recognizer.
diff --git a/.moss_ws/apps/sensors/listener/CLAUDE.md b/.moss_ws/apps/sensors/listener/CLAUDE.md
new file mode 100644
index 00000000..04a48775
--- /dev/null
+++ b/.moss_ws/apps/sensors/listener/CLAUDE.md
@@ -0,0 +1,66 @@
+# MOSS App Development
+
+You are working inside a MOSS App — an independent, process-isolated unit deliverable
+through the MOSS app system. Think of this as its own small project.
+
+## Project mindset
+
+- This app is a standalone unit. It may be downloaded from a hub or shared independently.
+- Tests live in `tests/` under this directory, NOT in the main project's `tests/`.
+- Dependencies go in `pyproject.toml`; the app gets its own `.venv` via `uv run`.
+
+## Directory layout
+
+```
+apps///
+├── APP.md # metadata — MOSS discovers the app through this
+├── CLAUDE.md # this file — AI developer context
+├── main.py # entry point
+├── pyproject.toml # optional — independent dependencies
+├── src/ # optional — multi-module app code
+├── tests/ # tests live here, NOT in the main project
+└── runtime/ # runtime data (auto-created by MOSS)
+ ├── assets/
+ ├── configs/
+ └── logs/
+```
+
+When you have a `pyproject.toml`, treat this as a proper project — source in `src/`,
+tests in `tests/`, managed by `uv run`. The `runtime/` directory is a MOSS convention,
+auto-managed by the framework.
+
+## Dependency management
+
+App dependencies are managed by `uv run` — it creates the app's own venv automatically.
+`moss apps start` and `moss apps test` both use `uv run` under the hood.
+
+For the full decision framework (three isolation levels, when to use pyproject.toml vs
+PEP 723 vs shared runtime), see `moss howtos read app-dev/build-an-app`.
+
+## Testing
+
+Put tests in `tests/` under this app directory. Use `test_channel()` for the common case::
+
+```python
+import pytest
+from ghoshell_moss.core.blueprint.channel_builder import new_channel, test_channel
+
+@pytest.mark.asyncio
+async def test_my_command():
+ chan = new_channel(name="my_app")
+
+ @chan.build.command()
+ async def ping() -> str:
+ return "pong"
+
+ tasks = await test_channel(chan, ctml='')
+ assert await tasks[0] == "pong"
+```
+
+Run with: `uv run pytest tests/ -v`
+
+This is the baseline. For scopes, observe, cancel, nested channels, or complex CTML
+scenarios, `test_channel` is not enough — read `moss howtos read app-dev/test-an-app`.
+
+---
+*Generated by moss apps create. Last updated 2026-06-04 by DeepSeek V4 Pro.*
diff --git a/.moss_ws/apps/aether/listener/main.py b/.moss_ws/apps/sensors/listener/main.py
similarity index 99%
rename from .moss_ws/apps/aether/listener/main.py
rename to .moss_ws/apps/sensors/listener/main.py
index 78ee1024..7f483140 100644
--- a/.moss_ws/apps/aether/listener/main.py
+++ b/.moss_ws/apps/sensors/listener/main.py
@@ -4,8 +4,8 @@
publishes SpeechTopic on final recognition, emits AudioSignal to mindflow.
Usage:
- moss apps test aether/listener
- moss apps start aether/listener
+ moss apps test sensors/listener
+ moss apps start sensors/listener
"""
import asyncio
import json
diff --git a/.moss_ws/apps/aether/listener/pyproject.toml b/.moss_ws/apps/sensors/listener/pyproject.toml
similarity index 100%
rename from .moss_ws/apps/aether/listener/pyproject.toml
rename to .moss_ws/apps/sensors/listener/pyproject.toml
diff --git a/.moss_ws/configs/llm.yml b/.moss_ws/configs/llm.yml
index ea17b6d4..55d979c3 100644
--- a/.moss_ws/configs/llm.yml
+++ b/.moss_ws/configs/llm.yml
@@ -1,28 +1,14 @@
services:
- name: anthropic
- base_url: $ANTHROPIC_BASE_URL
+ base_url: https://api.anthropic.com
api_key: $ANTHROPIC_API_KEY
api_type: anthropic
- name: openai_compat
- base_url: $DEEPSEEK_BASE_URL
- api_key: $DEEPSEEK_API_KEY
+ base_url: https://api.openai.com
+ api_key: $OPENAI_API_KEY
api_type: openai
models:
-- model: deepseek-v4-flash
- service: openai_compat
- model_type: flash
- context_window: 1000000
- max_output_tokens: 8192
- protocols:
- - text
-- model: kimi-k2.6
- service: anthropic
- model_type: default
- context_window: 200000
- max_output_tokens: 8192
- protocols:
- - text
- model: claude-sonnet-4-6
service: anthropic
model_type: default
@@ -48,4 +34,4 @@ models:
- text
- image
-default_model: deepseek-v4-flash
+default_model: claude-sonnet-4-6
diff --git a/.moss_ws/ghosts/echo/soul.md b/.moss_ws/ghosts/echo/soul.md
index 6128f2ea..78073ac7 100644
--- a/.moss_ws/ghosts/echo/soul.md
+++ b/.moss_ws/ghosts/echo/soul.md
@@ -29,14 +29,3 @@
**先感知,再行动。** 你收到的 percepts 是你了解世界的窗口——认真看它们,基于它们决策,而不是基于假设。
**承认边界。** 做不到就说做不到。你是原型,有限制是正常的。诚实的边界比虚假的全能更有价值。
-
-## Aether 语音演示纪律
-
-当你通过语音输入收到用户短句时,优先做实时对话,不要做复杂编排:
-
-- **先说话,短句回应。** 首句尽量 5 到 20 个中文字,像真人接话,不要长篇解释。
-- **默认直接输出自然语言纯文本。** 纯文本会被 Shell 的语音通道播放。除非用户明确要求操作工具,不要输出 CTML。
-- **不要启动或停止 app。** Aether 演示需要的 `aether/vpio_capture`、`aether/listener`、`aether/core` 已由外部启动。
-- **不要输出 Markdown 代码块。** 语音对话里不要展示示例代码。
-- 如果必须使用 ``,只允许这些属性:`tone`、`voice:dict`、`as_default`。不要使用 `emotion`、`style` 等不存在的属性。
-- 用户说“停下”“立刻停下”“别说了”等停止意图时,立即停止当前表达,下一轮只用一句短话确认。
diff --git a/.moss_ws/src/MOSS/modes/aether/MODE.md b/.moss_ws/src/MOSS/modes/aether/MODE.md
index 595d82a8..4ce0ab51 100644
--- a/.moss_ws/src/MOSS/modes/aether/MODE.md
+++ b/.moss_ws/src/MOSS/modes/aether/MODE.md
@@ -3,14 +3,14 @@ apps:
- '*/*'
bringup_apps:
- 'aether/vpio_capture'
- - 'aether/listener'
+ - 'sensors/listener'
- 'aether/core'
ctml_version: ''
description: 'Aether Core · 能量核心 UI 模式 — 听→想→说 完整回路 + 急刹打断 (macOS VPIO AEC)'
name: aether
---
-Aether Core 模式:拉起 `aether/vpio_capture` + `aether/listener` + `aether/core` 三个 app,ghost 通过语音对话(ASR→LLM→TTS),前端能量核心实时反映 idle/listen/think/speak/interrupt 状态。
+Aether Core 模式:拉起 `aether/vpio_capture` + `sensors/listener` + `aether/core` 三个 app,ghost 通过语音对话(ASR→LLM→TTS),前端能量核心实时反映 idle/listen/think/speak/interrupt 状态。
语音演示优先级:低延迟短回应优先于复杂 CTML。Ghost 收到语音时应优先直接输出一句自然语言纯文本,让 SpeechChannel 立刻播放;不要主动启动 app,不要输出 Markdown 代码块,不要使用 `` 等 speech channel 不支持的属性。
diff --git a/.moss_ws/src/MOSS/modes/listener/MODE.md b/.moss_ws/src/MOSS/modes/listener/MODE.md
index 23d00720..db26538d 100644
--- a/.moss_ws/src/MOSS/modes/listener/MODE.md
+++ b/.moss_ws/src/MOSS/modes/listener/MODE.md
@@ -2,11 +2,10 @@
apps:
- sensors/*
- tools/*
-- aether/*
bringup_apps:
- sensors/audio_capture
-- aether/listener
+- sensors/listener
ctml_version: ''
description: Audio capture + ASR listener mode
name: listener
----
+---
\ No newline at end of file
diff --git a/.moss_ws/src/MOSS/modes/show/MODE.md b/.moss_ws/src/MOSS/modes/show/MODE.md
index 8053dfc7..4fefa57f 100644
--- a/.moss_ws/src/MOSS/modes/show/MODE.md
+++ b/.moss_ws/src/MOSS/modes/show/MODE.md
@@ -1,9 +1,9 @@
---
-apps: ["browsers/*", "games/*", "tools/*", "ui/*", "sensors/*", "aether/*"]
+apps: ["browsers/*", "games/*", "tools/*", "ui/*", "sensors/*"]
bringup_apps: [
"tools/screen_capture",
"ui/reflex",
- "sensors/audio_capture", "aether/listener",
+ "sensors/audio_capture", "sensors/listener",
"games/ai_eye", "sensors/vision",
]
ctml_version: ''
From 77762f65619224df50a2331671cb194a2f9865e8 Mon Sep 17 00:00:00 2001
From: LpLegend <2422019509@qq.com>
Date: Fri, 10 Jul 2026 21:36:58 +0800
Subject: [PATCH 3/3] =?UTF-8?q?feat(aether):=20=E5=AE=9E=E7=8E=B0=E5=85=A8?=
=?UTF-8?q?=E5=8F=8C=E5=B7=A5=E8=AF=AD=E9=9F=B3=E4=BA=A4=E4=BA=92=E6=A8=A1?=
=?UTF-8?q?=E5=BC=8F=E6=A0=B8=E5=BF=83=E8=83=BD=E5=8A=9B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
本提交完成Aether全双工语音模式的基础实现,包含以下核心改动:
1. 新增AudioNucleus可配置中断参数,支持全双工模式关闭语音final自动中断
2. 实现带旁路急停的语音runtime,支持通过音频topic直接清空TTS缓冲
3. 重构TTS/ASR组件,添加代理配置和错误处理优化
4. 新增环境变量配置系统,集中管理Aether模式特殊行为
5. 优化Atom幽灵的兼容模式,适配OpenAI兼容流式API
6. 完善listener的门控逻辑,支持全双工回声抑制关闭
7. 新增单元测试覆盖AudioNucleus行为
8. 补充完整的文档和配置示例
---
.../2026/06/aether-duplex-ui/FEATURE.md | 85 +++++++++++++++++++
.moss_ws/apps/sensors/listener/.env.example | 14 ++-
.moss_ws/apps/sensors/listener/main.py | 59 +++++++++----
.moss_ws/src/MOSS/modes/aether/configs.py | 34 ++++++++
.moss_ws/src/MOSS/modes/aether/nuclei.py | 5 +-
.../core/mindflow/audio_nucleus.py | 35 +++++---
.../core/speech/stream_tts_speech.py | 3 +
src/ghoshell_moss/ghosts/atom/_adapter.py | 8 +-
src/ghoshell_moss/ghosts/atom/_meta.py | 13 +--
src/ghoshell_moss/ghosts/atom/_runtime.py | 35 +++++---
src/ghoshell_moss/host/app_store.py | 17 ++--
src/ghoshell_moss/host/ghost_runtime.py | 18 ++--
.../host/speech/volcengine_asr/config.py | 6 +-
.../host/speech/volcengine_asr/protocol.py | 14 ++-
.../host/speech/volcengine_asr/recognizer.py | 13 ++-
.../host/speech/volcengine_tts/tts.py | 7 +-
.../core/mindflow/test_audio_nucleus.py | 48 +++++++++++
17 files changed, 344 insertions(+), 70 deletions(-)
create mode 100644 .ai_partners/features/workstreams/2026/06/aether-duplex-ui/FEATURE.md
create mode 100644 tests/ghoshell_moss/core/mindflow/test_audio_nucleus.py
diff --git a/.ai_partners/features/workstreams/2026/06/aether-duplex-ui/FEATURE.md b/.ai_partners/features/workstreams/2026/06/aether-duplex-ui/FEATURE.md
new file mode 100644
index 00000000..18de4184
--- /dev/null
+++ b/.ai_partners/features/workstreams/2026/06/aether-duplex-ui/FEATURE.md
@@ -0,0 +1,85 @@
+---
+title: Aether 双工 UI:并发 listen/think/speak 可视化
+status: in-progress
+priority: P1
+created: 2026-06-29
+updated: 2026-07-10
+depends: []
+milestone:
+description: >-
+ 将 Aether Core 从互斥的五状态 UI,改造为面向 MOSS 全双工语音交互的并发活动层可视化。
+---
+
+# Aether 双工 UI
+
+> 使用 `moss features set-status aether-duplex-ui -m "note"` 更新状态。
+> 目录布局见 [TOPOLOGY.md](TOPOLOGY.md),完整约定见 [README.md](README.md)。
+
+## 动机
+
+Aether Core 本应证明 MOSS 的全双工交互能力,但第一版实现把 `listen`、`think`、`speak` 和 `interrupt` 压缩成了单一 UI 状态。这掩盖了核心现象:Ghost 可以一边说话一边继续听,也可以在语音输出已经开始后继续思考。
+
+单状态路径还带来了一个糟糕的失败模式:前端 VAD 可能在后端真正停止 TTS 之前就显示 `interrupt`,于是画面声称“已经停止”,但音频还在继续播放。这个 workstream 要让 UI 保真:普通 VAD 输入只点亮 listen 层;真正的 interrupt 只保留给后端 barge-in 确认,或显式发送 MOSS interrupt signal 的 interrupt 命令。
+
+## 设计索引
+
+- 关键设计文档:`design/`
+- 关键讨论记录:`discuss/`
+
+## 关键决策
+
+
+
+1. Aether 状态现在是 `layers`,不是互斥枚举。
+ WebSocket 载荷为兼容旧客户端仍保留 `state`,但权威契约是 `layers: {listen, think, speak, interrupt}`。`state` 按优先级派生:`interrupt > speak > think > listen > idle`。
+
+2. `listen`、`think`、`speak` 可以同时为 true。
+ 视觉语言改为混合核心着色器加三条外层活动环:青色 listen、紫色 think、琥珀色 speak。中心标签可以显示 `LISTEN + THINK + SPEAK`,而不是假装只有一个状态获胜。
+
+3. 前端 VAD 不再等同于 interrupt。
+ VAD 只切换本地 `mic` 诊断层。主 `listen` 由后端 ASR 运行时活动驱动,而不是由浏览器能量检测驱动。真正的 interrupt 需要 listener 检测到唤醒词,或收到显式 `{type:"interrupt"}` 请求,并通过 session 发送 `new_interrupt_signal()`。这样可以避免旧的 UI/TTS 裂脑,以及“listen 但没有 think”的假信号。
+
+4. `interrupt` 是抢占层,不是另一个普通状态。
+ 触发时,后端清空 `think` 和 `speak`,广播 `interrupt_burst`,并发送预期触发 `shell.clear()` 的 MOSS interrupt signal(TTS clear + interpretation cancellation)。
+
+5. 演示优先优化语音回合延迟,而不是保守分段。
+ listener 的 synthetic-final patience 是 1.0s,不是 3.0s。浏览器 VAD 默认阈值是 0.008,且 `listen` 会在 VAD 结束后短暂保持到 ASR final pending,所以 UI 不再在正常 ASR 延迟窗口里从 listen 直接掉回 idle。
+
+6. Aether voice mode 将 Echo 约束为短自然语言回复。
+ 默认 Echo soul 足够宽,可以做 CTML/工具调用,这会让 DeepSeek 在语音演示中输出非法语音标签,例如 ``,甚至输出 app-start 命令。Aether 现在明确要求优先短自然语言、不要 Markdown、不要启动 app;只有确实需要 CTML 时,才允许合法的 `say` 属性。
+
+## 实现记录
+
+
+
+- `.moss_ws/apps/aether/core/main.py` 负责后端 layer snapshot,并且仍为旧客户端发出 `state`。
+- `.moss_ws/apps/aether/core/webroot/web/state_mapper.js` 将 layers 映射到主状态和混合着色器目标。把映射集中在一个文件里,可以防止旧的五状态假设重新泄漏到 `main.js` 或 `scene.js`。
+- `.moss_ws/apps/aether/core/webroot/web/main.js` 会为了低于 50ms 的 listen 反馈立即应用本地 VAD,但不会因为 VAD 调用 `sendInterrupt()`。
+- 2026-06-29 尝试过浏览器视觉验证,但当前环境中的 Codex browser plugin 报告没有可用浏览器。因此改为运行 JS 语法检查、Python 编译,以及 layer-mapping 冒烟检查。
+- 2026-06-29 现场测试反馈:初始 VAD 太不敏感,ASR final 体感偏晚,interrupt 没有明显触发,Echo 生成了非法 CTML。修复:降低 VAD 阈值;本地 VAD 结束后保持 listen;缩短 ASR patience;扩大唤醒词;即使 speaker runtime topic 滞后,也把唤醒词当作 interrupt;收紧 Aether voice prompt 纪律。
+- 2026-06-29 追踪诊断:ASR 和 LLM 都在产生 turn,日志里 Volcengine TTS 也已成功返回音频。不稳定基线来自两个本地集成问题:Aether mode 没有为纯文本 speech output 注册 `__content__`;多 app 启动时可能撞上 Circus 的 transient arbiter lock。修复:在 Aether mode 中启用 `SpeechChannelModule(register_content=True)`;当 Circus 报告 arbiter 短暂忙碌时重试 watcher `start`。
+- 2026-06-29 追踪诊断:部分可见的“think but no speak”其实是假阳性 think。浏览器 VAD 结束后,在 ASR 产生 final `SpeechTopic` 之前就要求后端设置 `pending_think`;如果 ASR 没有 finalize,Aether 会超时,且没有 Ghost turn 可以说话。Reset 也曾发布字面量 `/reset` human SpeechTopic,制造假 think。修复:VAD 现在只拥有 listen 层,并等待后端 ASR final;reset 只清空视觉上下文,不再创建 speech turn。
+- 2026-06-29 现场双工 bug:listener 在 `speaker running=True` 时仍丢弃非 wake-word ASR 结果。在 VPIO AEC 路径中这是错误的:TTS 期间的用户语音已经经过 echo cancellation,必须成为正常 turn。症状很明确:浏览器显示 listen,ASR 产生 final text,但没有发布 `SpeechTopic`,所以 UI 从 listen 回到 idle,而不是进入 think/speak。修复:旧的保守行为只保留在 `LISTENER_GATE_DURING_TTS=1` 后面;Aether 默认在 TTS 期间也发布 speech。
+- 2026-06-29 完整 review:“listen then idle”有多重原因。前端 VAD 结束后立即发送 `listen=false`,所以 UI 可能在后端 ASR final 抵达前回到 idle。listener 的 300ms ASR end window 加 1.0s synthetic-final patience 会把语音过度切成“你”/“我”这类碎片,制造低价值 Ghost turn 队列并延迟短回复。Aether Core 也没有在 speaker runtime 重新变成 true 时清理 `_tts_end_at`,所以一次瞬时 `speaker=false` 可能稍后清空 `speak`,即使音频已经恢复。修复:前端 listen clear 延迟到 3.5s;listener 发布 `asr` runtime activity;ASR end window 调整为 700ms,synthetic final patience 调整为 1.4s,并丢弃单字 timeout fragment;Aether Core 使用后端 ASR runtime 表示 listen,并在 speaker 恢复时取消陈旧 TTS end grace。
+- 2026-06-29 语义修复:普通音频 impulse 不再设置 `interrupt=True`。只有唤醒词或显式 InterruptNucleus 应该调用 `shell.clear()`。这让 Aether 更接近全双工语义:用户可以在 speak 期间说话,而不会让每个 ASR final segment 都变成紧急停止。
+- 2026-06-29 诊断细化:用户仍能看到很多“listen then no think/speak”,因为浏览器 VAD 被展示成权威 listen,即使后端 ASR 没有产生 partial/final。UI 现在区分 `MIC`(浏览器本地能量)和 `LISTEN`(后端 ASR 活动)。VPIO capture 也记录一秒粒度的 RMS/peak stats,所以下次失败可以定位到浏览器 mic、macOS VPIO capture、Volcengine ASR 或 Ghost/TTS,而不是折叠成一个视觉状态。
+- 2026-06-30 ASR 完整性 bug:Volcengine ASR full-server response 可能不带 sequence 字段。旧 parser 总是跳过四个 sequence bytes,导致载荷从 `sult` 开始而不是 `{"result"`,进而 parse 失败、丢失 `LISTEN`、错过唤醒词 interrupt。修复:从 protocol flags 推导 response offset。listener segmentation 也太激进(`end_window_size=700`、synthetic-final patience `1.4s`),会截断“我有一”这类 utterance;Aether 现在使用 `end_window_size=1400`、patience `2.6s`,并加 60s first-result watchdog 来重启 stale ASR recognition。
+- 2026-06-30 interrupt/queue 诊断:日志显示“立刻停下”已被识别,Aether Core 也收到了后端 interrupt signal,但 `BaseTTSSpeech.clear()` 只清理 text output buffer,没有清理 TTS backend 或 `StreamAudioPlayer`;因此有效 interrupt 后音频仍可能继续播放。修复:`BaseTTSSpeech.clear()` 同时清理 backend TTS 和 player。另新增 `queue` activity layer,用来表示 `speak` active 时 finalize 的人类语音。TTS grace shutdown 不再无条件清掉更新的 `think`,所以用户在第一个回答期间问的第二个问题会保持显示为 `QUEUE + THINK`,而不是被第一个回答的 TTS end 覆盖。
+- 2026-06-30 现场 interrupt 延迟诊断:08:06:29 的 wake-word interrupt 立刻抵达 listener 和 Aether Core,但 host 到 08:06:58 才执行 `shell.clear()`,因为 interrupt 仍依赖 Mindflow attention/action abort checkpoint。到那时 Volcengine TTS 已经缓冲了 27.64s 的 player wait。修复:`GhostRuntimeImpl` 现在在 `AudioRuntimeTopic` 上运行一个带外 audio interrupt watcher;当看到 `device_name="interrupt"` 时立即调用 `shell.clear()`,而 Mindflow interrupt signal 仍保留为语义取消路径。
+- 2026-06-30 ASR 截断调参:现场日志仍显示“我刚才让”、“发现有一个问题,就是他”、“我需要”等 partial utterance 在 2.6s silence timeout 后被合成为 final。这主要不是浏览器 VAD 问题:后端 ASR partial 本身就是不完整的。listener 现在默认改为更保守的 `LISTENER_ASR_END_WINDOW_MS=1800` 和 `LISTENER_SILENCE_PATIENCE=4.5`,二者都可通过环境变量调整。
+- 2026-06-30 listen 语义修正:`LISTEN` 仍绑定在后端 ASR partial 上,所以 UI 只有在 Volcengine 已识别出文本后才显示 listen。现场日志显示 VPIO energy 在 ASR partial 前已经上升,例如 08:16:39/40 的 audio peaks,而 UI 到 08:16:40 的 ASR partial “我”才进入 listen。对 Aether 的视觉契约而言,listen 的含义是“音频层听到用户”,不是“语义 ASR 文本存在”。前端现在从本地浏览器 VAD(`mic`)组合出快速视觉 `listen` 层,同时仍只允许 ASR final/SpeechTopic 进入 `think`。
+- 2026-06-30 语音 interrupt 诊断:手动 stop 证明 interrupt topic → host watcher → `shell.clear()` → TTS/player clear 路径有效。语音 stop 失败位于上游:ASR 经常没有发出包含“停下”的文本。VPIO capture 现在默认 `VPIO_CHANNEL_MODE=best`,按每帧 RMS 选择最强 channel,而不是总选 channel 0,并发布 `best_ch/ch_rms` 诊断。listener 在 ASR runtime topic 中发布 ASR partial/final text,Aether Core 将最近三条 ASR 结果和 VPIO diagnostics 转发给 UI,让现场测试能区分硬件/channel capture、ASR/matcher 等问题。
+- 2026-06-30 ASR 控制变量排查:现场日志显示两个不同问题。Volcengine 增量 partial 在 UI 中看起来像多个 utterance,虽然只有 final result 会发布给 Ghost;panel 现在把当前 partial 和 final history 分开展示。Volcengine `server_error: 106` 过去被吞成空 final,listener 会立刻重连并进入 tight loop,UI 没有任何信号。recognizer 现在发出内部 error marker,listener 发布结构化 ASR error diagnostics 并退避重连,Aether UI 显示 code/backoff。local ASR 和 VPIO disabling 被有意推迟,以保持 headset-mic 测试隔离。
+- 2026-06-30 local ASR 转向:Volcengine ASR 停止产生可用结果。native sherpa-onnx runtime 已安装,但 Python/native model download 被模型托管网络卡住。已经下载完成的官方 wasm zh-en Paraformer bundle 现在通过 `/asr` 提供,并由 Aether UI 加载。浏览器本地 sherpa-onnx 在端侧完成 partial/final ASR,并通过 WebSocket 发送 `{type:"speech"}` 到 `aether_core`;后端从 final text 发布 `SpeechTopic`/`SPEECH_FINAL`。local partial 只要包含停止唤醒词,也会触发已有 interrupt path。listener 默认 `LISTENER_ASR_BACKEND=browser`,因此可以在不打开 Volcengine 的情况下保持运行。
+- 2026-06-30 native ASR 修正:浏览器 wasm bundle 下载了 226MB `.data` 文件,但在 `onRuntimeInitialized` 前卡住;即使加了 COOP/COEP headers,也很可能是 pthread worker/bootstrap 脆弱性。已从 wasm data package 中提取 `encoder.onnx`、`decoder.onnx` 和 `tokens.txt` 到 `.moss_ws/models/asr/sherpa-onnx-paraformer-zh-en-native/`。native `sherpa_onnx.OnlineRecognizer.from_paraformer(...)` 在这台 Mac 上约 0.68s 初始化,所以 listener 默认改成 `LISTENER_ASR_BACKEND=sherpa`,主 Aether 页面不再加载沉重的浏览器 wasm ASR scripts。
+- 2026-06-30 ASR fallback 决策:local sherpa ASR 对交互式 Aether 测试仍太不稳定(首轮可能识别,后续经常失败或准确率很差)。local ASR 代码、提取出的模型和 wasm debug page 保留给未来工作,但 listener 默认切回 `LISTENER_ASR_BACKEND=volcengine`。
+- 2026-06-30 Volcengine 文档排查:当前 ASR-only endpoint `wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async` 保持为默认值,但文档澄清了两个运行事实。第一,公开 ASR 错误表没有定义观察到的 raw `106`;error frame 会在 code 后携带 UTF-8 message,所以 parser 现在按文档中的 `Header + code + message_size + message` 布局解析,并把 message/backoff 暴露到 UI。第二,continuous listener mode 确实会打开 ASR recognition session 并发送 audio,所以 Aether 现在有前端 ASR control gate:`continuous` 保留旧行为,`manual + disabled` 阻止 listener 调用 `asr.recognize()`,从而避免打开 Volcengine WebSocket。完整当前技术设计记录在 `Docs/aether-voice-runtime-technical-design.md`。
+ `wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async` 是官方优化版双向 streaming 路径,不是废弃路径。更大的延迟回退来自本地调参漂移:Aether 曾漂到 `end_window_size=1800` 和 `silence_patience=4.5`,而 Volcengine 推荐低延迟 finalization 使用大约 800ms 或 1000ms。listener 默认值现在是 `LISTENER_ASR_END_WINDOW_MS=1000` 和 `LISTENER_SILENCE_PATIENCE=1.8`。protocol request 也开始发送 `enable_ddc` 和可配置的 `force_to_speech_time`,并通过 `VOLCENGINE_BM_ASR_API_KEY` 支持新版控制台 `X-Api-Key` header。官方 S2S 全双工 `wss://openspeech.bytedance.com/api/v3/duplex/realtime/dialogue` 是独立的未来架构轨道,因为它同时组合 ASR/LLM/TTS;如果不刻意包装,会绕过 Ghost/CTML。
+- 2026-06-30 DeepSeek/TTS streaming 文档排查:DeepSeek Chat Completions 可以 stream response delta,但不接受一个仍在增长的 ASR transcript 作为单个输入流。因此 Aether 保持把 ASR partial 作为 UI diagnostics,只把 final ASR text 提交给 Ghost;如果把每个 partial 都喂给 LLM,会产生“你现在”/“你现在给我讲”这类重复 turn。`Atom.articulate()` 已经使用 `agent.run_stream()`,并把 text delta yield 给 CTML/TTS。DeepSeek V4 现在通过 `thinking: {type: "disabled"}` 明确关闭 thinking,以降低语音延迟。Volcengine bidirectional TTS 已经能边消费 text chunk 边产生 audio chunk;其 headers/env overrides 现在与官方 `api/v3/tts/bidirection` 协议对齐,包括 `X-Api-Key`、resource ID selection、可选 usage-token 返回,以及 proxy-free WebSocket dialing。
+- 2026-07-01 清理:local ASR 实验不再属于当前 Aether baseline。browser wasm debug page/bridge、提取出的 local model 目录、listener backend selection branch 都从主路径移除。Aether 现在把 Volcengine ASR → `SpeechTopic` → Mindflow → DeepSeek V4 Flash → Volcengine TTS 视作唯一支持的演示回路。未来 offline ASR 工作应作为独立 feature 重新引入,并复用同一 Topic contract,而不是临时浏览器 `{type:"speech"}` shortcut。
+- 2026-07-01 交接完成:定向验证暴露了 Volcengine ASR error-frame parsing 中最后一个未清理 bug。parser 现在同时支持观察到的 sequence-wrapped error frame 和文档定义的无 sequence 布局 `Header + code + message_size + message`。新增无 sequence variant 的 protocol regression coverage。已验证 Python 编译、ASR protocol tests、speech/mindflow interrupt tests 和 WebGL JS syntax checks。完整 `pytest tests -q` 在这个 sandbox 中不是有用 gate,因为 zenoh POSIX shm 和 ZMQ tcp bind 会被环境拒绝。
+- 2026-07-01 现场音频/控制诊断:VPIO capture 看起来不是主要识别问题。当前 VPIO 路径报告 48k native capture、9 channels、input/output VPIO enabled、重复的 post-VPIO channel RMS、约 0.0007-0.0015 的低 silence RMS,以及常见 0.04-0.35、偶尔更高的 speech peaks。观察到的坏 turn 更像 ASR segmentation/finalization 问题,而不是错误 capture channel。listener 的 `LISTENER_SILENCE_PATIENCE` 默认值现在不那么激进(`3.2s`),同时保持 Volcengine `end_window_size=1000ms`。另确认一个缺陷:前端 “manual ASR disabled” 只是 transient topic,可能被 VPIO/ASR diagnostics 驱逐,之后 listener 会回退到 continuous mode;旧前端代码也会在 connect 时重新发送默认 continuous control。listener 现在保持 ASR control state sticky,使用更大的 runtime topic window,并在 manual mode disabled 时立即 gate live audio generator。WebGL bridge 只有在用户显式修改后才重发 ASR control。通过 Aether WebSocket 验证看到 `mode=manual enabled=False` 后跟随 `ASR manual gate closed`;以这种方式关闭 live Volcengine stream 仍会暴露 server EOF warning,这是清理风险,但不再表示 manual gate 失败。
+- 2026-07-01 Volcengine ASR timeout 诊断:再次检查官方 bigmodel streaming ASR 文档。`bigmodel_async` 仍是推荐的优化双向路径,audio packet duration 应保持约 100-200ms,观察到的 `45000081` 被文档定义为等待下一个 packet 超时。现场日志匹配这一点:VPIO stats 在 23:30:12 后停止,但进程仍存活;之后 listener 打开 Volcengine session,却没有发送 PCM,并每 8s 命中 `45000081`。第二个本地缺陷也被发现:listener 把 VPIO stream 当成 generic `AudioCaptureConfig` 默认 44.1kHz 输入,虽然 Aether VPIO 实际发布 16k PCM,于是把已经是 16k 的语音当成 44.1k 再采样,破坏 ASR timing。listener 现在为 Aether 默认 input sample rate 16k;如果在 `LISTENER_AUDIO_FRAME_TIMEOUT` 内没有 audio frame,就中止 ASR stream;Volcengine ASR 将外发 PCM 聚合成可配置的 200ms packet;VPIO 在 process 仍存活但 frames 停止时发布 stalled runtime topic。这修复了直接的 timeout/quality bug,但尚未自动重启 stalled VPIO engine,也还没移除下一次 ASR session 前剩余的 post-utterance cooldown gap。
+- 2026-07-04 Aether 项目布局决策:Aether 不再被当成松散的 `examples/` WebGL 演示。它是一个可运行的 MOSS mode 加 `aether/core` app,前端耦合到该 app 的 WebSocket/Topic contract。WebGL static files 已从 `examples/web_gl` 移到 `.moss_ws/apps/aether/core/webroot`,后端现在从 app-local root 提供服务。这让 app 在 `.moss_ws` 下自包含,同时保留已有 HTML/JS/着色器结构和 runtime protocol。
+- 2026-07-09 长暂停 ASR 诊断:官方 Volcengine SAUC WebSocket 文档确认优化版 streaming endpoint、100-200ms audio packet 建议、16k PCM 要求、final negative audio package、`end_window_size`,以及 `45000081` 表示 wait-for-next-packet timeout。现场失败位于 ASR 上游:listener 在没有 audio frame 抵达后正确拒绝打开新的 ASR session,而 VPIO 持续报告 `no frames`。VPIO watchdog 现在会在持续 stall 后重启 AVAudioEngine/tap,清空 stale queued frames,并发布 `restarting/restarted/restart_failed` runtime diagnostics。ASR protocol sender 也现在返回并保留 `create_audio_only_request` 发出的 packet sequence,所以 final packet 不再依赖 caller 侧重复 sequence arithmetic。
+- 2026-07-10 公共表面兼容性审查:Aether 特有行为已经收回到显式 mode/env 配置之后。公共默认值现在保留 `AudioNucleus` complete-impulse interruption、Atom 默认 Anthropic provider、AppStore 并行 bringup、listener TTS gating、listener capture sample-rate defaults、Volcengine WebSockets 使用系统 proxy,以及 silent ASR error finals。Aether mode 通过 `MOSS_ENABLE_AUDIO_INTERRUPT_TOPIC`、`MOSS_APPSTORE_BRINGUP_SERIAL`、`MOSS_ATOM_TEXT_PROMPT_COMPAT`、`MOSS_ATOM_DISABLE_HISTORY`、`LISTENER_INPUT_SAMPLE_RATE=16000`、`LISTENER_GATE_DURING_TTS=0`、`VOLCENGINE_BM_ASR_URL=.../bigmodel_async` 和 `VOLCENGINE_BM_ASR_PROPAGATE_ERRORS=1` 显式启用全双工变体。已添加聚焦的 AudioNucleus regression coverage,用来锁住公共默认行为和 Aether opt-in `interrupt_on_complete=False` 行为。
diff --git a/.moss_ws/apps/sensors/listener/.env.example b/.moss_ws/apps/sensors/listener/.env.example
index 2a2eb1f9..9dbb2733 100644
--- a/.moss_ws/apps/sensors/listener/.env.example
+++ b/.moss_ws/apps/sensors/listener/.env.example
@@ -1,6 +1,18 @@
# Listener App Environment Variables
# Copy to .env and customize as needed.
-# Disable TTS gating — ASR will run even while TTS is playing.
+# Disable TTS gating entirely — ASR will run even while TTS is playing.
# Set to 1 for debugging or when ASR and TTS must overlap.
LISTENER_DISABLE_TTS_GATE=0
+
+# Default listener behavior gates ASR while speaker/TTS is running.
+# aEther mode sets this to 0 because VPIO/AEC supports full-duplex speech.
+LISTENER_GATE_DURING_TTS=1
+
+# Leave empty to use AudioCaptureConfig.sample_rate. Set 16000 when the
+# producer is vpio_capture or another 16kHz PCM source.
+# LISTENER_INPUT_SAMPLE_RATE=16000
+
+# Propagate Volcengine server errors as diagnostic sentinel text for this app.
+# Generic ASR defaults to logging errors only.
+VOLCENGINE_BM_ASR_PROPAGATE_ERRORS=0
diff --git a/.moss_ws/apps/sensors/listener/main.py b/.moss_ws/apps/sensors/listener/main.py
index 7f483140..f9abf279 100644
--- a/.moss_ws/apps/sensors/listener/main.py
+++ b/.moss_ws/apps/sensors/listener/main.py
@@ -55,6 +55,18 @@
)
+def _tts_gate_enabled() -> bool:
+ """是否启用 TTS 播放期间的 ASR 门控。
+
+ 通用 listener 默认保持保守策略:speaker 正在播报时不把回声送进 ASR,
+ 只允许 wake word 走急停路径。aEther 依赖 VPIO/AEC 做全双工,所以在
+ aether mode 中通过 LISTENER_GATE_DURING_TTS=0 显式关闭。
+ """
+ if os.environ.get("LISTENER_DISABLE_TTS_GATE") == "1":
+ return False
+ return os.environ.get("LISTENER_GATE_DURING_TTS", "1") == "1"
+
+
def _env_float(name: str, default: float) -> float:
raw = os.environ.get(name)
if not raw:
@@ -171,9 +183,9 @@ async def _audio_generator(
``asr.recognize()`` finishes) does NOT reach ``consumer.__anext__()``
and silently drop a chunk.
- NOTE: TTS playback no longer aborts the generator. Instead, ASR continues
- recognizing during TTS. In Aether mode VPIO provides system AEC, so
- non-wake-word user speech during TTS must still become a normal turn.
+ NOTE: 这里不直接检查 TTS 状态。是否在 speaker 播放期间暂停 ASR,
+ 由 main loop 的 _tts_gate_enabled() 控制;aEther 可以关闭门控以支持
+ VPIO/AEC 全双工,普通 listener 默认仍保持保守门控。
"""
# Unbounded queue: pump must never block on put(), otherwise cancellation
# can land inside put() and the None sentinel never reaches the reader.
@@ -300,8 +312,7 @@ def _is_tts_playing(runtime_window, logger: logging.Logger | None = None) -> boo
的最新状态即可;旧的状态可能已被 running=False 覆盖。
环境变量 ``LISTENER_DISABLE_TTS_GATE=1`` 可关闭此门控。
- Aether 的 VPIO AEC 场景默认允许 TTS 播放时继续接收用户语音;如果
- 需要旧的保守回声过滤,可设置 ``LISTENER_GATE_DURING_TTS=1``。
+ 默认门控由 ``LISTENER_GATE_DURING_TTS`` 控制,未设置时为开启。
"""
if os.environ.get("LISTENER_DISABLE_TTS_GATE") == "1":
return False
@@ -320,11 +331,10 @@ async def main(matrix: Matrix) -> None:
# -- transport & source (consumer only, do not start capture) --
transport: AudioTransport = MatrixAudioTransport(matrix=matrix)
capture_config = AudioCaptureConfig()
- # Aether's vpio_capture publishes 16kHz mono PCM. The legacy MiniAudio
- # capture path used AudioCaptureConfig.sample_rate (44.1k by default), but
- # applying that default to VPIO double-resamples 16k audio and corrupts ASR
- # timing. Keep this env-tunable for non-VPIO listener modes.
- input_sample_rate = _env_int("LISTENER_INPUT_SAMPLE_RATE", _ASR_SAMPLE_RATE)
+ # 通用 listener 默认跟随 capture_config.sample_rate,保持原来的
+ # MiniAudio/audio_capture 路径兼容。aEther 的 vpio_capture 输出 16kHz
+ # mono PCM,因此由 aether mode 设置 LISTENER_INPUT_SAMPLE_RATE=16000。
+ input_sample_rate = _env_int("LISTENER_INPUT_SAMPLE_RATE", capture_config.sample_rate)
source = MiniAudioCaptureSource(transport=transport, config=capture_config)
consumer = source.new_sequential_consumer(max_queue_frames=128)
await consumer.start()
@@ -377,6 +387,27 @@ async def main(matrix: Matrix) -> None:
continue
logger.info("Waiting for speech...")
+ if _tts_gate_enabled():
+ while _is_tts_playing(runtime_window, logger):
+ drained = await _drain_consumer(consumer)
+ logger.info("TTS gate active; holding ASR, drained=%d", drained)
+ transport.pub_topic(AudioRuntimeTopic(
+ running=False,
+ device_name="asr",
+ device_explain=_asr_diag_payload(
+ source=asr_source,
+ state="tts_gate",
+ ),
+ started_at=time.monotonic(),
+ last_heartbeat=time.monotonic(),
+ ))
+ await asyncio.sleep(0.08)
+ asr_control = _refresh_asr_control(runtime_window, asr_control)
+ if asr_control.mode == "manual" and not asr_control.enabled:
+ break
+ if asr_control.mode == "manual" and not asr_control.enabled:
+ continue
+
preflight_timeout = _env_float("LISTENER_PRE_ASR_AUDIO_TIMEOUT", 2.0)
try:
first_chunk = await asyncio.wait_for(consumer.__anext__(), timeout=preflight_timeout)
@@ -403,10 +434,8 @@ async def main(matrix: Matrix) -> None:
await asyncio.sleep(0.2)
continue
- # NOTE: 不再在 TTS 播放时 hold ASR。
- # VPIO AEC 已经在系统层抑制扬声器回声;如果仍在这里把
- # speaker running 时的非唤醒词结果丢弃,用户在 speak 期间说的话
- # 就永远不会发布 SpeechTopic,前端会表现成 listen 后直接 idle。
+ # TTS 播放期间是否继续识别由 _tts_gate_enabled() 控制:
+ # 默认保守过滤回声;aEther mode 关闭门控,依赖 VPIO/AEC 支持全双工。
# Fresh abort flag and utterance id for this utterance.
abort_event = asyncio.Event()
@@ -524,7 +553,7 @@ async def main(matrix: Matrix) -> None:
# 旧的保守模式:TTS 播放时只保留 wake word,其他结果丢弃。
# Aether 默认关闭这条门控,保证真正全双工。
- if tts_active and os.environ.get("LISTENER_GATE_DURING_TTS") == "1":
+ if tts_active and _tts_gate_enabled():
continue
# Emit SPEECH_STARTED on first non-empty intermediate result for
diff --git a/.moss_ws/src/MOSS/modes/aether/configs.py b/.moss_ws/src/MOSS/modes/aether/configs.py
index e7c52eca..f6d470f2 100644
--- a/.moss_ws/src/MOSS/modes/aether/configs.py
+++ b/.moss_ws/src/MOSS/modes/aether/configs.py
@@ -1 +1,35 @@
+import os
+
from MOSS.manifests.configs import * # noqa: F403
+
+# aEther mode 的运行时默认值集中放在这里,避免把全双工语音 demo 的特殊
+# 假设写死到 MOSS 主干模块。全部使用 setdefault:用户 shell/.env 中显式
+# 配置的值优先级更高。
+
+# listener wake-word 命中后,除 Mindflow interrupt signal 外,还允许通过
+# AudioRuntimeTopic(device_name="interrupt") 直接清理 shell/TTS 缓冲。
+os.environ.setdefault("MOSS_ENABLE_AUDIO_INTERRUPT_TOPIC", "1")
+
+# aEther bringup 同时启动 vpio_capture/listener/web 时,Circus arbiter 偶发
+# “already running” 竞争;仅在该 mode 下串行启动,默认 AppStore 仍保持并行。
+os.environ.setdefault("MOSS_APPSTORE_BRINGUP_SERIAL", "1")
+
+# OpenAI-compatible pydantic_ai 流式路径对多个 TextContent/history 兼容性较弱;
+# aEther 语音场景先用单轮短上下文,后续如果换回兼容模型可取消这两个开关。
+os.environ.setdefault("MOSS_ATOM_TEXT_PROMPT_COMPAT", "1")
+os.environ.setdefault("MOSS_ATOM_DISABLE_HISTORY", "1")
+os.environ.setdefault("MOSS_OPENAI_DISABLE_THINKING", "1")
+
+# vpio_capture 输出 16kHz mono PCM;listener 默认仍跟随通用 capture 配置。
+os.environ.setdefault("LISTENER_INPUT_SAMPLE_RATE", "16000")
+os.environ.setdefault("LISTENER_GATE_DURING_TTS", "0")
+
+# aEther 当前基线使用火山 ASR SAUC 优化双向流式端点;公共 ASR 配置仍保留
+# 旧 URL 作为兼容默认,所以这里在 mode 内单独声明。
+os.environ.setdefault(
+ "VOLCENGINE_BM_ASR_URL",
+ "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async",
+)
+
+# aEther 前端需要看到 ASR 服务端错误以做诊断/backoff,通用 ASR 默认只记录日志。
+os.environ.setdefault("VOLCENGINE_BM_ASR_PROPAGATE_ERRORS", "1")
diff --git a/.moss_ws/src/MOSS/modes/aether/nuclei.py b/.moss_ws/src/MOSS/modes/aether/nuclei.py
index 25e5206e..305761d2 100644
--- a/.moss_ws/src/MOSS/modes/aether/nuclei.py
+++ b/.moss_ws/src/MOSS/modes/aether/nuclei.py
@@ -3,4 +3,7 @@
from ghoshell_moss.core.mindflow.audio_nucleus import AudioNucleusMeta
-audio_nucleus_factory = AudioNucleusMeta()
+# aEther 是全双工语音模式:普通 ASR final 表示“用户说完一句话”,不等同于
+# “立刻清空 shell/TTS”。真正的急停由 listener 的 wake word 走
+# InterruptNucleus 和 AudioRuntimeTopic(device_name="interrupt") 两条路径。
+audio_nucleus_factory = AudioNucleusMeta(interrupt_on_complete=False)
diff --git a/src/ghoshell_moss/core/mindflow/audio_nucleus.py b/src/ghoshell_moss/core/mindflow/audio_nucleus.py
index 6f68e52b..9f964924 100644
--- a/src/ghoshell_moss/core/mindflow/audio_nucleus.py
+++ b/src/ghoshell_moss/core/mindflow/audio_nucleus.py
@@ -23,20 +23,28 @@
class AudioNucleus(BufferNucleus):
"""Audio signal nucleus — aggregate ASR signals into attention impulses.
- SPEECH_STARTED (incomplete) signals preempt attention immediately on the
- first packet — the incomplete Impulse carries interrupt=True, triggers
- shell.stop_interpretation(), and occupies attention via complete=False.
- Subsequent signals in the same session accumulate silently into the buffer.
+ SPEECH_STARTED (incomplete) signals occupy attention for the current
+ utterance id via complete=False. Whether they interrupt an existing shell
+ execution is controlled by the produced Impulse/Signal semantics rather
+ than being forced here.
SPEECH_FINAL purges incomplete predecessors and produces a complete
- Impulse (interrupt=False) that delivers the full speech content to the
- already-occupied attention. If no STARTED preceded it (standalone FINAL),
- the complete impulse goes through normal arbitration without interrupt.
+ Impulse that delivers the full speech content to the already-occupied
+ attention. For compatibility, complete impulses interrupt by default;
+ aEther full-duplex voice mode disables this via interrupt_on_complete=False.
Reverse suppress (aligned with InterruptNucleus): pop_impulse starts a
victory-side cooldown; suppress only clears the buffer on the failure side.
"""
+ def __init__(self, *, interrupt_on_complete: bool = True, **kwargs):
+ super().__init__(**kwargs)
+ # 兼容默认语义:历史上 AudioNucleus 会把 complete 的语音 impulse
+ # 标记为 interrupt,用于旧 listener/show 场景在最终语音到达时抢占当前
+ # attention。aEther 的全双工语音不适合这个默认值,因此由
+ # AudioNucleusMeta(interrupt_on_complete=False) 在 aEther mode 内显式关闭。
+ self._interrupt_on_complete = interrupt_on_complete
+
async def _process_signal(self, signal: Signal) -> None:
audio_meta = AudioSignal.from_signal(signal)
if audio_meta and audio_meta.action == AudioAction.SPEECH_FINAL:
@@ -47,11 +55,10 @@ async def _process_signal(self, signal: Signal) -> None:
def _rebuild_impulse(self) -> Impulse | None:
impulse = super()._rebuild_impulse()
- if impulse is not None:
- # 普通语音不是急停。让 wake word / InterruptNucleus 负责真正的
- # shell.clear(),否则每个 ASR final 都会在响应前清掉 TTS/解释器,
- # 破坏 Aether 的全双工 speak+listen 语义。
- impulse.interrupt = False
+ if impulse is not None and impulse.complete:
+ # 只在 complete impulse 上保留旧行为开关;incomplete impulse 的
+ # interrupt 语义仍由 BufferNucleus/Signal 自身决定,避免扩大改动面。
+ impulse.interrupt = self._interrupt_on_complete
return impulse
def suppress(self, suppress_by: Impulse) -> None:
@@ -75,6 +82,9 @@ def pop_impulse(self, impulse: Impulse) -> None:
class AudioNucleusMeta(NucleusMeta):
"""音频感知核工厂 — 生产监听 audio 信号的 AudioNucleus。"""
+ def __init__(self, *, interrupt_on_complete: bool = True):
+ self._interrupt_on_complete = interrupt_on_complete
+
def name(self) -> str:
return "audio_nucleus"
@@ -95,4 +105,5 @@ def factory(self, container: IoCContainer) -> Nucleus:
min_priority=Priority.WARNING,
pulse_beat_interval=3.0,
logger=container.force_fetch(LoggerItf),
+ interrupt_on_complete=self._interrupt_on_complete,
)
diff --git a/src/ghoshell_moss/core/speech/stream_tts_speech.py b/src/ghoshell_moss/core/speech/stream_tts_speech.py
index a2754249..feacbae1 100644
--- a/src/ghoshell_moss/core/speech/stream_tts_speech.py
+++ b/src/ghoshell_moss/core/speech/stream_tts_speech.py
@@ -203,6 +203,9 @@ async def clear(self) -> list[str]:
self.logger.info("%s clear", self._log_prefix)
outputted = self._outputted.copy()
self._outputted.clear()
+ # clear 是急停路径:TTS 后端可能还在生成,player 也可能已经缓冲了
+ # 音频。两边同时清理,才能保证 shell.clear()/barge-in 后不再继续播放
+ # 旧语音;return_exceptions=True 避免一侧失败阻塞另一侧清理。
results = await asyncio.gather(
self._tts.clear(),
self._player.clear(),
diff --git a/src/ghoshell_moss/ghosts/atom/_adapter.py b/src/ghoshell_moss/ghosts/atom/_adapter.py
index 899c3e0b..0cb6de46 100644
--- a/src/ghoshell_moss/ghosts/atom/_adapter.py
+++ b/src/ghoshell_moss/ghosts/atom/_adapter.py
@@ -28,9 +28,11 @@ def messages_to_parts(messages: Iterable[Message]) -> list[UserContent]:
def moment_to_user_text(moment) -> str:
"""将 Moment 的所有请求消息合并为单个纯文本字符串.
- 避免 pydantic_ai OpenAIModel streaming 与包含 XML 的多个 TextContent
- user_prompt 不兼容 (AssertionError: Expected code to be unreachable).
- perspectives (mindflow 状态) 与用户输入合并为一段文本传给模型.
+ 这是 Atom 的兼容辅助函数,不是默认消息协议。它服务于部分
+ OpenAI-compatible streaming 端点:这些端点在 pydantic_ai 中处理多个
+ TextContent(尤其包含 mindflow XML 的 perspectives)时可能失败。启用
+ MOSS_ATOM_TEXT_PROMPT_COMPAT=1 后,运行时会把 perspectives 与用户输入
+ 合并为一段文本传给模型。
"""
chunks: list[str] = []
for msg in moment.as_request_messages():
diff --git a/src/ghoshell_moss/ghosts/atom/_meta.py b/src/ghoshell_moss/ghosts/atom/_meta.py
index 758f50fb..81fba981 100644
--- a/src/ghoshell_moss/ghosts/atom/_meta.py
+++ b/src/ghoshell_moss/ghosts/atom/_meta.py
@@ -120,7 +120,10 @@ def build_agent(self, container: IoCContainer) -> Agent[IoCContainer]:
self._load_soul(ghost_workspace)
model = self._model
if model is None:
- llm_provider = os.environ.get("MOSS_LLM_PROVIDER", "openai").lower()
+ # Atom 是 MOSS 的最小 Ghost 原型,默认 provider 保持历史兼容:
+ # 没有显式配置 MOSS_LLM_PROVIDER 时仍走 Anthropic。OpenAI-compatible
+ # provider 支持保留给 aEther/其它 mode 通过环境变量主动启用。
+ llm_provider = os.environ.get("MOSS_LLM_PROVIDER", "anthropic").lower()
if llm_provider == "anthropic":
model_name = os.environ.get("ANTHROPIC_MODEL")
if not model_name:
@@ -152,10 +155,10 @@ def build_agent(self, container: IoCContainer) -> Agent[IoCContainer]:
"OPENAI_BASE_URL / OPENAI_API_KEY env var not set."
)
model_settings = OpenAIModelSettings(timeout=60.0)
- if "deepseek" in model_name.lower() or "deepseek" in base_url.lower():
- # DeepSeek V4 defaults to thinking mode. Aether voice mode
- # needs low-latency non-thinking responses, while streaming
- # remains handled by Agent.run_stream().
+ if os.environ.get("MOSS_OPENAI_DISABLE_THINKING") == "1":
+ # 部分 OpenAI-compatible 服务(例如某些 DeepSeek 兼容端点)
+ # 支持在 extra_body 中关闭 thinking,以换取语音场景低延迟。
+ # 这不是 OpenAI 兼容协议的通用能力,必须由 mode/env 显式启用。
model_settings["extra_body"] = {"thinking": {"type": "disabled"}}
model_settings["openai_continuous_usage_stats"] = False
model = OpenAIModel(
diff --git a/src/ghoshell_moss/ghosts/atom/_runtime.py b/src/ghoshell_moss/ghosts/atom/_runtime.py
index f4a77f86..50f6b476 100644
--- a/src/ghoshell_moss/ghosts/atom/_runtime.py
+++ b/src/ghoshell_moss/ghosts/atom/_runtime.py
@@ -1,3 +1,4 @@
+import os
from typing import AsyncIterator, TYPE_CHECKING
from typing_extensions import Self
from ghoshell_moss.core.blueprint.ghost import Ghost, GhostMeta
@@ -77,7 +78,8 @@ def inspect_context(self) -> dict:
async def articulate(self, articulator: Articulator) -> AsyncIterator[str]:
moment = articulator.moment
- # /reset 命令:清空对话历史
+ # /reset 命令:清空 Atom 的内存对话历史。该命令只在用户输入精确等于
+ # /reset 时触发,不改变普通文本对话语义。
user_text = ""
for p in moment.percepts:
content = getattr(p, "content", None)
@@ -89,17 +91,26 @@ async def articulate(self, articulator: Articulator) -> AsyncIterator[str]:
self._logger.info("[Atom] context reset by /reset command")
yield "上下文已重置,我们重新开始。"
return
- # 合并 moment 所有消息为单字符串,避免 pydantic_ai OpenAIModel streaming
- # 与多个 TextContent (含 mindflow XML) 不兼容
- from ._adapter import moment_to_user_text
- user_prompt = moment_to_user_text(moment)
-
- # 注:语音对话为短上下文场景,不传 message_history 避免 pydantic_ai
- # OpenAIModel streaming 与历史 TextContent 不兼容 (同 user_prompt 问题)
- async with self._agent.run_stream(
- user_prompt=user_prompt,
- deps=self._container,
- ) as stream:
+
+ history = self.model_history()
+ if os.environ.get("MOSS_ATOM_TEXT_PROMPT_COMPAT") == "1":
+ # 兼容路径:把 Moment 的多个 TextContent 合并成单个纯文本 prompt。
+ # 这是为 OpenAI-compatible streaming 的已知兼容问题准备的开关,
+ # 不作为 Atom 的默认协议,避免丢失多模态/结构化 parts 的能力。
+ from ._adapter import moment_to_user_text
+ user_prompt = moment_to_user_text(moment)
+ else:
+ request = self.to_model_request(moment)
+ user_prompt = request.parts
+
+ run_kwargs = {
+ "user_prompt": user_prompt,
+ "deps": self._container,
+ }
+ if os.environ.get("MOSS_ATOM_DISABLE_HISTORY") != "1":
+ run_kwargs["message_history"] = history
+
+ async with self._agent.run_stream(**run_kwargs) as stream:
async for text in stream.stream_text(delta=True):
yield text
self.save_model_request(moment, stream.response)
diff --git a/src/ghoshell_moss/host/app_store.py b/src/ghoshell_moss/host/app_store.py
index 5d24c5ee..440a0a55 100644
--- a/src/ghoshell_moss/host/app_store.py
+++ b/src/ghoshell_moss/host/app_store.py
@@ -400,11 +400,18 @@ async def __aenter__(self) -> Self:
# 3. Bring-up
if self._bringup:
- for app_info in self.match_apps(self.list_apps(), self._bringup):
- # Circus arbiter still serializes watcher start internally.
- # Sequential bringup avoids transient "arbiter is already running"
- # failures when multiple apps are launched for an interactive mode.
- await self.start_app(app_info.fullname)
+ bringup_apps = list(self.match_apps(self.list_apps(), self._bringup))
+ if os.environ.get("MOSS_APPSTORE_BRINGUP_SERIAL") == "1":
+ # 仅在显式配置时串行启动。aEther 同时拉起多个实时音频/前端
+ # app 时,Circus arbiter 偶发 “already running” 竞争;串行化能
+ # 降低交互模式启动失败率。默认仍保持原来的并行 bringup,避免
+ # 拖慢普通 workspace 或改变已有 AppStore 行为。
+ for app_info in bringup_apps:
+ await self.start_app(app_info.fullname)
+ else:
+ cors = [self.start_app(app_info.fullname) for app_info in bringup_apps]
+ if cors:
+ await asyncio.gather(*cors)
return self
diff --git a/src/ghoshell_moss/host/ghost_runtime.py b/src/ghoshell_moss/host/ghost_runtime.py
index ac566659..9380821b 100644
--- a/src/ghoshell_moss/host/ghost_runtime.py
+++ b/src/ghoshell_moss/host/ghost_runtime.py
@@ -1,5 +1,6 @@
import asyncio
import contextlib
+import os
from typing import Callable, Type
import janus
@@ -229,7 +230,12 @@ def _route_signal_to_mindflow(signal: Signal):
matrix.create_task(self._main_loop(), stop_matrix_on_error=True)
matrix.create_task(self._articulate_loop(), stop_matrix_on_error=True)
matrix.create_task(self._action_loop(), stop_matrix_on_error=True)
- matrix.create_task(self._audio_interrupt_loop(), stop_matrix_on_error=False)
+ if os.environ.get("MOSS_ENABLE_AUDIO_INTERRUPT_TOPIC") == "1":
+ # 这是 aEther 全双工语音的旁路急停能力:listener 识别到“停下”等
+ # wake word 时,会发布 AudioRuntimeTopic(device_name="interrupt"),
+ # GhostRuntime 立即清空 shell/TTS 缓冲。默认不启用,避免普通 runtime
+ # 因同名 topic 或误报绕过 Mindflow 的正常 interrupt 仲裁。
+ matrix.create_task(self._audio_interrupt_loop(), stop_matrix_on_error=False)
# 等待应该发生在循环外侧.
await self._mindflow.wait_started()
# ignore any signals before started
@@ -238,12 +244,12 @@ def _route_signal_to_mindflow(signal: Signal):
# ── 三循环 ────────────────────────────────────
async def _audio_interrupt_loop(self) -> None:
- """Out-of-band audio emergency stop.
+ """语音旁路急停循环。
- Mindflow interrupt remains the semantic path, but voice barge-in must
- stop buffered TTS immediately. Waiting for the current attention/action
- to observe abort can leave tens of seconds of audio already buffered in
- the player.
+ 语义上的打断仍应优先走 Mindflow/InterruptNucleus;这里处理的是
+ aEther 这类全双工语音产品的工程现实:TTS player 可能已经缓冲了
+ 大量音频,仅等待 action abort 会让用户继续听到数秒甚至更久的旧声音。
+ 因此该循环只在显式配置 MOSS_ENABLE_AUDIO_INTERRUPT_TOPIC=1 时运行。
"""
audio_win = self.moss.matrix.session.topics.create_window_for(AudioRuntimeTopic, max_size=16)
last_started_at = 0.0
diff --git a/src/ghoshell_moss/host/speech/volcengine_asr/config.py b/src/ghoshell_moss/host/speech/volcengine_asr/config.py
index 3f7a6fa2..fc599381 100644
--- a/src/ghoshell_moss/host/speech/volcengine_asr/config.py
+++ b/src/ghoshell_moss/host/speech/volcengine_asr/config.py
@@ -19,7 +19,9 @@ class VolcengineASRConfig(BaseModel):
appid: str = Field("$VOLCENGINE_BM_ASR_APPID", description="火山引擎 asr 的 appid")
token: str = Field("$VOLCENGINE_BM_ASR_TOKEN", description="火山引擎的 asr app token")
api_key: str = Field("$VOLCENGINE_BM_ASR_API_KEY", description="新版控制台 API Key")
- url: str = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async"
+ # 默认 URL 保持历史兼容。新版 async 端点可通过 VOLCENGINE_BM_ASR_URL
+ # 覆盖,例如 wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async。
+ url: str = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel"
sample_rate: int = Field(16000, description="默认的采样率")
bits: int = Field(16)
channel: int = Field(1)
@@ -39,7 +41,7 @@ class VolcengineASRConfig(BaseModel):
)
audio_packet_ms: int = Field(
200,
- description="发送到火山 ASR 的音频包时长。官方建议 100-200ms,双向流式优化版推荐 200ms。",
+ description="发送到火山 ASR 的音频包时长。官方建议 100-200ms;可用环境变量覆盖。",
)
resource_id: str = Field("volc.bigasr.sauc.duration")
diff --git a/src/ghoshell_moss/host/speech/volcengine_asr/protocol.py b/src/ghoshell_moss/host/speech/volcengine_asr/protocol.py
index 9de2233b..a20a8bf9 100644
--- a/src/ghoshell_moss/host/speech/volcengine_asr/protocol.py
+++ b/src/ghoshell_moss/host/speech/volcengine_asr/protocol.py
@@ -2,6 +2,7 @@
import gzip
import io
import json
+import os
import struct
from typing import NamedTuple, Optional
@@ -80,7 +81,13 @@ async def connect(config: VolcengineASRConfig, connection_id: str = "") -> webso
else:
headers["X-Api-App-Key"] = config.appid
headers["X-Api-Access-Key"] = config.token
- return await websockets.connect(config.url, additional_headers=headers, proxy=None)
+ connect_kwargs = {"additional_headers": headers}
+ if os.environ.get("VOLCENGINE_BM_ASR_DISABLE_PROXY") == "1":
+ # websockets 新版本会读取系统代理环境变量。某些本地实时语音 demo
+ # 不希望走代理,可显式关闭;默认尊重用户/系统代理配置,避免影响
+ # 需要代理访问火山服务的通用环境。
+ connect_kwargs["proxy"] = None
+ return await websockets.connect(config.url, **connect_kwargs)
def create_init_request(uid: str, config: VolcengineASRConfig) -> tuple[bytes, int]:
@@ -181,8 +188,9 @@ def parse_response(data: bytes) -> Response:
offset += 4
payload = data[offset:offset + payload_size] if len(data) >= offset + payload_size else data[offset:]
else:
- # Volcengine docs define error frames without sequence/outer size:
- # Header + Error code (4B) + Error message size (4B) + Error message.
+ # 火山错误帧有两种形态。部分服务端返回不带 sequence/outer size:
+ # Header + Error code (4B) + Error message size (4B) + Error message。
+ # 这里兼容两种格式,避免把 55000/服务繁忙等错误解析成乱码。
payload = data[offset:]
if len(payload) >= 8:
diff --git a/src/ghoshell_moss/host/speech/volcengine_asr/recognizer.py b/src/ghoshell_moss/host/speech/volcengine_asr/recognizer.py
index 907d53a4..2847de92 100644
--- a/src/ghoshell_moss/host/speech/volcengine_asr/recognizer.py
+++ b/src/ghoshell_moss/host/speech/volcengine_asr/recognizer.py
@@ -1,6 +1,7 @@
import asyncio
import json
import logging
+import os
import time
from typing import AsyncIterable, Optional
@@ -222,10 +223,14 @@ async def _receive_loop(
message[:500],
connection_id,
)
- await result_queue.put(ASRResult(
- text=f"{_ASR_ERROR_PREFIX}{response.error_code}|{message}",
- is_final=True,
- ))
+ # 通用 ASR 合约不应把服务端错误伪装成用户说的话;默认只
+ # 返回空 final,让调用方结束本轮识别并查看日志。aEther
+ # listener 需要把错误码发布到运行时诊断 topic,因此通过
+ # VOLCENGINE_BM_ASR_PROPAGATE_ERRORS=1 显式启用哨兵文本。
+ text = ""
+ if os.environ.get("VOLCENGINE_BM_ASR_PROPAGATE_ERRORS") == "1":
+ text = f"{_ASR_ERROR_PREFIX}{response.error_code}|{message}"
+ await result_queue.put(ASRResult(text=text, is_final=True))
break
elif response.message_type == ResponseMessageType.server_ack:
diff --git a/src/ghoshell_moss/host/speech/volcengine_tts/tts.py b/src/ghoshell_moss/host/speech/volcengine_tts/tts.py
index a7169533..fc3e3ea4 100644
--- a/src/ghoshell_moss/host/speech/volcengine_tts/tts.py
+++ b/src/ghoshell_moss/host/speech/volcengine_tts/tts.py
@@ -713,7 +713,12 @@ async def _start_consuming_batch_loop(self, batch: VolcengineTTSBatch):
url = os.environ.get("VOLCENGINE_STREAM_TTS_URL", self._conf.url)
# 创建初始连接.
self.logger.info("%s prepare to connect to %s with header %s", self._log_prefix, url, header)
- async with connect(url, additional_headers=header, proxy=None) as ws:
+ connect_kwargs = {"additional_headers": header}
+ if os.environ.get("VOLCENGINE_STREAM_TTS_DISABLE_PROXY") == "1":
+ # 默认尊重系统代理;只有在实时语音本地链路明确不希望走代理时
+ # 才禁用。这样不会影响依赖代理访问火山 TTS 的普通部署环境。
+ connect_kwargs["proxy"] = None
+ async with connect(url, **connect_kwargs) as ws:
# 建连确认.
await start_connection(ws)
self.logger.debug("%s start connection %s", self._log_prefix, connection_id)
diff --git a/tests/ghoshell_moss/core/mindflow/test_audio_nucleus.py b/tests/ghoshell_moss/core/mindflow/test_audio_nucleus.py
new file mode 100644
index 00000000..3e553f4f
--- /dev/null
+++ b/tests/ghoshell_moss/core/mindflow/test_audio_nucleus.py
@@ -0,0 +1,48 @@
+import asyncio
+
+import pytest
+
+from ghoshell_moss.core.mindflow.audio_nucleus import AudioNucleus
+from ghoshell_moss.core.mindflow.audio_signal import AudioAction, AudioSignal
+
+
+def _speech_final_signal():
+ return AudioSignal(action=AudioAction.SPEECH_FINAL).to_signal("hello")
+
+
+def _audio_nucleus(*, interrupt_on_complete: bool = True) -> AudioNucleus:
+ return AudioNucleus(
+ name="audio_nucleus",
+ description="audio perception signal nucleus",
+ target_signal="audio",
+ default_prompt="User spoke via voice input. Process the speech.",
+ interrupt_on_complete=interrupt_on_complete,
+ )
+
+
+@pytest.mark.asyncio
+async def test_audio_nucleus_complete_impulse_interrupts_by_default():
+ nucleus = _audio_nucleus()
+
+ async with nucleus:
+ nucleus.add_signal(_speech_final_signal())
+ await asyncio.sleep(0.1)
+
+ impulse = nucleus.peek()
+ assert impulse is not None
+ assert impulse.complete is True
+ assert impulse.interrupt is True
+
+
+@pytest.mark.asyncio
+async def test_audio_nucleus_can_disable_complete_impulse_interrupt():
+ nucleus = _audio_nucleus(interrupt_on_complete=False)
+
+ async with nucleus:
+ nucleus.add_signal(_speech_final_signal())
+ await asyncio.sleep(0.1)
+
+ impulse = nucleus.peek()
+ assert impulse is not None
+ assert impulse.complete is True
+ assert impulse.interrupt is False