From dd2cd4d1c096069f50f63f17efec6528426ce1a6 Mon Sep 17 00:00:00 2001 From: Markus Klein Date: Sun, 14 Jun 2026 09:48:24 +0200 Subject: [PATCH 1/2] fix(realtime): migrate from disabled OpenAI beta Realtime API to GA OpenAI disabled the beta Realtime API shape server-side; clawbody fails with: error 4000 invalid_request_error.beta_api_shape_disabled. - Use client.realtime.connect (GA) instead of client.beta.realtime.connect - Rewrite session.update payload to GA schema: - type: realtime, output_modalities, nested audio.input/output - audio format objects {type: audio/pcm, rate: 24000} - turn_detection + transcription moved under audio.input - Update output audio event names to GA: response.audio.delta -> response.output_audio.delta response.audio_transcript.{delta,done} -> response.output_audio_transcript.{delta,done} Verified live: session.created + session.updated received, 8 tools registered. --- src/reachy_mini_openclaw/openai_realtime.py | 37 ++++++++++++--------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/src/reachy_mini_openclaw/openai_realtime.py b/src/reachy_mini_openclaw/openai_realtime.py index d6b61e7..20370a7 100644 --- a/src/reachy_mini_openclaw/openai_realtime.py +++ b/src/reachy_mini_openclaw/openai_realtime.py @@ -225,25 +225,30 @@ async def _run_session(self) -> None: # Fetch OpenClaw agent context (personality, memories, user info) system_instructions = await self._build_system_instructions() - async with self.client.beta.realtime.connect(model=model) as conn: + async with self.client.realtime.connect(model=model) as conn: # Configure session with OpenClaw's identity + robot body capabilities tools = self._build_tools() await conn.session.update( session={ - "modalities": ["text", "audio"], + "type": "realtime", "instructions": system_instructions, - "voice": get_session_voice(), - "input_audio_format": "pcm16", - "output_audio_format": "pcm16", - "input_audio_transcription": { - "model": "whisper-1", - }, - "turn_detection": { - "type": "server_vad", - "threshold": 0.5, - "prefix_padding_ms": 300, - "silence_duration_ms": 600, + "output_modalities": ["audio"], + "audio": { + "input": { + "format": {"type": "audio/pcm", "rate": 24000}, + "transcription": {"model": "whisper-1"}, + "turn_detection": { + "type": "server_vad", + "threshold": 0.5, + "prefix_padding_ms": 300, + "silence_duration_ms": 600, + }, + }, + "output": { + "format": {"type": "audio/pcm", "rate": 24000}, + "voice": get_session_voice(), + }, }, "tools": tools, "tool_choice": "auto", @@ -322,7 +327,7 @@ async def _handle_event(self, event: Any) -> None: logger.debug("Response started") # Audio output from TTS - if event_type == "response.audio.delta": + if event_type == "response.output_audio.delta": # Audio arriving means we have a response - stop thinking animation self.deps.movement_manager.set_processing(False) @@ -340,11 +345,11 @@ async def _handle_event(self, event: Any) -> None: await self.output_queue.put((OPENAI_SAMPLE_RATE, audio_data)) # Response text (for logging and UI) - if event_type == "response.audio_transcript.delta": + if event_type == "response.output_audio_transcript.delta": # Streaming transcript of what's being said pass # Could log incrementally if needed - if event_type == "response.audio_transcript.done": + if event_type == "response.output_audio_transcript.done": response_text = event.transcript logger.info("Assistant: %s", response_text[:100] if len(response_text) > 100 else response_text) self._last_assistant_response = response_text # Track for sync From 43c6095b0677ee298e822550143cfe40551e13e6 Mon Sep 17 00:00:00 2001 From: Markus Klein Date: Sun, 14 Jun 2026 10:13:12 +0200 Subject: [PATCH 2/2] fix(gateway): Ed25519 device identity for OpenClaw 6.x device-auth OpenClaw 6.x rejects device-less operator clients (CONTROL_UI_DEVICE_IDENTITY_REQUIRED) and grants device-less loopback clients an empty scope set, so chat.send fails. - New device_identity.py: persistent Ed25519 keypair, device id = sha256(raw pubkey) hex, sign challenge nonce -> base64url; cache device token. - Bridge connect now sends device{id,publicKey,signature,signedAt,nonce} and signs the V2 device-auth payload (gateway verifies V3 or V2 with same sig). - Widen protocol negotiation to min=3..max=4 (fixes 'protocol mismatch'). - Request scopes operator.read+operator.write (no 'chat' scope in 6.x; chat.send is write-scoped). First connect -> pairing; approve via 'openclaw devices approve '. Token persisted for reconnects. Verified live against OpenClaw 2026.6.5: connected with scopes, agent context retrieved (3364 chars), no fallback identity. --- src/reachy_mini_openclaw/device_identity.py | 127 ++++++++++++++++++++ src/reachy_mini_openclaw/openclaw_bridge.py | 90 +++++++++++--- 2 files changed, 202 insertions(+), 15 deletions(-) create mode 100644 src/reachy_mini_openclaw/device_identity.py diff --git a/src/reachy_mini_openclaw/device_identity.py b/src/reachy_mini_openclaw/device_identity.py new file mode 100644 index 0000000..758c654 --- /dev/null +++ b/src/reachy_mini_openclaw/device_identity.py @@ -0,0 +1,127 @@ +"""Device identity for the OpenClaw gateway (6.x device-auth). + +OpenClaw 6.x requires WebSocket clients to present a device identity (an +Ed25519 keypair) to receive operator scopes; device-less connections are +granted an empty scope set and cannot send chat. This module persists an +Ed25519 keypair, derives the device id (sha256 fingerprint of the raw public +key) and signs the gateway connect challenge. + +References (openclaw/openclaw): +- src/infra/device-identity.ts (keypair, fingerprint = sha256(raw pubkey).hex, + signature = Ed25519 over UTF-8 payload, base64url) +- packages/gateway-client/src/device-auth.ts (signed payload format) +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import logging +import os +from pathlib import Path +from typing import Optional + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +logger = logging.getLogger(__name__) + + +def _b64url_encode(data: bytes) -> str: + return base64.urlsafe_b64encode(data).decode("ascii").rstrip("=") + + +def _b64url_decode(s: str) -> bytes: + return base64.urlsafe_b64decode(s + "=" * (-len(s) % 4)) + + +class DeviceIdentity: + """Persistent Ed25519 device identity + cached gateway device token.""" + + def __init__(self, private_key: Ed25519PrivateKey, path: Path): + self._private_key = private_key + self._path = path + self.public_raw = private_key.public_key().public_bytes( + serialization.Encoding.Raw, serialization.PublicFormat.Raw + ) + # device id = sha256(raw public key) hex (matches fingerprintPublicKey) + self.device_id = hashlib.sha256(self.public_raw).hexdigest() + self.public_key_b64url = _b64url_encode(self.public_raw) + self.device_token: Optional[str] = None + + @classmethod + def load_or_create(cls, path: Path) -> "DeviceIdentity": + if path.exists(): + try: + data = json.loads(path.read_text()) + pk = Ed25519PrivateKey.from_private_bytes( + _b64url_decode(data["privateKey"]) + ) + ident = cls(pk, path) + ident.device_token = data.get("deviceToken") + logger.info("Loaded device identity: %s", ident.device_id) + return ident + except Exception as e: # noqa: BLE001 + logger.warning("Device identity unreadable (%s); regenerating", e) + ident = cls(Ed25519PrivateKey.generate(), path) + ident._save() + logger.info("Generated new device identity: %s", ident.device_id) + return ident + + def _save(self) -> None: + priv_raw = self._private_key.private_bytes( + serialization.Encoding.Raw, + serialization.PrivateFormat.Raw, + serialization.NoEncryption(), + ) + payload = { + "deviceId": self.device_id, + "privateKey": _b64url_encode(priv_raw), + "publicKey": self.public_key_b64url, + "deviceToken": self.device_token, + } + tmp = self._path.with_suffix(".tmp") + tmp.write_text(json.dumps(payload)) + os.chmod(tmp, 0o600) + tmp.replace(self._path) + + def set_device_token(self, token: Optional[str]) -> None: + if token and token != self.device_token: + self.device_token = token + self._save() + + def sign_payload(self, payload: str) -> str: + """Ed25519-sign a UTF-8 payload, return base64url (no padding).""" + return _b64url_encode(self._private_key.sign(payload.encode("utf-8"))) + + +def build_device_auth_payload_v2( + *, + device_id: str, + client_id: str, + client_mode: str, + role: str, + scopes: list[str], + signed_at_ms: int, + token: str, + nonce: str, +) -> str: + """Build the V2 device-auth payload (pipe-joined). + + The gateway verifies the signature against both the V3 and V2 payloads, so + signing V2 is sufficient and avoids the V3-only platform/deviceFamily fields. + """ + return "|".join( + [ + "v2", + device_id, + client_id, + client_mode, + role, + ",".join(scopes), + str(signed_at_ms), + token or "", + nonce, + ] + ) diff --git a/src/reachy_mini_openclaw/openclaw_bridge.py b/src/reachy_mini_openclaw/openclaw_bridge.py index af512b1..aa563ac 100644 --- a/src/reachy_mini_openclaw/openclaw_bridge.py +++ b/src/reachy_mini_openclaw/openclaw_bridge.py @@ -10,18 +10,27 @@ import json import asyncio import logging +import time import uuid +from pathlib import Path from typing import Optional, Any, AsyncIterator from dataclasses import dataclass import websockets from reachy_mini_openclaw.config import config +from reachy_mini_openclaw.device_identity import ( + DeviceIdentity, + build_device_auth_payload_v2, +) logger = logging.getLogger(__name__) -# Protocol version supported by this client +# Protocol version range supported by this client. OpenClaw 6.x negotiates +# within [minProtocol, maxProtocol]; pinning a single value caused +# "protocol mismatch" against newer gateways. PROTOCOL_VERSION = 3 +MAX_PROTOCOL_VERSION = 4 @dataclass @@ -104,6 +113,15 @@ def __init__( # Events keyed by runId -> list of event payloads self._run_events: dict[str, asyncio.Queue] = {} + # Persistent Ed25519 device identity (required by OpenClaw 6.x for + # operator scopes). Path overridable via CLAWBODY_DEVICE_IDENTITY. + ident_path = Path( + os.getenv("CLAWBODY_DEVICE_IDENTITY") + or os.path.expanduser("~/.clawbody/device-identity.json") + ) + ident_path.parent.mkdir(parents=True, exist_ok=True) + self.device = DeviceIdentity.load_or_create(ident_path) + # ------------------------------------------------------------------ # URL helpers # ------------------------------------------------------------------ @@ -146,13 +164,35 @@ async def connect(self) -> bool: close_timeout=5, ) - # 1. Receive challenge + # 1. Receive challenge (carries the nonce we must sign) raw = await asyncio.wait_for(self._ws.recv(), timeout=10) challenge = json.loads(raw) if challenge.get("event") != "connect.challenge": logger.warning("Unexpected first frame: %s", challenge.get("event")) + nonce = (challenge.get("payload") or {}).get("nonce", "") + + # 2. Build the device-signed connect request. OpenClaw 6.x requires + # a device identity for operator scopes; sign the challenge nonce + # with our persistent Ed25519 device key. + client_id = "openclaw-control-ui" + client_mode = "webchat" + role = "operator" + # 6.x: chat frames are covered by operator.read/operator.write; + # there is no separate "chat" scope (approve rejects it). + scopes = ["operator.read", "operator.write"] + signed_at_ms = int(time.time() * 1000) + auth_payload = build_device_auth_payload_v2( + device_id=self.device.device_id, + client_id=client_id, + client_mode=client_mode, + role=role, + scopes=scopes, + signed_at_ms=signed_at_ms, + token=self.gateway_token or "", + nonce=nonce, + ) + signature = self.device.sign_payload(auth_payload) - # 2. Send connect request req_id = str(uuid.uuid4()) connect_req = { "type": "req", @@ -160,16 +200,23 @@ async def connect(self) -> bool: "method": "connect", "params": { "minProtocol": PROTOCOL_VERSION, - "maxProtocol": PROTOCOL_VERSION, + "maxProtocol": MAX_PROTOCOL_VERSION, "auth": {"token": self.gateway_token} if self.gateway_token else {}, "client": { - "id": "openclaw-control-ui", + "id": client_id, "version": "1.0.0", - "platform": "linux", - "mode": "webchat", + "platform": "darwin", + "mode": client_mode, + }, + "role": role, + "scopes": scopes, + "device": { + "id": self.device.device_id, + "publicKey": self.device.public_key_b64url, + "signature": signature, + "signedAt": signed_at_ms, + "nonce": nonce, }, - "role": "operator", - "scopes": ["chat", "operator.write", "operator.read"], }, } await self._ws.send(json.dumps(connect_req)) @@ -183,10 +230,16 @@ async def connect(self) -> bool: payload = hello.get("payload", {}) server = payload.get("server", {}) self._conn_id = server.get("connId") + # Cache the device token for future reconnects (optional). + self.device.set_device_token( + (payload.get("auth") or {}).get("deviceToken") + ) + granted = (payload.get("auth") or {}).get("scopes") logger.info( - "Connected to OpenClaw gateway (server=%s, connId=%s)", + "Connected to OpenClaw gateway (server=%s, connId=%s, scopes=%s)", server.get("host", "?"), self._conn_id, + granted, ) # Start background listener self._listener_task = asyncio.create_task( @@ -195,11 +248,18 @@ async def connect(self) -> bool: return True else: err = hello.get("error", {}) - logger.error( - "OpenClaw connect failed: %s - %s", - err.get("code"), - err.get("message"), - ) + code = err.get("code") + message = err.get("message", "") + logger.error("OpenClaw connect failed: %s - %s", code, message) + blob = f"{code} {message} {err.get('details')}".lower() + if "pair" in blob or "not-paired" in blob or "approve" in blob: + logger.error( + "Device pairing required. On the gateway host run:\n" + " openclaw nodes pending\n" + " openclaw nodes approve \n" + " (this device id: %s)", + self.device.device_id, + ) await self._close_ws() return False