Fix ClawBody for current OpenAI Realtime (GA) + OpenClaw 6.x gateway device-auth

src/reachy_mini_openclaw/device_identity.py

@@ -0,0 +1,127 @@
+"""Device identity for the OpenClaw gateway (6.x device-auth).
+
+OpenClaw 6.x requires WebSocket clients to present a device identity (an
+Ed25519 keypair) to receive operator scopes; device-less connections are
+granted an empty scope set and cannot send chat. This module persists an
+Ed25519 keypair, derives the device id (sha256 fingerprint of the raw public
+key) and signs the gateway connect challenge.
+
+References (openclaw/openclaw):
+- src/infra/device-identity.ts (keypair, fingerprint = sha256(raw pubkey).hex,
+  signature = Ed25519 over UTF-8 payload, base64url)
+- packages/gateway-client/src/device-auth.ts (signed payload format)
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Optional
+
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+logger = logging.getLogger(__name__)
+
+
+def _b64url_encode(data: bytes) -> str:
+    return base64.urlsafe_b64encode(data).decode("ascii").rstrip("=")
+
+
+def _b64url_decode(s: str) -> bytes:
+    return base64.urlsafe_b64decode(s + "=" * (-len(s) % 4))
+
+
+class DeviceIdentity:
+    """Persistent Ed25519 device identity + cached gateway device token."""
+
+    def __init__(self, private_key: Ed25519PrivateKey, path: Path):
+        self._private_key = private_key
+        self._path = path
+        self.public_raw = private_key.public_key().public_bytes(
+            serialization.Encoding.Raw, serialization.PublicFormat.Raw
+        )
+        # device id = sha256(raw public key) hex (matches fingerprintPublicKey)
+        self.device_id = hashlib.sha256(self.public_raw).hexdigest()
+        self.public_key_b64url = _b64url_encode(self.public_raw)
+        self.device_token: Optional[str] = None
+
+    @classmethod
+    def load_or_create(cls, path: Path) -> "DeviceIdentity":
+        if path.exists():
+            try:
+                data = json.loads(path.read_text())
+                pk = Ed25519PrivateKey.from_private_bytes(
+                    _b64url_decode(data["privateKey"])
+                )
+                ident = cls(pk, path)
+                ident.device_token = data.get("deviceToken")
+                logger.info("Loaded device identity: %s", ident.device_id)
+                return ident
+            except Exception as e:  # noqa: BLE001
+                logger.warning("Device identity unreadable (%s); regenerating", e)
+        ident = cls(Ed25519PrivateKey.generate(), path)
+        ident._save()
+        logger.info("Generated new device identity: %s", ident.device_id)
+        return ident
+
+    def _save(self) -> None:
+        priv_raw = self._private_key.private_bytes(
+            serialization.Encoding.Raw,
+            serialization.PrivateFormat.Raw,
+            serialization.NoEncryption(),
+        )
+        payload = {
+            "deviceId": self.device_id,
+            "privateKey": _b64url_encode(priv_raw),
+            "publicKey": self.public_key_b64url,
+            "deviceToken": self.device_token,
+        }
+        tmp = self._path.with_suffix(".tmp")
+        tmp.write_text(json.dumps(payload))
+        os.chmod(tmp, 0o600)
+        tmp.replace(self._path)
+
+    def set_device_token(self, token: Optional[str]) -> None:
+        if token and token != self.device_token:
+            self.device_token = token
+            self._save()
+
+    def sign_payload(self, payload: str) -> str:
+        """Ed25519-sign a UTF-8 payload, return base64url (no padding)."""
+        return _b64url_encode(self._private_key.sign(payload.encode("utf-8")))
+
+
+def build_device_auth_payload_v2(
+    *,
+    device_id: str,
+    client_id: str,
+    client_mode: str,
+    role: str,
+    scopes: list[str],
+    signed_at_ms: int,
+    token: str,
+    nonce: str,
+) -> str:
+    """Build the V2 device-auth payload (pipe-joined).
+
+    The gateway verifies the signature against both the V3 and V2 payloads, so
+    signing V2 is sufficient and avoids the V3-only platform/deviceFamily fields.
+    """
+    return "|".join(
+        [
+            "v2",
+            device_id,
+            client_id,
+            client_mode,
+            role,
+            ",".join(scopes),
+            str(signed_at_ms),
+            token or "",
+            nonce,
+        ]
+    )

src/reachy_mini_openclaw/openai_realtime.py

@@ -225,25 +225,30 @@ async def _run_session(self) -> None:
         # Fetch OpenClaw agent context (personality, memories, user info)
         system_instructions = await self._build_system_instructions()
 
-        async with self.client.beta.realtime.connect(model=model) as conn:
+        async with self.client.realtime.connect(model=model) as conn:
             # Configure session with OpenClaw's identity + robot body capabilities
             tools = self._build_tools()
 
             await conn.session.update(
                 session={
-                    "modalities": ["text", "audio"],
+                    "type": "realtime",
                     "instructions": system_instructions,
-                    "voice": get_session_voice(),
-                    "input_audio_format": "pcm16",
-                    "output_audio_format": "pcm16",
-                    "input_audio_transcription": {
-                        "model": "whisper-1",
-                    },
-                    "turn_detection": {
-                        "type": "server_vad",
-                        "threshold": 0.5,
-                        "prefix_padding_ms": 300,
-                        "silence_duration_ms": 600,
+                    "output_modalities": ["audio"],
+                    "audio": {
+                        "input": {
+                            "format": {"type": "audio/pcm", "rate": 24000},
+                            "transcription": {"model": "whisper-1"},
+                            "turn_detection": {
+                                "type": "server_vad",
+                                "threshold": 0.5,
+                                "prefix_padding_ms": 300,
+                                "silence_duration_ms": 600,
+                            },
+                        },
+                        "output": {
+                            "format": {"type": "audio/pcm", "rate": 24000},
+                            "voice": get_session_voice(),
+                        },
                     },
                     "tools": tools,
                     "tool_choice": "auto",
@@ -322,7 +327,7 @@ async def _handle_event(self, event: Any) -> None:
             logger.debug("Response started")
 
         # Audio output from TTS
-        if event_type == "response.audio.delta":
+        if event_type == "response.output_audio.delta":
             # Audio arriving means we have a response - stop thinking animation
             self.deps.movement_manager.set_processing(False)
 
@@ -340,11 +345,11 @@ async def _handle_event(self, event: Any) -> None:
             await self.output_queue.put((OPENAI_SAMPLE_RATE, audio_data))
 
         # Response text (for logging and UI)
-        if event_type == "response.audio_transcript.delta":
+        if event_type == "response.output_audio_transcript.delta":
             # Streaming transcript of what's being said
             pass  # Could log incrementally if needed
 
-        if event_type == "response.audio_transcript.done":
+        if event_type == "response.output_audio_transcript.done":
             response_text = event.transcript
             logger.info("Assistant: %s", response_text[:100] if len(response_text) > 100 else response_text)
             self._last_assistant_response = response_text  # Track for sync

src/reachy_mini_openclaw/openclaw_bridge.py

@@ -10,18 +10,27 @@
 import json
 import asyncio
 import logging
+import time
 import uuid
+from pathlib import Path
 from typing import Optional, Any, AsyncIterator
 from dataclasses import dataclass
 
 import websockets
 
 from reachy_mini_openclaw.config import config
+from reachy_mini_openclaw.device_identity import (
+    DeviceIdentity,
+    build_device_auth_payload_v2,
+)
 
 logger = logging.getLogger(__name__)
 
-# Protocol version supported by this client
+# Protocol version range supported by this client. OpenClaw 6.x negotiates
+# within [minProtocol, maxProtocol]; pinning a single value caused
+# "protocol mismatch" against newer gateways.
 PROTOCOL_VERSION = 3
+MAX_PROTOCOL_VERSION = 4
 
 
 @dataclass
@@ -104,6 +113,15 @@ def __init__(
         # Events keyed by runId -> list of event payloads
         self._run_events: dict[str, asyncio.Queue] = {}
 
+        # Persistent Ed25519 device identity (required by OpenClaw 6.x for
+        # operator scopes). Path overridable via CLAWBODY_DEVICE_IDENTITY.
+        ident_path = Path(
+            os.getenv("CLAWBODY_DEVICE_IDENTITY")
+            or os.path.expanduser("~/.clawbody/device-identity.json")
+        )
+        ident_path.parent.mkdir(parents=True, exist_ok=True)
+        self.device = DeviceIdentity.load_or_create(ident_path)
+
     # ------------------------------------------------------------------
     # URL helpers
     # ------------------------------------------------------------------
@@ -146,30 +164,59 @@ async def connect(self) -> bool:
                 close_timeout=5,
             )
 
-            # 1. Receive challenge
+            # 1. Receive challenge (carries the nonce we must sign)
             raw = await asyncio.wait_for(self._ws.recv(), timeout=10)
             challenge = json.loads(raw)
             if challenge.get("event") != "connect.challenge":
                 logger.warning("Unexpected first frame: %s", challenge.get("event"))
+            nonce = (challenge.get("payload") or {}).get("nonce", "")
+
+            # 2. Build the device-signed connect request. OpenClaw 6.x requires
+            # a device identity for operator scopes; sign the challenge nonce
+            # with our persistent Ed25519 device key.
+            client_id = "openclaw-control-ui"
+            client_mode = "webchat"
+            role = "operator"
+            # 6.x: chat frames are covered by operator.read/operator.write;
+            # there is no separate "chat" scope (approve rejects it).
+            scopes = ["operator.read", "operator.write"]
+            signed_at_ms = int(time.time() * 1000)
+            auth_payload = build_device_auth_payload_v2(
+                device_id=self.device.device_id,
+                client_id=client_id,
+                client_mode=client_mode,
+                role=role,
+                scopes=scopes,
+                signed_at_ms=signed_at_ms,
+                token=self.gateway_token or "",
+                nonce=nonce,
+            )
+            signature = self.device.sign_payload(auth_payload)
 
-            # 2. Send connect request
             req_id = str(uuid.uuid4())
             connect_req = {
                 "type": "req",
                 "id": req_id,
                 "method": "connect",
                 "params": {
                     "minProtocol": PROTOCOL_VERSION,
-                    "maxProtocol": PROTOCOL_VERSION,
+                    "maxProtocol": MAX_PROTOCOL_VERSION,
                     "auth": {"token": self.gateway_token} if self.gateway_token else {},
                     "client": {
-                        "id": "openclaw-control-ui",
+                        "id": client_id,
                         "version": "1.0.0",
-                        "platform": "linux",
-                        "mode": "webchat",
+                        "platform": "darwin",
+                        "mode": client_mode,
+                    },
+                    "role": role,
+                    "scopes": scopes,
+                    "device": {
+                        "id": self.device.device_id,
+                        "publicKey": self.device.public_key_b64url,
+                        "signature": signature,
+                        "signedAt": signed_at_ms,
+                        "nonce": nonce,
                     },
-                    "role": "operator",
-                    "scopes": ["chat", "operator.write", "operator.read"],
                 },
             }
             await self._ws.send(json.dumps(connect_req))
@@ -183,10 +230,16 @@ async def connect(self) -> bool:
                 payload = hello.get("payload", {})
                 server = payload.get("server", {})
                 self._conn_id = server.get("connId")
+                # Cache the device token for future reconnects (optional).
+                self.device.set_device_token(
+                    (payload.get("auth") or {}).get("deviceToken")
+                )
+                granted = (payload.get("auth") or {}).get("scopes")
                 logger.info(
-                    "Connected to OpenClaw gateway (server=%s, connId=%s)",
+                    "Connected to OpenClaw gateway (server=%s, connId=%s, scopes=%s)",
                     server.get("host", "?"),
                     self._conn_id,
+                    granted,
                 )
                 # Start background listener
                 self._listener_task = asyncio.create_task(
@@ -195,11 +248,18 @@ async def connect(self) -> bool:
                 return True
             else:
                 err = hello.get("error", {})
-                logger.error(
-                    "OpenClaw connect failed: %s - %s",
-                    err.get("code"),
-                    err.get("message"),
-                )
+                code = err.get("code")
+                message = err.get("message", "")
+                logger.error("OpenClaw connect failed: %s - %s", code, message)
+                blob = f"{code} {message} {err.get('details')}".lower()
+                if "pair" in blob or "not-paired" in blob or "approve" in blob:
+                    logger.error(
+                        "Device pairing required. On the gateway host run:\n"
+                        "    openclaw nodes pending\n"
+                        "    openclaw nodes approve <requestId>\n"
+                        "  (this device id: %s)",
+                        self.device.device_id,
+                    )
                 await self._close_ws()
                 return False