feat: ui updates

client/joinly_client/agent.py

@@ -2,8 +2,9 @@
 import contextlib
 import json
 import logging
+from collections.abc import Awaitable, Callable
 from dataclasses import replace
-from typing import Any, Self
+from typing import Any, Literal, Self
 
 from pydantic_ai import BinaryContent
 from pydantic_ai.direct import model_request
@@ -26,6 +27,8 @@
 
 logger = logging.getLogger(__name__)
 
+AgentStatus = Literal["llm_call", "tool_call"]
+
 
 class ConversationalToolAgent:
     """A conversational agent implementation to interact with joinly."""
@@ -41,6 +44,7 @@ def __init__(  # noqa: PLR0913
         max_tool_result_chars: int = 2048,
         max_ephemeral_tool_result_chars: int = 16384,
         max_agent_iter: int | None = 15,
+        on_status: Callable[[AgentStatus | None], Awaitable[None]] | None = None,
     ) -> None:
         """Initialize the conversational agent with a model.
 
@@ -59,6 +63,7 @@ def __init__(  # noqa: PLR0913
                 tool results, truncated directly after the call. Defaults to 16384.
             max_agent_iter (int | None): The maximum number of iterations for the agent.
                 Defaults to 15.
+            on_status: Optional callback invoked with agent status changes.
         """
         if not tools:
             msg = "At least one tool must be provided to the agent."
@@ -68,6 +73,7 @@ def __init__(  # noqa: PLR0913
         self._prompt = prompt or get_prompt()
         self._tools = tools
         self._tool_executor = tool_executor
+        self._on_status = on_status
         self._messages: list[ModelMessage] = []
         self._max_messages = max_messages
         self._max_tool_result_chars = max_tool_result_chars
@@ -109,6 +115,11 @@ async def on_utterance(self, segments: list[TranscriptSegment]) -> None:
                 await self._run_task
         self._run_task = asyncio.create_task(self._run_loop(segments))
 
+    async def _set_status(self, status: AgentStatus | None) -> None:
+        """Signal a status change if a callback is registered."""
+        if self._on_status:
+            await self._on_status(status)
+
     async def _run_loop(self, segments: list[TranscriptSegment]) -> None:
         """Run the agent loop with the provided segments.
 
@@ -132,22 +143,27 @@ async def _run_loop(self, segments: list[TranscriptSegment]) -> None:
             self._messages, max_chars=self._max_tool_result_chars
         )
         self._messages = self._omit_binary_tool_results(self._messages)
-        while self._max_agent_iter is None or iteration < self._max_agent_iter:
-            self._messages = self._limit_messages(
-                self._messages, max_messages=self._max_messages
-            )
-            self._messages = self._truncate_tool_results(
-                self._messages, max_chars=self._max_ephemeral_tool_result_chars
-            )
+        try:
+            while self._max_agent_iter is None or iteration < self._max_agent_iter:
+                self._messages = self._limit_messages(
+                    self._messages, max_messages=self._max_messages
+                )
+                self._messages = self._truncate_tool_results(
+                    self._messages, max_chars=self._max_ephemeral_tool_result_chars
+                )
 
-            response = await self._call_llm(self._messages)
-            request = await self._call_tools(response)
-            self._messages.append(response)
-            if request:
-                self._messages.append(request)
-            if self._check_end_turn(response, request):
-                break
-            iteration += 1
+                await self._set_status("llm_call")
+                response = await self._call_llm(self._messages)
+                await self._set_status(None)
+                request = await self._call_tools(response)
+                self._messages.append(response)
+                if request:
+                    self._messages.append(request)
+                if self._check_end_turn(response, request):
+                    break
+                iteration += 1
+        finally:
+            await self._set_status(None)
 
     async def _call_llm(self, messages: list[ModelMessage]) -> ModelResponse:
         """Call the LLM with the current messages.
@@ -216,7 +232,14 @@ async def _call_tools(self, response: ModelResponse) -> ModelRequest | None:
         if not tool_calls:
             return None
 
-        results = await asyncio.gather(*[self._call_tool(t) for t in tool_calls])
+        signal = any(t.tool_name != "end_turn" for t in tool_calls)
+        if signal:
+            await self._set_status("tool_call")
+        try:
+            results = await asyncio.gather(*[self._call_tool(t) for t in tool_calls])
+        finally:
+            if signal:
+                await self._set_status(None)
 
         parts: list[ModelRequestPart] = [tool_return for tool_return, _ in results]
         parts.extend(user_part for _, user_part in results if user_part)

client/joinly_client/client.py

@@ -11,21 +11,31 @@
 from fastmcp.client.transports import StreamableHttpTransport
 from mcp import ClientSession, McpError, ResourceUpdatedNotification, ServerNotification
 from mcp.types import Tool
-from pydantic import AnyUrl
+from pydantic import AnyUrl, BaseModel
 
 from joinly_client.types import (
     MeetingChatHistory,
     MeetingParticipantList,
     SpeakerRole,
+    ToolExecutor,
     Transcript,
     TranscriptSegment,
+    UIAnimation,
+    UIAnimationContent,
+    UIUpdate,
     Usage,
     VideoSnapshot,
 )
 from joinly_client.utils import is_async_context, name_in_transcript
 
 logger = logging.getLogger(__name__)
 
+
+class _UIUpdateNotification(BaseModel):
+    method: str = "notifications/joinly_ui_update"
+    params: UIUpdate | None = None
+
+
 TRANSCRIPT_URL = AnyUrl("transcript://live")
 SEGMENTS_URL = AnyUrl("transcript://live/segments")
 USAGE_URL = AnyUrl("usage://current")
@@ -467,3 +477,65 @@ async def unmute(self) -> None:
             raise RuntimeError(msg)
 
         await self.client.call_tool("unmute_yourself")
+
+    @property
+    def supports_ui_update(self) -> bool:
+        """Check if the server supports joinly_ui_update notifications."""
+        caps = self.client.initialize_result.capabilities
+        return bool(caps.experimental and "joinly_ui_update" in caps.experimental)
+
+    async def on_agent_status(self, status: str | None) -> None:
+        """Map an agent status to a UI animation."""
+        _map: dict[str, UIAnimation] = {"llm_call": "thinking", "tool_call": "busy"}
+        await self.set_ui_animation(_map.get(status or ""))
+
+    async def set_ui_animation(self, animation: UIAnimation | None) -> None:
+        """Set a UI animation by name, or clear overlay with None."""
+        await self.send_ui_update(
+            UIUpdate(content=UIAnimationContent(animation=animation))
+        )
+
+    async def send_ui_update(self, update: UIUpdate) -> None:
+        """Send a UI update notification to the server.
+
+        Does nothing if the server does not advertise the joinly_ui_update
+        experimental capability.
+
+        Args:
+            update: The UI update to send.
+        """
+        if not self.supports_ui_update:
+            return
+        await self.session.send_notification(
+            _UIUpdateNotification(params=update)  # type: ignore[arg-type]
+        )
+
+    def create_agent(
+        self,
+        llm: Any,  # noqa: ANN401
+        tools: list[Any],
+        tool_executor: ToolExecutor,
+        **kwargs: Any,  # noqa: ANN401
+    ) -> Any:  # noqa: ANN401
+        """Create a ConversationalToolAgent wired to this client.
+
+        Connects the status callback and registers the agent's utterance
+        handler.
+
+        Args:
+            llm: The language model to use.
+            tools: Tool definitions for the agent.
+            tool_executor: Callable that executes tool calls.
+            **kwargs: Forwarded to ``ConversationalToolAgent``.
+        """
+        from joinly_client.agent import ConversationalToolAgent
+
+        agent = ConversationalToolAgent(
+            llm,
+            tools,
+            tool_executor,
+            on_status=self.on_agent_status,
+            **kwargs,
+        )
+        self.add_utterance_callback(agent.on_utterance)
+        return agent

client/joinly_client/main.py

@@ -9,7 +9,6 @@
 from dotenv import load_dotenv
 from fastmcp import Client, FastMCP
 
-from joinly_client.agent import ConversationalToolAgent
 from joinly_client.client import JoinlyClient
 from joinly_client.types import McpClientConfig, TranscriptSegment
 from joinly_client.utils import get_llm, get_prompt, load_tools
@@ -371,7 +370,7 @@ async def log_segments(segments: list[TranscriptSegment]) -> None:
                 },
             }
         )
-        agent = ConversationalToolAgent(
+        agent = client.create_agent(
             llm,
             tools,
             tool_executor,
@@ -381,7 +380,6 @@ async def log_segments(segments: list[TranscriptSegment]) -> None:
                 name=client.name,
             ),
         )
-        client.add_utterance_callback(agent.on_utterance)
         async with agent:
             await client.join_meeting(meeting_url)
             try:

client/joinly_client/types.py

@@ -12,6 +12,9 @@
     SpeakerRole,
     Transcript,
     TranscriptSegment,
+    UIAnimation,
+    UIAnimationContent,
+    UIUpdate,
     Usage,
     VideoSnapshot,
 )
@@ -26,6 +29,9 @@
     "SpeakerRole",
     "Transcript",
     "TranscriptSegment",
+    "UIAnimation",
+    "UIAnimationContent",
+    "UIUpdate",
     "Usage",
     "VideoSnapshot",
 ]

common/joinly_common/types.py

@@ -1,7 +1,7 @@
 from collections.abc import Iterable
 from decimal import ROUND_HALF_UP, Decimal
 from enum import Enum
-from typing import Literal
+from typing import Annotated, Literal
 
 from pydantic import (
     BaseModel,
@@ -302,3 +302,48 @@ def merge(self, other: "Usage") -> "Usage":
     def __str__(self) -> str:
         """Return a string representation of the Usage instance."""
         return "\n".join(f"{service}: {usage}" for service, usage in self.root.items())
+
+
+UITarget = Literal["overlay", "camera"]
+
+
+UIAnimation = Literal["thinking", "busy"]
+
+
+class UIAnimationContent(BaseModel):
+    """Predefined animation content. None stops the animation."""
+
+    type: Literal["animation"] = "animation"
+    animation: UIAnimation | None = None
+    target: Literal["overlay"] = "overlay"
+
+
+class UICsp(BaseModel):
+    """CSP restrictions for HTML content (aligned with MCP Apps spec)."""
+
+    connect_domains: list[str] = Field(default_factory=list, alias="connectDomains")
+    resource_domains: list[str] = Field(default_factory=list, alias="resourceDomains")
+    frame_domains: list[str] = Field(default_factory=list, alias="frameDomains")
+
+    model_config = ConfigDict(populate_by_name=True)
+
+
+class UIHtmlContent(BaseModel):
+    """Custom HTML content. None clears the content."""
+
+    type: Literal["html"] = "html"
+    html: str | None = None
+    target: UITarget = "overlay"
+    csp: UICsp | None = None
+
+
+UIContent = Annotated[
+    UIAnimationContent | UIHtmlContent,
+    Field(discriminator="type"),
+]
+
+
+class UIUpdate(BaseModel):
+    """A UI update notification."""
+
+    content: UIContent

joinly/core.py

@@ -10,6 +10,7 @@
     SpeechWindow,
     Transcript,
     TranscriptSegment,
+    UIUpdate,
     VideoSnapshot,
 )
 from joinly.utils.clock import Clock
@@ -247,6 +248,14 @@ async def stop_sharing(self) -> None:
         """Stop sharing screen in the meeting."""
         ...
 
+    async def update_ui(self, update: UIUpdate) -> None:
+        """Update the UI on the meeting provider.
+
+        Args:
+            update: The UI update to apply.
+        """
+        ...
+
 
 class TranscriptionController(Protocol):
     """Protocol for controlling transcription processes.

joinly/providers/base.py

@@ -3,6 +3,7 @@
     MeetingChatHistory,
     MeetingParticipant,
     ProviderNotSupportedError,
+    UIUpdate,
 )
 
 
@@ -58,3 +59,6 @@ async def stop_sharing(self) -> None:
         """Stop sharing screen in the meeting."""
         msg = "Provider does not support stopping screen share."
         raise ProviderNotSupportedError(msg)
+
+    async def update_ui(self, update: UIUpdate) -> None:
+        """Update the UI on the meeting provider."""