From ede3061b6d0c6d7d50fd26dabca82875aa421d9a Mon Sep 17 00:00:00 2001 From: dbrockmann Date: Wed, 11 Mar 2026 18:45:02 +0000 Subject: [PATCH 1/7] feat: add joinly ui updates types --- common/joinly_common/types.py | 44 ++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/common/joinly_common/types.py b/common/joinly_common/types.py index 0b7df67..1a06812 100644 --- a/common/joinly_common/types.py +++ b/common/joinly_common/types.py @@ -1,7 +1,7 @@ from collections.abc import Iterable from decimal import ROUND_HALF_UP, Decimal from enum import Enum -from typing import Literal +from typing import Annotated, Literal from pydantic import ( BaseModel, @@ -302,3 +302,45 @@ def merge(self, other: "Usage") -> "Usage": def __str__(self) -> str: """Return a string representation of the Usage instance.""" return "\n".join(f"{service}: {usage}" for service, usage in self.root.items()) + + +UITarget = Literal["overlay", "camera"] + + +class UIAnimationContent(BaseModel): + """Predefined animation content. None stops the animation.""" + + type: Literal["animation"] = "animation" + animation: Literal["thinking", "searching"] | None = None + target: Literal["overlay"] = "overlay" + + +class UICsp(BaseModel): + """CSP restrictions for HTML content (aligned with MCP Apps spec).""" + + connect_domains: list[str] = Field(default_factory=list, alias="connectDomains") + resource_domains: list[str] = Field(default_factory=list, alias="resourceDomains") + frame_domains: list[str] = Field(default_factory=list, alias="frameDomains") + + model_config = ConfigDict(populate_by_name=True) + + +class UIHtmlContent(BaseModel): + """Custom HTML content. None clears the content.""" + + type: Literal["html"] = "html" + html: str | None = None + target: UITarget = "overlay" + csp: UICsp | None = None + + +UIContent = Annotated[ + UIAnimationContent | UIHtmlContent, + Field(discriminator="type"), +] + + +class UIUpdate(BaseModel): + """A UI update notification.""" + + content: UIContent From 48a55fdc70233a20727eb43136b93e45137e1b02 Mon Sep 17 00:00:00 2001 From: dbrockmann Date: Wed, 11 Mar 2026 18:50:39 +0000 Subject: [PATCH 2/7] feat: display ui updates via mcp --- joinly/core.py | 9 ++ joinly/providers/base.py | 4 + joinly/providers/browser/camera_feed.py | 146 +++++++++++++++++-- joinly/providers/browser/meeting_provider.py | 14 +- joinly/server.py | 40 ++++- joinly/session.py | 9 ++ joinly/types.py | 6 + 7 files changed, 211 insertions(+), 17 deletions(-) diff --git a/joinly/core.py b/joinly/core.py index 84e2f43..d07f9f7 100644 --- a/joinly/core.py +++ b/joinly/core.py @@ -10,6 +10,7 @@ SpeechWindow, Transcript, TranscriptSegment, + UIUpdate, VideoSnapshot, ) from joinly.utils.clock import Clock @@ -247,6 +248,14 @@ async def stop_sharing(self) -> None: """Stop sharing screen in the meeting.""" ... + async def update_ui(self, update: UIUpdate) -> None: + """Update the UI on the meeting provider. + + Args: + update: The UI update to apply. + """ + ... + class TranscriptionController(Protocol): """Protocol for controlling transcription processes. diff --git a/joinly/providers/base.py b/joinly/providers/base.py index 0b8aab6..f312492 100644 --- a/joinly/providers/base.py +++ b/joinly/providers/base.py @@ -3,6 +3,7 @@ MeetingChatHistory, MeetingParticipant, ProviderNotSupportedError, + UIUpdate, ) @@ -58,3 +59,6 @@ async def stop_sharing(self) -> None: """Stop sharing screen in the meeting.""" msg = "Provider does not support stopping screen share." raise ProviderNotSupportedError(msg) + + async def update_ui(self, update: UIUpdate) -> None: + """Update the UI on the meeting provider.""" diff --git a/joinly/providers/browser/camera_feed.py b/joinly/providers/browser/camera_feed.py index 858448b..17e148a 100644 --- a/joinly/providers/browser/camera_feed.py +++ b/joinly/providers/browser/camera_feed.py @@ -140,6 +140,115 @@ } }""" +# Thinking: rotating arc segments with soft glow around the logo +_FX_THINKING = """\ +function fxThinking(ctx, cx, cy, logoW, logoH, t, alpha) { + const r = Math.max(logoW, logoH) * 0.62; + const pulse = 0.5 + 0.5 * Math.sin(t * 2.0); + + // Outer glow ring — subtle breathing + ctx.globalAlpha = (0.06 + pulse * 0.06) * alpha; + ctx.strokeStyle = '#ffffff'; + ctx.lineWidth = H * 0.012; + ctx.beginPath(); + ctx.arc(cx, cy, r + H * 0.006, 0, Math.PI * 2); + ctx.stroke(); + + // Rotating arc segments — 3 arcs at different speeds + for (let i = 0; i < 3; i++) { + const speed = 1.2 + i * 0.4; + const dir = i % 2 ? -1 : 1; + const base = t * speed * dir + i * Math.PI * 0.667; + const len = Math.PI * (0.3 + 0.15 * Math.sin(t * 1.5 + i)); + ctx.globalAlpha = (0.2 + (1 - i * 0.25) * 0.25) * alpha; + ctx.strokeStyle = '#ffffff'; + ctx.lineWidth = 2 - i * 0.4; + ctx.beginPath(); + ctx.arc(cx, cy, r + H * (0.002 + i * 0.006), + base, base + len); + ctx.stroke(); + } + + // Orbiting dots — 2 dots at different orbits + for (let i = 0; i < 2; i++) { + const a = t * (1.6 + i * 0.5) + i * Math.PI; + const orbitR = r + H * (0.01 + i * 0.008); + const dx = Math.cos(a) * orbitR; + const dy = Math.sin(a) * orbitR; + const dotPulse = 0.5 + 0.5 * Math.sin(t * 3 + i * 2); + ctx.globalAlpha = (0.35 + dotPulse * 0.4) * alpha; + ctx.fillStyle = '#ffffff'; + ctx.beginPath(); + ctx.arc(cx + dx, cy + dy, + H * (0.005 + dotPulse * 0.002), 0, Math.PI * 2); + ctx.fill(); + } +}""" + +# Searching: radar sweep with trailing particles +_FX_SEARCHING = """\ +function fxSearching(ctx, cx, y, t, alpha) { + const w = H * 0.08; + const speed = 0.6; + const p = (t * speed) % 2; + const dir = p <= 1 ? 1 : -1; + const norm = p <= 1 ? p : p - 1; + const ease = norm < 0.5 + ? 2 * norm * norm + : 1 - 2 * (1 - norm) * (1 - norm); + const x = dir > 0 + ? cx - w + ease * w * 2 + : cx + w - ease * w * 2; + + // Glow line + const grad = ctx.createLinearGradient( + x - H * 0.015, y, x + H * 0.015, y); + grad.addColorStop(0, 'rgba(255,255,255,0)'); + grad.addColorStop(0.5, 'rgba(255,255,255,1)'); + grad.addColorStop(1, 'rgba(255,255,255,0)'); + ctx.globalAlpha = 0.5 * alpha; + ctx.strokeStyle = grad; + ctx.lineWidth = 2; + ctx.beginPath(); + ctx.moveTo(x, y - H * 0.018); + ctx.lineTo(x, y + H * 0.018); + ctx.stroke(); + + // Centre dot + ctx.globalAlpha = 0.6 * alpha; + ctx.fillStyle = '#ffffff'; + ctx.beginPath(); + ctx.arc(x, y, H * 0.004, 0, Math.PI * 2); + ctx.fill(); + + // Trail particles + for (let i = 1; i <= 5; i++) { + const d = i * 0.04; + const tn = p <= 1 ? Math.max(0, p - d) : Math.max(0, (p - 1) - d); + const te = tn < 0.5 + ? 2 * tn * tn + : 1 - 2 * (1 - tn) * (1 - tn); + const tx = dir > 0 + ? cx - w + te * w * 2 + : cx + w - te * w * 2; + const fade = (1 - i / 6); + ctx.globalAlpha = fade * 0.35 * alpha; + ctx.fillStyle = '#ffffff'; + ctx.beginPath(); + ctx.arc(tx, y, H * (0.004 - i * 0.0004), 0, Math.PI * 2); + ctx.fill(); + } + + // Static endpoint markers + ctx.globalAlpha = 0.12 * alpha; + ctx.fillStyle = '#ffffff'; + for (const ex of [cx - w, cx + w]) { + ctx.beginPath(); + ctx.arc(ex, y, H * 0.003, 0, Math.PI * 2); + ctx.fill(); + } +}""" + # Reading: dot sweeping back and forth with a trail _FX_READING = """\ function fxReading(ctx, cx, y, t, alpha) { @@ -175,11 +284,19 @@ {fx_typing} {fx_share} {fx_reading} + {fx_thinking} + {fx_searching} const FX = {{ send_chat_message: fxTyping, get_chat_history: fxReading, get_participants: fxReading, + searching: fxSearching, + }}; + + const FX_BG = {{ + thinking: fxThinking, + share_screen: fxShare, }}; function _initCanvas() {{ @@ -241,13 +358,15 @@ statusAlpha += (wantAlpha - statusAlpha) * 0.12; if (status) statusT += 0.02; - // Share screen — behind the logo - if (statusAlpha > 0.02 - && status === 'share_screen') {{ - ctx.save(); - fxShare(ctx, cx, cy, logoW, logoH, - statusT, statusAlpha); - ctx.restore(); + // Background effects — behind the logo + if (statusAlpha > 0.02) {{ + const bgFn = FX_BG[status]; + if (bgFn) {{ + ctx.save(); + bgFn(ctx, cx, cy, logoW, logoH, + statusT, statusAlpha); + ctx.restore(); + }} }} // Speaking — behind the logo @@ -271,9 +390,8 @@ logoW, logoH ); - // Other status effects — in front of logo - if (statusAlpha > 0.02 - && status !== 'share_screen') {{ + // Foreground effects — below the logo + if (statusAlpha > 0.02) {{ const fn = FX[status]; if (fn) {{ ctx.save(); @@ -366,14 +484,16 @@ async def install(self, meeting_page: Page) -> None: fx_typing=_FX_TYPING, fx_share=_FX_SHARE, fx_reading=_FX_READING, + fx_thinking=_FX_THINKING, + fx_searching=_FX_SEARCHING, ) await meeting_page.add_init_script(script) - def set_status(self, status: str) -> None: - """Set a status label on the camera feed (e.g. 'typing...').""" + def set_effect(self, name: str | None) -> None: + """Set the active visual effect, or None to clear.""" page = self._meeting_page if page and not page.is_closed(): - safe = status.replace("'", "\\'") + safe = (name or "").replace("'", "\\'") task = asyncio.ensure_future( page.evaluate(f"window.__setStatus?.('{safe}')") ) diff --git a/joinly/providers/browser/meeting_provider.py b/joinly/providers/browser/meeting_provider.py index 9d525b8..d5e23f8 100644 --- a/joinly/providers/browser/meeting_provider.py +++ b/joinly/providers/browser/meeting_provider.py @@ -30,6 +30,9 @@ MeetingChatHistory, MeetingParticipant, ProviderNotSupportedError, + UIAnimationContent, + UIHtmlContent, + UIUpdate, VideoSnapshot, ) @@ -193,7 +196,7 @@ async def _action_guard( raise RuntimeError(msg) async with self._lock: - self._camera_feed.set_status(action) + self._camera_feed.set_effect(action) try: yield self._page, self._platform_controller except Exception as e: @@ -205,7 +208,7 @@ async def _action_guard( else: logger.info("Successfully performed '%s'.", action) finally: - self._camera_feed.set_status("") + self._camera_feed.set_effect(None) async def _get_platform_controller(self, url: str) -> BrowserPlatformController: """Get the platform-specific meeting controller based on the URL. @@ -400,6 +403,13 @@ async def stop_sharing(self) -> None: finally: await self._cleanup_content_page() + async def update_ui(self, update: UIUpdate) -> None: + """Update the UI on the camera feed.""" + if isinstance(update.content, UIAnimationContent): + self._camera_feed.set_effect(update.content.animation) + elif isinstance(update.content, UIHtmlContent): + logger.warning("HTML UI content not yet supported") + async def snapshot(self) -> VideoSnapshot: """Take a snapshot of the current video frame. diff --git a/joinly/server.py b/joinly/server.py index 6b2ccae..7b4412b 100644 --- a/joinly/server.py +++ b/joinly/server.py @@ -4,11 +4,12 @@ from collections.abc import AsyncIterator, Callable from contextlib import asynccontextmanager from dataclasses import dataclass -from typing import Annotated, Literal +from typing import Annotated, Any, Literal, Union, get_args from fastmcp import Context, FastMCP +from mcp import types as mcp_types from mcp.types import ImageContent -from pydantic import AnyUrl, Field, ValidationError +from pydantic import AnyUrl, BaseModel, Field, ValidationError from starlette.requests import Request from starlette.responses import JSONResponse @@ -21,6 +22,7 @@ SpeakerRole, SpeechInterruptedError, Transcript, + UIUpdate, Usage, ) from joinly.utils.usage import get_usage, reset_usage, set_usage @@ -31,6 +33,31 @@ SEGMENTS_URL = AnyUrl("transcript://live/segments") +class _UIUpdateNotification(BaseModel): + method: Literal["notifications/joinly_ui_update"] = "notifications/joinly_ui_update" + params: UIUpdate | None = None + + +def _patch_client_notifications(*types_: type) -> None: + field = mcp_types.ClientNotification.model_fields["root"] + current = get_args(field.annotation) + field.annotation = Union[(*current, *types_)] # type: ignore[assignment] + mcp_types.ClientNotification.model_rebuild(force=True) + + +_patch_client_notifications(_UIUpdateNotification) + + +def _patch_experimental(server: FastMCP, extra: dict[str, dict[str, Any]]) -> None: + """Advertise experimental capabilities on a FastMCP server.""" + _orig = server._mcp_server.get_capabilities # noqa: SLF001 + + def _get_capabilities(opts: Any, exp: dict | None = None) -> Any: # noqa: ANN401 + return _orig(opts, {**(exp or {}), **extra}) + + server._mcp_server.get_capabilities = _get_capabilities # type: ignore[assignment] # noqa: SLF001 + + @dataclass class SessionContext: """Context for the meeting session.""" @@ -105,6 +132,14 @@ async def _handle_unsubscribe_resource(url: AnyUrl) -> None: _remover[url]() _remover.pop(url) + async def _handle_ui_update(notification: _UIUpdateNotification) -> None: + if not notification.params: + return + logger.debug("UI update: %s", notification.params.content) + await meeting_session.update_ui(notification.params) + + server._mcp_server.notification_handlers[_UIUpdateNotification] = _handle_ui_update # noqa: SLF001 + try: yield SessionContext(meeting_session=meeting_session) finally: @@ -122,6 +157,7 @@ async def _handle_unsubscribe_resource(url: AnyUrl) -> None: mcp = FastMCP("joinly", lifespan=session_lifespan) +_patch_experimental(mcp, {"joinly_ui_update": {}}) @mcp.resource( diff --git a/joinly/session.py b/joinly/session.py index 3ece0ea..51001cf 100644 --- a/joinly/session.py +++ b/joinly/session.py @@ -12,6 +12,7 @@ MeetingChatHistory, MeetingParticipant, Transcript, + UIUpdate, VideoSnapshot, ) from joinly.utils.clock import Clock @@ -181,3 +182,11 @@ async def mute(self) -> None: async def unmute(self) -> None: """Unmute yourself in the meeting.""" await self._meeting_provider.unmute() + + async def update_ui(self, update: UIUpdate) -> None: + """Update the UI on the meeting provider. + + Args: + update: The UI update to apply. + """ + await self._meeting_provider.update_ui(update) diff --git a/joinly/types.py b/joinly/types.py index b77e6dc..75987f9 100644 --- a/joinly/types.py +++ b/joinly/types.py @@ -9,6 +9,9 @@ SpeakerRole, Transcript, TranscriptSegment, + UIAnimationContent, + UIHtmlContent, + UIUpdate, Usage, VideoSnapshot, ) @@ -22,6 +25,9 @@ "SpeakerRole", "Transcript", "TranscriptSegment", + "UIAnimationContent", + "UIHtmlContent", + "UIUpdate", "Usage", "VideoSnapshot", ] From cb287be09f484ddae6e897082ab84e49a94b9121 Mon Sep 17 00:00:00 2001 From: dbrockmann Date: Wed, 11 Mar 2026 19:37:39 +0000 Subject: [PATCH 3/7] refactor: rename searching to busy and define UIAnimation as own type --- common/joinly_common/types.py | 5 ++++- joinly/providers/browser/camera_feed.py | 12 ++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/common/joinly_common/types.py b/common/joinly_common/types.py index 1a06812..510ce49 100644 --- a/common/joinly_common/types.py +++ b/common/joinly_common/types.py @@ -307,11 +307,14 @@ def __str__(self) -> str: UITarget = Literal["overlay", "camera"] +UIAnimation = Literal["thinking", "busy"] + + class UIAnimationContent(BaseModel): """Predefined animation content. None stops the animation.""" type: Literal["animation"] = "animation" - animation: Literal["thinking", "searching"] | None = None + animation: UIAnimation | None = None target: Literal["overlay"] = "overlay" diff --git a/joinly/providers/browser/camera_feed.py b/joinly/providers/browser/camera_feed.py index 17e148a..e6695f0 100644 --- a/joinly/providers/browser/camera_feed.py +++ b/joinly/providers/browser/camera_feed.py @@ -185,9 +185,9 @@ } }""" -# Searching: radar sweep with trailing particles -_FX_SEARCHING = """\ -function fxSearching(ctx, cx, y, t, alpha) { +# Busy: radar sweep with trailing particles +_FX_BUSY = """\ +function fxBusy(ctx, cx, y, t, alpha) { const w = H * 0.08; const speed = 0.6; const p = (t * speed) % 2; @@ -285,13 +285,13 @@ {fx_share} {fx_reading} {fx_thinking} - {fx_searching} + {fx_busy} const FX = {{ send_chat_message: fxTyping, get_chat_history: fxReading, get_participants: fxReading, - searching: fxSearching, + busy: fxBusy, }}; const FX_BG = {{ @@ -485,7 +485,7 @@ async def install(self, meeting_page: Page) -> None: fx_share=_FX_SHARE, fx_reading=_FX_READING, fx_thinking=_FX_THINKING, - fx_searching=_FX_SEARCHING, + fx_busy=_FX_BUSY, ) await meeting_page.add_init_script(script) From e2d32920c02e75cc08aa1e6a355985f8c833f3f6 Mon Sep 17 00:00:00 2001 From: dbrockmann Date: Wed, 11 Mar 2026 19:43:38 +0000 Subject: [PATCH 4/7] feat: agent on status callback for status updates --- client/joinly_client/agent.py | 49 +++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/client/joinly_client/agent.py b/client/joinly_client/agent.py index 59f01ee..e7096c5 100644 --- a/client/joinly_client/agent.py +++ b/client/joinly_client/agent.py @@ -2,8 +2,9 @@ import contextlib import json import logging +from collections.abc import Awaitable, Callable from dataclasses import replace -from typing import Any, Self +from typing import Any, Literal, Self from pydantic_ai import BinaryContent from pydantic_ai.direct import model_request @@ -26,6 +27,8 @@ logger = logging.getLogger(__name__) +AgentStatus = Literal["llm_call", "tool_call"] + class ConversationalToolAgent: """A conversational agent implementation to interact with joinly.""" @@ -41,6 +44,7 @@ def __init__( # noqa: PLR0913 max_tool_result_chars: int = 2048, max_ephemeral_tool_result_chars: int = 16384, max_agent_iter: int | None = 15, + on_status: Callable[[AgentStatus | None], Awaitable[None]] | None = None, ) -> None: """Initialize the conversational agent with a model. @@ -59,6 +63,7 @@ def __init__( # noqa: PLR0913 tool results, truncated directly after the call. Defaults to 16384. max_agent_iter (int | None): The maximum number of iterations for the agent. Defaults to 15. + on_status: Optional callback invoked with agent status changes. """ if not tools: msg = "At least one tool must be provided to the agent." @@ -68,6 +73,7 @@ def __init__( # noqa: PLR0913 self._prompt = prompt or get_prompt() self._tools = tools self._tool_executor = tool_executor + self._on_status = on_status self._messages: list[ModelMessage] = [] self._max_messages = max_messages self._max_tool_result_chars = max_tool_result_chars @@ -109,6 +115,11 @@ async def on_utterance(self, segments: list[TranscriptSegment]) -> None: await self._run_task self._run_task = asyncio.create_task(self._run_loop(segments)) + async def _set_status(self, status: AgentStatus | None) -> None: + """Signal a status change if a callback is registered.""" + if self._on_status: + await self._on_status(status) + async def _run_loop(self, segments: list[TranscriptSegment]) -> None: """Run the agent loop with the provided segments. @@ -132,22 +143,28 @@ async def _run_loop(self, segments: list[TranscriptSegment]) -> None: self._messages, max_chars=self._max_tool_result_chars ) self._messages = self._omit_binary_tool_results(self._messages) - while self._max_agent_iter is None or iteration < self._max_agent_iter: - self._messages = self._limit_messages( - self._messages, max_messages=self._max_messages - ) - self._messages = self._truncate_tool_results( - self._messages, max_chars=self._max_ephemeral_tool_result_chars - ) + try: + while self._max_agent_iter is None or iteration < self._max_agent_iter: + self._messages = self._limit_messages( + self._messages, max_messages=self._max_messages + ) + self._messages = self._truncate_tool_results( + self._messages, max_chars=self._max_ephemeral_tool_result_chars + ) - response = await self._call_llm(self._messages) - request = await self._call_tools(response) - self._messages.append(response) - if request: - self._messages.append(request) - if self._check_end_turn(response, request): - break - iteration += 1 + await self._set_status("llm_call") + response = await self._call_llm(self._messages) + await self._set_status("tool_call") + request = await self._call_tools(response) + await self._set_status(None) + self._messages.append(response) + if request: + self._messages.append(request) + if self._check_end_turn(response, request): + break + iteration += 1 + finally: + await self._set_status(None) async def _call_llm(self, messages: list[ModelMessage]) -> ModelResponse: """Call the LLM with the current messages. From 33455eecbe6a8df06c24bc677a64c5716480b004 Mon Sep 17 00:00:00 2001 From: dbrockmann Date: Wed, 11 Mar 2026 20:20:17 +0000 Subject: [PATCH 5/7] feat: not set tool call status for end turn --- client/joinly_client/agent.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/client/joinly_client/agent.py b/client/joinly_client/agent.py index e7096c5..3c530a9 100644 --- a/client/joinly_client/agent.py +++ b/client/joinly_client/agent.py @@ -154,9 +154,8 @@ async def _run_loop(self, segments: list[TranscriptSegment]) -> None: await self._set_status("llm_call") response = await self._call_llm(self._messages) - await self._set_status("tool_call") - request = await self._call_tools(response) await self._set_status(None) + request = await self._call_tools(response) self._messages.append(response) if request: self._messages.append(request) @@ -233,7 +232,14 @@ async def _call_tools(self, response: ModelResponse) -> ModelRequest | None: if not tool_calls: return None - results = await asyncio.gather(*[self._call_tool(t) for t in tool_calls]) + signal = any(t.tool_name != "end_turn" for t in tool_calls) + if signal: + await self._set_status("tool_call") + try: + results = await asyncio.gather(*[self._call_tool(t) for t in tool_calls]) + finally: + if signal: + await self._set_status(None) parts: list[ModelRequestPart] = [tool_return for tool_return, _ in results] parts.extend(user_part for _, user_part in results if user_part) From fc316e49939ff48710342c26cfad127ef9483547 Mon Sep 17 00:00:00 2001 From: dbrockmann Date: Wed, 11 Mar 2026 20:21:46 +0000 Subject: [PATCH 6/7] feat: client ui update logic --- client/joinly_client/client.py | 43 +++++++++++++++++++++++++++++++++- client/joinly_client/types.py | 6 +++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/client/joinly_client/client.py b/client/joinly_client/client.py index 588524e..7ca6061 100644 --- a/client/joinly_client/client.py +++ b/client/joinly_client/client.py @@ -11,7 +11,7 @@ from fastmcp.client.transports import StreamableHttpTransport from mcp import ClientSession, McpError, ResourceUpdatedNotification, ServerNotification from mcp.types import Tool -from pydantic import AnyUrl +from pydantic import AnyUrl, BaseModel from joinly_client.types import ( MeetingChatHistory, @@ -19,6 +19,9 @@ SpeakerRole, Transcript, TranscriptSegment, + UIAnimation, + UIAnimationContent, + UIUpdate, Usage, VideoSnapshot, ) @@ -26,6 +29,12 @@ logger = logging.getLogger(__name__) + +class _UIUpdateNotification(BaseModel): + method: str = "notifications/joinly_ui_update" + params: UIUpdate | None = None + + TRANSCRIPT_URL = AnyUrl("transcript://live") SEGMENTS_URL = AnyUrl("transcript://live/segments") USAGE_URL = AnyUrl("usage://current") @@ -467,3 +476,35 @@ async def unmute(self) -> None: raise RuntimeError(msg) await self.client.call_tool("unmute_yourself") + + @property + def supports_ui_update(self) -> bool: + """Check if the server supports joinly_ui_update notifications.""" + caps = self.client.initialize_result.capabilities + return bool(caps.experimental and "joinly_ui_update" in caps.experimental) + + async def on_agent_status(self, status: str | None) -> None: + """Map an agent status to a UI animation.""" + _map: dict[str, UIAnimation] = {"llm_call": "thinking", "tool_call": "busy"} + await self.set_ui_animation(_map.get(status or "")) + + async def set_ui_animation(self, animation: UIAnimation | None) -> None: + """Set a UI animation by name, or clear overlay with None.""" + await self.send_ui_update( + UIUpdate(content=UIAnimationContent(animation=animation)) + ) + + async def send_ui_update(self, update: UIUpdate) -> None: + """Send a UI update notification to the server. + + Does nothing if the server does not advertise the joinly_ui_update + experimental capability. + + Args: + update: The UI update to send. + """ + if not self.supports_ui_update: + return + await self.session.send_notification( + _UIUpdateNotification(params=update) # type: ignore[arg-type] + ) diff --git a/client/joinly_client/types.py b/client/joinly_client/types.py index adf463e..cc6afde 100644 --- a/client/joinly_client/types.py +++ b/client/joinly_client/types.py @@ -12,6 +12,9 @@ SpeakerRole, Transcript, TranscriptSegment, + UIAnimation, + UIAnimationContent, + UIUpdate, Usage, VideoSnapshot, ) @@ -26,6 +29,9 @@ "SpeakerRole", "Transcript", "TranscriptSegment", + "UIAnimation", + "UIAnimationContent", + "UIUpdate", "Usage", "VideoSnapshot", ] From 4c569d190f71e90571c539e05057f2553d55d82f Mon Sep 17 00:00:00 2001 From: dbrockmann Date: Wed, 11 Mar 2026 20:22:07 +0000 Subject: [PATCH 7/7] feat: add wiring helper to client --- client/joinly_client/client.py | 31 +++++++++++++++++++++++++++++++ client/joinly_client/main.py | 4 +--- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/client/joinly_client/client.py b/client/joinly_client/client.py index 7ca6061..9542b3e 100644 --- a/client/joinly_client/client.py +++ b/client/joinly_client/client.py @@ -17,6 +17,7 @@ MeetingChatHistory, MeetingParticipantList, SpeakerRole, + ToolExecutor, Transcript, TranscriptSegment, UIAnimation, @@ -508,3 +509,33 @@ async def send_ui_update(self, update: UIUpdate) -> None: await self.session.send_notification( _UIUpdateNotification(params=update) # type: ignore[arg-type] ) + + def create_agent( + self, + llm: Any, # noqa: ANN401 + tools: list[Any], + tool_executor: ToolExecutor, + **kwargs: Any, # noqa: ANN401 + ) -> Any: # noqa: ANN401 + """Create a ConversationalToolAgent wired to this client. + + Connects the status callback and registers the agent's utterance + handler. + + Args: + llm: The language model to use. + tools: Tool definitions for the agent. + tool_executor: Callable that executes tool calls. + **kwargs: Forwarded to ``ConversationalToolAgent``. + """ + from joinly_client.agent import ConversationalToolAgent + + agent = ConversationalToolAgent( + llm, + tools, + tool_executor, + on_status=self.on_agent_status, + **kwargs, + ) + self.add_utterance_callback(agent.on_utterance) + return agent diff --git a/client/joinly_client/main.py b/client/joinly_client/main.py index 6acac9e..f5e2ec1 100644 --- a/client/joinly_client/main.py +++ b/client/joinly_client/main.py @@ -9,7 +9,6 @@ from dotenv import load_dotenv from fastmcp import Client, FastMCP -from joinly_client.agent import ConversationalToolAgent from joinly_client.client import JoinlyClient from joinly_client.types import McpClientConfig, TranscriptSegment from joinly_client.utils import get_llm, get_prompt, load_tools @@ -371,7 +370,7 @@ async def log_segments(segments: list[TranscriptSegment]) -> None: }, } ) - agent = ConversationalToolAgent( + agent = client.create_agent( llm, tools, tool_executor, @@ -381,7 +380,6 @@ async def log_segments(segments: list[TranscriptSegment]) -> None: name=client.name, ), ) - client.add_utterance_callback(agent.on_utterance) async with agent: await client.join_meeting(meeting_url) try: