From 665f20d8ed92bc3aea8987c8a0d9259173305679 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 31 Jul 2026 00:53:51 +0000 Subject: [PATCH 1/2] =?UTF-8?q?feat(desktop):=20gated=20ui=5Fscreenshot=20?= =?UTF-8?q?=E2=80=94=20the=20visual=20representation=20(D3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The desktop UI's third representation (ADR-062 D-4): pixels for the residue structure cannot answer — a rendering bug, a layout question, "why does this look wrong". It is also the most sensitive artifact the app can emit, so three rules bind every call and all three are decided in one electron-free module (uicapture.ts): - REFUSAL BY TABLE. ui_policy.ts's `capture` column decides, reading the same projected focus snapshot the agent can already see. A split refuses if EITHER pane refuses, so the vault cannot ride along in the half the user is not focused on; an undeclared surface and an unpublished screen refuse too — we decline what we cannot classify. Guests go through the same table via surfaceForPartition, so a new partition opts in by naming its row instead of inheriting one. - PER-CALL APPROVAL, ALWAYS. A `desktop_action` attention card, never a session grant. The hub raises it for relayed calls (D5); a LOCAL call never touches the hub, so the desktop raises the same kind itself and parks on it — 2 minutes, not the hub's 10, because this leg holds an MCP call open over stdio. `via: 'hub'` skips the local card, or a remote agent would be double-prompted. Signed out means no way to ask, which means refused — never captured unasked. - FAIL CLOSED. No answer in the window, a dismissed row, an unreadable resolution: all deny. The window leg re-checks the refusal after the park, because the approval was for what was on screen when the card went up. Read SCOPE, action CLASS: the tool sits in READ_TOOLS so the local kimi-code loop (read token only) can reach it — the consent that governs a capture is the card, not the spawn flag — but it audits on every leg (local browser reads do not) and hub-mirrors on every leg (hub-leg reads are ring-only). DESKTOP_ACTION_TOOL_NAMES is that distinction, in one place. The AttentionDock grows a desktop_action branch showing what was asked for — surfaces, or the fragment-stripped guest URL — with Allow once / Reject and a line saying why there is no session option. 21 new node --test cases (283 pass). The Electron halves (capturePage, the hub approval round trip) are unexercised here — no display, no Electron binary — and need the desktop/Playwright pass. Co-Authored-By: Claude Opus 5 (1M context) --- desktop/electron/src/browserbridge.test.ts | 19 +- desktop/electron/src/browserbridge.ts | 111 +++++++- desktop/electron/src/browserbridge_host.ts | 26 ++ desktop/electron/src/desktopui.ts | 7 + desktop/electron/src/main.ts | 5 + desktop/electron/src/uicapture.test.ts | 147 +++++++++++ desktop/electron/src/uicapture.ts | 187 ++++++++++++++ desktop/electron/src/uicapture_host.ts | 241 ++++++++++++++++++ desktop/electron/src/uiscreenshot.test.ts | 232 +++++++++++++++++ desktop/src/i18n/index.ts | 4 + desktop/src/surfaces/AttentionDock.tsx | 25 ++ docs/plans/desktop-ui-context-and-pointing.md | 72 ++++-- 12 files changed, 1037 insertions(+), 39 deletions(-) create mode 100644 desktop/electron/src/uicapture.test.ts create mode 100644 desktop/electron/src/uicapture.ts create mode 100644 desktop/electron/src/uicapture_host.ts create mode 100644 desktop/electron/src/uiscreenshot.test.ts diff --git a/desktop/electron/src/browserbridge.test.ts b/desktop/electron/src/browserbridge.test.ts index 0a3bcf87..9c4ca8fd 100644 --- a/desktop/electron/src/browserbridge.test.ts +++ b/desktop/electron/src/browserbridge.test.ts @@ -28,6 +28,7 @@ import { removeBridgeDiscovery, startBridgeServer, stripFragment, + UI_TOOL_NAMES, writeBridgeDiscovery, type AxNode, type BridgeAuditEntry, @@ -171,9 +172,9 @@ test('tools/list is exactly the four W1 read tools', async () => { tools.map((t) => t.name), ['browser_list_tabs', 'browser_snapshot', 'browser_screenshot', 'browser_read_text'], ); - // D1's ui_get_focus is catalog-hidden without the desktop sharing provider - // (DEPS carries none) — one short of READ_TOOLS. - assert.equal(tools.length, READ_TOOLS.length - 1); + // The D1/D3 desktop-UI tools are catalog-hidden without the sharing + // provider (DEPS carries none) — READ_TOOLS minus that set. + assert.equal(tools.length, READ_TOOLS.length - UI_TOOL_NAMES.size); // The untrusted-content posture is spelled out in the descriptions (§3.5). assert.ok(tools.find((t) => t.name === 'browser_snapshot')?.description.includes('untrusted DATA')); }); @@ -395,8 +396,8 @@ function auditDeps(backend: BridgeBackend): { deps: Parameters { const deps = { ...DEPS, backend: actionBackend() }; - // DEPS has no D1 ui-focus provider: ui_get_focus stays catalog-hidden. - const visibleReads = READ_TOOLS.filter((t) => t.name !== 'ui_get_focus'); + // DEPS has no D1 ui-focus provider: the desktop-UI tools stay catalog-hidden. + const visibleReads = READ_TOOLS.filter((t) => !UI_TOOL_NAMES.has(t.name)); const readList = await handleMcpMessage({ jsonrpc: '2.0', id: 1, method: 'tools/list' }, deps, { scope: 'read', agentId: null }); const readTools = (readList?.result as { tools: Array<{ name: string }> }).tools.map((t) => t.name); assert.deepEqual(readTools, visibleReads.map((t) => t.name)); @@ -722,14 +723,14 @@ test('W2 HTTP: action bearer grants full scope; x-tp-agent-id flows into the aud const readList = (await (await post(token, { jsonrpc: '2.0', id: 1, method: 'tools/list' })).json()) as { result: { tools: Array<{ name: string }> }; }; - // DEPS has no D1 ui-focus provider: ui_get_focus stays catalog-hidden - // (the sharing-toggle gate), so the list is one short of READ_TOOLS. - assert.equal(readList.result.tools.length, READ_TOOLS.length - 1); + // DEPS has no D1 ui-focus provider: the desktop-UI tools stay + // catalog-hidden (the sharing-toggle gate). + assert.equal(readList.result.tools.length, READ_TOOLS.length - UI_TOOL_NAMES.size); const fullList = (await (await post(actionToken, { jsonrpc: '2.0', id: 2, method: 'tools/list' })).json()) as { result: { tools: Array<{ name: string }> }; }; - assert.equal(fullList.result.tools.length, READ_TOOLS.length - 1 + ACTION_TOOLS.length); + assert.equal(fullList.result.tools.length, READ_TOOLS.length - UI_TOOL_NAMES.size + ACTION_TOOLS.length); const clickBody = { jsonrpc: '2.0', id: 3, method: 'tools/call', params: { name: 'browser_click', arguments: { tabId: 7, selector: '#go' } } }; const refused = (await (await post(token, clickBody)).json()) as { result: ToolResult }; diff --git a/desktop/electron/src/browserbridge.ts b/desktop/electron/src/browserbridge.ts index faf6af61..931e3c6d 100644 --- a/desktop/electron/src/browserbridge.ts +++ b/desktop/electron/src/browserbridge.ts @@ -47,6 +47,7 @@ import type { AddressInfo } from 'node:net'; // Explicit .ts extension: this module runs under `node --test`'s strip-only // loader (which resolves like plain ESM) as well as esbuild. import { partitionPolicy } from './webtab_policy.ts'; +import { parseScreenshotArgs } from './uicapture.ts'; // ── Targets / backend ──────────────────────────────────────────────────────── @@ -420,8 +421,34 @@ export const READ_TOOLS: readonly McpToolDef[] = [ 'What the TermiPod desktop user is currently looking at: the workbench surface(s) on screen plus focus state (open tab, focused agent, Inspect file + selection, terminal pane) as a compact JSON snapshot with captured_at. With a split, `surface` is the PRIMARY pane, `secondary` the pinned pane, and `active_pane` names the pane the user is in — resolve "this/here" against the active pane, not `surface` alone. Ids, paths and fragment-stripped URLs only — never message bodies, vault material, or settings values. Call this when the user references what is on their screen ("this", "here", "what I\'m looking at", "why is this failing") or when grounding in the user\'s current view would materially change the answer; do NOT call by default on every turn.', inputSchema: { type: 'object', properties: {}, additionalProperties: false }, }, + // D3 (plan §3.3, ADR-062 D-4): the VISUAL representation — pixels for the + // residue structure cannot answer. Read-SCOPED on purpose (the local + // kimi-code loop holds only the read token) but action-CLASS in every other + // respect: per-call approval, no session grant, audited + hub-mirrored. + { + name: 'ui_screenshot', + description: + 'Capture a PNG screenshot of the TermiPod desktop window, or of one embedded browser tab (tabId from browser_list_tabs). EVERY call raises an approval card the desktop user must accept — there is no standing grant, so use this only when pixels are the answer: a rendering bug, a layout question, "why does this look wrong". For what the user is looking at, ui_get_focus is cheaper, precise and needs no approval; for page content, browser_snapshot. Captures are refused outright while a sensitive surface (vault, settings) is on screen.', + inputSchema: { + type: 'object', + properties: { + tabId: { type: 'integer', description: 'Capture this embedded tab instead of the whole desktop window.' }, + }, + additionalProperties: false, + }, + }, ]; +/// The D1/D3 desktop-UI tools, which the sharing toggle gates as a set: off +/// means neither appears in any catalog and both refuse on call. +export const UI_TOOL_NAMES: ReadonlySet = new Set(['ui_get_focus', 'ui_screenshot']); + +/// Desktop-UI tools that are ACTION class despite living in READ_TOOLS (the +/// bearer scope they need and the consent they need are different questions — +/// see the ui_screenshot definition). They audit + hub-mirror like an action +/// on every leg, and their own handler owns the approval gate. +export const DESKTOP_ACTION_TOOL_NAMES: ReadonlySet = new Set(['ui_screenshot']); + /// D1: the MCP resource uri mirroring ui_get_focus (ADR-062 D-6). list + read /// only — subscriptions are deliberately NOT implemented: the relay forwards /// frames verbatim so notification delivery would work transport-wise, but @@ -547,11 +574,14 @@ const ACTION_TOOL_NAMES: ReadonlySet = new Set(ACTION_TOOLS.map((t) => t const READ_TOOL_NAMES: ReadonlySet = new Set(READ_TOOLS.map((t) => t.name)); /// Whether an audit entry should be mirrored to the hub as an agent event. -/// Actions always mirror (W2 contract). Hub-leg READS stay ring-only: the -/// hub routed the call, so a mirror row adds no information — the ring -/// entry exists purely so Settings → Remote driving can show (and revoke) -/// read-only remote sessions. +/// Actions always mirror (W2 contract) — including the desktop-UI action +/// tools, which sit in READ_TOOLS for scope reasons but are actions for +/// consent and audit (D3). Hub-leg READS stay ring-only: the hub routed the +/// call, so a mirror row adds no information — the ring entry exists purely +/// so Settings → Remote driving can show (and revoke) read-only remote +/// sessions. export function shouldMirrorAudit(entry: BridgeAuditEntry): boolean { + if (DESKTOP_ACTION_TOOL_NAMES.has(entry.tool)) return true; return !(entry.via === 'hub' && READ_TOOL_NAMES.has(entry.tool)); } @@ -574,8 +604,34 @@ export interface McpServerDeps { /// D1: the main-side focus cache (the renderer's last projected push), or /// null before the first push — the tool then answers an empty snapshot. getUiFocus?: () => Record | null; + /// D3: the gated screenshot (uicapture_host.ts). Absent means the capability + /// is not wired at all, which refuses like any other unavailable tool. The + /// provider owns the policy refusal AND the per-call approval — this module + /// only resolves the target and shapes the result. + captureUi?: (req: UiCaptureRequest) => Promise; +} + +/// One `ui_screenshot` request, with the target already resolved against the +/// live guest registry (so a guessed tabId can never name the app:// shell) +/// and the caller's identity attached for the approval card + audit. +export interface UiCaptureRequest { + /// null = the desktop window itself. + tabId: number | null; + /// The guest's fragment-stripped URL, when capturing a tab. + url: string | null; + /// The guest's partition, when capturing a tab — the policy key. + partition: string | null; + agentId: string; + agentHandle: string; + /// 'hub' calls arrive pre-approved (the hub raises the desktop_action card + /// before routing — D5), so the desktop must not raise a second one. + via: 'local' | 'hub'; } +export type UiCaptureResult = + | { ok: true; data_b64: string; width: number; height: number } + | { ok: false; code: string; message: string }; + interface JsonRpcRequest { jsonrpc?: string; id?: unknown; @@ -893,7 +949,7 @@ async function uploadFiles(deps: McpServerDeps, target: BridgeTarget, args: Reco } } -async function runTool(deps: McpServerDeps, name: string, args: Record): Promise { +async function runTool(deps: McpServerDeps, ctx: BridgeRequestContext, name: string, args: Record): Promise { const { backend } = deps; switch (name) { case 'browser_list_tabs': { @@ -1039,6 +1095,33 @@ async function runTool(deps: McpServerDeps, name: string, args: Record): Promise { - if (!ACTION_TOOL_NAMES.has(name) && ctx.via !== 'hub') return runTool(deps, name, args); + // D3: a desktop-UI action tool is audited on EVERY leg, local included — a + // screenshot of the user's own screen is exactly what the Settings audit + // view exists to show, and unlike a browser read it is neither cheap nor + // frequent, so the ring will not churn. + if (!ACTION_TOOL_NAMES.has(name) && !DESKTOP_ACTION_TOOL_NAMES.has(name) && ctx.via !== 'hub') { + return runTool(deps, ctx, name, args); + } const tabId = typeof args.tabId === 'number' && Number.isInteger(args.tabId) ? args.tabId : null; const target = tabId === null ? undefined : deps.backend.listTargets().find((t) => t.tabId === tabId); const entry: BridgeAuditEntry = { @@ -1077,7 +1166,7 @@ async function callTool(deps: McpServerDeps, ctx: BridgeRequestContext, name: st }; if (ctx.agentHandle !== undefined) entry.agent_handle = ctx.agentHandle; try { - const out = await runTool(deps, name, args); + const out = await runTool(deps, ctx, name, args); entry.ok = true; return out; } catch (e) { @@ -1234,10 +1323,10 @@ export async function handleMcpMessage( return rpcResult(id, {}); case 'tools/list': { // Scope gate #1: a read-scoped session never SEES the action tools. - // Consent gate (D1): ui_get_focus is catalog-visible only while the - // desktop's UI context sharing toggle is on. - const reads = - deps.uiFocusAvailable?.() === true ? READ_TOOLS : READ_TOOLS.filter((t) => t.name !== 'ui_get_focus'); + // Consent gate (D1/D3): the desktop-UI tools are catalog-visible only + // while the sharing toggle is on — off means no publisher and no tool + // in any catalog. + const reads = deps.uiFocusAvailable?.() === true ? READ_TOOLS : READ_TOOLS.filter((t) => !UI_TOOL_NAMES.has(t.name)); return rpcResult(id, { tools: ctx.scope === 'full' ? [...reads, ...ACTION_TOOLS] : reads }); } case 'resources/list': { diff --git a/desktop/electron/src/browserbridge_host.ts b/desktop/electron/src/browserbridge_host.ts index 310f2e93..e65872cc 100644 --- a/desktop/electron/src/browserbridge_host.ts +++ b/desktop/electron/src/browserbridge_host.ts @@ -62,6 +62,8 @@ import { type HubInvokePayload, type HubInvokeResult, type McpServerDeps, + type UiCaptureRequest, + type UiCaptureResult, } from './browserbridge'; import { policyForGuest } from './webtab'; import { keychainGetLocal } from './ipc/keychain'; @@ -273,6 +275,24 @@ export function setUiFocusProvider(p: UiFocusProvider): void { uiFocusProvider = p; } +/// D3: the gated screenshot provider (uicapture_host.ts), plugged in the same +/// way and for the same reason — it needs the hub context and the focus cache +/// this module and desktopui.ts own, so it registers instead of being imported. +type UiCaptureProvider = (req: UiCaptureRequest) => Promise; + +let uiCaptureProvider: UiCaptureProvider | null = null; + +export function setUiCaptureProvider(p: UiCaptureProvider): void { + uiCaptureProvider = p; +} + +/// The hub identity the audit mirror and the D3 approval card both post as. +/// Non-secret: the bearer is fetched from the keychain per use, never held +/// here (and never handed across IPC). +export function currentHubContext(): { baseUrl: string; teamId: string; profileId: string } | null { + return hubContext; +} + // ── Enable/disable lifecycle ───────────────────────────────────────────────── let server: BridgeServer | null = null; @@ -298,6 +318,12 @@ async function enable(): Promise { // ui_get_focus on the next tools/list without a server restart. uiFocusAvailable: () => uiFocusProvider?.available() ?? false, getUiFocus: () => uiFocusProvider?.snapshot() ?? null, + // D3: read through the holder on every call, so a provider registered + // after the server started (module init order) is still reachable. + captureUi: (req) => + uiCaptureProvider !== null + ? uiCaptureProvider(req) + : Promise.resolve({ ok: false, code: 'UI_UNAVAILABLE', message: 'the capture provider is not wired' }), }; server = await startBridgeServer({ ...mcpDeps, token, actionToken }); writeBridgeDiscovery(os.homedir(), { diff --git a/desktop/electron/src/desktopui.ts b/desktop/electron/src/desktopui.ts index 5d33cfa4..b1403c4e 100644 --- a/desktop/electron/src/desktopui.ts +++ b/desktop/electron/src/desktopui.ts @@ -40,6 +40,13 @@ export function isUiSharingEnabled(): boolean { /// that crosses IPC), so main's job is just to hold and serve it. let focusCache: Record | null = null; +/// D3's capture path reads the same cache to learn which surfaces are on +/// screen — the refusal decision must be made from the SAME view of the +/// screen the agent can already see, not from a second source of truth. +export function currentFocusSnapshot(): Record | null { + return sharingEnabled ? focusCache : null; +} + /// A runaway publisher can't fill memory: snapshots are ref-sized (a handful /// of ids/paths); anything bigger is a bug, not a bigger allowance. const MAX_SNAPSHOT_BYTES = 16384; diff --git a/desktop/electron/src/main.ts b/desktop/electron/src/main.ts index 53a2ad86..f8bbb31c 100644 --- a/desktop/electron/src/main.ts +++ b/desktop/electron/src/main.ts @@ -28,6 +28,9 @@ import { disposeKimiWebWin } from './kimiwebwin'; import { disposeRerun } from './rerun'; import { disposeBrowserBridge } from './browserbridge_host'; import { disposeAnnotations } from './annotation_host'; +// D3: importing the module registers the gated-screenshot provider on the +// bridge server; setShellWindow tells it which window "the desktop" means. +import { setShellWindow } from './uicapture_host'; import { initEvents } from './events'; // The frontend build. In dev (`electron .` from desktop/electron) it resolves to @@ -72,6 +75,7 @@ function createWindow(): void { }, }); mainWindow = win; + setShellWindow(win); // A `window.open` from any embedded content would spawn a child window that // INHERITS this window's webPreferences — preload included — handing // `__ELECTRON_BRIDGE__` and the whole command allowlist to remote content. @@ -89,6 +93,7 @@ function createWindow(): void { win.once('ready-to-show', () => win.show()); win.on('closed', () => { if (mainWindow === win) mainWindow = null; + setShellWindow(null); }); void win.loadURL(`${APP_ORIGIN}/index.html`); } diff --git a/desktop/electron/src/uicapture.test.ts b/desktop/electron/src/uicapture.test.ts new file mode 100644 index 00000000..30031dfe --- /dev/null +++ b/desktop/electron/src/uicapture.test.ts @@ -0,0 +1,147 @@ +/// Tests for the gated-screenshot core (D3 — plan §3.3, ADR-062 D-3/D-4). +/// The three rules that make a screenshot safe are all decided in uicapture.ts, +/// so they are all provable here without Electron: refusal by table, per-call +/// approval (never a session grant), and fail-closed decision reading. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { UI_POLICY, uiPolicyFor } from '../../src/state/ui_policy.ts'; +import { + captureApprovalCard, + captureDenialMessage, + captureRefusal, + parseScreenshotArgs, + readCaptureDecision, + surfaceForPartition, + visibleSurfaces, +} from './uicapture.ts'; + +// ── What is on screen ──────────────────────────────────────────────────────── + +test('visibleSurfaces reads both panes, and answers "unknown" rather than "empty"', () => { + assert.deepEqual(visibleSurfaces({ surface: 'read', captured_at: 'x' }), ['read']); + assert.deepEqual( + visibleSurfaces({ surface: 'read', secondary: { surface: 'debug' }, active_pane: 'secondary', captured_at: 'x' }), + ['read', 'debug'], + ); + // No snapshot yet, or one without a usable surface: null, NOT [] — the + // caller must be able to tell "nothing sensitive on screen" from "we have no + // idea what is on screen", because those get opposite answers. + assert.equal(visibleSurfaces(null), null); + assert.equal(visibleSurfaces({ captured_at: 'x' }), null); + assert.equal(visibleSurfaces({ surface: '', captured_at: 'x' }), null); + // A malformed secondary is ignored rather than fabricating a pane. + assert.deepEqual(visibleSurfaces({ surface: 'read', secondary: { nope: 1 } }), ['read']); +}); + +// ── Refusal by table ───────────────────────────────────────────────────────── + +test('captureRefusal: the policy table decides, and refuses what it does not declare', () => { + assert.equal(captureRefusal(['read']), null); + assert.equal(captureRefusal(['read', 'debug', 'replay']), null); + // A pixel capture of settings shows its VALUES even though its snapshot row + // emits only the surface id — the columns are independent (ADR-062 D-3). + assert.equal(captureRefusal(['settings']), 'settings'); + assert.equal(captureRefusal(['vault']), 'vault'); + // The gate covers BOTH panes: a split cannot launder the vault past it by + // putting it in the half the user is not focused on. + assert.equal(captureRefusal(['read', 'vault']), 'vault'); + // Default-correct for the unknown: a surface with no row is not capturable. + assert.equal(captureRefusal(['surface-added-next-quarter']), 'surface-added-next-quarter'); +}); + +test('the sensitive surfaces are exactly the ones that refuse pixels', () => { + const refusing = Object.entries(UI_POLICY) + .filter(([, row]) => row.capture === 'refuse') + .map(([surface]) => surface) + .sort(); + // Stated as an invariant, not as a snapshot of today's table: adding a row + // that refuses capture is a deliberate privacy decision and should have to + // touch this line. + assert.deepEqual(refusing, ['settings', 'vault']); +}); + +test('surfaceForPartition maps every guest kind onto a real policy row', () => { + assert.equal(surfaceForPartition('persist:webtab'), 'read'); + assert.equal(surfaceForPartition('kimiweb'), 'kimiweb'); + assert.equal(surfaceForPartition('rerunweb'), 'replay'); + // An unlisted partition opts IN by naming its surface — it never inherits + // one (the webtab_policy discipline). + assert.equal(surfaceForPartition('persist:something-new'), null); + assert.equal(surfaceForPartition(null), null); + // Every mapping must land on a declared row, or the guest leg would refuse + // for a reason nobody intended. + for (const partition of ['persist:webtab', 'kimiweb', 'rerunweb']) { + const surface = surfaceForPartition(partition); + assert.ok(surface !== null && uiPolicyFor(surface) !== null, partition); + } +}); + +// ── Arguments ──────────────────────────────────────────────────────────────── + +test('parseScreenshotArgs: absent tabId means the window; junk is refused', () => { + assert.deepEqual(parseScreenshotArgs({}), { tabId: null }); + assert.deepEqual(parseScreenshotArgs({ tabId: null }), { tabId: null }); + assert.deepEqual(parseScreenshotArgs({ tabId: 7 }), { tabId: 7 }); + assert.ok('error' in parseScreenshotArgs({ tabId: '7' })); + assert.ok('error' in parseScreenshotArgs({ tabId: 1.5 })); +}); + +// ── The approval card ──────────────────────────────────────────────────────── + +test('the approval card names what was asked for and carries no pixels', () => { + const card = captureApprovalCard({ + agentId: 'ag_1', + agentHandle: 'kimi-1', + scope: 'window', + surfaces: ['read', 'debug'], + url: null, + }); + assert.match(card.summary, /kimi-1/); + assert.match(card.summary, /read \+ debug/); + assert.equal(card.payload.tool, 'ui_screenshot'); + assert.deepEqual(card.payload.surfaces, ['read', 'debug']); + // §3.3: screenshots never get a standing grant, so the card says so in the + // payload as well as by omitting the session option in the UI. + assert.equal(card.payload.session_grant, false); + // The payload is a REFERENCE to what was requested — the image does not + // exist yet, and nothing content-shaped may ride along (ADR-062 D-2). + for (const key of Object.keys(card.payload)) { + assert.ok(!['image', 'data', 'data_b64', 'png', 'preview'].includes(key), key); + } +}); + +test('the card falls back to the agent id, then to a neutral subject', () => { + const byId = captureApprovalCard({ agentId: 'ag_1', agentHandle: '', scope: 'window', surfaces: ['read'], url: null }); + assert.match(byId.summary, /^ag_1 wants/); + const anon = captureApprovalCard({ agentId: '', agentHandle: '', scope: 'tab', surfaces: ['read'], url: 'https://a.b/c' }); + assert.match(anon.summary, /^An agent wants/); + assert.match(anon.summary, /https:\/\/a\.b\/c/); +}); + +// ── Fail-closed decisions ──────────────────────────────────────────────────── + +test('readCaptureDecision: only an explicit trailing approve approves', () => { + assert.equal(readCaptureDecision({ status: 'open', decisions: [] }), 'pending'); + // Quorum not yet met: an approve on a still-open row is not a decision. + assert.equal(readCaptureDecision({ status: 'open', decisions: [{ decision: 'approve' }] }), 'pending'); + assert.equal(readCaptureDecision({ status: 'resolved', decisions: [{ decision: 'approve' }] }), 'approve'); + assert.equal(readCaptureDecision({ status: 'resolved', decisions: [{ decision: 'reject' }] }), 'deny'); + // Dismissed through /resolve — resolved with no decision at all. Denies. + assert.equal(readCaptureDecision({ status: 'resolved' }), 'deny'); + assert.equal(readCaptureDecision({ status: 'resolved', decisions: [] }), 'deny'); + // The LAST decision wins (an override after a reject, say). + assert.equal(readCaptureDecision({ status: 'resolved', decisions: [{ decision: 'reject' }, { decision: 'approve' }] }), 'approve'); + assert.equal(readCaptureDecision({ status: 'resolved', decisions: [{ decision: 'approve' }, { decision: 'reject' }] }), 'deny'); + // Shapes we cannot read: an unparseable resolved row denies; a body we + // cannot read at all is not evidence of anything, so it keeps waiting. + assert.equal(readCaptureDecision({ status: 'resolved', decisions: 'nope' }), 'deny'); + assert.equal(readCaptureDecision({ status: 'resolved', decisions: [42] }), 'deny'); + assert.equal(readCaptureDecision(null), 'pending'); + assert.equal(readCaptureDecision('nope'), 'pending'); +}); + +test('denial messages point the agent at the cheaper representation', () => { + assert.match(captureDenialMessage('denied'), /denied/); + assert.match(captureDenialMessage('timeout'), /approval window/); + assert.match(captureDenialMessage('unavailable'), /ui_get_focus/); +}); diff --git a/desktop/electron/src/uicapture.ts b/desktop/electron/src/uicapture.ts new file mode 100644 index 00000000..5f096790 --- /dev/null +++ b/desktop/electron/src/uicapture.ts @@ -0,0 +1,187 @@ +/// Gated desktop screenshot — electron-free core (D3 — +/// docs/plans/desktop-ui-context-and-pointing.md §3.3, ADR-062 D-4). +/// +/// The visual representation of the desktop UI entity: pixels for the residue +/// structure cannot answer (a rendering bug, a layout question). It is the +/// most sensitive artifact the app can emit — a frame of everything the user +/// sees — so three rules bind every call, and all three are decided here: +/// +/// 1. REFUSAL BY TABLE. A capture is refused outright when any surface on +/// screen carries `capture: 'refuse'` in ui_policy.ts (vault, settings — +/// a pixel capture of settings shows its VALUES even though its snapshot +/// row emits only the surface id). An undeclared surface refuses too, and +/// so does an unknown screen (no focus snapshot yet): we refuse what we +/// cannot classify rather than capture it (ADR-062 D-3). +/// 2. PER-CALL APPROVAL, ALWAYS. Never a session grant — §3.3 is explicit +/// that screenshots get no standing consent. The card is a hub +/// attention_item of kind `desktop_action`, so the decision lands in the +/// director's inbox on every client, and the local leg raises it itself +/// (the hub leg's card is raised hub-side — D5). +/// 3. FAIL CLOSED. No decision inside the window, a dismissed row, a +/// resolution we cannot read: all deny. +/// +/// The live halves (capturePage, the hub round trip) are uicapture_host.ts; +/// everything below is pure so `node --test` covers the policy without +/// Electron. +import { uiPolicyFor } from '../../src/state/ui_policy.ts'; +import { KIMIWEB_PARTITION, RERUNWEB_PARTITION, WEBTAB_PARTITION } from './webtab_policy.ts'; + +// ── The refusal rule ───────────────────────────────────────────────────────── + +/// Which ui_policy row governs pixels of a `` guest. A guest is not a +/// workbench job, but it is painted inside one, and the capture question is +/// the same question — so it is answered by the same table rather than by a +/// second rule living in the capture path (ADR-062 D-3: nothing else in the +/// pipeline makes policy decisions). +/// +/// An unlisted partition maps to no row and therefore refuses: a future guest +/// kind opts in by naming its surface here, exactly as `webtab_policy` makes a +/// new partition state its `bridge` capability instead of inheriting one. +export function surfaceForPartition(partition: string | null): string | null { + switch (partition) { + case WEBTAB_PARTITION: + return 'read'; + case KIMIWEB_PARTITION: + return 'kimiweb'; + case RERUNWEB_PARTITION: + return 'replay'; + default: + return null; + } +} + +/// The surfaces a full-window capture would expose: the primary pane plus the +/// pinned one when a split is on screen. Read from the SAME projected focus +/// snapshot the agent can already see (desktopui.ts's cache) — no second +/// source of truth about what is on screen. +/// +/// Returns null when there is no usable snapshot: the answer is "unknown", +/// which the caller must treat as refusal, not as "nothing sensitive". +export function visibleSurfaces(snapshot: Record | null): string[] | null { + if (snapshot === null) return null; + const primary = snapshot.surface; + if (typeof primary !== 'string' || primary === '') return null; + const out = [primary]; + const secondary = snapshot.secondary; + if (secondary !== null && typeof secondary === 'object' && !Array.isArray(secondary)) { + const s = (secondary as Record).surface; + if (typeof s === 'string' && s !== '') out.push(s); + } + return out; +} + +/// The first surface that refuses pixels, or null when every one of them +/// allows. A surface with NO policy row refuses — the table is the allowlist, +/// so an undeclared surface is never capturable (the same posture the +/// projection takes in ui_policy.ts rule 1). +export function captureRefusal(surfaces: readonly string[]): string | null { + for (const surface of surfaces) { + if (uiPolicyFor(surface)?.capture !== 'allow') return surface; + } + return null; +} + +// ── Arguments ──────────────────────────────────────────────────────────────── + +/// `ui_screenshot` takes at most a tabId: absent means the desktop window +/// itself, present means that `` guest (validated against the live +/// registry host-side — an id alone never reaches the app:// shell). +export interface UiScreenshotArgs { + tabId: number | null; +} + +export function parseScreenshotArgs(args: Record): UiScreenshotArgs | { error: string } { + const raw = args.tabId; + if (raw === undefined || raw === null) return { tabId: null }; + if (typeof raw !== 'number' || !Number.isInteger(raw)) { + return { error: 'tabId must be an integer from browser_list_tabs (omit it to capture the desktop window)' }; + } + return { tabId: raw }; +} + +// ── The approval card ──────────────────────────────────────────────────────── + +/// How long the tool parks on the card before failing closed. Deliberately +/// SHORTER than the hub's 10-minute browser_action window: this leg is a +/// local MCP tool call held open over stdio, and an agent client that gives up +/// mid-park would leave the user approving a call nobody is waiting for. Two +/// minutes is long enough to notice the card, short enough that a stale one is +/// rare — and a denial is retryable, unlike a capture. +export const UI_CAPTURE_APPROVAL_TIMEOUT_MS = 120_000; + +/// How often the local leg re-reads the card while parked. +export const UI_CAPTURE_POLL_MS = 1500; + +export interface CaptureApprovalRequest { + agentId: string; + agentHandle: string; + /// 'window' (the whole desktop) or 'tab' (one embedded guest). + scope: 'window' | 'tab'; + /// The surfaces on screen, for the window scope; the guest URL for a tab. + surfaces: readonly string[]; + url: string | null; +} + +/// The card's human sentence + its machine payload. The payload carries a +/// UIRef-shaped description of WHAT was asked for — surfaces, tab, url — and +/// never a pixel: the approver decides from the reference, and the image only +/// exists after they say yes. +export function captureApprovalCard(req: CaptureApprovalRequest): { summary: string; payload: Record } { + const who = req.agentHandle !== '' ? req.agentHandle : req.agentId !== '' ? req.agentId : 'An agent'; + const what = + req.scope === 'tab' + ? `the embedded tab ${req.url ?? ''}`.trimEnd() + : req.surfaces.length > 0 + ? `the desktop window (${req.surfaces.join(' + ')})` + : 'the desktop window'; + return { + summary: `${who} wants a screenshot of ${what}`, + payload: { + tool: 'ui_screenshot', + scope: req.scope, + surfaces: [...req.surfaces], + url: req.url, + agent_id: req.agentId, + // Pinned in the payload so the card can never be re-read as a standing + // grant: screenshots are per-call, always (plan §3.3). + session_grant: false, + }, + }; +} + +// ── Reading the decision back ──────────────────────────────────────────────── + +export type CaptureOutcome = 'approve' | 'deny' | 'pending'; + +/// Interpret one `GET /attention/{id}` row. `open` means keep waiting; +/// anything else resolves, and only an explicit trailing `approve` decision +/// approves — a row dismissed through /resolve (no decision at all), a reject, +/// or a shape we cannot parse all deny. +export function readCaptureDecision(row: unknown): CaptureOutcome { + if (row === null || typeof row !== 'object' || Array.isArray(row)) return 'pending'; + const r = row as Record; + const status = typeof r.status === 'string' ? r.status : 'open'; + if (status === 'open') return 'pending'; + const decisions = r.decisions; + if (!Array.isArray(decisions) || decisions.length === 0) return 'deny'; + const last = decisions[decisions.length - 1]; + if (last === null || typeof last !== 'object') return 'deny'; + const decision = (last as Record).decision; + return decision === 'approve' ? 'approve' : 'deny'; +} + +/// The denial sentence an agent sees, by cause. Kept here (not in the host) so +/// the wording is testable and identical on both legs. +export function captureDenialMessage(cause: 'denied' | 'timeout' | 'unavailable'): string { + switch (cause) { + case 'denied': + return 'the desktop user denied this screenshot'; + case 'timeout': + return 'no decision on the screenshot request within the approval window — denied'; + case 'unavailable': + return ( + 'this desktop cannot ask for screenshot approval — it is not signed in to a hub, ' + + 'and ui_screenshot is per-call approved, always. Use ui_get_focus for structure instead' + ); + } +} diff --git a/desktop/electron/src/uicapture_host.ts b/desktop/electron/src/uicapture_host.ts new file mode 100644 index 00000000..f17c29e9 --- /dev/null +++ b/desktop/electron/src/uicapture_host.ts @@ -0,0 +1,241 @@ +/// Gated desktop screenshot — Electron main-process half (D3 — +/// docs/plans/desktop-ui-context-and-pointing.md §3.3). The policy (which +/// surfaces refuse pixels, how a decision reads, what the card says) is the +/// electron-free uicapture.ts; this module owns the live halves: +/// +/// - the CAPTURE: `capturePage` on the shell window or on one `` +/// guest, downscaled to the browser_screenshot discipline (PNG, ≤1568px +/// max edge) and returned as base64 — never written to disk, never +/// cached: the image exists for one tool result (ADR-062 D-7); +/// - the APPROVAL: a hub `desktop_action` attention_item raised per call, +/// polled until it resolves. The LOCAL leg raises it here; a hub-relayed +/// call arrives pre-approved (the hub raises the same card kind before it +/// routes — D5) and must not raise a second one. +/// +/// Fail-closed everywhere: no hub to ask, no answer in the window, an +/// unreadable resolution, an unknown surface on screen — all deny. The tool +/// registration + audit live on the bridge server (browserbridge.ts); this +/// module plugs itself in through setUiCaptureProvider so the import edge +/// stays one-directional. +import { webContents, type BrowserWindow, type NativeImage, type WebContents } from 'electron'; +import { fitWithin, MAX_IMAGE_EDGE } from './annotation'; +import { currentHubContext, setUiCaptureProvider } from './browserbridge_host'; +import { currentFocusSnapshot, isUiSharingEnabled } from './desktopui'; +import { keychainGetLocal } from './ipc/keychain'; +import { + captureApprovalCard, + captureDenialMessage, + captureRefusal, + readCaptureDecision, + surfaceForPartition, + visibleSurfaces, + UI_CAPTURE_APPROVAL_TIMEOUT_MS, + UI_CAPTURE_POLL_MS, + type CaptureOutcome, +} from './uicapture'; +import type { UiCaptureRequest, UiCaptureResult } from './browserbridge'; + +// ── The shell window ───────────────────────────────────────────────────────── + +/// The app's own window, registered by main.ts at creation (and cleared on +/// close). Deliberately NOT discovered via `BrowserWindow.getAllWindows()`: +/// the kimi-web detached window is a BrowserWindow too, and "capture whatever +/// window we found first" is exactly the kind of ambient targeting a +/// screenshot tool must not have. +let shellWindow: BrowserWindow | null = null; + +export function setShellWindow(win: BrowserWindow | null): void { + shellWindow = win; +} + +// ── Guest resolution ───────────────────────────────────────────────────────── + +/// The tabId was already validated against the bridge's live registry +/// (`requireTarget` in browserbridge.ts, so a guessed id can never name the +/// app:// shell); this only re-resolves it to a live webContents. +function liveGuest(tabId: number): WebContents | null { + const wc = webContents.fromId(tabId); + if (wc === undefined || wc.isDestroyed() || wc.getType() !== 'webview') return null; + return wc; +} + +// ── The hub approval round trip ────────────────────────────────────────────── + +interface HubLeg { + baseUrl: string; + teamId: string; + token: string; +} + +/// The hub identity + bearer for the approval card, or null when the desktop +/// is signed out. Same sourcing as the audit mirror: non-secret context pushed +/// by the renderer, the token read from the main-process keychain at use time +/// (never over IPC). +async function hubLeg(): Promise { + const ctx = currentHubContext(); + if (ctx === null) return null; + const token = await keychainGetLocal(`hub_token_${ctx.profileId}`); + if (token === null || token === '') return null; + return { baseUrl: ctx.baseUrl, teamId: ctx.teamId, token }; +} + +function teamUrl(leg: HubLeg, suffix: string): string { + return `${leg.baseUrl}/v1/teams/${encodeURIComponent(leg.teamId)}${suffix}`; +} + +/// Raise the per-call card. Returns its id, or null when the hub refused it — +/// which denies the capture (we never fall back to capturing unasked). +async function raiseCard(leg: HubLeg, summary: string, payload: Record, agentHandle: string): Promise { + try { + const res = await fetch(teamUrl(leg, '/attention'), { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${leg.token}` }, + body: JSON.stringify({ + scope_kind: 'team', + scope_id: leg.teamId, + kind: 'desktop_action', + summary, + severity: 'minor', + actor_handle: agentHandle, + pending_payload: payload, + }), + signal: AbortSignal.timeout(10_000), + }); + if (!res.ok) return null; + const body = (await res.json()) as { id?: unknown }; + return typeof body.id === 'string' && body.id !== '' ? body.id : null; + } catch { + return null; + } +} + +/// Poll one card until it resolves or the window closes. A transport error is +/// NOT a decision — it just costs one poll interval, and the deadline still +/// governs. +async function awaitCard(leg: HubLeg, id: string, deadline: number, now: () => number, sleep: (ms: number) => Promise): Promise { + for (;;) { + if (now() >= deadline) return 'pending'; + await sleep(UI_CAPTURE_POLL_MS); + let outcome: CaptureOutcome = 'pending'; + try { + const res = await fetch(teamUrl(leg, `/attention/${encodeURIComponent(id)}`), { + headers: { authorization: `Bearer ${leg.token}` }, + signal: AbortSignal.timeout(10_000), + }); + if (res.ok) outcome = readCaptureDecision(await res.json()); + } catch { + /* transport hiccup — try again inside the deadline */ + } + if (outcome !== 'pending') return outcome; + } +} + +/// Best-effort tidy-up so an unanswered card does not loiter in the inbox +/// after the agent gave up. `desktop_action` owes no agent reply, so /resolve +/// (the dismiss path) is the right verb. +async function dismissCard(leg: HubLeg, id: string): Promise { + try { + await fetch(teamUrl(leg, `/attention/${encodeURIComponent(id)}/resolve`), { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${leg.token}` }, + body: JSON.stringify({ resolved_by: 'desktop' }), + signal: AbortSignal.timeout(5_000), + }); + } catch { + /* the row stays open; the director can dismiss it by hand */ + } +} + +function sleepMs(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +// ── The capture ────────────────────────────────────────────────────────────── + +function toPngBase64(source: NativeImage): { data_b64: string; width: number; height: number } { + const size = source.getSize(); + const fit = fitWithin(size.width, size.height, MAX_IMAGE_EDGE); + const image = fit.width !== size.width || fit.height !== size.height ? source.resize({ ...fit, quality: 'good' }) : source; + return { data_b64: image.toPNG().toString('base64'), width: fit.width, height: fit.height }; +} + +function refused(surface: string): UiCaptureResult { + return { + ok: false, + code: 'CAPTURE_REFUSED', + message: + surface === '' + ? 'the desktop has not published what is on screen yet — a capture of an unclassified screen is refused' + : `surface '${surface}' refuses pixel capture by policy (ui_policy.ts capture column) — ask for structure instead (ui_get_focus)`, + }; +} + +async function capture(req: UiCaptureRequest): Promise { + if (!isUiSharingEnabled()) { + return { ok: false, code: 'UI_UNAVAILABLE', message: 'UI context sharing is off on the desktop (Settings → Assistant)' }; + } + + // 1. The table decides what may be captured at all — before any approval is + // asked for, so a refused surface never costs the user a card. + const scope: 'window' | 'tab' = req.tabId === null ? 'window' : 'tab'; + let surfaces: string[]; + if (scope === 'tab') { + const surface = surfaceForPartition(req.partition); + if (surface === null) return refused(req.partition ?? ''); + surfaces = [surface]; + } else { + const onScreen = visibleSurfaces(currentFocusSnapshot()); + if (onScreen === null) return refused(''); + surfaces = onScreen; + } + const blocked = captureRefusal(surfaces); + if (blocked !== null) return refused(blocked); + + // 2. Per-call approval. A hub-relayed call already carries one (D5). + if (req.via !== 'hub') { + const leg = await hubLeg(); + if (leg === null) return { ok: false, code: 'UI_APPROVAL_UNAVAILABLE', message: captureDenialMessage('unavailable') }; + const card = captureApprovalCard({ + agentId: req.agentId, + agentHandle: req.agentHandle, + scope, + surfaces, + url: req.url, + }); + const id = await raiseCard(leg, card.summary, card.payload, req.agentHandle); + if (id === null) { + return { ok: false, code: 'UI_APPROVAL_UNAVAILABLE', message: captureDenialMessage('unavailable') }; + } + const outcome = await awaitCard(leg, id, Date.now() + UI_CAPTURE_APPROVAL_TIMEOUT_MS, () => Date.now(), sleepMs); + if (outcome !== 'approve') { + if (outcome === 'pending') void dismissCard(leg, id); + return { ok: false, code: 'CAPTURE_DENIED', message: captureDenialMessage(outcome === 'pending' ? 'timeout' : 'denied') }; + } + } + + // 3. Pixels. Re-checked after the park: the user may have moved to the vault + // while the card sat open, and the approval was for what was on screen + // when it was raised. + if (scope === 'window') { + const after = visibleSurfaces(currentFocusSnapshot()); + if (after === null) return refused(''); + const moved = captureRefusal(after); + if (moved !== null) return refused(moved); + } + + try { + if (scope === 'tab' && req.tabId !== null) { + const guest = liveGuest(req.tabId); + if (guest === null) return { ok: false, code: 'TARGET_GONE', message: `tab ${String(req.tabId)} was destroyed` }; + return { ok: true, ...toPngBase64(await guest.capturePage()) }; + } + if (shellWindow === null || shellWindow.isDestroyed()) { + return { ok: false, code: 'NO_WINDOW', message: 'the desktop window is not open' }; + } + return { ok: true, ...toPngBase64(await shellWindow.webContents.capturePage()) }; + } catch (e) { + return { ok: false, code: 'CAPTURE_FAILED', message: e instanceof Error ? e.message : String(e) }; + } +} + +setUiCaptureProvider(capture); diff --git a/desktop/electron/src/uiscreenshot.test.ts b/desktop/electron/src/uiscreenshot.test.ts new file mode 100644 index 00000000..6ba8332f --- /dev/null +++ b/desktop/electron/src/uiscreenshot.test.ts @@ -0,0 +1,232 @@ +/// Tests for the D3 bridge surface (docs/plans/desktop-ui-context-and-pointing.md +/// §3.3): `ui_screenshot`'s catalog gating, target resolution, error shaping +/// and — the part that is easy to get wrong — the audit posture. The policy +/// itself (what refuses, how a decision reads) is uicapture.test.ts; the +/// capture provider is faked here exactly like the CDP backend. `node --test`. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + dispatchHubInvoke, + handleMcpMessage, + READ_TOOLS, + shouldMirrorAudit, + type BridgeAuditEntry, + type BridgeBackend, + type BridgeTarget, + type McpServerDeps, + type UiCaptureRequest, + type UiCaptureResult, +} from './browserbridge.ts'; + +const KIMIWEB_URL = 'http://127.0.0.1:17331/#token=secret-in-the-fragment'; + +const TABS: BridgeTarget[] = [ + { tabId: 7, url: 'https://arxiv.org/abs/2401.00001', title: 'a paper', partition: 'persist:webtab', bridge: 'full' }, + { tabId: 9, url: KIMIWEB_URL, title: 'Kimi', partition: 'kimiweb', bridge: 'read' }, +]; + +const backend: BridgeBackend = { + listTargets: () => TABS, + sendCommand: async () => { + throw new Error('unexpected CDP call — ui_screenshot captures main-side, never over the guest debugger'); + }, +}; + +const PNG_B64 = 'aGVsbG8='; + +interface Harness { + deps: McpServerDeps; + seen: UiCaptureRequest[]; + audit: BridgeAuditEntry[]; +} + +function harness(opts: { sharing?: boolean; result?: UiCaptureResult; provider?: boolean } = {}): Harness { + const seen: UiCaptureRequest[] = []; + const audit: BridgeAuditEntry[] = []; + const result = opts.result ?? { ok: true, data_b64: PNG_B64, width: 800, height: 600 }; + const deps: McpServerDeps = { + backend, + serverInfo: { name: 'termipod-browser', version: '0.0.0-test' }, + uiFocusAvailable: () => opts.sharing !== false, + getUiFocus: () => ({ surface: 'read', captured_at: 'x' }), + onAction: (e) => audit.push(e), + ...(opts.provider === false + ? {} + : { + captureUi: async (req: UiCaptureRequest): Promise => { + seen.push(req); + return result; + }, + }), + }; + return { deps, seen, audit }; +} + +interface ToolResult { + content: Array<{ type?: string; text?: string; data?: string; mimeType?: string }>; + isError?: boolean; +} + +async function call(deps: McpServerDeps, args: Record = {}): Promise { + const res = await handleMcpMessage( + { jsonrpc: '2.0', id: 1, method: 'tools/call', params: { name: 'ui_screenshot', arguments: args } }, + deps, + { scope: 'read', agentId: 'ag_1', agentHandle: 'kimi-1' }, + ); + return res?.result as ToolResult; +} + +// ── Catalog + consent gating ───────────────────────────────────────────────── + +test('ui_screenshot is a READ-SCOPE tool gated by the sharing toggle', async () => { + // Read scope on purpose: the local kimi-code loop holds only the read token, + // and the consent that matters for a capture is the per-call card, not a + // spawn flag. (Its ACTION-ness lives in the approval + audit, below.) + assert.ok(READ_TOOLS.some((t) => t.name === 'ui_screenshot')); + + const on = await handleMcpMessage({ jsonrpc: '2.0', id: 1, method: 'tools/list' }, harness().deps, { scope: 'read', agentId: null }); + const listed = (on?.result as { tools: Array<{ name: string; description: string }> }).tools; + const tool = listed.find((t) => t.name === 'ui_screenshot'); + assert.ok(tool !== undefined, 'toggle on must list the tool'); + // The description is the only steering an agent gets: it must say the + // approval is per-call and point at the cheaper representations (§3.3/D-4). + assert.match(tool.description, /approval card/); + assert.match(tool.description, /ui_get_focus/); + assert.match(tool.description, /browser_snapshot/); + + const off = await handleMcpMessage({ jsonrpc: '2.0', id: 2, method: 'tools/list' }, harness({ sharing: false }).deps, { + scope: 'read', + agentId: null, + }); + const hidden = (off?.result as { tools: Array<{ name: string }> }).tools; + assert.ok(!hidden.some((t) => t.name === 'ui_screenshot'), 'toggle off must hide the tool'); +}); + +test('the toggle gate binds the CALL, not only the catalog', async () => { + const h = harness({ sharing: false }); + const out = await call(h.deps); + assert.equal(out.isError, true); + assert.match(out.content[0]?.text ?? '', /UI_UNAVAILABLE/); + assert.equal(h.seen.length, 0, 'a refused call never reaches the capture provider'); +}); + +test('a build with no capture provider refuses instead of throwing', async () => { + const out = await call(harness({ provider: false }).deps); + assert.equal(out.isError, true); + assert.match(out.content[0]?.text ?? '', /UI_UNAVAILABLE/); +}); + +// ── Target resolution ──────────────────────────────────────────────────────── + +test('no tabId captures the desktop window; the caller identity rides along', async () => { + const h = harness(); + const out = await call(h.deps); + assert.equal(out.isError, undefined); + assert.deepEqual(out.content, [{ type: 'image', data: PNG_B64, mimeType: 'image/png' }]); + assert.deepEqual(h.seen, [{ tabId: null, url: null, partition: null, agentId: 'ag_1', agentHandle: 'kimi-1', via: 'local' }]); +}); + +test('a tabId resolves against the live registry, fragment-stripped', async () => { + const h = harness(); + await call(h.deps, { tabId: 9 }); + // The guest URL reaches the approval card, so it must be stripped on the way + // — a kimiweb hash carries a bearer (ADR-059 D-5). + assert.deepEqual(h.seen[0], { + tabId: 9, + url: 'http://127.0.0.1:17331/', + partition: 'kimiweb', + agentId: 'ag_1', + agentHandle: 'kimi-1', + via: 'local', + }); +}); + +test('an unknown or malformed tabId is refused before the provider runs', async () => { + const h = harness(); + const gone = await call(h.deps, { tabId: 12345 }); + assert.equal(gone.isError, true); + assert.match(gone.content[0]?.text ?? '', /TARGET_GONE/); + + const bad = await call(h.deps, { tabId: 'seven' }); + assert.equal(bad.isError, true); + assert.match(bad.content[0]?.text ?? '', /INVALID_PARAMS/); + assert.equal(h.seen.length, 0); +}); + +test('a provider refusal surfaces its code and message verbatim', async () => { + const h = harness({ result: { ok: false, code: 'CAPTURE_REFUSED', message: "surface 'vault' refuses pixel capture" } }); + const out = await call(h.deps); + assert.equal(out.isError, true); + assert.match(out.content[0]?.text ?? '', /CAPTURE_REFUSED: surface 'vault' refuses pixel capture/); +}); + +// ── Audit posture (the part that is easy to get wrong) ─────────────────────── + +test('a LOCAL ui_screenshot is audited — unlike a local browser read', async () => { + const h = harness(); + await call(h.deps); + assert.equal(h.audit.length, 1, 'exactly one entry per call'); + assert.equal(h.audit[0]?.tool, 'ui_screenshot'); + assert.equal(h.audit[0]?.via, 'local'); + assert.equal(h.audit[0]?.ok, true); + assert.equal(h.audit[0]?.agent_id, 'ag_1'); + + // The contrast that makes the rule visible: a local browser READ is not + // audited at all (same-machine, high frequency — the ring would churn). + const reads = harness(); + await handleMcpMessage( + { jsonrpc: '2.0', id: 1, method: 'tools/call', params: { name: 'browser_list_tabs', arguments: {} } }, + reads.deps, + { scope: 'read', agentId: 'ag_1' }, + ); + assert.equal(reads.audit.length, 0); +}); + +test('a failed capture is audited too, with its code', async () => { + const h = harness({ result: { ok: false, code: 'CAPTURE_DENIED', message: 'the desktop user denied this screenshot' } }); + await call(h.deps); + assert.equal(h.audit.length, 1); + assert.equal(h.audit[0]?.ok, false); + assert.equal(h.audit[0]?.error, 'CAPTURE_DENIED'); +}); + +test('ui_screenshot mirrors to the hub on every leg; ui_get_focus does not', () => { + const entry = (tool: string, via: 'local' | 'hub'): BridgeAuditEntry => ({ + ts: '2026-07-31T00:00:00Z', + tool, + agent_id: 'ag_1', + via, + tab_id: null, + url: null, + partition: null, + args: {}, + ok: true, + error: null, + }); + // A capture is an action for audit purposes even though it sits in + // READ_TOOLS — the hub mirror is where the user's own event stream records + // that a frame of their screen was handed over. + assert.equal(shouldMirrorAudit(entry('ui_screenshot', 'hub')), true); + assert.equal(shouldMirrorAudit(entry('ui_screenshot', 'local')), true); + // Hub-leg reads stay ring-only (the hub already knows it routed them). + assert.equal(shouldMirrorAudit(entry('ui_get_focus', 'hub')), false); + assert.equal(shouldMirrorAudit(entry('browser_snapshot', 'hub')), false); +}); + +// ── The hub leg ────────────────────────────────────────────────────────────── + +test('a hub-relayed capture is marked via:hub so the desktop raises no second card', async () => { + const h = harness(); + const out = await dispatchHubInvoke(h.deps, { tool: 'ui_screenshot', args: {}, agent_id: 'ag_9', agent_handle: 'remote-1' }, new Set()); + assert.equal(out.ok, true); + assert.equal(h.seen[0]?.via, 'hub', 'the hub gated this call already (D5) — a second card would double-prompt'); + assert.equal(h.seen[0]?.agentId, 'ag_9'); + assert.equal(h.seen[0]?.agentHandle, 'remote-1'); +}); + +test('a revoked agent cannot capture, and never reaches the provider', async () => { + const h = harness(); + const out = await dispatchHubInvoke(h.deps, { tool: 'ui_screenshot', args: {}, agent_id: 'ag_9' }, new Set(['ag_9'])); + assert.equal(out.ok, false); + assert.equal(h.seen.length, 0); +}); diff --git a/desktop/src/i18n/index.ts b/desktop/src/i18n/index.ts index aaecca77..410b1959 100644 --- a/desktop/src/i18n/index.ts +++ b/desktop/src/i18n/index.ts @@ -429,6 +429,9 @@ const en: Dict = { 'att.replyPlaceholder': 'Reply…', 'att.answer': 'Answer', 'att.dismiss': 'Dismiss', + // D3: the desktop screenshot card. No "Allow session" button exists for it — + // this line says why, so the missing option reads as a decision. + 'att.perCallOnly': 'Screenshots are approved one call at a time — there is no standing grant.', 'kanban.todo': 'To do', 'kanban.in_progress': 'In progress', @@ -2491,6 +2494,7 @@ const zh: Dict = { 'att.replyPlaceholder': '回复…', 'att.answer': '回答', 'att.dismiss': '忽略', + 'att.perCallOnly': '截屏需逐次批准，不提供长期授权。', 'kanban.todo': '待办', 'kanban.in_progress': '进行中', diff --git a/desktop/src/surfaces/AttentionDock.tsx b/desktop/src/surfaces/AttentionDock.tsx index 5387b607..6919adbc 100644 --- a/desktop/src/surfaces/AttentionDock.tsx +++ b/desktop/src/surfaces/AttentionDock.tsx @@ -75,6 +75,19 @@ function AttentionCard({ item }: { item: Entity }): JSX.Element { {preview(pending['args']) !== '' &&
{preview(pending['args'])}
} )} + {/* D3 (docs/plans/desktop-ui-context-and-pointing.md §3.3): the gated + desktop screenshot. The card describes WHAT was asked for — which + surfaces are on screen, or which embedded tab — because the image + does not exist until this decision says yes. */} + {kind === 'desktop_action' && pending !== undefined && ( +
+ {str(pending, 'tool') ?? 'desktop tool'} + {str(pending, 'scope') !== undefined && · {str(pending, 'scope')}} + {preview(pending['surfaces']) !== '' &&
{preview(pending['surfaces'])}
} + {str(pending, 'url') !== undefined &&
{str(pending, 'url')}
} +
{t('att.perCallOnly')}
+
+ )} {changeKind !== undefined && (
{preview(item['change_spec'] ?? item['target_ref'])}
)} @@ -122,6 +135,18 @@ function AttentionCard({ item }: { item: Entity }): JSX.Element { {t('att.reject')} + ) : kind === 'desktop_action' ? ( + // Deliberately NO "Allow session" — a screenshot never gets a standing + // grant (plan §3.3, ADR-062 D-4). The tool parks on this decision and + // fails closed if it does not arrive. +
+ + +
) : (