diff --git a/desktop/electron/src/browserbridge.test.ts b/desktop/electron/src/browserbridge.test.ts index 0a3bcf87..9c4ca8fd 100644 --- a/desktop/electron/src/browserbridge.test.ts +++ b/desktop/electron/src/browserbridge.test.ts @@ -28,6 +28,7 @@ import { removeBridgeDiscovery, startBridgeServer, stripFragment, + UI_TOOL_NAMES, writeBridgeDiscovery, type AxNode, type BridgeAuditEntry, @@ -171,9 +172,9 @@ test('tools/list is exactly the four W1 read tools', async () => { tools.map((t) => t.name), ['browser_list_tabs', 'browser_snapshot', 'browser_screenshot', 'browser_read_text'], ); - // D1's ui_get_focus is catalog-hidden without the desktop sharing provider - // (DEPS carries none) — one short of READ_TOOLS. - assert.equal(tools.length, READ_TOOLS.length - 1); + // The D1/D3 desktop-UI tools are catalog-hidden without the sharing + // provider (DEPS carries none) — READ_TOOLS minus that set. + assert.equal(tools.length, READ_TOOLS.length - UI_TOOL_NAMES.size); // The untrusted-content posture is spelled out in the descriptions (§3.5). assert.ok(tools.find((t) => t.name === 'browser_snapshot')?.description.includes('untrusted DATA')); }); @@ -395,8 +396,8 @@ function auditDeps(backend: BridgeBackend): { deps: Parameters { const deps = { ...DEPS, backend: actionBackend() }; - // DEPS has no D1 ui-focus provider: ui_get_focus stays catalog-hidden. - const visibleReads = READ_TOOLS.filter((t) => t.name !== 'ui_get_focus'); + // DEPS has no D1 ui-focus provider: the desktop-UI tools stay catalog-hidden. + const visibleReads = READ_TOOLS.filter((t) => !UI_TOOL_NAMES.has(t.name)); const readList = await handleMcpMessage({ jsonrpc: '2.0', id: 1, method: 'tools/list' }, deps, { scope: 'read', agentId: null }); const readTools = (readList?.result as { tools: Array<{ name: string }> }).tools.map((t) => t.name); assert.deepEqual(readTools, visibleReads.map((t) => t.name)); @@ -722,14 +723,14 @@ test('W2 HTTP: action bearer grants full scope; x-tp-agent-id flows into the aud const readList = (await (await post(token, { jsonrpc: '2.0', id: 1, method: 'tools/list' })).json()) as { result: { tools: Array<{ name: string }> }; }; - // DEPS has no D1 ui-focus provider: ui_get_focus stays catalog-hidden - // (the sharing-toggle gate), so the list is one short of READ_TOOLS. - assert.equal(readList.result.tools.length, READ_TOOLS.length - 1); + // DEPS has no D1 ui-focus provider: the desktop-UI tools stay + // catalog-hidden (the sharing-toggle gate). + assert.equal(readList.result.tools.length, READ_TOOLS.length - UI_TOOL_NAMES.size); const fullList = (await (await post(actionToken, { jsonrpc: '2.0', id: 2, method: 'tools/list' })).json()) as { result: { tools: Array<{ name: string }> }; }; - assert.equal(fullList.result.tools.length, READ_TOOLS.length - 1 + ACTION_TOOLS.length); + assert.equal(fullList.result.tools.length, READ_TOOLS.length - UI_TOOL_NAMES.size + ACTION_TOOLS.length); const clickBody = { jsonrpc: '2.0', id: 3, method: 'tools/call', params: { name: 'browser_click', arguments: { tabId: 7, selector: '#go' } } }; const refused = (await (await post(token, clickBody)).json()) as { result: ToolResult }; diff --git a/desktop/electron/src/browserbridge.ts b/desktop/electron/src/browserbridge.ts index faf6af61..931e3c6d 100644 --- a/desktop/electron/src/browserbridge.ts +++ b/desktop/electron/src/browserbridge.ts @@ -47,6 +47,7 @@ import type { AddressInfo } from 'node:net'; // Explicit .ts extension: this module runs under `node --test`'s strip-only // loader (which resolves like plain ESM) as well as esbuild. import { partitionPolicy } from './webtab_policy.ts'; +import { parseScreenshotArgs } from './uicapture.ts'; // ── Targets / backend ──────────────────────────────────────────────────────── @@ -420,8 +421,34 @@ export const READ_TOOLS: readonly McpToolDef[] = [ 'What the TermiPod desktop user is currently looking at: the workbench surface(s) on screen plus focus state (open tab, focused agent, Inspect file + selection, terminal pane) as a compact JSON snapshot with captured_at. With a split, `surface` is the PRIMARY pane, `secondary` the pinned pane, and `active_pane` names the pane the user is in — resolve "this/here" against the active pane, not `surface` alone. Ids, paths and fragment-stripped URLs only — never message bodies, vault material, or settings values. Call this when the user references what is on their screen ("this", "here", "what I\'m looking at", "why is this failing") or when grounding in the user\'s current view would materially change the answer; do NOT call by default on every turn.', inputSchema: { type: 'object', properties: {}, additionalProperties: false }, }, + // D3 (plan §3.3, ADR-062 D-4): the VISUAL representation — pixels for the + // residue structure cannot answer. Read-SCOPED on purpose (the local + // kimi-code loop holds only the read token) but action-CLASS in every other + // respect: per-call approval, no session grant, audited + hub-mirrored. + { + name: 'ui_screenshot', + description: + 'Capture a PNG screenshot of the TermiPod desktop window, or of one embedded browser tab (tabId from browser_list_tabs). EVERY call raises an approval card the desktop user must accept — there is no standing grant, so use this only when pixels are the answer: a rendering bug, a layout question, "why does this look wrong". For what the user is looking at, ui_get_focus is cheaper, precise and needs no approval; for page content, browser_snapshot. Captures are refused outright while a sensitive surface (vault, settings) is on screen.', + inputSchema: { + type: 'object', + properties: { + tabId: { type: 'integer', description: 'Capture this embedded tab instead of the whole desktop window.' }, + }, + additionalProperties: false, + }, + }, ]; +/// The D1/D3 desktop-UI tools, which the sharing toggle gates as a set: off +/// means neither appears in any catalog and both refuse on call. +export const UI_TOOL_NAMES: ReadonlySet = new Set(['ui_get_focus', 'ui_screenshot']); + +/// Desktop-UI tools that are ACTION class despite living in READ_TOOLS (the +/// bearer scope they need and the consent they need are different questions — +/// see the ui_screenshot definition). They audit + hub-mirror like an action +/// on every leg, and their own handler owns the approval gate. +export const DESKTOP_ACTION_TOOL_NAMES: ReadonlySet = new Set(['ui_screenshot']); + /// D1: the MCP resource uri mirroring ui_get_focus (ADR-062 D-6). list + read /// only — subscriptions are deliberately NOT implemented: the relay forwards /// frames verbatim so notification delivery would work transport-wise, but @@ -547,11 +574,14 @@ const ACTION_TOOL_NAMES: ReadonlySet = new Set(ACTION_TOOLS.map((t) => t const READ_TOOL_NAMES: ReadonlySet = new Set(READ_TOOLS.map((t) => t.name)); /// Whether an audit entry should be mirrored to the hub as an agent event. -/// Actions always mirror (W2 contract). Hub-leg READS stay ring-only: the -/// hub routed the call, so a mirror row adds no information — the ring -/// entry exists purely so Settings → Remote driving can show (and revoke) -/// read-only remote sessions. +/// Actions always mirror (W2 contract) — including the desktop-UI action +/// tools, which sit in READ_TOOLS for scope reasons but are actions for +/// consent and audit (D3). Hub-leg READS stay ring-only: the hub routed the +/// call, so a mirror row adds no information — the ring entry exists purely +/// so Settings → Remote driving can show (and revoke) read-only remote +/// sessions. export function shouldMirrorAudit(entry: BridgeAuditEntry): boolean { + if (DESKTOP_ACTION_TOOL_NAMES.has(entry.tool)) return true; return !(entry.via === 'hub' && READ_TOOL_NAMES.has(entry.tool)); } @@ -574,8 +604,34 @@ export interface McpServerDeps { /// D1: the main-side focus cache (the renderer's last projected push), or /// null before the first push — the tool then answers an empty snapshot. getUiFocus?: () => Record | null; + /// D3: the gated screenshot (uicapture_host.ts). Absent means the capability + /// is not wired at all, which refuses like any other unavailable tool. The + /// provider owns the policy refusal AND the per-call approval — this module + /// only resolves the target and shapes the result. + captureUi?: (req: UiCaptureRequest) => Promise; +} + +/// One `ui_screenshot` request, with the target already resolved against the +/// live guest registry (so a guessed tabId can never name the app:// shell) +/// and the caller's identity attached for the approval card + audit. +export interface UiCaptureRequest { + /// null = the desktop window itself. + tabId: number | null; + /// The guest's fragment-stripped URL, when capturing a tab. + url: string | null; + /// The guest's partition, when capturing a tab — the policy key. + partition: string | null; + agentId: string; + agentHandle: string; + /// 'hub' calls arrive pre-approved (the hub raises the desktop_action card + /// before routing — D5), so the desktop must not raise a second one. + via: 'local' | 'hub'; } +export type UiCaptureResult = + | { ok: true; data_b64: string; width: number; height: number } + | { ok: false; code: string; message: string }; + interface JsonRpcRequest { jsonrpc?: string; id?: unknown; @@ -893,7 +949,7 @@ async function uploadFiles(deps: McpServerDeps, target: BridgeTarget, args: Reco } } -async function runTool(deps: McpServerDeps, name: string, args: Record): Promise { +async function runTool(deps: McpServerDeps, ctx: BridgeRequestContext, name: string, args: Record): Promise { const { backend } = deps; switch (name) { case 'browser_list_tabs': { @@ -1039,6 +1095,33 @@ async function runTool(deps: McpServerDeps, name: string, args: Record): Promise { - if (!ACTION_TOOL_NAMES.has(name) && ctx.via !== 'hub') return runTool(deps, name, args); + // D3: a desktop-UI action tool is audited on EVERY leg, local included — a + // screenshot of the user's own screen is exactly what the Settings audit + // view exists to show, and unlike a browser read it is neither cheap nor + // frequent, so the ring will not churn. + if (!ACTION_TOOL_NAMES.has(name) && !DESKTOP_ACTION_TOOL_NAMES.has(name) && ctx.via !== 'hub') { + return runTool(deps, ctx, name, args); + } const tabId = typeof args.tabId === 'number' && Number.isInteger(args.tabId) ? args.tabId : null; const target = tabId === null ? undefined : deps.backend.listTargets().find((t) => t.tabId === tabId); const entry: BridgeAuditEntry = { @@ -1077,7 +1166,7 @@ async function callTool(deps: McpServerDeps, ctx: BridgeRequestContext, name: st }; if (ctx.agentHandle !== undefined) entry.agent_handle = ctx.agentHandle; try { - const out = await runTool(deps, name, args); + const out = await runTool(deps, ctx, name, args); entry.ok = true; return out; } catch (e) { @@ -1234,10 +1323,10 @@ export async function handleMcpMessage( return rpcResult(id, {}); case 'tools/list': { // Scope gate #1: a read-scoped session never SEES the action tools. - // Consent gate (D1): ui_get_focus is catalog-visible only while the - // desktop's UI context sharing toggle is on. - const reads = - deps.uiFocusAvailable?.() === true ? READ_TOOLS : READ_TOOLS.filter((t) => t.name !== 'ui_get_focus'); + // Consent gate (D1/D3): the desktop-UI tools are catalog-visible only + // while the sharing toggle is on — off means no publisher and no tool + // in any catalog. + const reads = deps.uiFocusAvailable?.() === true ? READ_TOOLS : READ_TOOLS.filter((t) => !UI_TOOL_NAMES.has(t.name)); return rpcResult(id, { tools: ctx.scope === 'full' ? [...reads, ...ACTION_TOOLS] : reads }); } case 'resources/list': { diff --git a/desktop/electron/src/browserbridge_host.ts b/desktop/electron/src/browserbridge_host.ts index 310f2e93..e65872cc 100644 --- a/desktop/electron/src/browserbridge_host.ts +++ b/desktop/electron/src/browserbridge_host.ts @@ -62,6 +62,8 @@ import { type HubInvokePayload, type HubInvokeResult, type McpServerDeps, + type UiCaptureRequest, + type UiCaptureResult, } from './browserbridge'; import { policyForGuest } from './webtab'; import { keychainGetLocal } from './ipc/keychain'; @@ -273,6 +275,24 @@ export function setUiFocusProvider(p: UiFocusProvider): void { uiFocusProvider = p; } +/// D3: the gated screenshot provider (uicapture_host.ts), plugged in the same +/// way and for the same reason — it needs the hub context and the focus cache +/// this module and desktopui.ts own, so it registers instead of being imported. +type UiCaptureProvider = (req: UiCaptureRequest) => Promise; + +let uiCaptureProvider: UiCaptureProvider | null = null; + +export function setUiCaptureProvider(p: UiCaptureProvider): void { + uiCaptureProvider = p; +} + +/// The hub identity the audit mirror and the D3 approval card both post as. +/// Non-secret: the bearer is fetched from the keychain per use, never held +/// here (and never handed across IPC). +export function currentHubContext(): { baseUrl: string; teamId: string; profileId: string } | null { + return hubContext; +} + // ── Enable/disable lifecycle ───────────────────────────────────────────────── let server: BridgeServer | null = null; @@ -298,6 +318,12 @@ async function enable(): Promise { // ui_get_focus on the next tools/list without a server restart. uiFocusAvailable: () => uiFocusProvider?.available() ?? false, getUiFocus: () => uiFocusProvider?.snapshot() ?? null, + // D3: read through the holder on every call, so a provider registered + // after the server started (module init order) is still reachable. + captureUi: (req) => + uiCaptureProvider !== null + ? uiCaptureProvider(req) + : Promise.resolve({ ok: false, code: 'UI_UNAVAILABLE', message: 'the capture provider is not wired' }), }; server = await startBridgeServer({ ...mcpDeps, token, actionToken }); writeBridgeDiscovery(os.homedir(), { diff --git a/desktop/electron/src/desktopui.ts b/desktop/electron/src/desktopui.ts index 5d33cfa4..b1403c4e 100644 --- a/desktop/electron/src/desktopui.ts +++ b/desktop/electron/src/desktopui.ts @@ -40,6 +40,13 @@ export function isUiSharingEnabled(): boolean { /// that crosses IPC), so main's job is just to hold and serve it. let focusCache: Record | null = null; +/// D3's capture path reads the same cache to learn which surfaces are on +/// screen — the refusal decision must be made from the SAME view of the +/// screen the agent can already see, not from a second source of truth. +export function currentFocusSnapshot(): Record | null { + return sharingEnabled ? focusCache : null; +} + /// A runaway publisher can't fill memory: snapshots are ref-sized (a handful /// of ids/paths); anything bigger is a bug, not a bigger allowance. const MAX_SNAPSHOT_BYTES = 16384; diff --git a/desktop/electron/src/main.ts b/desktop/electron/src/main.ts index 53a2ad86..f8bbb31c 100644 --- a/desktop/electron/src/main.ts +++ b/desktop/electron/src/main.ts @@ -28,6 +28,9 @@ import { disposeKimiWebWin } from './kimiwebwin'; import { disposeRerun } from './rerun'; import { disposeBrowserBridge } from './browserbridge_host'; import { disposeAnnotations } from './annotation_host'; +// D3: importing the module registers the gated-screenshot provider on the +// bridge server; setShellWindow tells it which window "the desktop" means. +import { setShellWindow } from './uicapture_host'; import { initEvents } from './events'; // The frontend build. In dev (`electron .` from desktop/electron) it resolves to @@ -72,6 +75,7 @@ function createWindow(): void { }, }); mainWindow = win; + setShellWindow(win); // A `window.open` from any embedded content would spawn a child window that // INHERITS this window's webPreferences — preload included — handing // `__ELECTRON_BRIDGE__` and the whole command allowlist to remote content. @@ -89,6 +93,7 @@ function createWindow(): void { win.once('ready-to-show', () => win.show()); win.on('closed', () => { if (mainWindow === win) mainWindow = null; + setShellWindow(null); }); void win.loadURL(`${APP_ORIGIN}/index.html`); } diff --git a/desktop/electron/src/uicapture.test.ts b/desktop/electron/src/uicapture.test.ts new file mode 100644 index 00000000..30031dfe --- /dev/null +++ b/desktop/electron/src/uicapture.test.ts @@ -0,0 +1,147 @@ +/// Tests for the gated-screenshot core (D3 — plan §3.3, ADR-062 D-3/D-4). +/// The three rules that make a screenshot safe are all decided in uicapture.ts, +/// so they are all provable here without Electron: refusal by table, per-call +/// approval (never a session grant), and fail-closed decision reading. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { UI_POLICY, uiPolicyFor } from '../../src/state/ui_policy.ts'; +import { + captureApprovalCard, + captureDenialMessage, + captureRefusal, + parseScreenshotArgs, + readCaptureDecision, + surfaceForPartition, + visibleSurfaces, +} from './uicapture.ts'; + +// ── What is on screen ──────────────────────────────────────────────────────── + +test('visibleSurfaces reads both panes, and answers "unknown" rather than "empty"', () => { + assert.deepEqual(visibleSurfaces({ surface: 'read', captured_at: 'x' }), ['read']); + assert.deepEqual( + visibleSurfaces({ surface: 'read', secondary: { surface: 'debug' }, active_pane: 'secondary', captured_at: 'x' }), + ['read', 'debug'], + ); + // No snapshot yet, or one without a usable surface: null, NOT [] — the + // caller must be able to tell "nothing sensitive on screen" from "we have no + // idea what is on screen", because those get opposite answers. + assert.equal(visibleSurfaces(null), null); + assert.equal(visibleSurfaces({ captured_at: 'x' }), null); + assert.equal(visibleSurfaces({ surface: '', captured_at: 'x' }), null); + // A malformed secondary is ignored rather than fabricating a pane. + assert.deepEqual(visibleSurfaces({ surface: 'read', secondary: { nope: 1 } }), ['read']); +}); + +// ── Refusal by table ───────────────────────────────────────────────────────── + +test('captureRefusal: the policy table decides, and refuses what it does not declare', () => { + assert.equal(captureRefusal(['read']), null); + assert.equal(captureRefusal(['read', 'debug', 'replay']), null); + // A pixel capture of settings shows its VALUES even though its snapshot row + // emits only the surface id — the columns are independent (ADR-062 D-3). + assert.equal(captureRefusal(['settings']), 'settings'); + assert.equal(captureRefusal(['vault']), 'vault'); + // The gate covers BOTH panes: a split cannot launder the vault past it by + // putting it in the half the user is not focused on. + assert.equal(captureRefusal(['read', 'vault']), 'vault'); + // Default-correct for the unknown: a surface with no row is not capturable. + assert.equal(captureRefusal(['surface-added-next-quarter']), 'surface-added-next-quarter'); +}); + +test('the sensitive surfaces are exactly the ones that refuse pixels', () => { + const refusing = Object.entries(UI_POLICY) + .filter(([, row]) => row.capture === 'refuse') + .map(([surface]) => surface) + .sort(); + // Stated as an invariant, not as a snapshot of today's table: adding a row + // that refuses capture is a deliberate privacy decision and should have to + // touch this line. + assert.deepEqual(refusing, ['settings', 'vault']); +}); + +test('surfaceForPartition maps every guest kind onto a real policy row', () => { + assert.equal(surfaceForPartition('persist:webtab'), 'read'); + assert.equal(surfaceForPartition('kimiweb'), 'kimiweb'); + assert.equal(surfaceForPartition('rerunweb'), 'replay'); + // An unlisted partition opts IN by naming its surface — it never inherits + // one (the webtab_policy discipline). + assert.equal(surfaceForPartition('persist:something-new'), null); + assert.equal(surfaceForPartition(null), null); + // Every mapping must land on a declared row, or the guest leg would refuse + // for a reason nobody intended. + for (const partition of ['persist:webtab', 'kimiweb', 'rerunweb']) { + const surface = surfaceForPartition(partition); + assert.ok(surface !== null && uiPolicyFor(surface) !== null, partition); + } +}); + +// ── Arguments ──────────────────────────────────────────────────────────────── + +test('parseScreenshotArgs: absent tabId means the window; junk is refused', () => { + assert.deepEqual(parseScreenshotArgs({}), { tabId: null }); + assert.deepEqual(parseScreenshotArgs({ tabId: null }), { tabId: null }); + assert.deepEqual(parseScreenshotArgs({ tabId: 7 }), { tabId: 7 }); + assert.ok('error' in parseScreenshotArgs({ tabId: '7' })); + assert.ok('error' in parseScreenshotArgs({ tabId: 1.5 })); +}); + +// ── The approval card ──────────────────────────────────────────────────────── + +test('the approval card names what was asked for and carries no pixels', () => { + const card = captureApprovalCard({ + agentId: 'ag_1', + agentHandle: 'kimi-1', + scope: 'window', + surfaces: ['read', 'debug'], + url: null, + }); + assert.match(card.summary, /kimi-1/); + assert.match(card.summary, /read \+ debug/); + assert.equal(card.payload.tool, 'ui_screenshot'); + assert.deepEqual(card.payload.surfaces, ['read', 'debug']); + // §3.3: screenshots never get a standing grant, so the card says so in the + // payload as well as by omitting the session option in the UI. + assert.equal(card.payload.session_grant, false); + // The payload is a REFERENCE to what was requested — the image does not + // exist yet, and nothing content-shaped may ride along (ADR-062 D-2). + for (const key of Object.keys(card.payload)) { + assert.ok(!['image', 'data', 'data_b64', 'png', 'preview'].includes(key), key); + } +}); + +test('the card falls back to the agent id, then to a neutral subject', () => { + const byId = captureApprovalCard({ agentId: 'ag_1', agentHandle: '', scope: 'window', surfaces: ['read'], url: null }); + assert.match(byId.summary, /^ag_1 wants/); + const anon = captureApprovalCard({ agentId: '', agentHandle: '', scope: 'tab', surfaces: ['read'], url: 'https://a.b/c' }); + assert.match(anon.summary, /^An agent wants/); + assert.match(anon.summary, /https:\/\/a\.b\/c/); +}); + +// ── Fail-closed decisions ──────────────────────────────────────────────────── + +test('readCaptureDecision: only an explicit trailing approve approves', () => { + assert.equal(readCaptureDecision({ status: 'open', decisions: [] }), 'pending'); + // Quorum not yet met: an approve on a still-open row is not a decision. + assert.equal(readCaptureDecision({ status: 'open', decisions: [{ decision: 'approve' }] }), 'pending'); + assert.equal(readCaptureDecision({ status: 'resolved', decisions: [{ decision: 'approve' }] }), 'approve'); + assert.equal(readCaptureDecision({ status: 'resolved', decisions: [{ decision: 'reject' }] }), 'deny'); + // Dismissed through /resolve — resolved with no decision at all. Denies. + assert.equal(readCaptureDecision({ status: 'resolved' }), 'deny'); + assert.equal(readCaptureDecision({ status: 'resolved', decisions: [] }), 'deny'); + // The LAST decision wins (an override after a reject, say). + assert.equal(readCaptureDecision({ status: 'resolved', decisions: [{ decision: 'reject' }, { decision: 'approve' }] }), 'approve'); + assert.equal(readCaptureDecision({ status: 'resolved', decisions: [{ decision: 'approve' }, { decision: 'reject' }] }), 'deny'); + // Shapes we cannot read: an unparseable resolved row denies; a body we + // cannot read at all is not evidence of anything, so it keeps waiting. + assert.equal(readCaptureDecision({ status: 'resolved', decisions: 'nope' }), 'deny'); + assert.equal(readCaptureDecision({ status: 'resolved', decisions: [42] }), 'deny'); + assert.equal(readCaptureDecision(null), 'pending'); + assert.equal(readCaptureDecision('nope'), 'pending'); +}); + +test('denial messages point the agent at the cheaper representation', () => { + assert.match(captureDenialMessage('denied'), /denied/); + assert.match(captureDenialMessage('timeout'), /approval window/); + assert.match(captureDenialMessage('unavailable'), /ui_get_focus/); +}); diff --git a/desktop/electron/src/uicapture.ts b/desktop/electron/src/uicapture.ts new file mode 100644 index 00000000..71302676 --- /dev/null +++ b/desktop/electron/src/uicapture.ts @@ -0,0 +1,202 @@ +/// Gated desktop screenshot — electron-free core (D3 — +/// docs/plans/desktop-ui-context-and-pointing.md §3.3, ADR-062 D-4). +/// +/// The visual representation of the desktop UI entity: pixels for the residue +/// structure cannot answer (a rendering bug, a layout question). It is the +/// most sensitive artifact the app can emit — a frame of everything the user +/// sees — so three rules bind every call, and all three are decided here: +/// +/// 1. REFUSAL BY TABLE. A capture is refused outright when any surface on +/// screen carries `capture: 'refuse'` in ui_policy.ts (vault, settings — +/// a pixel capture of settings shows its VALUES even though its snapshot +/// row emits only the surface id). An undeclared surface refuses too, and +/// so does an unknown screen (no focus snapshot yet): we refuse what we +/// cannot classify rather than capture it (ADR-062 D-3). +/// 2. PER-CALL APPROVAL, ALWAYS. Never a session grant — §3.3 is explicit +/// that screenshots get no standing consent. The card is a hub +/// attention_item of kind `desktop_action`, so the decision lands in the +/// director's inbox on every client, and the local leg raises it itself +/// (the hub leg's card is raised hub-side — D5). +/// 3. FAIL CLOSED. No decision inside the window, a dismissed row, a +/// resolution we cannot read: all deny. +/// +/// The live halves (capturePage, the hub round trip) are uicapture_host.ts; +/// everything below is pure so `node --test` covers the policy without +/// Electron. +import { uiPolicyFor } from '../../src/state/ui_policy.ts'; +import { KIMIWEB_PARTITION, RERUNWEB_PARTITION, WEBTAB_PARTITION } from './webtab_policy.ts'; + +// ── The refusal rule ───────────────────────────────────────────────────────── + +/// Which ui_policy row governs pixels of a `` guest. A guest is not a +/// workbench job, but it is painted inside one, and the capture question is +/// the same question — so it is answered by the same table rather than by a +/// second rule living in the capture path (ADR-062 D-3: nothing else in the +/// pipeline makes policy decisions). +/// +/// An unlisted partition maps to no row and therefore refuses: a future guest +/// kind opts in by naming its surface here, exactly as `webtab_policy` makes a +/// new partition state its `bridge` capability instead of inheriting one. +export function surfaceForPartition(partition: string | null): string | null { + switch (partition) { + case WEBTAB_PARTITION: + return 'read'; + case KIMIWEB_PARTITION: + return 'kimiweb'; + case RERUNWEB_PARTITION: + return 'replay'; + default: + return null; + } +} + +/// The surfaces a full-window capture would expose: the primary pane plus the +/// pinned one when a split is on screen. Read from the SAME projected focus +/// snapshot the agent can already see (desktopui.ts's cache) — no second +/// source of truth about what is on screen. +/// +/// Returns null when there is no usable snapshot: the answer is "unknown", +/// which the caller must treat as refusal, not as "nothing sensitive". +/// +/// INVARIANT this rests on: when the ACTIVE pane's row has an empty allowlist +/// (settings, vault), projectFocus rule 1 degrades the snapshot to +/// existence-only and DROPS `secondary` — so a hypothetical split with a +/// refusing surface active in the pinned pane would report only the primary +/// here, and "a split refuses if either pane refuses" would not hold. That +/// state is unreachable today because every `capture: 'refuse'` row is also +/// `splitEligible: false` (workbench.ts) — a refusing surface can only ever +/// be the primary, which rule 1 still reports as `surface`. If a refusing +/// surface ever becomes split-eligible, this helper must read the raw +/// (unprojected) focus state instead of the projection. +export function visibleSurfaces(snapshot: Record | null): string[] | null { + if (snapshot === null) return null; + const primary = snapshot.surface; + if (typeof primary !== 'string' || primary === '') return null; + const out = [primary]; + const secondary = snapshot.secondary; + if (secondary !== null && typeof secondary === 'object' && !Array.isArray(secondary)) { + const s = (secondary as Record).surface; + if (typeof s === 'string' && s !== '') out.push(s); + } + return out; +} + +/// The first surface that refuses pixels, or null when every one of them +/// allows. A surface with NO policy row refuses — the table is the allowlist, +/// so an undeclared surface is never capturable (the same posture the +/// projection takes in ui_policy.ts rule 1). +export function captureRefusal(surfaces: readonly string[]): string | null { + for (const surface of surfaces) { + if (uiPolicyFor(surface)?.capture !== 'allow') return surface; + } + return null; +} + +// ── Arguments ──────────────────────────────────────────────────────────────── + +/// `ui_screenshot` takes at most a tabId: absent means the desktop window +/// itself, present means that `` guest (validated against the live +/// registry host-side — an id alone never reaches the app:// shell). +export interface UiScreenshotArgs { + tabId: number | null; +} + +export function parseScreenshotArgs(args: Record): UiScreenshotArgs | { error: string } { + const raw = args.tabId; + if (raw === undefined || raw === null) return { tabId: null }; + if (typeof raw !== 'number' || !Number.isInteger(raw)) { + return { error: 'tabId must be an integer from browser_list_tabs (omit it to capture the desktop window)' }; + } + return { tabId: raw }; +} + +// ── The approval card ──────────────────────────────────────────────────────── + +/// How long the tool parks on the card before failing closed. Deliberately +/// SHORTER than the hub's 10-minute browser_action window: this leg is a +/// local MCP tool call held open over stdio, and an agent client that gives up +/// mid-park would leave the user approving a call nobody is waiting for. Two +/// minutes is long enough to notice the card, short enough that a stale one is +/// rare — and a denial is retryable, unlike a capture. +export const UI_CAPTURE_APPROVAL_TIMEOUT_MS = 120_000; + +/// How often the local leg re-reads the card while parked. +export const UI_CAPTURE_POLL_MS = 1500; + +export interface CaptureApprovalRequest { + agentId: string; + agentHandle: string; + /// 'window' (the whole desktop) or 'tab' (one embedded guest). + scope: 'window' | 'tab'; + /// The surfaces on screen, for the window scope; the guest URL for a tab. + surfaces: readonly string[]; + url: string | null; +} + +/// The card's human sentence + its machine payload. The payload carries a +/// UIRef-shaped description of WHAT was asked for — surfaces, tab, url — and +/// never a pixel: the approver decides from the reference, and the image only +/// exists after they say yes. +export function captureApprovalCard(req: CaptureApprovalRequest): { summary: string; payload: Record } { + const who = req.agentHandle !== '' ? req.agentHandle : req.agentId !== '' ? req.agentId : 'An agent'; + const what = + req.scope === 'tab' + ? `the embedded tab ${req.url ?? ''}`.trimEnd() + : req.surfaces.length > 0 + ? `the desktop window (${req.surfaces.join(' + ')})` + : 'the desktop window'; + return { + summary: `${who} wants a screenshot of ${what}`, + payload: { + tool: 'ui_screenshot', + scope: req.scope, + surfaces: [...req.surfaces], + url: req.url, + agent_id: req.agentId, + // Pinned in the payload so the card can never be re-read as a standing + // grant: screenshots are per-call, always (plan §3.3). + session_grant: false, + }, + }; +} + +// ── Reading the decision back ──────────────────────────────────────────────── + +export type CaptureOutcome = 'approve' | 'deny' | 'pending'; + +/// Interpret one `GET /attention/{id}` row. `open` means keep waiting; +/// anything else resolves, and only an explicit trailing `approve` decision +/// approves — a row dismissed through /resolve (no decision at all), a reject, +/// or a shape we cannot parse all deny. +export function readCaptureDecision(row: unknown): CaptureOutcome { + if (row === null || typeof row !== 'object' || Array.isArray(row)) return 'pending'; + const r = row as Record; + const status = typeof r.status === 'string' ? r.status : 'open'; + if (status === 'open') return 'pending'; + const decisions = r.decisions; + if (!Array.isArray(decisions) || decisions.length === 0) return 'deny'; + const last = decisions[decisions.length - 1]; + if (last === null || typeof last !== 'object') return 'deny'; + const decision = (last as Record).decision; + return decision === 'approve' ? 'approve' : 'deny'; +} + +/// The denial sentence an agent sees, by cause. Kept here (not in the host) so +/// the wording is testable and identical on both legs. +export function captureDenialMessage(cause: 'denied' | 'timeout' | 'unavailable' | 'raise_failed'): string { + switch (cause) { + case 'denied': + return 'the desktop user denied this screenshot'; + case 'timeout': + return 'no decision on the screenshot request within the approval window — denied'; + case 'unavailable': + return ( + 'this desktop cannot ask for screenshot approval — it is not signed in to a hub, ' + + 'and ui_screenshot is per-call approved, always. Use ui_get_focus for structure instead' + ); + case 'raise_failed': + // Distinct from 'unavailable': the desktop IS signed in, but the hub + // refused or errored the card — a retry may succeed, signing in won't. + return 'the hub did not accept the approval card for this screenshot — denied (transient hub error; retrying may work)'; + } +} diff --git a/desktop/electron/src/uicapture_host.ts b/desktop/electron/src/uicapture_host.ts new file mode 100644 index 00000000..078b87a9 --- /dev/null +++ b/desktop/electron/src/uicapture_host.ts @@ -0,0 +1,163 @@ +/// Gated desktop screenshot — Electron main-process half (D3 — +/// docs/plans/desktop-ui-context-and-pointing.md §3.3). The policy (which +/// surfaces refuse pixels, how a decision reads, what the card says) is the +/// electron-free uicapture.ts; this module owns the live halves: +/// +/// - the CAPTURE: `capturePage` on the shell window or on one `` +/// guest, downscaled to the browser_screenshot discipline (PNG, ≤1568px +/// max edge) and returned as base64 — never written to disk, never +/// cached: the image exists for one tool result (ADR-062 D-7); +/// - the APPROVAL: a hub `desktop_action` attention_item raised per call, +/// polled until it resolves. The LOCAL leg raises it here; a hub-relayed +/// call arrives pre-approved (the hub raises the same card kind before it +/// routes — D5) and must not raise a second one. +/// +/// Fail-closed everywhere: no hub to ask, no answer in the window, an +/// unreadable resolution, an unknown surface on screen — all deny. The tool +/// registration + audit live on the bridge server (browserbridge.ts); this +/// module plugs itself in through setUiCaptureProvider so the import edge +/// stays one-directional. +import { webContents, type BrowserWindow, type NativeImage, type WebContents } from 'electron'; +import { fitWithin, MAX_IMAGE_EDGE } from './annotation'; +import { currentHubContext, setUiCaptureProvider } from './browserbridge_host'; +import { currentFocusSnapshot, isUiSharingEnabled } from './desktopui'; +import { keychainGetLocal } from './ipc/keychain'; +import { + captureApprovalCard, + captureDenialMessage, + captureRefusal, + surfaceForPartition, + visibleSurfaces, +} from './uicapture'; +import { approveViaCard, type HubLeg } from './uicapture_hub'; +import type { UiCaptureRequest, UiCaptureResult } from './browserbridge'; + +// ── The shell window ───────────────────────────────────────────────────────── + +/// The app's own window, registered by main.ts at creation (and cleared on +/// close). Deliberately NOT discovered via `BrowserWindow.getAllWindows()`: +/// the kimi-web detached window is a BrowserWindow too, and "capture whatever +/// window we found first" is exactly the kind of ambient targeting a +/// screenshot tool must not have. +let shellWindow: BrowserWindow | null = null; + +export function setShellWindow(win: BrowserWindow | null): void { + shellWindow = win; +} + +// ── Guest resolution ───────────────────────────────────────────────────────── + +/// The tabId was already validated against the bridge's live registry +/// (`requireTarget` in browserbridge.ts, so a guessed id can never name the +/// app:// shell); this only re-resolves it to a live webContents. +function liveGuest(tabId: number): WebContents | null { + const wc = webContents.fromId(tabId); + if (wc === undefined || wc.isDestroyed() || wc.getType() !== 'webview') return null; + return wc; +} + +// ── The hub approval round trip ────────────────────────────────────────────── + +/// The hub identity + bearer for the approval card, or null when the desktop +/// is signed out. Same sourcing as the audit mirror: non-secret context pushed +/// by the renderer, the token read from the main-process keychain at use time +/// (never over IPC). The round trip itself (raise → poll → dismiss) lives in +/// uicapture_hub.ts, electron-free and test-pinned. +async function hubLeg(): Promise { + const ctx = currentHubContext(); + if (ctx === null) return null; + const token = await keychainGetLocal(`hub_token_${ctx.profileId}`); + if (token === null || token === '') return null; + return { baseUrl: ctx.baseUrl, teamId: ctx.teamId, token }; +} + +// ── The capture ────────────────────────────────────────────────────────────── + +function toPngBase64(source: NativeImage): { data_b64: string; width: number; height: number } { + const size = source.getSize(); + const fit = fitWithin(size.width, size.height, MAX_IMAGE_EDGE); + const image = fit.width !== size.width || fit.height !== size.height ? source.resize({ ...fit, quality: 'good' }) : source; + return { data_b64: image.toPNG().toString('base64'), width: fit.width, height: fit.height }; +} + +function refused(surface: string): UiCaptureResult { + return { + ok: false, + code: 'CAPTURE_REFUSED', + message: + surface === '' + ? 'the desktop has not published what is on screen yet — a capture of an unclassified screen is refused' + : `surface '${surface}' refuses pixel capture by policy (ui_policy.ts capture column) — ask for structure instead (ui_get_focus)`, + }; +} + +async function capture(req: UiCaptureRequest): Promise { + if (!isUiSharingEnabled()) { + return { ok: false, code: 'UI_UNAVAILABLE', message: 'UI context sharing is off on the desktop (Settings → Assistant)' }; + } + + // 1. The table decides what may be captured at all — before any approval is + // asked for, so a refused surface never costs the user a card. + const scope: 'window' | 'tab' = req.tabId === null ? 'window' : 'tab'; + let surfaces: string[]; + if (scope === 'tab') { + const surface = surfaceForPartition(req.partition); + if (surface === null) return refused(req.partition ?? ''); + surfaces = [surface]; + } else { + const onScreen = visibleSurfaces(currentFocusSnapshot()); + if (onScreen === null) return refused(''); + surfaces = onScreen; + } + const blocked = captureRefusal(surfaces); + if (blocked !== null) return refused(blocked); + + // 2. Per-call approval. A hub-relayed call already carries one (D5). + if (req.via !== 'hub') { + const leg = await hubLeg(); + if (leg === null) return { ok: false, code: 'UI_APPROVAL_UNAVAILABLE', message: captureDenialMessage('unavailable') }; + const card = captureApprovalCard({ + agentId: req.agentId, + agentHandle: req.agentHandle, + scope, + surfaces, + url: req.url, + }); + const verdict = await approveViaCard(leg, card, req.agentHandle); + if (verdict !== 'approve') { + // A hub that refused the card is a different diagnosis than "signed + // out" (checked above) — don't tell the agent to sign in when the hub + // just errored. + if (verdict === 'raise_failed') { + return { ok: false, code: 'UI_APPROVAL_UNAVAILABLE', message: captureDenialMessage('raise_failed') }; + } + return { ok: false, code: 'CAPTURE_DENIED', message: captureDenialMessage(verdict) }; + } + } + + // 3. Pixels. Re-checked after the park: the user may have moved to the vault + // while the card sat open, and the approval was for what was on screen + // when it was raised. + if (scope === 'window') { + const after = visibleSurfaces(currentFocusSnapshot()); + if (after === null) return refused(''); + const moved = captureRefusal(after); + if (moved !== null) return refused(moved); + } + + try { + if (scope === 'tab' && req.tabId !== null) { + const guest = liveGuest(req.tabId); + if (guest === null) return { ok: false, code: 'TARGET_GONE', message: `tab ${String(req.tabId)} was destroyed` }; + return { ok: true, ...toPngBase64(await guest.capturePage()) }; + } + if (shellWindow === null || shellWindow.isDestroyed()) { + return { ok: false, code: 'NO_WINDOW', message: 'the desktop window is not open' }; + } + return { ok: true, ...toPngBase64(await shellWindow.webContents.capturePage()) }; + } catch (e) { + return { ok: false, code: 'CAPTURE_FAILED', message: e instanceof Error ? e.message : String(e) }; + } +} + +setUiCaptureProvider(capture); diff --git a/desktop/electron/src/uicapture_hub.test.ts b/desktop/electron/src/uicapture_hub.test.ts new file mode 100644 index 00000000..a03f611b --- /dev/null +++ b/desktop/electron/src/uicapture_hub.test.ts @@ -0,0 +1,148 @@ +/// Tests for the ui_screenshot hub round trip (D3 — plan §3.3). Every seam is +/// injected, so raise → poll → dismiss sequencing is provable against a mocked +/// fetch — including the one wire shape that is load-bearing on the hub side: +/// the dismiss body must NOT name a `resolved_by` (the column REFERENCES +/// agents(id); a non-agent value FK-violates, the 500 is swallowed, and the +/// card would loiter in the director's inbox forever). +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { approveViaCard, dismissCard, raiseCard, teamUrl, type HubLeg } from './uicapture_hub.ts'; +import { UI_CAPTURE_POLL_MS } from './uicapture.ts'; + +const LEG: HubLeg = { baseUrl: 'https://hub.example', teamId: 'team_1', token: 'tok' }; +const CARD = { summary: 'agent wants a screenshot', payload: { tool: 'ui_screenshot', session_grant: false } }; + +interface Call { + url: string; + method: string; + body: unknown; +} + +/// A scripted fetch: each entry answers the next request; the log keeps what +/// was sent. `Response`s are real, so .ok/.json() behave like the wire. +function scriptedFetch(script: Array): { fetchFn: (url: string, init?: RequestInit) => Promise; calls: Call[] } { + const calls: Call[] = []; + return { + calls, + fetchFn: (url: string, init?: RequestInit) => { + calls.push({ + url, + method: init?.method ?? 'GET', + body: typeof init?.body === 'string' ? (JSON.parse(init.body) as unknown) : undefined, + }); + const next = script.shift() ?? new Error('script exhausted'); + return next instanceof Error ? Promise.reject(next) : Promise.resolve(next); + }, + }; +} + +function json(status: number, body: unknown): Response { + // 204 must be bodyless — the Response constructor enforces it. + if (status === 204) return new Response(null, { status }); + return new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } }); +} + +/// A clock that only sleep advances — the poll loop is deterministic. +function fakeClock(): { now: () => number; sleep: (ms: number) => Promise } { + let t = 0; + return { now: () => t, sleep: (ms: number) => ((t += ms), Promise.resolve()) }; +} + +// ── The FK-sensitive dismiss body ──────────────────────────────────────────── + +test('dismissCard sends an EMPTY body — resolved_by absent, never a non-agent id', async () => { + const { fetchFn, calls } = scriptedFetch([json(204, {})]); + await dismissCard(LEG, 'att_1', fetchFn); + assert.equal(calls.length, 1); + assert.equal(calls[0].url, teamUrl(LEG, '/attention/att_1/resolve')); + assert.equal(calls[0].method, 'POST'); + // {} on the wire: NULLIF('') → NULL is the hub's human/system-dismiss + // convention; any value here must be a real agents.id or the UPDATE + // FK-violates and the card never closes. + assert.deepEqual(calls[0].body, {}); +}); + +test('dismissCard swallows transport errors (best-effort tidy-up)', async () => { + const { fetchFn } = scriptedFetch([new Error('conn refused')]); + await dismissCard(LEG, 'att_1', fetchFn); // must not throw +}); + +// ── Raising ────────────────────────────────────────────────────────────────── + +test('raiseCard posts a desktop_action card and returns its id', async () => { + const { fetchFn, calls } = scriptedFetch([json(201, { id: 'att_9' })]); + const id = await raiseCard(LEG, CARD.summary, CARD.payload, 'pi-a', fetchFn); + assert.equal(id, 'att_9'); + const body = calls[0].body as Record; + assert.equal(body.kind, 'desktop_action'); + assert.equal(body.scope_kind, 'team'); + assert.equal(body.actor_handle, 'pi-a'); + assert.deepEqual(body.pending_payload, CARD.payload); +}); + +test('raiseCard maps hub refusal, bodyless success and transport error to null', async () => { + for (const script of [[json(500, {})], [json(201, {})], [new Error('down')]]) { + const { fetchFn } = scriptedFetch(script); + assert.equal(await raiseCard(LEG, CARD.summary, CARD.payload, 'pi-a', fetchFn), null); + } +}); + +// ── The whole approval leg ─────────────────────────────────────────────────── + +test('approveViaCard: approve resolves without a dismiss', async () => { + const { fetchFn, calls } = scriptedFetch([ + json(201, { id: 'att_1' }), + json(200, { status: 'resolved', decisions: [{ decision: 'approve' }] }), + ]); + const clock = fakeClock(); + assert.equal(await approveViaCard(LEG, CARD, 'pi-a', { ...clock, fetchFn }), 'approve'); + assert.equal(calls.filter((c) => c.url.endsWith('/resolve')).length, 0); +}); + +test('approveViaCard: a denied card is denied, and NOT dismissed (it already resolved)', async () => { + const { fetchFn, calls } = scriptedFetch([ + json(201, { id: 'att_1' }), + json(200, { status: 'resolved', decisions: [{ decision: 'reject' }] }), + ]); + const clock = fakeClock(); + assert.equal(await approveViaCard(LEG, CARD, 'pi-a', { ...clock, fetchFn }), 'denied'); + assert.equal(calls.filter((c) => c.url.endsWith('/resolve')).length, 0); +}); + +test('approveViaCard: timeout dismisses the card and denies', async () => { + // Two polls fit in the window; both come back open; then the deadline + // passes and the leg tidies up after itself. + const { fetchFn, calls } = scriptedFetch([ + json(201, { id: 'att_1' }), + json(200, { status: 'open' }), + json(200, { status: 'open' }), + json(204, {}), + ]); + const clock = fakeClock(); + const verdict = await approveViaCard(LEG, CARD, 'pi-a', { ...clock, fetchFn, timeoutMs: UI_CAPTURE_POLL_MS * 2 }); + assert.equal(verdict, 'timeout'); + // The dismiss is fire-and-forget; let it land before inspecting the log. + await new Promise((r) => setTimeout(r, 0)); + const dismiss = calls.filter((c) => c.url.endsWith('/attention/att_1/resolve')); + assert.equal(dismiss.length, 1); + assert.deepEqual(dismiss[0].body, {}); +}); + +test('approveViaCard: poll transport errors are not decisions — the deadline still governs', async () => { + const { fetchFn } = scriptedFetch([ + json(201, { id: 'att_1' }), + new Error('hiccup'), + new Error('hiccup'), + json(204, {}), + ]); + const clock = fakeClock(); + const verdict = await approveViaCard(LEG, CARD, 'pi-a', { ...clock, fetchFn, timeoutMs: UI_CAPTURE_POLL_MS * 2 }); + assert.equal(verdict, 'timeout'); +}); + +test('approveViaCard: a hub that refuses the card is raise_failed, and never polled', async () => { + const { fetchFn, calls } = scriptedFetch([json(500, {})]); + const clock = fakeClock(); + assert.equal(await approveViaCard(LEG, CARD, 'pi-a', { ...clock, fetchFn }), 'raise_failed'); + assert.equal(calls.length, 1); +}); diff --git a/desktop/electron/src/uicapture_hub.ts b/desktop/electron/src/uicapture_hub.ts new file mode 100644 index 00000000..d3e3fd5f --- /dev/null +++ b/desktop/electron/src/uicapture_hub.ts @@ -0,0 +1,152 @@ +/// The hub approval round trip for ui_screenshot (D3 — +/// docs/plans/desktop-ui-context-and-pointing.md §3.3), electron-free so +/// `node --test` can drive the raise → poll → dismiss sequencing against a +/// mocked fetch. The policy (card shape, decision reading, denial wording) +/// stays in uicapture.ts; the Electron halves (capturePage, the keychain +/// token) stay in uicapture_host.ts. +import { + readCaptureDecision, + UI_CAPTURE_APPROVAL_TIMEOUT_MS, + UI_CAPTURE_POLL_MS, + type CaptureOutcome, +} from './uicapture.ts'; + +export interface HubLeg { + baseUrl: string; + teamId: string; + token: string; +} + +export type FetchLike = (url: string, init?: RequestInit) => Promise; + +export function teamUrl(leg: HubLeg, suffix: string): string { + return `${leg.baseUrl}/v1/teams/${encodeURIComponent(leg.teamId)}${suffix}`; +} + +/// Raise the per-call card. Returns its id, or null when the hub refused it — +/// which denies the capture (we never fall back to capturing unasked). +/// +/// `actor_handle` names the requesting agent, but the hub honours the body +/// field only when the caller's token carries no handle of its own +/// (host-runner tokens — handleCreateAttention). This desktop's token belongs +/// to the signed-in user, so the ROW's actor stays the user — deliberately: +/// letting a user token attribute rows to arbitrary agents would be a +/// spoofing lever. The agent is named authoritatively in the summary and in +/// payload.agent_id. +export async function raiseCard( + leg: HubLeg, + summary: string, + payload: Record, + agentHandle: string, + fetchFn: FetchLike = fetch, +): Promise { + try { + const res = await fetchFn(teamUrl(leg, '/attention'), { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${leg.token}` }, + body: JSON.stringify({ + scope_kind: 'team', + scope_id: leg.teamId, + kind: 'desktop_action', + summary, + severity: 'minor', + actor_handle: agentHandle, + pending_payload: payload, + }), + signal: AbortSignal.timeout(10_000), + }); + if (!res.ok) return null; + const body = (await res.json()) as { id?: unknown }; + return typeof body.id === 'string' && body.id !== '' ? body.id : null; + } catch { + return null; + } +} + +/// Poll one card until it resolves or the window closes. A transport error is +/// NOT a decision — it just costs one poll interval, and the deadline still +/// governs. +export async function awaitCard( + leg: HubLeg, + id: string, + deadline: number, + now: () => number, + sleep: (ms: number) => Promise, + fetchFn: FetchLike = fetch, +): Promise { + for (;;) { + if (now() >= deadline) return 'pending'; + await sleep(UI_CAPTURE_POLL_MS); + let outcome: CaptureOutcome = 'pending'; + try { + const res = await fetchFn(teamUrl(leg, `/attention/${encodeURIComponent(id)}`), { + headers: { authorization: `Bearer ${leg.token}` }, + signal: AbortSignal.timeout(10_000), + }); + if (res.ok) outcome = readCaptureDecision(await res.json()); + } catch { + /* transport hiccup — try again inside the deadline */ + } + if (outcome !== 'pending') return outcome; + } +} + +/// Best-effort tidy-up so an unanswered card does not loiter in the inbox +/// after the agent gave up. `desktop_action` owes no agent reply, so /resolve +/// (the dismiss path) is the right verb. +/// +/// The body deliberately names NO `resolved_by`: the hub column REFERENCES +/// agents(id) (NULLIF('') → NULL is the human/system-dismiss convention in +/// handleResolveAttention), and any non-agent value FK-violates — a 500 this +/// catch would swallow, leaving the card open forever. Pinned by test. +export async function dismissCard(leg: HubLeg, id: string, fetchFn: FetchLike = fetch): Promise { + try { + await fetchFn(teamUrl(leg, `/attention/${encodeURIComponent(id)}/resolve`), { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${leg.token}` }, + body: JSON.stringify({}), + signal: AbortSignal.timeout(5_000), + }); + } catch { + /* the row stays open; the director can dismiss it by hand */ + } +} + +/// How one local capture's approval leg ends. `raise_failed` is a reachable +/// hub that refused/errored the card — a different diagnosis than "signed +/// out", which the caller checks before ever building a card. +export type ApprovalVerdict = 'approve' | 'denied' | 'timeout' | 'raise_failed'; + +export interface ApprovalDeps { + now?: () => number; + sleep?: (ms: number) => Promise; + fetchFn?: FetchLike; + timeoutMs?: number; +} + +function defaultSleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/// The whole approval leg: raise the card, park on it, tidy up on timeout. +/// Every seam is injectable so the sequencing — including the FK-sensitive +/// dismiss body — is provable without Electron or a hub. +export async function approveViaCard( + leg: HubLeg, + card: { summary: string; payload: Record }, + agentHandle: string, + deps: ApprovalDeps = {}, +): Promise { + const now = deps.now ?? (() => Date.now()); + const sleep = deps.sleep ?? defaultSleep; + const fetchFn = deps.fetchFn ?? fetch; + const id = await raiseCard(leg, card.summary, card.payload, agentHandle, fetchFn); + if (id === null) return 'raise_failed'; + const outcome = await awaitCard(leg, id, now() + (deps.timeoutMs ?? UI_CAPTURE_APPROVAL_TIMEOUT_MS), now, sleep, fetchFn); + if (outcome === 'approve') return 'approve'; + if (outcome === 'pending') { + void dismissCard(leg, id, fetchFn); + return 'timeout'; + } + return 'denied'; +} diff --git a/desktop/electron/src/uiscreenshot.test.ts b/desktop/electron/src/uiscreenshot.test.ts new file mode 100644 index 00000000..6ba8332f --- /dev/null +++ b/desktop/electron/src/uiscreenshot.test.ts @@ -0,0 +1,232 @@ +/// Tests for the D3 bridge surface (docs/plans/desktop-ui-context-and-pointing.md +/// §3.3): `ui_screenshot`'s catalog gating, target resolution, error shaping +/// and — the part that is easy to get wrong — the audit posture. The policy +/// itself (what refuses, how a decision reads) is uicapture.test.ts; the +/// capture provider is faked here exactly like the CDP backend. `node --test`. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + dispatchHubInvoke, + handleMcpMessage, + READ_TOOLS, + shouldMirrorAudit, + type BridgeAuditEntry, + type BridgeBackend, + type BridgeTarget, + type McpServerDeps, + type UiCaptureRequest, + type UiCaptureResult, +} from './browserbridge.ts'; + +const KIMIWEB_URL = 'http://127.0.0.1:17331/#token=secret-in-the-fragment'; + +const TABS: BridgeTarget[] = [ + { tabId: 7, url: 'https://arxiv.org/abs/2401.00001', title: 'a paper', partition: 'persist:webtab', bridge: 'full' }, + { tabId: 9, url: KIMIWEB_URL, title: 'Kimi', partition: 'kimiweb', bridge: 'read' }, +]; + +const backend: BridgeBackend = { + listTargets: () => TABS, + sendCommand: async () => { + throw new Error('unexpected CDP call — ui_screenshot captures main-side, never over the guest debugger'); + }, +}; + +const PNG_B64 = 'aGVsbG8='; + +interface Harness { + deps: McpServerDeps; + seen: UiCaptureRequest[]; + audit: BridgeAuditEntry[]; +} + +function harness(opts: { sharing?: boolean; result?: UiCaptureResult; provider?: boolean } = {}): Harness { + const seen: UiCaptureRequest[] = []; + const audit: BridgeAuditEntry[] = []; + const result = opts.result ?? { ok: true, data_b64: PNG_B64, width: 800, height: 600 }; + const deps: McpServerDeps = { + backend, + serverInfo: { name: 'termipod-browser', version: '0.0.0-test' }, + uiFocusAvailable: () => opts.sharing !== false, + getUiFocus: () => ({ surface: 'read', captured_at: 'x' }), + onAction: (e) => audit.push(e), + ...(opts.provider === false + ? {} + : { + captureUi: async (req: UiCaptureRequest): Promise => { + seen.push(req); + return result; + }, + }), + }; + return { deps, seen, audit }; +} + +interface ToolResult { + content: Array<{ type?: string; text?: string; data?: string; mimeType?: string }>; + isError?: boolean; +} + +async function call(deps: McpServerDeps, args: Record = {}): Promise { + const res = await handleMcpMessage( + { jsonrpc: '2.0', id: 1, method: 'tools/call', params: { name: 'ui_screenshot', arguments: args } }, + deps, + { scope: 'read', agentId: 'ag_1', agentHandle: 'kimi-1' }, + ); + return res?.result as ToolResult; +} + +// ── Catalog + consent gating ───────────────────────────────────────────────── + +test('ui_screenshot is a READ-SCOPE tool gated by the sharing toggle', async () => { + // Read scope on purpose: the local kimi-code loop holds only the read token, + // and the consent that matters for a capture is the per-call card, not a + // spawn flag. (Its ACTION-ness lives in the approval + audit, below.) + assert.ok(READ_TOOLS.some((t) => t.name === 'ui_screenshot')); + + const on = await handleMcpMessage({ jsonrpc: '2.0', id: 1, method: 'tools/list' }, harness().deps, { scope: 'read', agentId: null }); + const listed = (on?.result as { tools: Array<{ name: string; description: string }> }).tools; + const tool = listed.find((t) => t.name === 'ui_screenshot'); + assert.ok(tool !== undefined, 'toggle on must list the tool'); + // The description is the only steering an agent gets: it must say the + // approval is per-call and point at the cheaper representations (§3.3/D-4). + assert.match(tool.description, /approval card/); + assert.match(tool.description, /ui_get_focus/); + assert.match(tool.description, /browser_snapshot/); + + const off = await handleMcpMessage({ jsonrpc: '2.0', id: 2, method: 'tools/list' }, harness({ sharing: false }).deps, { + scope: 'read', + agentId: null, + }); + const hidden = (off?.result as { tools: Array<{ name: string }> }).tools; + assert.ok(!hidden.some((t) => t.name === 'ui_screenshot'), 'toggle off must hide the tool'); +}); + +test('the toggle gate binds the CALL, not only the catalog', async () => { + const h = harness({ sharing: false }); + const out = await call(h.deps); + assert.equal(out.isError, true); + assert.match(out.content[0]?.text ?? '', /UI_UNAVAILABLE/); + assert.equal(h.seen.length, 0, 'a refused call never reaches the capture provider'); +}); + +test('a build with no capture provider refuses instead of throwing', async () => { + const out = await call(harness({ provider: false }).deps); + assert.equal(out.isError, true); + assert.match(out.content[0]?.text ?? '', /UI_UNAVAILABLE/); +}); + +// ── Target resolution ──────────────────────────────────────────────────────── + +test('no tabId captures the desktop window; the caller identity rides along', async () => { + const h = harness(); + const out = await call(h.deps); + assert.equal(out.isError, undefined); + assert.deepEqual(out.content, [{ type: 'image', data: PNG_B64, mimeType: 'image/png' }]); + assert.deepEqual(h.seen, [{ tabId: null, url: null, partition: null, agentId: 'ag_1', agentHandle: 'kimi-1', via: 'local' }]); +}); + +test('a tabId resolves against the live registry, fragment-stripped', async () => { + const h = harness(); + await call(h.deps, { tabId: 9 }); + // The guest URL reaches the approval card, so it must be stripped on the way + // — a kimiweb hash carries a bearer (ADR-059 D-5). + assert.deepEqual(h.seen[0], { + tabId: 9, + url: 'http://127.0.0.1:17331/', + partition: 'kimiweb', + agentId: 'ag_1', + agentHandle: 'kimi-1', + via: 'local', + }); +}); + +test('an unknown or malformed tabId is refused before the provider runs', async () => { + const h = harness(); + const gone = await call(h.deps, { tabId: 12345 }); + assert.equal(gone.isError, true); + assert.match(gone.content[0]?.text ?? '', /TARGET_GONE/); + + const bad = await call(h.deps, { tabId: 'seven' }); + assert.equal(bad.isError, true); + assert.match(bad.content[0]?.text ?? '', /INVALID_PARAMS/); + assert.equal(h.seen.length, 0); +}); + +test('a provider refusal surfaces its code and message verbatim', async () => { + const h = harness({ result: { ok: false, code: 'CAPTURE_REFUSED', message: "surface 'vault' refuses pixel capture" } }); + const out = await call(h.deps); + assert.equal(out.isError, true); + assert.match(out.content[0]?.text ?? '', /CAPTURE_REFUSED: surface 'vault' refuses pixel capture/); +}); + +// ── Audit posture (the part that is easy to get wrong) ─────────────────────── + +test('a LOCAL ui_screenshot is audited — unlike a local browser read', async () => { + const h = harness(); + await call(h.deps); + assert.equal(h.audit.length, 1, 'exactly one entry per call'); + assert.equal(h.audit[0]?.tool, 'ui_screenshot'); + assert.equal(h.audit[0]?.via, 'local'); + assert.equal(h.audit[0]?.ok, true); + assert.equal(h.audit[0]?.agent_id, 'ag_1'); + + // The contrast that makes the rule visible: a local browser READ is not + // audited at all (same-machine, high frequency — the ring would churn). + const reads = harness(); + await handleMcpMessage( + { jsonrpc: '2.0', id: 1, method: 'tools/call', params: { name: 'browser_list_tabs', arguments: {} } }, + reads.deps, + { scope: 'read', agentId: 'ag_1' }, + ); + assert.equal(reads.audit.length, 0); +}); + +test('a failed capture is audited too, with its code', async () => { + const h = harness({ result: { ok: false, code: 'CAPTURE_DENIED', message: 'the desktop user denied this screenshot' } }); + await call(h.deps); + assert.equal(h.audit.length, 1); + assert.equal(h.audit[0]?.ok, false); + assert.equal(h.audit[0]?.error, 'CAPTURE_DENIED'); +}); + +test('ui_screenshot mirrors to the hub on every leg; ui_get_focus does not', () => { + const entry = (tool: string, via: 'local' | 'hub'): BridgeAuditEntry => ({ + ts: '2026-07-31T00:00:00Z', + tool, + agent_id: 'ag_1', + via, + tab_id: null, + url: null, + partition: null, + args: {}, + ok: true, + error: null, + }); + // A capture is an action for audit purposes even though it sits in + // READ_TOOLS — the hub mirror is where the user's own event stream records + // that a frame of their screen was handed over. + assert.equal(shouldMirrorAudit(entry('ui_screenshot', 'hub')), true); + assert.equal(shouldMirrorAudit(entry('ui_screenshot', 'local')), true); + // Hub-leg reads stay ring-only (the hub already knows it routed them). + assert.equal(shouldMirrorAudit(entry('ui_get_focus', 'hub')), false); + assert.equal(shouldMirrorAudit(entry('browser_snapshot', 'hub')), false); +}); + +// ── The hub leg ────────────────────────────────────────────────────────────── + +test('a hub-relayed capture is marked via:hub so the desktop raises no second card', async () => { + const h = harness(); + const out = await dispatchHubInvoke(h.deps, { tool: 'ui_screenshot', args: {}, agent_id: 'ag_9', agent_handle: 'remote-1' }, new Set()); + assert.equal(out.ok, true); + assert.equal(h.seen[0]?.via, 'hub', 'the hub gated this call already (D5) — a second card would double-prompt'); + assert.equal(h.seen[0]?.agentId, 'ag_9'); + assert.equal(h.seen[0]?.agentHandle, 'remote-1'); +}); + +test('a revoked agent cannot capture, and never reaches the provider', async () => { + const h = harness(); + const out = await dispatchHubInvoke(h.deps, { tool: 'ui_screenshot', args: {}, agent_id: 'ag_9' }, new Set(['ag_9'])); + assert.equal(out.ok, false); + assert.equal(h.seen.length, 0); +}); diff --git a/desktop/src/i18n/index.ts b/desktop/src/i18n/index.ts index aaecca77..410b1959 100644 --- a/desktop/src/i18n/index.ts +++ b/desktop/src/i18n/index.ts @@ -429,6 +429,9 @@ const en: Dict = { 'att.replyPlaceholder': 'Reply…', 'att.answer': 'Answer', 'att.dismiss': 'Dismiss', + // D3: the desktop screenshot card. No "Allow session" button exists for it — + // this line says why, so the missing option reads as a decision. + 'att.perCallOnly': 'Screenshots are approved one call at a time — there is no standing grant.', 'kanban.todo': 'To do', 'kanban.in_progress': 'In progress', @@ -2491,6 +2494,7 @@ const zh: Dict = { 'att.replyPlaceholder': '回复…', 'att.answer': '回答', 'att.dismiss': '忽略', + 'att.perCallOnly': '截屏需逐次批准，不提供长期授权。', 'kanban.todo': '待办', 'kanban.in_progress': '进行中', diff --git a/desktop/src/surfaces/AttentionDock.tsx b/desktop/src/surfaces/AttentionDock.tsx index 5387b607..6919adbc 100644 --- a/desktop/src/surfaces/AttentionDock.tsx +++ b/desktop/src/surfaces/AttentionDock.tsx @@ -75,6 +75,19 @@ function AttentionCard({ item }: { item: Entity }): JSX.Element { {preview(pending['args']) !== '' &&
{preview(pending['args'])}
} )} + {/* D3 (docs/plans/desktop-ui-context-and-pointing.md §3.3): the gated + desktop screenshot. The card describes WHAT was asked for — which + surfaces are on screen, or which embedded tab — because the image + does not exist until this decision says yes. */} + {kind === 'desktop_action' && pending !== undefined && ( +
+ {str(pending, 'tool') ?? 'desktop tool'} + {str(pending, 'scope') !== undefined && · {str(pending, 'scope')}} + {preview(pending['surfaces']) !== '' &&
{preview(pending['surfaces'])}
} + {str(pending, 'url') !== undefined &&
{str(pending, 'url')}
} +
{t('att.perCallOnly')}
+
+ )} {changeKind !== undefined && (
{preview(item['change_spec'] ?? item['target_ref'])}
)} @@ -122,6 +135,18 @@ function AttentionCard({ item }: { item: Entity }): JSX.Element { {t('att.reject')} + ) : kind === 'desktop_action' ? ( + // Deliberately NO "Allow session" — a screenshot never gets a standing + // grant (plan §3.3, ADR-062 D-4). The tool parks on this decision and + // fails closed if it does not arrive. +
+ + +
) : (