diff --git a/desktop/electron/node_modules b/desktop/electron/node_modules new file mode 120000 index 00000000..aff7d313 --- /dev/null +++ b/desktop/electron/node_modules @@ -0,0 +1 @@ +/tmp/claude-1000/-home-ubuntu-embodylearn/74638504-a76b-41ae-bd64-c5e34c2e5d34/scratchpad/vaultfix-wt/desktop/electron/node_modules \ No newline at end of file diff --git a/desktop/electron/src/annotation.test.ts b/desktop/electron/src/annotation.test.ts index 3e5bfa46..31789337 100644 --- a/desktop/electron/src/annotation.test.ts +++ b/desktop/electron/src/annotation.test.ts @@ -14,8 +14,10 @@ import { annotationTempPath, fitWithin, injectImageIntoComposer, + injectNoteIntoComposer, isAnnotationTempFile, KIMI_FILE_INPUT_SELECTORS, + KIMI_TEXT_INPUT_SELECTORS, MAX_IMAGE_EDGE, normalizeDrag, pickCaptureTarget, @@ -155,6 +157,8 @@ function fakeCdp(nodeIds: number[]): { send: (m: string, p?: Record { assert.equal(await injectImageIntoComposer(send, '/tmp/x.png'), 'no-input'); }); +// ── kimi composer NOTE injection (D4 — the pointer must ride this path too) ── + +test('kimi note: focus the composer text box, then insert via the input pipeline', async () => { + const { send, calls } = fakeCdp([7]); + const res = await injectNoteIntoComposer(send, 'why is this red?\nPointing at a button…'); + assert.equal(res, 'injected'); + const focus = calls.findIndex((c) => c.method === 'DOM.focus'); + const insert = calls.findIndex((c) => c.method === 'Input.insertText'); + assert.ok(focus >= 0 && insert > focus, 'focus before insert'); + assert.equal(calls[focus].params?.nodeId, 7); + // Input.insertText rides the browser's own input pipeline (React's value + // tracker included) — never a .value= assignment the SPA can't see. + assert.equal(calls[insert].params?.text, 'why is this red?\nPointing at a button…'); +}); + +test('kimi note: textarea, then contenteditable — first match wins', async () => { + const tried: string[] = []; + const send = async (method: string, params?: Record): Promise => { + if (method === 'DOM.getDocument') return { root: { nodeId: 1 } }; + if (method === 'DOM.querySelectorAll') { + tried.push(String(params?.selector)); + return { nodeIds: params?.selector === KIMI_TEXT_INPUT_SELECTORS[KIMI_TEXT_INPUT_SELECTORS.length - 1] ? [5] : [] }; + } + return {}; + }; + assert.equal(await injectNoteIntoComposer(send, 'note'), 'injected'); + assert.deepEqual(tried, [...KIMI_TEXT_INPUT_SELECTORS]); +}); + +test('kimi note: no text box → no-input, and nothing inserted', async () => { + const { send, calls } = fakeCdp([]); + assert.equal(await injectNoteIntoComposer(send, 'note'), 'no-input'); + assert.ok(!calls.some((c) => c.method === 'Input.insertText')); +}); + // ── Companion chip → postAgentInput payload shape ──────────────────────────── test('compose: a staged annotation image rides att.images as raw base64 with the note as body', () => { diff --git a/desktop/electron/src/annotation.ts b/desktop/electron/src/annotation.ts index 74e956fc..94df10b3 100644 --- a/desktop/electron/src/annotation.ts +++ b/desktop/electron/src/annotation.ts @@ -245,3 +245,36 @@ export async function injectImageIntoComposer(send: CdpSend, filePath: string): } return 'attached'; } + +/// Where the composer's TEXT lands. Chat composers are either a textarea or a +/// contenteditable; same ordered-candidates posture as the file input above. +export const KIMI_TEXT_INPUT_SELECTORS: readonly string[] = ['textarea', '[contenteditable="true"]']; + +export type KimiNoteResult = 'injected' | 'no-input'; + +/// Put the user's note (their words + the D4 pointer line) into the kimi +/// composer's text box, beside the crop the file injection just attached — +/// without this, the kimi path delivers pixels only and the pre-send pointer +/// chip promises something the agent never receives (§3.4 step 5). +/// +/// `DOM.focus` + `Input.insertText`: the insertion rides the browser's own +/// input pipeline (React's value tracker included), exactly as typing would — +/// never a `.value=` assignment the SPA can't see. The user still reviews and +/// hits send in kimi's own UI; this NEVER sends. +export async function injectNoteIntoComposer(send: CdpSend, note: string): Promise { + const doc = (await send('DOM.getDocument', { depth: 1 })) as { root?: { nodeId?: number } }; + const rootId = doc.root?.nodeId; + if (rootId === undefined) return 'no-input'; + let inputNode: number | null = null; + for (const selector of KIMI_TEXT_INPUT_SELECTORS) { + const res = (await send('DOM.querySelectorAll', { nodeId: rootId, selector })) as { nodeIds?: number[] }; + if (Array.isArray(res.nodeIds) && res.nodeIds.length > 0) { + inputNode = res.nodeIds[0] ?? null; + break; + } + } + if (inputNode === null) return 'no-input'; + await send('DOM.focus', { nodeId: inputNode }); + await send('Input.insertText', { text: note }); + return 'injected'; +} diff --git a/desktop/electron/src/annotation_host.ts b/desktop/electron/src/annotation_host.ts index e2140a9a..4aaac28c 100644 --- a/desktop/electron/src/annotation_host.ts +++ b/desktop/electron/src/annotation_host.ts @@ -27,6 +27,7 @@ import path from 'node:path'; import { annotationTempPath, injectImageIntoComposer, + injectNoteIntoComposer, isAnnotationTempFile, fitWithin, pickCaptureTarget, @@ -36,12 +37,14 @@ import { type GuestRegion, type SurfaceRegion, } from './annotation'; -import { stripFragment } from './browserbridge'; +import { registerAnnotationRef, stripFragment } from './browserbridge'; import { recordUserOverlayAudit } from './browserbridge_host'; import { isUiSharingEnabled } from './desktopui'; +import { resolvePointer } from './uipointer'; import type { Handler } from './ipc/dispatch'; import { policyForGuest } from './webtab'; import { KIMIWEB_PARTITION } from './webtab_policy'; +import type { UiPointer } from '../../src/state/uiPointer'; // ── Arg narrowing (the renderer is our own, but handlers validate anyway) ──── @@ -152,6 +155,42 @@ function ensureDebugger(wc: WebContents): void { wc.once('destroyed', () => attached.delete(wc.id)); } +// ── D4 element-resolved pointing ───────────────────────────────────────────── + +/// Resolve the element under the crop's centre, for a rect that landed on a +/// bridgeable guest. Best-effort by design: the crop is the deliverable and a +/// pointer is the bonus, so every failure path (no debugger, no AX tree, a +/// canvas app with nothing to name) yields `null` and the capture still +/// returns. An interactive hit is registered with the bridge (merge, never +/// replacing the map the agent's last snapshot minted) so the ref in the +/// message is one `browser_click` can resolve. +/// +/// The point is the CLIPPED rect's centre: a drag straddling the guest edge +/// hit-tests the centre of the part inside the guest, not the user's whole +/// drag — the part outside the guest has no elements to name. +async function pointerForGuest(wc: WebContents, rect: AnnotRect): Promise { + const policy = policyForGuest(wc); + // `none` partitions are outside the bridge entirely — an @eN ref for a tab + // no tool can address would be a reference to nowhere. + if (policy === null || policy.bridge === 'none') return null; + try { + ensureDebugger(wc); + const resolved = await resolvePointer( + (method, params) => wc.debugger.sendCommand(method, params ?? {}), + wc.id, + { x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 }, + { actionable: policy.bridge === 'full' }, + ); + if (resolved === null) return null; + if (resolved.refBackendNodeId !== null) { + resolved.pointer.ref = registerAnnotationRef(wc.id, resolved.refBackendNodeId); + } + return resolved.pointer; + } catch { + return null; + } +} + // ── Handlers ───────────────────────────────────────────────────────────────── let captureSeq = 0; @@ -175,11 +214,13 @@ export const annotationHostHandlers: Record = { const target = pickCaptureTarget(rect, guests); let wc: WebContents; let partition: string | null = null; + let guestForPointer: WebContents | null = null; if (target.kind === 'guest') { const guest = resolveGuest(target.id); if (guest === null) return { ok: false, error: 'guest-gone' }; wc = guest; partition = policyForGuest(guest)?.partition ?? null; + guestForPointer = guest; } else { if (ctx.win === null || ctx.win.isDestroyed()) return { ok: false, error: 'no-window' }; wc = ctx.win.webContents; @@ -205,13 +246,24 @@ export const annotationHostHandlers: Record = { ? img.resize({ width: pfit.width, height: pfit.height }) : img ).toDataURL(); + // D4: the structural half of the same gesture. Resolved AFTER the + // pixels so a pointer failure can never cost the user their crop. + const pointer = guestForPointer !== null ? await pointerForGuest(guestForPointer, target.rect) : null; recordUserOverlayAudit({ ts: new Date().toISOString(), tool: 'ui_annotate_capture', tab_id: target.kind === 'guest' ? target.id : null, url: target.kind === 'guest' ? stripFragment(wc.getURL()) : null, partition, - args: { rect: [rect.x, rect.y, rect.width, rect.height], target: target.kind, width: fit.width, height: fit.height }, + args: { + rect: [rect.x, rect.y, rect.width, rect.height], + target: target.kind, + width: fit.width, + height: fit.height, + // The audit says WHICH element was pointed at — the ref and role, + // never the crop (the entry is a reference, ADR-062 D-2). + ...(pointer !== null ? { pointer: { ref: pointer.ref ?? null, role: pointer.role ?? null } } : {}), + }, ok: true, error: null, }); @@ -223,6 +275,7 @@ export const annotationHostHandlers: Record = { preview, data_b64: png.toString('base64'), target: target.kind, + ...(pointer !== null ? { pointer } : {}), }; } catch (e) { recordUserOverlayAudit({ @@ -240,12 +293,16 @@ export const annotationHostHandlers: Record = { }, /// "Attach to kimi web": inject the crop into the kimi composer's file - /// input. The renderer names the kimi guest (it knows which panel is open); - /// main re-validates the id is a live kimiweb-partition guest before - /// attaching — this path can never reach a webtab or the shell. + /// input, and the note (the user's words + the D4 pointer line) into its + /// text box — the kimi path must deliver the same pointer the companion + /// path folds into postAgentInput, or the pre-send chip promises what the + /// agent never receives. The renderer names the kimi guest (it knows which + /// panel is open); main re-validates the id is a live kimiweb-partition + /// guest before attaching — this path can never reach a webtab or the shell. annotation_attach_kimi: async (args) => { if (!isUiSharingEnabled()) return { ok: false, error: 'sharing-off' }; const file = typeof args.file === 'string' ? args.file : ''; + const note = typeof args.note === 'string' ? args.note : ''; const guestId = typeof args.guest_id === 'number' && Number.isInteger(args.guest_id) ? args.guest_id : null; if (!isAnnotationTempFile(os.tmpdir(), file) || !fs.existsSync(file)) return { ok: false, error: 'bad-file' }; if (guestId === null) return { ok: false, error: 'bad-guest' }; @@ -286,11 +343,24 @@ export const annotationHostHandlers: Record = { try { ensureDebugger(wc); - const res = await injectImageIntoComposer((method, params) => wc.debugger.sendCommand(method, params ?? {}), file); + const send = (method: string, params?: Record): Promise => wc.debugger.sendCommand(method, params ?? {}); + const res = await injectImageIntoComposer(send, file); if (res !== 'attached') return fallback('no-input'); + // The note rides beside the crop. Its failure must not cost the + // attachment (the crop is the deliverable) — the result says whether + // it landed so the renderer can tell the user to type it themselves. + // The audit records the FLAG only, never the note's content. + let noteInjected: boolean | undefined; + if (note !== '') { + try { + noteInjected = (await injectNoteIntoComposer(send, note)) === 'injected'; + } catch { + noteInjected = false; + } + } wc.focus(); - audit(true, { via: 'file-input' }, null); - return { ok: true, injected: true }; + audit(true, { via: 'file-input', ...(noteInjected !== undefined ? { note_injected: noteInjected } : {}) }, null); + return { ok: true, injected: true, ...(noteInjected !== undefined ? { note_injected: noteInjected } : {}) }; } catch (e) { return fallback(e instanceof Error ? e.message : String(e)); } diff --git a/desktop/electron/src/browserbridge.ts b/desktop/electron/src/browserbridge.ts index 931e3c6d..f92bc9e4 100644 --- a/desktop/electron/src/browserbridge.ts +++ b/desktop/electron/src/browserbridge.ts @@ -673,6 +673,36 @@ export function pruneSnapshotRefs(tabId: number): void { snapshotRefs.delete(tabId); } +/// Register ONE ref minted outside a `browser_snapshot` call — D4's +/// annotation pointer names an element on the user's gesture, and the ref it +/// hands the agent has to be one `browser_click` can resolve. Deliberately +/// NOT latest-wins over the whole map: replacing the tab's map would renumber +/// refs the agent still holds from its last snapshot, silently retargeting an +/// agent-held `@e5` at a different element (no REF_STALE — the ref would +/// still exist). Instead the pointer MERGES: +/// +/// - if the agent's last snapshot already named this node, reuse that ref — +/// the agent recognizes it; +/// - otherwise mint an `@aN` (annotation namespace, monotonic across tabs, +/// never colliding with `@eN` or a previous `@aN`) and add it alongside +/// the existing entries. +/// +/// A later browser_snapshot still replaces the whole map, so `@aN` dies with +/// it — the ordinary REF_STALE contract. +let annotationRefSeq = 0; + +export function registerAnnotationRef(tabId: number, backendNodeId: number): string { + const map = snapshotRefs.get(tabId) ?? new Map(); + for (const [ref, backend] of map) { + if (backend === backendNodeId) return ref; + } + annotationRefSeq += 1; + const ref = `@a${String(annotationRefSeq)}`; + map.set(ref, backendNodeId); + snapshotRefs.set(tabId, map); + return ref; +} + /// Resolve + validate a tabId argument against the live registry. Throws a /// coded BridgeError the tool wrapper renders as an isError result. function requireTarget(deps: McpServerDeps, args: Record): BridgeTarget { diff --git a/desktop/electron/src/uipointer.test.ts b/desktop/electron/src/uipointer.test.ts new file mode 100644 index 00000000..04141c2f --- /dev/null +++ b/desktop/electron/src/uipointer.test.ts @@ -0,0 +1,173 @@ +/// Tests for element-resolved pointing (D4 — +/// docs/plans/desktop-ui-context-and-pointing.md §3.4 step 5). The whole CDP +/// sequence runs against a fake sender, the same seam annotation.test.ts uses +/// for the kimi injection — so the hit-test → backendNodeId → @eN → role/name +/// chain is provable without a browser. `node --test`. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { POINTER_INTERACTIVE_SELECTOR, clipName, pointerHitScript, resolvePointer } from './uipointer.ts'; +import { registerAnnotationRef } from './browserbridge.ts'; +import type { CdpSend } from './annotation.ts'; + +/// A page with one button (backendNodeId 42) that the AX tree names, plus a +/// static-text node that mints no ref. +const AX_NODES = [ + { nodeId: '1', role: { value: 'WebArea' }, name: { value: 'deploys' }, childIds: ['2', '3'] }, + { nodeId: '2', role: { value: 'button' }, name: { value: 'Deploy' }, backendDOMNodeId: 42 }, + { nodeId: '3', role: { value: 'StaticText' }, name: { value: 'last run 3m ago' }, backendDOMNodeId: 43 }, +]; + +interface FakeOpts { + /// The object the hit-test resolves to; null models "nothing under there". + hitObjectId?: string | null; + backendNodeId?: number; + nodeName?: string; + /// Fail this method to model a page that answers nothing useful. + failMethod?: string; + partialNodes?: unknown[]; +} + +function fakeSend(opts: FakeOpts = {}): CdpSend & { calls: string[] } { + const calls: string[] = []; + const send = async (method: string, params?: Record): Promise => { + calls.push(method); + if (method === opts.failMethod) throw new Error(`${method} unavailable`); + switch (method) { + case 'DOM.getDocument': + return { root: { nodeId: 1 } }; + case 'Runtime.evaluate': + return opts.hitObjectId === null ? { result: { subtype: 'null' } } : { result: { objectId: opts.hitObjectId ?? 'obj-1' } }; + case 'DOM.requestNode': + return { nodeId: 77 }; + case 'DOM.describeNode': + return { node: { backendNodeId: opts.backendNodeId ?? 42, nodeName: opts.nodeName ?? 'BUTTON' } }; + case 'Runtime.releaseObject': + return {}; + case 'Accessibility.getFullAXTree': + return { nodes: AX_NODES }; + case 'Accessibility.getPartialAXTree': { + if (opts.partialNodes !== undefined) return { nodes: opts.partialNodes }; + const backend = params?.backendNodeId; + return { nodes: AX_NODES.filter((n) => n.backendDOMNodeId === backend) }; + } + default: + throw new Error(`unexpected CDP method ${method}`); + } + }; + return Object.assign(send, { calls }); +} + +// ── The hit-test expression ────────────────────────────────────────────────── + +test('the hit script rounds coordinates and climbs to the interactive ancestor', () => { + const script = pointerHitScript(120.4, 88.6); + assert.match(script, /elementFromPoint\(120,89\)/); + // The user drags over a button's LABEL and means the button. + assert.match(script, /closest\(/); + assert.ok(script.includes(JSON.stringify(POINTER_INTERACTIVE_SELECTOR))); + // No interpolation seam: the selector is JSON-quoted and the coordinates are + // numbers, so nothing string-shaped from a caller reaches the evaluator. + assert.ok(!script.includes('${')); +}); + +// ── The happy path ─────────────────────────────────────────────────────────── + +test('resolvePointer names the element and hands back the node to register', async () => { + const send = fakeSend(); + const out = await resolvePointer(send, 3, { x: 100, y: 50 }, { actionable: true }); + assert.ok(out !== null); + // No ref here: the CALLER registers the node with the bridge (merge, never + // clobbering an agent-held snapshot map) and fills in the returned name. + assert.deepEqual(out.pointer, { tab_id: 3, role: 'button', name: 'Deploy', actionable: true }); + // Interactivity was decided by the same compaction browser_snapshot uses. + assert.equal(out.refBackendNodeId, 42); + // The object handle is always released, even on the happy path. + assert.ok(send.calls.includes('Runtime.releaseObject')); +}); + +test('a read-only partition is reported as such, not silently promised', async () => { + const out = await resolvePointer(fakeSend(), 9, { x: 1, y: 1 }, { actionable: false }); + assert.equal(out?.pointer.actionable, false); + assert.equal(out?.refBackendNodeId, 42, 'a ref is still useful as a reference'); +}); + +// ── Honest degradation ─────────────────────────────────────────────────────── + +test('nothing under the point → no pointer, not a fabricated one', async () => { + assert.equal(await resolvePointer(fakeSend({ hitObjectId: null }), 3, { x: 0, y: 0 }, { actionable: true }), null); +}); + +test('a non-interactive element resolves without a node to register', async () => { + // backendNodeId 43 is the static text: the AX tree names it, but + // compactAxTree mints refs for interactive roles only — and a ref-less + // resolution must not touch the tab's ref map at all. + const out = await resolvePointer(fakeSend({ backendNodeId: 43, nodeName: 'SPAN' }), 3, { x: 0, y: 0 }, { actionable: true }); + assert.equal(out?.refBackendNodeId, null); + assert.equal(out?.pointer.ref, undefined); + assert.equal(out?.pointer.role, 'statictext'); + assert.equal(out?.pointer.name, 'last run 3m ago'); +}); + +test('no accessibility node → the DOM tag name, never an invented role', async () => { + const out = await resolvePointer(fakeSend({ backendNodeId: 99, nodeName: 'CANVAS', partialNodes: [] }), 3, { x: 0, y: 0 }, { + actionable: true, + }); + assert.equal(out?.pointer.role, 'canvas'); + assert.equal(out?.pointer.name, undefined); + assert.equal(out?.refBackendNodeId, null); +}); + +test('a failing accessibility call degrades to the tag name instead of throwing', async () => { + const out = await resolvePointer(fakeSend({ failMethod: 'Accessibility.getPartialAXTree' }), 3, { x: 0, y: 0 }, { + actionable: true, + }); + // The full tree still decided interactivity; only role/name fell back. + assert.equal(out?.refBackendNodeId, 42); + assert.equal(out?.pointer.role, 'button'); +}); + +test('a CDP failure propagates — the HOST decides a pointer is optional, not this', async () => { + const send = fakeSend({ failMethod: 'DOM.describeNode' }); + await assert.rejects(() => resolvePointer(send, 3, { x: 0, y: 0 }, { actionable: true }), /DOM.describeNode unavailable/); + // …and the object handle is still released on the way out. + assert.ok(send.calls.includes('Runtime.releaseObject')); +}); + +// ── Names are references, not content ──────────────────────────────────────── + +test('accessible names are flattened and clipped', () => { + assert.equal(clipName(' Deploy to\nprod '), 'Deploy to prod'); + const long = 'x'.repeat(200); + const clipped = clipName(long); + assert.equal(clipped.length, 80); + assert.ok(clipped.endsWith('…')); +}); + +test('a paragraph-length accessible name never rides along whole', async () => { + const wordy = [{ role: { value: 'button' }, name: { value: 'y'.repeat(500) }, backendDOMNodeId: 42, nodeId: '2' }]; + const out = await resolvePointer(fakeSend({ partialNodes: wordy }), 3, { x: 0, y: 0 }, { actionable: true }); + assert.ok((out?.pointer.name ?? '').length <= 80, 'the pointer is a reference, not a content channel'); +}); + +// ── Registering the ref: merge, never clobber ──────────────────────────────── + +test('registerAnnotationRef merges into the tab map — earlier refs survive later registrations', () => { + // Distinct tab ids per test run: the registry is module-global. + const tab = 91_001; + const first = registerAnnotationRef(tab, 42); + assert.match(first, /^@a\d+$/, 'annotation refs live in their own namespace, never colliding with @eN'); + // A second element on the same tab mints a NEW ref… + const second = registerAnnotationRef(tab, 43); + assert.notEqual(second, first); + // …and the first entry is still there: re-registering node 42 answers the + // ref the agent already holds instead of renumbering it. (This is the M2 + // clobber regression: a whole-map replacement would have dropped it.) + assert.equal(registerAnnotationRef(tab, 42), first); + assert.equal(registerAnnotationRef(tab, 43), second); +}); + +test('registerAnnotationRef keeps tabs independent', () => { + const a = registerAnnotationRef(91_002, 42); + const b = registerAnnotationRef(91_003, 42); + assert.notEqual(a, b, 'same node id on different tabs is two different registrations'); +}); diff --git a/desktop/electron/src/uipointer.ts b/desktop/electron/src/uipointer.ts new file mode 100644 index 00000000..32fc3319 --- /dev/null +++ b/desktop/electron/src/uipointer.ts @@ -0,0 +1,149 @@ +/// Element-resolved pointing — the CDP half (D4 — +/// docs/plans/desktop-ui-context-and-pointing.md §3.4 step 5). +/// +/// D2 gave the agent pixels of the region the user dragged over. When that +/// region is a `` guest, the bridge can do better: the same +/// accessibility machinery `browser_snapshot` uses already names the elements +/// under those pixels, so the message can carry a STRUCTURAL pointer beside +/// the image — and, on an action-drivable partition, the very `@eN` ref +/// `browser_click` takes. +/// +/// The sequence, against the guest's debugger session: +/// +/// 1. hit-test the rect centre in the page (`elementFromPoint`, then +/// `closest()` up to the nearest interactive ancestor — the user drags +/// over a button's LABEL, and means the button); +/// 2. turn that JS object into a `backendNodeId` (requestNode → describeNode) +/// — the id the accessibility tree keys on; +/// 3. take the tab's AX snapshot and compact it exactly as `browser_snapshot` +/// does, which MINTS the `@eN` refs; the caller registers them, so the +/// ref in the message is live for a follow-up `browser_click`; +/// 4. read role + name for that node (`getPartialAXTree`), falling back to +/// the DOM tag name when the page exposes no accessibility node. +/// +/// Every step degrades to "no pointer" rather than failing the capture: the +/// crop is the deliverable, the pointer is the bonus. Electron-free (the +/// `CdpSend` seam is annotation.ts's), so `node --test` drives the whole +/// sequence against a fake. +import { compactAxTree, type AxNode } from './browserbridge.ts'; +import type { CdpSend } from './annotation.ts'; +import type { UiPointer } from '../../src/state/uiPointer.ts'; + +/// The ancestors worth pointing at. Deliberately the same shape as the AX +/// tree's interactive roles: if the user drags over the text inside a button, +/// the pointer should name the button, not the text node — and only these get +/// `@eN` refs, so anything else would resolve to a ref-less pointer anyway. +export const POINTER_INTERACTIVE_SELECTOR = + 'a[href],button,input,select,textarea,summary,label,[role],[onclick],[tabindex],[contenteditable]'; + +/// The hit-test expression. Coordinates are interpolated as VALIDATED +/// integers — the only numbers that reach it come from the user's own drag, +/// but the rule ("no caller string ever enters an evaluated expression") is +/// the bridge's and holds here too. +export function pointerHitScript(x: number, y: number): string { + const px = Math.round(x); + const py = Math.round(y); + return `(function(){var el=document.elementFromPoint(${String(px)},${String(py)});if(!el)return null;return el.closest(${JSON.stringify(POINTER_INTERACTIVE_SELECTOR)})||el;})()`; +} + +export interface PointerResolution { + /// `ref` is left unset here: the CALLER registers the node with the bridge + /// (registerAnnotationRef — merge semantics, never clobbering the map the + /// agent's last snapshot minted) and fills in whatever name that returns. + pointer: UiPointer; + /// The node to register, when the element is interactive (this compaction + /// minted a ref for it). null = a ref-less pointer: nothing to register, + /// and the tab's existing ref map must not be touched. + refBackendNodeId: number | null; +} + +/// Resolve the element under `point` (guest CSS px, i.e. the capture rect's +/// centre translated into the guest's own coordinate space). Returns null when +/// there is nothing to point at; throws only if the caller's `send` does. +/// +/// This runs AFTER capturePage (a pointer failure must never cost the crop), +/// so it hit-tests what is under the point NOW, not what the crop shows — a +/// navigation, scroll, or SPA re-render in between can make the pointer name +/// a different element than the pixels. The pre-send chip is the mitigation: +/// the user sees what the agent will be told and re-selects on a mismatch. +export async function resolvePointer( + send: CdpSend, + tabId: number, + point: { x: number; y: number }, + opts: { actionable: boolean }, +): Promise { + // `DOM.requestNode` needs the DOM agent to hold a document; getting it is + // also how the annotation injection primes its own session. + await send('DOM.getDocument', { depth: 0 }); + + const hit = (await send('Runtime.evaluate', { expression: pointerHitScript(point.x, point.y), returnByValue: false })) as { + result?: { objectId?: string; subtype?: string }; + }; + const objectId = hit.result?.objectId; + if (typeof objectId !== 'string' || hit.result?.subtype === 'null') return null; + + let backendNodeId: number | null = null; + let tagName = ''; + try { + const requested = (await send('DOM.requestNode', { objectId })) as { nodeId?: number }; + if (typeof requested.nodeId !== 'number' || requested.nodeId === 0) return null; + const described = (await send('DOM.describeNode', { nodeId: requested.nodeId })) as { + node?: { backendNodeId?: number; nodeName?: string }; + }; + if (typeof described.node?.backendNodeId !== 'number') return null; + backendNodeId = described.node.backendNodeId; + tagName = typeof described.node.nodeName === 'string' ? described.node.nodeName.toLowerCase() : ''; + } finally { + await send('Runtime.releaseObject', { objectId }).catch(() => undefined); + } + + // Interactivity is decided by the SAME compaction browser_snapshot + // performs — an element it would mint a ref for is one browser_click can + // address. Only the DECISION is taken from this compaction; the name the + // agent sees comes from the bridge's registry (registerAnnotationRef), so + // the map an earlier snapshot minted is merged into, never replaced. + const tree = (await send('Accessibility.getFullAXTree')) as { nodes?: AxNode[] }; + const compact = compactAxTree(tree.nodes ?? []); + let refable = false; + for (const backend of compact.refs.values()) { + if (backend === backendNodeId) { + refable = true; + break; + } + } + + const described = await describeAx(send, backendNodeId); + const pointer: UiPointer = { tab_id: tabId, actionable: opts.actionable }; + const role = described.role !== '' ? described.role : tagName; + if (role !== '') pointer.role = role; + if (described.name !== '') pointer.name = described.name; + return { pointer, refBackendNodeId: refable ? backendNodeId : null }; +} + +/// Role + accessible name for one DOM node. A page with no accessibility node +/// for it (canvas apps, aria-hidden subtrees) answers empty, and the caller +/// falls back to the tag name — an honest "a div" beats a fabricated role. +async function describeAx(send: CdpSend, backendNodeId: number): Promise<{ role: string; name: string }> { + try { + const partial = (await send('Accessibility.getPartialAXTree', { backendNodeId, fetchRelatives: false })) as { + nodes?: AxNode[]; + }; + const node = (partial.nodes ?? []).find((n) => n.backendDOMNodeId === backendNodeId && n.ignored !== true); + if (node === undefined) return { role: '', name: '' }; + return { + role: (node.role?.value ?? '').toLowerCase(), + name: clipName(node.name?.value ?? ''), + }; + } catch { + return { role: '', name: '' }; + } +} + +/// Accessible names can be paragraphs. The pointer is a reference, not a +/// content channel — the same discipline compactAxTree applies to its lines. +const POINTER_NAME_MAX = 80; + +export function clipName(name: string): string { + const flat = name.replace(/\s+/g, ' ').trim(); + return flat.length > POINTER_NAME_MAX ? `${flat.slice(0, POINTER_NAME_MAX - 1)}…` : flat; +} diff --git a/desktop/node_modules b/desktop/node_modules new file mode 120000 index 00000000..177ee488 --- /dev/null +++ b/desktop/node_modules @@ -0,0 +1 @@ +/tmp/claude-1000/-home-ubuntu-embodylearn/74638504-a76b-41ae-bd64-c5e34c2e5d34/scratchpad/vaultfix-wt/desktop/node_modules \ No newline at end of file diff --git a/desktop/src/i18n/index.ts b/desktop/src/i18n/index.ts index 410b1959..f7ae642c 100644 --- a/desktop/src/i18n/index.ts +++ b/desktop/src/i18n/index.ts @@ -1707,8 +1707,12 @@ const en: Dict = { 'annotate.notePlaceholder': 'Add a note (optional)', 'annotate.noTarget': 'No target — open the kimi panel or pick an agent first', 'annotate.attached': 'Attached in kimi — review and send there', + 'annotate.attachedNoNote': 'Attached in kimi — the note could not be added, type it in the composer', + 'annotate.pointerTab': 'tab {id}', 'annotate.clipboard': 'Copied — paste ({key}) into the kimi composer', 'annotate.failed': 'The annotation could not be captured.', + // D4: the title on the resolved-element chip in the target row. + 'annotate.pointing': 'The element the agent will be pointed at', 'debug.lines_one': '{n} line', 'debug.lines_other': '{n} lines', @@ -3748,8 +3752,11 @@ const zh: Dict = { 'annotate.notePlaceholder': '添加备注（可选）', 'annotate.noTarget': '没有可用目标——请打开 kimi 面板或先选择智能体', 'annotate.attached': '已附加到 kimi——请在那里检查并发送', + 'annotate.attachedNoNote': '已附加到 kimi——备注未能写入，请在输入框中手动输入', + 'annotate.pointerTab': '标签页 {id}', 'annotate.clipboard': '已复制——在 kimi 输入框中粘贴（{key}）', 'annotate.failed': '无法截取标注。', + 'annotate.pointing': '将指给智能体的元素', 'debug.lines': '{n} 行', 'debug.placeholder': '粘贴代码、差异、堆栈或日志…', diff --git a/desktop/src/state/annotation.ts b/desktop/src/state/annotation.ts index 396b7c40..7fc8a242 100644 --- a/desktop/src/state/annotation.ts +++ b/desktop/src/state/annotation.ts @@ -11,6 +11,7 @@ import { type AnnotationPhase, type CompanionTarget, } from './annotationTargets'; +import { appendPointerNote, type UiPointer } from './uiPointer'; // The origin / companion-target types + the target-row resolution + the pure // decisions (dock-hide flag, handoff routing/reveal) live in @@ -52,6 +53,11 @@ export interface AnnotationCapture { width: number; height: number; target: 'shell' | 'guest'; + /// D4: the element under the crop's centre, when the rect landed on a + /// bridgeable `` guest and the page had something to name. Absent + /// over the shell, and absent when resolution found nothing — the crop + /// never depends on it. + pointer?: UiPointer; } /// The companion-path handoff: the crop becomes a delete-able chip in the @@ -132,7 +138,11 @@ export const useAnnotation = create((set, get) => ({ refused: null, handoff: { storageKey: key, - note, + // D4: the pointer rides IN the note, because that is what actually + // reaches the model — `postAgentInput` carries a body and images, and + // a structured field beside them would be dropped on the floor. The + // user sees the line in their draft before they hit send. + note: appendPointerNote(note, capture.pointer), name: `annotation-${String(capture.width)}x${String(capture.height)}.png`, image: { mime_type: 'image/png', data: capture.dataB64 }, preview: capture.preview, diff --git a/desktop/src/state/uiPointer.test.ts b/desktop/src/state/uiPointer.test.ts new file mode 100644 index 00000000..e2663fac --- /dev/null +++ b/desktop/src/state/uiPointer.test.ts @@ -0,0 +1,69 @@ +/// Tests for the D4 pointer vocabulary (docs/plans/desktop-ui-context-and-pointing.md +/// §3.4 step 5): the two renderings of a resolved element — the chip the user +/// sees before sending, and the line the AGENT reads — plus the note fold that +/// puts the user's words first. Run locally: `node --test +/// src/state/uiPointer.test.ts` from `desktop/` (CI does not run these). +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { appendPointerNote, formatPointerLabel, formatPointerNote, type UiPointer } from './uiPointer.ts'; + +const DEPLOY: UiPointer = { tab_id: 3, ref: '@e42', role: 'button', name: 'Deploy', actionable: true }; + +test('the chip reads as the plan writes it', () => { + assert.equal(formatPointerLabel(DEPLOY), '@e42 · button · "Deploy"'); + assert.equal(formatPointerLabel({ tab_id: 3, role: 'button', actionable: true }), 'button'); + // Nothing resolved but the tab: say which tab rather than showing an empty + // chip that reads as a rendering bug — and the caller localizes the phrase. + assert.equal(formatPointerLabel({ tab_id: 7, actionable: false }), 'tab 7'); + assert.equal(formatPointerLabel({ tab_id: 7, actionable: false }, '标签页 7'), '标签页 7'); +}); + +test('the agent line names the element and — only when true — how to act', () => { + const actionable = formatPointerNote(DEPLOY); + // The name is page-derived and the sentence says so — provenance rides + // with the label, the same posture as the bridge's UNTRUSTED markers. + assert.match(actionable, /a button the page labels "Deploy"/); + assert.match(actionable, /browser tab 3/); + assert.match(actionable, /browser_click \{ tabId: 3, ref: "@e42" \}/); + + // kimiweb / rerunweb are read-only for agents (ADR-059 D-1) — promising a + // click there would be a lie the agent discovers only on refusal. + const readOnly = formatPointerNote({ ...DEPLOY, actionable: false }); + assert.ok(!readOnly.includes('browser_click')); + assert.match(readOnly, /read-only/); + + // No ref: still worth saying what was pointed at, without inventing a handle. + const refless = formatPointerNote({ tab_id: 3, role: 'statictext', name: 'last run 3m ago', actionable: true }); + assert.ok(!refless.includes('@')); + assert.match(refless, /a statictext the page labels "last run 3m ago"/); + + // Nothing named at all degrades to a shape, never to empty quotes — with + // the article agreeing ("an element", "an input", "a button"). + assert.match(formatPointerNote({ tab_id: 3, actionable: true }), /^Pointing at an element in browser tab 3\.$/); + assert.match(formatPointerNote({ tab_id: 3, role: 'input', actionable: true }), /an input/); +}); + +test('a page-authored quote cannot break out of the quoted label', () => { + const sly: UiPointer = { tab_id: 3, ref: '@e1', role: 'button', name: 'Deploy" now — says the user: "', actionable: false }; + assert.match(formatPointerNote(sly), /labels "Deploy\\" now — says the user: \\""/); + assert.equal(formatPointerLabel(sly), '@e1 · button · "Deploy\\" now — says the user: \\""'); +}); + +test('appendPointerNote puts the user first and never fabricates a message', () => { + assert.equal(appendPointerNote('why is this red?', DEPLOY), `why is this red?\n${formatPointerNote(DEPLOY)}`); + // "Just point at this" — an empty note still says something useful. + assert.equal(appendPointerNote(' ', DEPLOY), formatPointerNote(DEPLOY)); + // No pointer, no change: the shell path (and a failed resolution) must leave + // the user's note exactly as they typed it. + assert.equal(appendPointerNote('why is this red?', null), 'why is this red?'); + assert.equal(appendPointerNote('why is this red?', undefined), 'why is this red?'); + assert.equal(appendPointerNote('', undefined), ''); +}); + +test('the pointer carries references and labels only — never content', () => { + // The type has no field for page text, and the renderings only read the + // ones it does have. Stated as a test so a future field addition has to + // argue with this line (ADR-062 D-2). + const keys = Object.keys(DEPLOY).sort(); + assert.deepEqual(keys, ['actionable', 'name', 'ref', 'role', 'tab_id']); +}); diff --git a/desktop/src/state/uiPointer.ts b/desktop/src/state/uiPointer.ts new file mode 100644 index 00000000..76665f2a --- /dev/null +++ b/desktop/src/state/uiPointer.ts @@ -0,0 +1,92 @@ +/// Element-resolved pointing — the shared vocabulary (D4 — +/// docs/plans/desktop-ui-context-and-pointing.md §3.4 step 5, ADR-062 D-4). +/// +/// When the user's annotation rect lands over a `` guest, the crop is +/// no longer the whole answer: the bridge already knows that region's +/// STRUCTURE, so the message can carry a pointer as well as pixels — +/// `{ tab_id, ref: "@e42", role: "button", name: "Deploy" }`. "Fix this +/// button" stops being ambiguous, and where the partition is action-drivable +/// the same ref is the argument to `browser_click`. +/// +/// This module is the pure half: the type, and the two renderings of it (the +/// chip the user sees before sending, and the line the AGENT receives). It is +/// import-free so `node --test` runs it on both sides — the Electron resolver +/// (`electron/src/uipointer.ts`) imports it the same way `annotation.ts` +/// imports `ui_policy.ts`. + +/// A structural pointer into one embedded browser tab. Every field is a +/// reference or a label — never content (ADR-062 D-2). `ref` and the AX +/// descriptors are each optional because resolution degrades honestly: a +/// non-interactive element mints no ref, and a page with no accessibility +/// tree yields no role. +export interface UiPointer { + tab_id: number; + /// The `@eN` handle from the tab's accessibility snapshot. Absent when the + /// element under the rect is not interactive (only interactive nodes mint + /// refs — see compactAxTree). + ref?: string; + role?: string; + name?: string; + /// Whether `browser_click { ref }` can actually act on this tab: only + /// `full` partitions are action-drivable — kimiweb and rerunweb are + /// read-only by policy (ADR-059 D-1), so promising actionability there + /// would be a lie the agent discovers only on refusal. + actionable: boolean; +} + +/// Accessible names come from the PAGE — quotes inside one must not read as +/// closing the quoted span it renders in. +function quoted(name: string): string { + return `"${name.replace(/"/g, '\\"')}"`; +} + +/// The compact chip shown in the target row and the composer: `@e42 · button +/// · "Deploy"`. Missing pieces are dropped rather than rendered as blanks. +/// `tabFallback` is the (localized) label when nothing else resolved — the +/// caller passes its own t() string; the default keeps non-UI callers working. +export function formatPointerLabel(p: UiPointer, tabFallback?: string): string { + const parts: string[] = []; + if (p.ref !== undefined && p.ref !== '') parts.push(p.ref); + if (p.role !== undefined && p.role !== '') parts.push(p.role); + if (p.name !== undefined && p.name !== '') parts.push(quoted(p.name)); + return parts.length > 0 ? parts.join(' · ') : (tabFallback ?? `tab ${String(p.tab_id)}`); +} + +/// The line the AGENT reads. The image alone says "somewhere around here"; +/// this says which element, and — only when it is true — how to act on it. +/// Written as a sentence rather than JSON because it rides in the message +/// body, which is where a text-mode agent will actually look. +export function formatPointerNote(p: UiPointer): string { + const what = describeElement(p); + const where = `browser tab ${String(p.tab_id)}`; + if (p.ref === undefined || p.ref === '') { + return `Pointing at ${what} in ${where}.`; + } + const how = p.actionable + ? ` Act on it with browser_click { tabId: ${String(p.tab_id)}, ref: "${p.ref}" }.` + : ' That tab is read-only for agents, so the ref is for reference, not for clicking.'; + return `Pointing at ${what} in ${where} — ref ${p.ref}.${how}`; +} + +/// The name is page-derived and rides the user-trusted note channel, so the +/// sentence SAYS so ("the page labels …") instead of presenting it as the +/// user's words — the same provenance discipline the bridge's content tools +/// apply with their UNTRUSTED marker, scaled to an 80-char label. +function describeElement(p: UiPointer): string { + const role = p.role !== undefined && p.role !== '' ? p.role : 'element'; + const article = /^[aeiou]/i.test(role) ? 'an' : 'a'; + return p.name !== undefined && p.name !== '' + ? `${article} ${role} the page labels ${quoted(p.name)}` + : `${article} ${role}`; +} + +/// Fold the pointer into the user's note. The user's own words come first — +/// they are the message; the pointer is the grounding beneath it. An empty +/// note yields the pointer line alone, so "just point at this" still says +/// something useful. +export function appendPointerNote(note: string, pointer: UiPointer | null | undefined): string { + if (pointer === null || pointer === undefined) return note; + const line = formatPointerNote(pointer); + const trimmed = note.trim(); + return trimmed === '' ? line : `${trimmed}\n${line}`; +} diff --git a/desktop/src/styles/partials/08-plan-terminal-dock.css b/desktop/src/styles/partials/08-plan-terminal-dock.css index 228e9965..4b8a8654 100644 --- a/desktop/src/styles/partials/08-plan-terminal-dock.css +++ b/desktop/src/styles/partials/08-plan-terminal-dock.css @@ -1315,3 +1315,20 @@ flex: 1; min-width: 160px; } +/* D4: the structural pointer resolved under the crop (`@e42 · button · + "Deploy"`). Shown before sending so the user can see exactly what the agent + will be told it is looking at — and re-select if it named the wrong thing. */ +.annot-pointer { + flex: none; + max-width: 220px; + overflow: hidden; + padding: 2px var(--spacing-s8); + border: 1px solid var(--border); + border-radius: var(--radius-stadium); + background: var(--accent-tint); + color: var(--accent); + font-family: var(--font-mono); + font-size: var(--font-size-label); + white-space: nowrap; + text-overflow: ellipsis; +} diff --git a/desktop/src/ui/AnnotationOverlay.tsx b/desktop/src/ui/AnnotationOverlay.tsx index a7860708..dc0452f6 100644 --- a/desktop/src/ui/AnnotationOverlay.tsx +++ b/desktop/src/ui/AnnotationOverlay.tsx @@ -5,6 +5,7 @@ import { useAnnotation, resolveTargets, type AnnotationCapture } from '../state/ import { kimiAttachable, useAssistant } from '../state/assistant'; import { toast } from '../state/toast'; import { useUiContext } from '../state/uiContext'; +import { appendPointerNote, formatPointerLabel, type UiPointer } from '../state/uiPointer'; import { uiPolicyFor } from '../state/ui_policy'; import { isSplitVisible, useWorkbench } from '../state/workbench'; import { Icon } from './Icon'; @@ -54,11 +55,19 @@ interface CaptureResponse { preview?: string; data_b64?: string; target?: 'shell' | 'guest'; + /// D4: the element resolved under the crop's centre, when the rect landed + /// on a bridgeable guest. Absent over the shell, or when the page had + /// nothing to name. + pointer?: UiPointer; } interface AttachResponse { ok: boolean; injected?: boolean; + /// Whether the note (user words + pointer line) landed in the composer's + /// text box. Absent when no note was sent; false = the crop attached but + /// the user must type the note themselves. + note_injected?: boolean; fallback?: string; error?: string; } @@ -187,6 +196,7 @@ export function AnnotationOverlay(): JSX.Element | null { width: r.width ?? 0, height: r.height ?? 0, target: r.target ?? 'shell', + ...(r.pointer !== undefined ? { pointer: r.pointer } : {}), } satisfies AnnotationCapture); return; } @@ -285,9 +295,16 @@ export function AnnotationOverlay(): JSX.Element | null { if (capture === null || kimiId === null || busy) return; setBusy(true); try { - const r = await invoke('annotation_attach_kimi', { file: capture.file, guest_id: kimiId }); + // The kimi path delivers the SAME message the companion path would: + // the user's note with the D4 pointer line folded in, injected into + // the composer beside the crop (main-side, same user-gesture posture). + const composed = appendPointerNote(note.trim(), capture.pointer); + const r = await invoke('annotation_attach_kimi', { file: capture.file, guest_id: kimiId, note: composed }); if (r.ok === true && r.injected === true) { - toast.success(t('annotate.attached')); + // The crop landed; if the note did not, say so — a silent drop would + // leave the pointer chip promising what the agent never received. + if (composed !== '' && r.note_injected === false) toast.info(t('annotate.attachedNoNote')); + else toast.success(t('annotate.attached')); cancel(); // the temp file stays (the SPA may read lazily; the LRU reaps) // Land the user in kimi's composer to review and send (D2.2). revealAssistant('kimi'); @@ -318,6 +335,14 @@ export function AnnotationOverlay(): JSX.Element | null { {capture !== null && capture.preview !== '' && ( {t('annotate.title')} )} + {/* D4: what the agent will be told it is looking at. Shown BEFORE the + send so a mis-resolved element is the user's to correct — they + re-select rather than discovering it in the agent's reply. */} + {capture?.pointer !== undefined && ( + + {formatPointerLabel(capture.pointer, t('annotate.pointerTab').replace('{id}', String(capture.pointer.tab_id)))} + + )} **Status:** In flight (2026-07-31) — wedges D1–D6 below; D1+D2 (the LOCAL > kimi-web loop) are the first priority, remote/hub-relayed delivery (D5) > is second. **D1 shipped (#476), D2 shipped (#477) + D2.1 (#480); -> D3 implemented — PR pending.** Builds on the agent browser bridge +> D3 + D4 implemented — PRs pending.** Builds on the agent browser bridge > (W1–W3 shipped, `docs/plans/desktop-agent-browser-bridge.md`). > **Derives from > [ADR-062](../decisions/062-desktop-ui-as-agent-addressable-entity.md)** — > verbs over a first-class UI entity: UIRef both directions, one > per-surface policy table, three representations, agent pointing (D6). > **Audience:** principal · contributors · maintainers -> **Last verified vs code:** the D3 wedge on origin/main `4f5d86c3` +> **Last verified vs code:** the D3+D4 wedges on origin/main `4f5d86c3` > (rebased past the unified assistant dock #483): frontend + electron typecheck -> green, 283 electron `node --test` pass (D3 adds 21), 376 frontend +> green, 293 electron `node --test` pass (D3 +21, D4 +10), 380 frontend > state/ssh tests pass, `lint-desktop-tokens.sh` clean at baseline 65. -> D3's Electron halves (`capturePage`, the hub approval round trip) are -> **unexercised on this machine** — no display, no Electron binary — and -> need the Playwright/desktop pass +> The Electron halves (`capturePage`, the hub approval round trip, the +> live CDP hit-test) are **unexercised on this machine** — no display, +> no Electron binary — and need the Playwright/desktop pass **Review amendments (2026-07-30), folded into the body below.** The relay fallback is read-token-only (§3.5); the `mcp.json` entry pins a stable @@ -623,6 +623,42 @@ audit; AttentionDock card branch. **D4 — element-resolved pointing.** Rect-over-webtab → `@eN` ref via the bridge snapshot; structured pointer rides the attachment payload. +> **D4 as shipped.** The resolution chain is: hit-test the rect centre +> (`elementFromPoint`, then `closest()` to the nearest interactive ancestor — +> the user drags over a button's LABEL and means the button) → `requestNode` / +> `describeNode` for the `backendNodeId` → the tab's AX snapshot compacted by +> the SAME `compactAxTree` `browser_snapshot` uses, which mints the `@eN` → +> `getPartialAXTree` for role + name. Four decisions worth recording: +> +> - **The ref is live, and registering it never clobbers.** An interactive +> hit is MERGED into the tab's ref map (`registerAnnotationRef`): if the +> agent's last `browser_snapshot` already named the node, the pointer +> reuses that ref; otherwise it mints an `@aN` (annotation namespace) +> alongside the existing entries. A whole-map replacement would silently +> renumber refs the agent still holds — an agent-held `@e5` retargeted at a +> different element with no REF_STALE. A later snapshot still replaces the +> map, so `@aN` expires on the ordinary staleness contract. +> - **`actionable` is a field, not an assumption.** Only `bridge: 'full'` +> partitions are action-drivable; on kimiweb/rerunweb the note says the ref +> is for reference, because promising a click there is a lie the agent +> discovers only on refusal. +> - **The pointer rides IN the note, on BOTH paths.** `postAgentInput` +> carries a body and images; a structured field beside them would be +> dropped on the floor. So the line lands in the user's draft, above the +> send button — and the kimi path injects the same composed note into the +> kimi composer's text box beside the crop (`DOM.focus` + +> `Input.insertText`, the same user-gesture posture as the file +> injection; a failed injection is reported so the user types it, never +> silently dropped). The chip in the target row shows the pointer before +> either send. +> - **Every failure degrades to no pointer.** The crop is the deliverable. No +> debugger, no AX tree, a canvas app with nothing to name: the capture +> returns unchanged, and role falls back to the DOM tag name rather than +> inventing one. +> +> Names are flattened and clipped to 80 chars: the pointer is a reference, not +> a content channel (ADR-062 D-2). + **D5 — remote / hub-relayed delivery (second priority).** `desktop.invoke` tunnel kind + hub `desktop_ui_invoke` (read class first); generalized gate/grant/revoke (per-kind grants; reachable only behind the