feat(desktop): element-resolved pointing — the crop names an element (D4)

desktop/electron/node_modules

@@ -0,0 +1 @@
+/tmp/claude-1000/-home-ubuntu-embodylearn/74638504-a76b-41ae-bd64-c5e34c2e5d34/scratchpad/vaultfix-wt/desktop/electron/node_modules

desktop/electron/src/annotation.test.ts

@@ -14,8 +14,10 @@ import {
   annotationTempPath,
   fitWithin,
   injectImageIntoComposer,
+  injectNoteIntoComposer,
   isAnnotationTempFile,
   KIMI_FILE_INPUT_SELECTORS,
+  KIMI_TEXT_INPUT_SELECTORS,
   MAX_IMAGE_EDGE,
   normalizeDrag,
   pickCaptureTarget,
@@ -155,6 +157,8 @@ function fakeCdp(nodeIds: number[]): { send: (m: string, p?: Record<string, unkn
       if (method === 'DOM.setFileInputFiles') return {};
       if (method === 'Runtime.callFunctionOn') return {};
       if (method === 'Runtime.releaseObject') return {};
+      if (method === 'DOM.focus') return {};
+      if (method === 'Input.insertText') return {};
       throw new Error(`unexpected ${method}`);
     },
   };
@@ -214,6 +218,41 @@ test('kimi injection: an unresolvable node → no-input', async () => {
   assert.equal(await injectImageIntoComposer(send, '/tmp/x.png'), 'no-input');
 });
 
+// ── kimi composer NOTE injection (D4 — the pointer must ride this path too) ──
+
+test('kimi note: focus the composer text box, then insert via the input pipeline', async () => {
+  const { send, calls } = fakeCdp([7]);
+  const res = await injectNoteIntoComposer(send, 'why is this red?\nPointing at a button…');
+  assert.equal(res, 'injected');
+  const focus = calls.findIndex((c) => c.method === 'DOM.focus');
+  const insert = calls.findIndex((c) => c.method === 'Input.insertText');
+  assert.ok(focus >= 0 && insert > focus, 'focus before insert');
+  assert.equal(calls[focus].params?.nodeId, 7);
+  // Input.insertText rides the browser's own input pipeline (React's value
+  // tracker included) — never a .value= assignment the SPA can't see.
+  assert.equal(calls[insert].params?.text, 'why is this red?\nPointing at a button…');
+});
+
+test('kimi note: textarea, then contenteditable — first match wins', async () => {
+  const tried: string[] = [];
+  const send = async (method: string, params?: Record<string, unknown>): Promise<unknown> => {
+    if (method === 'DOM.getDocument') return { root: { nodeId: 1 } };
+    if (method === 'DOM.querySelectorAll') {
+      tried.push(String(params?.selector));
+      return { nodeIds: params?.selector === KIMI_TEXT_INPUT_SELECTORS[KIMI_TEXT_INPUT_SELECTORS.length - 1] ? [5] : [] };
+    }
+    return {};
+  };
+  assert.equal(await injectNoteIntoComposer(send, 'note'), 'injected');
+  assert.deepEqual(tried, [...KIMI_TEXT_INPUT_SELECTORS]);
+});
+
+test('kimi note: no text box → no-input, and nothing inserted', async () => {
+  const { send, calls } = fakeCdp([]);
+  assert.equal(await injectNoteIntoComposer(send, 'note'), 'no-input');
+  assert.ok(!calls.some((c) => c.method === 'Input.insertText'));
+});
+
 // ── Companion chip → postAgentInput payload shape ────────────────────────────
 
 test('compose: a staged annotation image rides att.images as raw base64 with the note as body', () => {

desktop/electron/src/annotation.ts

@@ -245,3 +245,36 @@ export async function injectImageIntoComposer(send: CdpSend, filePath: string):
   }
   return 'attached';
 }
+
+/// Where the composer's TEXT lands. Chat composers are either a textarea or a
+/// contenteditable; same ordered-candidates posture as the file input above.
+export const KIMI_TEXT_INPUT_SELECTORS: readonly string[] = ['textarea', '[contenteditable="true"]'];
+
+export type KimiNoteResult = 'injected' | 'no-input';
+
+/// Put the user's note (their words + the D4 pointer line) into the kimi
+/// composer's text box, beside the crop the file injection just attached —
+/// without this, the kimi path delivers pixels only and the pre-send pointer
+/// chip promises something the agent never receives (§3.4 step 5).
+///
+/// `DOM.focus` + `Input.insertText`: the insertion rides the browser's own
+/// input pipeline (React's value tracker included), exactly as typing would —
+/// never a `.value=` assignment the SPA can't see. The user still reviews and
+/// hits send in kimi's own UI; this NEVER sends.
+export async function injectNoteIntoComposer(send: CdpSend, note: string): Promise<KimiNoteResult> {
+  const doc = (await send('DOM.getDocument', { depth: 1 })) as { root?: { nodeId?: number } };
+  const rootId = doc.root?.nodeId;
+  if (rootId === undefined) return 'no-input';
+  let inputNode: number | null = null;
+  for (const selector of KIMI_TEXT_INPUT_SELECTORS) {
+    const res = (await send('DOM.querySelectorAll', { nodeId: rootId, selector })) as { nodeIds?: number[] };
+    if (Array.isArray(res.nodeIds) && res.nodeIds.length > 0) {
+      inputNode = res.nodeIds[0] ?? null;
+      break;
+    }
+  }
+  if (inputNode === null) return 'no-input';
+  await send('DOM.focus', { nodeId: inputNode });
+  await send('Input.insertText', { text: note });
+  return 'injected';
+}

desktop/electron/src/annotation_host.ts

@@ -27,6 +27,7 @@ import path from 'node:path';
 import {
   annotationTempPath,
   injectImageIntoComposer,
+  injectNoteIntoComposer,
   isAnnotationTempFile,
   fitWithin,
   pickCaptureTarget,
@@ -36,12 +37,14 @@ import {
   type GuestRegion,
   type SurfaceRegion,
 } from './annotation';
-import { stripFragment } from './browserbridge';
+import { registerAnnotationRef, stripFragment } from './browserbridge';
 import { recordUserOverlayAudit } from './browserbridge_host';
 import { isUiSharingEnabled } from './desktopui';
+import { resolvePointer } from './uipointer';
 import type { Handler } from './ipc/dispatch';
 import { policyForGuest } from './webtab';
 import { KIMIWEB_PARTITION } from './webtab_policy';
+import type { UiPointer } from '../../src/state/uiPointer';
 
 // ── Arg narrowing (the renderer is our own, but handlers validate anyway) ────
 
@@ -152,6 +155,42 @@ function ensureDebugger(wc: WebContents): void {
   wc.once('destroyed', () => attached.delete(wc.id));
 }
 
+// ── D4 element-resolved pointing ─────────────────────────────────────────────
+
+/// Resolve the element under the crop's centre, for a rect that landed on a
+/// bridgeable guest. Best-effort by design: the crop is the deliverable and a
+/// pointer is the bonus, so every failure path (no debugger, no AX tree, a
+/// canvas app with nothing to name) yields `null` and the capture still
+/// returns. An interactive hit is registered with the bridge (merge, never
+/// replacing the map the agent's last snapshot minted) so the ref in the
+/// message is one `browser_click` can resolve.
+///
+/// The point is the CLIPPED rect's centre: a drag straddling the guest edge
+/// hit-tests the centre of the part inside the guest, not the user's whole
+/// drag — the part outside the guest has no elements to name.
+async function pointerForGuest(wc: WebContents, rect: AnnotRect): Promise<UiPointer | null> {
+  const policy = policyForGuest(wc);
+  // `none` partitions are outside the bridge entirely — an @eN ref for a tab
+  // no tool can address would be a reference to nowhere.
+  if (policy === null || policy.bridge === 'none') return null;
+  try {
+    ensureDebugger(wc);
+    const resolved = await resolvePointer(
+      (method, params) => wc.debugger.sendCommand(method, params ?? {}),
+      wc.id,
+      { x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 },
+      { actionable: policy.bridge === 'full' },
+    );
+    if (resolved === null) return null;
+    if (resolved.refBackendNodeId !== null) {
+      resolved.pointer.ref = registerAnnotationRef(wc.id, resolved.refBackendNodeId);
+    }
+    return resolved.pointer;
+  } catch {
+    return null;
+  }
+}
+
 // ── Handlers ─────────────────────────────────────────────────────────────────
 
 let captureSeq = 0;
@@ -175,11 +214,13 @@ export const annotationHostHandlers: Record<string, Handler> = {
     const target = pickCaptureTarget(rect, guests);
     let wc: WebContents;
     let partition: string | null = null;
+    let guestForPointer: WebContents | null = null;
     if (target.kind === 'guest') {
       const guest = resolveGuest(target.id);
       if (guest === null) return { ok: false, error: 'guest-gone' };
       wc = guest;
       partition = policyForGuest(guest)?.partition ?? null;
+      guestForPointer = guest;
     } else {
       if (ctx.win === null || ctx.win.isDestroyed()) return { ok: false, error: 'no-window' };
       wc = ctx.win.webContents;
@@ -205,13 +246,24 @@ export const annotationHostHandlers: Record<string, Handler> = {
         ? img.resize({ width: pfit.width, height: pfit.height })
         : img
       ).toDataURL();
+      // D4: the structural half of the same gesture. Resolved AFTER the
+      // pixels so a pointer failure can never cost the user their crop.
+      const pointer = guestForPointer !== null ? await pointerForGuest(guestForPointer, target.rect) : null;
       recordUserOverlayAudit({
         ts: new Date().toISOString(),
         tool: 'ui_annotate_capture',
         tab_id: target.kind === 'guest' ? target.id : null,
         url: target.kind === 'guest' ? stripFragment(wc.getURL()) : null,
         partition,
-        args: { rect: [rect.x, rect.y, rect.width, rect.height], target: target.kind, width: fit.width, height: fit.height },
+        args: {
+          rect: [rect.x, rect.y, rect.width, rect.height],
+          target: target.kind,
+          width: fit.width,
+          height: fit.height,
+          // The audit says WHICH element was pointed at — the ref and role,
+          // never the crop (the entry is a reference, ADR-062 D-2).
+          ...(pointer !== null ? { pointer: { ref: pointer.ref ?? null, role: pointer.role ?? null } } : {}),
+        },
         ok: true,
         error: null,
       });
@@ -223,6 +275,7 @@ export const annotationHostHandlers: Record<string, Handler> = {
         preview,
         data_b64: png.toString('base64'),
         target: target.kind,
+        ...(pointer !== null ? { pointer } : {}),
       };
     } catch (e) {
       recordUserOverlayAudit({
@@ -240,12 +293,16 @@ export const annotationHostHandlers: Record<string, Handler> = {
   },
 
   /// "Attach to kimi web": inject the crop into the kimi composer's file
-  /// input. The renderer names the kimi guest (it knows which panel is open);
-  /// main re-validates the id is a live kimiweb-partition guest before
-  /// attaching — this path can never reach a webtab or the shell.
+  /// input, and the note (the user's words + the D4 pointer line) into its
+  /// text box — the kimi path must deliver the same pointer the companion
+  /// path folds into postAgentInput, or the pre-send chip promises what the
+  /// agent never receives. The renderer names the kimi guest (it knows which
+  /// panel is open); main re-validates the id is a live kimiweb-partition
+  /// guest before attaching — this path can never reach a webtab or the shell.
   annotation_attach_kimi: async (args) => {
     if (!isUiSharingEnabled()) return { ok: false, error: 'sharing-off' };
     const file = typeof args.file === 'string' ? args.file : '';
+    const note = typeof args.note === 'string' ? args.note : '';
     const guestId = typeof args.guest_id === 'number' && Number.isInteger(args.guest_id) ? args.guest_id : null;
     if (!isAnnotationTempFile(os.tmpdir(), file) || !fs.existsSync(file)) return { ok: false, error: 'bad-file' };
     if (guestId === null) return { ok: false, error: 'bad-guest' };
@@ -286,11 +343,24 @@ export const annotationHostHandlers: Record<string, Handler> = {
 
     try {
       ensureDebugger(wc);
-      const res = await injectImageIntoComposer((method, params) => wc.debugger.sendCommand(method, params ?? {}), file);
+      const send = (method: string, params?: Record<string, unknown>): Promise<unknown> => wc.debugger.sendCommand(method, params ?? {});
+      const res = await injectImageIntoComposer(send, file);
       if (res !== 'attached') return fallback('no-input');
+      // The note rides beside the crop. Its failure must not cost the
+      // attachment (the crop is the deliverable) — the result says whether
+      // it landed so the renderer can tell the user to type it themselves.
+      // The audit records the FLAG only, never the note's content.
+      let noteInjected: boolean | undefined;
+      if (note !== '') {
+        try {
+          noteInjected = (await injectNoteIntoComposer(send, note)) === 'injected';
+        } catch {
+          noteInjected = false;
+        }
+      }
       wc.focus();
-      audit(true, { via: 'file-input' }, null);
-      return { ok: true, injected: true };
+      audit(true, { via: 'file-input', ...(noteInjected !== undefined ? { note_injected: noteInjected } : {}) }, null);
+      return { ok: true, injected: true, ...(noteInjected !== undefined ? { note_injected: noteInjected } : {}) };
     } catch (e) {
       return fallback(e instanceof Error ? e.message : String(e));
     }

desktop/electron/src/browserbridge.ts

@@ -673,6 +673,36 @@ export function pruneSnapshotRefs(tabId: number): void {
   snapshotRefs.delete(tabId);
 }
 
+/// Register ONE ref minted outside a `browser_snapshot` call — D4's
+/// annotation pointer names an element on the user's gesture, and the ref it
+/// hands the agent has to be one `browser_click` can resolve. Deliberately
+/// NOT latest-wins over the whole map: replacing the tab's map would renumber
+/// refs the agent still holds from its last snapshot, silently retargeting an
+/// agent-held `@e5` at a different element (no REF_STALE — the ref would
+/// still exist). Instead the pointer MERGES:
+///
+///   - if the agent's last snapshot already named this node, reuse that ref —
+///     the agent recognizes it;
+///   - otherwise mint an `@aN` (annotation namespace, monotonic across tabs,
+///     never colliding with `@eN` or a previous `@aN`) and add it alongside
+///     the existing entries.
+///
+/// A later browser_snapshot still replaces the whole map, so `@aN` dies with
+/// it — the ordinary REF_STALE contract.
+let annotationRefSeq = 0;
+
+export function registerAnnotationRef(tabId: number, backendNodeId: number): string {
+  const map = snapshotRefs.get(tabId) ?? new Map<string, number>();
+  for (const [ref, backend] of map) {
+    if (backend === backendNodeId) return ref;
+  }
+  annotationRefSeq += 1;
+  const ref = `@a${String(annotationRefSeq)}`;
+  map.set(ref, backendNodeId);
+  snapshotRefs.set(tabId, map);
+  return ref;
+}
+
 /// Resolve + validate a tabId argument against the live registry. Throws a
 /// coded BridgeError the tool wrapper renders as an isError result.
 function requireTarget(deps: McpServerDeps, args: Record<string, unknown>): BridgeTarget {