feat(desktop,hub): agent pointing — ref-chips + ui_highlight (D6)

desktop/electron/src/browserbridge.ts

@@ -227,6 +227,15 @@ export function redactBridgeArgs(tool: string, args: Record<string, unknown>): R
       out[k] = v.length > 200 ? `${v.slice(0, 200)}… (${String(v.length)} chars)` : v;
     } else if (tool === 'browser_send_keys' && k === 'keys' && typeof v === 'string' && v.length === 1) {
       out[k] = '<redacted char>';
+    } else if (tool === 'ui_highlight' && k === 'note' && typeof v === 'string') {
+      // The ring must not out-store what the marker shows: the tool clips the
+      // note to one short line, so the audit entry does too.
+      out[k] = v.length > 140 ? `${v.slice(0, 140)}… (${String(v.length)} chars)` : v;
+    } else if (tool === 'ui_highlight' && k === 'ref' && v !== null && typeof v === 'object') {
+      // An agent-authored object of arbitrary depth — audit the shape, capped,
+      // rather than retaining the whole structure in memory for 50 entries.
+      const json = JSON.stringify(v) ?? '';
+      out[k] = json.length > 200 ? `${json.slice(0, 200)}… (${String(json.length)} chars)` : json;
     } else if (k === 'url' && typeof v === 'string') {
       out[k] = stripFragment(v);
     } else {
@@ -437,17 +446,42 @@ export const READ_TOOLS: readonly McpToolDef[] = [
       additionalProperties: false,
     },
   },
+  // D6 (plan §3.4b, ADR-062 D-5): deixis is symmetric — the agent points back.
+  // NON-ACTUATING by construction: it draws a glow and expires. No approval
+  // card (it takes no action with the user's authority) but audited like one,
+  // because "an agent drew on my screen" is exactly what the audit view is for.
+  {
+    name: 'ui_highlight',
+    description:
+      'Point the TermiPod desktop user at something on their own screen: an ephemeral, visibly attributed glow over a surface, with an optional note. NON-ACTUATING — it never focuses, scrolls, clicks or types, and it expires on its own; the user\'s click is the only actuator. `ref` is a UIRef, either the JSON shape ui_get_focus returns ({"surface":"replay","entity":{"dataset_id":"ds_1"}}) or its URI spelling ("ui://replay?dataset_id=ds_1"). Use it when words alone would make the user hunt ("the failing pane", "that row"). Refused over surfaces the desktop marks non-annotatable. You can also write a ui:// reference inline in your reply — it renders as a chip the user can click.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        ref: {
+          description: 'A UIRef: the JSON shape from ui_get_focus, or a "ui://<surface>?<ids>" string.',
+        },
+        note: { type: 'string', description: 'One short line shown with the highlight (<=140 chars).' },
+        ttl_ms: { type: 'integer', description: 'How long it stays up (default 8000, max 30000).' },
+      },
+      required: ['ref'],
+      additionalProperties: false,
+    },
+  },
 ];
 
-/// The D1/D3 desktop-UI tools, which the sharing toggle gates as a set: off
-/// means neither appears in any catalog and both refuse on call.
-export const UI_TOOL_NAMES: ReadonlySet<string> = new Set(['ui_get_focus', 'ui_screenshot']);
+/// The D1/D3/D6 desktop-UI tools, which the sharing toggle gates as a set: off
+/// means none appears in any catalog and all refuse on call.
+export const UI_TOOL_NAMES: ReadonlySet<string> = new Set(['ui_get_focus', 'ui_screenshot', 'ui_highlight']);
 
 /// Desktop-UI tools that are ACTION class despite living in READ_TOOLS (the
 /// bearer scope they need and the consent they need are different questions —
 /// see the ui_screenshot definition). They audit + hub-mirror like an action
-/// on every leg, and their own handler owns the approval gate.
-export const DESKTOP_ACTION_TOOL_NAMES: ReadonlySet<string> = new Set(['ui_screenshot']);
+/// on every leg, and their own handler owns whatever gate it needs.
+///
+/// `ui_highlight` is here for the AUDIT, not for an approval: ADR-062 D-5 is
+/// explicit that a highlight needs no card (consent is the sharing toggle plus
+/// the policy bit) but is audited like an action.
+export const DESKTOP_ACTION_TOOL_NAMES: ReadonlySet<string> = new Set(['ui_screenshot', 'ui_highlight']);
 
 /// D1: the MCP resource uri mirroring ui_get_focus (ADR-062 D-6). list + read
 /// only — subscriptions are deliberately NOT implemented: the relay forwards
@@ -609,6 +643,10 @@ export interface McpServerDeps {
   /// provider owns the policy refusal AND the per-call approval — this module
   /// only resolves the target and shapes the result.
   captureUi?: (req: UiCaptureRequest) => Promise<UiCaptureResult>;
+  /// D6: the agent-pointing highlight (uihighlight_host.ts). The provider owns
+  /// the policy bit, the rate limit and the TTL; this module only shapes the
+  /// call and the answer.
+  highlightUi?: (req: UiHighlightRequest) => Promise<UiHighlightResult>;
 }
 
 /// One `ui_screenshot` request, with the target already resolved against the
@@ -632,6 +670,24 @@ export type UiCaptureResult =
   | { ok: true; data_b64: string; width: number; height: number }
   | { ok: false; code: string; message: string };
 
+/// D6: one `ui_highlight` call, with the caller's identity attached — a
+/// highlight is ATTRIBUTED on screen ("kimi-1 points here"), which is half of
+/// why it needs no approval card.
+export interface UiHighlightRequest {
+  /// The raw `ref` argument, JSON or `ui://` string — parsed by the provider
+  /// through the shared grammar, so the tool and the transcript chip point at
+  /// the same thing by construction.
+  ref: unknown;
+  note: string;
+  ttlMs: number | null;
+  agentId: string;
+  agentHandle: string;
+  // No `via`: the leg is recorded by the audit ring at the callTool wrapper,
+  // and the highlight itself treats local and relayed callers identically.
+}
+
+export type UiHighlightResult = { ok: true; surface: string; ttl_ms: number } | { ok: false; code: string; message: string };
+
 interface JsonRpcRequest {
   jsonrpc?: string;
   id?: unknown;
@@ -1152,11 +1208,44 @@ async function runTool(deps: McpServerDeps, ctx: BridgeRequestContext, name: str
       if (!res.ok) throw new BridgeError(res.code, res.message);
       return { content: [{ type: 'image', data: res.data_b64, mimeType: 'image/png' }] };
     }
+    case 'ui_highlight': {
+      if (deps.uiFocusAvailable?.() !== true) {
+        throw new BridgeError(
+          'UI_UNAVAILABLE',
+          'UI context sharing is off on the desktop (Settings → Assistant) — the desktop UI is not addressable',
+        );
+      }
+      if (deps.highlightUi === undefined) {
+        throw new BridgeError('UI_UNAVAILABLE', 'this desktop build cannot render agent highlights');
+      }
+      const note = typeof args.note === 'string' ? args.note.slice(0, HIGHLIGHT_NOTE_MAX) : '';
+      let ttlMs: number | null = null;
+      if (args.ttl_ms !== undefined) {
+        if (typeof args.ttl_ms !== 'number' || !Number.isInteger(args.ttl_ms) || args.ttl_ms <= 0) {
+          throw new BridgeError('INVALID_PARAMS', 'ttl_ms must be a positive integer (milliseconds)');
+        }
+        ttlMs = args.ttl_ms;
+      }
+      const res = await deps.highlightUi({
+        ref: args.ref,
+        note,
+        ttlMs,
+        agentId: ctx.agentId ?? '',
+        agentHandle: ctx.agentHandle ?? '',
+      });
+      if (!res.ok) throw new BridgeError(res.code, res.message);
+      // The answer says what the user will SEE, so the agent can describe it
+      // ("I've highlighted the replay panel") rather than guess.
+      return textContent(`highlighted ${res.surface} for ${String(Math.round(res.ttl_ms / 1000))}s — the user sees an attributed marker; nothing was focused or clicked`);
+    }
     default:
       throw new BridgeError('UNKNOWN_TOOL', `unknown tool '${name}'`);
   }
 }
 
+/// A highlight note is a caption, not a message channel: one short line.
+const HIGHLIGHT_NOTE_MAX = 140;
+
 /// The focus answer as JSON text. Never blocks: before the renderer's first
 /// push the cache is null and the answer is an explicit empty snapshot —
 /// `captured_at: null` tells the agent there is nothing fresh, rather than

desktop/electron/src/browserbridge_host.ts

@@ -66,6 +66,8 @@ import {
   type TunnelClass,
   type UiCaptureRequest,
   type UiCaptureResult,
+  type UiHighlightRequest,
+  type UiHighlightResult,
 } from './browserbridge';
 import { policyForGuest } from './webtab';
 import { keychainGetLocal } from './ipc/keychain';
@@ -288,6 +290,16 @@ export function setUiCaptureProvider(p: UiCaptureProvider): void {
   uiCaptureProvider = p;
 }
 
+/// D6: the agent-pointing provider (uihighlight_host.ts), registered the same
+/// way as D3's capture provider.
+type UiHighlightProvider = (req: UiHighlightRequest) => Promise<UiHighlightResult>;
+
+let uiHighlightProvider: UiHighlightProvider | null = null;
+
+export function setUiHighlightProvider(p: UiHighlightProvider): void {
+  uiHighlightProvider = p;
+}
+
 /// The hub identity the audit mirror and the D3 approval card both post as.
 /// Non-secret: the bearer is fetched from the keychain per use, never held
 /// here (and never handed across IPC).
@@ -326,6 +338,10 @@ async function enable(): Promise<void> {
       uiCaptureProvider !== null
         ? uiCaptureProvider(req)
         : Promise.resolve<UiCaptureResult>({ ok: false, code: 'UI_UNAVAILABLE', message: 'the capture provider is not wired' }),
+    highlightUi: (req) =>
+      uiHighlightProvider !== null
+        ? uiHighlightProvider(req)
+        : Promise.resolve<UiHighlightResult>({ ok: false, code: 'UI_UNAVAILABLE', message: 'the highlight provider is not wired' }),
   };
   server = await startBridgeServer({ ...mcpDeps, token, actionToken });
   writeBridgeDiscovery(os.homedir(), {

desktop/electron/src/main.ts

@@ -31,6 +31,8 @@ import { disposeAnnotations } from './annotation_host';
 // D3: importing the module registers the gated-screenshot provider on the
 // bridge server; setShellWindow tells it which window "the desktop" means.
 import { setShellWindow } from './uicapture_host';
+// D6: importing the module registers the agent-highlight provider.
+import './uihighlight_host';
 import { initEvents } from './events';
 
 // The frontend build. In dev (`electron .` from desktop/electron) it resolves to

desktop/electron/src/uicapture_host.ts

@@ -45,6 +45,13 @@ export function setShellWindow(win: BrowserWindow | null): void {
   shellWindow = win;
 }
 
+/// The shell's webContents, for main→renderer pushes (D6's highlight orders).
+/// Null when the window is closed — a push with nowhere to land is a refusal,
+/// not a crash.
+export function shellWebContents(): WebContents | null {
+  return shellWindow !== null && !shellWindow.isDestroyed() ? shellWindow.webContents : null;
+}
+
 // ── Guest resolution ─────────────────────────────────────────────────────────
 
 /// The tabId was already validated against the bridge's live registry

desktop/electron/src/uihighlight.test.ts

@@ -0,0 +1,150 @@
+/// Tests for agent pointing (D6 — docs/plans/desktop-ui-context-and-pointing.md
+/// §3.4b, ADR-062 D-5). `ui_highlight` is the weakest capability in the plan —
+/// it draws a glow and expires — so what needs proving is not that it works
+/// but that it cannot become attention spam or fake UI: the policy bit binds,
+/// the attribution is never empty, the TTL is ours, and the rate limit counts
+/// abuse rather than refusals. `node --test`.
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  clampHighlightTtl,
+  clipHighlightNote,
+  decideHighlight,
+  pruneHighlightHistory,
+  HIGHLIGHT_NOTE_MAX,
+  HIGHLIGHT_RATE_LIMIT,
+  HIGHLIGHT_RATE_WINDOW_MS,
+  HIGHLIGHT_TTL_DEFAULT_MS,
+  HIGHLIGHT_TTL_MAX_MS,
+  type HighlightInput,
+} from './uihighlight.ts';
+
+const NOW = 1_800_000_000_000;
+
+function input(over: Partial<HighlightInput> = {}): HighlightInput {
+  return {
+    ref: 'ui://replay?dataset_id=ds_1&episode_id=ep_2&cursor=1234',
+    note: 'this episode',
+    ttlMs: null,
+    agentId: 'ag_1',
+    agentHandle: 'kimi-1',
+    now: NOW,
+    recent: [],
+    id: 'hl-1',
+    iso: '2026-07-31T00:00:00.000Z',
+    ...over,
+  };
+}
+
+// ── The policy bit binds ─────────────────────────────────────────────────────
+
+test('the highlight column decides which surfaces may be drawn over', () => {
+  const ok = decideHighlight(input());
+  assert.equal(ok.ok, true);
+  if (ok.ok) assert.equal(ok.order.ref.surface, 'replay');
+
+  // Vault refuses everything, always (ADR-062 D-3).
+  const vault = decideHighlight(input({ ref: 'ui://vault' }));
+  assert.equal(vault.ok, false);
+  if (!vault.ok) assert.equal(vault.code, 'HIGHLIGHT_REFUSED');
+
+  // Settings allows highlight even though it refuses CAPTURE — the columns
+  // are independent because the sensitivities are.
+  assert.equal(decideHighlight(input({ ref: 'ui://settings' })).ok, true);
+
+  // A surface with no row is not annotatable: the table is the allowlist.
+  const unknown = decideHighlight(input({ ref: 'ui://surface-from-the-future' }));
+  assert.equal(unknown.ok, false);
+  if (!unknown.ok) assert.equal(unknown.code, 'UNKNOWN_SURFACE');
+});
+
+test('both UIRef spellings resolve to the same thing', () => {
+  const uri = decideHighlight(input({ ref: 'ui://replay?dataset_id=ds_1' }));
+  const json = decideHighlight(input({ ref: { surface: 'replay', entity: { dataset_id: 'ds_1' } } }));
+  assert.equal(uri.ok && json.ok, true);
+  if (uri.ok && json.ok) assert.deepEqual(uri.order.ref, json.order.ref);
+});
+
+test('an unparseable ref is refused with an actionable message', () => {
+  for (const bad of [null, 42, 'not a ref', {}, { surface: 'Replay!' }, []]) {
+    const out = decideHighlight(input({ ref: bad }));
+    assert.equal(out.ok, false, JSON.stringify(bad));
+    if (!out.ok) {
+      assert.equal(out.code, 'INVALID_REF');
+      // The message must show the shape, not just name it.
+      assert.match(out.message, /ui:\/\//);
+    }
+  }
+});
+
+// ── Attribution, TTL, note ───────────────────────────────────────────────────
+
+test('a highlight is never unattributed', () => {
+  const handle = decideHighlight(input());
+  assert.equal(handle.ok && handle.order.by, 'kimi-1');
+  const byId = decideHighlight(input({ agentHandle: '' }));
+  assert.equal(byId.ok && byId.order.by, 'ag_1');
+  // Even a caller with no identity at all gets a subject — an anonymous glow
+  // is the fake-UI failure mode the plan's risk section names.
+  const anon = decideHighlight(input({ agentHandle: '', agentId: '' }));
+  assert.equal(anon.ok && anon.order.by, 'an agent');
+});
+
+test('the TTL is ours, not the caller argument', () => {
+  assert.equal(clampHighlightTtl(null), HIGHLIGHT_TTL_DEFAULT_MS);
+  assert.equal(clampHighlightTtl(2000), 2000);
+  // An agent cannot pin a marker to the screen.
+  assert.equal(clampHighlightTtl(60 * 60 * 1000), HIGHLIGHT_TTL_MAX_MS);
+  // …nor make one flash by for a frame.
+  assert.equal(clampHighlightTtl(1), 500);
+  const out = decideHighlight(input({ ttlMs: 999_999 }));
+  assert.equal(out.ok && out.order.ttl_ms, HIGHLIGHT_TTL_MAX_MS);
+});
+
+test('the note is a caption, not a message channel', () => {
+  assert.equal(clipHighlightNote('  look   at\nthis '), 'look at this');
+  const long = clipHighlightNote('y'.repeat(500));
+  assert.equal(long.length, HIGHLIGHT_NOTE_MAX);
+  assert.ok(long.endsWith('…'));
+  const out = decideHighlight(input({ note: 'z'.repeat(1000) }));
+  assert.ok(out.ok && out.order.note.length <= HIGHLIGHT_NOTE_MAX);
+});
+
+// ── Rate limiting ────────────────────────────────────────────────────────────
+
+test('an agent cannot paper the screen', () => {
+  const recent = Array.from({ length: HIGHLIGHT_RATE_LIMIT }, (_, i) => NOW - i * 1000);
+  const out = decideHighlight(input({ recent }));
+  assert.equal(out.ok, false);
+  if (!out.ok) {
+    assert.equal(out.code, 'HIGHLIGHT_RATE_LIMITED');
+    // The refusal suggests the alternative rather than just saying no.
+    assert.match(out.message, /words/);
+  }
+
+  // One short of the limit still passes…
+  assert.equal(decideHighlight(input({ recent: recent.slice(1) })).ok, true);
+  // …and the window slides: old timestamps do not count.
+  const stale = recent.map((t) => t - HIGHLIGHT_RATE_WINDOW_MS);
+  assert.equal(decideHighlight(input({ recent: stale })).ok, true);
+});
+
+test('history pruning keeps the store bounded without a sweeper', () => {
+  const recent = [NOW - HIGHLIGHT_RATE_WINDOW_MS * 2, NOW - 1000, NOW];
+  assert.deepEqual(pruneHighlightHistory(recent, NOW), [NOW - 1000, NOW]);
+  assert.deepEqual(pruneHighlightHistory([], NOW), []);
+});
+
+test('a refusal does not consume the budget', () => {
+  // The limit exists to stop a loop, not to punish a mistake: an agent that
+  // mistypes a ref five times must still be able to point once it gets it
+  // right. (The host enforces this by writing back the PRUNED history on a
+  // refusal and the appended one only on success — this test pins the
+  // decision half: a refusal returns before the counter is read.)
+  const recent = Array.from({ length: HIGHLIGHT_RATE_LIMIT - 1 }, () => NOW);
+  const bad = decideHighlight(input({ ref: 'nonsense', recent }));
+  assert.equal(bad.ok, false);
+  if (!bad.ok) assert.equal(bad.code, 'INVALID_REF', 'the ref check must precede the rate check');
+  const refused = decideHighlight(input({ ref: 'ui://vault', recent }));
+  if (!refused.ok) assert.equal(refused.code, 'HIGHLIGHT_REFUSED', 'policy refusal must precede the rate check');
+});