diff --git a/desktop/electron/src/browserbridge.ts b/desktop/electron/src/browserbridge.ts index e28200a1..d38336a6 100644 --- a/desktop/electron/src/browserbridge.ts +++ b/desktop/electron/src/browserbridge.ts @@ -227,6 +227,15 @@ export function redactBridgeArgs(tool: string, args: Record): R out[k] = v.length > 200 ? `${v.slice(0, 200)}… (${String(v.length)} chars)` : v; } else if (tool === 'browser_send_keys' && k === 'keys' && typeof v === 'string' && v.length === 1) { out[k] = ''; + } else if (tool === 'ui_highlight' && k === 'note' && typeof v === 'string') { + // The ring must not out-store what the marker shows: the tool clips the + // note to one short line, so the audit entry does too. + out[k] = v.length > 140 ? `${v.slice(0, 140)}… (${String(v.length)} chars)` : v; + } else if (tool === 'ui_highlight' && k === 'ref' && v !== null && typeof v === 'object') { + // An agent-authored object of arbitrary depth — audit the shape, capped, + // rather than retaining the whole structure in memory for 50 entries. + const json = JSON.stringify(v) ?? ''; + out[k] = json.length > 200 ? `${json.slice(0, 200)}… (${String(json.length)} chars)` : json; } else if (k === 'url' && typeof v === 'string') { out[k] = stripFragment(v); } else { @@ -437,17 +446,42 @@ export const READ_TOOLS: readonly McpToolDef[] = [ additionalProperties: false, }, }, + // D6 (plan §3.4b, ADR-062 D-5): deixis is symmetric — the agent points back. + // NON-ACTUATING by construction: it draws a glow and expires. No approval + // card (it takes no action with the user's authority) but audited like one, + // because "an agent drew on my screen" is exactly what the audit view is for. + { + name: 'ui_highlight', + description: + 'Point the TermiPod desktop user at something on their own screen: an ephemeral, visibly attributed glow over a surface, with an optional note. NON-ACTUATING — it never focuses, scrolls, clicks or types, and it expires on its own; the user\'s click is the only actuator. `ref` is a UIRef, either the JSON shape ui_get_focus returns ({"surface":"replay","entity":{"dataset_id":"ds_1"}}) or its URI spelling ("ui://replay?dataset_id=ds_1"). Use it when words alone would make the user hunt ("the failing pane", "that row"). Refused over surfaces the desktop marks non-annotatable. You can also write a ui:// reference inline in your reply — it renders as a chip the user can click.', + inputSchema: { + type: 'object', + properties: { + ref: { + description: 'A UIRef: the JSON shape from ui_get_focus, or a "ui://?" string.', + }, + note: { type: 'string', description: 'One short line shown with the highlight (<=140 chars).' }, + ttl_ms: { type: 'integer', description: 'How long it stays up (default 8000, max 30000).' }, + }, + required: ['ref'], + additionalProperties: false, + }, + }, ]; -/// The D1/D3 desktop-UI tools, which the sharing toggle gates as a set: off -/// means neither appears in any catalog and both refuse on call. -export const UI_TOOL_NAMES: ReadonlySet = new Set(['ui_get_focus', 'ui_screenshot']); +/// The D1/D3/D6 desktop-UI tools, which the sharing toggle gates as a set: off +/// means none appears in any catalog and all refuse on call. +export const UI_TOOL_NAMES: ReadonlySet = new Set(['ui_get_focus', 'ui_screenshot', 'ui_highlight']); /// Desktop-UI tools that are ACTION class despite living in READ_TOOLS (the /// bearer scope they need and the consent they need are different questions — /// see the ui_screenshot definition). They audit + hub-mirror like an action -/// on every leg, and their own handler owns the approval gate. -export const DESKTOP_ACTION_TOOL_NAMES: ReadonlySet = new Set(['ui_screenshot']); +/// on every leg, and their own handler owns whatever gate it needs. +/// +/// `ui_highlight` is here for the AUDIT, not for an approval: ADR-062 D-5 is +/// explicit that a highlight needs no card (consent is the sharing toggle plus +/// the policy bit) but is audited like an action. +export const DESKTOP_ACTION_TOOL_NAMES: ReadonlySet = new Set(['ui_screenshot', 'ui_highlight']); /// D1: the MCP resource uri mirroring ui_get_focus (ADR-062 D-6). list + read /// only — subscriptions are deliberately NOT implemented: the relay forwards @@ -609,6 +643,10 @@ export interface McpServerDeps { /// provider owns the policy refusal AND the per-call approval — this module /// only resolves the target and shapes the result. captureUi?: (req: UiCaptureRequest) => Promise; + /// D6: the agent-pointing highlight (uihighlight_host.ts). The provider owns + /// the policy bit, the rate limit and the TTL; this module only shapes the + /// call and the answer. + highlightUi?: (req: UiHighlightRequest) => Promise; } /// One `ui_screenshot` request, with the target already resolved against the @@ -632,6 +670,24 @@ export type UiCaptureResult = | { ok: true; data_b64: string; width: number; height: number } | { ok: false; code: string; message: string }; +/// D6: one `ui_highlight` call, with the caller's identity attached — a +/// highlight is ATTRIBUTED on screen ("kimi-1 points here"), which is half of +/// why it needs no approval card. +export interface UiHighlightRequest { + /// The raw `ref` argument, JSON or `ui://` string — parsed by the provider + /// through the shared grammar, so the tool and the transcript chip point at + /// the same thing by construction. + ref: unknown; + note: string; + ttlMs: number | null; + agentId: string; + agentHandle: string; + // No `via`: the leg is recorded by the audit ring at the callTool wrapper, + // and the highlight itself treats local and relayed callers identically. +} + +export type UiHighlightResult = { ok: true; surface: string; ttl_ms: number } | { ok: false; code: string; message: string }; + interface JsonRpcRequest { jsonrpc?: string; id?: unknown; @@ -1152,11 +1208,44 @@ async function runTool(deps: McpServerDeps, ctx: BridgeRequestContext, name: str if (!res.ok) throw new BridgeError(res.code, res.message); return { content: [{ type: 'image', data: res.data_b64, mimeType: 'image/png' }] }; } + case 'ui_highlight': { + if (deps.uiFocusAvailable?.() !== true) { + throw new BridgeError( + 'UI_UNAVAILABLE', + 'UI context sharing is off on the desktop (Settings → Assistant) — the desktop UI is not addressable', + ); + } + if (deps.highlightUi === undefined) { + throw new BridgeError('UI_UNAVAILABLE', 'this desktop build cannot render agent highlights'); + } + const note = typeof args.note === 'string' ? args.note.slice(0, HIGHLIGHT_NOTE_MAX) : ''; + let ttlMs: number | null = null; + if (args.ttl_ms !== undefined) { + if (typeof args.ttl_ms !== 'number' || !Number.isInteger(args.ttl_ms) || args.ttl_ms <= 0) { + throw new BridgeError('INVALID_PARAMS', 'ttl_ms must be a positive integer (milliseconds)'); + } + ttlMs = args.ttl_ms; + } + const res = await deps.highlightUi({ + ref: args.ref, + note, + ttlMs, + agentId: ctx.agentId ?? '', + agentHandle: ctx.agentHandle ?? '', + }); + if (!res.ok) throw new BridgeError(res.code, res.message); + // The answer says what the user will SEE, so the agent can describe it + // ("I've highlighted the replay panel") rather than guess. + return textContent(`highlighted ${res.surface} for ${String(Math.round(res.ttl_ms / 1000))}s — the user sees an attributed marker; nothing was focused or clicked`); + } default: throw new BridgeError('UNKNOWN_TOOL', `unknown tool '${name}'`); } } +/// A highlight note is a caption, not a message channel: one short line. +const HIGHLIGHT_NOTE_MAX = 140; + /// The focus answer as JSON text. Never blocks: before the renderer's first /// push the cache is null and the answer is an explicit empty snapshot — /// `captured_at: null` tells the agent there is nothing fresh, rather than diff --git a/desktop/electron/src/browserbridge_host.ts b/desktop/electron/src/browserbridge_host.ts index 53b50979..ef0d062d 100644 --- a/desktop/electron/src/browserbridge_host.ts +++ b/desktop/electron/src/browserbridge_host.ts @@ -66,6 +66,8 @@ import { type TunnelClass, type UiCaptureRequest, type UiCaptureResult, + type UiHighlightRequest, + type UiHighlightResult, } from './browserbridge'; import { policyForGuest } from './webtab'; import { keychainGetLocal } from './ipc/keychain'; @@ -288,6 +290,16 @@ export function setUiCaptureProvider(p: UiCaptureProvider): void { uiCaptureProvider = p; } +/// D6: the agent-pointing provider (uihighlight_host.ts), registered the same +/// way as D3's capture provider. +type UiHighlightProvider = (req: UiHighlightRequest) => Promise; + +let uiHighlightProvider: UiHighlightProvider | null = null; + +export function setUiHighlightProvider(p: UiHighlightProvider): void { + uiHighlightProvider = p; +} + /// The hub identity the audit mirror and the D3 approval card both post as. /// Non-secret: the bearer is fetched from the keychain per use, never held /// here (and never handed across IPC). @@ -326,6 +338,10 @@ async function enable(): Promise { uiCaptureProvider !== null ? uiCaptureProvider(req) : Promise.resolve({ ok: false, code: 'UI_UNAVAILABLE', message: 'the capture provider is not wired' }), + highlightUi: (req) => + uiHighlightProvider !== null + ? uiHighlightProvider(req) + : Promise.resolve({ ok: false, code: 'UI_UNAVAILABLE', message: 'the highlight provider is not wired' }), }; server = await startBridgeServer({ ...mcpDeps, token, actionToken }); writeBridgeDiscovery(os.homedir(), { diff --git a/desktop/electron/src/main.ts b/desktop/electron/src/main.ts index f8bbb31c..982d1fd6 100644 --- a/desktop/electron/src/main.ts +++ b/desktop/electron/src/main.ts @@ -31,6 +31,8 @@ import { disposeAnnotations } from './annotation_host'; // D3: importing the module registers the gated-screenshot provider on the // bridge server; setShellWindow tells it which window "the desktop" means. import { setShellWindow } from './uicapture_host'; +// D6: importing the module registers the agent-highlight provider. +import './uihighlight_host'; import { initEvents } from './events'; // The frontend build. In dev (`electron .` from desktop/electron) it resolves to diff --git a/desktop/electron/src/uicapture_host.ts b/desktop/electron/src/uicapture_host.ts index 078b87a9..002f3520 100644 --- a/desktop/electron/src/uicapture_host.ts +++ b/desktop/electron/src/uicapture_host.ts @@ -45,6 +45,13 @@ export function setShellWindow(win: BrowserWindow | null): void { shellWindow = win; } +/// The shell's webContents, for main→renderer pushes (D6's highlight orders). +/// Null when the window is closed — a push with nowhere to land is a refusal, +/// not a crash. +export function shellWebContents(): WebContents | null { + return shellWindow !== null && !shellWindow.isDestroyed() ? shellWindow.webContents : null; +} + // ── Guest resolution ───────────────────────────────────────────────────────── /// The tabId was already validated against the bridge's live registry diff --git a/desktop/electron/src/uihighlight.test.ts b/desktop/electron/src/uihighlight.test.ts new file mode 100644 index 00000000..78b298cb --- /dev/null +++ b/desktop/electron/src/uihighlight.test.ts @@ -0,0 +1,150 @@ +/// Tests for agent pointing (D6 — docs/plans/desktop-ui-context-and-pointing.md +/// §3.4b, ADR-062 D-5). `ui_highlight` is the weakest capability in the plan — +/// it draws a glow and expires — so what needs proving is not that it works +/// but that it cannot become attention spam or fake UI: the policy bit binds, +/// the attribution is never empty, the TTL is ours, and the rate limit counts +/// abuse rather than refusals. `node --test`. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + clampHighlightTtl, + clipHighlightNote, + decideHighlight, + pruneHighlightHistory, + HIGHLIGHT_NOTE_MAX, + HIGHLIGHT_RATE_LIMIT, + HIGHLIGHT_RATE_WINDOW_MS, + HIGHLIGHT_TTL_DEFAULT_MS, + HIGHLIGHT_TTL_MAX_MS, + type HighlightInput, +} from './uihighlight.ts'; + +const NOW = 1_800_000_000_000; + +function input(over: Partial = {}): HighlightInput { + return { + ref: 'ui://replay?dataset_id=ds_1&episode_id=ep_2&cursor=1234', + note: 'this episode', + ttlMs: null, + agentId: 'ag_1', + agentHandle: 'kimi-1', + now: NOW, + recent: [], + id: 'hl-1', + iso: '2026-07-31T00:00:00.000Z', + ...over, + }; +} + +// ── The policy bit binds ───────────────────────────────────────────────────── + +test('the highlight column decides which surfaces may be drawn over', () => { + const ok = decideHighlight(input()); + assert.equal(ok.ok, true); + if (ok.ok) assert.equal(ok.order.ref.surface, 'replay'); + + // Vault refuses everything, always (ADR-062 D-3). + const vault = decideHighlight(input({ ref: 'ui://vault' })); + assert.equal(vault.ok, false); + if (!vault.ok) assert.equal(vault.code, 'HIGHLIGHT_REFUSED'); + + // Settings allows highlight even though it refuses CAPTURE — the columns + // are independent because the sensitivities are. + assert.equal(decideHighlight(input({ ref: 'ui://settings' })).ok, true); + + // A surface with no row is not annotatable: the table is the allowlist. + const unknown = decideHighlight(input({ ref: 'ui://surface-from-the-future' })); + assert.equal(unknown.ok, false); + if (!unknown.ok) assert.equal(unknown.code, 'UNKNOWN_SURFACE'); +}); + +test('both UIRef spellings resolve to the same thing', () => { + const uri = decideHighlight(input({ ref: 'ui://replay?dataset_id=ds_1' })); + const json = decideHighlight(input({ ref: { surface: 'replay', entity: { dataset_id: 'ds_1' } } })); + assert.equal(uri.ok && json.ok, true); + if (uri.ok && json.ok) assert.deepEqual(uri.order.ref, json.order.ref); +}); + +test('an unparseable ref is refused with an actionable message', () => { + for (const bad of [null, 42, 'not a ref', {}, { surface: 'Replay!' }, []]) { + const out = decideHighlight(input({ ref: bad })); + assert.equal(out.ok, false, JSON.stringify(bad)); + if (!out.ok) { + assert.equal(out.code, 'INVALID_REF'); + // The message must show the shape, not just name it. + assert.match(out.message, /ui:\/\//); + } + } +}); + +// ── Attribution, TTL, note ─────────────────────────────────────────────────── + +test('a highlight is never unattributed', () => { + const handle = decideHighlight(input()); + assert.equal(handle.ok && handle.order.by, 'kimi-1'); + const byId = decideHighlight(input({ agentHandle: '' })); + assert.equal(byId.ok && byId.order.by, 'ag_1'); + // Even a caller with no identity at all gets a subject — an anonymous glow + // is the fake-UI failure mode the plan's risk section names. + const anon = decideHighlight(input({ agentHandle: '', agentId: '' })); + assert.equal(anon.ok && anon.order.by, 'an agent'); +}); + +test('the TTL is ours, not the caller argument', () => { + assert.equal(clampHighlightTtl(null), HIGHLIGHT_TTL_DEFAULT_MS); + assert.equal(clampHighlightTtl(2000), 2000); + // An agent cannot pin a marker to the screen. + assert.equal(clampHighlightTtl(60 * 60 * 1000), HIGHLIGHT_TTL_MAX_MS); + // …nor make one flash by for a frame. + assert.equal(clampHighlightTtl(1), 500); + const out = decideHighlight(input({ ttlMs: 999_999 })); + assert.equal(out.ok && out.order.ttl_ms, HIGHLIGHT_TTL_MAX_MS); +}); + +test('the note is a caption, not a message channel', () => { + assert.equal(clipHighlightNote(' look at\nthis '), 'look at this'); + const long = clipHighlightNote('y'.repeat(500)); + assert.equal(long.length, HIGHLIGHT_NOTE_MAX); + assert.ok(long.endsWith('…')); + const out = decideHighlight(input({ note: 'z'.repeat(1000) })); + assert.ok(out.ok && out.order.note.length <= HIGHLIGHT_NOTE_MAX); +}); + +// ── Rate limiting ──────────────────────────────────────────────────────────── + +test('an agent cannot paper the screen', () => { + const recent = Array.from({ length: HIGHLIGHT_RATE_LIMIT }, (_, i) => NOW - i * 1000); + const out = decideHighlight(input({ recent })); + assert.equal(out.ok, false); + if (!out.ok) { + assert.equal(out.code, 'HIGHLIGHT_RATE_LIMITED'); + // The refusal suggests the alternative rather than just saying no. + assert.match(out.message, /words/); + } + + // One short of the limit still passes… + assert.equal(decideHighlight(input({ recent: recent.slice(1) })).ok, true); + // …and the window slides: old timestamps do not count. + const stale = recent.map((t) => t - HIGHLIGHT_RATE_WINDOW_MS); + assert.equal(decideHighlight(input({ recent: stale })).ok, true); +}); + +test('history pruning keeps the store bounded without a sweeper', () => { + const recent = [NOW - HIGHLIGHT_RATE_WINDOW_MS * 2, NOW - 1000, NOW]; + assert.deepEqual(pruneHighlightHistory(recent, NOW), [NOW - 1000, NOW]); + assert.deepEqual(pruneHighlightHistory([], NOW), []); +}); + +test('a refusal does not consume the budget', () => { + // The limit exists to stop a loop, not to punish a mistake: an agent that + // mistypes a ref five times must still be able to point once it gets it + // right. (The host enforces this by writing back the PRUNED history on a + // refusal and the appended one only on success — this test pins the + // decision half: a refusal returns before the counter is read.) + const recent = Array.from({ length: HIGHLIGHT_RATE_LIMIT - 1 }, () => NOW); + const bad = decideHighlight(input({ ref: 'nonsense', recent })); + assert.equal(bad.ok, false); + if (!bad.ok) assert.equal(bad.code, 'INVALID_REF', 'the ref check must precede the rate check'); + const refused = decideHighlight(input({ ref: 'ui://vault', recent })); + if (!refused.ok) assert.equal(refused.code, 'HIGHLIGHT_REFUSED', 'policy refusal must precede the rate check'); +}); diff --git a/desktop/electron/src/uihighlight.ts b/desktop/electron/src/uihighlight.ts new file mode 100644 index 00000000..b648036e --- /dev/null +++ b/desktop/electron/src/uihighlight.ts @@ -0,0 +1,129 @@ +/// Agent pointing — electron-free core (D6 — +/// docs/plans/desktop-ui-context-and-pointing.md §3.4b, ADR-062 D-5). +/// +/// Deixis is symmetric: a collaborating agent needs "look here" as much as +/// "what are you looking at". `ui_highlight` is the pointing half, and it is +/// deliberately the WEAKEST capability in the plan — it draws a glow and +/// expires. It never focuses, scrolls, clicks or types (the no-driving +/// non-goal), so it needs no approval card; consent is the sharing toggle plus +/// the policy table's `highlight` bit. +/// +/// What it does need is the discipline that keeps a non-actuating annotation +/// from becoming attention spam or fake UI (plan §6): +/// - the `highlight` COLUMN decides which surfaces may be drawn over at all; +/// - it is always ATTRIBUTED — the marker names the agent, so nothing an +/// agent draws can be mistaken for the app talking; +/// - it is TTL-bounded, and the bound is ours, not the caller's; +/// - it is RATE-LIMITED per agent, so a loop cannot paper the screen; +/// - the note is one short line, never a message channel. +/// +/// All of that is decided here so `node --test` proves it without Electron. +import { uiPolicyFor } from '../../src/state/ui_policy.ts'; +import { uiRefFromJson, type UiRef } from '../../src/state/uiRef.ts'; + +export const HIGHLIGHT_TTL_DEFAULT_MS = 8_000; +export const HIGHLIGHT_TTL_MAX_MS = 30_000; +export const HIGHLIGHT_NOTE_MAX = 140; + +/// Per-agent budget: how many highlights an agent may raise inside the window. +/// Generous enough for a real pointing conversation ("this one… no, this one") +/// and far short of anything that could paper the screen. +export const HIGHLIGHT_RATE_LIMIT = 6; +export const HIGHLIGHT_RATE_WINDOW_MS = 60_000; + +export function clampHighlightTtl(requested: number | null): number { + if (requested === null) return HIGHLIGHT_TTL_DEFAULT_MS; + return Math.max(500, Math.min(requested, HIGHLIGHT_TTL_MAX_MS)); +} + +export function clipHighlightNote(note: string): string { + const flat = note.replace(/\s+/g, ' ').trim(); + return flat.length > HIGHLIGHT_NOTE_MAX ? `${flat.slice(0, HIGHLIGHT_NOTE_MAX - 1)}…` : flat; +} + +/// What the renderer is asked to draw. Everything an overlay needs and nothing +/// more — no pixels, no content, and the attribution is not optional. +export interface HighlightOrder { + id: string; + ref: UiRef; + note: string; + /// Who is pointing. Rendered on the marker; an unnamed agent shows as its id + /// rather than as nobody, because an unattributed glow is the failure mode + /// the risk section names. + by: string; + ttl_ms: number; + at: string; +} + +export type HighlightDecision = + | { ok: true; order: HighlightOrder } + | { ok: false; code: string; message: string }; + +export interface HighlightInput { + ref: unknown; + note: string; + ttlMs: number | null; + agentId: string; + agentHandle: string; + /// Monotonic-ish clock, injected for the tests. + now: number; + /// Timestamps of this agent's recent highlights, newest last. The caller + /// owns the store; this function owns the rule. + recent: readonly number[]; + /// Unique id for the order (the caller supplies it — this module mints + /// nothing so it stays deterministic under test). + id: string; + iso: string; +} + +/// Decide one highlight. Order matters: parse, then policy, then rate — an +/// unparseable ref should not consume an agent's budget, and a refused surface +/// should not either (a refusal is the table talking, not abuse). +export function decideHighlight(input: HighlightInput): HighlightDecision { + const ref = uiRefFromJson(input.ref); + if (ref === null) { + return { + ok: false, + code: 'INVALID_REF', + message: + 'ref must be a UIRef: the JSON shape ui_get_focus returns ({"surface":"replay","entity":{…}}) or its URI spelling ("ui://replay?dataset_id=ds_1")', + }; + } + const row = uiPolicyFor(ref.surface); + if (row === null) { + return { ok: false, code: 'UNKNOWN_SURFACE', message: `'${ref.surface}' is not a surface this desktop declares` }; + } + if (row.highlight !== 'allow') { + return { + ok: false, + code: 'HIGHLIGHT_REFUSED', + message: `surface '${ref.surface}' refuses agent annotation by policy (ui_policy.ts highlight column)`, + }; + } + const window = input.recent.filter((t) => input.now - t < HIGHLIGHT_RATE_WINDOW_MS); + if (window.length >= HIGHLIGHT_RATE_LIMIT) { + return { + ok: false, + code: 'HIGHLIGHT_RATE_LIMITED', + message: `too many highlights (${String(HIGHLIGHT_RATE_LIMIT)} per minute) — say it in words instead`, + }; + } + const by = input.agentHandle !== '' ? input.agentHandle : input.agentId !== '' ? input.agentId : 'an agent'; + return { + ok: true, + order: { + id: input.id, + ref, + note: clipHighlightNote(input.note), + by, + ttl_ms: clampHighlightTtl(input.ttlMs), + at: input.iso, + }, + }; +} + +/// Drop timestamps outside the window — the caller's store stays bounded +/// without a sweeper. +export function pruneHighlightHistory(recent: readonly number[], now: number): number[] { + return recent.filter((t) => now - t < HIGHLIGHT_RATE_WINDOW_MS); +} diff --git a/desktop/electron/src/uihighlight_host.ts b/desktop/electron/src/uihighlight_host.ts new file mode 100644 index 00000000..fd6042f1 --- /dev/null +++ b/desktop/electron/src/uihighlight_host.ts @@ -0,0 +1,64 @@ +/// Agent pointing — Electron main-process half (D6 — +/// docs/plans/desktop-ui-context-and-pointing.md §3.4b). The rules live in the +/// electron-free uihighlight.ts; this module owns the two live parts: +/// +/// - the per-agent rate-limit store (in-memory, per app run — a hub restart +/// or an app restart forgiving an agent is fine: the limit exists to stop +/// a loop, not to punish); +/// - delivery: the order is pushed to the renderer over the shell's event +/// channel, where AgentHighlightOverlay paints it and lets it expire. +/// +/// Nothing here is stored and nothing is echoed back to the agent beyond +/// "drawn, for N seconds" — ADR-062 D-7: UI state is host-local and ephemeral. +import { setUiHighlightProvider } from './browserbridge_host'; +import { isUiSharingEnabled } from './desktopui'; +import { emit } from './events'; +import { shellWebContents } from './uicapture_host'; +import { decideHighlight, pruneHighlightHistory } from './uihighlight'; +import type { UiHighlightRequest, UiHighlightResult } from './browserbridge'; + +/// The renderer event name. One channel, one payload shape (a HighlightOrder). +export const HIGHLIGHT_EVENT = 'desktopui_highlight'; + +/// agentId → recent highlight timestamps. Bounded by the prune on every use, +/// so a chatty agent cannot grow it either. +const history = new Map(); + +let seq = 0; + +async function highlight(req: UiHighlightRequest): Promise { + if (!isUiSharingEnabled()) { + return { ok: false, code: 'UI_UNAVAILABLE', message: 'UI context sharing is off on the desktop (Settings → Assistant)' }; + } + const wc = shellWebContents(); + if (wc === null) { + return { ok: false, code: 'NO_WINDOW', message: 'the desktop window is not open' }; + } + const now = Date.now(); + const key = req.agentId !== '' ? req.agentId : 'unknown'; + const recent = pruneHighlightHistory(history.get(key) ?? [], now); + seq += 1; + const decision = decideHighlight({ + ref: req.ref, + note: req.note, + ttlMs: req.ttlMs, + agentId: req.agentId, + agentHandle: req.agentHandle, + now, + recent, + id: `hl-${String(now)}-${String(seq)}`, + iso: new Date(now).toISOString(), + }); + if (!decision.ok) { + // A refusal does NOT consume budget: a refused surface is the policy + // table talking, and an unparseable ref is a mistake, neither of which is + // the abuse the limit exists for. + history.set(key, recent); + return { ok: false, code: decision.code, message: decision.message }; + } + history.set(key, [...recent, now]); + emit(wc, HIGHLIGHT_EVENT, decision.order); + return { ok: true, surface: decision.order.ref.surface, ttl_ms: decision.order.ttl_ms }; +} + +setUiHighlightProvider(highlight); diff --git a/desktop/src/i18n/index.ts b/desktop/src/i18n/index.ts index f7ae642c..7c955e66 100644 --- a/desktop/src/i18n/index.ts +++ b/desktop/src/i18n/index.ts @@ -1713,6 +1713,11 @@ const en: Dict = { 'annotate.failed': 'The annotation could not be captured.', // D4: the title on the resolved-element chip in the target row. 'annotate.pointing': 'The element the agent will be pointed at', + // D6 — agent pointing. The attribution is not decoration: it is what keeps + // an agent's marker from reading as the app talking. + 'highlight.points': '{agent} points here', + 'highlight.goThere': 'Go there', + 'uiref.surfaceOnly': 'Opened the surface — this build can\u2019t jump to the exact item', 'debug.lines_one': '{n} line', 'debug.lines_other': '{n} lines', @@ -3757,6 +3762,9 @@ const zh: Dict = { 'annotate.clipboard': '已复制——在 kimi 输入框中粘贴（{key}）', 'annotate.failed': '无法截取标注。', 'annotate.pointing': '将指给智能体的元素', + 'highlight.points': '{agent} 指向这里', + 'highlight.goThere': '前往', + 'uiref.surfaceOnly': '已打开对应界面——当前版本无法定位到具体条目', 'debug.lines': '{n} 行', 'debug.placeholder': '粘贴代码、差异、堆栈或日志…', diff --git a/desktop/src/main.tsx b/desktop/src/main.tsx index 3fa72e5d..2936d858 100644 --- a/desktop/src/main.tsx +++ b/desktop/src/main.tsx @@ -65,6 +65,11 @@ async function boot(): Promise { // when off — the default. const { syncUiContextToMain } = await import('./state/uiContext'); syncUiContextToMain(); + // D6: listen for agent-pointing orders. Independent of the toggle above — + // main only ever pushes one while sharing is on, and a listener with nothing + // to hear costs nothing. + const { initAgentHighlights } = await import('./state/agentHighlight'); + initAgentHighlights(); } void boot(); diff --git a/desktop/src/state/agentHighlight.ts b/desktop/src/state/agentHighlight.ts new file mode 100644 index 00000000..728dd1de --- /dev/null +++ b/desktop/src/state/agentHighlight.ts @@ -0,0 +1,88 @@ +import { create } from 'zustand'; +import { listen } from '../bridge'; +import { isShell } from '../platform'; +import type { UiRef } from './uiRef'; + +/// Agent pointing, renderer half (D6 — +/// docs/plans/desktop-ui-context-and-pointing.md §3.4b, ADR-062 D-5). +/// +/// An agent calls `ui_highlight`; main decides (policy bit, rate limit, TTL) +/// and pushes ONE order down this channel. The store holds the live orders and +/// expires them; AgentHighlightOverlay paints them. +/// +/// The invariants this half must not break — the highlight is an ANNOTATION, +/// not a control: +/// - it never focuses, scrolls, clicks or types (nothing here calls into +/// the workbench store — contrast uiRefFocus.ts, which runs only from a +/// user's click on a chip); +/// - it is always attributed and always dismissible; +/// - it expires on its own, and the deadline is main's, not the agent's. + +export interface HighlightOrder { + id: string; + ref: UiRef; + note: string; + /// The agent, as shown on the marker. Never empty (main falls back to the + /// id, then to a generic subject) — an unattributed glow is the failure mode + /// the plan's risk section names. + by: string; + ttl_ms: number; + at: string; +} + +interface AgentHighlightState { + orders: HighlightOrder[]; + show: (order: HighlightOrder) => void; + dismiss: (id: string) => void; + clear: () => void; +} + +/// At most this many at once. A rate limit lives main-side too; this is the +/// visual bound — six markers is already a crowded screen, and the oldest +/// giving way keeps the newest (the one being talked about) visible. +const MAX_LIVE = 4; + +export const useAgentHighlight = create((set) => ({ + orders: [], + show: (order) => + set((s) => ({ orders: [...s.orders.filter((o) => o.id !== order.id), order].slice(-MAX_LIVE) })), + dismiss: (id) => set((s) => ({ orders: s.orders.filter((o) => o.id !== id) })), + clear: () => set({ orders: [] }), +})); + +/// Narrow one pushed payload. Main is our own process, but a renderer store +/// takes nothing on trust from an IPC boundary that an agent's arguments +/// reached (however filtered). +export function asHighlightOrder(value: unknown): HighlightOrder | null { + if (value === null || typeof value !== 'object' || Array.isArray(value)) return null; + const v = value as Record; + const ref = v.ref; + if (ref === null || typeof ref !== 'object' || Array.isArray(ref)) return null; + const surface = (ref as Record).surface; + if (typeof v.id !== 'string' || v.id === '' || typeof surface !== 'string' || surface === '') return null; + const params = (ref as Record).params; + // Main already clamps TTL (30s max) and clips the note (140) — re-applied + // here because a renderer store takes nothing on trust from an IPC payload + // an agent's arguments reached, however filtered (defense in depth). + const rawTtl = typeof v.ttl_ms === 'number' && Number.isFinite(v.ttl_ms) && v.ttl_ms > 0 ? v.ttl_ms : 8000; + const ttl = Math.min(rawTtl, 30_000); + const note = typeof v.note === 'string' ? v.note.slice(0, 140) : ''; + return { + id: v.id, + ref: { surface, params: params !== null && typeof params === 'object' ? (params as Record) : {} }, + note, + by: typeof v.by === 'string' && v.by !== '' ? v.by : 'an agent', + ttl_ms: ttl, + at: typeof v.at === 'string' ? v.at : '', + }; +} + +/// Subscribe to main's highlight channel (called once from main.tsx). No-op +/// outside the shell — a browser build has no agent pointing at it. +export function initAgentHighlights(): void { + if (!isShell()) return; + void listen('desktopui_highlight', (event) => { + const order = asHighlightOrder(event.payload); + if (order !== null) useAgentHighlight.getState().show(order); + }); +} diff --git a/desktop/src/state/inspect.ts b/desktop/src/state/inspect.ts index 73fb5457..67012c46 100644 --- a/desktop/src/state/inspect.ts +++ b/desktop/src/state/inspect.ts @@ -1,5 +1,5 @@ import { create } from 'zustand'; -import { looksLikeDot } from './dotGraph'; +import { looksLikeDot } from './dotGraph.ts'; /// The Inspect (J3) surface's open-tab model — the multi-source inspector shell /// that replaces the round-1 paste textarea. Each tab is a viewer over one diff --git a/desktop/src/state/uiContext.ts b/desktop/src/state/uiContext.ts index 92637b59..f0cc9119 100644 --- a/desktop/src/state/uiContext.ts +++ b/desktop/src/state/uiContext.ts @@ -8,6 +8,7 @@ import { useInspect } from './inspect'; import { useReplay } from './replay'; import { useTerminals } from '../terminal/store'; import { assembleRawFocus, createFocusSender, projectFocus, type FocusSources } from './ui_policy'; +import { useAgentHighlight } from './agentHighlight'; /// UI context sharing (D1 — docs/plans/desktop-ui-context-and-pointing.md /// §3.2/§3.7). Whether agents ON THIS HOST (the embedded kimi web panel and @@ -62,6 +63,10 @@ export const useUiContext = create((set) => ({ persistEnabled(v); set({ enabled: v }); setPublishing(v); + // Turning sharing OFF revokes the agents' presence on screen — any live + // highlight markers (D6) go with it rather than glowing out their TTL + // after the user withdrew consent. + if (!v) useAgentHighlight.getState().clear(); if (!isShell()) return; void invoke('desktopui_set_enabled', { enabled: v }).catch(() => undefined); }, diff --git a/desktop/src/state/uiRef.test.ts b/desktop/src/state/uiRef.test.ts new file mode 100644 index 00000000..74b2d493 --- /dev/null +++ b/desktop/src/state/uiRef.test.ts @@ -0,0 +1,114 @@ +/// Tests for the UIRef grammar (D6 — docs/plans/desktop-ui-context-and-pointing.md +/// §3.4b, ADR-062 D-2): the written form both directions of deixis share. Run +/// locally: `node --test src/state/uiRef.test.ts` from `desktop/`. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { formatUiRefUri, linkifyUiRefs, parseUiRefUri, uiRefFromJson, uiRefLabel } from './uiRef.ts'; + +test('the URI form round-trips', () => { + const ref = { surface: 'replay', params: { dataset_id: 'ds_1', episode_id: 'ep_2', cursor: '1234' } }; + const uri = formatUiRefUri(ref); + assert.equal(uri, 'ui://replay?dataset_id=ds_1&episode_id=ep_2&cursor=1234'); + assert.deepEqual(parseUiRefUri(uri), ref); + + // A bare surface is a valid ref: "look at the terminal" needs no ids. + assert.deepEqual(parseUiRefUri('ui://terminal'), { surface: 'terminal', params: {} }); + assert.equal(formatUiRefUri({ surface: 'terminal', params: {} }), 'ui://terminal'); + + // Values that need escaping survive the round trip. + const path = { surface: 'debug', params: { file: 'src/a b/c.ts', selection: '42,58' } }; + assert.deepEqual(parseUiRefUri(formatUiRefUri(path)), path); +}); + +test('the parser refuses what it cannot govern', () => { + // The surface keys the policy table — a ref we cannot key is a ref we + // cannot govern, so it is not a ref. + assert.equal(parseUiRefUri('ui://'), null); + assert.equal(parseUiRefUri('ui://Replay'), null); + assert.equal(parseUiRefUri('ui://../etc'), null); + assert.equal(parseUiRefUri('https://example.com'), null); + assert.equal(parseUiRefUri(''), null); + // Junk params are dropped, not fatal: a ref is a best-effort pointer. + assert.deepEqual(parseUiRefUri('ui://read?&=x&novalue&ok=1'), { surface: 'read', params: { ok: '1' } }); + // Values are bounded — a ref is a reference, not a payload. + const huge = `ui://read?x=${'y'.repeat(500)}`; + assert.deepEqual(parseUiRefUri(huge), { surface: 'read', params: {} }); +}); + +test('the JSON form (what ui_highlight takes) flattens to the same thing', () => { + // The ADR's nested shape, as ui_get_focus emits it. + assert.deepEqual(uiRefFromJson({ surface: 'replay', entity: { dataset_id: 'ds_1', cursor: 1234 } }), { + surface: 'replay', + params: { dataset_id: 'ds_1', cursor: '1234' }, + }); + assert.deepEqual(uiRefFromJson({ surface: 'debug', path: { file: 'src/foo.ts', selection: [42, 58] } }), { + surface: 'debug', + params: { file: 'src/foo.ts', selection: '42,58' }, + }); + // A string argument is the URI spelling — one grammar, two spellings. + assert.deepEqual(uiRefFromJson('ui://read?tab_id=wt_3'), { surface: 'read', params: { tab_id: 'wt_3' } }); + // Unknown blocks are ignored rather than rejected: refusing the whole ref + // over one stray key would make the round trip brittle. + assert.deepEqual(uiRefFromJson({ surface: 'read', mystery: { a: 'b' } }), { surface: 'read', params: { a: 'b' } }); + for (const bad of [null, 42, [], { entity: {} }, { surface: 'Nope!' }]) { + assert.equal(uiRefFromJson(bad), null, JSON.stringify(bad)); + } +}); + +test('the chip label reads as the plan writes it', () => { + assert.equal(uiRefLabel({ surface: 'replay', params: { episode_id: 'ep_9', cursor: '1234' } }), 'replay · ep_9 @ 1234'); + assert.equal(uiRefLabel({ surface: 'debug', params: { file: 'src/foo.ts', selection: '42,58' } }), 'src/foo.ts:42'); + assert.equal(uiRefLabel({ surface: 'debug', params: { file: 'src/foo.ts' } }), 'src/foo.ts'); + // Never an empty chip. + assert.equal(uiRefLabel({ surface: 'terminal', params: {} }), 'terminal'); +}); + +test('linkifyUiRefs turns agent prose into chips — and leaves code alone', () => { + const out = linkifyUiRefs('see ui://replay?episode_id=ep_9 for the drop'); + assert.equal(out, 'see [replay · ep_9](ui://replay?episode_id=ep_9) for the drop'); + + // THE thing it must never do: an agent explaining a URI inside a fence is + // SHOWING it, not pointing with it. + const fenced = 'try:\n```\nui://replay?dataset_id=ds_1\n```\ndone'; + assert.equal(linkifyUiRefs(fenced), fenced); + const inline = 'the form is `ui://replay?dataset_id=ds_1`, ok?'; + assert.equal(linkifyUiRefs(inline), inline); + // A fence that never closes still swallows its tail rather than linkifying + // half a code block. + const unclosed = 'oops\n```\nui://read\n'; + assert.equal(linkifyUiRefs(unclosed), unclosed); + + // Sentence punctuation is not part of the ref. + assert.match(linkifyUiRefs('go to ui://terminal.'), /\(ui:\/\/terminal\)\.$/); + // Several in one message all become chips. + const two = linkifyUiRefs('ui://read and ui://debug?file=a.ts'); + assert.equal((two.match(/\]\(ui:\/\//g) ?? []).length, 2); + // An unparseable token stays prose — better a literal string than a chip + // that points nowhere. + assert.equal(linkifyUiRefs('ui://'), 'ui://'); + // Text with no refs is returned untouched. + assert.equal(linkifyUiRefs('nothing here'), 'nothing here'); +}); + +test('malformed percent escapes are junk params, never a throw', () => { + // Regression: decodeURIComponent throws on a lone `%` / `%zz`, and this + // grammar reads AGENT PROSE — "100% done" is a sentence. The throw + // happened during Markdown render, so one such message crashed the + // transcript surface on every re-mount (persistent, agent-controlled). + assert.doesNotThrow(() => linkifyUiRefs('see ui://read?file=100% done')); + assert.doesNotThrow(() => linkifyUiRefs('x ui://read?a=%zz y')); + assert.doesNotThrow(() => parseUiRefUri('ui://read?a=%')); + // The pair that failed to decode is dropped; the ref itself survives — + // "a token we cannot parse stays prose" applies to the pair, not the ref. + assert.deepEqual(parseUiRefUri('ui://read?a=%zz&tab_id=wt_3'), { surface: 'read', params: { tab_id: 'wt_3' } }); + // And the linkified output still renders as a chip for the valid part. + assert.match(linkifyUiRefs('ui://read?a=%zz&tab_id=wt_3'), /^\[.*\]\(ui:\/\/read\?a=%zz&tab_id=wt_3\)$/); +}); + +test('a ref carries references only — never content', () => { + // Stated as an invariant: the type has no content field and the parser + // mints none. A future field addition has to argue with this line + // (ADR-062 D-2). + const ref = parseUiRefUri('ui://read?tab_id=wt_3'); + assert.deepEqual(Object.keys(ref ?? {}).sort(), ['params', 'surface']); +}); diff --git a/desktop/src/state/uiRef.ts b/desktop/src/state/uiRef.ts new file mode 100644 index 00000000..2ed8cf43 --- /dev/null +++ b/desktop/src/state/uiRef.ts @@ -0,0 +1,194 @@ +/// The UIRef grammar (D6 — docs/plans/desktop-ui-context-and-pointing.md +/// §3.4b, ADR-062 D-2). Deixis is symmetric: the user points at the agent with +/// the annotation overlay, and the agent points back — as a clickable chip in +/// the transcript, and as an ephemeral highlight over the surface itself. +/// +/// Both directions need one written form for "this thing on screen". ADR-062 +/// defines the UIRef as structured JSON; agents write prose, so the wire form +/// an agent can TYPE is a URI over the same fields: +/// +/// ui://replay?dataset_id=ds_1&episode_id=ep_2&cursor=1234 +/// ui://debug?file=src/foo.ts&selection=42,58 +/// ui://read?tab_id=wt_3 +/// +/// The scheme is the one already minted for the focus resource (`ui://focus`), +/// so nothing new is introduced — the surface takes the host slot and the +/// entity ids are query params. The JSON form (what `ui_highlight` takes as an +/// argument) converts to and from it, so a tool call and a transcript chip +/// point at the same thing by construction. +/// +/// Import-free on purpose: the transcript renderer, the highlight overlay and +/// the `node --test` suites all read it, and the Electron half converts the +/// tool's JSON through the same code. + +/// A reference to something on the desktop's screen. `surface` is a workbench +/// job id (or a pseudo-surface — `kimiweb`, `vault`); `params` are ids, paths, +/// fragment-stripped URLs and coordinates. Never content: a UIRef is a JOIN +/// KEY into the entity graph the agent already reaches with its own tools, not +/// a data export (ADR-062 D-2). +export interface UiRef { + surface: string; + params: Record; +} + +/// Params are `key=value` with a small vocabulary; anything longer than this +/// is not a reference. The cap also bounds what a malicious agent can push +/// into a chip label. +const MAX_PARAM_LEN = 200; +const MAX_PARAMS = 8; +const MAX_SURFACE_LEN = 32; + +/// Surfaces are workbench job ids: lowercase word characters. Anything else is +/// not parsed — the policy table is keyed by these, and a ref we cannot key is +/// a ref we cannot govern. +const SURFACE_RE = /^[a-z][a-z0-9_-]*$/; + +/// The token as it appears in agent prose. Deliberately strict about the tail: +/// a trailing `.` or `)` belongs to the sentence, not the ref. +export const UI_REF_TOKEN_RE = /ui:\/\/[a-z][a-z0-9_-]*(?:\?[A-Za-z0-9_=&,.:/@%+-]*)?/g; + +/// decodeURIComponent throws on malformed escapes (`%`, `%zz`) — and this +/// grammar reads AGENT PROSE, where "100% done" is a sentence, not an +/// escape. A throw here would take the whole transcript surface down on +/// render, and the message persists, so it would crash again on every +/// re-mount. A pair we cannot decode is junk, not an exception. +function safeDecode(part: string): string | null { + try { + return decodeURIComponent(part); + } catch { + return null; + } +} + +export function parseUiRefUri(uri: string): UiRef | null { + if (!uri.startsWith('ui://')) return null; + const rest = uri.slice('ui://'.length); + const q = rest.indexOf('?'); + const surface = (q < 0 ? rest : rest.slice(0, q)).trim(); + if (surface === '' || surface.length > MAX_SURFACE_LEN || !SURFACE_RE.test(surface)) return null; + const params: Record = {}; + if (q >= 0) { + for (const pair of rest.slice(q + 1).split('&')) { + if (pair === '') continue; + const eq = pair.indexOf('='); + if (eq <= 0) continue; + const key = safeDecode(pair.slice(0, eq)); + const value = safeDecode(pair.slice(eq + 1)); + if (key === null || value === null || key === '' || value === '' || value.length > MAX_PARAM_LEN) continue; + if (Object.keys(params).length >= MAX_PARAMS) break; + params[key] = value; + } + } + return { surface, params }; +} + +export function formatUiRefUri(ref: UiRef): string { + const entries = Object.entries(ref.params).filter(([k, v]) => k !== '' && v !== ''); + if (entries.length === 0) return `ui://${ref.surface}`; + const query = entries.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join('&'); + return `ui://${ref.surface}?${query}`; +} + +// ── The JSON form (ADR-062 D-2 / the ui_highlight argument) ────────────────── + +/// Convert the ADR's nested JSON shape into the flat one. The blocks are the +/// same vocabulary the focus projection emits (`ui_policy.ts`), so an agent +/// can hand back a slice of what `ui_get_focus` gave it and have it resolve. +/// Unknown blocks are ignored rather than rejected: a ref is a best-effort +/// pointer, and refusing the whole thing over one stray key would make the +/// round trip brittle. +export function uiRefFromJson(value: unknown): UiRef | null { + if (typeof value === 'string') return parseUiRefUri(value); + if (value === null || typeof value !== 'object' || Array.isArray(value)) return null; + const obj = value as Record; + const surface = obj.surface; + if (typeof surface !== 'string' || !SURFACE_RE.test(surface) || surface.length > MAX_SURFACE_LEN) return null; + const params: Record = {}; + const take = (key: string, raw: unknown): void => { + if (Object.keys(params).length >= MAX_PARAMS) return; + if (typeof raw === 'string' && raw !== '' && raw.length <= MAX_PARAM_LEN) params[key] = raw; + else if (typeof raw === 'number' && Number.isFinite(raw)) params[key] = String(raw); + else if (Array.isArray(raw) && raw.every((n) => typeof n === 'number')) params[key] = raw.join(','); + }; + for (const [key, raw] of Object.entries(obj)) { + if (key === 'surface') continue; + if (raw !== null && typeof raw === 'object' && !Array.isArray(raw)) { + // A block: `{ entity: {…} }`, `{ path: {…} }` — flatten its fields. + for (const [sub, subRaw] of Object.entries(raw as Record)) take(sub, subRaw); + } else { + take(key, raw); + } + } + return { surface, params }; +} + +// ── Rendering ──────────────────────────────────────────────────────────────── + +/// The chip's label. Reads as the plan writes it: `replay · ep_… @ 1234`, +/// `src/foo.ts:42`. Falls back to the surface name, never to an empty chip. +export function uiRefLabel(ref: UiRef): string { + const p = ref.params; + const file = p.file ?? p.path; + if (file !== undefined) { + const line = firstLine(p.selection); + return line === null ? file : `${file}:${String(line)}`; + } + const entity = p.episode_id ?? p.dataset_id ?? p.run_id ?? p.task_id ?? p.project_id ?? p.document_id ?? p.agent_id; + if (entity !== undefined) { + const cursor = p.cursor; + return cursor !== undefined ? `${ref.surface} · ${entity} @ ${cursor}` : `${ref.surface} · ${entity}`; + } + const tab = p.tab_id ?? p.pane_id; + return tab !== undefined ? `${ref.surface} · ${tab}` : ref.surface; +} + +function firstLine(selection: string | undefined): number | null { + if (selection === undefined) return null; + const head = Number.parseInt(selection.split(',')[0] ?? '', 10); + return Number.isFinite(head) ? head : null; +} + +/// Rewrite bare `ui://…` tokens in agent prose into markdown links, so the +/// transcript's existing link renderer can paint them as chips. +/// +/// Done as a string pre-pass rather than a markdown plugin because it must be +/// provable without a renderer, and because the ONE thing it must never do is +/// touch code: an agent explaining a URI inside a fence is showing it, not +/// pointing with it. Fenced blocks and inline-code spans are skipped verbatim. +export function linkifyUiRefs(text: string): string { + let out = ''; + for (const segment of splitCodeSpans(text)) { + if (segment.code) { + out += segment.text; + continue; + } + out += segment.text.replace(UI_REF_TOKEN_RE, (token) => { + const ref = parseUiRefUri(token); + // A token we cannot parse stays prose — better a literal string than a + // chip that points nowhere. + return ref === null ? token : `[${uiRefLabel(ref)}](${token})`; + }); + } + return out; +} + +interface Segment { + text: string; + code: boolean; +} + +/// Split on fenced blocks (``` … ```) and inline code spans (` … `). Kept +/// deliberately simple — it only has to be conservative: anything it calls +/// code is passed through untouched, which is the safe direction. +function splitCodeSpans(text: string): Segment[] { + const out: Segment[] = []; + const re = /```[\s\S]*?(?:```|$)|`[^`\n]*`/g; + let last = 0; + for (let m = re.exec(text); m !== null; m = re.exec(text)) { + if (m.index > last) out.push({ text: text.slice(last, m.index), code: false }); + out.push({ text: m[0], code: true }); + last = m.index + m[0].length; + } + if (last < text.length) out.push({ text: text.slice(last), code: false }); + return out; +} diff --git a/desktop/src/state/uiRefFocus.test.ts b/desktop/src/state/uiRefFocus.test.ts new file mode 100644 index 00000000..daae8553 --- /dev/null +++ b/desktop/src/state/uiRefFocus.test.ts @@ -0,0 +1,68 @@ +/// Tests for the UIRef → focus dispatch (D6 — plan §3.4b, §5 "ref-chip parse +/// + dispatch"). The three stores it drives are pure zustand, so a click's +/// whole effect — which tier it reached, what the stores now say — is provable +/// under `node --test`. Run locally from `desktop/` (CI does not run these). +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +// The Inspect store reads localStorage at module load; give node one before +// the module graph comes in (dynamic import below, after the stub). +(globalThis as { localStorage?: unknown }).localStorage = { + getItem: () => null, + setItem: () => undefined, + removeItem: () => undefined, +}; + +const { canFocusUiRef, focusUiRef } = await import('./uiRefFocus.ts'); +const { useWorkbench } = await import('./workbench.ts'); +const { useReplay } = await import('./replay.ts'); +const { useInspect } = await import('./inspect.ts'); + +test('canFocusUiRef: workbench jobs yes; pseudo-surfaces and junk no', () => { + assert.equal(canFocusUiRef({ surface: 'replay', params: {} }), true); + assert.equal(canFocusUiRef({ surface: 'debug', params: {} }), true); + // `vault` is a policy pseudo-surface, not a job — the chip renders but is + // not clickable, and dispatch refuses it outright. + assert.equal(canFocusUiRef({ surface: 'vault', params: {} }), false); + assert.equal(canFocusUiRef({ surface: 'nonsense', params: {} }), false); +}); + +test('an unknown surface is refused without touching the workbench', () => { + const before = useWorkbench.getState().job; + assert.equal(focusUiRef({ surface: 'nonsense', params: { dataset_id: 'ds_1' } }), 'unknown'); + assert.equal(useWorkbench.getState().job, before); +}); + +test('replay ref with dataset+project reaches the entity tier', () => { + const res = focusUiRef({ surface: 'replay', params: { dataset_id: 'ds_1', project_id: 'proj_1', episode_id: '3' } }); + assert.equal(res, 'entity'); + assert.equal(useWorkbench.getState().job, 'replay'); + assert.deepEqual(useReplay.getState().target, { datasetId: 'ds_1', projectId: 'proj_1', episode: 3 }); +}); + +test('replay ref with only a dataset preselects and still counts as entity', () => { + useReplay.setState({ target: null, selectedId: '' }); + assert.equal(focusUiRef({ surface: 'replay', params: { dataset_id: 'ds_9' } }), 'entity'); + assert.equal(useReplay.getState().selectedId, 'ds_9'); + assert.equal(useReplay.getState().target, null); +}); + +test('debug ref prefers the already-open tab; an unopened file stops at the surface', () => { + useInspect.setState({ + tabs: [{ id: 'tab_a', kind: 'code', path: 'src/foo.ts', title: 'foo.ts' } as never], + activeId: null, + }); + assert.equal(focusUiRef({ surface: 'debug', params: { file: 'src/foo.ts' } }), 'entity'); + assert.equal(useInspect.getState().activeId, 'tab_a'); + assert.equal(useWorkbench.getState().job, 'debug'); + + // The agent points at a file the user does not have open: the chip lands + // the user on Inspect and stops — honest about its depth, no new tab. + assert.equal(focusUiRef({ surface: 'debug', params: { file: 'src/other.ts' } }), 'surface'); + assert.equal(useInspect.getState().tabs.length, 1, 'dispatch never opens tabs'); +}); + +test('a bare surface ref switches the job and reports the surface tier', () => { + assert.equal(focusUiRef({ surface: 'read', params: {} }), 'surface'); + assert.equal(useWorkbench.getState().job, 'read'); +}); diff --git a/desktop/src/state/uiRefFocus.ts b/desktop/src/state/uiRefFocus.ts new file mode 100644 index 00000000..370ab68a --- /dev/null +++ b/desktop/src/state/uiRefFocus.ts @@ -0,0 +1,87 @@ +/// UIRef → focus dispatch (D6 — docs/plans/desktop-ui-context-and-pointing.md +/// §3.4b). The other half of a ref-chip: what happens when the user CLICKS it. +/// +/// The agent directs attention; **the click is the user's** (ADR-062 D-5). So +/// this module exists only behind a click handler — nothing here ever runs on +/// a ref merely being rendered, and there is deliberately no path from an +/// agent message to it. That is the whole no-actuation rule in one sentence: +/// an agent-emitted ref never focuses, scrolls, clicks or types. +/// +/// Resolution is honest about its own depth. Switching to the right surface +/// always works; entity-level focus works where a store already exposes a +/// setter for it (Replay's dataset/episode, Inspect's tabs by path), and where +/// one does not, the chip lands the user on the surface and stops. That beats +/// pretending, and it beats not shipping the chip at all. +// `.ts` extensions so `node --test` resolves the module graph — all three +// stores are pure zustand and this file is on the plan's §5 test list. +import { isKnownJob, useWorkbench, type JobId } from './workbench.ts'; +import { useReplay } from './replay.ts'; +import { useInspect } from './inspect.ts'; +import type { UiRef } from './uiRef.ts'; + +/// How far a click got. Returned so the caller can say something honest when +/// a ref points at a surface this build cannot open. +export type UiRefFocusResult = 'entity' | 'surface' | 'unknown'; + +/// Whether a ref can be focused at all — the chip renders either way (a +/// reference is worth reading), but an unfocusable one is not clickable. +export function canFocusUiRef(ref: UiRef): boolean { + return isKnownJob(ref.surface); +} + +export function focusUiRef(ref: UiRef): UiRefFocusResult { + if (!isKnownJob(ref.surface)) return 'unknown'; + const job = ref.surface as JobId; + // Entity focus is requested BEFORE the job switch: the Replay surface reads + // its target as it mounts, so setting the job first would race a surface + // that has not been told what to show yet. + const focused = focusEntity(job, ref); + useWorkbench.getState().setJob(job); + return focused ? 'entity' : 'surface'; +} + +function focusEntity(job: JobId, ref: UiRef): boolean { + const p = ref.params; + if (job === 'replay') { + const datasetId = p.dataset_id; + // `openRegistered` needs the project too — the surface defaults to the + // first project's library, so without it the dataset is never found. + const projectId = p.project_id; + if (datasetId !== undefined && projectId !== undefined) { + const episode = toIndex(p.episode ?? p.episode_id); + useReplay.getState().openRegistered({ + datasetId, + projectId, + ...(episode !== null ? { episode } : {}), + }); + return true; + } + if (datasetId !== undefined) { + // No project: the best we can do is preselect the id and let the + // surface's own library resolve it. + useReplay.getState().select(datasetId); + return true; + } + return false; + } + if (job === 'debug') { + const path = p.file ?? p.path; + if (path === undefined) return false; + const tabs = useInspect.getState().tabs; + // Prefer an already-open tab over a second one for the same file: the + // agent is pointing at what the user has, not asking for a new view. + const existing = tabs.find((t) => t.path === path); + if (existing !== undefined) { + useInspect.getState().setActive(existing.id); + return true; + } + return false; + } + return false; +} + +function toIndex(raw: string | undefined): number | null { + if (raw === undefined) return null; + const n = Number.parseInt(raw, 10); + return Number.isFinite(n) && n >= 0 ? n : null; +} diff --git a/desktop/src/styles/partials/08-plan-terminal-dock.css b/desktop/src/styles/partials/08-plan-terminal-dock.css index 4b8a8654..2ba5a7b2 100644 --- a/desktop/src/styles/partials/08-plan-terminal-dock.css +++ b/desktop/src/styles/partials/08-plan-terminal-dock.css @@ -1332,3 +1332,88 @@ white-space: nowrap; text-overflow: ellipsis; } + +/* ── D6 agent pointing (ui/AgentHighlightOverlay.tsx) ──────────────────────── + The agent's half of the pointing symmetry (ADR-062 D-5): an attributed, + expiring marker. Sits on the OVERLAY tier — deliberately BELOW `--z-menu` + (where the annotation layer lives) and below the toast tier, so an agent's + marker can never cover the Attention dock, a modal, or a consent dialog. + An agent must not be able to obscure the UI that governs it. */ +.agent-highlight-layer { + position: fixed; + right: var(--spacing-s16); + bottom: var(--spacing-s16); + z-index: var(--z-overlay); + display: flex; + flex-direction: column; + gap: var(--spacing-s8); + align-items: flex-end; + pointer-events: none; + max-width: min(420px, calc(100vw - 48px)); +} +/* The Attention dock is an in-flow region (z-auto); without this it shares + the bottom-right corner with the fixed highlight layer, and a marker's + card (pointer-events: auto) could paint over — and click-intercept — the + dock's bottom strip, where desktop_action approval cards render. The + dock therefore stacks ABOVE the highlight layer (and still below menus/ + modals): an agent's marker must never cover the UI that governs it. */ +.region.dock { + position: relative; + z-index: calc(var(--z-overlay) + 1); +} +.agent-highlight { + display: flex; + align-items: center; + gap: var(--spacing-s8); + padding: var(--spacing-s8) var(--spacing-s12); + border: 1px solid var(--accent); + border-left-width: 3px; + border-radius: var(--radius-md); + background: var(--surface); + box-shadow: var(--sh-3); + font-size: var(--font-size-label); + pointer-events: auto; +} +/* The attribution is load-bearing, not decoration: it is what keeps an + agent's marker from reading as the app talking. */ +.agent-highlight-who { + display: inline-flex; + align-items: center; + gap: 4px; + color: var(--accent); + white-space: nowrap; +} +.agent-highlight-ref { + font-family: var(--font-mono); + color: var(--text); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.agent-highlight-note { + color: var(--text-muted); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +/* D6 ref-chips: an agent-emitted `ui://…` rendered inline in the transcript. + Clicking focuses that surface — the agent directs attention, the user + actuates. A ref this build cannot focus renders disabled, still readable. */ +.md-uiref { + display: inline-flex; + align-items: baseline; + padding: 0 6px; + border: 1px solid var(--border); + border-radius: var(--radius-stadium); + background: var(--accent-tint); + color: var(--accent); + font-family: var(--font-mono); + font-size: 0.92em; + cursor: pointer; +} +.md-uiref:disabled { + color: var(--text-muted); + background: var(--surface-2); + cursor: default; +} diff --git a/desktop/src/ui/AgentHighlightOverlay.tsx b/desktop/src/ui/AgentHighlightOverlay.tsx new file mode 100644 index 00000000..03a5bce9 --- /dev/null +++ b/desktop/src/ui/AgentHighlightOverlay.tsx @@ -0,0 +1,76 @@ +import { useEffect } from 'react'; +import { useT } from '../i18n'; +import { toast } from '../state/toast'; +import { useAgentHighlight, type HighlightOrder } from '../state/agentHighlight'; +import { canFocusUiRef, focusUiRef } from '../state/uiRefFocus'; +import { uiRefLabel } from '../state/uiRef'; +import { Icon } from './Icon'; + +/// Agent pointing, the visible half (D6 — +/// docs/plans/desktop-ui-context-and-pointing.md §3.4b, ADR-062 D-5). +/// +/// An agent said "look here". This paints an attributed marker that expires — +/// and does NOTHING else. It is deliberately the weakest thing in the plan: +/// +/// - **Non-actuating.** Nothing here focuses, scrolls, clicks or types. The +/// marker offers the user a "Go there" button, and that button is the only +/// path to `focusUiRef` — the agent directs attention, the user actuates. +/// - **Attributed, always.** Every marker names the agent, so nothing an +/// agent draws can be mistaken for the app talking (the fake-UI risk). +/// - **Never over the consent UI.** It sits BELOW the modal/menu tier, so it +/// can never cover the Attention dock or an approval dialog — an agent +/// must not be able to obscure the thing that governs it. +/// - **Dismissible and self-expiring.** The TTL is main's, not the agent's. +/// +/// Mounted once at the app shell, next to AnnotationOverlay (the user's half +/// of the same symmetry). +function HighlightCard({ order }: { order: HighlightOrder }): JSX.Element { + const t = useT(); + const dismiss = useAgentHighlight((s) => s.dismiss); + const focusable = canFocusUiRef(order.ref); + + useEffect(() => { + const timer = setTimeout(() => dismiss(order.id), order.ttl_ms); + return () => clearTimeout(timer); + }, [order.id, order.ttl_ms, dismiss]); + + return ( +
+ + {/* Function replacement: `by` is agent-influenced, and a literal `$&` + in a handle must render as itself, not as a replace() pattern. */} + {t('highlight.points').replace('{agent}', () => order.by)} + + {uiRefLabel(order.ref)} + {order.note !== '' && {order.note}} + {focusable && ( + + )} + +
+ ); +} + +export function AgentHighlightOverlay(): JSX.Element | null { + const orders = useAgentHighlight((s) => s.orders); + if (orders.length === 0) return null; + return ( +
+ {orders.map((order) => ( + + ))} +
+ ); +} diff --git a/desktop/src/ui/AppShell.tsx b/desktop/src/ui/AppShell.tsx index d170c389..3dc6e82f 100644 --- a/desktop/src/ui/AppShell.tsx +++ b/desktop/src/ui/AppShell.tsx @@ -34,6 +34,7 @@ import { AssistantDock } from './AssistantDock'; import { useAssistant } from '../state/assistant'; import { useTerminals } from '../terminal/store'; import { ActivityBar } from './ActivityBar'; +import { AgentHighlightOverlay } from './AgentHighlightOverlay'; import { AnnotationOverlay } from './AnnotationOverlay'; import { CommandPalette, type Command } from './CommandPalette'; import { ConnectPanel } from './ConnectPanel'; @@ -408,6 +409,10 @@ export function AppShell(): JSX.Element { GLOBALLY by the status-bar chip / palette entry (D2.1); renders only while the UI-context-sharing toggle is on. */} + {/* D6: the agent's half of the pointing symmetry — attributed, expiring, + non-actuating, and painted BELOW the modal tier so it can never cover + the Attention dock (ADR-062 D-5). */} + ); } diff --git a/desktop/src/ui/EventCard.tsx b/desktop/src/ui/EventCard.tsx index 37feb4b8..e2219ed2 100644 --- a/desktop/src/ui/EventCard.tsx +++ b/desktop/src/ui/EventCard.tsx @@ -370,7 +370,7 @@ function InputTextBody({ p }: { p: Entity }): JSX.Element {
{label} - +
); } @@ -408,11 +408,14 @@ function bodyFor(ev: FeedEvent, t: TLookup, result?: Entity, callName?: string): const p = ev.payload; switch (ev.kind) { case 'text': { + // `text` is the agent's own prose — the PRIMARY carrier of D6 + // ref-chips ("an agent-emitted UIRef in a reply renders as a + // clickable chip"). uiRefs must ride both branches of the fold. const body = str(p, 'text') ?? ''; - if (!isFoldable(body)) return ; + if (!isFoldable(body)) return ; return ( - + ); } @@ -528,7 +531,7 @@ function bodyFor(ev: FeedEvent, t: TLookup, result?: Entity, callName?: string): return ; } const text = str(p, 'text'); - if (text !== undefined) return ; + if (text !== undefined) return ; return (
{ev.kind} diff --git a/desktop/src/ui/Markdown.tsx b/desktop/src/ui/Markdown.tsx index 6004aee0..9921853f 100644 --- a/desktop/src/ui/Markdown.tsx +++ b/desktop/src/ui/Markdown.tsx @@ -1,4 +1,7 @@ import { isValidElement, memo, useEffect, useState, type ReactNode } from 'react'; +import { linkifyUiRefs, parseUiRefUri } from '../state/uiRef'; +import { canFocusUiRef, focusUiRef } from '../state/uiRefFocus'; +import { toast } from '../state/toast'; import ReactMarkdown, { defaultUrlTransform, type Components } from 'react-markdown'; import { Icon } from './Icon'; import { useT } from '../i18n'; @@ -18,6 +21,10 @@ import rehypeHighlight from 'rehype-highlight'; // `` can only paint — it can't execute script — so this is XSS-safe. function urlTransform(url: string): string { if (/^data:image\//i.test(url) || url.startsWith(NOTE_ATT_SCHEME)) return url; + // D6 ref-chips: `ui://?…` is our own in-app reference scheme, and + // the default transform would strip it. Safe by construction — it never + // reaches the network; clicking one calls focusUiRef and nothing else. + if (url.startsWith('ui://')) return url; return defaultUrlTransform(url); } @@ -201,6 +208,7 @@ export const Markdown = memo(function Markdown({ text, singleDollarMath = false, headingIds = false, + uiRefs = false, }: { text: string; // Enable `$…$` inline math + `\(…\)`/`\[…\]` LaTeX-delimiter normalization. @@ -210,9 +218,15 @@ export const Markdown = memo(function Markdown({ singleDollarMath?: boolean; // Stamp `id` on headings so the MarkdownReader outline can scroll to them. headingIds?: boolean; + // D6: render agent-emitted `ui://…` references as clickable focus chips. + // ON for agent transcripts only — prose surfaces have no desktop to point + // at, and a chip in a document would be a dead control. + uiRefs?: boolean; }): JSX.Element { const openLink = useOpenLink(); - const src = singleDollarMath ? normalizeMath(text) : text; + const t = useT(); + const withRefs = uiRefs ? linkifyUiRefs(text) : text; + const src = singleDollarMath ? normalizeMath(withRefs) : withRefs; const heading = (Tag: 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6'): Components[typeof Tag] => function H({ node, children }): JSX.Element { const id = headingIds ? slugify(hastText(node as HastLike)) || undefined : undefined; @@ -237,6 +251,38 @@ export const Markdown = memo(function Markdown({ img: ({ src, alt }) => , pre: ({ children }) => {children}, a: ({ children, href }) => { + // D6: an agent-emitted UIRef renders as a chip that focuses that + // surface when the USER clicks it — the agent directs attention, + // the user actuates (ADR-062 D-5). A ref this build cannot focus + // still renders, just not as a control: the reference is worth + // reading even when the jump is not available. + if (typeof href === 'string' && href.startsWith('ui://')) { + const ref = parseUiRefUri(href); + if (ref === null) return {children}; + const focusable = canFocusUiRef(ref); + return ( + + ); + } const external = typeof href === 'string' && /^(https?:|mailto:)/.test(href); return (