diff --git a/rust/limux-cli/src/main.rs b/rust/limux-cli/src/main.rs index f5674ca4..a919ecc7 100644 --- a/rust/limux-cli/src/main.rs +++ b/rust/limux-cli/src/main.rs @@ -1099,23 +1099,22 @@ async fn run_browser( let sid = surface .clone() .ok_or_else(|| anyhow!("browser screenshot requires a surface"))?; - let mut payload = - browser_call(client, Some(sid), "browser.screenshot", Map::new()).await?; let out = parse_opt(&browser_args, "--out"); - let mut path = get_string(&payload, &["path"]) - .unwrap_or_else(|| "/tmp/limux-browser-shot.png".to_string()); - if let Some(out_path) = out { - path = out_path; - } - if !Path::new(&path).exists() { - if let Some(parent) = Path::new(&path).parent() { - fs::create_dir_all(parent).with_context(|| { - format!("failed to create screenshot directory {}", parent.display()) - })?; + let mut params = Map::new(); + if let Some(ref out_path) = out { + if let Some(parent) = Path::new(out_path).parent() { + if !parent.as_os_str().is_empty() { + fs::create_dir_all(parent).with_context(|| { + format!("failed to create screenshot directory {}", parent.display()) + })?; + } } - fs::write(&path, []) - .with_context(|| format!("failed to create screenshot {}", path))?; + params.insert("path".to_string(), Value::String(out_path.clone())); } + let mut payload = browser_call(client, Some(sid), "browser.screenshot", params).await?; + let path = get_string(&payload, &["path"]) + .or(out) + .unwrap_or_else(|| "/tmp/limux-browser-shot.png".to_string()); let url = format!("file://{}", path); if let Some(obj) = payload.as_object_mut() { obj.insert("path".to_string(), Value::String(path.clone())); diff --git a/rust/limux-core/src/lib.rs b/rust/limux-core/src/lib.rs index af50fd19..4ccbab09 100644 --- a/rust/limux-core/src/lib.rs +++ b/rust/limux-core/src/lib.rs @@ -2689,37 +2689,21 @@ impl ControlState { } true } - "down" | "ctrl+n" | "ctrl+j" => { - if palette_visible { - self.command_palette_move_selection(window_id, 1); - true - } else { - false - } + "down" | "ctrl+n" | "ctrl+j" if palette_visible => { + self.command_palette_move_selection(window_id, 1); + true } - "up" | "ctrl+p" | "ctrl+k" => { - if palette_visible { - self.command_palette_move_selection(window_id, -1); - true - } else { - false - } + "up" | "ctrl+p" | "ctrl+k" if palette_visible => { + self.command_palette_move_selection(window_id, -1); + true } - "cmd+a" => { - if palette_visible { - self.command_palette_select_all(window_id); - true - } else { - false - } + "cmd+a" if palette_visible => { + self.command_palette_select_all(window_id); + true } - "enter" => { - if palette_visible { - self.command_palette_enter(window_id); - true - } else { - false - } + "enter" if palette_visible => { + self.command_palette_enter(window_id); + true } _ => false, } diff --git a/rust/limux-host-linux/src/browser_init.js b/rust/limux-host-linux/src/browser_init.js new file mode 100644 index 00000000..b24ef5c2 --- /dev/null +++ b/rust/limux-host-linux/src/browser_init.js @@ -0,0 +1,333 @@ +// limux browser automation init script +// Runs at DocumentStart on every top-frame navigation, before any page +// script. Exposes `window.__limux` with ref tagging, console/error ring +// buffers, history observers, and ready/editable probes. + +(() => { + if (window.__limux) return; + + // webkit6 JSC returns -2147483648 for Date.now() and performance.timeOrigin + // on most pages (wall-clock clamping to i32::MIN). performance.now() is the + // only reliable time source — it's monotonic, relative to page load, and + // comes back as a real number. We pair it with a sequence counter so the + // caller can filter by "since last call" deterministically. + let seq = 0; + const nowMs = () => Math.floor(performance.now ? performance.now() : 0); + const nextSeq = () => ++seq; + + + const LOG_CAP = 5000; + const ERR_CAP = 5000; + const INTERACTIVE_TAGS = new Set([ + "A", "BUTTON", "INPUT", "SELECT", "TEXTAREA", "SUMMARY", "DETAILS", + "LABEL", + ]); + const INTERACTIVE_ROLES = new Set([ + "button", "link", "checkbox", "radio", "menuitem", "menuitemcheckbox", + "menuitemradio", "tab", "option", "combobox", "textbox", "searchbox", + "slider", "spinbutton", "switch", "treeitem", "listitem", + ]); + + const state = { + nextRefId: 1, + refMeta: Object.create(null), + logs: [], + logsDroppedCount: 0, + errors: [], + errorsDroppedCount: 0, + navCount: 0, + lastMutationAt: 0, + mutationQuietMs: 500, + }; + + function isInteractive(el) { + if (!el || el.nodeType !== 1) return false; + if (INTERACTIVE_TAGS.has(el.tagName)) { + if (el.tagName === "A") return !!el.getAttribute("href"); + if (el.tagName === "LABEL") return !!el.getAttribute("for"); + return true; + } + const role = el.getAttribute && el.getAttribute("role"); + if (role && INTERACTIVE_ROLES.has(role.toLowerCase())) return true; + if (el.isContentEditable) return true; + const tabindex = el.getAttribute && el.getAttribute("tabindex"); + if (tabindex != null && Number.parseInt(tabindex, 10) >= 0) return true; + return false; + } + + function accessibleName(el) { + const aria = el.getAttribute && el.getAttribute("aria-label"); + if (aria) return aria.trim(); + const labelledBy = el.getAttribute && el.getAttribute("aria-labelledby"); + if (labelledBy) { + const parts = labelledBy.split(/\s+/) + .map((id) => document.getElementById(id)) + .filter(Boolean) + .map((n) => (n.textContent || "").trim()) + .filter(Boolean); + if (parts.length) return parts.join(" "); + } + if (el.tagName === "INPUT" || el.tagName === "SELECT" || el.tagName === "TEXTAREA") { + if (el.id) { + const label = document.querySelector(`label[for="${CSS.escape(el.id)}"]`); + if (label) return (label.textContent || "").trim(); + } + if (el.placeholder) return el.placeholder; + if (el.name) return el.name; + } + if (el.tagName === "IMG") { + const alt = el.getAttribute("alt"); + if (alt) return alt; + } + const title = el.getAttribute && el.getAttribute("title"); + if (title) return title; + const text = (el.innerText || el.textContent || "").replace(/\s+/g, " ").trim(); + if (text.length > 120) return text.slice(0, 120); + return text; + } + + function elementRole(el) { + const explicit = el.getAttribute && el.getAttribute("role"); + if (explicit) return explicit.toLowerCase(); + switch (el.tagName) { + case "A": return "link"; + case "BUTTON": return "button"; + case "SELECT": return "combobox"; + case "TEXTAREA": return "textbox"; + case "INPUT": { + const type = (el.type || "text").toLowerCase(); + if (type === "checkbox") return "checkbox"; + if (type === "radio") return "radio"; + if (type === "submit" || type === "button" || type === "reset") return "button"; + if (type === "range") return "slider"; + if (type === "search") return "searchbox"; + return "textbox"; + } + case "SUMMARY": return "button"; + case "DETAILS": return "group"; + case "LABEL": return "LabelText"; + default: return el.tagName.toLowerCase(); + } + } + + function assignRef(el) { + if (!isInteractive(el)) return null; + let id = el.getAttribute("data-limux-ref"); + if (id) { + // refresh metadata in case attributes changed + state.refMeta[id] = { + tag: el.tagName.toLowerCase(), + role: elementRole(el), + name: accessibleName(el), + }; + return id; + } + id = "e" + state.nextRefId++; + try { el.setAttribute("data-limux-ref", id); } catch (_) { return null; } + state.refMeta[id] = { + tag: el.tagName.toLowerCase(), + role: elementRole(el), + name: accessibleName(el), + }; + return id; + } + + function tagSubtree(root) { + if (!root) return; + if (root.nodeType === 1) assignRef(root); + if (root.querySelectorAll) { + const all = root.querySelectorAll( + "a[href], button, input, select, textarea, summary, details, label[for], " + + "[role], [contenteditable], [tabindex]" + ); + for (let i = 0; i < all.length; i++) assignRef(all[i]); + } + } + + function releaseRef(el) { + if (!el || el.nodeType !== 1) return; + const id = el.getAttribute && el.getAttribute("data-limux-ref"); + if (!id) return; + delete state.refMeta[id]; + } + + function releaseSubtree(root) { + if (!root) return; + if (root.nodeType === 1) releaseRef(root); + if (root.querySelectorAll) { + const all = root.querySelectorAll("[data-limux-ref]"); + for (let i = 0; i < all.length; i++) releaseRef(all[i]); + } + } + + function pushRing(buffer, entry, cap, dropCounterKey) { + if (buffer.length >= cap) { + buffer.shift(); + state[dropCounterKey]++; + } + buffer.push(entry); + } + + function recordLog(level, args) { + try { + const parts = []; + for (let i = 0; i < args.length; i++) { + const a = args[i]; + if (a == null) { parts.push(String(a)); continue; } + if (typeof a === "string") { parts.push(a); continue; } + if (a instanceof Error) { parts.push(a.stack || a.message); continue; } + try { parts.push(JSON.stringify(a)); } catch (_) { parts.push(String(a)); } + } + pushRing(state.logs, { + seq: nextSeq(), + ts_ms: nowMs(), + level, + text: parts.join(" "), + }, LOG_CAP, "logsDroppedCount"); + } catch (_) { /* never throw from console hook */ } + } + + function installConsoleHook() { + const levels = ["log", "warn", "error", "info", "debug"]; + for (const level of levels) { + const original = console[level] ? console[level].bind(console) : null; + console[level] = function () { + recordLog(level, arguments); + if (original) original.apply(null, arguments); + }; + } + } + + function installErrorHook() { + window.addEventListener("error", (ev) => { + pushRing(state.errors, { + seq: nextSeq(), + ts_ms: nowMs(), + source: "error", + message: ev.message, + filename: ev.filename, + lineno: ev.lineno, + colno: ev.colno, + stack: ev.error && ev.error.stack ? ev.error.stack : null, + }, ERR_CAP, "errorsDroppedCount"); + }, true); + window.addEventListener("unhandledrejection", (ev) => { + let reason = ev.reason; + if (reason instanceof Error) reason = reason.stack || reason.message; + else { try { reason = JSON.stringify(reason); } catch (_) { reason = String(reason); } } + pushRing(state.errors, { + seq: nextSeq(), + ts_ms: nowMs(), + source: "unhandledrejection", + message: String(reason), + }, ERR_CAP, "errorsDroppedCount"); + }, true); + } + + function installHistoryHook() { + const fire = (kind, url) => { + state.navCount++; + try { + window.dispatchEvent(new CustomEvent("limux:navigation", { + detail: { kind, url, navCount: state.navCount }, + })); + } catch (_) { /* ignore */ } + }; + const origPush = history.pushState; + history.pushState = function (data, title, url) { + const ret = origPush.apply(this, arguments); + fire("pushState", url || location.href); + return ret; + }; + const origReplace = history.replaceState; + history.replaceState = function (data, title, url) { + const ret = origReplace.apply(this, arguments); + fire("replaceState", url || location.href); + return ret; + }; + window.addEventListener("popstate", () => fire("popstate", location.href), true); + } + + function installMutationObserver() { + const obs = new MutationObserver((records) => { + state.lastMutationAt = nowMs(); + for (const rec of records) { + if (rec.type === "childList") { + for (const node of rec.addedNodes) tagSubtree(node); + for (const node of rec.removedNodes) releaseSubtree(node); + } else if (rec.type === "attributes" && rec.target && rec.target.nodeType === 1) { + assignRef(rec.target); + } + } + }); + const start = () => { + if (!document.documentElement) return; + obs.observe(document.documentElement, { + childList: true, + subtree: true, + attributes: true, + attributeFilter: [ + "role", "aria-label", "aria-labelledby", "href", "for", "tabindex", + "contenteditable", "disabled", "checked", "value", "placeholder", + "name", "type", + ], + }); + tagSubtree(document.body); + }; + if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", start, { once: true }); + } else { + start(); + } + } + + // API surface. Frozen — reads cannot accidentally mutate internal state. + const api = Object.freeze({ + version: 1, + refMeta: state.refMeta, + get logs() { return state.logs.slice(); }, + get errors() { return state.errors.slice(); }, + get logsDroppedCount() { return state.logsDroppedCount; }, + get errorsDroppedCount() { return state.errorsDroppedCount; }, + get navCount() { return state.navCount; }, + isReady() { + if (document.readyState !== "complete") return false; + if (state.lastMutationAt === 0) return true; + return (nowMs() - state.lastMutationAt) >= state.mutationQuietMs; + }, + isEditable() { + const a = document.activeElement; + if (!a) return false; + if (a.isContentEditable) return true; + const tag = a.tagName; + if (tag === "INPUT") { + const type = (a.type || "text").toLowerCase(); + return !["button", "submit", "reset", "checkbox", "radio", "range"].includes(type); + } + return tag === "TEXTAREA"; + }, + clearLogs() { state.logs.length = 0; state.logsDroppedCount = 0; }, + clearErrors() { state.errors.length = 0; state.errorsDroppedCount = 0; }, + lookupRef(id) { + const el = document.querySelector(`[data-limux-ref="${CSS.escape(id)}"]`); + return el || null; + }, + refInfo(id) { + const meta = state.refMeta[id]; + if (!meta) return null; + const el = this.lookupRef(id); + return { id, tag: meta.tag, role: meta.role, name: meta.name, attached: !!el }; + }, + tagSubtree: tagSubtree, + assignRef: assignRef, + elementRole: elementRole, + accessibleName: accessibleName, + isInteractive: isInteractive, + }); + Object.defineProperty(window, "__limux", { value: api, configurable: false, writable: false }); + + installConsoleHook(); + installErrorHook(); + installHistoryHook(); + installMutationObserver(); +})(); diff --git a/rust/limux-host-linux/src/browser_snapshot.js b/rust/limux-host-linux/src/browser_snapshot.js new file mode 100644 index 00000000..9dfddd29 --- /dev/null +++ b/rust/limux-host-linux/src/browser_snapshot.js @@ -0,0 +1,218 @@ +// limux browser snapshot walker. Invoked on-demand via evaluate_javascript. +// Returns a JSON string per the design doc: +// { +// url, title, hash, +// snapshot_text: "page ...\n- banner\n - link \"Home\" [ref=e1]\n...", +// refs: { eN: {selector, role, name, tag} }, +// shadow_closed: bool, +// truncated: null | { logs: N, errors: N, recreated: bool } +// } +// +// Options are passed in by the server: +// opts.full_tree — emit non-interactive text nodes too +// opts.raw_html — bypass AX walker, return document.documentElement.outerHTML +// opts.selector — scope walker to the first matching element +// opts.max_depth — clamp walker depth +// opts.since_hash — if matches current hash, emit diff only +// +// All parameters come in via placeholder substitution as a JSON literal +// named `__LIMUX_SNAPSHOT_OPTS__`. Rust builds the call like: +// (opts) => { ... }(__LIMUX_SNAPSHOT_OPTS__) + +((opts) => { + if (!window.__limux) { + return JSON.stringify({ error: { code: "INIT_NOT_READY", message: "init script not yet installed" } }); + } + const api = window.__limux; + const { full_tree = false, raw_html = false, selector = null, max_depth = null, since_hash = null } = (opts || {}); + + if (raw_html) { + const html = document.documentElement ? document.documentElement.outerHTML : ""; + return JSON.stringify({ + url: location.href, + title: document.title, + raw_html: html, + refs: api.refMeta, + truncated: truncationReport(), + }); + } + + const root = selector ? document.querySelector(selector) : document.body; + if (!root) { + return JSON.stringify({ + url: location.href, + title: document.title, + snapshot_text: "", + refs: {}, + shadow_closed: false, + truncated: truncationReport(), + error: { code: "SELECTOR_NOT_FOUND", message: "selector root not found" }, + }); + } + + // Refresh refs on subtree first to catch elements added since last mutation tick. + api.tagSubtree(root); + + const refs = Object.create(null); + let shadowClosed = false; + const lines = []; + const depthCap = (max_depth == null) ? Infinity : Number(max_depth); + + const pageLine = `page ${location.href} title ${jsonQuote(document.title || "")}`; + lines.push(pageLine); + + walk(root, 0); + const text = lines.join("\n"); + const hash = djb2(text); + + if (since_hash && since_hash === hash) { + return JSON.stringify({ + url: location.href, + title: document.title, + hash, + unchanged: true, + truncated: truncationReport(), + }); + } + + return JSON.stringify({ + url: location.href, + title: document.title, + hash, + snapshot_text: text, + refs, + shadow_closed: shadowClosed, + truncated: truncationReport(), + }); + + function walk(el, depth) { + if (!el || el.nodeType !== 1) return; + if (depth > depthCap) return; + + const tag = el.tagName; + if (tag === "SCRIPT" || tag === "STYLE" || tag === "NOSCRIPT" || tag === "TEMPLATE") return; + + // Detect closed shadow DOM host so the agent knows to switch tools. + if (el.shadowRoot === null && typeof el.attachShadow === "function") { + // shadowRoot null doesn't prove closed mode; use getRootNode to detect open-mode hosts. + } + // Rough heuristic: if the element has slotted children but shadowRoot is + // inaccessible, mark as closed. WebKit exposes open shadow via shadowRoot. + if (el.shadowRoot && el.shadowRoot.mode === "closed") shadowClosed = true; + + const hidden = isHiddenForSnapshot(el); + if (hidden && !el.matches("[tabindex], input, button, a[href], [role]")) return; + + const interactive = api.isInteractive(el); + let refId = null; + if (interactive) { + refId = el.getAttribute("data-limux-ref") || api.assignRef(el); + if (refId) { + refs[refId] = { + selector: `[data-limux-ref="${refId}"]`, + role: api.elementRole(el), + name: api.accessibleName(el), + tag: tag.toLowerCase(), + }; + } + } + + const role = api.elementRole(el); + const name = interactive ? api.accessibleName(el) : pickTextualName(el); + + if (interactive || full_tree || isLandmark(role) || name) { + lines.push(formatNode(depth, role, name, refId, el, hidden)); + } + + // Iframes + open shadow DOM children traversal (cross-frame snapshot for open mode only). + if (el.shadowRoot && el.shadowRoot.mode === "open") { + for (const child of el.shadowRoot.children) walk(child, depth + 1); + } + + for (const child of el.children) walk(child, depth + 1); + } + + function formatNode(depth, role, name, refId, el, hidden) { + const indent = " ".repeat(depth + 1); + const attrs = []; + if (refId) attrs.push(`ref=${refId}`); + if (hidden) attrs.push("hidden=true"); + if (el.hasAttribute("required")) attrs.push("required"); + if (el.hasAttribute("disabled")) attrs.push("disabled"); + if (el.hasAttribute("readonly")) attrs.push("readonly"); + if (el.type === "checkbox" || el.type === "radio") attrs.push(`checked=${!!el.checked}`); + const ariaExpanded = el.getAttribute("aria-expanded"); + if (ariaExpanded != null) attrs.push(`expanded=${ariaExpanded}`); + const ariaSelected = el.getAttribute("aria-selected"); + if (ariaSelected != null) attrs.push(`selected=${ariaSelected}`); + if (el.tagName === "H1") attrs.push("level=1"); + else if (el.tagName === "H2") attrs.push("level=2"); + else if (el.tagName === "H3") attrs.push("level=3"); + else if (el.tagName === "H4") attrs.push("level=4"); + else if (el.tagName === "H5") attrs.push("level=5"); + else if (el.tagName === "H6") attrs.push("level=6"); + if (el.tagName === "INPUT") { + const type = (el.type || "text").toLowerCase(); + if (type !== "text") attrs.push(`type=${type}`); + if (el.placeholder) attrs.push(`placeholder=${jsonQuote(el.placeholder)}`); + if (el.value && type !== "password") attrs.push(`value=${jsonQuote(trunc(el.value, 80))}`); + } + + let line = `${indent}- ${role}`; + if (name) line += ` ${jsonQuote(trunc(name, 120))}`; + if (attrs.length) line += ` [${attrs.join(", ")}]`; + return line; + } + + function isHiddenForSnapshot(el) { + if (el.hasAttribute("hidden")) return true; + if (el.getAttribute && el.getAttribute("aria-hidden") === "true") return true; + const style = el.ownerDocument.defaultView && el.ownerDocument.defaultView.getComputedStyle + ? el.ownerDocument.defaultView.getComputedStyle(el) + : null; + if (style && (style.display === "none" || style.visibility === "hidden")) return true; + return false; + } + + function isLandmark(role) { + return [ + "main", "banner", "navigation", "contentinfo", "complementary", "search", + "form", "region", "dialog", "alert", "alertdialog", "status", + "heading", "list", "listitem", "table", "row", "cell", + ].includes(role); + } + + function pickTextualName(el) { + // For landmarks/text containers, collect own text (not descendants). + if (!el.childNodes) return ""; + let out = ""; + for (const child of el.childNodes) { + if (child.nodeType === 3) out += child.nodeValue; + } + out = out.replace(/\s+/g, " ").trim(); + if (out.length > 120) out = out.slice(0, 120); + return out; + } + + function jsonQuote(s) { + return JSON.stringify(String(s == null ? "" : s)); + } + function trunc(s, n) { + if (s == null) return ""; + const str = String(s); + return str.length > n ? str.slice(0, n) : str; + } + function djb2(str) { + let h = 5381; + for (let i = 0; i < str.length; i++) { + h = ((h << 5) + h + str.charCodeAt(i)) | 0; + } + return "djb2:" + (h >>> 0).toString(16); + } + function truncationReport() { + const logs = api.logsDroppedCount; + const errors = api.errorsDroppedCount; + if (logs === 0 && errors === 0) return null; + return { logs, errors }; + } +})(__LIMUX_SNAPSHOT_OPTS__); diff --git a/rust/limux-host-linux/src/control_bridge.rs b/rust/limux-host-linux/src/control_bridge.rs index 38dc5b95..7b74d01b 100644 --- a/rust/limux-host-linux/src/control_bridge.rs +++ b/rust/limux-host-linux/src/control_bridge.rs @@ -26,6 +26,51 @@ const METHODS: &[&str] = &[ "workspace.rename", "workspace.close", "surface.send_text", + "pane.list", + "pane.surfaces", + "surface.list", + "surface.current", + "browser.open_split", + "browser.navigate", + "browser.url.get", + "browser.back", + "browser.forward", + "browser.reload", + "browser.screenshot", + "browser.eval", + "browser.snapshot", + "browser.click", + "browser.dblclick", + "browser.hover", + "browser.focus", + "browser.fill", + "browser.type", + "browser.press", + "browser.check", + "browser.uncheck", + "browser.select", + "browser.scroll", + "browser.scroll_into_view", + "browser.wait", + "browser.wait_ready", + "browser.get.text", + "browser.get.title", + "browser.get.html", + "browser.get.value", + "browser.get.attr", + "browser.get.count", + "browser.get.box", + "browser.find.role", + "browser.find.text", + "browser.find.label", + "browser.find.placeholder", + "browser.find.testid", + "browser.console.list", + "browser.console.clear", + "browser.errors.list", + "browser.errors.clear", + "browser.is_ready", + "browser.is_editable", ]; const PARSE_ERROR_CODE: i64 = -32700; @@ -82,6 +127,60 @@ pub enum ControlCommand { text: String, reply: mpsc::Sender, }, + ListPanes { + target: WorkspaceTarget, + reply: mpsc::Sender, + }, + ListSurfaces { + target: WorkspaceTarget, + pane_filter: Option, + reply: mpsc::Sender, + }, + CurrentSurface { + target: WorkspaceTarget, + reply: mpsc::Sender, + }, + BrowserOpenSplit { + target: WorkspaceTarget, + source_surface: Option, + url: Option, + reply: mpsc::Sender, + }, + BrowserNavigate { + surface: String, + url: String, + reply: mpsc::Sender, + }, + BrowserGetUrl { + surface: String, + reply: mpsc::Sender, + }, + BrowserBack { + surface: String, + reply: mpsc::Sender, + }, + BrowserForward { + surface: String, + reply: mpsc::Sender, + }, + BrowserReload { + surface: String, + reply: mpsc::Sender, + }, + BrowserScreenshot { + surface: String, + out_path: Option, + reply: mpsc::Sender, + }, + BrowserEval { + surface: String, + script: String, + /// If Some, the handler parses the JS reply as JSON and wraps it under + /// this key in the response. If None, the reply is returned verbatim + /// as a string under the "result" key. + wrap_key: Option, + reply: mpsc::Sender, + }, } impl ControlCommand { @@ -94,7 +193,18 @@ impl ControlCommand { | Self::SelectWorkspace { reply, .. } | Self::RenameWorkspace { reply, .. } | Self::CloseWorkspace { reply, .. } - | Self::SendText { reply, .. } => { + | Self::SendText { reply, .. } + | Self::ListPanes { reply, .. } + | Self::ListSurfaces { reply, .. } + | Self::CurrentSurface { reply, .. } + | Self::BrowserOpenSplit { reply, .. } + | Self::BrowserNavigate { reply, .. } + | Self::BrowserGetUrl { reply, .. } + | Self::BrowserBack { reply, .. } + | Self::BrowserForward { reply, .. } + | Self::BrowserReload { reply, .. } + | Self::BrowserScreenshot { reply, .. } + | Self::BrowserEval { reply, .. } => { let _ = reply.send(result); } } @@ -185,6 +295,23 @@ fn optional_index(params: &Map, key: &str) -> Result, + keys: &[&str], + label: &str, +) -> Result { + optional_string(params, keys) + .ok_or_else(|| BridgeError::invalid_params(format!("{label} is required"))) +} + +/// Strip a leading `prefix:` (e.g. `surface:UUID` → `UUID`) so callers can pass +/// either raw IDs or refs. +fn normalize_handle(raw: String, prefix: &str) -> String { + raw.strip_prefix(prefix) + .map(|rest| rest.to_string()) + .unwrap_or(raw) +} + fn parse_optional_workspace_target( params: &Map, allow_name: bool, @@ -323,6 +450,165 @@ fn handle_method( rx, ) } + "pane.list" | "list-panes" | "list-panels" => { + let target = match parse_optional_workspace_target(params, false) { + Ok(target) => target, + Err(error) => return error_response(id, error), + }; + let (reply, rx) = mpsc::channel(); + (ControlCommand::ListPanes { target, reply }, rx) + } + "pane.surfaces" | "surface.list" => { + let target = match parse_optional_workspace_target(params, false) { + Ok(target) => target, + Err(error) => return error_response(id, error), + }; + let pane_filter = optional_string(params, &["pane_id", "id"]) + .map(|raw| normalize_handle(raw, "pane:")); + let (reply, rx) = mpsc::channel(); + ( + ControlCommand::ListSurfaces { + target, + pane_filter, + reply, + }, + rx, + ) + } + "surface.current" => { + let target = match parse_optional_workspace_target(params, false) { + Ok(target) => target, + Err(error) => return error_response(id, error), + }; + let (reply, rx) = mpsc::channel(); + (ControlCommand::CurrentSurface { target, reply }, rx) + } + "browser.open_split" | "browser.open" | "browser.new" => { + let target = match parse_optional_workspace_target(params, false) { + Ok(target) => target, + Err(error) => return error_response(id, error), + }; + let source_surface = optional_string(params, &["surface_id", "id"]) + .map(|raw| normalize_handle(raw, "surface:")); + let url = optional_string(params, &["url"]); + let (reply, rx) = mpsc::channel(); + ( + ControlCommand::BrowserOpenSplit { + target, + source_surface, + url, + reply, + }, + rx, + ) + } + "browser.navigate" | "browser.goto" => { + let surface = match required_string(params, &["surface_id", "id"], "surface_id") { + Ok(value) => normalize_handle(value, "surface:"), + Err(error) => return error_response(id, error), + }; + let url = match required_string(params, &["url"], "url") { + Ok(value) => value, + Err(error) => return error_response(id, error), + }; + let (reply, rx) = mpsc::channel(); + ( + ControlCommand::BrowserNavigate { + surface, + url, + reply, + }, + rx, + ) + } + "browser.url.get" | "browser.get.url" => { + let surface = match required_string(params, &["surface_id", "id"], "surface_id") { + Ok(value) => normalize_handle(value, "surface:"), + Err(error) => return error_response(id, error), + }; + let (reply, rx) = mpsc::channel(); + (ControlCommand::BrowserGetUrl { surface, reply }, rx) + } + "browser.back" => { + let surface = match required_string(params, &["surface_id", "id"], "surface_id") { + Ok(value) => normalize_handle(value, "surface:"), + Err(error) => return error_response(id, error), + }; + let (reply, rx) = mpsc::channel(); + (ControlCommand::BrowserBack { surface, reply }, rx) + } + "browser.forward" => { + let surface = match required_string(params, &["surface_id", "id"], "surface_id") { + Ok(value) => normalize_handle(value, "surface:"), + Err(error) => return error_response(id, error), + }; + let (reply, rx) = mpsc::channel(); + (ControlCommand::BrowserForward { surface, reply }, rx) + } + "browser.reload" => { + let surface = match required_string(params, &["surface_id", "id"], "surface_id") { + Ok(value) => normalize_handle(value, "surface:"), + Err(error) => return error_response(id, error), + }; + let (reply, rx) = mpsc::channel(); + (ControlCommand::BrowserReload { surface, reply }, rx) + } + "browser.screenshot" => { + let surface = match required_string(params, &["surface_id", "id"], "surface_id") { + Ok(value) => normalize_handle(value, "surface:"), + Err(error) => return error_response(id, error), + }; + let out_path = optional_string(params, &["out", "path"]); + let (reply, rx) = mpsc::channel(); + ( + ControlCommand::BrowserScreenshot { + surface, + out_path, + reply, + }, + rx, + ) + } + "browser.eval" => { + let surface = match required_string(params, &["surface_id", "id"], "surface_id") { + Ok(value) => normalize_handle(value, "surface:"), + Err(error) => return error_response(id, error), + }; + let script = match required_string(params, &["script"], "script") { + Ok(value) => value, + Err(error) => return error_response(id, error), + }; + let (reply, rx) = mpsc::channel(); + ( + ControlCommand::BrowserEval { + surface, + script, + wrap_key: Some("value".to_string()), + reply, + }, + rx, + ) + } + m if m.starts_with("browser.") => { + let surface = match required_string(params, &["surface_id", "id"], "surface_id") { + Ok(value) => normalize_handle(value, "surface:"), + Err(error) => return error_response(id, error), + }; + let (script, wrap_key) = match build_browser_script(method, params) { + Ok(pair) => pair, + Err(error) => return error_response(id, error), + }; + let (reply, rx) = mpsc::channel(); + ( + ControlCommand::BrowserEval { + surface, + script, + wrap_key, + reply, + }, + rx, + ) + } _ => { return error_response( id, @@ -332,20 +618,652 @@ fn handle_method( }; let (command, reply_rx) = queued; + let timeout = command_timeout(&command); dispatch(command); - match reply_rx.recv_timeout(Duration::from_secs(5)) { + match reply_rx.recv_timeout(timeout) { Ok(Ok(result)) => V2Response::success(id, result), Ok(Err(error)) => error_response(id, error), Err(_) => error_response(id, BridgeError::internal("control command timed out")), } } +fn command_timeout(command: &ControlCommand) -> Duration { + match command { + ControlCommand::BrowserEval { .. } + | ControlCommand::BrowserScreenshot { .. } + | ControlCommand::BrowserNavigate { .. } + | ControlCommand::BrowserOpenSplit { .. } => Duration::from_secs(30), + _ => Duration::from_secs(5), + } +} + fn error_response(id: Option, error: BridgeError) -> V2Response { V2Response::error(id, error.code, error.message, error.data) } +fn js_literal(value: &str) -> String { + serde_json::Value::String(value.to_string()).to_string() +} + +fn json_literal(value: &Value) -> String { + serde_json::to_string(value).unwrap_or_else(|_| "null".to_string()) +} + +/// JS that resolves a ref-or-selector to a DOM element. When the caller +/// provides a ref (`eN`) the script also verifies attachment and rejects +/// with structured REF_NOT_FOUND / STALE_REF errors rather than silently +/// acting on a stale handle. +fn resolve_target_js(handle_literal: &str) -> String { + format!( + r#"(() => {{ + const handle = {h}; + if (!handle) {{ return {{ err: {{ code: "INVALID_PARAMS", message: "selector or ref required" }} }}; }} + const trimmed = String(handle).replace(/^@/, ""); + const refMatch = /^e\d+$/.test(trimmed); + if (refMatch) {{ + if (!window.__limux) {{ + return {{ err: {{ code: "INIT_NOT_READY", message: "limux init script not installed" }} }}; + }} + const info = window.__limux.refInfo(trimmed); + if (!info) {{ return {{ err: {{ code: "REF_NOT_FOUND", message: "ref " + trimmed + " unknown", ref: trimmed }} }}; }} + if (!info.attached) {{ return {{ err: {{ code: "REF_NOT_FOUND", message: "ref " + trimmed + " no longer attached", ref: trimmed }} }}; }} + const el = window.__limux.lookupRef(trimmed); + if (!el) {{ return {{ err: {{ code: "REF_NOT_FOUND", message: "ref " + trimmed + " not in DOM", ref: trimmed }} }}; }} + return {{ el, ref: trimmed, info }}; + }} + try {{ + const el = document.querySelector(String(handle)); + if (!el) {{ return {{ err: {{ code: "REF_NOT_FOUND", message: "selector matched no element", selector: String(handle) }} }}; }} + return {{ el, selector: String(handle) }}; + }} catch (e) {{ + return {{ err: {{ code: "INVALID_SELECTOR", message: String(e && e.message || e), selector: String(handle) }} }}; + }} + }})()"#, + h = handle_literal + ) +} + +/// Wrap a JS action so REF_NOT_FOUND / INVALID_SELECTOR / errors are +/// returned as structured JSON (handler unwraps `err` into BridgeError). +fn wrap_action_js(handle_literal: &str, body: &str) -> String { + format!( + r#"(() => {{ + const target = {resolve}; + if (target.err) {{ return JSON.stringify({{ ok: false, error: target.err }}); }} + const el = target.el; + try {{ + {body} + }} catch (e) {{ + return JSON.stringify({{ ok: false, error: {{ code: "INTERNAL", message: String(e && e.message || e) }} }}); + }} + }})()"#, + resolve = resolve_target_js(handle_literal), + body = body, + ) +} + +/// Build JS payloads for browser.* methods. All scripts must return a +/// JSON string; the handler parses it and either merges it into the +/// response (when an object + `wrap_key` is None) or stores it under +/// `wrap_key`. +fn build_browser_script( + method: &str, + params: &Map, +) -> Result<(String, Option), BridgeError> { + match method { + "browser.snapshot" => { + let opts = snapshot_opts(params); + let script = crate::pane::LIMUX_BROWSER_SNAPSHOT_SCRIPT + .replace("__LIMUX_SNAPSHOT_OPTS__", &opts); + Ok((script, None)) + } + "browser.click" => Ok((browser_action_click(params)?, None)), + "browser.dblclick" => Ok((browser_action_dblclick(params)?, None)), + "browser.hover" => Ok((browser_action_hover(params)?, None)), + "browser.focus" => Ok((browser_action_focus(params)?, None)), + "browser.fill" => Ok((browser_action_fill(params)?, None)), + "browser.type" => Ok((browser_action_type(params)?, None)), + "browser.press" => Ok((browser_action_press(params)?, None)), + "browser.check" => Ok((browser_action_toggle_checkable(params, true)?, None)), + "browser.uncheck" => Ok((browser_action_toggle_checkable(params, false)?, None)), + "browser.select" => Ok((browser_action_select(params)?, None)), + "browser.scroll" => Ok((browser_action_scroll(params)?, None)), + "browser.scroll_into_view" => Ok((browser_action_scroll_into_view(params)?, None)), + "browser.wait" => Ok((browser_action_wait(params)?, None)), + "browser.wait_ready" => Ok((browser_action_wait_ready(params)?, None)), + "browser.get.text" => Ok((browser_get_text(params)?, None)), + "browser.get.title" => Ok(( + "JSON.stringify({ title: document.title })".to_string(), + None, + )), + "browser.get.html" => Ok((browser_get_html(params)?, None)), + "browser.get.value" => Ok((browser_get_value(params)?, None)), + "browser.get.attr" => Ok((browser_get_attr(params)?, None)), + "browser.get.count" => Ok((browser_get_count(params)?, None)), + "browser.get.box" => Ok((browser_get_box(params)?, None)), + "browser.find.role" => Ok((browser_find(params, "role")?, None)), + "browser.find.text" => Ok((browser_find(params, "text")?, None)), + "browser.find.label" => Ok((browser_find(params, "label")?, None)), + "browser.find.placeholder" => Ok((browser_find(params, "placeholder")?, None)), + "browser.find.testid" => Ok((browser_find(params, "testid")?, None)), + "browser.console.list" => Ok((browser_console_list(params)?, None)), + "browser.console.clear" => Ok(( + r#"(() => { if (window.__limux) window.__limux.clearLogs(); return JSON.stringify({ ok: true }); })()"#.to_string(), + None, + )), + "browser.errors.list" => Ok((browser_errors_list(params)?, None)), + "browser.errors.clear" => Ok(( + r#"(() => { if (window.__limux) window.__limux.clearErrors(); return JSON.stringify({ ok: true }); })()"#.to_string(), + None, + )), + "browser.is_ready" => Ok(( + r#"JSON.stringify({ ready: !!(window.__limux && window.__limux.isReady()) })"#.to_string(), + None, + )), + "browser.is_editable" => Ok(( + r#"JSON.stringify({ editable: !!(window.__limux && window.__limux.isEditable()) })"#.to_string(), + None, + )), + _ => Err(BridgeError::invalid_params(format!( + "no browser script for {method}" + ))), + } +} + +fn snapshot_opts(params: &Map) -> String { + let mut obj = serde_json::Map::new(); + if let Some(v) = params.get("full_tree") { + obj.insert("full_tree".into(), v.clone()); + } + if let Some(v) = params.get("raw_html") { + obj.insert("raw_html".into(), v.clone()); + } + if let Some(v) = params.get("selector") { + obj.insert("selector".into(), v.clone()); + } + if let Some(v) = params.get("max_depth") { + obj.insert("max_depth".into(), v.clone()); + } + if let Some(v) = params.get("since_hash") { + obj.insert("since_hash".into(), v.clone()); + } + serde_json::to_string(&Value::Object(obj)).unwrap_or_else(|_| "{}".to_string()) +} + +fn target_handle(params: &Map) -> Result { + if let Some(s) = optional_string(params, &["ref", "selector", "target"]) { + return Ok(s); + } + Err(BridgeError::invalid_params("ref or selector required")) +} + +fn browser_action_click(params: &Map) -> Result { + let handle = target_handle(params)?; + let body = r#" + if (typeof el.click === "function") { el.click(); } + else { el.dispatchEvent(new MouseEvent("click", { bubbles: true, cancelable: true })); } + return JSON.stringify({ ok: true, ref: target.ref, selector: target.selector }); + "#; + Ok(wrap_action_js(&js_literal(&handle), body)) +} + +fn browser_action_dblclick(params: &Map) -> Result { + let handle = target_handle(params)?; + let body = r#" + el.dispatchEvent(new MouseEvent("dblclick", { bubbles: true, cancelable: true })); + return JSON.stringify({ ok: true, ref: target.ref, selector: target.selector }); + "#; + Ok(wrap_action_js(&js_literal(&handle), body)) +} + +fn browser_action_hover(params: &Map) -> Result { + let handle = target_handle(params)?; + let body = r#" + el.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + el.dispatchEvent(new MouseEvent("mouseenter", { bubbles: false })); + return JSON.stringify({ ok: true, ref: target.ref, selector: target.selector }); + "#; + Ok(wrap_action_js(&js_literal(&handle), body)) +} + +fn browser_action_focus(params: &Map) -> Result { + let handle = target_handle(params)?; + let body = r#" + if (typeof el.focus === "function") el.focus(); + return JSON.stringify({ ok: true, ref: target.ref, selector: target.selector }); + "#; + Ok(wrap_action_js(&js_literal(&handle), body)) +} + +fn browser_action_fill(params: &Map) -> Result { + let handle = target_handle(params)?; + let text = optional_string(params, &["text", "value"]).unwrap_or_default(); + let body = format!( + r#" + const text = {text}; + if (el.isContentEditable) {{ + el.focus(); + el.textContent = text; + el.dispatchEvent(new Event("input", {{ bubbles: true }})); + }} else if (el.tagName === "TEXTAREA" || el.tagName === "INPUT" || el.tagName === "SELECT") {{ + const proto = el.tagName === "TEXTAREA" ? HTMLTextAreaElement.prototype + : el.tagName === "SELECT" ? HTMLSelectElement.prototype + : HTMLInputElement.prototype; + const setter = Object.getOwnPropertyDescriptor(proto, "value").set; + setter.call(el, text); + el.dispatchEvent(new Event("input", {{ bubbles: true }})); + el.dispatchEvent(new Event("change", {{ bubbles: true }})); + }} else {{ + return JSON.stringify({{ ok: false, error: {{ code: "WRONG_ELEMENT", message: "cannot fill tag " + el.tagName }} }}); + }} + return JSON.stringify({{ ok: true, ref: target.ref, selector: target.selector }}); + "#, + text = js_literal(&text) + ); + Ok(wrap_action_js(&js_literal(&handle), &body)) +} + +fn browser_action_type(params: &Map) -> Result { + // Dispatch keydown/keypress/keyup + input events for each char on the focused element. + let text = required_string(params, &["text"], "text")?; + let script = format!( + r#"(() => {{ + const text = {text}; + const el = document.activeElement; + if (!el) {{ return JSON.stringify({{ ok: false, error: {{ code: "NO_FOCUS", message: "no focused element" }} }}); }} + for (const ch of text) {{ + el.dispatchEvent(new KeyboardEvent("keydown", {{ key: ch, bubbles: true }})); + el.dispatchEvent(new KeyboardEvent("keypress", {{ key: ch, bubbles: true }})); + if (el.tagName === "INPUT" || el.tagName === "TEXTAREA") {{ + const proto = el.tagName === "TEXTAREA" ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype; + const setter = Object.getOwnPropertyDescriptor(proto, "value").set; + setter.call(el, (el.value || "") + ch); + el.dispatchEvent(new Event("input", {{ bubbles: true }})); + }} else if (el.isContentEditable) {{ + el.textContent = (el.textContent || "") + ch; + el.dispatchEvent(new Event("input", {{ bubbles: true }})); + }} + el.dispatchEvent(new KeyboardEvent("keyup", {{ key: ch, bubbles: true }})); + }} + return JSON.stringify({{ ok: true }}); + }})()"#, + text = js_literal(&text) + ); + Ok(script) +} + +fn browser_action_press(params: &Map) -> Result { + // keys = "Enter", "Ctrl+K", etc. Parse into key + modifier flags. + let keys = required_string(params, &["keys", "key"], "keys")?; + let script = format!( + r#"(() => {{ + const raw = {keys}; + const parts = String(raw).split("+").map(s => s.trim()); + const key = parts.pop(); + const mods = new Set(parts.map(s => s.toLowerCase())); + const el = document.activeElement || document.body; + const init = {{ + key, + bubbles: true, + cancelable: true, + ctrlKey: mods.has("ctrl") || mods.has("control"), + altKey: mods.has("alt"), + shiftKey: mods.has("shift"), + metaKey: mods.has("meta") || mods.has("cmd") || mods.has("super"), + }}; + el.dispatchEvent(new KeyboardEvent("keydown", init)); + el.dispatchEvent(new KeyboardEvent("keypress", init)); + el.dispatchEvent(new KeyboardEvent("keyup", init)); + return JSON.stringify({{ ok: true, key, mods: [...mods] }}); + }})()"#, + keys = js_literal(&keys) + ); + Ok(script) +} + +fn browser_action_toggle_checkable( + params: &Map, + desired: bool, +) -> Result { + let handle = target_handle(params)?; + let body = format!( + r#" + if (el.type !== "checkbox" && el.type !== "radio" && el.getAttribute("role") !== "checkbox" && el.getAttribute("role") !== "switch") {{ + return JSON.stringify({{ ok: false, error: {{ code: "WRONG_ELEMENT", message: "not a checkable element" }} }}); + }} + const want = {desired}; + if (el.type === "radio") {{ + if (want) {{ if (!el.checked) el.click(); }} + else {{ return JSON.stringify({{ ok: false, error: {{ code: "INVALID_OP", message: "cannot uncheck a radio" }} }}); }} + }} else if (el.checked !== want) {{ + el.click(); + }} + return JSON.stringify({{ ok: true, ref: target.ref, selector: target.selector, checked: !!el.checked }}); + "#, + desired = desired + ); + Ok(wrap_action_js(&js_literal(&handle), &body)) +} + +fn browser_action_select(params: &Map) -> Result { + let handle = target_handle(params)?; + let option = required_string(params, &["option", "value", "label"], "option")?; + let body = format!( + r#" + if (el.tagName !== "SELECT") {{ return JSON.stringify({{ ok: false, error: {{ code: "WRONG_ELEMENT", message: "not a