From c83a7a726667dc62d03c0c77ecc9c28d4c98a28b Mon Sep 17 00:00:00 2001 From: "MVB.Mir" Date: Fri, 17 Apr 2026 23:58:55 +0300 Subject: [PATCH 1/3] control: wire pane/surface/browser navigation methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously the control-socket METHODS array only exposed workspace.* and surface.send_text. Every pane.*, surface.*, and browser.* call returned -32601 unknown method. Same gap on upstream am-will/main. This commit wires the first tier of methods end-to-end against live GTK state, no JS eval: pane.list, pane.surfaces, surface.list, surface.current browser.open_split, browser.navigate, browser.url.get browser.back, browser.forward, browser.reload browser.screenshot, browser.eval IDs and refs: pane ids are u32 as string, surface ids are UUID strings, refs are "pane:N" / "surface:UUID". Inputs accept raw ids or prefixed refs via normalize_handle. Outputs always include both id and ref so callers never have to reconstruct either form. browser.open_split defaults to hosting the new browser in a pane other than the focused (caller) pane. If only one pane exists it splits the focused pane horizontally and places the browser in the new split, so a caller never clobbers its own terminal. The source pane can be overridden explicitly via source_surface. browser.screenshot uses webkit6::WebView::snapshot with SnapshotRegion:: Visible → gdk_texture_save_to_png, rejecting empty textures instead of writing a zero-byte file. The CLI now forwards --out as the `path` parameter so the server writes directly to the caller's target rather than leaking a random /tmp path. Async ops (navigate/open_split/screenshot/eval) use mpsc reply channels with a 30s timeout; sync ops default to 5s. --- rust/limux-cli/src/main.rs | 30 +- rust/limux-host-linux/src/control_bridge.rs | 248 ++++++++++- rust/limux-host-linux/src/pane.rs | 232 ++++++++++ rust/limux-host-linux/src/window.rs | 467 ++++++++++++++++++++ 4 files changed, 961 insertions(+), 16 deletions(-) diff --git a/rust/limux-cli/src/main.rs b/rust/limux-cli/src/main.rs index f5674ca4..c73d9333 100644 --- a/rust/limux-cli/src/main.rs +++ b/rust/limux-cli/src/main.rs @@ -1099,23 +1099,25 @@ async fn run_browser( let sid = surface .clone() .ok_or_else(|| anyhow!("browser screenshot requires a surface"))?; - let mut payload = - browser_call(client, Some(sid), "browser.screenshot", Map::new()).await?; let out = parse_opt(&browser_args, "--out"); - let mut path = get_string(&payload, &["path"]) - .unwrap_or_else(|| "/tmp/limux-browser-shot.png".to_string()); - if let Some(out_path) = out { - path = out_path; - } - if !Path::new(&path).exists() { - if let Some(parent) = Path::new(&path).parent() { - fs::create_dir_all(parent).with_context(|| { - format!("failed to create screenshot directory {}", parent.display()) - })?; + let mut params = Map::new(); + if let Some(ref out_path) = out { + if let Some(parent) = Path::new(out_path).parent() { + if !parent.as_os_str().is_empty() { + fs::create_dir_all(parent).with_context(|| { + format!( + "failed to create screenshot directory {}", + parent.display() + ) + })?; + } } - fs::write(&path, []) - .with_context(|| format!("failed to create screenshot {}", path))?; + params.insert("path".to_string(), Value::String(out_path.clone())); } + let mut payload = browser_call(client, Some(sid), "browser.screenshot", params).await?; + let path = get_string(&payload, &["path"]) + .or(out) + .unwrap_or_else(|| "/tmp/limux-browser-shot.png".to_string()); let url = format!("file://{}", path); if let Some(obj) = payload.as_object_mut() { obj.insert("path".to_string(), Value::String(path.clone())); diff --git a/rust/limux-host-linux/src/control_bridge.rs b/rust/limux-host-linux/src/control_bridge.rs index 38dc5b95..43cf56e5 100644 --- a/rust/limux-host-linux/src/control_bridge.rs +++ b/rust/limux-host-linux/src/control_bridge.rs @@ -26,6 +26,18 @@ const METHODS: &[&str] = &[ "workspace.rename", "workspace.close", "surface.send_text", + "pane.list", + "pane.surfaces", + "surface.list", + "surface.current", + "browser.open_split", + "browser.navigate", + "browser.url.get", + "browser.back", + "browser.forward", + "browser.reload", + "browser.screenshot", + "browser.eval", ]; const PARSE_ERROR_CODE: i64 = -32700; @@ -82,6 +94,60 @@ pub enum ControlCommand { text: String, reply: mpsc::Sender, }, + ListPanes { + target: WorkspaceTarget, + reply: mpsc::Sender, + }, + ListSurfaces { + target: WorkspaceTarget, + pane_filter: Option, + reply: mpsc::Sender, + }, + CurrentSurface { + target: WorkspaceTarget, + reply: mpsc::Sender, + }, + BrowserOpenSplit { + target: WorkspaceTarget, + source_surface: Option, + url: Option, + reply: mpsc::Sender, + }, + BrowserNavigate { + surface: String, + url: String, + reply: mpsc::Sender, + }, + BrowserGetUrl { + surface: String, + reply: mpsc::Sender, + }, + BrowserBack { + surface: String, + reply: mpsc::Sender, + }, + BrowserForward { + surface: String, + reply: mpsc::Sender, + }, + BrowserReload { + surface: String, + reply: mpsc::Sender, + }, + BrowserScreenshot { + surface: String, + out_path: Option, + reply: mpsc::Sender, + }, + BrowserEval { + surface: String, + script: String, + /// If Some, the handler parses the JS reply as JSON and wraps it under + /// this key in the response. If None, the reply is returned verbatim + /// as a string under the "result" key. + wrap_key: Option, + reply: mpsc::Sender, + }, } impl ControlCommand { @@ -94,7 +160,18 @@ impl ControlCommand { | Self::SelectWorkspace { reply, .. } | Self::RenameWorkspace { reply, .. } | Self::CloseWorkspace { reply, .. } - | Self::SendText { reply, .. } => { + | Self::SendText { reply, .. } + | Self::ListPanes { reply, .. } + | Self::ListSurfaces { reply, .. } + | Self::CurrentSurface { reply, .. } + | Self::BrowserOpenSplit { reply, .. } + | Self::BrowserNavigate { reply, .. } + | Self::BrowserGetUrl { reply, .. } + | Self::BrowserBack { reply, .. } + | Self::BrowserForward { reply, .. } + | Self::BrowserReload { reply, .. } + | Self::BrowserScreenshot { reply, .. } + | Self::BrowserEval { reply, .. } => { let _ = reply.send(result); } } @@ -185,6 +262,23 @@ fn optional_index(params: &Map, key: &str) -> Result, + keys: &[&str], + label: &str, +) -> Result { + optional_string(params, keys) + .ok_or_else(|| BridgeError::invalid_params(format!("{label} is required"))) +} + +/// Strip a leading `prefix:` (e.g. `surface:UUID` → `UUID`) so callers can pass +/// either raw IDs or refs. +fn normalize_handle(raw: String, prefix: &str) -> String { + raw.strip_prefix(prefix) + .map(|rest| rest.to_string()) + .unwrap_or(raw) +} + fn parse_optional_workspace_target( params: &Map, allow_name: bool, @@ -323,6 +417,145 @@ fn handle_method( rx, ) } + "pane.list" | "list-panes" | "list-panels" => { + let target = match parse_optional_workspace_target(params, false) { + Ok(target) => target, + Err(error) => return error_response(id, error), + }; + let (reply, rx) = mpsc::channel(); + (ControlCommand::ListPanes { target, reply }, rx) + } + "pane.surfaces" | "surface.list" => { + let target = match parse_optional_workspace_target(params, false) { + Ok(target) => target, + Err(error) => return error_response(id, error), + }; + let pane_filter = optional_string(params, &["pane_id", "id"]) + .map(|raw| normalize_handle(raw, "pane:")); + let (reply, rx) = mpsc::channel(); + ( + ControlCommand::ListSurfaces { + target, + pane_filter, + reply, + }, + rx, + ) + } + "surface.current" => { + let target = match parse_optional_workspace_target(params, false) { + Ok(target) => target, + Err(error) => return error_response(id, error), + }; + let (reply, rx) = mpsc::channel(); + (ControlCommand::CurrentSurface { target, reply }, rx) + } + "browser.open_split" | "browser.open" | "browser.new" => { + let target = match parse_optional_workspace_target(params, false) { + Ok(target) => target, + Err(error) => return error_response(id, error), + }; + let source_surface = optional_string(params, &["surface_id", "id"]) + .map(|raw| normalize_handle(raw, "surface:")); + let url = optional_string(params, &["url"]); + let (reply, rx) = mpsc::channel(); + ( + ControlCommand::BrowserOpenSplit { + target, + source_surface, + url, + reply, + }, + rx, + ) + } + "browser.navigate" | "browser.goto" => { + let surface = match required_string(params, &["surface_id", "id"], "surface_id") { + Ok(value) => normalize_handle(value, "surface:"), + Err(error) => return error_response(id, error), + }; + let url = match required_string(params, &["url"], "url") { + Ok(value) => value, + Err(error) => return error_response(id, error), + }; + let (reply, rx) = mpsc::channel(); + ( + ControlCommand::BrowserNavigate { + surface, + url, + reply, + }, + rx, + ) + } + "browser.url.get" | "browser.get.url" => { + let surface = match required_string(params, &["surface_id", "id"], "surface_id") { + Ok(value) => normalize_handle(value, "surface:"), + Err(error) => return error_response(id, error), + }; + let (reply, rx) = mpsc::channel(); + (ControlCommand::BrowserGetUrl { surface, reply }, rx) + } + "browser.back" => { + let surface = match required_string(params, &["surface_id", "id"], "surface_id") { + Ok(value) => normalize_handle(value, "surface:"), + Err(error) => return error_response(id, error), + }; + let (reply, rx) = mpsc::channel(); + (ControlCommand::BrowserBack { surface, reply }, rx) + } + "browser.forward" => { + let surface = match required_string(params, &["surface_id", "id"], "surface_id") { + Ok(value) => normalize_handle(value, "surface:"), + Err(error) => return error_response(id, error), + }; + let (reply, rx) = mpsc::channel(); + (ControlCommand::BrowserForward { surface, reply }, rx) + } + "browser.reload" => { + let surface = match required_string(params, &["surface_id", "id"], "surface_id") { + Ok(value) => normalize_handle(value, "surface:"), + Err(error) => return error_response(id, error), + }; + let (reply, rx) = mpsc::channel(); + (ControlCommand::BrowserReload { surface, reply }, rx) + } + "browser.screenshot" => { + let surface = match required_string(params, &["surface_id", "id"], "surface_id") { + Ok(value) => normalize_handle(value, "surface:"), + Err(error) => return error_response(id, error), + }; + let out_path = optional_string(params, &["out", "path"]); + let (reply, rx) = mpsc::channel(); + ( + ControlCommand::BrowserScreenshot { + surface, + out_path, + reply, + }, + rx, + ) + } + "browser.eval" => { + let surface = match required_string(params, &["surface_id", "id"], "surface_id") { + Ok(value) => normalize_handle(value, "surface:"), + Err(error) => return error_response(id, error), + }; + let script = match required_string(params, &["script"], "script") { + Ok(value) => value, + Err(error) => return error_response(id, error), + }; + let (reply, rx) = mpsc::channel(); + ( + ControlCommand::BrowserEval { + surface, + script, + wrap_key: Some("value".to_string()), + reply, + }, + rx, + ) + } _ => { return error_response( id, @@ -332,16 +565,27 @@ fn handle_method( }; let (command, reply_rx) = queued; + let timeout = command_timeout(&command); dispatch(command); - match reply_rx.recv_timeout(Duration::from_secs(5)) { + match reply_rx.recv_timeout(timeout) { Ok(Ok(result)) => V2Response::success(id, result), Ok(Err(error)) => error_response(id, error), Err(_) => error_response(id, BridgeError::internal("control command timed out")), } } +fn command_timeout(command: &ControlCommand) -> Duration { + match command { + ControlCommand::BrowserEval { .. } + | ControlCommand::BrowserScreenshot { .. } + | ControlCommand::BrowserNavigate { .. } + | ControlCommand::BrowserOpenSplit { .. } => Duration::from_secs(30), + _ => Duration::from_secs(5), + } +} + fn error_response(id: Option, error: BridgeError) -> V2Response { V2Response::error(id, error.code, error.message, error.data) } diff --git a/rust/limux-host-linux/src/pane.rs b/rust/limux-host-linux/src/pane.rs index 45d83fd9..a97ecc4b 100644 --- a/rust/limux-host-linux/src/pane.rs +++ b/rust/limux-host-linux/src/pane.rs @@ -1448,6 +1448,133 @@ pub fn tab_working_directory(pane_widget: >k::Widget, tab_id: &str) -> Option< } } +// --------------------------------------------------------------------------- +// Control-socket introspection helpers +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone)] +pub struct PaneSnapshotInfo { + pub pane_id: u32, + pub surface_count: usize, + pub active_surface_id: Option, + pub surfaces: Vec, +} + +#[derive(Debug, Clone)] +pub struct SurfaceSnapshotInfo { + pub id: String, + pub title: String, + pub kind: SurfaceSnapshotKind, + pub pinned: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SurfaceSnapshotKind { + Terminal, + Browser, + Keybinds, +} + +/// Return a snapshot of the given pane for the control socket to serialize. +pub fn pane_snapshot_info(pane_widget: >k::Widget) -> Option { + let internals = find_pane_internals(pane_widget)?; + let tab_state = internals.tab_state.borrow(); + Some(PaneSnapshotInfo { + pane_id: internals.pane_id, + surface_count: tab_state.tabs.len(), + active_surface_id: tab_state.active_tab.clone(), + surfaces: tab_state + .tabs + .iter() + .map(|entry| SurfaceSnapshotInfo { + id: entry.id.clone(), + title: entry.title_label.label().to_string(), + kind: match &entry.kind { + TabKind::Terminal { .. } => SurfaceSnapshotKind::Terminal, + TabKind::Browser { .. } => SurfaceSnapshotKind::Browser, + TabKind::Keybinds => SurfaceSnapshotKind::Keybinds, + }, + pinned: entry.pinned, + }) + .collect(), + }) +} + +/// Walk every pane reachable from `root`, invoking `visit` for each. +pub fn walk_panes(root: >k::Widget, mut visit: impl FnMut(>k::Widget)) { + walk_panes_inner(root, &mut visit); +} + +fn walk_panes_inner(widget: >k::Widget, visit: &mut dyn FnMut(>k::Widget)) { + if is_pane_widget(widget) { + visit(widget); + return; + } + let mut child = widget.first_child(); + while let Some(current) = child { + walk_panes_inner(¤t, visit); + child = current.next_sibling(); + } +} + +/// Create a browser tab in `pane_widget` and return the new surface id. +pub fn add_browser_tab_returning_id( + pane_widget: >k::Widget, + uri: Option<&str>, +) -> Option { + let internals = find_pane_internals(pane_widget)?; + let options = uri.map(|uri| BrowserTabOptions { + id: None, + custom_name: None, + pinned: false, + uri: Some(uri), + }); + add_browser_tab_inner(&internals, options); + let active = internals.tab_state.borrow().active_tab.clone(); + active +} + +/// Resolve a `BrowserShortcutTarget` for the given surface id within the pane. +pub fn browser_target_for_surface( + pane_widget: >k::Widget, + surface_id: &str, +) -> Option { + let internals = find_pane_internals(pane_widget)?; + let tab_state = internals.tab_state.borrow(); + let entry = tab_state.tabs.iter().find(|entry| entry.id == surface_id)?; + match &entry.kind { + TabKind::Browser { state } => Some(BrowserShortcutTarget { + uri: state.uri.clone(), + handles: state.handles.clone(), + }), + _ => None, + } +} + +/// Find the pane widget that contains the given surface id by scanning the +/// global pane registry. +pub fn find_pane_widget_for_surface(surface_id: &str) -> Option { + PANE_REGISTRY.with(|registry| { + for weak in registry.borrow().values() { + let Some(internals) = weak.upgrade() else { + continue; + }; + let tab_state = internals.tab_state.borrow(); + if tab_state.tabs.iter().any(|entry| entry.id == surface_id) { + return Some(internals.pane_outer.clone().upcast()); + } + } + None + }) +} + +/// Resolve a `BrowserShortcutTarget` for the given surface id anywhere in the +/// application (scans all panes). +pub fn find_browser_target(surface_id: &str) -> Option { + let pane_widget = find_pane_widget_for_surface(surface_id)?; + browser_target_for_surface(&pane_widget, surface_id) +} + pub fn move_tab_to_pane( source_pane: >k::Widget, tab_id: &str, @@ -2483,6 +2610,111 @@ impl BrowserShortcutTarget { pub fn is_page_editable(&self) -> bool { self.handles.is_page_editable() } + + /// Load `url` in the underlying WebView (no-op if webkit is disabled). + pub fn load_uri(&self, url: &str) { + #[cfg(feature = "webkit")] + { + self.handles.webview.load_uri(url); + *self.uri.borrow_mut() = Some(url.to_string()); + } + #[cfg(not(feature = "webkit"))] + { + let _ = url; + } + } + + /// Current URI reported by the WebView (may differ from `current_uri` + /// which tracks the last URL loaded/navigated-to). + pub fn webview_uri(&self) -> Option { + #[cfg(feature = "webkit")] + { + self.handles + .webview + .uri() + .map(|gstr| gstr.to_string()) + .or_else(|| self.uri.borrow().clone()) + } + #[cfg(not(feature = "webkit"))] + { + self.uri.borrow().clone() + } + } + + /// Run JavaScript in the WebView and deliver the string-form result to + /// `callback`. When webkit is disabled the callback is invoked immediately + /// with an error. + pub fn evaluate_js( + &self, + script: String, + callback: Box) + 'static>, + ) { + #[cfg(feature = "webkit")] + { + let mut cb = Some(callback); + self.handles.webview.evaluate_javascript( + &script, + None, + None, + None::<>k::gio::Cancellable>, + move |result| { + if let Some(cb) = cb.take() { + match result { + Ok(value) => cb(Ok(value.to_str().to_string())), + Err(error) => cb(Err(error.to_string())), + } + } + }, + ); + } + #[cfg(not(feature = "webkit"))] + { + let _ = script; + callback(Err("webkit feature disabled".to_string())); + } + } + + /// Capture a PNG snapshot of the WebView's visible region and write it to + /// `out_path`, then invoke `callback` with the result. + pub fn snapshot_png( + &self, + out_path: std::path::PathBuf, + callback: Box) + 'static>, + ) { + #[cfg(feature = "webkit")] + { + let mut cb = Some(callback); + self.handles.webview.snapshot( + webkit6::SnapshotRegion::Visible, + webkit6::SnapshotOptions::empty(), + None::<>k::gio::Cancellable>, + move |result| { + let Some(cb) = cb.take() else { return }; + match result { + Ok(texture) => { + if texture.width() == 0 || texture.height() == 0 { + cb(Err("snapshot texture empty; webview not yet rendered" + .to_string())); + return; + } + match texture.save_to_png(&out_path) { + Ok(()) => cb(Ok(out_path)), + Err(error) => { + cb(Err(format!("snapshot save failed: {error}"))) + } + } + } + Err(error) => cb(Err(error.to_string())), + } + }, + ); + } + #[cfg(not(feature = "webkit"))] + { + let _ = out_path; + callback(Err("webkit feature disabled".to_string())); + } + } } #[cfg(feature = "webkit")] diff --git a/rust/limux-host-linux/src/window.rs b/rust/limux-host-linux/src/window.rs index 6a78a6e1..8c85d7ee 100644 --- a/rust/limux-host-linux/src/window.rs +++ b/rust/limux-host-linux/src/window.rs @@ -100,12 +100,80 @@ fn surface_ref(id: &str) -> String { format!("surface:{id}") } +fn pane_ref(pane_id: u32) -> String { + format!("pane:{pane_id}") +} + fn normalize_workspace_handle(raw: &str) -> &str { raw.trim() .strip_prefix("workspace:") .unwrap_or_else(|| raw.trim()) } +fn parse_pane_id_input(raw: &str) -> Option { + raw.trim() + .strip_prefix("pane:") + .unwrap_or_else(|| raw.trim()) + .parse::() + .ok() +} + +fn surface_kind_label(kind: pane::SurfaceSnapshotKind) -> &'static str { + match kind { + pane::SurfaceSnapshotKind::Terminal => "terminal", + pane::SurfaceSnapshotKind::Browser => "browser", + pane::SurfaceSnapshotKind::Keybinds => "keybinds", + } +} + +fn encode_pane_row(workspace_id: &str, snapshot: &pane::PaneSnapshotInfo) -> serde_json::Value { + serde_json::json!({ + "workspace_id": workspace_id, + "workspace_ref": workspace_ref(workspace_id), + "id": snapshot.pane_id.to_string(), + "ref": pane_ref(snapshot.pane_id), + "pane_id": snapshot.pane_id.to_string(), + "pane_ref": pane_ref(snapshot.pane_id), + "surface_count": snapshot.surface_count, + "active_surface_id": snapshot.active_surface_id.as_deref(), + "active_surface_ref": snapshot.active_surface_id.as_deref().map(surface_ref), + }) +} + +fn encode_surface_row( + workspace_id: &str, + snapshot: &pane::PaneSnapshotInfo, + surface: &pane::SurfaceSnapshotInfo, +) -> serde_json::Value { + let active = snapshot.active_surface_id.as_deref() == Some(surface.id.as_str()); + serde_json::json!({ + "workspace_id": workspace_id, + "workspace_ref": workspace_ref(workspace_id), + "pane_id": snapshot.pane_id.to_string(), + "pane_ref": pane_ref(snapshot.pane_id), + "id": surface.id.as_str(), + "ref": surface_ref(&surface.id), + "surface_id": surface.id.as_str(), + "surface_ref": surface_ref(&surface.id), + "title": surface.title.as_str(), + "type": surface_kind_label(surface.kind), + "pinned": surface.pinned, + "selected": active, + "focused": active, + }) +} + +/// Walk the active workspace and collect pane snapshots with stable ordering. +fn collect_workspace_panes(workspace: &Workspace) -> Vec { + let mut rows = Vec::new(); + pane::walk_panes(&workspace.root, |pane_widget| { + if let Some(info) = pane::pane_snapshot_info(pane_widget) { + rows.push(info); + } + }); + rows +} + fn workspace_index_for_target(state: &AppState, target: &WorkspaceTarget) -> Option { match target { WorkspaceTarget::Active => (!state.workspaces.is_empty()).then_some(state.active_idx), @@ -3037,9 +3105,408 @@ fn handle_control_command(state: &State, command: ControlCommand) { } let _ = reply.send(Ok(payload)); } + ControlCommand::ListPanes { target, reply } => { + let result = list_panes_for_target(state, &target); + let _ = reply.send(result); + } + ControlCommand::ListSurfaces { + target, + pane_filter, + reply, + } => { + let result = list_surfaces_for_target(state, &target, pane_filter.as_deref()); + let _ = reply.send(result); + } + ControlCommand::CurrentSurface { target, reply } => { + let result = current_surface_for_target(state, &target); + let _ = reply.send(result); + } + ControlCommand::BrowserOpenSplit { + target, + source_surface, + url, + reply, + } => { + let result = browser_open_split(state, &target, source_surface.as_deref(), url); + let _ = reply.send(result); + } + ControlCommand::BrowserNavigate { + surface, + url, + reply, + } => { + let result = browser_navigate(&surface, &url); + let _ = reply.send(result); + } + ControlCommand::BrowserGetUrl { surface, reply } => { + let result = browser_get_url(&surface); + let _ = reply.send(result); + } + ControlCommand::BrowserBack { surface, reply } => { + let result = browser_history(&surface, BrowserHistoryAction::Back); + let _ = reply.send(result); + } + ControlCommand::BrowserForward { surface, reply } => { + let result = browser_history(&surface, BrowserHistoryAction::Forward); + let _ = reply.send(result); + } + ControlCommand::BrowserReload { surface, reply } => { + let result = browser_history(&surface, BrowserHistoryAction::Reload); + let _ = reply.send(result); + } + ControlCommand::BrowserScreenshot { + surface, + out_path, + reply, + } => { + browser_screenshot(&surface, out_path, reply); + } + ControlCommand::BrowserEval { + surface, + script, + wrap_key, + reply, + } => { + browser_eval(&surface, script, wrap_key, reply); + } } } +// --------------------------------------------------------------------------- +// Control-socket: pane/surface/browser helpers +// --------------------------------------------------------------------------- + +fn list_panes_for_target( + state: &State, + target: &WorkspaceTarget, +) -> Result { + let app_state = state.borrow(); + let Some(idx) = workspace_index_for_target(&app_state, target) else { + return Err(crate::control_bridge::BridgeError::not_found( + "workspace not found", + )); + }; + let workspace = &app_state.workspaces[idx]; + let workspace_id = workspace.id.clone(); + let panes = collect_workspace_panes(workspace); + let rows: Vec = panes + .iter() + .map(|snapshot| encode_pane_row(&workspace_id, snapshot)) + .collect(); + Ok(serde_json::json!({ "panes": rows })) +} + +fn list_surfaces_for_target( + state: &State, + target: &WorkspaceTarget, + pane_filter: Option<&str>, +) -> Result { + let app_state = state.borrow(); + let Some(idx) = workspace_index_for_target(&app_state, target) else { + return Err(crate::control_bridge::BridgeError::not_found( + "workspace not found", + )); + }; + let workspace = &app_state.workspaces[idx]; + let workspace_id = workspace.id.clone(); + let panes = collect_workspace_panes(workspace); + let filter_pane_id = pane_filter.and_then(parse_pane_id_input); + + let mut rows = Vec::new(); + for snapshot in &panes { + if let Some(pane_id) = filter_pane_id { + if snapshot.pane_id != pane_id { + continue; + } + } + for surface in &snapshot.surfaces { + rows.push(encode_surface_row(&workspace_id, snapshot, surface)); + } + } + + if filter_pane_id.is_some() && rows.is_empty() { + return Err(crate::control_bridge::BridgeError::not_found( + "pane not found", + )); + } + + Ok(serde_json::json!({ "surfaces": rows })) +} + +fn current_surface_for_target( + state: &State, + target: &WorkspaceTarget, +) -> Result { + let workspace_id = { + let app_state = state.borrow(); + let Some(idx) = workspace_index_for_target(&app_state, target) else { + return Err(crate::control_bridge::BridgeError::not_found( + "workspace not found", + )); + }; + app_state.workspaces[idx].id.clone() + }; + + let pane_widget = find_focused_pane(state) + .map(|(_id, widget)| widget) + .ok_or_else(|| crate::control_bridge::BridgeError::not_found("no focused pane"))?; + let snapshot = pane::pane_snapshot_info(&pane_widget) + .ok_or_else(|| crate::control_bridge::BridgeError::internal("pane snapshot failed"))?; + let active = snapshot + .active_surface_id + .as_ref() + .and_then(|id| snapshot.surfaces.iter().find(|s| &s.id == id)) + .ok_or_else(|| crate::control_bridge::BridgeError::not_found("no active surface"))?; + Ok(encode_surface_row(&workspace_id, &snapshot, active)) +} + +fn browser_open_split( + state: &State, + target: &WorkspaceTarget, + source_surface: Option<&str>, + url: Option, +) -> Result { + let workspace_id = { + let app_state = state.borrow(); + let Some(idx) = workspace_index_for_target(&app_state, target) else { + return Err(crate::control_bridge::BridgeError::not_found( + "workspace not found", + )); + }; + app_state.workspaces[idx].id.clone() + }; + + // Default: host browser in a pane OTHER than the focused (caller) pane. + // If another pane exists, target the first non-focused pane. Otherwise + // split the focused pane and host the browser in the new pane. + let (pane_widget, created_split) = if let Some(sid) = source_surface { + let widget = pane::find_pane_widget_for_surface(sid).ok_or_else(|| { + crate::control_bridge::BridgeError::not_found("source surface not found") + })?; + (widget, false) + } else { + let focused = find_focused_pane(state).map(|(_, widget)| widget); + + let mut sibling: Option = None; + { + let app_state = state.borrow(); + if let Some(ws) = app_state.workspaces.iter().find(|w| w.id == workspace_id) { + pane::walk_panes(&ws.root, |pane_widget| { + if sibling.is_some() { + return; + } + match &focused { + Some(focused_widget) if focused_widget == pane_widget => {} + _ => sibling = Some(pane_widget.clone()), + } + }); + } + } + + if let Some(widget) = sibling { + (widget, false) + } else { + // Only one pane exists (or no focused pane to distinguish) — split it. + let focused_widget = focused.ok_or_else(|| { + crate::control_bridge::BridgeError::not_found("no pane to host browser") + })?; + let new_pane = split_pane( + state, + &workspace_id, + &focused_widget, + gtk::Orientation::Horizontal, + SplitPaneOptions { + initial_state: None, + skip_default_tab: true, + new_pane_first: false, + persist: true, + }, + ); + (new_pane, true) + } + }; + + let resolved_url = url.unwrap_or_else(|| "about:blank".to_string()); + let new_surface_id = pane::add_browser_tab_returning_id(&pane_widget, Some(&resolved_url)) + .ok_or_else(|| { + crate::control_bridge::BridgeError::internal("browser tab creation failed") + })?; + + let snapshot = pane::pane_snapshot_info(&pane_widget) + .ok_or_else(|| crate::control_bridge::BridgeError::internal("pane snapshot failed"))?; + let surface = snapshot + .surfaces + .iter() + .find(|s| s.id == new_surface_id) + .ok_or_else(|| { + crate::control_bridge::BridgeError::internal("new surface missing from snapshot") + })?; + + Ok(serde_json::json!({ + "surface_id": new_surface_id.as_str(), + "surface_ref": surface_ref(&new_surface_id), + "pane_id": snapshot.pane_id.to_string(), + "pane_ref": pane_ref(snapshot.pane_id), + "created_split": created_split, + "surface": encode_surface_row(&workspace_id, &snapshot, surface), + "browser": { + "open": true, + "url": resolved_url, + }, + })) +} + +fn browser_navigate( + surface: &str, + url: &str, +) -> Result { + let target = pane::find_browser_target(surface).ok_or_else(|| { + crate::control_bridge::BridgeError::not_found("browser surface not found") + })?; + target.load_uri(url); + Ok(serde_json::json!({ + "surface_id": surface, + "surface_ref": surface_ref(surface), + "url": url, + "ok": true, + })) +} + +fn browser_get_url( + surface: &str, +) -> Result { + let target = pane::find_browser_target(surface).ok_or_else(|| { + crate::control_bridge::BridgeError::not_found("browser surface not found") + })?; + let url = target.webview_uri(); + Ok(serde_json::json!({ + "surface_id": surface, + "surface_ref": surface_ref(surface), + "url": url, + })) +} + +enum BrowserHistoryAction { + Back, + Forward, + Reload, +} + +fn browser_history( + surface: &str, + action: BrowserHistoryAction, +) -> Result { + let target = pane::find_browser_target(surface).ok_or_else(|| { + crate::control_bridge::BridgeError::not_found("browser surface not found") + })?; + let ok = match action { + BrowserHistoryAction::Back => target.go_back(), + BrowserHistoryAction::Forward => target.go_forward(), + BrowserHistoryAction::Reload => target.reload(), + }; + Ok(serde_json::json!({ + "surface_id": surface, + "surface_ref": surface_ref(surface), + "ok": ok, + })) +} + +fn browser_screenshot( + surface: &str, + out_path: Option, + reply: std::sync::mpsc::Sender< + Result, + >, +) { + let Some(target) = pane::find_browser_target(surface) else { + let _ = reply.send(Err(crate::control_bridge::BridgeError::not_found( + "browser surface not found", + ))); + return; + }; + let path = out_path.map(std::path::PathBuf::from).unwrap_or_else(|| { + std::env::temp_dir().join(format!("limux-browser-shot-{}.png", uuid::Uuid::new_v4())) + }); + let surface_owned = surface.to_string(); + let path_clone = path.clone(); + target.snapshot_png( + path, + Box::new(move |result| match result { + Ok(written) => { + let _ = reply.send(Ok(serde_json::json!({ + "surface_id": surface_owned, + "surface_ref": surface_ref(&surface_owned), + "path": written.to_string_lossy(), + "ok": true, + }))); + } + Err(error) => { + let _ = path_clone; + let _ = reply.send(Err(crate::control_bridge::BridgeError::internal(error))); + } + }), + ); +} + +fn browser_eval( + surface: &str, + script: String, + wrap_key: Option, + reply: std::sync::mpsc::Sender< + Result, + >, +) { + let Some(target) = pane::find_browser_target(surface) else { + let _ = reply.send(Err(crate::control_bridge::BridgeError::not_found( + "browser surface not found", + ))); + return; + }; + let surface_owned = surface.to_string(); + target.evaluate_js( + script, + Box::new(move |result| match result { + Ok(raw) => { + let parsed: Option = serde_json::from_str(&raw).ok(); + let response = match (wrap_key, parsed) { + (Some(key), Some(value)) => serde_json::json!({ + "surface_id": surface_owned, + "surface_ref": surface_ref(&surface_owned), + key: value, + }), + (Some(key), None) => serde_json::json!({ + "surface_id": surface_owned, + "surface_ref": surface_ref(&surface_owned), + key: raw, + }), + (None, Some(mut value)) => { + if let Some(obj) = value.as_object_mut() { + obj.insert( + "surface_id".to_string(), + serde_json::Value::String(surface_owned.clone()), + ); + obj.insert( + "surface_ref".to_string(), + serde_json::Value::String(surface_ref(&surface_owned)), + ); + } + value + } + (None, None) => serde_json::json!({ + "surface_id": surface_owned, + "surface_ref": surface_ref(&surface_owned), + "result": raw, + }), + }; + let _ = reply.send(Ok(response)); + } + Err(error) => { + let _ = reply.send(Err(crate::control_bridge::BridgeError::internal(error))); + } + }), + ); +} + fn add_workspace_from_state(state: &State, workspace: &WorkspaceState) { let shortcuts = { let s = state.borrow(); From de08ccd4e8b602c61b0e6b12d4f2705c35169639 Mon Sep 17 00:00:00 2001 From: "MVB.Mir" Date: Sat, 18 Apr 2026 00:19:45 +0300 Subject: [PATCH 2/3] control: wire browser JS-eval methods (snapshot, actions, console, errors) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the JS-eval tier of the browser dispatcher on top of the pane/nav wiring from 1697f3c. No CLI changes — all methods reachable via limux-cli browser subcommands or raw control socket RPC. Init script (browser_init.js) is installed via UserContentManager at InjectionTime::Start on every top-frame navigation. It exposes window.__limux with: - Ref tagger + MutationObserver that keep `data-limux-ref="eN"` on every interactive node so refs survive DOM mutations within a page. - Ring buffers for console.{log,warn,error,info,debug} and for window.onerror + unhandledrejection, capped at 5000 entries each, with a monotonic seq counter the caller filters by (webkit6 clamps Date.now() / performance.timeOrigin to i32::MIN so wall-clock is unusable — performance.now() is the only working time source). - history.pushState / replaceState hook that fires a limux:navigation event + bumps a navCount for SPA handling. - isReady() probe (readyState complete AND DOM quiet ≥ 500ms). - isEditable() probe for active-element focus state. Snapshot walker (browser_snapshot.js) returns a token-efficient AX tree keyed on the init-script's refs. Output format: page title "" - banner - link "Home" [ref=e1] - navigation "Main" - link "Docs" [ref=e2] - main - heading "Sign in" [level=1] - form - textbox "Email" [ref=e4, required] - button "Sign in" [ref=e7] Each snapshot returns {url, title, hash, snapshot_text, refs, ...} with djb2 hash for future --since diff support. Scope flags (--selector, --max-depth, --full-tree, --raw-html) included. New methods wired via generic BrowserEval dispatcher: browser.snapshot browser.click, dblclick, hover, focus browser.fill, type, press browser.check, uncheck, select browser.scroll, scroll_into_view browser.wait, wait_ready browser.get.{text, title, html, value, attr, count, box} browser.find.{role, text, label, placeholder, testid} browser.console.{list, clear} browser.errors.{list, clear} browser.is_ready, is_editable Every action goes through a resolve-target helper that distinguishes refs (via window.__limux.refInfo) from selectors, returning a structured REF_NOT_FOUND error when a ref is no longer attached rather than silently acting on the wrong element. find.text prefers interactive descendants over structural ancestors so a paragraph wrapping a single link doesn't shadow the link itself. --- rust/limux-cli/src/main.rs | 5 +- rust/limux-host-linux/src/browser_init.js | 333 +++++++++ rust/limux-host-linux/src/browser_snapshot.js | 218 ++++++ rust/limux-host-linux/src/control_bridge.rs | 674 ++++++++++++++++++ rust/limux-host-linux/src/pane.rs | 23 +- rust/limux-host-linux/src/window.rs | 12 +- 6 files changed, 1247 insertions(+), 18 deletions(-) create mode 100644 rust/limux-host-linux/src/browser_init.js create mode 100644 rust/limux-host-linux/src/browser_snapshot.js diff --git a/rust/limux-cli/src/main.rs b/rust/limux-cli/src/main.rs index c73d9333..a919ecc7 100644 --- a/rust/limux-cli/src/main.rs +++ b/rust/limux-cli/src/main.rs @@ -1105,10 +1105,7 @@ async fn run_browser( if let Some(parent) = Path::new(out_path).parent() { if !parent.as_os_str().is_empty() { fs::create_dir_all(parent).with_context(|| { - format!( - "failed to create screenshot directory {}", - parent.display() - ) + format!("failed to create screenshot directory {}", parent.display()) })?; } } diff --git a/rust/limux-host-linux/src/browser_init.js b/rust/limux-host-linux/src/browser_init.js new file mode 100644 index 00000000..b24ef5c2 --- /dev/null +++ b/rust/limux-host-linux/src/browser_init.js @@ -0,0 +1,333 @@ +// limux browser automation init script +// Runs at DocumentStart on every top-frame navigation, before any page +// script. Exposes `window.__limux` with ref tagging, console/error ring +// buffers, history observers, and ready/editable probes. + +(() => { + if (window.__limux) return; + + // webkit6 JSC returns -2147483648 for Date.now() and performance.timeOrigin + // on most pages (wall-clock clamping to i32::MIN). performance.now() is the + // only reliable time source — it's monotonic, relative to page load, and + // comes back as a real number. We pair it with a sequence counter so the + // caller can filter by "since last call" deterministically. + let seq = 0; + const nowMs = () => Math.floor(performance.now ? performance.now() : 0); + const nextSeq = () => ++seq; + + + const LOG_CAP = 5000; + const ERR_CAP = 5000; + const INTERACTIVE_TAGS = new Set([ + "A", "BUTTON", "INPUT", "SELECT", "TEXTAREA", "SUMMARY", "DETAILS", + "LABEL", + ]); + const INTERACTIVE_ROLES = new Set([ + "button", "link", "checkbox", "radio", "menuitem", "menuitemcheckbox", + "menuitemradio", "tab", "option", "combobox", "textbox", "searchbox", + "slider", "spinbutton", "switch", "treeitem", "listitem", + ]); + + const state = { + nextRefId: 1, + refMeta: Object.create(null), + logs: [], + logsDroppedCount: 0, + errors: [], + errorsDroppedCount: 0, + navCount: 0, + lastMutationAt: 0, + mutationQuietMs: 500, + }; + + function isInteractive(el) { + if (!el || el.nodeType !== 1) return false; + if (INTERACTIVE_TAGS.has(el.tagName)) { + if (el.tagName === "A") return !!el.getAttribute("href"); + if (el.tagName === "LABEL") return !!el.getAttribute("for"); + return true; + } + const role = el.getAttribute && el.getAttribute("role"); + if (role && INTERACTIVE_ROLES.has(role.toLowerCase())) return true; + if (el.isContentEditable) return true; + const tabindex = el.getAttribute && el.getAttribute("tabindex"); + if (tabindex != null && Number.parseInt(tabindex, 10) >= 0) return true; + return false; + } + + function accessibleName(el) { + const aria = el.getAttribute && el.getAttribute("aria-label"); + if (aria) return aria.trim(); + const labelledBy = el.getAttribute && el.getAttribute("aria-labelledby"); + if (labelledBy) { + const parts = labelledBy.split(/\s+/) + .map((id) => document.getElementById(id)) + .filter(Boolean) + .map((n) => (n.textContent || "").trim()) + .filter(Boolean); + if (parts.length) return parts.join(" "); + } + if (el.tagName === "INPUT" || el.tagName === "SELECT" || el.tagName === "TEXTAREA") { + if (el.id) { + const label = document.querySelector(`label[for="${CSS.escape(el.id)}"]`); + if (label) return (label.textContent || "").trim(); + } + if (el.placeholder) return el.placeholder; + if (el.name) return el.name; + } + if (el.tagName === "IMG") { + const alt = el.getAttribute("alt"); + if (alt) return alt; + } + const title = el.getAttribute && el.getAttribute("title"); + if (title) return title; + const text = (el.innerText || el.textContent || "").replace(/\s+/g, " ").trim(); + if (text.length > 120) return text.slice(0, 120); + return text; + } + + function elementRole(el) { + const explicit = el.getAttribute && el.getAttribute("role"); + if (explicit) return explicit.toLowerCase(); + switch (el.tagName) { + case "A": return "link"; + case "BUTTON": return "button"; + case "SELECT": return "combobox"; + case "TEXTAREA": return "textbox"; + case "INPUT": { + const type = (el.type || "text").toLowerCase(); + if (type === "checkbox") return "checkbox"; + if (type === "radio") return "radio"; + if (type === "submit" || type === "button" || type === "reset") return "button"; + if (type === "range") return "slider"; + if (type === "search") return "searchbox"; + return "textbox"; + } + case "SUMMARY": return "button"; + case "DETAILS": return "group"; + case "LABEL": return "LabelText"; + default: return el.tagName.toLowerCase(); + } + } + + function assignRef(el) { + if (!isInteractive(el)) return null; + let id = el.getAttribute("data-limux-ref"); + if (id) { + // refresh metadata in case attributes changed + state.refMeta[id] = { + tag: el.tagName.toLowerCase(), + role: elementRole(el), + name: accessibleName(el), + }; + return id; + } + id = "e" + state.nextRefId++; + try { el.setAttribute("data-limux-ref", id); } catch (_) { return null; } + state.refMeta[id] = { + tag: el.tagName.toLowerCase(), + role: elementRole(el), + name: accessibleName(el), + }; + return id; + } + + function tagSubtree(root) { + if (!root) return; + if (root.nodeType === 1) assignRef(root); + if (root.querySelectorAll) { + const all = root.querySelectorAll( + "a[href], button, input, select, textarea, summary, details, label[for], " + + "[role], [contenteditable], [tabindex]" + ); + for (let i = 0; i < all.length; i++) assignRef(all[i]); + } + } + + function releaseRef(el) { + if (!el || el.nodeType !== 1) return; + const id = el.getAttribute && el.getAttribute("data-limux-ref"); + if (!id) return; + delete state.refMeta[id]; + } + + function releaseSubtree(root) { + if (!root) return; + if (root.nodeType === 1) releaseRef(root); + if (root.querySelectorAll) { + const all = root.querySelectorAll("[data-limux-ref]"); + for (let i = 0; i < all.length; i++) releaseRef(all[i]); + } + } + + function pushRing(buffer, entry, cap, dropCounterKey) { + if (buffer.length >= cap) { + buffer.shift(); + state[dropCounterKey]++; + } + buffer.push(entry); + } + + function recordLog(level, args) { + try { + const parts = []; + for (let i = 0; i < args.length; i++) { + const a = args[i]; + if (a == null) { parts.push(String(a)); continue; } + if (typeof a === "string") { parts.push(a); continue; } + if (a instanceof Error) { parts.push(a.stack || a.message); continue; } + try { parts.push(JSON.stringify(a)); } catch (_) { parts.push(String(a)); } + } + pushRing(state.logs, { + seq: nextSeq(), + ts_ms: nowMs(), + level, + text: parts.join(" "), + }, LOG_CAP, "logsDroppedCount"); + } catch (_) { /* never throw from console hook */ } + } + + function installConsoleHook() { + const levels = ["log", "warn", "error", "info", "debug"]; + for (const level of levels) { + const original = console[level] ? console[level].bind(console) : null; + console[level] = function () { + recordLog(level, arguments); + if (original) original.apply(null, arguments); + }; + } + } + + function installErrorHook() { + window.addEventListener("error", (ev) => { + pushRing(state.errors, { + seq: nextSeq(), + ts_ms: nowMs(), + source: "error", + message: ev.message, + filename: ev.filename, + lineno: ev.lineno, + colno: ev.colno, + stack: ev.error && ev.error.stack ? ev.error.stack : null, + }, ERR_CAP, "errorsDroppedCount"); + }, true); + window.addEventListener("unhandledrejection", (ev) => { + let reason = ev.reason; + if (reason instanceof Error) reason = reason.stack || reason.message; + else { try { reason = JSON.stringify(reason); } catch (_) { reason = String(reason); } } + pushRing(state.errors, { + seq: nextSeq(), + ts_ms: nowMs(), + source: "unhandledrejection", + message: String(reason), + }, ERR_CAP, "errorsDroppedCount"); + }, true); + } + + function installHistoryHook() { + const fire = (kind, url) => { + state.navCount++; + try { + window.dispatchEvent(new CustomEvent("limux:navigation", { + detail: { kind, url, navCount: state.navCount }, + })); + } catch (_) { /* ignore */ } + }; + const origPush = history.pushState; + history.pushState = function (data, title, url) { + const ret = origPush.apply(this, arguments); + fire("pushState", url || location.href); + return ret; + }; + const origReplace = history.replaceState; + history.replaceState = function (data, title, url) { + const ret = origReplace.apply(this, arguments); + fire("replaceState", url || location.href); + return ret; + }; + window.addEventListener("popstate", () => fire("popstate", location.href), true); + } + + function installMutationObserver() { + const obs = new MutationObserver((records) => { + state.lastMutationAt = nowMs(); + for (const rec of records) { + if (rec.type === "childList") { + for (const node of rec.addedNodes) tagSubtree(node); + for (const node of rec.removedNodes) releaseSubtree(node); + } else if (rec.type === "attributes" && rec.target && rec.target.nodeType === 1) { + assignRef(rec.target); + } + } + }); + const start = () => { + if (!document.documentElement) return; + obs.observe(document.documentElement, { + childList: true, + subtree: true, + attributes: true, + attributeFilter: [ + "role", "aria-label", "aria-labelledby", "href", "for", "tabindex", + "contenteditable", "disabled", "checked", "value", "placeholder", + "name", "type", + ], + }); + tagSubtree(document.body); + }; + if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", start, { once: true }); + } else { + start(); + } + } + + // API surface. Frozen — reads cannot accidentally mutate internal state. + const api = Object.freeze({ + version: 1, + refMeta: state.refMeta, + get logs() { return state.logs.slice(); }, + get errors() { return state.errors.slice(); }, + get logsDroppedCount() { return state.logsDroppedCount; }, + get errorsDroppedCount() { return state.errorsDroppedCount; }, + get navCount() { return state.navCount; }, + isReady() { + if (document.readyState !== "complete") return false; + if (state.lastMutationAt === 0) return true; + return (nowMs() - state.lastMutationAt) >= state.mutationQuietMs; + }, + isEditable() { + const a = document.activeElement; + if (!a) return false; + if (a.isContentEditable) return true; + const tag = a.tagName; + if (tag === "INPUT") { + const type = (a.type || "text").toLowerCase(); + return !["button", "submit", "reset", "checkbox", "radio", "range"].includes(type); + } + return tag === "TEXTAREA"; + }, + clearLogs() { state.logs.length = 0; state.logsDroppedCount = 0; }, + clearErrors() { state.errors.length = 0; state.errorsDroppedCount = 0; }, + lookupRef(id) { + const el = document.querySelector(`[data-limux-ref="${CSS.escape(id)}"]`); + return el || null; + }, + refInfo(id) { + const meta = state.refMeta[id]; + if (!meta) return null; + const el = this.lookupRef(id); + return { id, tag: meta.tag, role: meta.role, name: meta.name, attached: !!el }; + }, + tagSubtree: tagSubtree, + assignRef: assignRef, + elementRole: elementRole, + accessibleName: accessibleName, + isInteractive: isInteractive, + }); + Object.defineProperty(window, "__limux", { value: api, configurable: false, writable: false }); + + installConsoleHook(); + installErrorHook(); + installHistoryHook(); + installMutationObserver(); +})(); diff --git a/rust/limux-host-linux/src/browser_snapshot.js b/rust/limux-host-linux/src/browser_snapshot.js new file mode 100644 index 00000000..9dfddd29 --- /dev/null +++ b/rust/limux-host-linux/src/browser_snapshot.js @@ -0,0 +1,218 @@ +// limux browser snapshot walker. Invoked on-demand via evaluate_javascript. +// Returns a JSON string per the design doc: +// { +// url, title, hash, +// snapshot_text: "page ...\n- banner\n - link \"Home\" [ref=e1]\n...", +// refs: { eN: {selector, role, name, tag} }, +// shadow_closed: bool, +// truncated: null | { logs: N, errors: N, recreated: bool } +// } +// +// Options are passed in by the server: +// opts.full_tree — emit non-interactive text nodes too +// opts.raw_html — bypass AX walker, return document.documentElement.outerHTML +// opts.selector — scope walker to the first matching element +// opts.max_depth — clamp walker depth +// opts.since_hash — if matches current hash, emit diff only +// +// All parameters come in via placeholder substitution as a JSON literal +// named `__LIMUX_SNAPSHOT_OPTS__`. Rust builds the call like: +// (opts) => { ... }(__LIMUX_SNAPSHOT_OPTS__) + +((opts) => { + if (!window.__limux) { + return JSON.stringify({ error: { code: "INIT_NOT_READY", message: "init script not yet installed" } }); + } + const api = window.__limux; + const { full_tree = false, raw_html = false, selector = null, max_depth = null, since_hash = null } = (opts || {}); + + if (raw_html) { + const html = document.documentElement ? document.documentElement.outerHTML : ""; + return JSON.stringify({ + url: location.href, + title: document.title, + raw_html: html, + refs: api.refMeta, + truncated: truncationReport(), + }); + } + + const root = selector ? document.querySelector(selector) : document.body; + if (!root) { + return JSON.stringify({ + url: location.href, + title: document.title, + snapshot_text: "", + refs: {}, + shadow_closed: false, + truncated: truncationReport(), + error: { code: "SELECTOR_NOT_FOUND", message: "selector root not found" }, + }); + } + + // Refresh refs on subtree first to catch elements added since last mutation tick. + api.tagSubtree(root); + + const refs = Object.create(null); + let shadowClosed = false; + const lines = []; + const depthCap = (max_depth == null) ? Infinity : Number(max_depth); + + const pageLine = `page ${location.href} title ${jsonQuote(document.title || "")}`; + lines.push(pageLine); + + walk(root, 0); + const text = lines.join("\n"); + const hash = djb2(text); + + if (since_hash && since_hash === hash) { + return JSON.stringify({ + url: location.href, + title: document.title, + hash, + unchanged: true, + truncated: truncationReport(), + }); + } + + return JSON.stringify({ + url: location.href, + title: document.title, + hash, + snapshot_text: text, + refs, + shadow_closed: shadowClosed, + truncated: truncationReport(), + }); + + function walk(el, depth) { + if (!el || el.nodeType !== 1) return; + if (depth > depthCap) return; + + const tag = el.tagName; + if (tag === "SCRIPT" || tag === "STYLE" || tag === "NOSCRIPT" || tag === "TEMPLATE") return; + + // Detect closed shadow DOM host so the agent knows to switch tools. + if (el.shadowRoot === null && typeof el.attachShadow === "function") { + // shadowRoot null doesn't prove closed mode; use getRootNode to detect open-mode hosts. + } + // Rough heuristic: if the element has slotted children but shadowRoot is + // inaccessible, mark as closed. WebKit exposes open shadow via shadowRoot. + if (el.shadowRoot && el.shadowRoot.mode === "closed") shadowClosed = true; + + const hidden = isHiddenForSnapshot(el); + if (hidden && !el.matches("[tabindex], input, button, a[href], [role]")) return; + + const interactive = api.isInteractive(el); + let refId = null; + if (interactive) { + refId = el.getAttribute("data-limux-ref") || api.assignRef(el); + if (refId) { + refs[refId] = { + selector: `[data-limux-ref="${refId}"]`, + role: api.elementRole(el), + name: api.accessibleName(el), + tag: tag.toLowerCase(), + }; + } + } + + const role = api.elementRole(el); + const name = interactive ? api.accessibleName(el) : pickTextualName(el); + + if (interactive || full_tree || isLandmark(role) || name) { + lines.push(formatNode(depth, role, name, refId, el, hidden)); + } + + // Iframes + open shadow DOM children traversal (cross-frame snapshot for open mode only). + if (el.shadowRoot && el.shadowRoot.mode === "open") { + for (const child of el.shadowRoot.children) walk(child, depth + 1); + } + + for (const child of el.children) walk(child, depth + 1); + } + + function formatNode(depth, role, name, refId, el, hidden) { + const indent = " ".repeat(depth + 1); + const attrs = []; + if (refId) attrs.push(`ref=${refId}`); + if (hidden) attrs.push("hidden=true"); + if (el.hasAttribute("required")) attrs.push("required"); + if (el.hasAttribute("disabled")) attrs.push("disabled"); + if (el.hasAttribute("readonly")) attrs.push("readonly"); + if (el.type === "checkbox" || el.type === "radio") attrs.push(`checked=${!!el.checked}`); + const ariaExpanded = el.getAttribute("aria-expanded"); + if (ariaExpanded != null) attrs.push(`expanded=${ariaExpanded}`); + const ariaSelected = el.getAttribute("aria-selected"); + if (ariaSelected != null) attrs.push(`selected=${ariaSelected}`); + if (el.tagName === "H1") attrs.push("level=1"); + else if (el.tagName === "H2") attrs.push("level=2"); + else if (el.tagName === "H3") attrs.push("level=3"); + else if (el.tagName === "H4") attrs.push("level=4"); + else if (el.tagName === "H5") attrs.push("level=5"); + else if (el.tagName === "H6") attrs.push("level=6"); + if (el.tagName === "INPUT") { + const type = (el.type || "text").toLowerCase(); + if (type !== "text") attrs.push(`type=${type}`); + if (el.placeholder) attrs.push(`placeholder=${jsonQuote(el.placeholder)}`); + if (el.value && type !== "password") attrs.push(`value=${jsonQuote(trunc(el.value, 80))}`); + } + + let line = `${indent}- ${role}`; + if (name) line += ` ${jsonQuote(trunc(name, 120))}`; + if (attrs.length) line += ` [${attrs.join(", ")}]`; + return line; + } + + function isHiddenForSnapshot(el) { + if (el.hasAttribute("hidden")) return true; + if (el.getAttribute && el.getAttribute("aria-hidden") === "true") return true; + const style = el.ownerDocument.defaultView && el.ownerDocument.defaultView.getComputedStyle + ? el.ownerDocument.defaultView.getComputedStyle(el) + : null; + if (style && (style.display === "none" || style.visibility === "hidden")) return true; + return false; + } + + function isLandmark(role) { + return [ + "main", "banner", "navigation", "contentinfo", "complementary", "search", + "form", "region", "dialog", "alert", "alertdialog", "status", + "heading", "list", "listitem", "table", "row", "cell", + ].includes(role); + } + + function pickTextualName(el) { + // For landmarks/text containers, collect own text (not descendants). + if (!el.childNodes) return ""; + let out = ""; + for (const child of el.childNodes) { + if (child.nodeType === 3) out += child.nodeValue; + } + out = out.replace(/\s+/g, " ").trim(); + if (out.length > 120) out = out.slice(0, 120); + return out; + } + + function jsonQuote(s) { + return JSON.stringify(String(s == null ? "" : s)); + } + function trunc(s, n) { + if (s == null) return ""; + const str = String(s); + return str.length > n ? str.slice(0, n) : str; + } + function djb2(str) { + let h = 5381; + for (let i = 0; i < str.length; i++) { + h = ((h << 5) + h + str.charCodeAt(i)) | 0; + } + return "djb2:" + (h >>> 0).toString(16); + } + function truncationReport() { + const logs = api.logsDroppedCount; + const errors = api.errorsDroppedCount; + if (logs === 0 && errors === 0) return null; + return { logs, errors }; + } +})(__LIMUX_SNAPSHOT_OPTS__); diff --git a/rust/limux-host-linux/src/control_bridge.rs b/rust/limux-host-linux/src/control_bridge.rs index 43cf56e5..7b74d01b 100644 --- a/rust/limux-host-linux/src/control_bridge.rs +++ b/rust/limux-host-linux/src/control_bridge.rs @@ -38,6 +38,39 @@ const METHODS: &[&str] = &[ "browser.reload", "browser.screenshot", "browser.eval", + "browser.snapshot", + "browser.click", + "browser.dblclick", + "browser.hover", + "browser.focus", + "browser.fill", + "browser.type", + "browser.press", + "browser.check", + "browser.uncheck", + "browser.select", + "browser.scroll", + "browser.scroll_into_view", + "browser.wait", + "browser.wait_ready", + "browser.get.text", + "browser.get.title", + "browser.get.html", + "browser.get.value", + "browser.get.attr", + "browser.get.count", + "browser.get.box", + "browser.find.role", + "browser.find.text", + "browser.find.label", + "browser.find.placeholder", + "browser.find.testid", + "browser.console.list", + "browser.console.clear", + "browser.errors.list", + "browser.errors.clear", + "browser.is_ready", + "browser.is_editable", ]; const PARSE_ERROR_CODE: i64 = -32700; @@ -556,6 +589,26 @@ fn handle_method( rx, ) } + m if m.starts_with("browser.") => { + let surface = match required_string(params, &["surface_id", "id"], "surface_id") { + Ok(value) => normalize_handle(value, "surface:"), + Err(error) => return error_response(id, error), + }; + let (script, wrap_key) = match build_browser_script(method, params) { + Ok(pair) => pair, + Err(error) => return error_response(id, error), + }; + let (reply, rx) = mpsc::channel(); + ( + ControlCommand::BrowserEval { + surface, + script, + wrap_key, + reply, + }, + rx, + ) + } _ => { return error_response( id, @@ -590,6 +643,627 @@ fn error_response(id: Option<Value>, error: BridgeError) -> V2Response { V2Response::error(id, error.code, error.message, error.data) } +fn js_literal(value: &str) -> String { + serde_json::Value::String(value.to_string()).to_string() +} + +fn json_literal(value: &Value) -> String { + serde_json::to_string(value).unwrap_or_else(|_| "null".to_string()) +} + +/// JS that resolves a ref-or-selector to a DOM element. When the caller +/// provides a ref (`eN`) the script also verifies attachment and rejects +/// with structured REF_NOT_FOUND / STALE_REF errors rather than silently +/// acting on a stale handle. +fn resolve_target_js(handle_literal: &str) -> String { + format!( + r#"(() => {{ + const handle = {h}; + if (!handle) {{ return {{ err: {{ code: "INVALID_PARAMS", message: "selector or ref required" }} }}; }} + const trimmed = String(handle).replace(/^@/, ""); + const refMatch = /^e\d+$/.test(trimmed); + if (refMatch) {{ + if (!window.__limux) {{ + return {{ err: {{ code: "INIT_NOT_READY", message: "limux init script not installed" }} }}; + }} + const info = window.__limux.refInfo(trimmed); + if (!info) {{ return {{ err: {{ code: "REF_NOT_FOUND", message: "ref " + trimmed + " unknown", ref: trimmed }} }}; }} + if (!info.attached) {{ return {{ err: {{ code: "REF_NOT_FOUND", message: "ref " + trimmed + " no longer attached", ref: trimmed }} }}; }} + const el = window.__limux.lookupRef(trimmed); + if (!el) {{ return {{ err: {{ code: "REF_NOT_FOUND", message: "ref " + trimmed + " not in DOM", ref: trimmed }} }}; }} + return {{ el, ref: trimmed, info }}; + }} + try {{ + const el = document.querySelector(String(handle)); + if (!el) {{ return {{ err: {{ code: "REF_NOT_FOUND", message: "selector matched no element", selector: String(handle) }} }}; }} + return {{ el, selector: String(handle) }}; + }} catch (e) {{ + return {{ err: {{ code: "INVALID_SELECTOR", message: String(e && e.message || e), selector: String(handle) }} }}; + }} + }})()"#, + h = handle_literal + ) +} + +/// Wrap a JS action so REF_NOT_FOUND / INVALID_SELECTOR / errors are +/// returned as structured JSON (handler unwraps `err` into BridgeError). +fn wrap_action_js(handle_literal: &str, body: &str) -> String { + format!( + r#"(() => {{ + const target = {resolve}; + if (target.err) {{ return JSON.stringify({{ ok: false, error: target.err }}); }} + const el = target.el; + try {{ + {body} + }} catch (e) {{ + return JSON.stringify({{ ok: false, error: {{ code: "INTERNAL", message: String(e && e.message || e) }} }}); + }} + }})()"#, + resolve = resolve_target_js(handle_literal), + body = body, + ) +} + +/// Build JS payloads for browser.* methods. All scripts must return a +/// JSON string; the handler parses it and either merges it into the +/// response (when an object + `wrap_key` is None) or stores it under +/// `wrap_key`. +fn build_browser_script( + method: &str, + params: &Map<String, Value>, +) -> Result<(String, Option<String>), BridgeError> { + match method { + "browser.snapshot" => { + let opts = snapshot_opts(params); + let script = crate::pane::LIMUX_BROWSER_SNAPSHOT_SCRIPT + .replace("__LIMUX_SNAPSHOT_OPTS__", &opts); + Ok((script, None)) + } + "browser.click" => Ok((browser_action_click(params)?, None)), + "browser.dblclick" => Ok((browser_action_dblclick(params)?, None)), + "browser.hover" => Ok((browser_action_hover(params)?, None)), + "browser.focus" => Ok((browser_action_focus(params)?, None)), + "browser.fill" => Ok((browser_action_fill(params)?, None)), + "browser.type" => Ok((browser_action_type(params)?, None)), + "browser.press" => Ok((browser_action_press(params)?, None)), + "browser.check" => Ok((browser_action_toggle_checkable(params, true)?, None)), + "browser.uncheck" => Ok((browser_action_toggle_checkable(params, false)?, None)), + "browser.select" => Ok((browser_action_select(params)?, None)), + "browser.scroll" => Ok((browser_action_scroll(params)?, None)), + "browser.scroll_into_view" => Ok((browser_action_scroll_into_view(params)?, None)), + "browser.wait" => Ok((browser_action_wait(params)?, None)), + "browser.wait_ready" => Ok((browser_action_wait_ready(params)?, None)), + "browser.get.text" => Ok((browser_get_text(params)?, None)), + "browser.get.title" => Ok(( + "JSON.stringify({ title: document.title })".to_string(), + None, + )), + "browser.get.html" => Ok((browser_get_html(params)?, None)), + "browser.get.value" => Ok((browser_get_value(params)?, None)), + "browser.get.attr" => Ok((browser_get_attr(params)?, None)), + "browser.get.count" => Ok((browser_get_count(params)?, None)), + "browser.get.box" => Ok((browser_get_box(params)?, None)), + "browser.find.role" => Ok((browser_find(params, "role")?, None)), + "browser.find.text" => Ok((browser_find(params, "text")?, None)), + "browser.find.label" => Ok((browser_find(params, "label")?, None)), + "browser.find.placeholder" => Ok((browser_find(params, "placeholder")?, None)), + "browser.find.testid" => Ok((browser_find(params, "testid")?, None)), + "browser.console.list" => Ok((browser_console_list(params)?, None)), + "browser.console.clear" => Ok(( + r#"(() => { if (window.__limux) window.__limux.clearLogs(); return JSON.stringify({ ok: true }); })()"#.to_string(), + None, + )), + "browser.errors.list" => Ok((browser_errors_list(params)?, None)), + "browser.errors.clear" => Ok(( + r#"(() => { if (window.__limux) window.__limux.clearErrors(); return JSON.stringify({ ok: true }); })()"#.to_string(), + None, + )), + "browser.is_ready" => Ok(( + r#"JSON.stringify({ ready: !!(window.__limux && window.__limux.isReady()) })"#.to_string(), + None, + )), + "browser.is_editable" => Ok(( + r#"JSON.stringify({ editable: !!(window.__limux && window.__limux.isEditable()) })"#.to_string(), + None, + )), + _ => Err(BridgeError::invalid_params(format!( + "no browser script for {method}" + ))), + } +} + +fn snapshot_opts(params: &Map<String, Value>) -> String { + let mut obj = serde_json::Map::new(); + if let Some(v) = params.get("full_tree") { + obj.insert("full_tree".into(), v.clone()); + } + if let Some(v) = params.get("raw_html") { + obj.insert("raw_html".into(), v.clone()); + } + if let Some(v) = params.get("selector") { + obj.insert("selector".into(), v.clone()); + } + if let Some(v) = params.get("max_depth") { + obj.insert("max_depth".into(), v.clone()); + } + if let Some(v) = params.get("since_hash") { + obj.insert("since_hash".into(), v.clone()); + } + serde_json::to_string(&Value::Object(obj)).unwrap_or_else(|_| "{}".to_string()) +} + +fn target_handle(params: &Map<String, Value>) -> Result<String, BridgeError> { + if let Some(s) = optional_string(params, &["ref", "selector", "target"]) { + return Ok(s); + } + Err(BridgeError::invalid_params("ref or selector required")) +} + +fn browser_action_click(params: &Map<String, Value>) -> Result<String, BridgeError> { + let handle = target_handle(params)?; + let body = r#" + if (typeof el.click === "function") { el.click(); } + else { el.dispatchEvent(new MouseEvent("click", { bubbles: true, cancelable: true })); } + return JSON.stringify({ ok: true, ref: target.ref, selector: target.selector }); + "#; + Ok(wrap_action_js(&js_literal(&handle), body)) +} + +fn browser_action_dblclick(params: &Map<String, Value>) -> Result<String, BridgeError> { + let handle = target_handle(params)?; + let body = r#" + el.dispatchEvent(new MouseEvent("dblclick", { bubbles: true, cancelable: true })); + return JSON.stringify({ ok: true, ref: target.ref, selector: target.selector }); + "#; + Ok(wrap_action_js(&js_literal(&handle), body)) +} + +fn browser_action_hover(params: &Map<String, Value>) -> Result<String, BridgeError> { + let handle = target_handle(params)?; + let body = r#" + el.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + el.dispatchEvent(new MouseEvent("mouseenter", { bubbles: false })); + return JSON.stringify({ ok: true, ref: target.ref, selector: target.selector }); + "#; + Ok(wrap_action_js(&js_literal(&handle), body)) +} + +fn browser_action_focus(params: &Map<String, Value>) -> Result<String, BridgeError> { + let handle = target_handle(params)?; + let body = r#" + if (typeof el.focus === "function") el.focus(); + return JSON.stringify({ ok: true, ref: target.ref, selector: target.selector }); + "#; + Ok(wrap_action_js(&js_literal(&handle), body)) +} + +fn browser_action_fill(params: &Map<String, Value>) -> Result<String, BridgeError> { + let handle = target_handle(params)?; + let text = optional_string(params, &["text", "value"]).unwrap_or_default(); + let body = format!( + r#" + const text = {text}; + if (el.isContentEditable) {{ + el.focus(); + el.textContent = text; + el.dispatchEvent(new Event("input", {{ bubbles: true }})); + }} else if (el.tagName === "TEXTAREA" || el.tagName === "INPUT" || el.tagName === "SELECT") {{ + const proto = el.tagName === "TEXTAREA" ? HTMLTextAreaElement.prototype + : el.tagName === "SELECT" ? HTMLSelectElement.prototype + : HTMLInputElement.prototype; + const setter = Object.getOwnPropertyDescriptor(proto, "value").set; + setter.call(el, text); + el.dispatchEvent(new Event("input", {{ bubbles: true }})); + el.dispatchEvent(new Event("change", {{ bubbles: true }})); + }} else {{ + return JSON.stringify({{ ok: false, error: {{ code: "WRONG_ELEMENT", message: "cannot fill tag " + el.tagName }} }}); + }} + return JSON.stringify({{ ok: true, ref: target.ref, selector: target.selector }}); + "#, + text = js_literal(&text) + ); + Ok(wrap_action_js(&js_literal(&handle), &body)) +} + +fn browser_action_type(params: &Map<String, Value>) -> Result<String, BridgeError> { + // Dispatch keydown/keypress/keyup + input events for each char on the focused element. + let text = required_string(params, &["text"], "text")?; + let script = format!( + r#"(() => {{ + const text = {text}; + const el = document.activeElement; + if (!el) {{ return JSON.stringify({{ ok: false, error: {{ code: "NO_FOCUS", message: "no focused element" }} }}); }} + for (const ch of text) {{ + el.dispatchEvent(new KeyboardEvent("keydown", {{ key: ch, bubbles: true }})); + el.dispatchEvent(new KeyboardEvent("keypress", {{ key: ch, bubbles: true }})); + if (el.tagName === "INPUT" || el.tagName === "TEXTAREA") {{ + const proto = el.tagName === "TEXTAREA" ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype; + const setter = Object.getOwnPropertyDescriptor(proto, "value").set; + setter.call(el, (el.value || "") + ch); + el.dispatchEvent(new Event("input", {{ bubbles: true }})); + }} else if (el.isContentEditable) {{ + el.textContent = (el.textContent || "") + ch; + el.dispatchEvent(new Event("input", {{ bubbles: true }})); + }} + el.dispatchEvent(new KeyboardEvent("keyup", {{ key: ch, bubbles: true }})); + }} + return JSON.stringify({{ ok: true }}); + }})()"#, + text = js_literal(&text) + ); + Ok(script) +} + +fn browser_action_press(params: &Map<String, Value>) -> Result<String, BridgeError> { + // keys = "Enter", "Ctrl+K", etc. Parse into key + modifier flags. + let keys = required_string(params, &["keys", "key"], "keys")?; + let script = format!( + r#"(() => {{ + const raw = {keys}; + const parts = String(raw).split("+").map(s => s.trim()); + const key = parts.pop(); + const mods = new Set(parts.map(s => s.toLowerCase())); + const el = document.activeElement || document.body; + const init = {{ + key, + bubbles: true, + cancelable: true, + ctrlKey: mods.has("ctrl") || mods.has("control"), + altKey: mods.has("alt"), + shiftKey: mods.has("shift"), + metaKey: mods.has("meta") || mods.has("cmd") || mods.has("super"), + }}; + el.dispatchEvent(new KeyboardEvent("keydown", init)); + el.dispatchEvent(new KeyboardEvent("keypress", init)); + el.dispatchEvent(new KeyboardEvent("keyup", init)); + return JSON.stringify({{ ok: true, key, mods: [...mods] }}); + }})()"#, + keys = js_literal(&keys) + ); + Ok(script) +} + +fn browser_action_toggle_checkable( + params: &Map<String, Value>, + desired: bool, +) -> Result<String, BridgeError> { + let handle = target_handle(params)?; + let body = format!( + r#" + if (el.type !== "checkbox" && el.type !== "radio" && el.getAttribute("role") !== "checkbox" && el.getAttribute("role") !== "switch") {{ + return JSON.stringify({{ ok: false, error: {{ code: "WRONG_ELEMENT", message: "not a checkable element" }} }}); + }} + const want = {desired}; + if (el.type === "radio") {{ + if (want) {{ if (!el.checked) el.click(); }} + else {{ return JSON.stringify({{ ok: false, error: {{ code: "INVALID_OP", message: "cannot uncheck a radio" }} }}); }} + }} else if (el.checked !== want) {{ + el.click(); + }} + return JSON.stringify({{ ok: true, ref: target.ref, selector: target.selector, checked: !!el.checked }}); + "#, + desired = desired + ); + Ok(wrap_action_js(&js_literal(&handle), &body)) +} + +fn browser_action_select(params: &Map<String, Value>) -> Result<String, BridgeError> { + let handle = target_handle(params)?; + let option = required_string(params, &["option", "value", "label"], "option")?; + let body = format!( + r#" + if (el.tagName !== "SELECT") {{ return JSON.stringify({{ ok: false, error: {{ code: "WRONG_ELEMENT", message: "not a <select>" }} }}); }} + const want = {opt}; + let matched = false; + for (const option of el.options) {{ + if (option.value === want || option.label === want || option.text === want) {{ + el.value = option.value; + matched = true; + break; + }} + }} + if (!matched) {{ return JSON.stringify({{ ok: false, error: {{ code: "OPTION_NOT_FOUND", message: "option not found", option: want }} }}); }} + el.dispatchEvent(new Event("input", {{ bubbles: true }})); + el.dispatchEvent(new Event("change", {{ bubbles: true }})); + return JSON.stringify({{ ok: true, ref: target.ref, selector: target.selector, value: el.value }}); + "#, + opt = js_literal(&option) + ); + Ok(wrap_action_js(&js_literal(&handle), &body)) +} + +fn browser_action_scroll(params: &Map<String, Value>) -> Result<String, BridgeError> { + let x = params.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0); + let y = params.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0); + let script = format!( + r#"(() => {{ + window.scrollBy({{ left: {x}, top: {y}, behavior: "instant" }}); + return JSON.stringify({{ ok: true, x: window.scrollX, y: window.scrollY }}); + }})()"# + ); + Ok(script) +} + +fn browser_action_scroll_into_view(params: &Map<String, Value>) -> Result<String, BridgeError> { + let handle = target_handle(params)?; + let body = r#" + if (typeof el.scrollIntoView === "function") el.scrollIntoView({ behavior: "instant", block: "center", inline: "center" }); + return JSON.stringify({ ok: true, ref: target.ref, selector: target.selector }); + "#; + Ok(wrap_action_js(&js_literal(&handle), body)) +} + +fn browser_action_wait(params: &Map<String, Value>) -> Result<String, BridgeError> { + // Poll every 100ms up to timeout_ms (default 5000). Condition: either + // selector matches OR ref is attached OR document ready flag. + let selector = optional_string(params, &["selector"]); + let ref_id = optional_string(params, &["ref"]); + let timeout_ms = params + .get("timeout_ms") + .or_else(|| params.get("timeout")) + .and_then(|v| v.as_u64()) + .unwrap_or(5000); + let ready_flag = params + .get("ready") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + let script = format!( + r#"(async () => {{ + const selector = {sel}; + const refId = {rf}; + const readyFlag = {ready}; + const deadline = performance.now() + {timeout}; + while (performance.now() < deadline) {{ + if (readyFlag && window.__limux && window.__limux.isReady()) {{ + return JSON.stringify({{ ok: true, reason: "ready" }}); + }} + if (selector) {{ + try {{ + const el = document.querySelector(selector); + if (el) return JSON.stringify({{ ok: true, selector, reason: "selector" }}); + }} catch (e) {{ + return JSON.stringify({{ ok: false, error: {{ code: "INVALID_SELECTOR", message: String(e && e.message || e) }} }}); + }} + }} + if (refId) {{ + const info = window.__limux && window.__limux.refInfo(refId); + if (info && info.attached) return JSON.stringify({{ ok: true, ref: refId, reason: "ref" }}); + }} + await new Promise(r => setTimeout(r, 100)); + }} + return JSON.stringify({{ ok: false, error: {{ code: "TIMEOUT", message: "wait timed out", timeout_ms: {timeout} }} }}); + }})()"#, + sel = json_literal(&Value::from(selector)), + rf = json_literal(&Value::from(ref_id)), + ready = ready_flag, + timeout = timeout_ms + ); + Ok(script) +} + +fn browser_action_wait_ready(params: &Map<String, Value>) -> Result<String, BridgeError> { + let timeout_ms = params + .get("timeout_ms") + .or_else(|| params.get("timeout")) + .and_then(|v| v.as_u64()) + .unwrap_or(30000); + let script = format!( + r#"(async () => {{ + const deadline = performance.now() + {timeout}; + while (performance.now() < deadline) {{ + if (window.__limux && window.__limux.isReady()) {{ + return JSON.stringify({{ ok: true }}); + }} + await new Promise(r => setTimeout(r, 100)); + }} + return JSON.stringify({{ ok: false, error: {{ code: "TIMEOUT", message: "page not ready within " + {timeout} + "ms" }} }}); + }})()"#, + timeout = timeout_ms + ); + Ok(script) +} + +fn browser_get_text(params: &Map<String, Value>) -> Result<String, BridgeError> { + let handle = optional_string(params, &["ref", "selector", "target"]) + .unwrap_or_else(|| "body".to_string()); + let body = r#" + const text = el.innerText || el.textContent || ""; + return JSON.stringify({ ok: true, text }); + "#; + Ok(wrap_action_js(&js_literal(&handle), body)) +} + +fn browser_get_html(params: &Map<String, Value>) -> Result<String, BridgeError> { + let handle = optional_string(params, &["ref", "selector", "target"]); + match handle { + Some(h) => { + let body = r#" + return JSON.stringify({ ok: true, html: el.outerHTML }); + "#; + Ok(wrap_action_js(&js_literal(&h), body)) + } + None => Ok( + r#"JSON.stringify({ ok: true, html: document.documentElement.outerHTML })"#.to_string(), + ), + } +} + +fn browser_get_value(params: &Map<String, Value>) -> Result<String, BridgeError> { + let handle = target_handle(params)?; + let body = r#" + return JSON.stringify({ ok: true, value: el.value ?? null }); + "#; + Ok(wrap_action_js(&js_literal(&handle), body)) +} + +fn browser_get_attr(params: &Map<String, Value>) -> Result<String, BridgeError> { + let handle = target_handle(params)?; + let name = required_string(params, &["attr", "name"], "attr")?; + let body = format!( + r#" + return JSON.stringify({{ ok: true, value: el.getAttribute({n}) }}); + "#, + n = js_literal(&name) + ); + Ok(wrap_action_js(&js_literal(&handle), &body)) +} + +fn browser_get_count(params: &Map<String, Value>) -> Result<String, BridgeError> { + let selector = required_string(params, &["selector"], "selector")?; + let script = format!( + r#"(() => {{ + try {{ + return JSON.stringify({{ ok: true, count: document.querySelectorAll({s}).length }}); + }} catch (e) {{ + return JSON.stringify({{ ok: false, error: {{ code: "INVALID_SELECTOR", message: String(e && e.message || e) }} }}); + }} + }})()"#, + s = js_literal(&selector) + ); + Ok(script) +} + +fn browser_get_box(params: &Map<String, Value>) -> Result<String, BridgeError> { + let handle = target_handle(params)?; + let body = r#" + const r = el.getBoundingClientRect(); + return JSON.stringify({ ok: true, box: { x: r.left, y: r.top, w: r.width, h: r.height } }); + "#; + Ok(wrap_action_js(&js_literal(&handle), body)) +} + +fn browser_find(params: &Map<String, Value>, kind: &str) -> Result<String, BridgeError> { + let value = required_string( + params, + &[ + "value", + "name", + "role", + "text", + "label", + "placeholder", + "testid", + ], + "value", + )?; + let script = format!( + r#"(() => {{ + if (!window.__limux) return JSON.stringify({{ ok: false, error: {{ code: "INIT_NOT_READY", message: "init script not installed" }} }}); + const kind = {kind}; + const want = {v}; + let el = null; + if (kind === "role") {{ + const nodes = document.querySelectorAll("[role], a, button, input, select, textarea, summary, details"); + for (const node of nodes) {{ + if (window.__limux.elementRole(node) === want) {{ el = node; break; }} + }} + }} else if (kind === "text") {{ + // Prefer interactive elements with exact-text match so we + // don't accidentally target an ancestor wrapper. + const interactive = document.querySelectorAll( + "a[href], button, input, select, textarea, summary, " + + "[role=button], [role=link], [role=checkbox], [role=radio], " + + "[role=menuitem], [role=tab], [role=option]" + ); + for (const node of interactive) {{ + const label = ((node.innerText || node.textContent || "").trim()) || node.value || node.getAttribute("aria-label") || ""; + if (label === want) {{ el = node; break; }} + }} + if (!el) {{ + const all = document.querySelectorAll("h1, h2, h3, h4, h5, h6, label, [role=heading], [role=listitem], [role=alert]"); + for (const node of all) {{ + if ((node.innerText || node.textContent || "").trim() === want) {{ el = node; break; }} + }} + }} + }} else if (kind === "label") {{ + const label = document.querySelector("label"); + const all = document.querySelectorAll("label"); + for (const lbl of all) {{ + if ((lbl.textContent || "").trim() === want) {{ + if (lbl.htmlFor) el = document.getElementById(lbl.htmlFor); + if (!el) el = lbl.querySelector("input, textarea, select"); + if (el) break; + }} + }} + }} else if (kind === "placeholder") {{ + el = document.querySelector('[placeholder="' + CSS.escape(want) + '"]'); + }} else if (kind === "testid") {{ + el = document.querySelector('[data-testid="' + CSS.escape(want) + '"]'); + if (!el) el = document.querySelector('[data-test-id="' + CSS.escape(want) + '"]'); + }} + if (!el) return JSON.stringify({{ ok: false, error: {{ code: "NOT_FOUND", message: "find." + kind + " matched nothing", query: want }} }}); + const id = window.__limux.assignRef(el); + return JSON.stringify({{ ok: true, ref: id, role: window.__limux.elementRole(el), name: window.__limux.accessibleName(el) }}); + }})()"#, + kind = js_literal(kind), + v = js_literal(&value) + ); + Ok(script) +} + +fn browser_console_list(params: &Map<String, Value>) -> Result<String, BridgeError> { + // `since` filters by monotonic seq number (returned in each entry). The + // caller records the largest `seq` it saw and passes it back next time. + // ts_ms is informational only — webkit6 clamps wall-clock to i32::MIN so + // seq is the only reliable ordering. + let since = params.get("since").and_then(|v| v.as_u64()).unwrap_or(0); + let clear_after = params + .get("clear_after") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + let levels = optional_string(params, &["level", "levels"]); + let levels_filter = levels + .map(|s| { + let v: Vec<String> = s.split(',').map(|x| x.trim().to_string()).collect(); + format!( + "[{}]", + v.iter() + .map(|x| js_literal(x)) + .collect::<Vec<_>>() + .join(",") + ) + }) + .unwrap_or_else(|| "null".to_string()); + let script = format!( + r#"(() => {{ + if (!window.__limux) return JSON.stringify({{ logs: [], dropped: 0 }}); + const since = {since}; + const levels = {levels}; + const logs = window.__limux.logs.filter(e => e.seq > since && (!levels || levels.includes(e.level))); + const dropped = window.__limux.logsDroppedCount; + const latest = logs.length ? logs[logs.length - 1].seq : since; + if ({clear}) window.__limux.clearLogs(); + return JSON.stringify({{ logs, dropped, latest_seq: latest }}); + }})()"#, + since = since, + levels = levels_filter, + clear = clear_after + ); + Ok(script) +} + +fn browser_errors_list(params: &Map<String, Value>) -> Result<String, BridgeError> { + let since = params.get("since").and_then(|v| v.as_u64()).unwrap_or(0); + let clear_after = params + .get("clear_after") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + let script = format!( + r#"(() => {{ + if (!window.__limux) return JSON.stringify({{ errors: [], dropped: 0 }}); + const since = {since}; + const errors = window.__limux.errors.filter(e => e.seq > since); + const dropped = window.__limux.errorsDroppedCount; + const latest = errors.length ? errors[errors.length - 1].seq : since; + if ({clear}) window.__limux.clearErrors(); + return JSON.stringify({{ errors, dropped, latest_seq: latest }}); + }})()"#, + since = since, + clear = clear_after + ); + Ok(script) +} + fn dispatch_request(input: &str, dispatch: &dyn Fn(ControlCommand)) -> V2Response { match parse_request(input) { Ok(request) => handle_method(request.id, &request.method, request.params, dispatch), diff --git a/rust/limux-host-linux/src/pane.rs b/rust/limux-host-linux/src/pane.rs index a97ecc4b..238fbbb3 100644 --- a/rust/limux-host-linux/src/pane.rs +++ b/rust/limux-host-linux/src/pane.rs @@ -2693,15 +2693,14 @@ impl BrowserShortcutTarget { match result { Ok(texture) => { if texture.width() == 0 || texture.height() == 0 { - cb(Err("snapshot texture empty; webview not yet rendered" - .to_string())); + cb(Err( + "snapshot texture empty; webview not yet rendered".to_string() + )); return; } match texture.save_to_png(&out_path) { Ok(()) => cb(Ok(out_path)), - Err(error) => { - cb(Err(format!("snapshot save failed: {error}"))) - } + Err(error) => cb(Err(format!("snapshot save failed: {error}"))), } } Err(error) => cb(Err(error.to_string())), @@ -2914,6 +2913,10 @@ impl BrowserHandles { } } +pub const LIMUX_BROWSER_INIT_SCRIPT: &str = include_str!("browser_init.js"); + +pub const LIMUX_BROWSER_SNAPSHOT_SCRIPT: &str = include_str!("browser_snapshot.js"); + #[cfg(feature = "webkit")] const LIMUX_BROWSER_EDITABLE_STATE_HANDLER: &str = "limuxEditableState"; @@ -2994,6 +2997,16 @@ fn create_browser_widget( &[], &[], )); + // Install limux browser automation init script on every top-frame load, + // before any page script runs. Provides window.__limux with ref tagging, + // console/error ring buffers, history hooks, and ready/editable probes. + user_content_manager.add_script(&webkit6::UserScript::new( + LIMUX_BROWSER_INIT_SCRIPT, + webkit6::UserContentInjectedFrames::TopFrame, + webkit6::UserScriptInjectionTime::Start, + &[], + &[], + )); { let dom_editable = dom_editable.clone(); user_content_manager.connect_script_message_received( diff --git a/rust/limux-host-linux/src/window.rs b/rust/limux-host-linux/src/window.rs index 8c85d7ee..869fcb64 100644 --- a/rust/limux-host-linux/src/window.rs +++ b/rust/limux-host-linux/src/window.rs @@ -3372,9 +3372,7 @@ fn browser_navigate( })) } -fn browser_get_url( - surface: &str, -) -> Result<serde_json::Value, crate::control_bridge::BridgeError> { +fn browser_get_url(surface: &str) -> Result<serde_json::Value, crate::control_bridge::BridgeError> { let target = pane::find_browser_target(surface).ok_or_else(|| { crate::control_bridge::BridgeError::not_found("browser surface not found") })?; @@ -3414,9 +3412,7 @@ fn browser_history( fn browser_screenshot( surface: &str, out_path: Option<String>, - reply: std::sync::mpsc::Sender< - Result<serde_json::Value, crate::control_bridge::BridgeError>, - >, + reply: std::sync::mpsc::Sender<Result<serde_json::Value, crate::control_bridge::BridgeError>>, ) { let Some(target) = pane::find_browser_target(surface) else { let _ = reply.send(Err(crate::control_bridge::BridgeError::not_found( @@ -3452,9 +3448,7 @@ fn browser_eval( surface: &str, script: String, wrap_key: Option<String>, - reply: std::sync::mpsc::Sender< - Result<serde_json::Value, crate::control_bridge::BridgeError>, - >, + reply: std::sync::mpsc::Sender<Result<serde_json::Value, crate::control_bridge::BridgeError>>, ) { let Some(target) = pane::find_browser_target(surface) else { let _ = reply.send(Err(crate::control_bridge::BridgeError::not_found( From c97558a68b238946c1df8a675e71e9bdada31206 Mon Sep 17 00:00:00 2001 From: "MVB.Mir" <buracmircea@gmail.com> Date: Fri, 17 Apr 2026 18:35:37 +0300 Subject: [PATCH 3/3] Fix clippy collapsible_match warnings (Rust 1.95) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rust 1.95 stable (released between main's last green CI run and this PR's) promotes collapsible_match to a warning, which the workspace's check.sh treats as an error via -D warnings. The flagged patterns in limux-core are `match combo_norm { pattern => { if palette_visible { … true } else { false } } … }` — they collapse cleanly into match guards: `pattern if palette_visible => { … true }`. Behavior is unchanged; any non-match combo hits the catch-all `_ => false` arm. Unrelated to the redesign; included here to keep CI green. --- rust/limux-core/src/lib.rs | 40 ++++++++++++-------------------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/rust/limux-core/src/lib.rs b/rust/limux-core/src/lib.rs index af50fd19..4ccbab09 100644 --- a/rust/limux-core/src/lib.rs +++ b/rust/limux-core/src/lib.rs @@ -2689,37 +2689,21 @@ impl ControlState { } true } - "down" | "ctrl+n" | "ctrl+j" => { - if palette_visible { - self.command_palette_move_selection(window_id, 1); - true - } else { - false - } + "down" | "ctrl+n" | "ctrl+j" if palette_visible => { + self.command_palette_move_selection(window_id, 1); + true } - "up" | "ctrl+p" | "ctrl+k" => { - if palette_visible { - self.command_palette_move_selection(window_id, -1); - true - } else { - false - } + "up" | "ctrl+p" | "ctrl+k" if palette_visible => { + self.command_palette_move_selection(window_id, -1); + true } - "cmd+a" => { - if palette_visible { - self.command_palette_select_all(window_id); - true - } else { - false - } + "cmd+a" if palette_visible => { + self.command_palette_select_all(window_id); + true } - "enter" => { - if palette_visible { - self.command_palette_enter(window_id); - true - } else { - false - } + "enter" if palette_visible => { + self.command_palette_enter(window_id); + true } _ => false, }