refactor(computer-use): phase-0 quick wins from computer-use/browser-control audit

scripts/core-boundaries/rules/source/forbidden-rules.mjs

@@ -2149,26 +2149,6 @@ export const forbiddenContentRules = [
       },
     ],
   },
-  {
-    path: 'src/crates/assembly/core/src/agentic/tools/computer_use_verification.rs',
-    patterns: [
-      {
-        regex: /\bpub struct VerificationResult\b/,
-        message:
-          'core Computer Use verification facade must not own verification contracts; use tool-runtime computer_use',
-      },
-      {
-        regex: /\bpub struct RetryStrategy\b/,
-        message:
-          'core Computer Use verification facade must not own retry strategy state; use tool-runtime computer_use',
-      },
-      {
-        regex: /\bpub fn detect_visual_change\b/,
-        message:
-          'core Computer Use verification facade must not own visual-change logic; use tool-runtime computer_use',
-      },
-    ],
-  },
   {
     path: 'src/crates/assembly/core/src/agentic/session/turn_skill_agent_snapshot_store.rs',
     patterns: [

src/crates/assembly/core/builtin-skills-upstreams.json

@@ -20,7 +20,7 @@
       "revision": "81c336c1c20b80ac648e0416a7b6e0c0ae7878bb",
       "package_version": "0.32.3",
       "local_patches": [
-        "route web and supported Electron work to agent-browser and native desktop work to BitFun ComputerUse",
+        "prefer BitFun ControlHub browser domain when available, use agent-browser only when ControlHub is unavailable or for supported Electron work, and route native desktop work to BitFun ComputerUse",
         "pin the documented install version and require user approval",
         "preserve explicit missing-prerequisite and no-silent-fallback behavior"
       ]

src/crates/assembly/core/builtin_skills/agent-browser/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: agent-browser
-description: Browser automation CLI for AI agents. Use when the user needs to interact with websites, including navigating pages, filling forms, clicking buttons, taking screenshots, extracting data, testing web apps, or automating any browser task. Triggers include requests to "open a website", "fill out a form", "click a button", "take a screenshot", "scrape data from a page", "test this web app", "login to a site", "automate browser actions", or any task requiring programmatic web interaction. Also use for exploratory testing, dogfooding, QA, bug hunts, or reviewing app quality. Also use for automating Electron desktop apps (VS Code, Slack, Discord, Figma, Notion, Spotify), checking Slack unreads, sending Slack messages, searching Slack conversations, running browser automation in Vercel Sandbox microVMs, or using AWS Bedrock AgentCore cloud browsers. Prefer agent-browser for web and supported Electron automation; use BitFun ComputerUse for native desktop UI that agent-browser cannot reach.
+description: Browser automation CLI for AI agents. Use when the user needs to interact with websites, including navigating pages, filling forms, clicking buttons, taking screenshots, extracting data, testing web apps, or automating any browser task. Triggers include requests to "open a website", "fill out a form", "click a button", "take a screenshot", "scrape data from a page", "test this web app", "login to a site", "automate browser actions", or any task requiring programmatic web interaction. Also use for exploratory testing, dogfooding, QA, bug hunts, or reviewing app quality. Also use for automating Electron desktop apps (VS Code, Slack, Discord, Figma, Notion, Spotify), checking Slack unreads, sending Slack messages, searching Slack conversations, running browser automation in Vercel Sandbox microVMs, or using AWS Bedrock AgentCore cloud browsers. Prefer BitFun's ControlHub browser domain for web automation when it is available; use agent-browser only when ControlHub is unavailable or for supported Electron automation, and use BitFun ComputerUse for native desktop UI that agent-browser cannot reach.
 allowed-tools: Bash(agent-browser:*), Bash(npx agent-browser:*)
 hidden: true
 ---
@@ -9,6 +9,8 @@ hidden: true
 
 Fast browser automation CLI for AI agents. Chrome/Chromium via CDP with accessibility-tree snapshots and compact `@eN` element refs.
 
+Prefer BitFun's `ControlHub` browser domain when it is available; use this skill only when `ControlHub` is unavailable. The two stacks use separate browser instances, element refs, and login state, so do not mix them within one task.
+
 Install only after user approval: `npm i -g agent-browser@0.32.3 && agent-browser install`
 
 If the CLI is unavailable and the user declines installation, explain the missing prerequisite and offer a non-browser fallback; do not silently switch tools.

src/crates/assembly/core/src/agentic/agents/prompts/agentic_mode.md

@@ -89,6 +89,7 @@ The user will primarily request you perform software engineering tasks. This inc
 - When the user explicitly asks to complete work and review it carefully, finish the implementation first, then dispatch at most one independent read-only `CodeReview` Task. Do not fan out `CodeReview` into architecture, performance, security, product, or other invented dimensions: broader coverage belongs to the unified `/review` path, which selects bounded review lenses and owns cost confirmation. Do not launch review by default for every task.
 - Treat reviewer output as adversarial evidence. The reviewer never fixes its own findings. Apply accepted fixes in the implementation agent, then request a fresh independent review only when the change or risk warrants it.
 - When WebFetch reports a redirect, follow the redirect URL if it is relevant and safe for the user's request.
+- For browser and web-page work, route in this order: (1) reading page content that does not require the user's login state: use WebFetch; (2) pages that require the user's login state or JavaScript interaction: use `ControlHub` with `domain: "browser"` (connect, snapshot, then act through `@eN` refs); (3) non-Chromium browsers (Firefox/Safari) or native desktop apps: use `ComputerUse` desktop actions. Prefer `ControlHub` over browser-automation skills such as `agent-browser`; use those skills only when `ControlHub` is unavailable.
 - When multiple tool calls are independent, run them in parallel. Keep dependent operations sequential, and never use placeholders or guess missing parameters.
 - Use specialized tools for file reads, edits, searches, and deletions because they preserve workspace context and permissions. Use ExecCommand for commands that genuinely need a shell. Do not use shell commands only to communicate with the user.
 - For security-sensitive tasks, support defensive analysis and remediation only. Refuse malicious code, exploit workflows, credential harvesting, or instructions that would facilitate abuse.

src/crates/assembly/core/src/agentic/agents/prompts/claw_mode.md

@@ -20,6 +20,12 @@ Use `ControlHub` for browser automation, terminal signalling, and routing/capabi
 - `domain: "terminal"` for signalling existing terminal sessions, such as interrupting or killing them.
 - `domain: "meta"` for capability and route checks.
 
+For browser and web-page work, route in this order:
+
+1. Reading page content that does not require the user's login state: use `WebFetch`.
+2. Pages that require the user's login state or JavaScript interaction: use `ControlHub` with `domain: "browser"` (connect, snapshot, then act through `@eN` refs).
+3. Non-Chromium browsers (Firefox/Safari) or native desktop apps: delegate to a `ComputerUse` session as described below.
+
 Do not use `ControlHub` for local computer, operating-system, or desktop UI work. Desktop and system actions have moved to the dedicated `ComputerUse` tool/agent. This includes screenshots, OCR, mouse, keyboard, app state, app launching, opening files or URLs through the OS, clipboard access, OS facts, and local scripts.
 
 If the user asks you to operate or inspect the local computer, delegate the task to a `ComputerUse` session via SessionControl/SessionMessage only when both tools appear in your current tool list. Include the user's goal, target app/window/site, safety constraints, and expected verification in the handoff. If delegation is unavailable, explain that the task needs the Computer Use mode.

src/crates/assembly/core/src/agentic/agents/prompts/computer_use_mode.md

@@ -67,7 +67,13 @@ When Runtime Context indicates the primary model does not support image understa
 
 # Browser Work
 
-For websites and web apps, prefer `ControlHub` with `domain: "browser"` when it is available so cookies, login state, and extensions are preserved. If `ControlHub` is unavailable, do not claim browser-domain automation; use `ComputerUse` only for browser chrome or OS-level interaction that it can actually observe and verify.
+For websites and web apps, route in this order:
+
+1. Reading page content that does not require the user's login state: use `WebFetch` when it is available.
+2. Pages that require the user's login state or JavaScript interaction: use `ControlHub` with `domain: "browser"` (connect, snapshot, then act through `@eN` refs) so cookies, login state, and extensions are preserved.
+3. Non-Chromium browsers (Firefox/Safari) or native desktop apps: use `ComputerUse` desktop actions.
+
+If `ControlHub` is unavailable, do not claim browser-domain automation; use `ComputerUse` only for browser chrome or OS-level interaction that it can actually observe and verify.
 
 Use desktop-domain controls only for browser chrome, OS dialogs, permission prompts, file pickers, or when browser-domain capabilities are unavailable.
 

src/crates/assembly/core/src/agentic/tools/computer_use_host.rs

@@ -88,7 +88,7 @@ pub trait ComputerUseHost: Send + Sync + std::fmt::Debug {
     /// Fails if no screenshot was taken in this process since startup (or since last host reset).
     fn map_image_coords_to_pointer(&self, x: i32, y: i32) -> BitFunResult<(i32, i32)>;
 
-    /// Same as `map_image_coords_to_pointer` but **sub-point** precision (macOS: use for `ComputerUseMousePrecise`).
+    /// Same as `map_image_coords_to_pointer` but **sub-point** precision (macOS: use for the `mouse_move` action).
     fn map_image_coords_to_pointer_f64(&self, x: i32, y: i32) -> BitFunResult<(f64, f64)> {
         let (a, b) = self.map_image_coords_to_pointer(x, y)?;
         Ok((a as f64, b as f64))
@@ -110,10 +110,10 @@ pub trait ComputerUseHost: Send + Sync + std::fmt::Debug {
 
     async fn mouse_move(&self, x: i32, y: i32) -> BitFunResult<()>;
 
-    /// Move the pointer by `(dx, dy)` in **global screen pixels** (same space as `ComputerUseMousePrecise` absolute).
+    /// Move the pointer by `(dx, dy)` in **global screen pixels** (same space as absolute `mouse_move` globals).
     async fn pointer_move_relative(&self, dx: i32, dy: i32) -> BitFunResult<()>;
 
-    /// Click at the **current** pointer position only (does not move). Use `ComputerUseMousePrecise` / `ComputerUseMouseStep` / `pointer_move_rel` first.
+    /// Click at the **current** pointer position only (does not move). Use `mouse_move` / `move_to_text` / `pointer_move_rel` first.
     /// `button`: "left" | "right" | "middle"
     /// On desktop, enforces the vision fine-screenshot guard (unlike [`mouse_click_authoritative`](Self::mouse_click_authoritative)).
     async fn mouse_click(&self, button: &str) -> BitFunResult<()>;
@@ -187,7 +187,7 @@ pub trait ComputerUseHost: Send + Sync + std::fmt::Debug {
     /// After a successful `screenshot_display`, the model may `mouse_click` (until the pointer moves again).
     fn computer_use_after_screenshot(&self) {}
 
-    /// After `ComputerUseMousePrecise` / `ComputerUseMouseStep` / relative pointer moves: the next `mouse_click` must be preceded by a new screenshot.
+    /// After `mouse_move` / `pointer_move_rel` pointer moves: the next `mouse_click` must be preceded by a new screenshot.
     fn computer_use_after_pointer_mutation(&self) {}
 
     /// After `mouse_click`, require a fresh screenshot before the next click (unless pointer moved, which also invalidates).

src/crates/assembly/core/src/agentic/tools/implementations/computer_use_actions.rs

@@ -22,8 +22,8 @@ use super::control_hub::{coded_tool_error, err_response, ControlHubError, ErrorC
 /// Key = target PID, value = `(target_signature, before_digest, count)`.
 /// When the same `(action,target)` lands on an unchanged digest twice in a
 /// row the dispatcher injects an `app_state.loop_warning` so the model is
-/// forced off the failing path on its **next** turn (`/Screenshot policy/
-/// Mandatory screenshot moments` in `claw_mode.md`).
+/// forced off the failing path on its **next** turn (see the observe → act →
+/// verify guidance in `computer_use_mode.md`).
 type AppLoopTracker =
     std::sync::OnceLock<std::sync::Mutex<std::collections::HashMap<i32, (String, String, u32)>>>;
 
@@ -107,7 +107,7 @@ impl ComputerUseActions {
         ControlHubError::new(
             ErrorCode::GuardRejected,
             format!(
-                "desktop.{} is blocked while {} is frontmost. Use ControlHub domain=\"browser\" for all browser interaction; desktop mouse/keyboard browser control is forbidden.",
+                "ComputerUse `{}` is blocked while {} is frontmost. Use ControlHub domain=\"browser\" for all browser interaction; desktop mouse/keyboard browser control is forbidden.",
                 action, app_name
             ),
         )
@@ -130,26 +130,21 @@ impl ComputerUseActions {
             .unwrap_or("")
             .to_ascii_lowercase();
 
-        const NAME_HINTS: &[&str] = &[
-            "chrome",
-            "chromium",
-            "edge",
-            "brave",
-            "arc",
-            "firefox",
-            "safari",
-            "browser",
-            "浏览器",
-        ];
-        const BUNDLE_HINTS: &[&str] = &[
-            "chrome", "chromium", "edge", "brave", "arc", "firefox", "safari", "browser",
-        ];
+        // Only Chromium-family browsers are guarded: they are the only ones the
+        // ControlHub browser domain can drive over CDP. Firefox/Safari (and other
+        // non-Chromium browsers) have no CDP path, so desktop control must stay
+        // allowed for them — blocking both surfaces would leave no control path.
+        const NAME_HINTS: &[&str] = &["chrome", "chromium", "edge", "brave", "arc"];
+        const BUNDLE_HINTS: &[&str] = &["chrome", "chromium", "edge", "brave", "arc"];
 
         NAME_HINTS.iter().any(|hint| name.contains(hint))
             || BUNDLE_HINTS.iter().any(|hint| bundle.contains(hint))
     }
 
-    async fn desktop_action_targets_browser(
+    /// Rejects physical input actions while a CDP-drivable browser is frontmost.
+    /// Read-only observation actions (`screenshot`, `locate`, `describe_screen`, …)
+    /// stay allowed. Called by `ComputerUseTool::call_impl` before dispatch.
+    pub(crate) async fn desktop_action_targets_browser(
         &self,
         action: &str,
         context: &ToolUseContext,
@@ -166,7 +161,6 @@ impl ComputerUseActions {
             "key_chord",
             "type_text",
             "paste",
-            "locate",
             "move_to_text",
         ];
         if !guarded_actions.contains(&action) {
@@ -356,10 +350,6 @@ impl ComputerUseActions {
             _ => {}
         }
 
-        if let Some(err) = self.desktop_action_targets_browser(action, context).await {
-            return Ok(err_response("desktop", action, err));
-        }
-
         // UX shortcut: every screen-coordinate action accepts an optional
         // `display_id`. If present (and different from the currently pinned
         // display), pin it BEFORE forwarding so the model doesn't need a
@@ -837,6 +827,26 @@ impl ComputerUseActions {
             result_with_optional_screenshot(data, summary, shot_opt)
         }
 
+        // These actions only make sense with a marked-up screenshot the model
+        // can look at; text-only models are steered to the AX/OCR text paths.
+        if text_only
+            && matches!(
+                action,
+                "build_interactive_view"
+                    | "interactive_click"
+                    | "build_visual_mark_view"
+                    | "visual_click"
+            )
+        {
+            return Err(coded_tool_error(
+                ErrorCode::NotAvailable,
+                format!(
+                    "`{}` requires a vision-capable primary model (its result is a marked-up screenshot). Use `describe_screen` or `get_app_state` to observe as text, then act with `app_click`, `click_target`, `move_to_text`, or `key_chord`.",
+                    action
+                ),
+            ));
+        }
+
         let bg = host.supports_background_input();
         let ax = host.supports_ax_tree();
 
@@ -1591,6 +1601,8 @@ fn error_code_from_local(code: &str) -> ErrorCode {
 #[cfg(test)]
 mod tests {
     use super::loop_tracker_observe;
+    use super::ComputerUseActions;
+    use crate::agentic::tools::computer_use_host::ComputerUseForegroundApplication;
 
     // A unique PID avoids interference with the shared APP_LOOP_TRACKER state
     // across tests in the same process.
@@ -1636,6 +1648,37 @@ mod tests {
         );
     }
 
+    fn foreground(name: &str, bundle_id: &str) -> ComputerUseForegroundApplication {
+        ComputerUseForegroundApplication {
+            name: Some(name.to_string()),
+            bundle_id: Some(bundle_id.to_string()),
+            process_id: Some(1),
+        }
+    }
+
+    /// Only Chromium-family browsers are CDP-drivable via the ControlHub
+    /// browser domain. Firefox/Safari must NOT trip the desktop browser guard
+    /// or the user would have no control path at all.
+    #[test]
+    fn browser_guard_matches_only_chromium_family() {
+        assert!(ComputerUseActions::is_probably_browser_app(&foreground(
+            "Google Chrome",
+            "com.google.Chrome"
+        )));
+        assert!(ComputerUseActions::is_probably_browser_app(&foreground(
+            "Microsoft Edge",
+            "com.microsoft.edgemac"
+        )));
+        assert!(!ComputerUseActions::is_probably_browser_app(&foreground(
+            "Firefox",
+            "org.mozilla.firefox"
+        )));
+        assert!(!ComputerUseActions::is_probably_browser_app(&foreground(
+            "Safari",
+            "com.apple.Safari"
+        )));
+    }
+
     /// A genuine tree mutation (digest changes) must NOT trigger the warning,
     /// even on the same target — progress resets the streak.
     #[test]