diff --git a/extensions/browser-use/README.md b/extensions/browser-use/README.md index 5b9f6409cb2..0e72c6b129e 100644 --- a/extensions/browser-use/README.md +++ b/extensions/browser-use/README.md @@ -1,17 +1,22 @@ # browser-use -Minimal pi extension that enforces browser-only mode and routes a single `browser` tool through [`agent-browser`](https://github.com/vercel-labs/agent-browser). +Minimal pi extension that enforces browser-only mode and exposes a single Playwright-backed `browser` tool. + +The tool is text-only: it observes Chromium through the raw CDP accessibility tree, filters it into compact refs, and performs actions against those refs. It does not use screenshots, vision, or model-predicted screen coordinates. ## Requirements -Install `agent-browser` first: +Install dependencies from the repository root: ```bash -npm install -g agent-browser -agent-browser install +npm install ``` -The extension also works with `npx agent-browser`, but a global install is the simplest setup. +If Playwright has not installed Chromium yet, run: + +```bash +npx playwright install chromium +``` ## Usage @@ -21,32 +26,33 @@ pi -e ./extensions/browser-use/index.ts The extension exposes a single `browser` tool and blocks all non-browser tools for the session. -Start with a `goto` step: +Start with a `goto` action: ```json { - "steps": [ + "actions": [ { "type": "goto", "url": "https://example.com" }, - { "type": "snapshot", "interactive": true } + { "type": "snapshot" } ] } ``` -Use `snapshot` steps to collect refs like `@e1` when you need them, and batch short predictable flows into one call. +Each result includes a fresh accessibility tree with refs like `e1`. Refs are regenerated after every call, so use only refs from the latest result. ## Tools -The `browser` tool accepts a `steps` array. Supported step types are: +The `browser` tool accepts an `actions` array with 1 to 7 actions: -- navigation and session: `goto`, `navigation`, `tabs`, `close` -- page interaction: `click`, `type`, `fill`, `select`, `check`, `uncheck`, `hover`, `drag`, `upload`, `scroll`, `wait` -- inspection and capture: `snapshot`, `screenshot`, `pdf` -- keyboard and mouse: `keyboard`, `mouse` +- `goto`: navigate to a URL. +- `snapshot`: observe only. +- `click`: click a ref from the latest accessibility tree. +- `fill`: replace text in an editable ref. +- `press`: press a keyboard key, optionally after focusing a ref. +- `scroll`: scroll the page or a scrollable ref. +- `wait`: wait for milliseconds or a Playwright load state. ## Notes -- `snapshot` supports `interactive`, `urls`, `compact`, `depth`, and `selector`. -- `screenshot` supports `annotate` and `full`. -- `select` accepts either a single `value` or multiple `values`. -- `upload` targets a specific `ref` or selector and accepts `files`. -- Shared options such as `session`, `sessionName`, `profile`, `provider`, `engine`, `cdp`, and `headers` live at the top level of the `browser` tool call and apply to every step in the sequence. +- Observations come from CDP `Accessibility.getFullAXTree`, not Playwright AI snapshots. +- Actions target refs, not CSS selectors, role/name locators, screenshots, or coordinates. +- Set top-level `headed: true` on the first call to show the Chromium window. diff --git a/extensions/browser-use/index.ts b/extensions/browser-use/index.ts index c98ede8e1e1..a80e54722f9 100644 --- a/extensions/browser-use/index.ts +++ b/extensions/browser-use/index.ts @@ -1,1163 +1,514 @@ -import { mkdir, readFile } from "node:fs/promises"; -import { join } from "node:path"; -import { - StringEnum, - type ImageContent, - type TextContent, -} from "@mariozechner/pi-ai"; -import { Type, type Static, type TSchema } from "@sinclair/typebox"; +import { createRequire } from "node:module"; +import { type TextContent } from "@mariozechner/pi-ai"; +import { Type, type Static } from "typebox"; import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, truncateTail, type ExtensionAPI, - type ExtensionContext, type ToolCallEvent, } from "@mariozechner/pi-coding-agent"; const EXTENSION_NAME = "browser-use"; -const EXTENSION_PREFIX = "pi-browser-use"; const BROWSER_TOOL_NAME = "browser"; +const REF_ATTRIBUTE = "data-pi-browser-ref"; +const DEFAULT_WAIT_MS = 1000; +const DEFAULT_SCROLL_Y = 600; const DEFAULT_BROWSER_PROMPT_SNIPPET = - "`browser`: interact with the browser by running one or more sequential steps with agent-browser. Use this instead of shell tools."; -type BrowserStepType = - | "goto" - | "click" - | "type" - | "fill" - | "select" - | "check" - | "uncheck" - | "hover" - | "drag" - | "upload" - | "scroll" - | "wait" - | "close" - | "snapshot" - | "screenshot" - | "pdf" - | "navigation" - | "tabs" - | "keyboard" - | "mouse"; + "`browser`: interact with a Chromium browser using text-only accessibility-tree refs. Use this instead of shell tools."; const DEFAULT_BROWSER_GUIDELINES = [ "Use only the `browser` tool for browser interaction. Do not attempt to use bash, read, edit, write, grep, find, or ls.", - "Prefer one `browser` call with a few predictable `steps` over many tiny calls.", - "Use a `snapshot` step to collect refs like `@e1` before ref-based actions, or when the page structure changes enough that old refs may be stale.", - "You do not need a `wait` or `snapshot` step after every page-changing action. Add `wait` when navigation, async rendering, or downloads need confirmation before the next step.", - "Use a `screenshot` step only when visual context is needed. Set `annotate: true` when you want image labels that line up with snapshot refs.", + "Use `goto` first, then use refs from the returned accessibility tree for `click`, `fill`, `press`, and `scroll` actions.", + "Refs are regenerated after each browser call. Use only refs from the latest browser result.", + "Do not predict screen coordinates or rely on screenshots. The browser tool is text-only.", ]; -const BrowserProviderSchema = StringEnum(["ios", "browserbase", "kernel", "browseruse", "browserless", "agentcore"] as const, { - description: "Optional agent-browser provider.", -}); -const BrowserEngineSchema = StringEnum(["chrome", "lightpanda"] as const, { - description: "Optional agent-browser engine.", -}); -const MouseButtonSchema = StringEnum(["left", "right", "middle"] as const, { - description: "Mouse button to use for mouse commands.", -}); -const BrowserNavigationActionSchema = StringEnum(["back", "forward", "reload"] as const, { - description: "Navigation action to perform on the current page.", -}); -const BrowserTabsActionSchema = StringEnum(["list", "new", "select", "close"] as const, { - description: "Tab action to perform.", -}); -const BrowserKeyboardActionSchema = StringEnum(["press", "down", "up"] as const, { - description: "Keyboard action to perform.", -}); -const BrowserMouseActionSchema = StringEnum(["move", "down", "up", "wheel"] as const, { - description: "Mouse action to perform.", -}); -const BrowserScrollDirectionSchema = StringEnum(["up", "down", "left", "right"] as const, { - description: "Scroll direction.", -}); -const BrowserWaitLoadStateSchema = StringEnum(["load", "domcontentloaded", "networkidle"] as const, { - description: "Page load state to wait for.", -}); -const BrowserWaitStateSchema = StringEnum(["visible", "hidden", "attached", "detached"] as const, { - description: "Element state to wait for when `target` is used.", -}); - -const BrowserCommandNames = [ - "goto", - "click", - "type", - "fill", - "select", - "check", - "uncheck", - "hover", - "drag", - "upload", - "scroll", - "wait", - "close", - "snapshot", - "screenshot", - "pdf", - "go-back", - "go-forward", - "reload", - "tab-list", - "tab-new", - "tab-select", - "tab-close", - "press", - "keydown", - "keyup", - "mousemove", - "mousedown", - "mouseup", - "mousewheel", -] as const; +const LoadStateSchema = Type.Union([Type.Literal("load"), Type.Literal("domcontentloaded"), Type.Literal("networkidle")]); +const ScrollDirectionSchema = Type.Union([Type.Literal("up"), Type.Literal("down"), Type.Literal("left"), Type.Literal("right")]); + +const browserToolParameters = Type.Object( + { + headed: Type.Optional(Type.Boolean({ description: "Whether to show the Chromium window." })), + actions: Type.Array( + Type.Union([ + Type.Object({ type: Type.Literal("goto"), url: Type.String({ description: "URL to navigate to." }) }, { additionalProperties: false }), + Type.Object({ type: Type.Literal("snapshot") }, { additionalProperties: false }), + Type.Object({ type: Type.Literal("click"), ref: Type.String({ description: "Ref from the latest accessibility tree, such as e3." }) }, { additionalProperties: false }), + Type.Object( + { + type: Type.Literal("fill"), + ref: Type.String({ description: "Editable ref from the latest accessibility tree." }), + text: Type.String({ description: "Replacement text." }), + }, + { additionalProperties: false }, + ), + Type.Object( + { + type: Type.Literal("press"), + key: Type.String({ description: "Keyboard key name, such as Enter, Tab, or ArrowDown." }), + ref: Type.Optional(Type.String({ description: "Optional ref to focus before pressing the key." })), + }, + { additionalProperties: false }, + ), + Type.Object( + { + type: Type.Literal("scroll"), + ref: Type.Optional(Type.String({ description: "Optional scrollable ref. If omitted, scrolls the page." })), + direction: Type.Optional(ScrollDirectionSchema), + amount: Type.Optional(Type.Number({ description: "Scroll amount in pixels.", minimum: 0 })), + }, + { additionalProperties: false }, + ), + Type.Object( + { + type: Type.Literal("wait"), + ms: Type.Optional(Type.Number({ description: "Milliseconds to wait.", minimum: 0 })), + loadState: Type.Optional(LoadStateSchema), + }, + { additionalProperties: false }, + ), + ]), + { minItems: 1, maxItems: 7, description: "Browser actions to execute sequentially." }, + ), + }, + { additionalProperties: false }, +); -type BrowserCommandName = (typeof BrowserCommandNames)[number]; +type BrowserToolCallParams = Static; +type BrowserAction = BrowserToolCallParams["actions"][number]; +type LoadState = Static; +type ScrollDirection = Static; interface BrowserToolDetails { - sessionName: string; - steps: Array<{ - type: BrowserStepType; - command: BrowserCommandName; - invocation: string[]; - summary: string; - stdout?: string; - stderr?: string; - outputPath?: string; - }>; + actions: Array<{ type: BrowserAction["type"]; summary: string }>; + title: string; + url: string; + refCount: number; } -interface ResolvedCli { - command: string; - baseArgs: string[]; +interface BrowserType { + launch(options: { headless: boolean }): Promise; } -interface DriverState { - activeToolCallId?: string; - resolvedCli?: ResolvedCli; +interface Browser { + newContext(): Promise; + close(): Promise; } -interface PreparedCommand { - argv: string[]; - summary: string; - outputPath?: string; +interface BrowserContext { + newPage(): Promise; + newCDPSession(page: Page): Promise; + close(): Promise; } -interface AgentBrowserResponse { - success?: boolean; - data?: unknown; - error?: unknown; - warning?: unknown; +interface Page { + goto(url: string, options?: { waitUntil?: LoadState }): Promise; + title(): Promise; + url(): string; + waitForLoadState(state: LoadState): Promise; + waitForTimeout(ms: number): Promise; + evaluate(pageFunction: (arg: TArg) => unknown, arg: TArg): Promise; + locator(selector: string): Locator; + keyboard: { press(key: string): Promise }; } -interface CliOutput { - stdout: string; - stderr: string; - response?: AgentBrowserResponse; +interface Locator { + click(): Promise; + fill(text: string): Promise; + focus(): Promise; + evaluate(pageFunction: (element: Element, arg: TArg) => unknown, arg: TArg): Promise; } -const SharedCommandFields = { - session: Type.Optional( - Type.String({ - description: "Optional agent-browser session name. Defaults to a workspace-derived session name.", - minLength: 1, - }), - ), - headed: Type.Optional(Type.Boolean({ description: "Whether to show the browser window." })), - provider: Type.Optional(BrowserProviderSchema), - engine: Type.Optional(BrowserEngineSchema), - sessionName: Type.Optional( - Type.String({ - description: "Optional persisted state name for agent-browser `--session-name`.", - minLength: 1, - }), - ), - profile: Type.Optional( - Type.String({ - description: "Optional Chrome profile name or persistent profile path for agent-browser `--profile`.", - minLength: 1, - }), - ), - state: Type.Optional( - Type.String({ - description: "Optional storage state JSON path for agent-browser `--state`.", - minLength: 1, - }), - ), - autoConnect: Type.Optional(Type.Boolean({ description: "Whether to auto-connect to a running Chrome instance." })), - cdp: Type.Optional( - Type.String({ - description: "Optional Chrome DevTools Protocol port or WebSocket URL.", - minLength: 1, - }), - ), - device: Type.Optional( - Type.String({ - description: "Optional device name, typically for the iOS provider.", - minLength: 1, - }), - ), - allowFileAccess: Type.Optional(Type.Boolean({ description: "Whether to allow `file://` pages to access local files." })), - ignoreHttpsErrors: Type.Optional(Type.Boolean({ description: "Whether to ignore HTTPS certificate errors." })), - headers: Type.Optional( - Type.Record(Type.String(), Type.String(), { - description: "Optional HTTP headers to send for the navigated origin.", - }), - ), -} as const; +interface CdpSession { + send(method: string, params?: Record): Promise; + detach(): Promise; +} -function toolSchema>(properties: TProperties) { - return Type.Object( - { - ...properties, - ...SharedCommandFields, - }, - { additionalProperties: false }, - ); +interface DriverState { + activeToolCallId?: string; + browser?: Browser; + context?: BrowserContext; + page?: Page; + cdp?: CdpSession; + headed?: boolean; + refMap: Map; } -function browserStepSchema>(type: TType, properties: TProperties) { - return Type.Object( - { - type: Type.Literal(type), - ...properties, - }, - { additionalProperties: false }, - ); +interface AXValue { + value?: unknown; } -const browserToolParameters = toolSchema({ - steps: Type.Array( - Type.Union([ - browserStepSchema("goto", { - url: Type.String({ description: "URL to navigate to." }), - }), - browserStepSchema("click", { - ref: Type.String({ description: "Element ref, CSS selector, XPath, or other agent-browser locator." }), - newTab: Type.Optional(Type.Boolean({ description: "Open a link target in a new tab instead of the current tab." })), - }), - browserStepSchema("type", { - text: Type.String({ description: "Text to type." }), - insertText: Type.Optional(Type.Boolean({ description: "Insert text without key events." })), - }), - browserStepSchema("fill", { - ref: Type.String({ description: "Element ref or selector to fill." }), - text: Type.String({ description: "Replacement text." }), - }), - browserStepSchema("select", { - ref: Type.String({ description: "Select element ref or selector." }), - value: Type.Optional(Type.String({ description: "Single option value to select." })), - values: Type.Optional(Type.Array(Type.String(), { minItems: 1, description: "One or more option values to select." })), - }), - browserStepSchema("check", { - ref: Type.String({ description: "Checkbox or radio ref or selector." }), - }), - browserStepSchema("uncheck", { - ref: Type.String({ description: "Checkbox ref or selector." }), - }), - browserStepSchema("hover", { - ref: Type.String({ description: "Element ref or selector to hover." }), - }), - browserStepSchema("drag", { - startRef: Type.String({ description: "Source element ref or selector." }), - endRef: Type.String({ description: "Target element ref or selector." }), - }), - browserStepSchema("upload", { - ref: Type.String({ description: "File input ref or selector." }), - files: Type.Array(Type.String(), { minItems: 1, description: "One or more file paths to upload." }), - }), - browserStepSchema("scroll", { - direction: Type.Optional(BrowserScrollDirectionSchema), - amount: Type.Optional(Type.Number({ description: "Optional number of pixels to scroll.", minimum: 0 })), - selector: Type.Optional(Type.String({ description: "Optional selector for a specific scrollable container." })), - }), - browserStepSchema("wait", { - ms: Type.Optional(Type.Number({ description: "Milliseconds to wait.", minimum: 0 })), - target: Type.Optional(Type.String({ description: "Element ref or selector to wait for." })), - text: Type.Optional(Type.String({ description: "Page text to wait for." })), - urlPattern: Type.Optional(Type.String({ description: "URL pattern to wait for." })), - loadState: Type.Optional(BrowserWaitLoadStateSchema), - expression: Type.Optional(Type.String({ description: "JavaScript expression to wait for." })), - waitState: Type.Optional(BrowserWaitStateSchema), - downloadPath: Type.Optional(Type.String({ description: "Path to save the next download to." })), - timeout: Type.Optional(Type.Number({ description: "Optional download timeout in milliseconds.", minimum: 0 })), - }), - browserStepSchema("close", {}), - browserStepSchema("snapshot", { - interactive: Type.Optional(Type.Boolean({ description: "Only include interactive elements." })), - urls: Type.Optional(Type.Boolean({ description: "Include href URLs for links in the snapshot." })), - compact: Type.Optional(Type.Boolean({ description: "Remove empty structural elements." })), - depth: Type.Optional(Type.Number({ description: "Optional maximum tree depth.", minimum: 1 })), - selector: Type.Optional(Type.String({ description: "Optional selector to scope the snapshot." })), - }), - browserStepSchema("screenshot", { - ref: Type.Optional(Type.String({ description: "Optional element ref or selector for an element screenshot." })), - full: Type.Optional(Type.Boolean({ description: "Capture the full page instead of only the viewport." })), - annotate: Type.Optional(Type.Boolean({ description: "Overlay numbered labels that line up with snapshot refs." })), - }), - browserStepSchema("pdf", {}), - browserStepSchema("navigation", { - action: BrowserNavigationActionSchema, - }), - browserStepSchema("tabs", { - action: BrowserTabsActionSchema, - url: Type.Optional(Type.String({ description: "Optional URL for `new`." })), - index: Type.Optional(Type.Number({ description: "Tab index for `select`. Optional for `close`.", minimum: 0 })), - }), - browserStepSchema("keyboard", { - action: BrowserKeyboardActionSchema, - key: Type.String({ description: "Keyboard key name, such as Enter or ArrowDown." }), - }), - browserStepSchema("mouse", { - action: BrowserMouseActionSchema, - x: Type.Optional(Type.Number({ description: "X coordinate for `move`." })), - y: Type.Optional(Type.Number({ description: "Y coordinate for `move`." })), - button: Type.Optional(MouseButtonSchema), - dx: Type.Optional(Type.Number({ description: "Optional horizontal wheel delta for `wheel`." })), - dy: Type.Optional(Type.Number({ description: "Vertical wheel delta for `wheel`." })), - }), - ]), - { - minItems: 1, - maxItems: 16, - description: "Browser steps to execute sequentially in one tool call.", - }, - ), -}); +interface AXProperty { + name: string; + value?: AXValue; +} -type BrowserToolCallParams = Static; -type BrowserStep = BrowserToolCallParams["steps"][number]; +interface AXNode { + nodeId: string; + ignored: boolean; + role?: AXValue; + name?: AXValue; + description?: AXValue; + value?: AXValue; + properties?: AXProperty[]; + parentId?: string; + childIds?: string[]; + backendDOMNodeId?: number; +} -type BrowserSharedParams = { - session?: string; - headed?: boolean; - provider?: Static; - engine?: Static; - sessionName?: string; - profile?: string; - state?: string; - autoConnect?: boolean; - cdp?: string; - device?: string; - allowFileAccess?: boolean; - ignoreHttpsErrors?: boolean; - headers?: Record; -}; +interface GetFullAXTreeResponse { + nodes: AXNode[]; +} -type BrowserToolParams = BrowserSharedParams & { - command: BrowserCommandName; - url?: string; - ref?: string; - text?: string; - value?: string; - values?: string[]; - startRef?: string; - endRef?: string; - files?: string[]; - index?: number; - key?: string; - x?: number; - y?: number; - dx?: number; - dy?: number; - button?: Static; - direction?: Static; - amount?: number; - selector?: string; - target?: string; - ms?: number; - urlPattern?: string; - loadState?: Static; - expression?: string; - waitState?: Static; - downloadPath?: string; - timeout?: number; - interactive?: boolean; - urls?: boolean; - compact?: boolean; - depth?: number; - annotate?: boolean; - full?: boolean; - insertText?: boolean; - newTab?: boolean; -}; +interface ResolveNodeResponse { + object: { objectId?: string }; +} + +interface CallFunctionOnResponse { + exceptionDetails?: { + text?: string; + exception?: { + description?: string; + value?: unknown; + }; + }; +} + +const ACTIONABLE_ROLES = new Set([ + "button", + "checkbox", + "combobox", + "link", + "menuitem", + "menuitemcheckbox", + "menuitemradio", + "option", + "radio", + "scrollbar", + "searchbox", + "slider", + "spinbutton", + "switch", + "tab", + "textbox", + "treeitem", +]); +const TEXT_ROLES = new Set(["heading", "image", "img", "paragraph", "StaticText", "text", "LabelText", "listitem", "cell"]); +const STATE_NAMES = new Set(["checked", "disabled", "editable", "expanded", "focusable", "focused", "pressed", "required", "selected"]); + +const require = createRequire(import.meta.url); +const { chromium } = require("playwright") as { chromium: BrowserType }; function shouldBlockTool(event: ToolCallEvent): boolean { return event.toolName !== BROWSER_TOOL_NAME; } -function sanitizeSessionName(value: string): string { - return value - .toLowerCase() - .replace(/[^a-z0-9]+/g, "-") - .replace(/^-+|-+$/g, "") - .slice(0, 48); +function normalizeRef(ref: string): string { + return ref.startsWith("@") ? ref.slice(1) : ref; } -function getSessionName(ctx: ExtensionContext, params: BrowserSharedParams): string { - if (params.session) { - return params.session; - } - const derived = sanitizeSessionName(ctx.cwd); - return derived ? `${EXTENSION_PREFIX}-${derived}` : `${EXTENSION_PREFIX}-workspace`; +function axValueToString(value: AXValue | undefined): string { + const raw = value?.value; + if (typeof raw === "string") return raw.trim(); + if (typeof raw === "number" || typeof raw === "boolean") return String(raw); + return ""; } -function outputDir(cwd: string): string { - return join(cwd, ".agent-browser", EXTENSION_PREFIX); +function axValueToBoolean(value: AXValue | undefined): boolean { + return value?.value === true || value?.value === "true"; } -function outputPath(cwd: string, toolCallId: string, extension: "png" | "pdf"): string { - return join(outputDir(cwd), `${toolCallId}.${extension}`); +function getProperty(node: AXNode, name: string): AXProperty | undefined { + return node.properties?.find((property) => property.name === name); } -function describeCommand(command: BrowserCommandName, details: string[] = []): string { - const parts = details.filter((detail) => detail.length > 0); - return parts.length > 0 ? `${command} ${parts.join(" ")}` : command; +function hasInterestingState(node: AXNode): boolean { + return (node.properties ?? []).some((property) => STATE_NAMES.has(property.name) && axValueToString(property.value).length > 0); } -function requireStringField( - params: BrowserToolParams, - command: BrowserCommandName, - field: "url" | "ref" | "text" | "value" | "startRef" | "endRef" | "key" | "selector" | "target" | "urlPattern" | "expression" | "downloadPath", -): string { - const value = params[field]; - if (typeof value === "string" && value.length > 0) { - return value; - } - throw new Error(`browser ${command} requires \`${field}\`.`); +function isActionable(node: AXNode): boolean { + const role = axValueToString(node.role); + return ( + ACTIONABLE_ROLES.has(role) || + axValueToBoolean(getProperty(node, "editable")?.value) || + axValueToBoolean(getProperty(node, "focusable")?.value) || + axValueToBoolean(getProperty(node, "selectable")?.value) + ); } -function requireNumberField( - params: BrowserToolParams, - command: BrowserCommandName, - field: "index" | "x" | "y" | "dx" | "dy" | "amount" | "ms" | "timeout" | "depth", -): number { - const value = params[field]; - if (typeof value === "number" && Number.isFinite(value)) { - return value; - } - throw new Error(`browser ${command} requires \`${field}\`.`); +function hasTextContent(node: AXNode): boolean { + return [node.name, node.value, node.description].some((value) => axValueToString(value).length > 0); } -function requireStringArrayField(params: BrowserToolParams, command: BrowserCommandName, field: "files" | "values"): string[] { - const value = params[field]; - if (Array.isArray(value) && value.length > 0 && value.every((entry) => typeof entry === "string" && entry.length > 0)) { - return value; - } - throw new Error(`browser ${command} requires \`${field}\`.`); +function shouldRenderNode(node: AXNode): boolean { + if (node.ignored) return false; + const role = axValueToString(node.role); + if (!role || role === "none" || role === "presentation") return false; + return isActionable(node) || TEXT_ROLES.has(role) || hasTextContent(node) || hasInterestingState(node); } -function summarizeValue(value: string, maxLength = 48): string { - const compact = value.replace(/\s+/g, " ").trim(); - if (compact.length <= maxLength) { - return JSON.stringify(compact); - } - return JSON.stringify(`${compact.slice(0, maxLength - 3)}...`); +function shouldRefNode(node: AXNode): boolean { + return !node.ignored && typeof node.backendDOMNodeId === "number" && isActionable(node); } -function truncateOutput(label: string, text: string): string | undefined { - if (!text.trim()) { - return undefined; +function formatNodeLine(node: AXNode, ref: string | undefined): string { + const role = axValueToString(node.role) || "node"; + const parts = ref ? [`[${ref}]`, role] : [role]; + const name = axValueToString(node.name); + const value = axValueToString(node.value); + const description = axValueToString(node.description); + if (name) parts.push(JSON.stringify(name)); + if (value && value !== name) parts.push(`value=${JSON.stringify(value)}`); + if (description && description !== name) parts.push(`description=${JSON.stringify(description)}`); + for (const property of node.properties ?? []) { + if (!STATE_NAMES.has(property.name)) continue; + const propertyValue = axValueToString(property.value); + if (propertyValue) parts.push(`${property.name}=${propertyValue}`); } - const truncation = truncateTail(text, { maxLines: DEFAULT_MAX_LINES, maxBytes: DEFAULT_MAX_BYTES }); - let content = truncation.content.trim(); - if (!truncation.truncated) { - return content; - } - content += `\n\n[${label} truncated: showing ${truncation.outputLines} of ${truncation.totalLines} lines`; - content += ` (${formatSize(truncation.outputBytes)} of ${formatSize(truncation.totalBytes)})]`; - return content; + return parts.join(" "); } function truncateSnapshot(text: string): string { const truncation = truncateTail(text, { maxLines: DEFAULT_MAX_LINES, maxBytes: DEFAULT_MAX_BYTES }); - if (!truncation.truncated) { - return truncation.content.trim(); - } + if (!truncation.truncated) return truncation.content.trim(); let content = truncation.content.trim(); - content += `\n\n[Snapshot truncated: showing ${truncation.outputLines} of ${truncation.totalLines} lines`; + content += `\n\n[Accessibility tree truncated: showing ${truncation.outputLines} of ${truncation.totalLines} lines`; content += ` (${formatSize(truncation.outputBytes)} of ${formatSize(truncation.totalBytes)})]`; return content; } -function pushBooleanFlag(argv: string[], flag: string, value: boolean | undefined): void { - if (value === undefined) { - return; - } - argv.push(flag); - if (!value) { - argv.push("false"); +async function ensureDriver(state: DriverState, headed: boolean | undefined): Promise<{ page: Page; cdp: CdpSession }> { + if (state.page && state.cdp) { + if (headed !== undefined && headed !== state.headed) throw new Error("Browser is already running; `headed` can only be set on the first browser call."); + return { page: state.page, cdp: state.cdp }; } + state.headed = headed ?? false; + state.browser = await chromium.launch({ headless: !state.headed }); + state.context = await state.browser.newContext(); + state.page = await state.context.newPage(); + state.cdp = await state.context.newCDPSession(state.page); + return { page: state.page, cdp: state.cdp }; } -function buildSharedArgs(params: BrowserSharedParams, sessionName: string): string[] { - const argv = ["--session", sessionName]; - pushBooleanFlag(argv, "--headed", params.headed); - if (params.provider) argv.push("--provider", params.provider); - if (params.engine) argv.push("--engine", params.engine); - if (params.sessionName) argv.push("--session-name", params.sessionName); - if (params.profile) argv.push("--profile", params.profile); - if (params.state) argv.push("--state", params.state); - pushBooleanFlag(argv, "--auto-connect", params.autoConnect); - if (params.cdp) argv.push("--cdp", params.cdp); - if (params.device) argv.push("--device", params.device); - pushBooleanFlag(argv, "--allow-file-access", params.allowFileAccess); - pushBooleanFlag(argv, "--ignore-https-errors", params.ignoreHttpsErrors); - if (params.headers && Object.keys(params.headers).length > 0) { - argv.push("--headers", JSON.stringify(params.headers)); - } - return argv; +async function closeDriver(state: DriverState): Promise { + const { browser, cdp, context } = state; + state.browser = undefined; + state.context = undefined; + state.page = undefined; + state.cdp = undefined; + state.refMap.clear(); + await cdp?.detach().catch(() => undefined); + await context?.close().catch(() => undefined); + await browser?.close().catch(() => undefined); } -function getSelectValues(params: BrowserToolParams): string[] { - if (Array.isArray(params.values) && params.values.length > 0) { - return requireStringArrayField(params, "select", "values"); - } - if (typeof params.value === "string" && params.value.length > 0) { - return [params.value]; - } - throw new Error("browser select requires `value` or `values`."); +async function resolveObjectId(cdp: CdpSession, backendDOMNodeId: number): Promise { + const response = await cdp.send("DOM.resolveNode", { backendNodeId: backendDOMNodeId }); + if (!response.object.objectId) throw new Error("Ref no longer resolves to a DOM node."); + return response.object.objectId; } -function buildWaitCommand(params: BrowserToolParams): PreparedCommand { - const modes = [ - params.ms !== undefined ? "ms" : undefined, - typeof params.target === "string" && params.target.length > 0 ? "target" : undefined, - typeof params.text === "string" && params.text.length > 0 ? "text" : undefined, - typeof params.urlPattern === "string" && params.urlPattern.length > 0 ? "urlPattern" : undefined, - params.loadState ? "loadState" : undefined, - typeof params.expression === "string" && params.expression.length > 0 ? "expression" : undefined, - typeof params.downloadPath === "string" && params.downloadPath.length > 0 ? "downloadPath" : undefined, - ].filter(Boolean); - if (modes.length !== 1) { - throw new Error( - "browser wait requires exactly one of `ms`, `target`, `text`, `urlPattern`, `loadState`, `expression`, or `downloadPath`.", - ); - } - if (params.waitState && !params.target) { - throw new Error("browser wait only accepts `waitState` when `target` is provided."); - } - if (params.timeout !== undefined && !params.downloadPath) { - throw new Error("browser wait only accepts `timeout` when `downloadPath` is provided."); - } - if (params.ms !== undefined) { - const ms = requireNumberField(params, "wait", "ms"); - return { argv: ["wait", String(ms)], summary: describeCommand("wait", [`${ms}ms`]) }; - } - if (params.target) { - const target = requireStringField(params, "wait", "target"); - const argv = ["wait", target]; - if (params.waitState) { - argv.push("--state", params.waitState); +async function clearRefMarkers(page: Page): Promise { + await page.evaluate((attribute) => { + for (const element of Array.from(document.querySelectorAll(`[${attribute}]`))) { + element.removeAttribute(attribute); } - return { - argv, - summary: describeCommand("wait", [target, params.waitState ? `state=${params.waitState}` : ""]), - }; - } - if (params.text) { - const text = requireStringField(params, "wait", "text"); - return { - argv: ["wait", "--text", text], - summary: describeCommand("wait", [`text=${summarizeValue(text)}`]), - }; - } - if (params.urlPattern) { - const urlPattern = requireStringField(params, "wait", "urlPattern"); - return { - argv: ["wait", "--url", urlPattern], - summary: describeCommand("wait", [`url=${summarizeValue(urlPattern)}`]), - }; - } - if (params.loadState) { - return { - argv: ["wait", "--load", params.loadState], - summary: describeCommand("wait", [`load=${params.loadState}`]), - }; - } - if (params.expression) { - requireStringField(params, "wait", "expression"); - return { - argv: ["wait", "--fn", params.expression], - summary: describeCommand("wait", ["fn"]), - }; - } - const downloadPath = requireStringField(params, "wait", "downloadPath"); - const argv = ["wait", "--download", downloadPath]; - if (params.timeout !== undefined) { - argv.push("--timeout", String(requireNumberField(params, "wait", "timeout"))); + }, REF_ATTRIBUTE); +} + +function locatorForRef(page: Page, ref: string): Locator { + return page.locator(`[${REF_ATTRIBUTE}="${ref}"]`); +} + +async function markRef(state: DriverState, page: Page, ref: string): Promise { + if (!state.cdp) throw new Error("Browser is not running."); + const normalizedRef = normalizeRef(ref); + const backendDOMNodeId = state.refMap.get(normalizedRef); + if (backendDOMNodeId === undefined) throw new Error(`Unknown browser ref ${ref}. Use a ref from the latest browser result.`); + await clearRefMarkers(page); + const objectId = await resolveObjectId(state.cdp, backendDOMNodeId); + const response = await state.cdp.send("Runtime.callFunctionOn", { + objectId, + functionDeclaration: `function (attribute, ref) { + if (!(this instanceof Element)) throw new Error("Ref does not resolve to an Element."); + this.setAttribute(attribute, ref); + }`, + arguments: [{ value: REF_ATTRIBUTE }, { value: normalizedRef }], + awaitPromise: true, + }); + if (response.exceptionDetails) { + throw new Error(response.exceptionDetails.exception?.description ?? response.exceptionDetails.text ?? "Failed to mark browser ref."); } - return { - argv, - summary: describeCommand("wait", ["download"]), - }; + return locatorForRef(page, normalizedRef); } -function buildCommand(params: BrowserToolParams, sessionName: string, toolCallId: string, cwd: string): PreparedCommand { - const argv = buildSharedArgs(params, sessionName); +async function clickRef(state: DriverState, page: Page, ref: string): Promise { + await (await markRef(state, page, ref)).click(); +} - switch (params.command) { - case "goto": { - const url = requireStringField(params, "goto", "url"); - argv.push("open", url); - return { argv, summary: describeCommand("goto", [summarizeValue(url, 72)]) }; - } - case "click": { - const ref = requireStringField(params, "click", "ref"); - argv.push("click", ref); - if (params.newTab) argv.push("--new-tab"); - return { argv, summary: describeCommand("click", [ref]) }; - } - case "type": { - const text = requireStringField(params, "type", "text"); - argv.push("keyboard", params.insertText ? "inserttext" : "type", text); - return { - argv, - summary: describeCommand("type", [params.insertText ? "inserttext" : "", `${text.length} chars`]), - }; - } - case "fill": { - const ref = requireStringField(params, "fill", "ref"); - const text = requireStringField(params, "fill", "text"); - argv.push("fill", ref, text); - return { argv, summary: describeCommand("fill", [ref, `${text.length} chars`]) }; - } - case "select": { - const ref = requireStringField(params, "select", "ref"); - const values = getSelectValues(params); - argv.push("select", ref, ...values); - return { - argv, - summary: describeCommand("select", [ref, values.length === 1 ? summarizeValue(values[0]) : `${values.length} values`]), - }; - } - case "check": { - const ref = requireStringField(params, "check", "ref"); - argv.push("check", ref); - return { argv, summary: describeCommand("check", [ref]) }; - } - case "uncheck": { - const ref = requireStringField(params, "uncheck", "ref"); - argv.push("uncheck", ref); - return { argv, summary: describeCommand("uncheck", [ref]) }; - } - case "hover": { - const ref = requireStringField(params, "hover", "ref"); - argv.push("hover", ref); - return { argv, summary: describeCommand("hover", [ref]) }; - } - case "drag": { - const startRef = requireStringField(params, "drag", "startRef"); - const endRef = requireStringField(params, "drag", "endRef"); - argv.push("drag", startRef, endRef); - return { argv, summary: describeCommand("drag", [startRef, "->", endRef]) }; - } - case "upload": { - const ref = requireStringField(params, "upload", "ref"); - const files = requireStringArrayField(params, "upload", "files"); - argv.push("upload", ref, ...files); - return { argv, summary: describeCommand("upload", [ref, `${files.length} file${files.length === 1 ? "" : "s"}`]) }; - } - case "scroll": { - const direction = params.direction ?? "down"; - argv.push("scroll", direction); - if (params.amount !== undefined) { - argv.push(String(requireNumberField(params, "scroll", "amount"))); - } - if (params.selector) { - argv.push("--selector", params.selector); - } - return { - argv, - summary: describeCommand("scroll", [direction, params.amount !== undefined ? String(params.amount) : ""]), - }; - } - case "wait": { - const waitCommand = buildWaitCommand(params); - return { - argv: [...argv, ...waitCommand.argv], - summary: waitCommand.summary, - }; - } - case "close": - argv.push("close"); - return { argv, summary: describeCommand("close") }; - case "snapshot": { - argv.push("snapshot"); - if (params.interactive) argv.push("--interactive"); - if (params.urls) argv.push("--urls"); - if (params.compact) argv.push("--compact"); - if (params.depth !== undefined) argv.push("--depth", String(requireNumberField(params, "snapshot", "depth"))); - if (params.selector) argv.push("--selector", params.selector); - return { argv, summary: describeCommand("snapshot", [params.interactive ? "interactive" : ""]) }; - } - case "screenshot": { - const path = outputPath(cwd, toolCallId, "png"); - argv.push("screenshot"); - if (params.ref) argv.push(params.ref); - if (params.full) argv.push("--full"); - if (params.annotate) argv.push("--annotate"); - argv.push(path); - return { - argv, - summary: describeCommand("screenshot", [ - params.ref ?? "", - params.full ? "full" : "", - params.annotate ? "annotate" : "", - ]), - outputPath: path, - }; - } - case "pdf": { - const path = outputPath(cwd, toolCallId, "pdf"); - argv.push("pdf", path); - return { argv, summary: describeCommand("pdf"), outputPath: path }; - } - case "go-back": - argv.push("back"); - return { argv, summary: describeCommand("go-back") }; - case "go-forward": - argv.push("forward"); - return { argv, summary: describeCommand("go-forward") }; - case "reload": - argv.push("reload"); - return { argv, summary: describeCommand("reload") }; - case "tab-list": - argv.push("tab", "list"); - return { argv, summary: describeCommand("tab-list") }; - case "tab-new": - argv.push("tab", "new"); - if (params.url) argv.push(params.url); - return { argv, summary: describeCommand("tab-new", params.url ? [summarizeValue(params.url, 72)] : []) }; - case "tab-select": { - const index = requireNumberField(params, "tab-select", "index"); - argv.push("tab", String(index)); - return { argv, summary: describeCommand("tab-select", [String(index)]) }; - } - case "tab-close": - argv.push("tab", "close"); - if (params.index !== undefined) argv.push(String(params.index)); - return { argv, summary: describeCommand("tab-close", params.index !== undefined ? [String(params.index)] : []) }; - case "press": { - const key = requireStringField(params, "press", "key"); - argv.push("press", key); - return { argv, summary: describeCommand("press", [key]) }; - } - case "keydown": { - const key = requireStringField(params, "keydown", "key"); - argv.push("keydown", key); - return { argv, summary: describeCommand("keydown", [key]) }; - } - case "keyup": { - const key = requireStringField(params, "keyup", "key"); - argv.push("keyup", key); - return { argv, summary: describeCommand("keyup", [key]) }; - } - case "mousemove": { - const x = requireNumberField(params, "mousemove", "x"); - const y = requireNumberField(params, "mousemove", "y"); - argv.push("mouse", "move", String(x), String(y)); - return { argv, summary: describeCommand("mousemove", [String(x), String(y)]) }; - } - case "mousedown": - argv.push("mouse", "down"); - if (params.button) argv.push(params.button); - return { argv, summary: describeCommand("mousedown", params.button ? [params.button] : []) }; - case "mouseup": - argv.push("mouse", "up"); - if (params.button) argv.push(params.button); - return { argv, summary: describeCommand("mouseup", params.button ? [params.button] : []) }; - case "mousewheel": { - const dy = requireNumberField(params, "mousewheel", "dy"); - argv.push("mouse", "wheel", String(dy)); - if (params.dx !== undefined) argv.push(String(params.dx)); - return { - argv, - summary: describeCommand("mousewheel", [String(dy), params.dx !== undefined ? String(params.dx) : ""]), - }; - } - } +async function fillRef(state: DriverState, page: Page, ref: string, text: string): Promise { + await (await markRef(state, page, ref)).fill(text); } -function parseCliResponse(stdout: string): AgentBrowserResponse | undefined { - const trimmed = stdout.trim(); - if (!trimmed) { - return undefined; - } - try { - return JSON.parse(trimmed) as AgentBrowserResponse; - } catch { - return undefined; - } +async function focusRef(state: DriverState, page: Page, ref: string): Promise { + await (await markRef(state, page, ref)).focus(); } -function formatUnknown(label: string, value: unknown): string | undefined { - if (value === undefined || value === null) { - return undefined; - } - if (typeof value === "string") { - return truncateOutput(label, value); - } - try { - return truncateOutput(label, JSON.stringify(value, null, 2)); - } catch { - return truncateOutput(label, String(value)); +function scrollDelta(direction: ScrollDirection | undefined, amount: number | undefined): { x: number; y: number } { + const value = amount ?? DEFAULT_SCROLL_Y; + switch (direction ?? "down") { + case "up": + return { x: 0, y: -value }; + case "left": + return { x: -value, y: 0 }; + case "right": + return { x: value, y: 0 }; + case "down": + return { x: 0, y: value }; } + throw new Error(`Unsupported scroll direction: ${String(direction)}`); } -function formatCliError(response: AgentBrowserResponse | undefined, stdout: string, stderr: string): string { - const parts = [ - formatUnknown("error", response?.error), - formatUnknown("warning", response?.warning), - truncateOutput("stderr", stderr), - !response ? truncateOutput("stdout", stdout) : undefined, - ].filter((value): value is string => typeof value === "string" && value.length > 0); - return parts.join("\n\n") || "agent-browser failed"; +async function scrollRef(state: DriverState, page: Page, ref: string, direction: ScrollDirection | undefined, amount: number | undefined): Promise { + const delta = scrollDelta(direction, amount); + const locator = await markRef(state, page, ref); + await locator.evaluate((element, value) => { + element.scrollIntoView({ block: "center", inline: "center" }); + element.scrollBy(value.x, value.y); + }, delta); } -function extractResponseSections( - response: AgentBrowserResponse | undefined, - stdout: string, - stderr: string, - prepared: PreparedCommand, -): { - output?: string; - warnings?: string; - snapshot?: string; -} { - const data = response?.data; - let output: string | undefined; - let snapshot: string | undefined; - - if (data && typeof data === "object" && !Array.isArray(data)) { - const record = { ...(data as Record) }; - if (typeof record.snapshot === "string") { - snapshot = truncateSnapshot(record.snapshot); - delete record.snapshot; - if (record.refs && typeof record.refs === "object" && !Array.isArray(record.refs)) { - record.refCount = Object.keys(record.refs as Record).length; - delete record.refs; - } - } - output = Object.keys(record).length > 0 ? formatUnknown("output", record) : undefined; - } else if (response) { - output = formatUnknown("output", data); - } else if (stdout.trim()) { - output = truncateOutput("stdout", stdout); - } +async function scrollPage(page: Page, direction: ScrollDirection | undefined, amount: number | undefined): Promise { + const delta = scrollDelta(direction, amount); + await page.evaluate((value) => { + const target = globalThis as { scrollBy?: (x: number, y: number) => void }; + target.scrollBy?.(value.x, value.y); + }, delta); +} - if (!output && prepared.outputPath) { - output = formatUnknown("output", { path: prepared.outputPath }); - } +async function renderAccessibilityTree(page: Page, cdp: CdpSession, state: DriverState): Promise<{ text: string; refCount: number }> { + await clearRefMarkers(page); + const response = await cdp.send("Accessibility.getFullAXTree"); + const nodesById = new Map(response.nodes.map((node) => [node.nodeId, node])); + const roots = response.nodes.filter((node) => !node.parentId || !nodesById.has(node.parentId)); + const lines: string[] = []; + const nextRefMap = new Map(); + let refIndex = 0; + const visited = new Set(); - const warnings = - [ - formatUnknown("warning", response?.warning), - truncateOutput("stderr", stderr), - ].filter((value): value is string => typeof value === "string" && value.length > 0).join("\n\n") || undefined; + const visit = (node: AXNode, depth: number): void => { + if (visited.has(node.nodeId)) return; + visited.add(node.nodeId); + const renderSelf = shouldRenderNode(node); + const ref = renderSelf && shouldRefNode(node) ? `e${++refIndex}` : undefined; + if (ref && node.backendDOMNodeId !== undefined) nextRefMap.set(ref, node.backendDOMNodeId); + if (renderSelf) lines.push(`${" ".repeat(depth)}- ${formatNodeLine(node, ref)}`); + const childDepth = renderSelf ? depth + 1 : depth; + for (const childId of node.childIds ?? []) { + const child = nodesById.get(childId); + if (child) visit(child, childDepth); + } + }; - return { output, warnings, snapshot }; + for (const root of roots) visit(root, 0); + state.refMap = nextRefMap; + return { text: truncateSnapshot(lines.join("\n") || "(accessibility tree is empty)"), refCount: nextRefMap.size }; } -async function resolveCli(pi: ExtensionAPI, state: DriverState, ctx: ExtensionContext): Promise { - if (state.resolvedCli) { - return state.resolvedCli; - } - const result = await pi.exec( - "bash", - [ - "-lc", - "if command -v agent-browser >/dev/null 2>&1; then printf 'agent-browser'; elif command -v npx >/dev/null 2>&1; then printf 'npx'; else exit 1; fi", - ], - { cwd: ctx.cwd, signal: ctx.signal, timeout: 5000 }, - ); - if (result.code !== 0) { - throw new Error("agent-browser is not installed. Install it with npm install -g agent-browser && agent-browser install."); +function actionSummary(action: BrowserAction): string { + switch (action.type) { + case "goto": + return `Opened ${JSON.stringify(action.url)}`; + case "snapshot": + return "Captured accessibility tree"; + case "click": + return `Clicked ${action.ref}`; + case "fill": + return `Filled ${action.ref} with ${action.text.length} characters`; + case "press": + return action.ref ? `Focused ${action.ref} and pressed ${action.key}` : `Pressed ${action.key}`; + case "scroll": + return action.ref ? `Scrolled ${action.ref}` : "Scrolled page"; + case "wait": + return action.loadState ? `Waited for ${action.loadState}` : `Waited ${action.ms ?? DEFAULT_WAIT_MS}ms`; } - const command = result.stdout.trim(); - state.resolvedCli = command === "agent-browser" ? { command, baseArgs: [] } : { command: "npx", baseArgs: ["-y", "agent-browser"] }; - return state.resolvedCli; + throw new Error("Unsupported browser action."); } -async function execCli(pi: ExtensionAPI, state: DriverState, ctx: ExtensionContext, argv: string[]): Promise { - const cli = await resolveCli(pi, state, ctx); - const result = await pi.exec(cli.command, [...cli.baseArgs, "--json", ...argv], { - cwd: ctx.cwd, - signal: ctx.signal, - timeout: 120000, - }); - const response = parseCliResponse(result.stdout); - if (result.code !== 0 || response?.success === false) { - throw new Error(formatCliError(response, result.stdout, result.stderr)); +async function executeAction(state: DriverState, page: Page, action: BrowserAction): Promise { + switch (action.type) { + case "goto": + await page.goto(action.url, { waitUntil: "domcontentloaded" }); + break; + case "snapshot": + break; + case "click": + await clickRef(state, page, action.ref); + break; + case "fill": + await fillRef(state, page, action.ref, action.text); + break; + case "press": + if (action.ref) await focusRef(state, page, action.ref); + await page.keyboard.press(action.key); + break; + case "scroll": + if (action.ref) await scrollRef(state, page, action.ref, action.direction, action.amount); + else await scrollPage(page, action.direction, action.amount); + break; + case "wait": + if (action.ms !== undefined && action.loadState !== undefined) throw new Error("browser wait accepts either `ms` or `loadState`, not both."); + if (action.loadState) await page.waitForLoadState(action.loadState); + else await page.waitForTimeout(action.ms ?? DEFAULT_WAIT_MS); + break; } - return { - stdout: result.stdout, - stderr: result.stderr, - response, - }; -} - -async function readImage(path: string): Promise { - const data = await readFile(path, { encoding: "base64" }); - return { type: "image", mimeType: "image/png", data }; -} - -function formatStepContent(stepNumber: number, summary: string, stdout?: string, stderr?: string, snapshot?: string): string { - const sections = [`${stepNumber}. ${summary}`]; - if (stdout) sections.push(`Output:\n${stdout}`); - if (stderr) sections.push(`Warnings:\n${stderr}`); - if (snapshot) sections.push(`Snapshot:\n${snapshot}`); - return sections.join("\n\n"); } async function runBrowserTool( - pi: ExtensionAPI, state: DriverState, - ctx: ExtensionContext, toolCallId: string, params: BrowserToolCallParams, -): Promise<{ content: (TextContent | ImageContent)[]; details: BrowserToolDetails }> { - if (state.activeToolCallId && state.activeToolCallId !== toolCallId) { - throw new Error("Browser tool does not allow parallel execution."); - } +): Promise<{ content: TextContent[]; details: BrowserToolDetails }> { + if (state.activeToolCallId && state.activeToolCallId !== toolCallId) throw new Error("Browser tool does not allow parallel execution."); state.activeToolCallId = toolCallId; try { - await mkdir(outputDir(ctx.cwd), { recursive: true }); - const shared = getSharedParams(params); - const sessionName = getSessionName(ctx, shared); - const details: BrowserToolDetails = { - sessionName, - steps: [], - }; - const contentSections: string[] = []; - let image: ImageContent | undefined; - for (const [index, step] of params.steps.entries()) { - const stepParams = toBrowserParams(step, shared); - const prepared = buildCommand(stepParams, sessionName, `${toolCallId}-${index + 1}`, ctx.cwd); - const output = await execCli(pi, state, ctx, prepared.argv); - const sections = extractResponseSections(output.response, output.stdout, output.stderr, prepared); - if (step.type === "screenshot" && prepared.outputPath) { - image = await readImage(prepared.outputPath); - } - details.steps.push({ - type: step.type, - command: stepParams.command, - invocation: prepared.argv, - summary: prepared.summary, - stdout: sections.output, - stderr: sections.warnings, - outputPath: prepared.outputPath, - }); - contentSections.push(formatStepContent(index + 1, prepared.summary, sections.output, sections.warnings, sections.snapshot)); + const { page, cdp } = await ensureDriver(state, params.headed); + const summaries: Array<{ type: BrowserAction["type"]; summary: string }> = []; + for (const action of params.actions) { + await executeAction(state, page, action); + summaries.push({ type: action.type, summary: actionSummary(action) }); } - const content: (TextContent | ImageContent)[] = [{ type: "text", text: contentSections.join("\n\n") }]; - if (image) { - content.push(image); - } - return { content, details }; + const [title, snapshot] = await Promise.all([page.title(), renderAccessibilityTree(page, cdp, state)]); + const url = page.url(); + const sections = [ + summaries.map((entry, index) => `${index + 1}. ${entry.summary}`).join("\n"), + `URL: ${url}`, + `Title: ${title || "(untitled)"}`, + `Refs: ${snapshot.refCount}`, + `Accessibility tree:\n${snapshot.text}`, + ].filter((section) => section.length > 0); + return { + content: [{ type: "text", text: sections.join("\n\n") }], + details: { actions: summaries, title, url, refCount: snapshot.refCount }, + }; } finally { - if (state.activeToolCallId === toolCallId) { - state.activeToolCallId = undefined; - } - } -} - -function getSharedParams(params: BrowserSharedParams): BrowserSharedParams { - return { - session: params.session, - headed: params.headed, - provider: params.provider, - engine: params.engine, - sessionName: params.sessionName, - profile: params.profile, - state: params.state, - autoConnect: params.autoConnect, - cdp: params.cdp, - device: params.device, - allowFileAccess: params.allowFileAccess, - ignoreHttpsErrors: params.ignoreHttpsErrors, - headers: params.headers, - }; -} - -function toBrowserParams(step: BrowserStep, shared: BrowserSharedParams): BrowserToolParams { - switch (step.type) { - case "goto": - return { - ...shared, - command: "goto", - url: step.url, - }; - case "click": - return { - ...shared, - command: "click", - ref: step.ref, - newTab: step.newTab, - }; - case "type": - return { - ...shared, - command: "type", - text: step.text, - insertText: step.insertText, - }; - case "fill": - return { - ...shared, - command: "fill", - ref: step.ref, - text: step.text, - }; - case "select": - return { - ...shared, - command: "select", - ref: step.ref, - value: step.value, - values: step.values, - }; - case "check": - return { - ...shared, - command: "check", - ref: step.ref, - }; - case "uncheck": - return { - ...shared, - command: "uncheck", - ref: step.ref, - }; - case "hover": - return { - ...shared, - command: "hover", - ref: step.ref, - }; - case "drag": - return { - ...shared, - command: "drag", - startRef: step.startRef, - endRef: step.endRef, - }; - case "upload": - return { - ...shared, - command: "upload", - ref: step.ref, - files: step.files, - }; - case "scroll": - return { - ...shared, - command: "scroll", - direction: step.direction, - amount: step.amount, - selector: step.selector, - }; - case "wait": - return { - ...shared, - command: "wait", - ms: step.ms, - target: step.target, - text: step.text, - urlPattern: step.urlPattern, - loadState: step.loadState, - expression: step.expression, - waitState: step.waitState, - downloadPath: step.downloadPath, - timeout: step.timeout, - }; - case "close": - return { - ...shared, - command: "close", - }; - case "snapshot": - return { - ...shared, - command: "snapshot", - interactive: step.interactive, - urls: step.urls, - compact: step.compact, - depth: step.depth, - selector: step.selector, - }; - case "screenshot": - return { - ...shared, - command: "screenshot", - ref: step.ref, - full: step.full, - annotate: step.annotate, - }; - case "pdf": - return { - ...shared, - command: "pdf", - }; - case "navigation": - return { - ...shared, - command: step.action === "back" ? "go-back" : step.action === "forward" ? "go-forward" : "reload", - }; - case "tabs": - switch (step.action) { - case "list": - return { ...shared, command: "tab-list" }; - case "new": - return { ...shared, command: "tab-new", url: step.url }; - case "select": - if (step.index === undefined) { - throw new Error("browser steps `tabs` with action `select` requires `index`."); - } - return { ...shared, command: "tab-select", index: step.index }; - case "close": - return { ...shared, command: "tab-close", index: step.index }; - } - case "keyboard": - return { - ...shared, - command: step.action === "press" ? "press" : step.action === "down" ? "keydown" : "keyup", - key: step.key, - }; - case "mouse": - switch (step.action) { - case "move": - if (step.x === undefined || step.y === undefined) { - throw new Error("browser steps `mouse` with action `move` requires `x` and `y`."); - } - return { ...shared, command: "mousemove", x: step.x, y: step.y }; - case "down": - return { ...shared, command: "mousedown", button: step.button }; - case "up": - return { ...shared, command: "mouseup", button: step.button }; - case "wheel": - if (step.dy === undefined) { - throw new Error("browser steps `mouse` with action `wheel` requires `dy`."); - } - return { ...shared, command: "mousewheel", dx: step.dx, dy: step.dy }; - } + if (state.activeToolCallId === toolCallId) state.activeToolCallId = undefined; } } export default function registerBrowserUseExtension(pi: ExtensionAPI): void { - const state: DriverState = {}; + const state: DriverState = { refMap: new Map() }; pi.registerTool({ name: BROWSER_TOOL_NAME, label: "Browser", - description: "Interact with the browser by executing one or more sequential steps with agent-browser.", + description: "Interact with Chromium using a text-only accessibility tree and ref-based actions.", promptSnippet: DEFAULT_BROWSER_PROMPT_SNIPPET, promptGuidelines: DEFAULT_BROWSER_GUIDELINES, parameters: browserToolParameters, - async execute(toolCallId, params, _signal, _onUpdate, ctx) { - return runBrowserTool(pi, state, ctx, toolCallId, params); + async execute(toolCallId, params) { + return runBrowserTool(state, toolCallId, params); }, }); @@ -1165,13 +516,12 @@ export default function registerBrowserUseExtension(pi: ExtensionAPI): void { pi.setActiveTools([BROWSER_TOOL_NAME]); }); + pi.on("session_shutdown", async () => { + await closeDriver(state); + }); + pi.on("tool_call", async (event) => { - if (!shouldBlockTool(event)) { - return; - } - return { - block: true, - reason: `${EXTENSION_NAME} enforces browser-only mode. ${event.toolName} is disabled.`, - }; + if (!shouldBlockTool(event)) return; + return { block: true, reason: `${EXTENSION_NAME} enforces browser-only mode. ${event.toolName} is disabled.` }; }); } diff --git a/package-lock.json b/package-lock.json index 5660d7dcb00..486c3c03252 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,8 @@ "dependencies": { "@mariozechner/jiti": "^2.6.5", "@mariozechner/pi-coding-agent": "^0.30.2", - "get-east-asian-width": "^1.4.0" + "get-east-asian-width": "^1.4.0", + "playwright": "^1.59.1" }, "devDependencies": { "@biomejs/biome": "2.3.5", @@ -7027,6 +7028,50 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/playwright": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", + "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.59.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz", + "integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/postcss": { "version": "8.5.12", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.12.tgz", diff --git a/package.json b/package.json index 00d964141d6..1c82b87a6f4 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,8 @@ "dependencies": { "@mariozechner/jiti": "^2.6.5", "@mariozechner/pi-coding-agent": "^0.30.2", - "get-east-asian-width": "^1.4.0" + "get-east-asian-width": "^1.4.0", + "playwright": "^1.59.1" }, "overrides": { "rimraf": "6.1.2",