From d430e03ccc8ddd0d2f686c7f45a27375da5489d8 Mon Sep 17 00:00:00 2001 From: EdgeStorage Date: Thu, 28 May 2026 21:31:04 +0800 Subject: [PATCH] feat: add playwright runtime support --- README.md | 24 +- README.zh-CN.md | 24 +- extension/runtime/click-routing.ts | 2 + extension/runtime/debugger-executor.ts | 4 + extension/runtime/execution-helpers.ts | 7 + .../injected/playwright-locator.injected.ts | 585 +++++++++ .../playwright-shim-helpers.injected.ts | 257 ++++ .../playwright-shim-types.injected.ts | 18 + .../injected/playwright-shim.injected.ts | 1090 +++++++++++++++++ .../injected/script-runtime.injected.ts | 56 +- extension/runtime/managed-input-bridge.ts | 199 ++- lib/cli-parser.ts | 20 +- package.json | 1 + pnpm-lock.yaml | 10 + scripts/generate-script-runtime.ts | 6 + skills/web-cap/SKILL.md | 179 ++- tests/background-click-routing.test.ts | 82 ++ tests/execution-helpers.test.ts | 84 ++ tests/web-cap-cli.test.ts | 9 + 19 files changed, 2529 insertions(+), 128 deletions(-) create mode 100644 extension/runtime/injected/playwright-locator.injected.ts create mode 100644 extension/runtime/injected/playwright-shim-helpers.injected.ts create mode 100644 extension/runtime/injected/playwright-shim-types.injected.ts create mode 100644 extension/runtime/injected/playwright-shim.injected.ts diff --git a/README.md b/README.md index e74e9fa..3bbd9e5 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,22 @@ Web Cap is a local-first browser automation toolkit for agents. It lets agents i Agents interact with Web Cap through the `web-cap` CLI. The CLI manages the required local runtime automatically, so users do not need a separate startup command. +## Quick Use + +1. Install the Web Cap skill with the `skills` CLI: + + ```bash + npx skills add edgestorage/web-cap + ``` + +2. Install the Web Cap browser extension: + + - Open the latest GitHub Release. + - Download the Chrome extension zip asset, named like `*chrome*.zip`. + - Open `chrome://extensions` in Chrome. + - Enable Developer mode. + - Drag the downloaded zip file into the extensions page. + ## Features - Browser extension runtime for real Chrome/Firefox tabs. @@ -80,7 +96,7 @@ The browser extension connects to the local runtime and executes commands agains - pnpm 9.x - A Chromium-based browser or Firefox for extension development -## Quick Start +## Development Quick Start Install dependencies: @@ -110,12 +126,6 @@ pnpm cli script-search "inspect page" --type read --site generic-web pnpm cli script-get builtin.page.inspect ``` -Install the Web Cap agent skill with the `skills` CLI: - -```bash -npx skills add edgestorage/web-cap --skill web-cap -a codex -``` - A typical agent flow is: 1. Use `script-search` to find a reusable script. diff --git a/README.zh-CN.md b/README.zh-CN.md index 59a65bf..13f0423 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -6,6 +6,22 @@ Web Cap 是一个本地优先的浏览器自动化工具，面向 agent 使用 Agent 通过 `web-cap` CLI 使用 Web Cap。CLI 会自动管理所需的本地运行时，用户不需要额外的启动命令。 +## 快速使用 + +1. 通过 `skills` CLI 安装 Web Cap skill： + + ```bash + npx skills add edgestorage/web-cap + ``` + +2. 安装 Web Cap 浏览器扩展： + + - 打开最新的 GitHub Release。 + - 下载 Chrome 扩展 zip 产物，文件名类似 `*chrome*.zip`。 + - 在 Chrome 中打开 `chrome://extensions`。 + - 开启开发者模式。 + - 将下载的 zip 文件拖到扩展程序页面中。 + ## 功能特性 - 面向真实 Chrome/Firefox 标签页的浏览器扩展运行时。 @@ -80,7 +96,7 @@ Browser extension - pnpm 9.x - 用于扩展开发的 Chromium 系浏览器或 Firefox -## 快速开始 +## 开发快速开始 安装依赖： @@ -110,12 +126,6 @@ pnpm cli script-search "inspect page" --type read --site generic-web pnpm cli script-get builtin.page.inspect ``` -通过 `skills` CLI 安装 Web Cap agent skill： - -```bash -npx skills add edgestorage/web-cap --skill web-cap -a codex -``` - 典型 agent 流程是： 1. 用 `script-search` 查找可复用脚本。 diff --git a/extension/runtime/click-routing.ts b/extension/runtime/click-routing.ts index e4ea172..dce6614 100644 --- a/extension/runtime/click-routing.ts +++ b/extension/runtime/click-routing.ts @@ -75,6 +75,8 @@ function scriptUsesManagedClick(code: string): boolean { } return ( + /\bpage\s*\.\s*mouse\b/.test(code) || + /\.\s*dragTo\s*\(/.test(code) || /\.\s*click\s*\(/.test(code) || /new\s+MouseEvent\s*\(/.test(code) || /dispatchEvent\s*\(\s*new\s+MouseEvent/.test(code) diff --git a/extension/runtime/debugger-executor.ts b/extension/runtime/debugger-executor.ts index 88a791a..34bd838 100644 --- a/extension/runtime/debugger-executor.ts +++ b/extension/runtime/debugger-executor.ts @@ -51,6 +51,8 @@ export class DebuggerScriptExecutor { await this.managedInputBridgeFactory.createManagedKeyboardBridge(target, executionScope); const windowBridge = await this.managedInputBridgeFactory.createManagedWindowBridge(target, executionScope); + const browserBridge = + await this.managedInputBridgeFactory.createManagedBrowserBridge(target, executionScope); let evaluation: DebuggerEvaluateResult; try { evaluation = await this.client.sendCommand(target, 'Runtime.evaluate', { @@ -59,6 +61,7 @@ export class DebuggerScriptExecutor { managedKeyboardBridgeFunctionName: keyboardBridge.bridgeFunctionName, managedWindowBridgeFunctionName: windowBridge.bridgeFunctionName, managedTimerBridgeFunctionName: timerBridge.bridgeFunctionName, + managedBrowserBridgeFunctionName: browserBridge.bridgeFunctionName, }), awaitPromise: true, returnByValue: true, @@ -66,6 +69,7 @@ export class DebuggerScriptExecutor { allowUnsafeEvalBlockedByCSP: true, }); } finally { + await browserBridge.dispose(); await timerBridge.dispose(); await windowBridge.dispose(); await keyboardBridge.dispose(); diff --git a/extension/runtime/execution-helpers.ts b/extension/runtime/execution-helpers.ts index c4a3a3d..dc5c0d2 100644 --- a/extension/runtime/execution-helpers.ts +++ b/extension/runtime/execution-helpers.ts @@ -16,6 +16,7 @@ export interface ScriptExecutionExpressionOptions { managedKeyboardBridgeFunctionName?: string; managedWindowBridgeFunctionName?: string; managedTimerBridgeFunctionName?: string; + managedBrowserBridgeFunctionName?: string; } export function scriptToFunctionExpression(code: string): string { @@ -323,6 +324,10 @@ export function buildScriptExecutionExpression( options.managedTimerBridgeFunctionName === undefined ? null : String(options.managedTimerBridgeFunctionName); + const managedBrowserBridgeFunctionName = + options.managedBrowserBridgeFunctionName === undefined + ? null + : String(options.managedBrowserBridgeFunctionName); const scripts = new Map(); for (const item of scriptRegistry) { scripts.set(item.id, item); @@ -345,6 +350,7 @@ export function buildScriptExecutionExpression( const managedKeyboardBridgeFunctionName = ${JSON.stringify(managedKeyboardBridgeFunctionName)}; const managedWindowBridgeFunctionName = ${JSON.stringify(managedWindowBridgeFunctionName)}; const managedTimerBridgeFunctionName = ${JSON.stringify(managedTimerBridgeFunctionName)}; + const managedBrowserBridgeFunctionName = ${JSON.stringify(managedBrowserBridgeFunctionName)}; const timerBridge = managedTimerBridgeFunctionName ? globalThis[managedTimerBridgeFunctionName] : null; const nativeSetTimeout = globalThis.setTimeout.bind(globalThis); const nativeClearTimeout = globalThis.clearTimeout.bind(globalThis); @@ -400,6 +406,7 @@ export function buildScriptExecutionExpression( managedKeyboardBridgeFunctionName, managedWindowBridgeFunctionName, managedTimerBridgeFunctionName, + managedBrowserBridgeFunctionName, scriptFactories, }); })() diff --git a/extension/runtime/injected/playwright-locator.injected.ts b/extension/runtime/injected/playwright-locator.injected.ts new file mode 100644 index 0000000..3b4cb8b --- /dev/null +++ b/extension/runtime/injected/playwright-locator.injected.ts @@ -0,0 +1,585 @@ +/* eslint-disable */ +import type { LocatorQuery, PlaywrightShimDeps, RuntimeMethodTable, ScriptPlaywrightLocator, ScriptPlaywrightPage } from './playwright-shim-types.injected'; +import { accessibleName, cssEscape, hideHighlightOverlay, implicitRole, isVisibleElement, notImplemented, pressKeyOnElement, showHighlightOverlay, smallestTextMatches, textMatches, timeoutFromOptions, waitForLocator } from './playwright-shim-helpers.injected'; + +const PLAYWRIGHT_LOCATOR_METHODS = [ + 'elementHandle', + 'highlight', + 'toString', + 'all', + 'allInnerTexts', + 'allTextContents', + 'and', + 'ariaSnapshot', + 'blur', + 'boundingBox', + 'check', + 'clear', + 'click', + 'contentFrame', + 'count', + 'dblclick', + 'describe', + 'description', + 'dispatchEvent', + 'dragTo', + 'drop', + 'elementHandles', + 'fill', + 'filter', + 'first', + 'focus', + 'frameLocator', + 'getAttribute', + 'getByAltText', + 'getByLabel', + 'getByPlaceholder', + 'getByRole', + 'getByTestId', + 'getByText', + 'getByTitle', + 'hideHighlight', + 'hover', + 'innerHTML', + 'innerText', + 'inputValue', + 'isChecked', + 'isDisabled', + 'isEditable', + 'isEnabled', + 'isHidden', + 'isVisible', + 'last', + 'locator', + 'normalize', + 'nth', + 'or', + 'page', + 'press', + 'pressSequentially', + 'screenshot', + 'scrollIntoViewIfNeeded', + 'selectOption', + 'selectText', + 'setChecked', + 'setInputFiles', + 'tap', + 'textContent', + 'type', + 'uncheck', + 'waitFor', +]; + + +export function createLocator( + query: LocatorQuery, + label: string, + pageApi: ScriptPlaywrightPage, + deps: PlaywrightShimDeps, +): ScriptPlaywrightLocator { + const elementCenter = (element: HTMLElement) => { + const rect = element.getBoundingClientRect(); + return { + x: rect.left + rect.width / 2, + y: rect.top + rect.height / 2, + }; + }; + const requireElement = async (options?: { timeout?: number; state?: 'attached' | 'visible' }) => { + const element = await waitForLocator(query, label, deps.wait, { + timeout: timeoutFromOptions(options), + state: options?.state ?? 'visible', + }); + if (!element) { + throw new Error(`Locator ${label} did not resolve to an element.`); + } + return element; + }; + + const implementation: ScriptPlaywrightLocator = { + __query: query, + __description: '', + async evaluate(pageFunction: unknown, arg?: unknown, options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options), state: 'attached' }); + if (typeof pageFunction === 'function') { + return await Promise.resolve(pageFunction(element, arg)); + } + if (typeof pageFunction === 'string') { + return (0, eval)(pageFunction); + } + throw new Error('locator.evaluate requires a function or string expression.'); + }, + async evaluateAll(pageFunction: unknown, arg?: unknown) { + if (typeof pageFunction !== 'function') { + throw new Error('locator.evaluateAll requires a function.'); + } + return await Promise.resolve(pageFunction(query(), arg)); + }, + async all() { + return query().map((_, index) => createLocator(() => query().slice(index, index + 1), `${label}.nth(${index})`, pageApi, deps)); + }, + async allInnerTexts() { + return query().map((element) => (element instanceof HTMLElement ? element.innerText : element.textContent ?? '')); + }, + async allTextContents() { + return query().map((element) => element.textContent ?? ''); + }, + and(locator: RuntimeMethodTable) { + if (typeof locator?.__query !== 'function') { + throw new Error('locator.and requires a Web Cap locator.'); + } + return createLocator( + () => { + const otherElements = new Set(locator.__query() as Element[]); + return query().filter((element) => otherElements.has(element)); + }, + `${label}.and(${String(locator)})`, + pageApi, + deps, + ); + }, + async boundingBox() { + const element = await requireElement(); + if (!(element instanceof HTMLElement)) { + return null; + } + const rect = element.getBoundingClientRect(); + return { x: rect.x, y: rect.y, width: rect.width, height: rect.height }; + }, + async check(options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options) }); + if (!(element instanceof HTMLInputElement) || element.type !== 'checkbox') { + throw new Error(`Locator ${label} did not resolve to a checkbox input.`); + } + if (!element.checked) { + element.click(); + await deps.waitForManagedInput(); + } + }, + async contentFrame(options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options), state: 'attached' }); + if (!(element instanceof HTMLIFrameElement)) { + return null; + } + return typeof pageApi.__frameForElement === 'function' + ? pageApi.__frameForElement(element) + : null; + }, + async clear(options?: unknown) { + await implementation.fill('', options); + }, + async click(options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options) }); + if (!(element instanceof HTMLElement)) { + throw new Error(`Locator ${label} did not resolve to an HTMLElement.`); + } + element.scrollIntoView?.({ block: 'center', inline: 'center' }); + element.click(); + await deps.waitForManagedInput(); + }, + async dblclick(options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options) }); + if (!(element instanceof HTMLElement)) { + throw new Error(`Locator ${label} did not resolve to an HTMLElement.`); + } + element.scrollIntoView?.({ block: 'center', inline: 'center' }); + element.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true })); + element.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true })); + element.dispatchEvent(new MouseEvent('dblclick', { bubbles: true, cancelable: true })); + await deps.waitForManagedInput(); + }, + async count() { + return query().length; + }, + async dispatchEvent(type: unknown, eventInit: unknown = {}, options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options), state: 'attached' }); + element.dispatchEvent(new Event(String(type), { bubbles: true, ...(eventInit as EventInit) })); + await deps.waitForManagedInput(); + }, + describe(description: unknown) { + const next = createLocator(query, label, pageApi, deps); + next.__description = String(description ?? ''); + return next; + }, + description() { + return String(implementation.__description ?? ''); + }, + async dragTo(target: RuntimeMethodTable, options?: unknown) { + const source = await requireElement({ timeout: timeoutFromOptions(options) }); + const targetElement = typeof target?.elementHandle === 'function' + ? await target.elementHandle(options) + : null; + if (!(source instanceof HTMLElement) || !(targetElement instanceof HTMLElement)) { + throw new Error('locator.dragTo requires source and target HTMLElements.'); + } + if (pageApi.mouse) { + const sourcePoint = elementCenter(source); + const targetPoint = elementCenter(targetElement); + await pageApi.mouse.move(sourcePoint.x, sourcePoint.y); + await pageApi.mouse.down(); + await pageApi.mouse.move(targetPoint.x, targetPoint.y, { steps: 8 }); + await pageApi.mouse.up(); + return; + } + const dataTransfer = typeof DataTransfer !== 'undefined' ? new DataTransfer() : undefined; + const eventInit = { bubbles: true, cancelable: true, dataTransfer } as DragEventInit; + const createDragLikeEvent = (type: string) => + typeof DragEvent !== 'undefined' + ? new DragEvent(type, eventInit) + : new MouseEvent(type, eventInit); + source.dispatchEvent(createDragLikeEvent('dragstart')); + targetElement.dispatchEvent(createDragLikeEvent('dragenter')); + targetElement.dispatchEvent(createDragLikeEvent('dragover')); + targetElement.dispatchEvent(createDragLikeEvent('drop')); + source.dispatchEvent(createDragLikeEvent('dragend')); + await deps.waitForManagedInput(); + }, + async drop(options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options) }); + if (!(element instanceof HTMLElement)) { + throw new Error(`Locator ${label} did not resolve to an HTMLElement.`); + } + const dataTransfer = typeof DataTransfer !== 'undefined' ? new DataTransfer() : undefined; + const eventInit = { bubbles: true, cancelable: true, dataTransfer } as DragEventInit; + element.dispatchEvent( + typeof DragEvent !== 'undefined' + ? new DragEvent('drop', eventInit) + : new MouseEvent('drop', eventInit), + ); + await deps.waitForManagedInput(); + }, + async elementHandle(options?: unknown) { + return await requireElement({ timeout: timeoutFromOptions(options), state: 'attached' }); + }, + async elementHandles() { + return query(); + }, + fill: async (value: unknown, options?: unknown) => { + const element = await requireElement({ timeout: timeoutFromOptions(options) }); + await deps.typeIntoElement(element, value); + await deps.waitForManagedInput(); + }, + filter(options: { hasText?: unknown; hasNotText?: unknown } = {}) { + return createLocator( + () => + query().filter((element) => { + if (options.hasText !== undefined && !textMatches(element.textContent, options.hasText)) { + return false; + } + if (options.hasNotText !== undefined && textMatches(element.textContent, options.hasNotText)) { + return false; + } + return true; + }), + `${label}.filter()`, + pageApi, + deps, + ); + }, + first() { + return createLocator(() => query().slice(0, 1), `${label}.first()`, pageApi, deps); + }, + async focus(options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options) }); + if (element instanceof HTMLElement) { + element.focus(); + } + }, + async blur(options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options), state: 'attached' }); + if (element instanceof HTMLElement) { + element.blur(); + } + }, + async getAttribute(name: unknown, options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options), state: 'attached' }); + return element.getAttribute(String(name)); + }, + async highlight(options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options) }); + showHighlightOverlay(element); + }, + getByAltText(text: unknown, options: { exact?: boolean } = {}) { + return createLocator( + () => query().flatMap((root) => [...root.querySelectorAll('img, area')]).filter((element) => textMatches(element.getAttribute('alt'), text, options.exact)), + `${label}.getByAltText(${String(text)})`, + pageApi, + deps, + ); + }, + getByLabel(text: unknown, options: { exact?: boolean } = {}) { + return createLocator( + () => + query() + .flatMap((root) => [root, ...root.querySelectorAll('*')]) + .filter((element) => { + if ( + !(element instanceof HTMLInputElement) && + !(element instanceof HTMLTextAreaElement) && + !(element instanceof HTMLSelectElement) + ) { + return false; + } + return textMatches(accessibleName(element), text, options.exact); + }), + `${label}.getByLabel(${String(text)})`, + pageApi, + deps, + ); + }, + getByPlaceholder(text: unknown, options: { exact?: boolean } = {}) { + return createLocator( + () => query().flatMap((root) => [root, ...root.querySelectorAll('*')]).filter((element) => textMatches(element.getAttribute('placeholder'), text, options.exact)), + `${label}.getByPlaceholder(${String(text)})`, + pageApi, + deps, + ); + }, + getByRole(role: unknown, options: { name?: unknown; exact?: boolean } = {}) { + return createLocator( + () => + query() + .flatMap((root) => [root, ...root.querySelectorAll('*')]) + .filter((element) => { + if (implicitRole(element) !== String(role)) { + return false; + } + return options.name === undefined || textMatches(accessibleName(element), options.name, options.exact); + }), + `${label}.getByRole(${String(role)})`, + pageApi, + deps, + ); + }, + getByTestId(testId: unknown) { + return implementation.locator(`[data-testid="${cssEscape(String(testId))}"]`); + }, + getByText(text: unknown, options: { exact?: boolean } = {}) { + return createLocator( + () => smallestTextMatches(query(), text, options.exact), + `${label}.getByText(${String(text)})`, + pageApi, + deps, + ); + }, + getByTitle(text: unknown, options: { exact?: boolean } = {}) { + return createLocator( + () => query().flatMap((root) => [root, ...root.querySelectorAll('*')]).filter((element) => textMatches(element.getAttribute('title'), text, options.exact)), + `${label}.getByTitle(${String(text)})`, + pageApi, + deps, + ); + }, + async hover(options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options) }); + if (element instanceof HTMLElement && pageApi.mouse) { + const point = elementCenter(element); + await pageApi.mouse.move(point.x, point.y); + return; + } + element.dispatchEvent(new MouseEvent('mouseover', { bubbles: true })); + await deps.waitForManagedInput(); + }, + async hideHighlight() { + hideHighlightOverlay(); + }, + async innerHTML(options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options), state: 'attached' }); + return element.innerHTML; + }, + async innerText(options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options), state: 'attached' }); + return element instanceof HTMLElement ? element.innerText : element.textContent ?? ''; + }, + async inputValue(options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options), state: 'attached' }); + if ( + element instanceof HTMLInputElement || + element instanceof HTMLTextAreaElement || + element instanceof HTMLSelectElement + ) { + return element.value; + } + throw new Error(`Locator ${label} did not resolve to an input, textarea, or select.`); + }, + async isChecked() { + const element = query()[0]; + return element instanceof HTMLInputElement ? element.checked : false; + }, + async isDisabled() { + const element = query()[0]; + return element instanceof HTMLButtonElement || element instanceof HTMLInputElement || element instanceof HTMLSelectElement || element instanceof HTMLTextAreaElement + ? element.disabled + : false; + }, + async isEditable() { + const element = query()[0]; + return deps.isEditableElement(element); + }, + async isEnabled() { + return !(await implementation.isDisabled()); + }, + async isHidden() { + return !(await implementation.isVisible()); + }, + async isVisible() { + return isVisibleElement(query()[0]); + }, + last() { + return createLocator(() => query().slice(-1), `${label}.last()`, pageApi, deps); + }, + locator(selector: unknown) { + if (typeof selector !== 'string') { + throw new Error('locator.locator only supports string selectors in Web Cap script runtime.'); + } + return createLocator( + () => query().flatMap((element) => [...element.querySelectorAll(selector)]), + `${label}.locator(${selector})`, + pageApi, + deps, + ); + }, + nth(index: unknown) { + const normalizedIndex = Math.max(Math.trunc(Number(index)), 0); + return createLocator(() => query().slice(normalizedIndex, normalizedIndex + 1), `${label}.nth(${normalizedIndex})`, pageApi, deps); + }, + or(locator: RuntimeMethodTable) { + if (typeof locator?.__query !== 'function') { + throw new Error('locator.or requires a Web Cap locator.'); + } + return createLocator( + () => { + const seen = new Set(); + const elements = [...query(), ...(locator.__query() as Element[])]; + return elements.filter((element) => { + if (seen.has(element)) { + return false; + } + seen.add(element); + return true; + }); + }, + `${label}.or(${String(locator)})`, + pageApi, + deps, + ); + }, + page() { + return pageApi; + }, + async press(key: unknown, options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options) }); + await pressKeyOnElement(element, key, deps); + }, + async pressSequentially(text: unknown, options?: { delay?: number; timeout?: number }) { + const element = await requireElement({ timeout: timeoutFromOptions(options) }); + const delay = Math.max(Number(options?.delay ?? 0), 0); + for (const char of String(text ?? '')) { + await pressKeyOnElement(element, char, deps); + if (delay > 0) { + await deps.wait(delay); + } + } + }, + async scrollIntoViewIfNeeded(options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options) }); + if (element instanceof HTMLElement) { + element.scrollIntoView?.({ block: 'center', inline: 'center' }); + } + }, + async screenshot(options: { type?: unknown; quality?: unknown } = {}) { + const element = await requireElement(); + if (!(element instanceof HTMLElement)) { + throw new Error(`Locator ${label} did not resolve to an HTMLElement.`); + } + if (!deps.browserCommand) { + throw new Error('locator.screenshot requires the debugger CDP bridge.'); + } + element.scrollIntoView?.({ block: 'center', inline: 'center' }); + const rect = element.getBoundingClientRect(); + const format = options.type === 'jpeg' ? 'jpeg' : 'png'; + const params: Record = { + format, + fromSurface: true, + clip: { + x: rect.left + window.scrollX, + y: rect.top + window.scrollY, + width: Math.max(rect.width, 1), + height: Math.max(rect.height, 1), + scale: 1, + }, + }; + if (format === 'jpeg' && options.quality !== undefined) { + params.quality = Number(options.quality); + } + const result = await deps.browserCommand('Page.captureScreenshot', params) as { data?: string }; + return result.data ?? result; + }, + async setChecked(checked: unknown, options?: unknown) { + const shouldBeChecked = Boolean(checked); + const element = await requireElement({ timeout: timeoutFromOptions(options) }); + if (!(element instanceof HTMLInputElement) || element.type !== 'checkbox') { + throw new Error(`Locator ${label} did not resolve to a checkbox input.`); + } + if (element.checked !== shouldBeChecked) { + element.click(); + await deps.waitForManagedInput(); + } + }, + async selectOption(values: unknown, options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options) }); + if (!(element instanceof HTMLSelectElement)) { + throw new Error(`Locator ${label} did not resolve to a select element.`); + } + const selectedValues = Array.isArray(values) ? values.map(String) : [String(values)]; + for (const option of [...element.options]) { + option.selected = selectedValues.includes(option.value) || selectedValues.includes(option.label); + } + element.dispatchEvent(new Event('input', { bubbles: true })); + element.dispatchEvent(new Event('change', { bubbles: true })); + await deps.waitForManagedInput(); + return selectedValues; + }, + async selectText(options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options) }); + if (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) { + element.select(); + return; + } + const selection = globalThis.getSelection?.(); + if (!selection || !(element instanceof HTMLElement)) { + return; + } + const range = document.createRange(); + range.selectNodeContents(element); + selection.removeAllRanges(); + selection.addRange(range); + }, + async tap(options?: unknown) { + await implementation.click(options); + }, + async textContent(options?: unknown) { + const element = await requireElement({ timeout: timeoutFromOptions(options), state: 'attached' }); + return element.textContent; + }, + async type(text: unknown, options?: { delay?: number; timeout?: number }) { + await implementation.pressSequentially(text, options); + }, + async uncheck(options?: unknown) { + await implementation.setChecked(false, options); + }, + async waitFor(options: { state?: 'attached' | 'detached' | 'visible' | 'hidden'; timeout?: number } = {}) { + await waitForLocator(query, label, deps.wait, options); + }, + toString() { + return label; + }, + }; + + for (const method of PLAYWRIGHT_LOCATOR_METHODS) { + if (!(method in implementation)) { + implementation[method] = notImplemented(`locator.${method}`); + } + } + + return implementation; +} diff --git a/extension/runtime/injected/playwright-shim-helpers.injected.ts b/extension/runtime/injected/playwright-shim-helpers.injected.ts new file mode 100644 index 0000000..1b4738f --- /dev/null +++ b/extension/runtime/injected/playwright-shim-helpers.injected.ts @@ -0,0 +1,257 @@ +/* eslint-disable */ +import type { LocatorQuery, PlaywrightShimDeps } from './playwright-shim-types.injected'; + +export function notImplemented(apiName: string) { + return () => { + throw new Error(`${apiName} is part of the Playwright API surface but is not implemented by Web Cap script runtime yet.`); + }; +} + +export function timeoutFromOptions(options: unknown, defaultMs = 5000) { + const timeout = + options && typeof options === 'object' && 'timeout' in options + ? Number((options as { timeout?: unknown }).timeout) + : defaultMs; + return Math.max(Number.isFinite(timeout) ? timeout : defaultMs, 0); +} + +function normalizeText(value: unknown) { + return String(value ?? '').replace(/\s+/g, ' ').trim(); +} + +export function textMatches(value: unknown, expected: unknown, exact = false) { + const text = normalizeText(value); + if (expected instanceof RegExp) { + return expected.test(text); + } + const normalizedExpected = normalizeText(expected); + return exact ? text === normalizedExpected : text.includes(normalizedExpected); +} + +export function smallestTextMatches(roots: Element[], expected: unknown, exact = false) { + const matches = roots.flatMap((root) => { + const candidates = [root, ...root.querySelectorAll('*')]; + const matchingCandidates = candidates.filter((element) => textMatches(element.textContent, expected, exact)); + return matchingCandidates.filter( + (element) => !matchingCandidates.some((other) => other !== element && element.contains(other)), + ); + }); + return matches.filter((element, index) => matches.indexOf(element) === index); +} + +export function cssEscape(value: string) { + const cssObject = (globalThis as typeof globalThis & { CSS?: { escape?: (value: string) => string } }).CSS; + if (typeof cssObject?.escape === 'function') { + return cssObject.escape(value); + } + return value.replace(/["\\]/g, '\\$&'); +} + +export function isVisibleElement(element: unknown) { + if (!(element instanceof HTMLElement)) { + return false; + } + const style = globalThis.getComputedStyle?.(element); + if (!style) { + return false; + } + if ( + element.hidden || + style.display === 'none' || + style.visibility === 'hidden' || + style.visibility === 'collapse' || + Number(style.opacity) === 0 + ) { + return false; + } + const rect = element.getBoundingClientRect(); + return rect.width > 0 && rect.height > 0; +} + +const HIGHLIGHT_OVERLAY_ID = '__web-cap-playwright-highlight'; + +export function hideHighlightOverlay() { + document.getElementById(HIGHLIGHT_OVERLAY_ID)?.remove(); +} + +export function showHighlightOverlay(element: Element) { + hideHighlightOverlay(); + if (!(element instanceof HTMLElement)) { + return; + } + const rect = element.getBoundingClientRect(); + const overlay = document.createElement('div'); + overlay.id = HIGHLIGHT_OVERLAY_ID; + Object.assign(overlay.style, { + position: 'absolute', + left: `${rect.left + window.scrollX}px`, + top: `${rect.top + window.scrollY}px`, + width: `${rect.width}px`, + height: `${rect.height}px`, + border: '2px solid #2563eb', + boxShadow: '0 0 0 99999px rgba(37, 99, 235, 0.12)', + pointerEvents: 'none', + zIndex: '2147483647', + }); + document.documentElement.appendChild(overlay); +} + +export function accessibleName(element: Element) { + const ariaLabel = element.getAttribute('aria-label'); + if (ariaLabel) { + return ariaLabel; + } + const labelledBy = element.getAttribute('aria-labelledby'); + if (labelledBy) { + return labelledBy + .split(/\s+/) + .map((id) => document.getElementById(id)?.textContent ?? '') + .join(' ') + .trim(); + } + if ( + element instanceof HTMLInputElement || + element instanceof HTMLTextAreaElement || + element instanceof HTMLSelectElement + ) { + return [...(element.labels ?? [])].map((label) => label.textContent ?? '').join(' ').trim(); + } + if (element instanceof HTMLImageElement) { + return element.alt || element.title || ''; + } + return element.textContent ?? ''; +} + +export function implicitRole(element: Element) { + const role = element.getAttribute('role'); + if (role) { + return role; + } + const tagName = element.tagName.toLowerCase(); + if (tagName === 'button') { + return 'button'; + } + if (tagName === 'a' && element instanceof HTMLAnchorElement && element.href) { + return 'link'; + } + if (tagName === 'img') { + return 'img'; + } + if (tagName === 'input') { + const type = (element.getAttribute('type') || 'text').toLowerCase(); + if (type === 'button' || type === 'submit' || type === 'reset') { + return 'button'; + } + if (type === 'checkbox') { + return 'checkbox'; + } + if (type === 'radio') { + return 'radio'; + } + if (type === 'search') { + return 'searchbox'; + } + return 'textbox'; + } + if (tagName === 'textarea') { + return 'textbox'; + } + if (tagName === 'select') { + return 'combobox'; + } + return ''; +} + +function allElements() { + return [...document.querySelectorAll('*')]; +} + +function keyboardInfoForKey(key: string) { + const aliases: Record = { + Enter: { key: 'Enter', code: 'Enter', keyCode: 13 }, + Escape: { key: 'Escape', code: 'Escape', keyCode: 27 }, + Backspace: { key: 'Backspace', code: 'Backspace', keyCode: 8 }, + Delete: { key: 'Delete', code: 'Delete', keyCode: 46 }, + Tab: { key: 'Tab', code: 'Tab', keyCode: 9 }, + Space: { key: ' ', code: 'Space', keyCode: 32 }, + }; + if (aliases[key]) { + return aliases[key]; + } + if (key.length === 1) { + const upper = key.toUpperCase(); + return { key, code: `Key${upper}`, keyCode: upper.charCodeAt(0) }; + } + return { key, code: key, keyCode: 0 }; +} + +function applyPressedKeyToEditable(element: HTMLElement, key: string) { + if (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) { + if (key === 'Backspace') { + element.value = element.value.slice(0, -1); + } else if (key.length === 1) { + element.value += key; + } else { + return; + } + element.dispatchEvent(new InputEvent('input', { bubbles: true, data: key.length === 1 ? key : null })); + return; + } + + if (element.isContentEditable && key.length === 1) { + element.textContent = `${element.textContent ?? ''}${key}`; + element.dispatchEvent(new InputEvent('input', { bubbles: true, data: key })); + } +} + +export async function pressKeyOnElement(element: unknown, key: unknown, deps: PlaywrightShimDeps) { + if (!(element instanceof HTMLElement)) { + throw new Error('Keyboard press target must be an HTMLElement.'); + } + const normalizedKey = String(key); + const keyInfo = keyboardInfoForKey(normalizedKey); + const eventInit = { + key: keyInfo.key, + code: keyInfo.code, + keyCode: keyInfo.keyCode, + which: keyInfo.keyCode, + bubbles: true, + cancelable: true, + }; + element.scrollIntoView?.({ block: 'center', inline: 'center' }); + element.focus?.(); + element.dispatchEvent(new KeyboardEvent('keydown', eventInit)); + if (keyInfo.key.length === 1 || keyInfo.key === 'Enter') { + element.dispatchEvent(new KeyboardEvent('keypress', eventInit)); + } + if (deps.useDomKeyboardFallback()) { + applyPressedKeyToEditable(element, keyInfo.key); + } + element.dispatchEvent(new KeyboardEvent('keyup', eventInit)); + await deps.waitForManagedInput(); +} + +export async function waitForLocator( + query: LocatorQuery, + label: string, + wait: PlaywrightShimDeps['wait'], + options: { state?: 'attached' | 'detached' | 'visible' | 'hidden'; timeout?: number } = {}, +) { + const state = options.state ?? 'visible'; + const timeout = timeoutFromOptions(options); + const startedAt = Date.now(); + while (Date.now() - startedAt <= timeout) { + const element = query()[0] ?? null; + const visible = isVisibleElement(element); + if ( + (state === 'attached' && element) || + (state === 'detached' && !element) || + (state === 'visible' && visible) || + (state === 'hidden' && (!element || !visible)) + ) { + return element; + } + await wait(50); + } + throw new Error(`Timed out after ${timeout}ms waiting for ${label} to be ${state}.`); +} diff --git a/extension/runtime/injected/playwright-shim-types.injected.ts b/extension/runtime/injected/playwright-shim-types.injected.ts new file mode 100644 index 0000000..f40d979 --- /dev/null +++ b/extension/runtime/injected/playwright-shim-types.injected.ts @@ -0,0 +1,18 @@ +/* eslint-disable */ +import type { Locator as PlaywrightLocator, Page as PlaywrightPage } from 'playwright-core'; + +export type RuntimeMethodTable = Record; +export type ScriptPlaywrightPage = RuntimeMethodTable & { __playwrightPageType?: PlaywrightPage }; +export type ScriptPlaywrightLocator = RuntimeMethodTable & { __playwrightLocatorType?: PlaywrightLocator }; +export type LocatorQuery = () => Element[]; + +export type PlaywrightShimDeps = { + wait(ms: number): Promise; + typeIntoElement(element: unknown, value: unknown): Promise; + isEditableElement(element: unknown): boolean; + useDomKeyboardFallback(): boolean; + browserCommand?(method: string, params?: Record): Promise; + browserEvent?(method: string, params?: Record, timeoutMs?: number): Promise; + recordEvidenceEvent?(type: string, value: unknown): void; + waitForManagedInput(): Promise; +}; diff --git a/extension/runtime/injected/playwright-shim.injected.ts b/extension/runtime/injected/playwright-shim.injected.ts new file mode 100644 index 0000000..8ace170 --- /dev/null +++ b/extension/runtime/injected/playwright-shim.injected.ts @@ -0,0 +1,1090 @@ +/* eslint-disable */ +import type { PlaywrightShimDeps, RuntimeMethodTable, ScriptPlaywrightPage } from './playwright-shim-types.injected'; +import { createLocator } from './playwright-locator.injected'; +import { cssEscape, hideHighlightOverlay, notImplemented, timeoutFromOptions, waitForLocator } from './playwright-shim-helpers.injected'; + +export function createPlaywrightPageApi(deps: PlaywrightShimDeps): ScriptPlaywrightPage { +let defaultTimeoutMs = 5000; +async function browserCommand(method: string, params: Record = {}) { + if (!deps.browserCommand) { + throw new Error(`page.${method} requires the debugger CDP bridge.`); + } + return await deps.browserCommand(method, params) as T; +} + +async function browserEvent(method: string, params: Record = {}, timeoutMs = defaultTimeoutMs) { + if (!deps.browserEvent) { + throw new Error(`page.${method} requires the debugger CDP bridge.`); + } + return await deps.browserEvent(method, params, timeoutMs) as T; +} + +function serializeUrlMatcher(matcher: unknown) { + if (typeof matcher === 'string') { + return { url: matcher }; + } + if (matcher instanceof RegExp) { + return { regexSource: matcher.source, regexFlags: matcher.flags }; + } + return {}; +} + +function frameDocument(frameElement: HTMLIFrameElement | null) { + return frameElement ? frameElement.contentDocument : document; +} + +type FrameMetadata = { + id?: string; + parentId?: string; + name?: string; + url?: string; +}; + +const PLAYWRIGHT_PAGE_METHODS = [ + '$', + '$$', + 'waitForSelector', + 'exposeBinding', + 'removeAllListeners', + 'on', + 'once', + 'addListener', + 'removeListener', + 'off', + 'prependListener', + 'addLocatorHandler', + 'addScriptTag', + 'addStyleTag', + 'ariaSnapshot', + 'bringToFront', + 'cancelPickLocator', + 'check', + 'clearConsoleMessages', + 'clearPageErrors', + 'click', + 'close', + 'consoleMessages', + 'content', + 'context', + 'dblclick', + 'dispatchEvent', + 'dragAndDrop', + 'emulateMedia', + 'exposeFunction', + 'fill', + 'focus', + 'frame', + 'frameLocator', + 'frames', + 'getAttribute', + 'getByAltText', + 'getByLabel', + 'getByPlaceholder', + 'getByRole', + 'getByTestId', + 'getByText', + 'getByTitle', + 'goBack', + 'goForward', + 'goto', + 'hideHighlight', + 'hover', + 'innerHTML', + 'innerText', + 'inputValue', + 'isChecked', + 'isClosed', + 'isDisabled', + 'isEditable', + 'isEnabled', + 'isHidden', + 'isVisible', + 'locator', + 'mainFrame', + 'opener', + 'pageErrors', + 'pause', + 'pdf', + 'pickLocator', + 'press', + 'reload', + 'removeLocatorHandler', + 'requestGC', + 'requests', + 'route', + 'routeFromHAR', + 'routeWebSocket', + 'screenshot', + 'selectOption', + 'setChecked', + 'setContent', + 'setDefaultNavigationTimeout', + 'setDefaultTimeout', + 'setExtraHTTPHeaders', + 'setInputFiles', + 'setViewportSize', + 'tap', + 'textContent', + 'title', + 'type', + 'uncheck', + 'unroute', + 'unrouteAll', + 'url', + 'video', + 'viewportSize', + 'waitForEvent', + 'waitForLoadState', + 'waitForNavigation', + 'waitForRequest', + 'waitForResponse', + 'waitForTimeout', + 'waitForURL', + 'workers', +]; + +function createPageApi(): ScriptPlaywrightPage { + const pageApi: ScriptPlaywrightPage = {}; + const mouseState = { + x: 0, + y: 0, + buttons: 0, + }; + const mouseButtonName = (button: unknown) => { + const value = String(button ?? 'left'); + return value === 'right' || value === 'middle' || value === 'back' || value === 'forward' + ? value + : 'left'; + }; + const mouseButtonsMask = (button: string) => + button === 'right' ? 2 : button === 'middle' ? 4 : button === 'back' ? 8 : button === 'forward' ? 16 : 1; + const recordMouseAction = (action: string, value: Record = {}) => { + deps.recordEvidenceEvent?.('managed_mouse', { + action, + x: mouseState.x, + y: mouseState.y, + buttons: mouseState.buttons, + ...value, + }); + }; + const dispatchMouseEvent = async ( + type: string, + x: number, + y: number, + options: { button?: unknown; buttons?: number; clickCount?: unknown; deltaX?: unknown; deltaY?: unknown } = {}, + ) => { + const button = mouseButtonName(options.button); + await browserCommand('Input.dispatchMouseEvent', { + type, + x, + y, + button, + buttons: options.buttons ?? mouseState.buttons, + clickCount: Math.max(Math.trunc(Number(options.clickCount ?? 1)), 1), + deltaX: Number(options.deltaX ?? 0), + deltaY: Number(options.deltaY ?? 0), + pointerType: 'mouse', + }); + }; + const mouseApi: RuntimeMethodTable = { + async move(x: unknown, y: unknown, options: { steps?: unknown } = {}) { + const targetX = Number(x); + const targetY = Number(y); + if (!Number.isFinite(targetX) || !Number.isFinite(targetY)) { + throw new Error('page.mouse.move requires finite x and y coordinates.'); + } + const steps = Math.max(Math.trunc(Number(options.steps ?? 1)), 1); + const startX = mouseState.x; + const startY = mouseState.y; + for (let index = 1; index <= steps; index += 1) { + const nextX = startX + ((targetX - startX) * index) / steps; + const nextY = startY + ((targetY - startY) * index) / steps; + await dispatchMouseEvent('mouseMoved', nextX, nextY, { buttons: mouseState.buttons }); + } + mouseState.x = targetX; + mouseState.y = targetY; + recordMouseAction('move', { steps }); + }, + async down(options: { button?: unknown; clickCount?: unknown } = {}) { + const button = mouseButtonName(options.button); + mouseState.buttons |= mouseButtonsMask(button); + await dispatchMouseEvent('mousePressed', mouseState.x, mouseState.y, { + button, + buttons: mouseState.buttons, + clickCount: options.clickCount, + }); + recordMouseAction('down', { button }); + }, + async up(options: { button?: unknown; clickCount?: unknown } = {}) { + const button = mouseButtonName(options.button); + const nextButtons = mouseState.buttons & ~mouseButtonsMask(button); + await dispatchMouseEvent('mouseReleased', mouseState.x, mouseState.y, { + button, + buttons: nextButtons, + clickCount: options.clickCount, + }); + mouseState.buttons = nextButtons; + recordMouseAction('up', { button }); + }, + async click(x: unknown, y: unknown, options: { button?: unknown; clickCount?: unknown; delay?: unknown } = {}) { + await mouseApi.move(x, y); + await mouseApi.down(options); + const delay = Math.max(Number(options.delay ?? 0), 0); + if (delay > 0) { + await deps.wait(delay); + } + await mouseApi.up(options); + }, + async dblclick(x: unknown, y: unknown, options: { button?: unknown; delay?: unknown } = {}) { + await mouseApi.click(x, y, { ...options, clickCount: 1 }); + await mouseApi.click(x, y, { ...options, clickCount: 2 }); + }, + async wheel(deltaX: unknown, deltaY: unknown) { + await dispatchMouseEvent('mouseWheel', mouseState.x, mouseState.y, { + buttons: mouseState.buttons, + deltaX, + deltaY, + }); + recordMouseAction('wheel', { + deltaX: Number(deltaX ?? 0), + deltaY: Number(deltaY ?? 0), + }); + }, + }; + const sameOriginFrameElements = () => + [...document.querySelectorAll('iframe')] + .filter((element): element is HTMLIFrameElement => element instanceof HTMLIFrameElement) + .filter((element) => Boolean(element.contentDocument)); + const findSameOriginFrameElement = (metadata: FrameMetadata) => + sameOriginFrameElements().find((element) => { + const frameName = element.name || element.id || ''; + const frameUrl = element.contentDocument?.location?.href ?? element.src ?? ''; + return Boolean( + (metadata.name && frameName === metadata.name) || + (metadata.url && frameUrl === metadata.url), + ); + }) ?? null; + const flattenFrameTree = (tree: RuntimeMethodTable | undefined, parentId?: string): FrameMetadata[] => { + if (!tree?.frame || typeof tree.frame !== 'object') { + return []; + } + const frame = tree.frame as RuntimeMethodTable; + return [ + { + id: typeof frame.id === 'string' ? frame.id : '', + parentId, + name: typeof frame.name === 'string' ? frame.name : '', + url: typeof frame.url === 'string' ? frame.url : '', + }, + ...((Array.isArray(tree.childFrames) ? tree.childFrames : []) as RuntimeMethodTable[]).flatMap((child) => + flattenFrameTree(child, typeof frame.id === 'string' ? frame.id : undefined), + ), + ]; + }; + const readFrameMetadata = async () => { + if (!deps.browserCommand) { + return [ + { name: '', url: globalThis.location?.href ?? '' }, + ...sameOriginFrameElements().map((element) => ({ + name: element.name || element.id || '', + url: element.contentDocument?.location?.href ?? element.src ?? '', + })), + ]; + } + const result = await browserCommand<{ frameTree?: RuntimeMethodTable }>('Page.getFrameTree'); + return flattenFrameTree(result.frameTree); + }; + const frameExecutionContextId = async (frameId: string) => { + const result = await browserCommand<{ executionContextId?: number }>('Page.createIsolatedWorld', { + frameId, + worldName: '__webCapPlaywrightFrame', + grantUniveralAccess: true, + }); + if (!result.executionContextId) { + throw new Error(`Could not create execution context for frame ${frameId}.`); + } + return result.executionContextId; + }; + const evaluateInFrame = async (frameId: string, expression: string) => { + const contextId = await frameExecutionContextId(frameId); + const result = await browserCommand<{ result?: { value?: T }; exceptionDetails?: { text?: string } }>('Runtime.evaluate', { + expression, + contextId, + awaitPromise: true, + returnByValue: true, + }); + if (result.exceptionDetails) { + throw new Error(result.exceptionDetails.text ?? `Frame evaluation failed for ${frameId}.`); + } + return result.result?.value as T; + }; + const createCdpFrameLocator = (frameId: string, selector: string, locatorLabel: string, queryExpression?: string): RuntimeMethodTable => { + const selectorJson = JSON.stringify(selector); + const queryAllExpression = queryExpression ?? `[...document.querySelectorAll(${selectorJson})]`; + const readOne = async (body: string) => + await evaluateInFrame(frameId, `(() => { + const element = (${queryAllExpression})[0] ?? null; + if (!element) return null; + ${body} + })()`); + const locatorApi: RuntimeMethodTable = { + async count() { + return await evaluateInFrame(frameId, `(${queryAllExpression}).length`); + }, + async textContent() { + return await readOne('return element.textContent;'); + }, + async innerText() { + return await readOne('return element instanceof HTMLElement ? element.innerText : element.textContent;'); + }, + async allTextContents() { + return await evaluateInFrame(frameId, `(${queryAllExpression}).map((element) => element.textContent ?? '')`); + }, + async allInnerTexts() { + return await evaluateInFrame(frameId, `(${queryAllExpression}).map((element) => element instanceof HTMLElement ? element.innerText : element.textContent ?? '')`); + }, + async isVisible() { + return Boolean(await readOne(`const style = getComputedStyle(element); + const rect = element.getBoundingClientRect(); + return style.display !== 'none' && style.visibility !== 'hidden' && rect.width > 0 && rect.height > 0;`)); + }, + async waitFor(options: { timeout?: number } = {}) { + const timeout = timeoutFromOptions(options, defaultTimeoutMs); + const startedAt = Date.now(); + while (Date.now() - startedAt <= timeout) { + if (await locatorApi.count() > 0) { + return; + } + await deps.wait(50); + } + throw new Error(`Timed out after ${timeout}ms waiting for ${locatorLabel}.`); + }, + async click(options?: { timeout?: number }) { + await locatorApi.waitFor(options); + const rect = await readOne<{ left: number; top: number; width: number; height: number } | null>( + `const rect = element.getBoundingClientRect(); + return { left: rect.left, top: rect.top, width: rect.width, height: rect.height };`, + ); + if (!rect) { + throw new Error(`${locatorLabel} did not resolve to an element.`); + } + const owner = await browserCommand<{ backendNodeId?: number }>('DOM.getFrameOwner', { frameId }); + const box = owner.backendNodeId + ? await browserCommand<{ model?: { content?: number[] } }>('DOM.getBoxModel', { backendNodeId: owner.backendNodeId }) + : null; + const content = box?.model?.content ?? [0, 0]; + const x = Number(content[0] ?? 0) + rect.left + rect.width / 2; + const y = Number(content[1] ?? 0) + rect.top + rect.height / 2; + await browserCommand('Input.dispatchMouseEvent', { type: 'mouseMoved', x, y, button: 'left', buttons: 0 }); + await browserCommand('Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button: 'left', buttons: 1, clickCount: 1 }); + await browserCommand('Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button: 'left', buttons: 0, clickCount: 1 }); + }, + async hover(options?: { timeout?: number }) { + await locatorApi.waitFor(options); + const rect = await readOne<{ left: number; top: number; width: number; height: number } | null>( + `const rect = element.getBoundingClientRect(); + return { left: rect.left, top: rect.top, width: rect.width, height: rect.height };`, + ); + if (!rect) { + throw new Error(`${locatorLabel} did not resolve to an element.`); + } + const owner = await browserCommand<{ backendNodeId?: number }>('DOM.getFrameOwner', { frameId }); + const box = owner.backendNodeId + ? await browserCommand<{ model?: { content?: number[] } }>('DOM.getBoxModel', { backendNodeId: owner.backendNodeId }) + : null; + const content = box?.model?.content ?? [0, 0]; + const x = Number(content[0] ?? 0) + rect.left + rect.width / 2; + const y = Number(content[1] ?? 0) + rect.top + rect.height / 2; + await browserCommand('Input.dispatchMouseEvent', { type: 'mouseMoved', x, y, button: 'left', buttons: 0 }); + }, + async dblclick(options?: { timeout?: number }) { + await locatorApi.click(options); + await locatorApi.click(options); + }, + async tap(options?: { timeout?: number }) { + await locatorApi.click(options); + }, + async fill(value: unknown, options?: { timeout?: number }) { + await locatorApi.waitFor(options); + const valueJson = JSON.stringify(String(value ?? '')); + await evaluateInFrame(frameId, `(() => { + const element = (${queryAllExpression})[0] ?? null; + if (!(element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement)) { + throw new Error('Frame locator fill target is not editable.'); + } + element.focus(); + element.value = ${valueJson}; + element.dispatchEvent(new Event('input', { bubbles: true })); + element.dispatchEvent(new Event('change', { bubbles: true })); + })()`); + }, + async type(value: unknown, options?: { timeout?: number }) { + await locatorApi.fill(value, options); + }, + async press(key: unknown, options?: { timeout?: number }) { + await locatorApi.waitFor(options); + const keyJson = JSON.stringify(String(key ?? '')); + await evaluateInFrame(frameId, `(() => { + const element = (${queryAllExpression})[0] ?? null; + if (!(element instanceof HTMLElement)) throw new Error('Frame locator press target is not an HTMLElement.'); + element.focus(); + const key = ${keyJson}; + element.dispatchEvent(new KeyboardEvent('keydown', { key, bubbles: true, cancelable: true })); + element.dispatchEvent(new KeyboardEvent('keyup', { key, bubbles: true, cancelable: true })); + })()`); + }, + async check(options?: { timeout?: number }) { + await locatorApi.setChecked(true, options); + }, + async uncheck(options?: { timeout?: number }) { + await locatorApi.setChecked(false, options); + }, + async setChecked(checked: unknown, options?: { timeout?: number }) { + await locatorApi.waitFor(options); + await evaluateInFrame(frameId, `(() => { + const element = (${queryAllExpression})[0] ?? null; + if (!(element instanceof HTMLInputElement) || element.type !== 'checkbox') { + throw new Error('Frame locator checkbox target is not a checkbox input.'); + } + const checked = ${Boolean(checked)}; + if (element.checked !== checked) { + element.checked = checked; + element.dispatchEvent(new Event('input', { bubbles: true })); + element.dispatchEvent(new Event('change', { bubbles: true })); + } + })()`); + }, + async selectOption(values: unknown, options?: { timeout?: number }) { + await locatorApi.waitFor(options); + const selectedValues = Array.isArray(values) ? values.map(String) : [String(values)]; + const valuesJson = JSON.stringify(selectedValues); + await evaluateInFrame(frameId, `(() => { + const element = (${queryAllExpression})[0] ?? null; + if (!(element instanceof HTMLSelectElement)) { + throw new Error('Frame locator select target is not a select element.'); + } + const values = new Set(${valuesJson}); + for (const option of [...element.options]) { + option.selected = values.has(option.value) || values.has(option.label); + } + element.dispatchEvent(new Event('input', { bubbles: true })); + element.dispatchEvent(new Event('change', { bubbles: true })); + })()`); + return selectedValues; + }, + locator(innerSelector: unknown) { + if (typeof innerSelector !== 'string') { + throw new Error('frame locator only supports string selectors in Web Cap script runtime.'); + } + return createCdpFrameLocator(frameId, `${selector} ${innerSelector}`, `${locatorLabel}.locator(${innerSelector})`); + }, + toString() { + return locatorLabel; + }, + }; + return locatorApi; + }; + const createFrameApi = (frameElement: HTMLIFrameElement | null, metadata: FrameMetadata = {}): RuntimeMethodTable => { + const frameApi: RuntimeMethodTable = { + frameElement() { + return frameElement; + }, + _id() { + return metadata.id ?? ''; + }, + _parentId() { + return metadata.parentId ?? ''; + }, + parentFrame() { + return null; + }, + name() { + return metadata.name ?? frameElement?.name ?? frameElement?.id ?? ''; + }, + url() { + return metadata.url ?? frameDocument(frameElement)?.location?.href ?? ''; + }, + async title() { + return frameDocument(frameElement)?.title ?? ''; + }, + locator(selector: unknown) { + if (typeof selector !== 'string') { + throw new Error('frame.locator only supports string selectors in Web Cap script runtime.'); + } + if (!frameDocument(frameElement) && metadata.id) { + return createCdpFrameLocator(metadata.id, selector, `frame(${metadata.id}).locator(${selector})`); + } + return createLocator( + () => [...(frameDocument(frameElement)?.querySelectorAll(selector) ?? [])], + `frame.locator(${selector})`, + pageApi, + deps, + ); + }, + async waitForSelector(selector: unknown, options: { state?: 'attached' | 'detached' | 'visible' | 'hidden'; timeout?: number } = {}) { + if (!frameDocument(frameElement) && metadata.id) { + const locator = createCdpFrameLocator(metadata.id, String(selector), `frame(${metadata.id}).locator(${String(selector)})`); + await locator.waitFor(options); + return null; + } + return await waitForLocator( + () => [...(frameDocument(frameElement)?.querySelectorAll(String(selector)) ?? [])], + `frame.waitForSelector(${String(selector)})`, + deps.wait, + { timeout: defaultTimeoutMs, ...options }, + ); + }, + }; + frameApi.getByText = (text: unknown, options: { exact?: boolean } = {}) => { + if (!frameDocument(frameElement) && metadata.id) { + const textJson = JSON.stringify(String(text ?? '')); + const exact = options.exact === true; + return createCdpFrameLocator( + metadata.id, + '*', + `frame(${metadata.id}).getByText(${String(text)})`, + `[...document.querySelectorAll('*')].filter((element) => { + const value = String(element.textContent ?? '').replace(/\\s+/g, ' ').trim(); + return ${exact} ? value === ${textJson} : value.includes(${textJson}); + })`, + ); + } + return (frameApi.locator('body') as RuntimeMethodTable).getByText(text, options); + }; + frameApi.getByTestId = (testId: unknown) => { + const selector = `[data-testid="${cssEscape(String(testId))}"]`; + return frameApi.locator(selector); + }; + const cdpAttributeTextLocator = (attribute: string, text: unknown, exact = false, labelName = attribute) => { + if (!metadata.id) { + return null; + } + const textJson = JSON.stringify(String(text ?? '')); + const attributeJson = JSON.stringify(attribute); + return createCdpFrameLocator( + metadata.id, + '*', + `frame(${metadata.id}).getBy${labelName}(${String(text)})`, + `[...document.querySelectorAll('*')].filter((element) => { + const value = String(element.getAttribute(${attributeJson}) ?? '').replace(/\\s+/g, ' ').trim(); + return ${exact} ? value === ${textJson} : value.includes(${textJson}); + })`, + ); + }; + frameApi.getByPlaceholder = (text: unknown, options: { exact?: boolean } = {}) => { + if (!frameDocument(frameElement) && metadata.id) { + return cdpAttributeTextLocator('placeholder', text, options.exact, 'Placeholder'); + } + return (frameApi.locator('body') as RuntimeMethodTable).getByPlaceholder(text, options); + }; + frameApi.getByTitle = (text: unknown, options: { exact?: boolean } = {}) => { + if (!frameDocument(frameElement) && metadata.id) { + return cdpAttributeTextLocator('title', text, options.exact, 'Title'); + } + return (frameApi.locator('body') as RuntimeMethodTable).getByTitle(text, options); + }; + frameApi.getByAltText = (text: unknown, options: { exact?: boolean } = {}) => { + if (!frameDocument(frameElement) && metadata.id) { + const textJson = JSON.stringify(String(text ?? '')); + return createCdpFrameLocator( + metadata.id, + 'img, area', + `frame(${metadata.id}).getByAltText(${String(text)})`, + `[...document.querySelectorAll('img, area')].filter((element) => { + const value = String(element.getAttribute('alt') ?? '').replace(/\\s+/g, ' ').trim(); + return ${options.exact === true} ? value === ${textJson} : value.includes(${textJson}); + })`, + ); + } + return (frameApi.locator('body') as RuntimeMethodTable).getByAltText(text, options); + }; + frameApi.getByLabel = (text: unknown, options: { exact?: boolean } = {}) => { + if (!frameDocument(frameElement) && metadata.id) { + const textJson = JSON.stringify(String(text ?? '')); + return createCdpFrameLocator( + metadata.id, + 'input, textarea, select', + `frame(${metadata.id}).getByLabel(${String(text)})`, + `[...document.querySelectorAll('input, textarea, select')].filter((element) => { + const labels = element.labels ? [...element.labels].map((label) => label.textContent ?? '').join(' ') : ''; + const aria = element.getAttribute('aria-label') ?? ''; + const value = String(aria || labels).replace(/\\s+/g, ' ').trim(); + return ${options.exact === true} ? value === ${textJson} : value.includes(${textJson}); + })`, + ); + } + return (frameApi.locator('body') as RuntimeMethodTable).getByLabel(text, options); + }; + frameApi.getByRole = (role: unknown, options: { name?: unknown; exact?: boolean } = {}) => { + if (!frameDocument(frameElement) && metadata.id) { + const roleJson = JSON.stringify(String(role)); + const nameJson = JSON.stringify(options.name === undefined ? '' : String(options.name)); + const hasName = options.name !== undefined; + return createCdpFrameLocator( + metadata.id, + '*', + `frame(${metadata.id}).getByRole(${String(role)})`, + `[...document.querySelectorAll('*')].filter((element) => { + const explicitRole = element.getAttribute('role') || ''; + const tag = element.tagName.toLowerCase(); + const inputType = (element.getAttribute('type') || 'text').toLowerCase(); + const implicitRole = + tag === 'button' ? 'button' : + tag === 'a' && element.href ? 'link' : + tag === 'img' ? 'img' : + tag === 'textarea' ? 'textbox' : + tag === 'select' ? 'combobox' : + tag === 'input' && ['button', 'submit', 'reset'].includes(inputType) ? 'button' : + tag === 'input' && inputType === 'checkbox' ? 'checkbox' : + tag === 'input' && inputType === 'radio' ? 'radio' : + tag === 'input' && inputType === 'search' ? 'searchbox' : + tag === 'input' ? 'textbox' : ''; + if ((explicitRole || implicitRole) !== ${roleJson}) return false; + if (!${hasName}) return true; + const value = String(element.getAttribute('aria-label') || element.textContent || '').replace(/\\s+/g, ' ').trim(); + return ${options.exact === true} ? value === ${nameJson} : value.includes(${nameJson}); + })`, + ); + } + return (frameApi.locator('body') as RuntimeMethodTable).getByRole(role, options); + }; + for (const method of [] as string[]) { + frameApi[method] = (...args: unknown[]) => + (frameApi.locator('body') as RuntimeMethodTable)[method](...args); + } + return frameApi; + }; + pageApi.__frameForElement = createFrameApi; + Object.assign(pageApi, { + mouse: mouseApi, + async $(selector: unknown) { + return document.querySelector(String(selector)); + }, + async $$(selector: unknown) { + return [...document.querySelectorAll(String(selector))]; + }, + async evaluate(pageFunction: unknown, arg?: unknown) { + if (typeof pageFunction === 'function') { + return await Promise.resolve(pageFunction(arg)); + } + if (typeof pageFunction === 'string') { + return (0, eval)(pageFunction); + } + throw new Error('page.evaluate requires a function or string expression.'); + }, + async waitForSelector(selector: unknown, options: { state?: 'attached' | 'detached' | 'visible' | 'hidden'; timeout?: number } = {}) { + return await waitForLocator(() => [...document.querySelectorAll(String(selector))], `page.waitForSelector(${String(selector)})`, deps.wait, { timeout: defaultTimeoutMs, ...options }); + }, + async addScriptTag(options: { content?: unknown; type?: unknown; url?: unknown } = {}) { + const script = document.createElement('script'); + if (options.type !== undefined) { + script.type = String(options.type); + } + if (options.content !== undefined) { + script.textContent = String(options.content); + } + if (options.url !== undefined) { + await new Promise((resolve, reject) => { + script.addEventListener('load', () => resolve(), { once: true }); + script.addEventListener('error', () => reject(new Error(`Failed to load script ${String(options.url)}`)), { once: true }); + script.src = String(options.url); + document.head.appendChild(script); + }); + return script; + } + document.head.appendChild(script); + return script; + }, + async addStyleTag(options: { content?: unknown; url?: unknown } = {}) { + if (options.url !== undefined) { + const link = document.createElement('link'); + link.rel = 'stylesheet'; + await new Promise((resolve, reject) => { + link.addEventListener('load', () => resolve(), { once: true }); + link.addEventListener('error', () => reject(new Error(`Failed to load stylesheet ${String(options.url)}`)), { once: true }); + link.href = String(options.url); + document.head.appendChild(link); + }); + return link; + } + const style = document.createElement('style'); + style.textContent = String(options.content ?? ''); + document.head.appendChild(style); + return style; + }, + async bringToFront() { + await browserCommand('Page.bringToFront'); + }, + async check(selector: unknown, options?: unknown) { + await (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).setChecked(true, options); + }, + async click(selector: unknown, options?: unknown) { + await (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).click(options); + }, + async close() { + globalThis.close?.(); + }, + async content() { + return document.documentElement.outerHTML; + }, + async dblclick(selector: unknown, options?: unknown) { + await (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).dblclick(options); + }, + async dispatchEvent(selector: unknown, type: unknown, eventInit?: unknown, options?: unknown) { + await (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).dispatchEvent(type, eventInit, options); + }, + async dragAndDrop(source: unknown, target: unknown, options?: unknown) { + const sourceLocator = (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(source)); + const targetLocator = (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(target)); + await sourceLocator.dragTo(targetLocator, options); + }, + async fill(selector: unknown, value: unknown, options?: unknown) { + await (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).fill(value, options); + }, + async emulateMedia(options: { media?: unknown; colorScheme?: unknown; reducedMotion?: unknown } = {}) { + const features = []; + if (options.colorScheme !== undefined) { + features.push({ name: 'prefers-color-scheme', value: String(options.colorScheme) }); + } + if (options.reducedMotion !== undefined) { + features.push({ name: 'prefers-reduced-motion', value: String(options.reducedMotion) }); + } + await browserCommand('Emulation.setEmulatedMedia', { + media: options.media === undefined ? '' : String(options.media), + features, + }); + }, + async focus(selector: unknown, options?: unknown) { + await (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).focus(options); + }, + async frame(options: unknown = {}) { + const frames = await (pageApi.frames as () => Promise)(); + if (typeof options === 'string') { + return frames.find((frame) => frame.name() === options || frame.url() === options) ?? null; + } + const name = options && typeof options === 'object' && 'name' in options ? String((options as { name?: unknown }).name ?? '') : ''; + const url = options && typeof options === 'object' && 'url' in options ? (options as { url?: unknown }).url : undefined; + return frames.find((frame) => { + if (name && frame.name() !== name) { + return false; + } + if (typeof url === 'string' && frame.url() !== url) { + return false; + } + if (url instanceof RegExp && !url.test(frame.url())) { + return false; + } + return true; + }) ?? null; + }, + frameLocator(selector: unknown) { + if (typeof selector !== 'string') { + throw new Error('page.frameLocator only supports string selectors in Web Cap script runtime.'); + } + const queryFrameDocuments = () => + [...document.querySelectorAll(selector)] + .filter((element): element is HTMLIFrameElement => element instanceof HTMLIFrameElement) + .map((element) => element.contentDocument) + .filter((item): item is Document => Boolean(item)); + const frameLocatorApi: RuntimeMethodTable = { + locator(innerSelector: unknown) { + if (typeof innerSelector !== 'string') { + throw new Error('frameLocator.locator only supports string selectors in Web Cap script runtime.'); + } + return createLocator( + () => queryFrameDocuments().flatMap((frameDoc) => [...frameDoc.querySelectorAll(innerSelector)]), + `page.frameLocator(${selector}).locator(${innerSelector})`, + pageApi, + deps, + ); + }, + }; + for (const method of ['getByAltText', 'getByLabel', 'getByPlaceholder', 'getByRole', 'getByTestId', 'getByText', 'getByTitle']) { + frameLocatorApi[method] = (...args: unknown[]) => + (frameLocatorApi.locator('body') as RuntimeMethodTable)[method](...args); + } + return frameLocatorApi; + }, + async frames() { + const metadata = await readFrameMetadata(); + if (metadata.length === 0) { + return [createFrameApi(null, { name: '', url: globalThis.location?.href ?? '' })]; + } + const frameApis = metadata.map((frame, index) => + createFrameApi(index === 0 ? null : findSameOriginFrameElement(frame), frame), + ); + const frameById = new Map(frameApis.map((frame) => [frame._id(), frame])); + for (const frame of frameApis) { + frame.parentFrame = () => frameById.get(frame._parentId()) ?? null; + } + return frameApis; + }, + async goBack() { + globalThis.history?.back(); + return null; + }, + async goForward() { + globalThis.history?.forward(); + return null; + }, + getAttribute(selector: unknown, name: unknown, options?: unknown) { + return (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).getAttribute(name, options); + }, + getByAltText(text: unknown, options?: unknown) { + return (pageApi.locator as (selector: string) => RuntimeMethodTable)('body').getByAltText(text, options); + }, + getByLabel(text: unknown, options?: unknown) { + return (pageApi.locator as (selector: string) => RuntimeMethodTable)('body').getByLabel(text, options); + }, + getByPlaceholder(text: unknown, options?: unknown) { + return (pageApi.locator as (selector: string) => RuntimeMethodTable)('body').getByPlaceholder(text, options); + }, + getByRole(role: unknown, options?: unknown) { + return (pageApi.locator as (selector: string) => RuntimeMethodTable)('body').getByRole(role, options); + }, + getByTestId(testId: unknown) { + return (pageApi.locator as (selector: string) => RuntimeMethodTable)(`[data-testid="${cssEscape(String(testId))}"]`); + }, + getByText(text: unknown, options?: unknown) { + return (pageApi.locator as (selector: string) => RuntimeMethodTable)('body').getByText(text, options); + }, + getByTitle(text: unknown, options?: unknown) { + return (pageApi.locator as (selector: string) => RuntimeMethodTable)('body').getByTitle(text, options); + }, + async hover(selector: unknown, options?: unknown) { + await (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).hover(options); + }, + async hideHighlight() { + hideHighlightOverlay(); + }, + innerHTML(selector: unknown, options?: unknown) { + return (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).innerHTML(options); + }, + innerText(selector: unknown, options?: unknown) { + return (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).innerText(options); + }, + inputValue(selector: unknown, options?: unknown) { + return (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).inputValue(options); + }, + isChecked(selector: unknown) { + return (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).isChecked(); + }, + isClosed() { + return false; + }, + isDisabled(selector: unknown) { + return (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).isDisabled(); + }, + isEditable(selector: unknown) { + return (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).isEditable(); + }, + isEnabled(selector: unknown) { + return (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).isEnabled(); + }, + isHidden(selector: unknown) { + return (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).isHidden(); + }, + isVisible(selector: unknown) { + return (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).isVisible(); + }, + locator(selector: unknown) { + if (typeof selector !== 'string') { + throw new Error('page.locator only supports string selectors in Web Cap script runtime.'); + } + return createLocator(() => [...document.querySelectorAll(selector)], `page.locator(${selector})`, pageApi, deps); + }, + async mainFrame() { + const frames = await (pageApi.frames as () => Promise)(); + return frames[0] ?? createFrameApi(null, { name: '', url: globalThis.location?.href ?? '' }); + }, + async goto(url: unknown) { + globalThis.location.href = String(url); + return null; + }, + async press(selector: unknown, key: unknown, options?: unknown) { + const locator = (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)); + await locator.press(key, options); + }, + async selectOption(selector: unknown, values: unknown, options?: unknown) { + return await (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).selectOption(values, options); + }, + async setChecked(selector: unknown, checked: unknown, options?: unknown) { + await (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).setChecked(checked, options); + }, + async setContent(html: unknown) { + document.open(); + document.write(String(html)); + document.close(); + }, + async tap(selector: unknown, options?: unknown) { + await (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).tap(options); + }, + textContent(selector: unknown, options?: unknown) { + return (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).textContent(options); + }, + async title() { + return document.title; + }, + async type(selector: unknown, text: unknown, options?: unknown) { + await (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).type(text, options); + }, + async uncheck(selector: unknown, options?: unknown) { + await (pageApi.locator as (selector: string) => RuntimeMethodTable)(String(selector)).uncheck(options); + }, + url() { + return globalThis.location?.href ?? ''; + }, + viewportSize() { + return { width: window.innerWidth, height: window.innerHeight }; + }, + async reload() { + globalThis.location.reload(); + return null; + }, + async requestGC() { + await browserCommand('HeapProfiler.collectGarbage'); + }, + async pdf(options: Record = {}) { + const result = await browserCommand<{ data?: string }>('Page.printToPDF', options); + return result.data ?? result; + }, + async screenshot(options: { type?: unknown; quality?: unknown; fullPage?: unknown } = {}) { + const format = options.type === 'jpeg' ? 'jpeg' : 'png'; + const params: Record = { format, fromSurface: true }; + if (format === 'jpeg' && options.quality !== undefined) { + params.quality = Number(options.quality); + } + if (options.fullPage === true) { + const metrics = await browserCommand<{ + contentSize?: { x?: number; y?: number; width?: number; height?: number }; + }>('Page.getLayoutMetrics'); + const contentSize = metrics.contentSize; + if (contentSize) { + params.captureBeyondViewport = true; + params.clip = { + x: Number(contentSize.x ?? 0), + y: Number(contentSize.y ?? 0), + width: Number(contentSize.width ?? window.innerWidth), + height: Number(contentSize.height ?? window.innerHeight), + scale: 1, + }; + } + } + const result = await browserCommand<{ data?: string }>('Page.captureScreenshot', params); + return result.data ?? result; + }, + async setDefaultNavigationTimeout(timeout: unknown) { + defaultTimeoutMs = Math.max(Number(timeout) || 0, 0); + }, + async setDefaultTimeout(timeout: unknown) { + defaultTimeoutMs = Math.max(Number(timeout) || 0, 0); + }, + async setExtraHTTPHeaders(headers: Record = {}) { + await browserCommand('Network.enable'); + await browserCommand('Network.setExtraHTTPHeaders', { headers }); + }, + async setViewportSize(size: { width?: unknown; height?: unknown }) { + const width = Math.max(Math.trunc(Number(size?.width ?? window.innerWidth)), 1); + const height = Math.max(Math.trunc(Number(size?.height ?? window.innerHeight)), 1); + await browserCommand('Emulation.setDeviceMetricsOverride', { + width, + height, + deviceScaleFactor: window.devicePixelRatio || 1, + mobile: false, + }); + }, + async waitForLoadState(state: unknown = 'load', options: { timeout?: number } = {}) { + const target = String(state); + const isReady = () => + target === 'domcontentloaded' + ? document.readyState === 'interactive' || document.readyState === 'complete' + : document.readyState === 'complete'; + if (isReady()) { + if (target === 'networkidle') { + await deps.wait(500); + } + return; + } + const timeout = timeoutFromOptions(options, defaultTimeoutMs); + await Promise.race([ + new Promise((resolve) => { + const eventName = target === 'domcontentloaded' ? 'DOMContentLoaded' : 'load'; + globalThis.addEventListener(eventName, () => resolve(), { once: true }); + }), + deps.wait(timeout).then(() => { + throw new Error(`Timed out after ${timeout}ms waiting for load state ${target}.`); + }), + ]); + if (target === 'networkidle') { + await deps.wait(500); + } + }, + async waitForNavigation(options: { timeout?: number } = {}) { + await browserCommand('Page.enable'); + const event = await browserEvent('Page.frameNavigated', {}, timeoutFromOptions(options, defaultTimeoutMs)); + await (pageApi.waitForLoadState as (state?: unknown, options?: { timeout?: number }) => Promise)('load', options).catch(() => undefined); + return event; + }, + async waitForEvent(event: unknown, options: { timeout?: number; predicate?: unknown } = {}) { + const eventName = String(event); + if (eventName === 'request') { + return await (pageApi.waitForRequest as (urlOrPredicate?: unknown, options?: { timeout?: number }) => Promise)(undefined, options); + } + if (eventName === 'response') { + return await (pageApi.waitForResponse as (urlOrPredicate?: unknown, options?: { timeout?: number }) => Promise)(undefined, options); + } + if (eventName === 'load') { + await (pageApi.waitForLoadState as (state?: unknown, options?: { timeout?: number }) => Promise)('load', options); + return { type: 'load' }; + } + if (eventName === 'domcontentloaded') { + await (pageApi.waitForLoadState as (state?: unknown, options?: { timeout?: number }) => Promise)('domcontentloaded', options); + return { type: 'domcontentloaded' }; + } + if (eventName === 'framenavigated') { + await browserCommand('Page.enable'); + return await browserEvent('Page.frameNavigated', {}, timeoutFromOptions(options, defaultTimeoutMs)); + } + throw new Error(`page.waitForEvent(${eventName}) is not implemented by Web Cap script runtime yet.`); + }, + async waitForRequest(urlOrPredicate: unknown, options: { timeout?: number } = {}) { + await browserCommand('Network.enable'); + return await browserEvent('Network.requestWillBeSent', serializeUrlMatcher(urlOrPredicate), timeoutFromOptions(options, defaultTimeoutMs)); + }, + async waitForResponse(urlOrPredicate: unknown, options: { timeout?: number } = {}) { + await browserCommand('Network.enable'); + return await browserEvent('Network.responseReceived', serializeUrlMatcher(urlOrPredicate), timeoutFromOptions(options, defaultTimeoutMs)); + }, + async waitForTimeout(timeout: unknown) { + await deps.wait(Number(timeout) || 0); + }, + async waitForURL(url: unknown, options: { timeout?: number } = {}) { + const timeout = timeoutFromOptions(options, defaultTimeoutMs); + const startedAt = Date.now(); + while (Date.now() - startedAt <= timeout) { + const currentUrl = globalThis.location?.href ?? ''; + if ( + (typeof url === 'string' && currentUrl === url) || + (url instanceof RegExp && url.test(currentUrl)) || + (typeof url === 'function' && url(new URL(currentUrl))) + ) { + return; + } + await deps.wait(50); + } + throw new Error(`Timed out after ${timeout}ms waiting for URL ${String(url)}.`); + }, + }); + + for (const method of PLAYWRIGHT_PAGE_METHODS) { + if (!(method in pageApi)) { + pageApi[method] = notImplemented(`page.${method}`); + } + } + + return pageApi; +} + + +return createPageApi(); +} diff --git a/extension/runtime/injected/script-runtime.injected.ts b/extension/runtime/injected/script-runtime.injected.ts index 9574132..6728ea7 100644 --- a/extension/runtime/injected/script-runtime.injected.ts +++ b/extension/runtime/injected/script-runtime.injected.ts @@ -1,5 +1,7 @@ /* eslint-disable */ // Mechanically extracted from execution-helpers.ts. Keep behavior changes out of this file. +import { createPlaywrightPageApi } from './playwright-shim.injected'; +import type { ScriptPlaywrightPage } from './playwright-shim-types.injected'; import { installManagedClickHook } from './managed-click.injected'; import { captureVisibleElementsDiff } from './visible-elements.injected'; @@ -72,12 +74,14 @@ type RuntimeApi = { get(scriptId: string): RuntimeJsonObject; list(): RuntimeJsonObject[]; call(scriptId: string, nestedInput?: RuntimeJsonObject): Promise; + page: ScriptPlaywrightPage; typeIntoElement(element: unknown, value: unknown): Promise; waitForManagedInput(): Promise; }; type ManagedKeyboardBridge = (payload: RuntimeJsonObject) => unknown; type ManagedWindowBridge = (payload: RuntimeJsonObject) => unknown; +type ManagedBrowserBridge = (payload: RuntimeJsonObject) => unknown; export interface ScriptRuntimeArgs { scriptDefinition: RuntimeScript; @@ -86,6 +90,7 @@ export interface ScriptRuntimeArgs { managedClickBridgeFunctionName: string | null; managedKeyboardBridgeFunctionName: string | null; managedWindowBridgeFunctionName: string | null; + managedBrowserBridgeFunctionName: string | null; scriptFactories: Record unknown>; } @@ -96,6 +101,7 @@ export async function runScriptRuntime({ managedClickBridgeFunctionName, managedKeyboardBridgeFunctionName, managedWindowBridgeFunctionName, + managedBrowserBridgeFunctionName, scriptFactories, }: ScriptRuntimeArgs) { function validateScalarField( @@ -199,10 +205,10 @@ export async function runScriptRuntime({ const POST_ACTION_VISIBLE_DIFF_DELAY_MS = 200; const MAX_SCROLL_TARGETS = 200; - function hasManagedClickEventSince(eventStartIndex: number) { + function hasPostActionEventSince(eventStartIndex: number) { return context.evidence.events .slice(eventStartIndex) - .some((event) => event.type === 'managed_click'); + .some((event) => event.type === 'managed_click' || event.type === 'managed_mouse'); } function toSchemaSummary(item: RuntimeScript): RuntimeJsonObject { @@ -443,6 +449,10 @@ export async function runScriptRuntime({ return getRuntimeBridge(managedWindowBridgeFunctionName); } + function getManagedBrowserBridge(): ManagedBrowserBridge | null { + return getRuntimeBridge(managedBrowserBridgeFunctionName); + } + function isEditableElement(element: unknown) { const hasInputClass = typeof HTMLInputElement !== 'undefined'; const hasTextareaClass = typeof HTMLTextAreaElement !== 'undefined'; @@ -650,7 +660,7 @@ export async function runScriptRuntime({ }; } - function createApi(): RuntimeApi { + function createApi(page: ScriptPlaywrightPage): RuntimeApi { return { get(scriptId: string) { const nested = context.registry.get(scriptId); @@ -665,6 +675,7 @@ export async function runScriptRuntime({ async call(scriptId: string, nestedInput: RuntimeJsonObject = {}) { return await executeScriptById(scriptId, nestedInput, true); }, + page, async typeIntoElement(element: unknown, value: unknown) { await typeIntoElement(element, value); }, @@ -685,7 +696,33 @@ export async function runScriptRuntime({ ); } - const cap = createApi(); + const page = createPlaywrightPageApi({ + wait, + typeIntoElement, + isEditableElement, + useDomKeyboardFallback: () => !getManagedKeyboardBridge(), + browserCommand: async (method, params = {}) => { + const bridgeFunction = getManagedBrowserBridge(); + if (!bridgeFunction) { + throw new Error('Browser-level Playwright API requires the debugger CDP bridge.'); + } + return await Promise.resolve(bridgeFunction({ action: 'command', method, params })); + }, + browserEvent: async (method, params = {}, timeoutMs) => { + const bridgeFunction = getManagedBrowserBridge(); + if (!bridgeFunction) { + throw new Error('Browser-level Playwright API requires the debugger CDP bridge.'); + } + return await Promise.resolve(bridgeFunction({ action: 'waitForEvent', method, params, timeoutMs })); + }, + recordEvidenceEvent: (type, value) => { + context.evidence.events.push({ type, value }); + }, + waitForManagedInput: async () => { + await context.pendingAsyncOperations; + }, + }); + const cap = createApi(page); const scriptFunction = scriptFactories[item.id]; if (typeof scriptFunction !== 'function') { throw new Error( @@ -693,9 +730,11 @@ export async function runScriptRuntime({ ); } - const runtimeGlobal = globalThis as typeof globalThis & { cap?: RuntimeApi }; + const runtimeGlobal = globalThis as typeof globalThis & { cap?: RuntimeApi; page?: ScriptPlaywrightPage }; const previousCap = runtimeGlobal.cap; + const previousPage = runtimeGlobal.page; runtimeGlobal.cap = cap; + runtimeGlobal.page = page; const visibleElementsStartedAt = Date.now(); const beforeSnapshotStartedAt = Date.now(); const beforeVisibleElements = context.visibleElementsTracker.snapshot(); @@ -716,10 +755,15 @@ export async function runScriptRuntime({ } else { runtimeGlobal.cap = previousCap; } + if (previousPage === undefined) { + delete runtimeGlobal.page; + } else { + runtimeGlobal.page = previousPage; + } } let postActionDelayMs = 0; - if (typeof document !== 'undefined' && hasManagedClickEventSince(scriptEventStartIndex)) { + if (typeof document !== 'undefined' && hasPostActionEventSince(scriptEventStartIndex)) { const postActionDelayStartedAt = Date.now(); await wait(POST_ACTION_VISIBLE_DIFF_DELAY_MS); postActionDelayMs = Date.now() - postActionDelayStartedAt; diff --git a/extension/runtime/managed-input-bridge.ts b/extension/runtime/managed-input-bridge.ts index 3162f07..28c0283 100644 --- a/extension/runtime/managed-input-bridge.ts +++ b/extension/runtime/managed-input-bridge.ts @@ -45,6 +45,14 @@ interface DebuggerManagedTimerPayload { delayMs?: number; } +interface DebuggerManagedBrowserPayload { + id: string; + action?: 'command' | 'waitForEvent'; + method?: string; + params?: Record; + timeoutMs?: number; +} + interface PointerPosition { x: number; y: number; @@ -261,6 +269,47 @@ export class ManagedInputBridgeFactory { ); } + async createManagedBrowserBridge( + target: DebuggeeTarget, + scope: ManagedInputBridgeExecutionScope = this.createExecutionScope(), + ): Promise { + const bridgeSuffix = this.createBridgeSuffix(scope, 'browser'); + const bindingName = `__webCapDebuggerBrowserBinding_${bridgeSuffix}`; + const bridgeFunctionName = `__webCapManagedBrowserBridge_${bridgeSuffix}`; + const resolverStoreName = `__webCapManagedBrowserResolvers_${bridgeSuffix}`; + + await this.client.sendCommand(target, 'Runtime.addBinding', { + name: bindingName, + }); + + const listener = ( + source: DebuggeeTarget, + method: string, + params?: Record, + ) => { + if (source.tabId !== target.tabId || method !== 'Runtime.bindingCalled') { + return; + } + + const event = params as RuntimeBindingCalledEvent | undefined; + if (event?.name !== bindingName || !event.payload) { + return; + } + + void this.handleManagedBrowserBinding(target, resolverStoreName, event.payload); + }; + this.client.getChromeApi()?.debugger?.onEvent?.addListener(listener); + + await this.client.sendCommand(target, 'Runtime.evaluate', { + expression: this.buildBrowserBridgeInstaller(bindingName, bridgeFunctionName, resolverStoreName), + awaitPromise: true, + returnByValue: true, + allowUnsafeEvalBlockedByCSP: true, + }); + + return this.createDisposableBridge(target, listener, bindingName, bridgeFunctionName, resolverStoreName); + } + private createDisposableBridge( target: DebuggeeTarget, listener: ( @@ -435,6 +484,47 @@ export class ManagedInputBridgeFactory { this.timerHandles.set(timerKey, timer); } + private async handleManagedBrowserBinding( + target: DebuggeeTarget, + resolverStoreName: string, + payloadJson: string, + ): Promise { + let payload: DebuggerManagedBrowserPayload | undefined; + try { + payload = JSON.parse(payloadJson) as DebuggerManagedBrowserPayload; + } catch { + return; + } + + if (!payload?.id || !payload.method) { + return; + } + + try { + const result = + payload.action === 'waitForEvent' + ? await this.waitForDebuggerEvent( + target, + String(payload.method), + this.readRecord(payload.params) ?? {}, + Math.max(Number(payload.timeoutMs ?? 5000), 0), + ) + : await this.client.sendCommand( + target, + String(payload.method), + this.readRecord(payload.params) ?? {}, + ); + await this.resolveManagedPromise(target, resolverStoreName, payload.id, result); + } catch (error) { + await this.rejectManagedPromise( + target, + resolverStoreName, + payload.id, + error instanceof Error ? error.message : String(error), + ); + } + } + private async dispatchManagedMouse( target: DebuggeeTarget, payload: DebuggerManagedClickPayload, @@ -522,6 +612,73 @@ export class ManagedInputBridgeFactory { this.pointerPositions.set(target.tabId, { x, y }); } + private async waitForDebuggerEvent( + target: DebuggeeTarget, + eventMethod: string, + matcher: Record, + timeoutMs: number, + ): Promise> { + return await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + cleanup(); + reject(new Error(`Timed out after ${timeoutMs}ms waiting for ${eventMethod}.`)); + }, timeoutMs); + const cleanup = () => { + clearTimeout(timeout); + this.client.getChromeApi()?.debugger?.onEvent?.removeListener(listener); + }; + const listener = ( + source: DebuggeeTarget, + method: string, + params?: Record, + ) => { + if (source.tabId !== target.tabId || method !== eventMethod) { + return; + } + if (!this.matchesDebuggerEvent(params ?? {}, matcher)) { + return; + } + cleanup(); + resolve(params ?? {}); + }; + this.client.getChromeApi()?.debugger?.onEvent?.addListener(listener); + }); + } + + private matchesDebuggerEvent( + params: Record, + matcher: Record, + ): boolean { + const url = this.readEventUrl(params); + const expectedUrl = typeof matcher.url === 'string' ? matcher.url : ''; + if (expectedUrl && url !== expectedUrl) { + return false; + } + const regexSource = typeof matcher.regexSource === 'string' ? matcher.regexSource : ''; + if (regexSource) { + const regex = new RegExp( + regexSource, + typeof matcher.regexFlags === 'string' ? matcher.regexFlags : '', + ); + if (!regex.test(url)) { + return false; + } + } + return true; + } + + private readEventUrl(params: Record): string { + const request = this.readRecord(params.request); + if (typeof request?.url === 'string') { + return request.url; + } + const response = this.readRecord(params.response); + if (typeof response?.url === 'string') { + return response.url; + } + return typeof params.url === 'string' ? params.url : ''; + } + private async sendMouseEvent( target: DebuggeeTarget, payload: Record & { debug?: Record }, @@ -725,6 +882,7 @@ export class ManagedInputBridgeFactory { target: DebuggeeTarget, resolverStoreName: string, id: string, + value?: unknown, ): Promise { await this.client.sendCommand(target, 'Runtime.evaluate', { expression: ` @@ -735,7 +893,7 @@ export class ManagedInputBridgeFactory { return; } delete store[${JSON.stringify(id)}]; - entry.resolve(); + entry.resolve(${JSON.stringify(value ?? null)}); })(); `, awaitPromise: true, @@ -1050,6 +1208,45 @@ export class ManagedInputBridgeFactory { })); }); }; +})(); + `; + } + + private buildBrowserBridgeInstaller( + bindingName: string, + bridgeFunctionName: string, + resolverStoreName: string, + ): string { + return ` +(() => { + const bindingName = ${JSON.stringify(bindingName)}; + const bridgeFunctionName = ${JSON.stringify(bridgeFunctionName)}; + const resolverStoreName = ${JSON.stringify(resolverStoreName)}; + const binding = globalThis[bindingName]; + if (typeof binding !== 'function') { + throw new Error(\`Debugger binding \${bindingName} was not installed.\`); + } + + globalThis[resolverStoreName] = Object.create(null); + globalThis[bridgeFunctionName] = (payload) => { + const id = + typeof payload?.id === 'string' && payload.id.length > 0 + ? payload.id + : \`\${Date.now()}-\${Math.random().toString(16).slice(2)}\`; + return new Promise((resolve, reject) => { + globalThis[resolverStoreName][id] = { resolve, reject }; + binding(JSON.stringify({ + id, + action: payload?.action === 'waitForEvent' ? 'waitForEvent' : 'command', + method: typeof payload?.method === 'string' ? payload.method : '', + params: + payload && typeof payload.params === 'object' && payload.params !== null + ? payload.params + : {}, + timeoutMs: Number(payload?.timeoutMs ?? 0), + })); + }); + }; })(); `; } diff --git a/lib/cli-parser.ts b/lib/cli-parser.ts index eba4a28..97dd573 100644 --- a/lib/cli-parser.ts +++ b/lib/cli-parser.ts @@ -319,6 +319,9 @@ function scriptExecutionHelp(): string { return one JSON object, and can use cap.call(...) inside the script to call reusable capabilities. + Scripts also receive a Playwright-style page API as global page and cap.page. +${scriptRuntimeApiHelp(' ')} + --script Script source code to run in the browser tab. --script-file Read script source code from a file. --input JSON object passed to the script. Defaults to {}. @@ -329,6 +332,15 @@ function scriptExecutionHelp(): string { `; } +function scriptRuntimeApiHelp(indent = ''): string { + return `${indent}Use page.locator(...) and locator actions such as click(), fill(), count(), +${indent}textContent(), first(), nth(), waitFor(), and getByRole()/getByText(). + +${indent}Example: +${indent} await page.getByRole('button', { name: 'Login' }).click(); +${indent} await page.locator('input[name=email]').fill(input.email);`; +} + function createCommandParser(commandName: CliCommandName): Command { switch (commandName) { case 'mcp': @@ -412,7 +424,7 @@ function createScriptGetParser(): Command { function createScriptExecuteParser(): Command { return createParser('script-execute') .description( - 'Run JavaScript in the selected browser tab with JSON input and observable page evidence.', + 'Run JavaScript in the selected browser tab with JSON input, observable page evidence, and Playwright-style page/locator helpers.', ) .option('--script ', 'Script source code to run in the browser tab.') .option('--script-file ', 'Read script source code from a file.') @@ -446,7 +458,11 @@ function createWaitEventsParser(): Command { } function helpForCommand(command: Command): CliCommand { - return { name: 'help', text: command.helpInformation() }; + const runtimeHelp = + command.name() === 'script-execute' + ? `\nRuntime script APIs:\n page / cap.page Playwright-style Page helper for the current tab.\n page.locator() Create a Playwright-style Locator helper.\n${scriptRuntimeApiHelp(' ')}\n` + : ''; + return { name: 'help', text: `${command.helpInformation()}${runtimeHelp}` }; } function createParser(name: string): Command { diff --git a/package.json b/package.json index f344dc4..7671083 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "@wxt-dev/module-vue": "^1.0.2", "cross-spawn": "^7.0.6", "eslint": "^10.4.0", + "playwright-core": "1.60.0", "rollup": "^4.60.4", "rollup-plugin-esbuild": "^6.2.1", "tsx": "^4.21.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9346fae..440387e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -72,6 +72,9 @@ importers: eslint: specifier: ^10.4.0 version: 10.4.0(jiti@2.6.1) + playwright-core: + specifier: 1.60.0 + version: 1.60.0 rollup: specifier: ^4.60.4 version: 4.60.4 @@ -2179,6 +2182,11 @@ packages: pkg-types@2.3.0: resolution: {integrity: sha512-SIqCzDRg0s9npO5XQ3tNZioRY1uK06lA41ynBC1YmFTmnY6FjUjVt6s4LoADmwoig1qqD0oK8h1p/8mlMx8Oig==} + playwright-core@1.60.0: + resolution: {integrity: sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA==} + engines: {node: '>=18'} + hasBin: true + postcss@8.5.10: resolution: {integrity: sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==} engines: {node: ^10 || ^12 || >=14} @@ -4818,6 +4826,8 @@ snapshots: exsolve: 1.0.8 pathe: 2.0.3 + playwright-core@1.60.0: {} + postcss@8.5.10: dependencies: nanoid: 3.3.11 diff --git a/scripts/generate-script-runtime.ts b/scripts/generate-script-runtime.ts index 0f9cbe6..497dfed 100644 --- a/scripts/generate-script-runtime.ts +++ b/scripts/generate-script-runtime.ts @@ -8,6 +8,10 @@ const currentDir = dirname(fileURLToPath(import.meta.url)); const runtimeSourcePaths = [ resolve(currentDir, '../extension/runtime/injected/visible-elements.injected.ts'), resolve(currentDir, '../extension/runtime/injected/managed-click.injected.ts'), + resolve(currentDir, '../extension/runtime/injected/playwright-shim-types.injected.ts'), + resolve(currentDir, '../extension/runtime/injected/playwright-shim-helpers.injected.ts'), + resolve(currentDir, '../extension/runtime/injected/playwright-locator.injected.ts'), + resolve(currentDir, '../extension/runtime/injected/playwright-shim.injected.ts'), resolve(currentDir, '../extension/runtime/injected/script-runtime.injected.ts'), ]; const generatedPath = resolve(currentDir, '../extension/runtime/injected/script-runtime.generated.ts'); @@ -33,7 +37,9 @@ function transpileRuntimeModule(source: string): string { return transpiled.outputText .replace(/^\s*import\s+.+?;\s*$/gm, '') .replace(/^export\s+(async\s+function\s+runScriptRuntime)/m, '$1') + .replace(/^export\s+(async\s+function\s+)/gm, '$1') .replace(/^export\s+(function\s+)/gm, '$1') + .replace(/^\s*export\s+\{\};\s*$/gm, '') .trim(); } diff --git a/skills/web-cap/SKILL.md b/skills/web-cap/SKILL.md index aa4adfd..130bdce 100644 --- a/skills/web-cap/SKILL.md +++ b/skills/web-cap/SKILL.md @@ -1,85 +1,79 @@ --- name: web-cap -description: Use when installing, configuring, or using the web-cap CLI from the web-capability npm package for local-first browser automation. Applies to running browser automation from the command line, connecting the browser extension runtime, inspecting tabs, searching/executing/registering reusable page scripts, and optionally starting the MCP adapter with `web-cap mcp`. +description: Use when you need to operate or inspect a real browser tab through local-first browser automation, including clicking, typing, reading page state, extracting data, navigating pages, or running reusable scripts against live browser sessions. --- # Web Cap CLI -## What Web Cap Is +## Overview -Web Cap is a command-line browser automation toolkit for agents. The npm package is `web-capability`; the installed CLI command is `web-cap`. +Web Cap is a command-line browser automation toolkit for agents. The npm package is `web-capability`, and the installed CLI command is `web-cap`. -Start from the CLI workflow. MCP is only one CLI subcommand (`web-cap mcp`) for tools that need a stdio MCP adapter. +## Check And Install -## Install - -Install the package from npm when the project does not already provide it: +Check whether the Web Cap browser extension/runtime is connected before running page automation: ```bash -npm install -g web-capability +web-cap session-status ``` -For local development inside the Web Cap repository, use: +If the `web-cap` command is not available, install the CLI: ```bash -pnpm install -pnpm cli --help +npm install -g web-capability ``` -The browser side still requires the Web Cap extension/runtime to be loaded and connected. Use `web-cap session-status` first to check whether a browser runtime is connected. +The browser side requires the Web Cap extension/runtime to be loaded and connected. Re-run `web-cap session-status` after installation or extension setup before executing page scripts. -## Core CLI Workflow +## Run Scripts -Use the CLI directly before considering MCP: +Prefer running scripts directly with `script-execute`: ```bash web-cap session-status -web-cap script-search "inspect page" -web-cap script-get builtin.page.inspect -web-cap script-execute --script "async () => ({ ok: true, title: document.title, url: location.href })" +web-cap script-execute --script "export default async function () { return { ok: true, title: document.title, url: location.href }; }" ``` -In this repository, replace `web-cap` with `pnpm cli` when running against source: - -```bash -pnpm cli session-status -pnpm cli script-search "inspect page" -pnpm cli script-get builtin.page.inspect -``` - -## Working Rules - Treat Web Cap scripts as reusable browser capabilities, not just throwaway snippets: 1. Check browser context with `session-status` when the active tab matters. -2. Search existing scripts when the task is common, repeated, risky, or site-specific. -3. Inspect promising scripts with `script-get`. -4. Execute scripts with `script-execute`. -5. Register reliable scripts with `script-register` or `script-execute --register`. +2. Prefer script files for anything longer than a tiny one-off read. +3. Execute scripts with `script-execute --script-file ` and pass variable data through `--input` or `--input-file`. +4. Return structured JSON objects, including `ok`, `url`, and `title` when useful. -Prefer the least permanent option that still makes the task reliable. +Use one-off inline scripts only for very small reads such as `document.title`, `location.href`, or a short visible text fragment. -## When To Search First +## Script Files -Run `script-search` before writing new JavaScript when any of these are true: +For reusable automation, write a normal script file and call it whenever needed: -- The task is common or generic: extract visible text, click by text, fill a form, gather links, inspect page state, summarize a table, handle pagination, or scrape repeated cards. -- The target site is known or stable enough that a site-specific script may already exist. -- The operation is risky or stateful: buying, deleting, sending, submitting, changing settings, auth flows, admin pages, or anything where a tested script is safer. -- The same operation will likely be repeated across tabs, sites, or future turns. -- You are unsure of page structure and need a quicker way to discover available built-ins. - -Use broad search terms first, then site-specific filters when useful: +```javascript +// scripts/read-page-summary.js +export default async function (input) { + const heading = await page.locator("h1").first().textContent().catch(() => ""); + const links = await page.locator("a").evaluateAll((items, limit) => + items.slice(0, limit).map((el) => ({ + text: (el.textContent || "").replace(/\s+/g, " ").trim(), + href: el.href || "" + })), + input.limit ?? 20 + ); + + return { + ok: true, + url: location.href, + title: document.title, + heading, + links + }; +} +``` ```bash -web-cap script-search "extract visible page text" -web-cap script-search "click element by text" --site bilibili.com -web-cap script-search "notifications messages list" --site message.bilibili.com +web-cap script-execute --script-file scripts/read-page-summary.js --input '{"limit":10}' ``` -Skip search for tiny one-off reads where direct DOM inspection is faster and harmless, such as reading `document.title`, `location.href`, or a small visible text fragment. - -## Execute Scripts +## Script Guidelines Write scripts as small browser functions with clear boundaries: @@ -94,78 +88,53 @@ Write scripts as small browser functions with clear boundaries: Example one-off read: ```bash -web-cap script-execute --script "async () => ({ ok: true, title: document.title, url: location.href, text: document.body.innerText.slice(0, 4000) })" +web-cap script-execute --script "export default async function () { return { ok: true, title: document.title, url: location.href, text: document.body.innerText.slice(0, 4000) }; }" ``` -Example structured page scan: +## Page Operations + +When a script operates on page content, prefer the Playwright-compatible runtime APIs exposed as global `page` and `cap.page`. Use `page.locator(...)`, role/text helpers, and locator actions instead of hand-rolled DOM clicks or form mutations. + +Example form interaction: ```javascript -async (input) => { - const visible = (el) => { - const s = getComputedStyle(el); - const r = el.getBoundingClientRect(); - return s.display !== "none" && s.visibility !== "hidden" && r.width > 0 && r.height > 0; +export default async function (input) { + await page.locator('input[name="email"]').fill(input.email); + await page.locator('input[name="password"]').fill(input.password); + await page.getByRole("button", { name: "Login" }).click(); + + return { + ok: true, + url: location.href, + title: document.title }; - const items = [...document.querySelectorAll("a, button, [role=button], li, [class*=item]")] - .filter(visible) - .map((el) => ({ - text: (el.innerText || el.textContent || "").replace(/\s+/g, " ").trim(), - href: el.href || "", - aria: el.getAttribute("aria-label") || "" - })) - .filter((x) => x.text || x.href || x.aria); - return { ok: true, url: location.href, title: document.title, items }; } ``` -## Register Reusable Scripts - -Register a script when a searched-for reusable capability did not already exist and the temporary script has become reliable enough to preserve. - -Register after: - -- The script has a stable purpose and input/output contract. -- It is parameterized through input instead of hard-coded to the current user request. -- It returns structured JSON output. -- It has been tested once on the live page or workflow. - -Do not register when: - -- The code contains one-off user data, temporary selectors, or task-specific constants. -- The script mutates live state without explicit safety design and clear inputs. -- The page structure is still unknown and the code is exploratory. -- The operation was too trivial to search for in the first place. - -## Reuse Scripts - -After `script-search`, use `script-get` for promising scripts before execution. Then call by script id or compose from another script via `cap.call`. - -Temporary script ids from recent `script-execute` runs can also be reused while they remain in local history. Use temporary reuse when iterating on the same page during one investigation; promote to a registered script once the pattern is reusable. - -When composing scripts, keep the outer inline script small: +Example repeated item extraction: ```javascript -async (input) => { - const page = await cap.call("extract-visible-page-state", { - includeLinks: true, - maxItems: 100 - }); - return { ok: true, source: page.url, items: page.items.filter((x) => /消息|通知/.test(x.text)) }; +export default async function (input) { + const cards = page.locator(input.cardSelector); + const count = Math.min(await cards.count(), input.limit ?? 20); + const items = []; + + for (let index = 0; index < count; index += 1) { + const card = cards.nth(index); + items.push({ + title: await card.locator(input.titleSelector).first().textContent().catch(() => ""), + href: await card.locator("a").first().getAttribute("href").catch(() => "") + }); + } + + return { ok: true, url: location.href, title: document.title, items }; } ``` ## Stateful Site Actions -For actions that change a user's account or site state, prefer browser-visible UI operations such as clicking the page's own buttons and verifying the resulting page state. Avoid calling a site's private or semi-private HTTP APIs directly, even from the same-origin page context, unless the user explicitly asks for API-based execution or the UI path is unavailable and the tradeoff is explained first. - -When opening pages or tabs only to perform an operation, close those temporary pages after the operation and verification are complete, unless the user asked to keep them open or the page is useful context for the next step. - -## MCP Is Optional +For actions that change a user's account or site state, prefer browser-visible UI operations such as clicking the page's own buttons and verifying the resulting page state. -Use MCP only when the surrounding agent/tooling expects a stdio MCP server: +Avoid calling a site's private or semi-private HTTP APIs directly, even from the same-origin page context, unless the user explicitly asks for API-based execution or the UI path is unavailable and the tradeoff is explained first. -```bash -web-cap mcp -``` - -Do not frame Web Cap as primarily MCP. The normal path is the npm package plus the `web-cap` command-line interface. +When opening pages or tabs only to perform an operation, close those temporary pages after the operation and verification are complete, unless the user asked to keep them open or the page is useful context for the next step. diff --git a/tests/background-click-routing.test.ts b/tests/background-click-routing.test.ts index 77d2d09..dc998e9 100644 --- a/tests/background-click-routing.test.ts +++ b/tests/background-click-routing.test.ts @@ -162,6 +162,88 @@ export default async function () { expect(scriptRequiresBrowserLevelClick(script, [])).toBe(true); }); + it('routes Playwright mouse scripts to debugger', () => { + const script = scriptDefinitionSchema.parse({ + id: 'playwright.mouse', + name: 'Playwright Mouse', + version: '1.0.0', + status: 'active', + type: 'act', + summary: 'Uses Playwright-style page.mouse.', + target: { + site: 'generic-web', + urlPatterns: ['http://*', 'https://*'], + pageHints: [], + }, + tags: ['test'], + inputSchema: { + type: 'object', + properties: {}, + required: [], + additionalProperties: false, + }, + outputSchema: { + type: 'object', + properties: {}, + required: [], + additionalProperties: true, + }, + script: { + timeoutMs: 1_000, + code: ` +export default async function () { + await page.mouse.move(10, 20); + await page.mouse.down(); + await page.mouse.up(); + return { ok: true }; +} + `.trim(), + }, + }); + + expect(scriptRequiresBrowserLevelClick(script, [])).toBe(true); + }); + + it('routes Playwright drag scripts to debugger', () => { + const script = scriptDefinitionSchema.parse({ + id: 'playwright.drag', + name: 'Playwright Drag', + version: '1.0.0', + status: 'active', + type: 'act', + summary: 'Uses Playwright-style locator.dragTo.', + target: { + site: 'generic-web', + urlPatterns: ['http://*', 'https://*'], + pageHints: [], + }, + tags: ['test'], + inputSchema: { + type: 'object', + properties: {}, + required: [], + additionalProperties: false, + }, + outputSchema: { + type: 'object', + properties: {}, + required: [], + additionalProperties: true, + }, + script: { + timeoutMs: 1_000, + code: ` +export default async function () { + await page.locator('#source').dragTo(page.locator('#target')); + return { ok: true }; +} + `.trim(), + }, + }); + + expect(scriptRequiresBrowserLevelClick(script, [])).toBe(true); + }); + it('keeps non-click scripts on the user-script path', () => { const script = scriptDefinitionSchema.parse({ id: 'read.only', diff --git a/tests/execution-helpers.test.ts b/tests/execution-helpers.test.ts index 70f998f..d09673c 100644 --- a/tests/execution-helpers.test.ts +++ b/tests/execution-helpers.test.ts @@ -101,6 +101,7 @@ describe('execution helpers', () => { expect(scriptRuntimeSource).toContain('async function runScriptRuntime'); expect(scriptRuntimeSource).toContain('MutationObserver'); expect(scriptRuntimeSource).toContain('managed_click'); + expect(scriptRuntimeSource).toContain('managed_mouse'); expect(scriptRuntimeSource).toContain('function installManagedClickHook('); expect(scriptRuntimeSource).toContain('async waitForManagedInput()'); expect(scriptRuntimeSource).toContain('function managedMouseDispatch('); @@ -360,6 +361,89 @@ export default async function (input) { }); }); + it('records Playwright mouse actions as managed action evidence', async () => { + const commands: Array<{ method: string; params: Record }> = []; + const bridgeName = '__webCapTestBrowserBridge'; + (globalThis as typeof globalThis & Record)[bridgeName] = ( + payload: Record, + ) => { + if (payload.action === 'command') { + commands.push({ + method: String(payload.method), + params: payload.params as Record, + }); + } + return {}; + }; + + try { + const script = scriptDefinitionSchema.parse({ + id: 'mouse.click', + name: 'Mouse Click', + version: '1.0.0', + status: 'active', + type: 'act', + summary: 'Clicks through the Playwright mouse shim.', + target: { + site: 'generic-web', + urlPatterns: ['http://*', 'https://*'], + pageHints: [], + }, + tags: ['test'], + inputSchema: { + type: 'object', + properties: {}, + required: [], + additionalProperties: false, + }, + outputSchema: { + type: 'object', + properties: { + ok: { type: 'boolean' }, + }, + required: ['ok'], + additionalProperties: false, + }, + script: { + timeoutMs: 1_000, + code: ` +export default async function () { + await page.mouse.click(12, 34); + return { ok: true }; +} + `.trim(), + }, + }); + + const expression = buildScriptExecutionExpression(script, {}, [], { + managedBrowserBridgeFunctionName: bridgeName, + }); + const response = (await eval(expression)) as { + ok: boolean; + evidence?: { events: Array<{ type: string; value: Record }> }; + }; + + expect(response.ok).toBe(true); + expect(commands.map((command) => command.method)).toEqual([ + 'Input.dispatchMouseEvent', + 'Input.dispatchMouseEvent', + 'Input.dispatchMouseEvent', + ]); + expect(response.evidence?.events).toContainEqual({ + type: 'managed_mouse', + value: { + action: 'up', + x: 12, + y: 34, + buttons: 0, + button: 'left', + }, + }); + } finally { + delete (globalThis as typeof globalThis & Record)[bridgeName]; + } + }); + it('routes user script setTimeout through the managed timer bridge when provided', async () => { const originalSetTimeout = globalThis.setTimeout; const originalClearTimeout = globalThis.clearTimeout; diff --git a/tests/web-cap-cli.test.ts b/tests/web-cap-cli.test.ts index a40c547..f9fb8b2 100644 --- a/tests/web-cap-cli.test.ts +++ b/tests/web-cap-cli.test.ts @@ -185,6 +185,9 @@ describe('WEB_CAP CLI', () => { expect(stdout).toContain('web-cap script-execute --script '); expect(stdout).toContain('Runs JavaScript in the selected browser tab.'); expect(stdout).toContain('use cap.call(...) inside the script'); + expect(stdout).toContain('Playwright-style page API as global page and cap.page'); + expect(stdout).toContain("await page.getByRole('button', { name: 'Login' }).click();"); + expect(stdout).toContain("await page.locator('input[name=email]').fill(input.email);"); expect(stdout).toContain('--script-file '); expect(stdout).toContain('--timeout-ms '); expect(stdout).not.toContain('--definition-file'); @@ -204,6 +207,10 @@ describe('WEB_CAP CLI', () => { expect(stderr).toBe(''); expect(stdout).toContain('Usage: script-execute [options]'); expect(stdout).toContain('Run JavaScript in the selected browser tab'); + expect(stdout).toContain('Playwright-style page/locator helpers'); + expect(stdout).toContain('Runtime script APIs:'); + expect(stdout).toContain('page / cap.page'); + expect(stdout).toContain('page.locator()'); expect(stdout).not.toContain('inline script code through the local runtime daemon'); expect(stdout).toContain('--script '); expect(stdout).toContain('--script-file '); @@ -224,6 +231,8 @@ describe('WEB_CAP CLI', () => { expect(stderr).toBe(''); expect(stdout).toContain('Usage: script-execute [options]'); expect(stdout).toContain('--script '); + expect(stdout).toContain('Runtime script APIs:'); + expect(stdout).toContain('page.locator()'); expect(stdout).not.toContain('Usage: web-cap [options]'); expect(stdout).not.toContain('browser-new-tab'); });