From 1a95e19a162f1da12fd822bab3f419ae83fbc472 Mon Sep 17 00:00:00 2001 From: EdgeStorage Date: Sat, 30 May 2026 23:41:28 +0800 Subject: [PATCH 1/3] feat: add browser screenshot capture --- README.md | 13 +- README.zh-CN.md | 13 +- extension/entrypoints/background.ts | 147 +++++++- extension/runtime/browser-command-handler.ts | 76 ++++- extension/runtime/debugger-executor.ts | 6 + extension/runtime/execution-helpers.ts | 17 + .../injected/playwright-locator.injected.ts | 9 +- .../playwright-shim-types.injected.ts | 7 + .../injected/playwright-shim.injected.ts | 9 +- .../injected/script-runtime.injected.ts | 63 +++- extension/runtime/user-script-executor.ts | 2 + lib/cli-parser.ts | 38 ++- lib/cli.ts | 10 + lib/server/agent/contracts.ts | 3 + lib/server/app-rpc.ts | 13 + lib/server/app.ts | 6 + lib/server/browser/command-contracts.ts | 2 + lib/server/browser/screenshot-store.ts | 185 ++++++++++ lib/server/browser/session-service.ts | 51 +++ .../runtime/websocket-runtime-bridge.ts | 317 ++++++++++++++++-- lib/server/tool-contracts.ts | 38 ++- shared/browser-command-contracts.ts | 12 +- shared/protocol.ts | 37 ++ skills/web-cap/SKILL.md | 8 + tests/browser-command-contracts.test.ts | 21 ++ tests/web-cap-app.test.ts | 159 ++++++++- tests/web-cap-cli.test.ts | 49 ++- 27 files changed, 1263 insertions(+), 48 deletions(-) create mode 100644 lib/server/browser/screenshot-store.ts diff --git a/README.md b/README.md index 8d9886a..3003800 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Compared with action-first browser tools, Web Cap focuses on: - In-page execution, so scripts can work directly with the DOM and page state. - Reusable capabilities, so successful scripts can be searched, inspected, and called again. -- Composable scripts, so one script can call another through `cap.call(...)`. +- Deprecated: composable scripts, where one script calls another through `cap.call(...)`. - Optional post-execution observation, so script runs can return evidence about what changed on the page when evidence collection is enabled. - Local persistence, so agent-learned workflows can survive beyond a single run. - CLI access, so agents can use the same browser capabilities from normal command-line workflows. @@ -135,7 +135,7 @@ A typical agent flow is: Execute script code in the selected browser tab. Scripts receive one object argument and return one JSON object. -`script-execute` accepts optional execution settings such as `--timeout-ms`, `--script-file`, `--input-file`, and `--register`. During execution, scripts can call other scripts through `cap.call(scriptId, input)`. `--register` saves the inline script only after execution succeeds with `ok: true`. +`script-execute` accepts optional execution settings such as `--timeout-ms`, `--script-file`, `--input-file`, and `--register`. `--register` saves the inline script only after execution succeeds with `ok: true`. ### Browser commands @@ -147,21 +147,22 @@ Scripts are JavaScript functions with JSON-compatible inputs and outputs: ```js export default async function (input) { - const page = await cap.call('builtin.page.inspect', {}); + const heading = await page.locator('h1').first().textContent().catch(() => ''); return { ok: true, - title: page.title, + title: document.title, + heading, selector: input.selector, }; } ``` -The runtime injects `cap` while the script executes. +The runtime injects `page` and `cap.page` while the script executes. Available runtime helpers: -- `cap.call(scriptId, input)` - call a built-in or registered script. +- Deprecated: `cap.call(scriptId, input)` - call a built-in or registered script. - `cap.get(scriptId)` - read one script schema summary. - `cap.list()` - list callable script schema summaries. diff --git a/README.zh-CN.md b/README.zh-CN.md index 6c68db4..fc8faaa 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -41,7 +41,7 @@ Agent 可以在页面内运行 JavaScript，组合内置能力，并把有用脚 - 页面内执行，脚本可以直接访问 DOM 和页面状态。 - 能力复用，成功脚本可以被搜索、查看并再次调用。 -- 脚本组合，一个脚本可以通过 `cap.call(...)` 调用另一个脚本。 +- 已废弃：脚本组合，即一个脚本通过 `cap.call(...)` 调用另一个脚本。 - 可选的执行后观察，在启用证据采集时脚本运行可以返回页面变化证据。 - 本地持久化，让 agent 学到的工作流不只存在于单次运行中。 - 命令行访问，让 agent 可以在普通 CLI 工作流中使用同一套浏览器能力。 @@ -135,7 +135,7 @@ pnpm cli session-status 在选定的浏览器标签页中执行脚本。脚本接收一个对象参数，并返回一个 JSON 对象。 -`script-execute` 支持 `--timeout-ms`、`--script-file`、`--input-file`、`--register` 等可选执行配置。脚本执行期间，可以通过 `cap.call(scriptId, input)` 调用其他脚本。`--register` 只会在执行成功且结果包含 `ok: true` 时保存内联脚本。 +`script-execute` 支持 `--timeout-ms`、`--script-file`、`--input-file`、`--register` 等可选执行配置。`--register` 只会在执行成功且结果包含 `ok: true` 时保存内联脚本。 ### 浏览器命令 @@ -147,21 +147,22 @@ Web Cap 还包括 `browser-new-tab`、`session-status`、`wait-events` 等命令 ```js export default async function (input) { - const page = await cap.call('builtin.page.inspect', {}); + const heading = await page.locator('h1').first().textContent().catch(() => ''); return { ok: true, - title: page.title, + title: document.title, + heading, selector: input.selector, }; } ``` -运行时会在脚本执行期间注入 `cap`。 +运行时会在脚本执行期间注入 `page` 和 `cap.page`。 可用 runtime helper： -- `cap.call(scriptId, input)` - 调用内置脚本或已注册脚本。 +- 已废弃：`cap.call(scriptId, input)` - 调用内置脚本或已注册脚本。 - `cap.get(scriptId)` - 读取某个脚本的 schema 摘要。 - `cap.list()` - 列出当前可调用脚本的 schema 摘要。 diff --git a/extension/entrypoints/background.ts b/extension/entrypoints/background.ts index b48311a..54ec332 100644 --- a/extension/entrypoints/background.ts +++ b/extension/entrypoints/background.ts @@ -8,6 +8,7 @@ import { type ExecutionEvidenceEvent, type ExecutionEvidenceOption, type RuntimeEnvelope, + type RuntimeScreenshotArtifactPayload, type ScriptExecutionHistoryEntry, type RuntimeTabSnapshot, } from '@shared/protocol'; @@ -20,6 +21,7 @@ import { import { isDebuggerFallbackEligibleError, isExecutionInterruptedByNavigationError, + type ScriptScreenshotArtifact, type ScriptExecutionResponse, } from '../runtime/execution-helpers'; import { BrowserCommandHandler } from '../runtime/browser-command-handler'; @@ -95,7 +97,7 @@ class RuntimeClient { }, sendTabSnapshot: () => this.sendTabSnapshot(), toTabSnapshot: (tab) => this.toTabSnapshot(tab), - }); + }, this.debuggerExecutor.getDebuggerClient()); start(): void { this.connect(); @@ -174,6 +176,7 @@ class RuntimeClient { envelope.payload.tabId, envelope.payload.activateTab, envelope.payload.evidence ?? [], + envelope.payload.screenshotArtifactBasePath, ); break; case 'browser_command': @@ -203,6 +206,7 @@ class RuntimeClient { tabId?: number, activateTab?: boolean, evidenceOptions: ExecutionEvidenceOption[] = ['common'], + screenshotArtifactBasePath?: string, ): Promise { const selectedTab = tabId ? await browser.tabs.get(tabId) : await this.getActiveTab(); if (!selectedTab?.id || !selectedTab.url) { @@ -249,6 +253,7 @@ class RuntimeClient { input, scriptRegistry, evidenceOptions, + screenshotArtifactBasePath, ); } catch (error) { if (!isExecutionInterruptedByNavigationError(error)) { @@ -317,6 +322,10 @@ class RuntimeClient { result, evidence, status: response.status ?? 'succeeded', + screenshotArtifacts: this.sendBinaryScreenshotArtifacts( + response.screenshotArtifacts ?? [], + requestId, + ), }, { sessionId: this.sessionId, requestId }, ), @@ -359,6 +368,7 @@ class RuntimeClient { input: Record, scriptRegistry: ScriptDefinition[], evidence: ExecutionEvidenceOption[], + screenshotArtifactBasePath?: string, ): Promise { const requiresBrowserLevelClick = scriptRequiresBrowserLevelClick( scriptDefinition, @@ -386,6 +396,7 @@ class RuntimeClient { input, scriptRegistry, evidence, + screenshotArtifactBasePath, ); } @@ -400,6 +411,7 @@ class RuntimeClient { input, scriptRegistry, evidence, + screenshotArtifactBasePath, ); } catch (error) { if ( @@ -415,6 +427,7 @@ class RuntimeClient { input, scriptRegistry, evidence, + screenshotArtifactBasePath, ); } } @@ -458,7 +471,11 @@ class RuntimeClient { createRuntimeEnvelope( 'browser_command_result', { - result: response.result, + result: this.extractBinaryScreenshotArtifacts( + response.result, + requestId, + 'metadata', + ) as Record, }, { sessionId: this.sessionId, requestId }, ), @@ -886,11 +903,137 @@ class RuntimeClient { ); } + private extractBinaryScreenshotArtifacts( + value: unknown, + requestId: string, + resultShape: 'path' | 'metadata', + ): unknown { + if (isScreenshotArtifact(value)) { + const transferId = crypto.randomUUID(); + const bytes = decodeBase64(value.data); + const type = value.type === 'jpeg' ? 'jpeg' : 'png'; + const mimeType = typeof value.mimeType === 'string' + ? value.mimeType + : type === 'jpeg' + ? 'image/jpeg' + : 'image/png'; + + this.send( + createRuntimeEnvelope( + 'binary_payload_start', + { + transferId, + kind: 'screenshot', + mimeType, + type, + byteLength: bytes.byteLength, + resultShape, + }, + { sessionId: this.sessionId, requestId }, + ), + ); + this.sendBinary(bytes); + + return { + __webCapType: 'screenshot_transfer', + transferId, + resultShape, + }; + } + + if (Array.isArray(value)) { + return value.map((item) => this.extractBinaryScreenshotArtifacts(item, requestId, resultShape)); + } + + if (isRecord(value)) { + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [ + key, + this.extractBinaryScreenshotArtifacts(item, requestId, resultShape), + ]), + ); + } + + return value; + } + + private sendBinaryScreenshotArtifacts( + artifacts: ScriptScreenshotArtifact[], + requestId: string, + ): RuntimeScreenshotArtifactPayload[] { + return artifacts.map((artifact) => { + const transferId = crypto.randomUUID(); + const bytes = decodeBase64(artifact.data); + const type = artifact.type === 'jpeg' ? 'jpeg' : 'png'; + const mimeType = typeof artifact.mimeType === 'string' + ? artifact.mimeType + : type === 'jpeg' + ? 'image/jpeg' + : 'image/png'; + + this.send( + createRuntimeEnvelope( + 'binary_payload_start', + { + transferId, + kind: 'screenshot', + mimeType, + type, + byteLength: bytes.byteLength, + resultShape: 'metadata', + path: artifact.path, + }, + { sessionId: this.sessionId, requestId }, + ), + ); + this.sendBinary(bytes); + + return { + kind: 'screenshot', + path: artifact.path, + transferId, + mimeType, + type, + }; + }); + } + private send(envelope: RuntimeEnvelope): void { if (this.socket?.readyState === WebSocket.OPEN) { this.socket.send(JSON.stringify(envelope)); } } + + private sendBinary(bytes: Uint8Array): void { + if (this.socket?.readyState === WebSocket.OPEN) { + this.socket.send(bytes); + } + } +} + +function isScreenshotArtifact(value: unknown): value is { + data: string; + mimeType?: string; + type?: string; +} { + return ( + isRecord(value) && + value.__webCapType === 'screenshot' && + typeof value.data === 'string' + ); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function decodeBase64(value: string): Uint8Array { + const binary = atob(value); + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index += 1) { + bytes[index] = binary.charCodeAt(index); + } + return bytes; } function applyExecutionTabIndicatorScript(titlePrefix: string): void { diff --git a/extension/runtime/browser-command-handler.ts b/extension/runtime/browser-command-handler.ts index 34ca135..4bec29d 100644 --- a/extension/runtime/browser-command-handler.ts +++ b/extension/runtime/browser-command-handler.ts @@ -6,6 +6,7 @@ import { browserCommandInputSchemas, normalizeWaitEventsDurationMs, } from '@shared/browser-command-contracts'; +import { ChromeDebuggerClient } from './chrome-debugger-client'; interface BrowserCommandResponse { ok: boolean; @@ -28,7 +29,10 @@ interface BrowserCommandHandlerOptions { } export class BrowserCommandHandler { - constructor(private readonly options: BrowserCommandHandlerOptions) {} + constructor( + private readonly options: BrowserCommandHandlerOptions, + private readonly debuggerClient = new ChromeDebuggerClient(), + ) {} async execute( tabId: number, @@ -45,6 +49,10 @@ export class BrowserCommandHandler { return results[0]?.result ?? { ok: false, error: 'Page script returned no result.' }; } + if (command === 'browser_screenshot') { + return await this.captureScreenshot(tabId, input); + } + if (command === 'create_tab') { const parsed = browserCommandInputSchemas.create_tab.parse(input ?? {}); const createdTab = await browser.tabs.create({ @@ -69,6 +77,72 @@ export class BrowserCommandHandler { return { ok: false, error: `Browser command ${command} is not supported directly.` }; } + private async captureScreenshot( + tabId: number, + input: Record, + ): Promise { + if (!this.debuggerClient.isAvailable()) { + return { + ok: false, + error: 'chrome.debugger is not available in this browser runtime.', + }; + } + + const parsed = browserCommandInputSchemas.browser_screenshot.parse(input ?? {}); + const format = parsed.type ?? 'png'; + const params: Record = { + format, + fromSurface: true, + }; + if (format === 'jpeg' && parsed.quality !== undefined) { + params.quality = parsed.quality; + } + if (parsed.omitBackground !== undefined) { + params.omitBackground = parsed.omitBackground; + } + + const data = await this.debuggerClient.withAttachedDebugger(tabId, async (target) => { + if (parsed.fullPage === true) { + const metrics = await this.debuggerClient.sendCommand<{ + contentSize?: { x?: number; y?: number; width?: number; height?: number }; + }>(target, 'Page.getLayoutMetrics'); + const contentSize = metrics.contentSize; + if (contentSize) { + params.captureBeyondViewport = true; + params.clip = { + x: Number(contentSize.x ?? 0), + y: Number(contentSize.y ?? 0), + width: Math.max(Number(contentSize.width ?? 1), 1), + height: Math.max(Number(contentSize.height ?? 1), 1), + scale: 1, + }; + } + } + + const result = await this.debuggerClient.sendCommand<{ data?: string }>( + target, + 'Page.captureScreenshot', + params, + ); + return result.data; + }); + + if (!data) { + return { ok: false, error: 'Browser screenshot returned no image data.' }; + } + + return { + ok: true, + result: { + __webCapType: 'screenshot', + data, + mimeType: format === 'jpeg' ? 'image/jpeg' : 'image/png', + type: format, + encoding: 'base64', + }, + }; + } + private async waitForBrowserEvents( tabId: number, input: Record, diff --git a/extension/runtime/debugger-executor.ts b/extension/runtime/debugger-executor.ts index a65fd28..f0d18eb 100644 --- a/extension/runtime/debugger-executor.ts +++ b/extension/runtime/debugger-executor.ts @@ -26,12 +26,17 @@ export class DebuggerScriptExecutor { return this.client.isAvailable(); } + getDebuggerClient(): ChromeDebuggerClient { + return this.client; + } + async executeScript( tabId: number, scriptDefinition: ScriptDefinition, input: Record, scriptRegistry: ScriptDefinition[], evidence: ExecutionEvidenceOption[] = [], + screenshotArtifactBasePath?: string, ): Promise { if (!this.isAvailable()) { throw new Error('chrome.debugger is not available in this browser runtime.'); @@ -64,6 +69,7 @@ export class DebuggerScriptExecutor { managedWindowBridgeFunctionName: windowBridge.bridgeFunctionName, managedTimerBridgeFunctionName: timerBridge.bridgeFunctionName, managedBrowserBridgeFunctionName: browserBridge.bridgeFunctionName, + screenshotArtifactBasePath, evidence, }), awaitPromise: true, diff --git a/extension/runtime/execution-helpers.ts b/extension/runtime/execution-helpers.ts index f906653..7e9ed35 100644 --- a/extension/runtime/execution-helpers.ts +++ b/extension/runtime/execution-helpers.ts @@ -7,11 +7,21 @@ import type { import { scriptRuntimeSource } from './injected/script-runtime.generated'; import ts from 'typescript'; +export interface ScriptScreenshotArtifact { + kind: 'screenshot'; + path: string; + data: string; + mimeType: string; + type: 'png' | 'jpeg'; + encoding: 'base64'; +} + export interface ScriptExecutionResponse { ok: boolean; status?: 'succeeded' | 'interrupted'; result?: Record; evidence?: ExecutionEvidence; + screenshotArtifacts?: ScriptScreenshotArtifact[]; error?: string; } @@ -21,6 +31,7 @@ export interface ScriptExecutionExpressionOptions { managedWindowBridgeFunctionName?: string; managedTimerBridgeFunctionName?: string; managedBrowserBridgeFunctionName?: string; + screenshotArtifactBasePath?: string; evidence?: ExecutionEvidenceOption[]; } @@ -429,6 +440,10 @@ export function buildScriptExecutionExpression( options.managedBrowserBridgeFunctionName === undefined ? null : String(options.managedBrowserBridgeFunctionName); + const screenshotArtifactBasePath = + options.screenshotArtifactBasePath === undefined + ? null + : String(options.screenshotArtifactBasePath); const evidence = options.evidence ?? []; const scripts = new Map(); for (const item of scriptRegistry) { @@ -453,6 +468,7 @@ export function buildScriptExecutionExpression( const managedWindowBridgeFunctionName = ${JSON.stringify(managedWindowBridgeFunctionName)}; const managedTimerBridgeFunctionName = ${JSON.stringify(managedTimerBridgeFunctionName)}; const managedBrowserBridgeFunctionName = ${JSON.stringify(managedBrowserBridgeFunctionName)}; + const screenshotArtifactBasePath = ${JSON.stringify(screenshotArtifactBasePath)}; const evidence = ${JSON.stringify(evidence)}; const timerBridge = managedTimerBridgeFunctionName ? globalThis[managedTimerBridgeFunctionName] : null; const nativeSetTimeout = globalThis.setTimeout.bind(globalThis); @@ -510,6 +526,7 @@ export function buildScriptExecutionExpression( managedWindowBridgeFunctionName, managedTimerBridgeFunctionName, managedBrowserBridgeFunctionName, + screenshotArtifactBasePath, evidence, scriptFactories, }); diff --git a/extension/runtime/injected/playwright-locator.injected.ts b/extension/runtime/injected/playwright-locator.injected.ts index 5b8c3ff..c2381ad 100644 --- a/extension/runtime/injected/playwright-locator.injected.ts +++ b/extension/runtime/injected/playwright-locator.injected.ts @@ -512,7 +512,14 @@ export function createLocator( params.quality = Number(options.quality); } const result = await deps.browserCommand('Page.captureScreenshot', params) as { data?: string }; - return result.data ?? result; + if (typeof result.data !== 'string' || result.data.length === 0) { + throw new Error('Page.captureScreenshot returned no image data.'); + } + const data = result.data; + const mimeType = format === 'jpeg' ? 'image/jpeg' : 'image/png'; + return deps.createScreenshotArtifact + ? deps.createScreenshotArtifact({ data, mimeType, type: format }) + : { data, mimeType, type: format }; }, async setChecked(checked: unknown, options?: unknown) { const shouldBeChecked = Boolean(checked); diff --git a/extension/runtime/injected/playwright-shim-types.injected.ts b/extension/runtime/injected/playwright-shim-types.injected.ts index f40d979..221263b 100644 --- a/extension/runtime/injected/playwright-shim-types.injected.ts +++ b/extension/runtime/injected/playwright-shim-types.injected.ts @@ -6,6 +6,12 @@ export type ScriptPlaywrightPage = RuntimeMethodTable & { __playwrightPageType?: export type ScriptPlaywrightLocator = RuntimeMethodTable & { __playwrightLocatorType?: PlaywrightLocator }; export type LocatorQuery = () => Element[]; +export interface RuntimeScreenshotArtifactInput { + data: string; + mimeType: string; + type: 'png' | 'jpeg'; +} + export type PlaywrightShimDeps = { wait(ms: number): Promise; typeIntoElement(element: unknown, value: unknown): Promise; @@ -13,6 +19,7 @@ export type PlaywrightShimDeps = { useDomKeyboardFallback(): boolean; browserCommand?(method: string, params?: Record): Promise; browserEvent?(method: string, params?: Record, timeoutMs?: number): Promise; + createScreenshotArtifact?(input: RuntimeScreenshotArtifactInput): Record; recordEvidenceEvent?(type: string, value: unknown): void; waitForManagedInput(): Promise; }; diff --git a/extension/runtime/injected/playwright-shim.injected.ts b/extension/runtime/injected/playwright-shim.injected.ts index a67de1d..c37933a 100644 --- a/extension/runtime/injected/playwright-shim.injected.ts +++ b/extension/runtime/injected/playwright-shim.injected.ts @@ -981,7 +981,14 @@ function createPageApi(): ScriptPlaywrightPage { } } const result = await browserCommand<{ data?: string }>('Page.captureScreenshot', params); - return result.data ?? result; + if (typeof result.data !== 'string' || result.data.length === 0) { + throw new Error('Page.captureScreenshot returned no image data.'); + } + const data = result.data; + const mimeType = format === 'jpeg' ? 'image/jpeg' : 'image/png'; + return deps.createScreenshotArtifact + ? deps.createScreenshotArtifact({ data, mimeType, type: format }) + : { data, mimeType, type: format }; }, async setDefaultNavigationTimeout(timeout: unknown) { defaultTimeoutMs = Math.max(Number(timeout) || 0, 0); diff --git a/extension/runtime/injected/script-runtime.injected.ts b/extension/runtime/injected/script-runtime.injected.ts index 8fd26b1..d789e98 100644 --- a/extension/runtime/injected/script-runtime.injected.ts +++ b/extension/runtime/injected/script-runtime.injected.ts @@ -1,7 +1,10 @@ /* eslint-disable */ // Mechanically extracted from execution-helpers.ts. Keep behavior changes out of this file. import { createPlaywrightPageApi } from './playwright-shim.injected'; -import type { ScriptPlaywrightPage } from './playwright-shim-types.injected'; +import type { + RuntimeScreenshotArtifactInput, + ScriptPlaywrightPage, +} from './playwright-shim-types.injected'; import { installManagedClickHook } from './managed-click.injected'; import { captureVisibleElementsDiff } from './visible-elements.injected'; @@ -53,11 +56,21 @@ interface RuntimeEvidence { }; } +interface RuntimeScreenshotArtifact { + kind: 'screenshot'; + path: string; + data: string; + mimeType: string; + type: 'png' | 'jpeg'; + encoding: 'base64'; +} + interface RuntimeContext { registry: Map; evidence: RuntimeEvidence; state: RuntimeJsonObject; callStack: string[]; + screenshotArtifacts: RuntimeScreenshotArtifact[]; pendingAsyncOperations: Promise; visibleElementsTracker: { start(): void; @@ -93,6 +106,7 @@ export interface ScriptRuntimeArgs { managedKeyboardBridgeFunctionName: string | null; managedWindowBridgeFunctionName: string | null; managedBrowserBridgeFunctionName: string | null; + screenshotArtifactBasePath: string | null; evidence: RuntimeEvidenceOption[]; scriptFactories: Record unknown>; } @@ -105,6 +119,7 @@ export async function runScriptRuntime({ managedKeyboardBridgeFunctionName, managedWindowBridgeFunctionName, managedBrowserBridgeFunctionName, + screenshotArtifactBasePath, evidence, scriptFactories, }: ScriptRuntimeArgs) { @@ -261,6 +276,7 @@ export async function runScriptRuntime({ evidence: createEvidence(collectEvents || collectVisibleElements), state: {}, callStack: [], + screenshotArtifacts: [], pendingAsyncOperations: Promise.resolve(), visibleElementsTracker: captureVisibleElementsDiff(), }; @@ -503,6 +519,49 @@ export async function runScriptRuntime({ return getRuntimeBridge(managedBrowserBridgeFunctionName); } + function createScreenshotFileName(type: 'png' | 'jpeg'): string { + const extension = type === 'jpeg' ? 'jpg' : 'png'; + const bytes = new Uint8Array(8); + if (typeof globalThis.crypto?.getRandomValues === 'function') { + globalThis.crypto.getRandomValues(bytes); + } else { + for (let index = 0; index < bytes.length; index += 1) { + bytes[index] = Math.floor(Math.random() * 256); + } + } + const binary = Array.from(bytes, (byte) => String.fromCharCode(byte)).join(''); + const id = btoa(binary).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, ''); + return `s-${id}.${extension}`; + } + + function joinArtifactPath(basePath: string, fileName: string): string { + return `${basePath.replace(/[\\/]+$/, '')}/${fileName}`; + } + + function createScreenshotArtifact(input: RuntimeScreenshotArtifactInput): RuntimeJsonObject { + if (!screenshotArtifactBasePath) { + return { + data: input.data, + mimeType: input.mimeType, + type: input.type, + }; + } + + const path = joinArtifactPath( + screenshotArtifactBasePath, + createScreenshotFileName(input.type), + ); + context.screenshotArtifacts.push({ + kind: 'screenshot', + path, + data: input.data, + mimeType: input.mimeType, + type: input.type, + encoding: 'base64', + }); + return { path }; + } + function isEditableElement(element: unknown) { const hasInputClass = typeof HTMLInputElement !== 'undefined'; const hasTextareaClass = typeof HTMLTextAreaElement !== 'undefined'; @@ -800,6 +859,7 @@ export async function runScriptRuntime({ } return await Promise.resolve(bridgeFunction({ action: 'waitForEvent', method, params, timeoutMs })); }, + createScreenshotArtifact, recordEvidenceEvent: (type, value) => { context.evidence.events.push({ type, value }); }, @@ -979,6 +1039,7 @@ export async function runScriptRuntime({ status: interrupted ? 'interrupted' : 'succeeded', result, evidence: buildResponseEvidence(), + screenshotArtifacts: context.screenshotArtifacts, }; } catch (error) { return { diff --git a/extension/runtime/user-script-executor.ts b/extension/runtime/user-script-executor.ts index 23b9f81..654f68f 100644 --- a/extension/runtime/user-script-executor.ts +++ b/extension/runtime/user-script-executor.ts @@ -38,6 +38,7 @@ export class UserScriptExecutor { input: Record, scriptRegistry: ScriptDefinition[], evidence: ExecutionEvidenceOption[] = [], + screenshotArtifactBasePath?: string, ): Promise { const chromeApi = this.getChromeApi(); if (!chromeApi?.userScripts?.execute) { @@ -51,6 +52,7 @@ export class UserScriptExecutor { js: [ { code: buildScriptExecutionExpression(scriptDefinition, input, scriptRegistry, { + screenshotArtifactBasePath, evidence, }), }, diff --git a/lib/cli-parser.ts b/lib/cli-parser.ts index 2d86ca2..8aaf222 100644 --- a/lib/cli-parser.ts +++ b/lib/cli-parser.ts @@ -24,6 +24,13 @@ export interface BrowserNewTabCliOptions extends JsonOutputCliOptions { active?: boolean; } +export interface BrowserScreenshotCliOptions extends JsonOutputCliOptions { + tabId?: number; + type?: 'png' | 'jpeg'; + quality?: number; + omitBackground?: boolean; +} + export interface WaitEventsCliOptions { durationMs?: number; tabId?: number; @@ -42,6 +49,7 @@ export type CliCommand = | { name: 'config'; options: ConfigCliOptions } | { name: 'session-status'; options: JsonOutputCliOptions } | { name: 'script-execute'; options: ScriptExecuteCliOptions } + | { name: 'browser-screenshot'; options: BrowserScreenshotCliOptions } | { name: 'browser-new-tab'; options: BrowserNewTabCliOptions } | { name: 'wait-events'; options: WaitEventsCliOptions }; @@ -70,6 +78,8 @@ export function parseCliArgs(argv: string[]): CliCommand { return parseSessionStatusArgs(args); case 'script-execute': return parseScriptExecuteArgs(args); + case 'browser-screenshot': + return parseBrowserScreenshotArgs(args); case 'browser-new-tab': return parseBrowserNewTabArgs(args); case 'wait-events': @@ -231,6 +241,12 @@ function parseBrowserNewTabArgs(args: string[]): CliCommand { return options === 'help' ? helpForCommand(command) : { name: 'browser-new-tab', options }; } +function parseBrowserScreenshotArgs(args: string[]): CliCommand { + const command = createBrowserScreenshotParser(); + const options = parseCommander(command, args); + return options === 'help' ? helpForCommand(command) : { name: 'browser-screenshot', options }; +} + function parseWaitEventsArgs(args: string[]): CliCommand { const command = createWaitEventsParser(); const options = parseCommander(command, args); @@ -255,8 +271,7 @@ function scriptExecutionHelp(): string { web-cap script-execute --tab-id --script-file [--input-file ] [--pretty] Runs JavaScript in the selected browser tab. Scripts receive one JSON object, - return one JSON object, and can use cap.call(...) inside the script to call - reusable capabilities. + return one JSON object, and can use the Playwright-style page API. Scripts also receive a Playwright-style page API as global page and cap.page. ${scriptRuntimeApiHelp(' ')} @@ -293,6 +308,8 @@ function createCommandParser(commandName: CliCommandName): Command { return createSessionStatusParser(); case 'script-execute': return createScriptExecuteParser(); + case 'browser-screenshot': + return createBrowserScreenshotParser(); case 'browser-new-tab': return createBrowserNewTabParser(); case 'wait-events': @@ -307,6 +324,7 @@ function cliCommandNames(): CliCommandName[] { 'config', 'session-status', 'script-execute', + 'browser-screenshot', 'browser-new-tab', 'wait-events', ]; @@ -362,6 +380,15 @@ function createBrowserNewTabParser(): Command { .option('--active ', 'Whether the tab should be activated.', parseBooleanOption); } +function createBrowserScreenshotParser(): Command { + return createJsonOutputParser('browser-screenshot') + .description('Capture a screenshot from the connected browser tab.') + .option('--tab-id ', 'Browser tab id to target. Defaults to the active tab.', parseIntegerOption) + .option('--type ', 'Screenshot image format. Defaults to png.', parseScreenshotTypeOption) + .option('--quality <0-100>', 'JPEG quality. Only applies when --type jpeg.', parseIntegerOption) + .option('--omit-background ', 'Hide the default white background when supported.', parseBooleanOption); +} + function createWaitEventsParser(): Command { return createParser('wait-events') .description('Wait while the user completes a browser action and stream the resulting interaction path as JSON Lines.') @@ -476,6 +503,13 @@ function parseIntegerOption(value: string): number { return parsed; } +function parseScreenshotTypeOption(value: string): 'png' | 'jpeg' { + if (value !== 'png' && value !== 'jpeg') { + throw new InvalidArgumentError('Expected png or jpeg.'); + } + return value; +} + function parseBooleanOption(value: string): boolean { if (value === 'true') { return true; diff --git a/lib/cli.ts b/lib/cli.ts index 09ba59b..dbac464 100644 --- a/lib/cli.ts +++ b/lib/cli.ts @@ -160,6 +160,8 @@ function coreToolNameForCommand(command: CliCommand): CoreToolName | undefined { return 'session_status'; case 'script-execute': return 'script_execute'; + case 'browser-screenshot': + return 'browser_screenshot'; case 'browser-new-tab': return 'browser_new_tab'; default: @@ -176,6 +178,13 @@ function buildCoreToolInput( return {}; case 'script-execute': return scriptExecuteRequest!; + case 'browser-screenshot': + return { + tabId: command.options.tabId, + type: command.options.type, + quality: command.options.quality, + omitBackground: command.options.omitBackground, + }; case 'browser-new-tab': return { url: command.options.url, @@ -190,6 +199,7 @@ function jsonOutputOptionsForCommand(command: CliCommand): JsonOutputCliOptions switch (command.name) { case 'session-status': case 'script-execute': + case 'browser-screenshot': case 'browser-new-tab': case 'config': return command.options; diff --git a/lib/server/agent/contracts.ts b/lib/server/agent/contracts.ts index 36ddfd3..61ad44a 100644 --- a/lib/server/agent/contracts.ts +++ b/lib/server/agent/contracts.ts @@ -1,12 +1,14 @@ import type { ScriptDefinition } from '@shared/script-schema'; import type { BrowserCommandResult, + BrowserScreenshotResult, ScriptExecutionHistoryEntry, ScriptExecutionResult, ExecutionEvidenceOption, RuntimeSessionSnapshot, } from '@shared/protocol'; import type { + BrowserScreenshotInput, CreateTabInput, WaitEventsInput, } from '@shared/browser-command-contracts'; @@ -33,6 +35,7 @@ export interface WebCapAgentService { scriptExecute(request: ExecuteScriptRequest): Promise; scriptHistoryList(limit?: number): Promise; scriptRegistryList(): Promise; + browserScreenshot(input: BrowserScreenshotInput): Promise; browserNewTab(input: CreateTabInput): Promise; browserWaitEvents( input: WaitEventsInput, diff --git a/lib/server/app-rpc.ts b/lib/server/app-rpc.ts index f1bd886..6b17d8c 100644 --- a/lib/server/app-rpc.ts +++ b/lib/server/app-rpc.ts @@ -5,11 +5,13 @@ import type { } from '@shared/script-schema'; import type { BrowserCommandResult, + BrowserScreenshotResult, RuntimeSessionSnapshot, ScriptExecutionHistoryEntry, } from '@shared/protocol'; import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@shared/protocol'; import type { + BrowserScreenshotInput, CreateTabInput, WaitEventsInput, } from '@shared/browser-command-contracts'; @@ -215,6 +217,10 @@ export class WebCapRpcServer { const params = parseRpcInput(request.method, request.params); return await this.app.browserNewTab(params); } + case 'browserScreenshot': { + const params = parseRpcInput(request.method, request.params); + return await this.app.browserScreenshot(params); + } case 'browserWaitEvents': { const params = parseRpcInput(request.method, request.params); return await this.app.browserWaitEvents(params, emitEvent); @@ -269,6 +275,13 @@ export class WebCapRpcClient { )) as BrowserCommandResult; } + async browserScreenshot(input: BrowserScreenshotInput): Promise { + return (await this.requestWithRuntimeReconnectGrace( + 'browserScreenshot', + input as Record, + )) as BrowserScreenshotResult; + } + async browserWaitEvents( input: WaitEventsInput, onEvent?: (event: Record) => void, diff --git a/lib/server/app.ts b/lib/server/app.ts index 0d056aa..cb7b010 100644 --- a/lib/server/app.ts +++ b/lib/server/app.ts @@ -3,9 +3,11 @@ import { } from '@shared/script-schema'; import type { BrowserCommandResult, + BrowserScreenshotResult, ScriptExecutionHistoryEntry, } from '@shared/protocol'; import type { CreateTabInput, WaitEventsInput } from '@shared/browser-command-contracts'; +import type { BrowserScreenshotInput } from '@shared/browser-command-contracts'; import { builtinScripts, builtinScriptRecords } from './scripts/builtin-scripts'; import type { ExecuteScriptRequest, @@ -91,6 +93,10 @@ export class WebCapAgentApp implements WebCapAgentService { return await this.scriptRegistryService.buildRegisteredScriptRegistry(); } + async browserScreenshot(input: BrowserScreenshotInput): Promise { + return await this.browserSessionService.screenshot(input); + } + async browserNewTab(input: CreateTabInput): Promise { return await this.browserSessionService.newTab(input); } diff --git a/lib/server/browser/command-contracts.ts b/lib/server/browser/command-contracts.ts index 731e0ea..8d94dd4 100644 --- a/lib/server/browser/command-contracts.ts +++ b/lib/server/browser/command-contracts.ts @@ -5,6 +5,7 @@ import { browserCommandInputSchemas, browserCommandRequestSchemas, normalizeWaitEventsDurationMs, + type BrowserScreenshotInput, type ContractedBrowserCommandName, type CreateTabInput, type WaitEventsInput, @@ -17,6 +18,7 @@ export { browserCommandInputSchemas, browserCommandRequestSchemas, normalizeWaitEventsDurationMs, + type BrowserScreenshotInput, type ContractedBrowserCommandName, type CreateTabInput, type WaitEventsInput, diff --git a/lib/server/browser/screenshot-store.ts b/lib/server/browser/screenshot-store.ts new file mode 100644 index 0000000..49cd41a --- /dev/null +++ b/lib/server/browser/screenshot-store.ts @@ -0,0 +1,185 @@ +import { randomBytes } from 'node:crypto'; +import { mkdir, readdir, rm, stat, writeFile } from 'node:fs/promises'; +import { basename, relative, resolve, sep, join } from 'node:path'; +import { resolveWebCapStateDir } from '../state-dir'; + +const SCREENSHOT_DIR_NAME = 'temp-screenshots'; +const SCREENSHOT_RETENTION_MS = 24 * 60 * 60 * 1000; +const SCREENSHOT_FILE_PATTERN = /^(?:s-[A-Za-z0-9_-]{11}|screenshot-.*)\.(?:png|jpe?g)$/i; + +interface ScreenshotStoreEnvironment { + WEB_CAP_STATE_DIR?: string; +} + +interface RuntimeScreenshotResult { + data: string; + mimeType: string; + type: 'png' | 'jpeg'; +} + +export interface StoredScreenshotResult { + path: string; + mimeType: string; + type: 'png' | 'jpeg'; + encoding: 'file'; + sizeBytes: number; +} + +export async function storeBrowserScreenshot( + result: Record, + env: ScreenshotStoreEnvironment = process.env, +): Promise { + const screenshot = parseRuntimeScreenshotResult(result); + return await storeBrowserScreenshotBytes( + Buffer.from(screenshot.data, 'base64'), + screenshot, + env, + ); +} + +export async function storeBrowserScreenshotBytes( + bytes: Buffer, + screenshot: Omit, + env: ScreenshotStoreEnvironment = process.env, +): Promise { + const directory = resolveScreenshotDirectory(env); + await mkdir(directory, { recursive: true }); + await cleanupExpiredScreenshots(directory).catch((error) => { + console.warn('WEB_CAP failed to clean expired screenshots:', error); + }); + + const filePath = join(directory, createScreenshotFileName(screenshot.type)); + await writeFile(filePath, bytes); + + return { + path: filePath, + mimeType: screenshot.mimeType, + type: screenshot.type, + encoding: 'file', + sizeBytes: bytes.byteLength, + }; +} + +export async function storeBrowserScreenshotBytesAtPath( + bytes: Buffer, + screenshot: Omit, + filePath: string, + env: ScreenshotStoreEnvironment = process.env, +): Promise { + const directory = resolveScreenshotDirectory(env); + await mkdir(directory, { recursive: true }); + await cleanupExpiredScreenshots(directory).catch((error) => { + console.warn('WEB_CAP failed to clean expired screenshots:', error); + }); + + const safePath = resolveSafeScreenshotPath(directory, filePath); + await writeFile(safePath, bytes); + return { + path: safePath, + mimeType: screenshot.mimeType, + type: screenshot.type, + encoding: 'file', + sizeBytes: bytes.byteLength, + }; +} + +export async function storeScriptScreenshotArtifacts( + value: unknown, + env: ScreenshotStoreEnvironment = process.env, +): Promise { + if (isRuntimeScreenshotMarker(value)) { + return (await storeBrowserScreenshot(value, env)).path; + } + + if (Array.isArray(value)) { + return await Promise.all(value.map((item) => storeScriptScreenshotArtifacts(item, env))); + } + + if (isRecord(value)) { + const entries = await Promise.all( + Object.entries(value).map(async ([key, nested]) => [ + key, + await storeScriptScreenshotArtifacts(nested, env), + ]), + ); + return Object.fromEntries(entries); + } + + return value; +} + +export function resolveScreenshotDirectory( + env: ScreenshotStoreEnvironment = process.env, +): string { + return join(resolveWebCapStateDir(env), SCREENSHOT_DIR_NAME); +} + +async function cleanupExpiredScreenshots(directory: string): Promise { + const now = Date.now(); + const entries = await readdir(directory, { withFileTypes: true }); + await Promise.all( + entries.map(async (entry) => { + if (!entry.isFile() || !SCREENSHOT_FILE_PATTERN.test(entry.name)) { + return; + } + + const filePath = join(directory, entry.name); + const fileStat = await stat(filePath); + if (now - fileStat.mtimeMs <= SCREENSHOT_RETENTION_MS) { + return; + } + + await rm(filePath, { force: true }); + }), + ); +} + +function parseRuntimeScreenshotResult(result: Record): RuntimeScreenshotResult { + if (typeof result.data !== 'string' || result.data.length === 0) { + throw new Error('Browser screenshot returned no image data.'); + } + + const type: 'png' | 'jpeg' = result.type === 'jpeg' ? 'jpeg' : 'png'; + const mimeType = typeof result.mimeType === 'string' + ? result.mimeType + : type === 'jpeg' + ? 'image/jpeg' + : 'image/png'; + return { + data: result.data, + mimeType, + type, + }; +} + +function resolveSafeScreenshotPath(directory: string, candidatePath: string): string { + const resolvedDirectory = resolve(directory); + const resolvedPath = resolve(candidatePath); + const relativePath = relative(resolvedDirectory, resolvedPath); + if ( + relativePath.startsWith('..') || + relativePath === '' || + relativePath.includes(sep) || + !SCREENSHOT_FILE_PATTERN.test(basename(resolvedPath)) + ) { + throw new Error('Browser screenshot artifact path is not allowed.'); + } + return resolvedPath; +} + +function isRuntimeScreenshotMarker(value: unknown): value is Record { + return ( + isRecord(value) && + value.__webCapType === 'screenshot' && + typeof value.data === 'string' + ); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function createScreenshotFileName(type: 'png' | 'jpeg'): string { + const id = randomBytes(8).toString('base64url'); + return `s-${id}.${type === 'jpeg' ? 'jpg' : 'png'}`; +} diff --git a/lib/server/browser/session-service.ts b/lib/server/browser/session-service.ts index c45166d..c0246bb 100644 --- a/lib/server/browser/session-service.ts +++ b/lib/server/browser/session-service.ts @@ -1,5 +1,6 @@ import { type BrowserCommandResult, + type BrowserScreenshotResult, type RuntimeConnectionSnapshot, type RuntimeSessionSnapshot, type RuntimeTabSnapshot, @@ -9,9 +10,11 @@ import { RuntimeBridgeError } from '../runtime/runtime-bridge'; import { parseBrowserCommandRequest, timeoutForBrowserCommand, + type BrowserScreenshotInput, type CreateTabInput, type WaitEventsInput, } from './command-contracts'; +import { storeBrowserScreenshot } from './screenshot-store'; export interface ExecutionTarget { runtime?: RuntimeConnectionSnapshot; @@ -25,6 +28,30 @@ export class BrowserSessionService { return this.runtimeBridge.getSessionStatus(); } + async screenshot(input: BrowserScreenshotInput): Promise { + this.assertConnected(); + const parsed = parseBrowserCommandRequest('browser_screenshot', input); + const commandResult = await this.runtimeBridge.executeBrowserCommand( + 'browser_screenshot', + { + type: parsed.type, + quality: parsed.quality, + fullPage: parsed.fullPage, + omitBackground: parsed.omitBackground, + }, + { tabId: parsed.tabId }, + ); + if (commandResult.result.encoding === 'file') { + return summarizeScreenshotResult(commandResult); + } + + const stored = await storeBrowserScreenshot(commandResult.result); + return summarizeScreenshotResult({ + ...commandResult, + result: { ...stored }, + }); + } + async newTab(input: CreateTabInput): Promise { this.assertConnected(); const parsed = parseBrowserCommandRequest('create_tab', input); @@ -71,6 +98,30 @@ export class BrowserSessionService { } } +function summarizeScreenshotResult(commandResult: BrowserCommandResult): BrowserScreenshotResult { + if (typeof commandResult.result.path !== 'string' || commandResult.result.path.length === 0) { + throw new RuntimeBridgeError( + 'Browser screenshot did not return a file path.', + 'EXECUTION_FAILED', + ); + } + + return { + result: { + path: commandResult.result.path, + ...(typeof commandResult.result.sizeBytes === 'number' + ? { sizeBytes: commandResult.result.sizeBytes } + : {}), + }, + timingMs: commandResult.timingMs, + tab: { + tabId: commandResult.tab.tabId, + url: commandResult.tab.url, + title: commandResult.tab.title, + }, + }; +} + function selectRuntimeForTab( runtimes: RuntimeConnectionSnapshot[] | undefined, tabId?: number, diff --git a/lib/server/runtime/websocket-runtime-bridge.ts b/lib/server/runtime/websocket-runtime-bridge.ts index 97c6c50..85c761b 100644 --- a/lib/server/runtime/websocket-runtime-bridge.ts +++ b/lib/server/runtime/websocket-runtime-bridge.ts @@ -1,4 +1,5 @@ import { randomUUID } from 'node:crypto'; +import { resolve as resolvePath } from 'node:path'; import { WebSocketServer, type WebSocket } from 'ws'; import type { ScriptDefinition } from '@shared/script-schema'; import { @@ -21,6 +22,12 @@ import { } from '@shared/protocol'; import { RuntimeBridge, RuntimeBridgeError, type BrowserCommandOptions } from './runtime-bridge'; import { DEFAULT_BROWSER_COMMAND_TIMEOUT_MS } from '../browser/command-contracts'; +import { + resolveScreenshotDirectory, + storeBrowserScreenshotBytes, + storeBrowserScreenshotBytesAtPath, + type StoredScreenshotResult, +} from '../browser/screenshot-store'; const EXECUTION_RESPONSE_GRACE_MS = 5_000; @@ -51,6 +58,23 @@ interface RuntimeConnection { snapshot: RuntimeConnectionSnapshot; } +interface PendingBinaryTransfer { + sessionId: string; + requestId: string; + transferId: string; + kind: 'screenshot'; + mimeType: string; + type: 'png' | 'jpeg'; + byteLength: number; + resultShape: 'path' | 'metadata'; + path?: string; +} + +interface StoredBinaryTransfer { + resultShape: 'path' | 'metadata'; + stored: StoredScreenshotResult; +} + export interface WebSocketRuntimeBridgeOptions { port?: number; onRuntimeCountChanged?: (runtimeCount: number) => void; @@ -63,6 +87,8 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { private activeSessionId?: string; private pendingExecutions = new Map(); private pendingBrowserCommands = new Map(); + private pendingBinaryTransfers = new Map(); + private storedBinaryTransfers = new Map>(); private port: number; private scriptHistoryLoader?: () => Promise; private scriptRegistryLoader?: () => Promise; @@ -125,6 +151,7 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { ), ); this.pendingExecutions.delete(requestId); + this.cleanupBinaryTransfersForRequest(pending.sessionId, requestId); } for (const [requestId, pending] of this.pendingBrowserCommands) { @@ -136,6 +163,7 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { ), ); this.pendingBrowserCommands.delete(requestId); + this.cleanupBinaryTransfersForRequest(pending.sessionId, requestId); } await new Promise((resolve) => { @@ -151,6 +179,8 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { this.server = undefined; this.runtimes.clear(); this.socketSessions.clear(); + this.pendingBinaryTransfers.clear(); + this.storedBinaryTransfers.clear(); this.activeSessionId = undefined; } @@ -204,6 +234,7 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { const startedAt = Date.now(); const timer = setTimeout(() => { this.pendingExecutions.delete(requestId); + this.cleanupBinaryTransfersForRequest(target.sessionId, requestId); resolve({ scriptId: scriptDefinition.id, status: 'interrupted', @@ -255,6 +286,7 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { tabId: tab.tabId, activateTab: options.activateTab, evidence, + screenshotArtifactBasePath: resolveScreenshotDirectory(), }, { requestId, @@ -292,6 +324,7 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { const timeoutMs = options.timeoutMs ?? DEFAULT_BROWSER_COMMAND_TIMEOUT_MS; const timer = setTimeout(() => { this.pendingBrowserCommands.delete(requestId); + this.cleanupBinaryTransfersForRequest(target.sessionId, requestId); reject( new RuntimeBridgeError( `Browser command ${command} timed out after ${timeoutMs}ms.`, @@ -360,13 +393,13 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { } private attachClient(socket: WebSocket): void { - socket.on('message', (buffer: Buffer) => { - const parsed = JSON.parse(buffer.toString()) as RuntimeEnvelope; - this.handleEnvelope(socket, parsed); + socket.on('message', (buffer: Buffer, isBinary: boolean) => { + void this.handleSocketMessage(socket, buffer, isBinary); }); socket.on('close', () => { const sessionId = this.socketSessions.get(socket); + this.pendingBinaryTransfers.delete(socket); this.removeRuntime(socket); this.rejectPendingForSession( sessionId, @@ -376,6 +409,7 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { socket.on('error', () => { const sessionId = this.socketSessions.get(socket); + this.pendingBinaryTransfers.delete(socket); this.removeRuntime(socket); this.rejectPendingForSession( sessionId, @@ -384,6 +418,20 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { }); } + private async handleSocketMessage( + socket: WebSocket, + buffer: Buffer, + isBinary: boolean, + ): Promise { + if (isBinary) { + await this.handleBinaryFrame(socket, buffer); + return; + } + + const parsed = JSON.parse(buffer.toString()) as RuntimeEnvelope; + await this.handleEnvelope(socket, parsed); + } + private findRuntimeForExecution( tabId?: number, ): { sessionId: string; runtime: RuntimeConnection } | undefined { @@ -411,7 +459,7 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { return { sessionId, runtime }; } - private handleEnvelope(socket: WebSocket, envelope: RuntimeEnvelope): void { + private async handleEnvelope(socket: WebSocket, envelope: RuntimeEnvelope): Promise { const runtime = this.runtimeForSocket(socket); if (runtime) { runtime.snapshot.lastSeenAt = envelope.timestamp; @@ -419,7 +467,7 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { switch (envelope.type) { case 'hello': - void this.handleHello(socket, envelope.payload); + await this.handleHello(socket, envelope.payload); break; case 'heartbeat': break; @@ -433,22 +481,52 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { this.activeSessionId = runtime.snapshot.sessionId; } break; + case 'binary_payload_start': + if (runtime) { + this.pendingBinaryTransfers.set(socket, { + sessionId: runtime.snapshot.sessionId, + requestId: envelope.requestId, + transferId: envelope.payload.transferId, + kind: envelope.payload.kind, + mimeType: envelope.payload.mimeType, + type: envelope.payload.type, + byteLength: envelope.payload.byteLength, + resultShape: envelope.payload.resultShape, + path: envelope.payload.path, + }); + } + break; case 'execution_result': { const pending = this.pendingExecutions.get(envelope.requestId); if (!pending || runtime?.snapshot.sessionId !== pending.sessionId) { return; } - clearTimeout(pending.timer); - this.pendingExecutions.delete(envelope.requestId); - pending.resolve({ - scriptId: pending.scriptDefinition.id, - status: envelope.payload.status ?? inferExecutionStatus(envelope.payload.evidence), - result: envelope.payload.result, - evidence: summarizeExecutionEvidence(envelope.payload.evidence), - timingMs: Date.now() - pending.startedAt, - tab: summarizeExecutionTab(pending.tab, pending.includeTabInResult), - }); + try { + await this.awaitScreenshotArtifactTransfers( + envelope.payload.screenshotArtifacts, + pending.sessionId, + envelope.requestId, + ); + clearTimeout(pending.timer); + this.pendingExecutions.delete(envelope.requestId); + pending.resolve({ + scriptId: pending.scriptDefinition.id, + status: envelope.payload.status ?? inferExecutionStatus(envelope.payload.evidence), + result: envelope.payload.result, + evidence: summarizeExecutionEvidence(envelope.payload.evidence), + timingMs: Date.now() - pending.startedAt, + tab: summarizeExecutionTab(pending.tab, pending.includeTabInResult), + }); + this.cleanupBinaryTransfersForRequest(pending.sessionId, envelope.requestId); + } catch (error) { + clearTimeout(pending.timer); + this.pendingExecutions.delete(envelope.requestId); + pending.reject( + error instanceof Error ? error : new Error(String(error)), + ); + this.cleanupBinaryTransfersForRequest(pending.sessionId, envelope.requestId); + } break; } case 'browser_command_result': { @@ -457,14 +535,29 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { return; } - clearTimeout(pending.timer); - this.pendingBrowserCommands.delete(envelope.requestId); - pending.resolve({ - command: pending.command, - result: envelope.payload.result, - timingMs: Date.now() - pending.startedAt, - tab: pending.tab, - }); + try { + const result = await this.materializeBinaryTransferArtifacts( + envelope.payload.result, + pending.sessionId, + envelope.requestId, + ); + clearTimeout(pending.timer); + this.pendingBrowserCommands.delete(envelope.requestId); + pending.resolve({ + command: pending.command, + result, + timingMs: Date.now() - pending.startedAt, + tab: pending.tab, + }); + this.cleanupBinaryTransfersForRequest(pending.sessionId, envelope.requestId); + } catch (error) { + clearTimeout(pending.timer); + this.pendingBrowserCommands.delete(envelope.requestId); + pending.reject( + error instanceof Error ? error : new Error(String(error)), + ); + this.cleanupBinaryTransfersForRequest(pending.sessionId, envelope.requestId); + } break; } case 'browser_command_event': { @@ -490,6 +583,7 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { ), ); this.pendingExecutions.delete(envelope.requestId); + this.cleanupBinaryTransfersForRequest(pending.sessionId, envelope.requestId); } const browserCommand = envelope.requestId @@ -509,6 +603,7 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { ), ); this.pendingBrowserCommands.delete(envelope.requestId); + this.cleanupBinaryTransfersForRequest(browserCommand.sessionId, envelope.requestId); } break; } @@ -517,6 +612,170 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { } } + private async handleBinaryFrame(socket: WebSocket, buffer: Buffer): Promise { + const transfer = this.pendingBinaryTransfers.get(socket); + if (!transfer) { + console.warn('WEB_CAP received an unexpected binary payload.'); + return; + } + + this.pendingBinaryTransfers.delete(socket); + if (buffer.byteLength !== transfer.byteLength) { + const error = new RuntimeBridgeError( + `Binary payload ${transfer.transferId} size mismatch: expected ${transfer.byteLength} bytes, received ${buffer.byteLength}.`, + 'EXECUTION_FAILED', + ); + this.rejectPendingForRequest(transfer.sessionId, transfer.requestId, error); + return; + } + + const storage = ( + transfer.path + ? storeBrowserScreenshotBytesAtPath(buffer, { + mimeType: transfer.mimeType, + type: transfer.type, + }, transfer.path) + : storeBrowserScreenshotBytes(buffer, { + mimeType: transfer.mimeType, + type: transfer.type, + }) + ).then((stored) => ({ + resultShape: transfer.resultShape, + stored, + })); + storage.catch(() => undefined); + this.storedBinaryTransfers.set(this.binaryTransferKey(transfer), storage); + } + + private async awaitScreenshotArtifactTransfers( + artifacts: { transferId: string; path: string }[] | undefined, + sessionId: string, + requestId: string, + ): Promise { + if (!artifacts?.length) { + return; + } + await Promise.all( + artifacts.map(async (artifact) => { + const transfer = await this.readStoredBinaryTransfer( + sessionId, + requestId, + artifact.transferId, + ); + if (resolvePath(transfer.stored.path) !== resolvePath(artifact.path)) { + throw new RuntimeBridgeError( + `Screenshot artifact ${artifact.transferId} path did not match its binary payload path.`, + 'EXECUTION_FAILED', + ); + } + }), + ); + } + + private async materializeBinaryTransferArtifacts( + value: unknown, + sessionId: string, + requestId: string, + ): Promise> { + const materialized = await this.materializeBinaryTransferValue(value, sessionId, requestId); + if (!isRecord(materialized)) { + throw new RuntimeBridgeError( + 'Browser runtime returned a non-object result after binary payload materialization.', + 'EXECUTION_FAILED', + ); + } + return materialized; + } + + private async materializeBinaryTransferValue( + value: unknown, + sessionId: string, + requestId: string, + ): Promise { + if (isBinaryTransferMarker(value)) { + const transfer = await this.readStoredBinaryTransfer(sessionId, requestId, value.transferId); + return transfer.resultShape === 'metadata' + ? { ...transfer.stored } + : transfer.stored.path; + } + + if (Array.isArray(value)) { + return await Promise.all( + value.map((item) => this.materializeBinaryTransferValue(item, sessionId, requestId)), + ); + } + + if (isRecord(value)) { + const entries = await Promise.all( + Object.entries(value).map(async ([key, item]) => [ + key, + await this.materializeBinaryTransferValue(item, sessionId, requestId), + ]), + ); + return Object.fromEntries(entries); + } + + return value; + } + + private async readStoredBinaryTransfer( + sessionId: string, + requestId: string, + transferId: string, + ): Promise { + const key = this.binaryTransferKey({ sessionId, requestId, transferId }); + const transfer = this.storedBinaryTransfers.get(key); + if (!transfer) { + throw new RuntimeBridgeError( + `Binary payload ${transferId} was not received for request ${requestId}.`, + 'EXECUTION_FAILED', + ); + } + return await transfer; + } + + private cleanupBinaryTransfersForRequest(sessionId: string, requestId: string): void { + const prefix = `${sessionId}:${requestId}:`; + for (const [socket, transfer] of this.pendingBinaryTransfers) { + if (transfer.sessionId === sessionId && transfer.requestId === requestId) { + this.pendingBinaryTransfers.delete(socket); + } + } + + for (const [key, transfer] of this.storedBinaryTransfers) { + if (key.startsWith(prefix)) { + transfer.catch(() => undefined); + this.storedBinaryTransfers.delete(key); + } + } + } + + private binaryTransferKey(input: { + sessionId: string; + requestId: string; + transferId: string; + }): string { + return `${input.sessionId}:${input.requestId}:${input.transferId}`; + } + + private rejectPendingForRequest(sessionId: string, requestId: string, error: Error): void { + const execution = this.pendingExecutions.get(requestId); + if (execution?.sessionId === sessionId) { + clearTimeout(execution.timer); + execution.reject(error); + this.pendingExecutions.delete(requestId); + } + + const browserCommand = this.pendingBrowserCommands.get(requestId); + if (browserCommand?.sessionId === sessionId) { + clearTimeout(browserCommand.timer); + browserCommand.reject(error); + this.pendingBrowserCommands.delete(requestId); + } + + this.cleanupBinaryTransfersForRequest(sessionId, requestId); + } + private async handleHello(socket: WebSocket, payload: RuntimeHelloPayload): Promise { this.removeRuntime(socket); const sessionId = randomUUID(); @@ -627,6 +886,7 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { clearTimeout(pending.timer); pending.reject(error); this.pendingExecutions.delete(requestId); + this.cleanupBinaryTransfersForRequest(sessionId, requestId); } for (const [requestId, pending] of this.pendingBrowserCommands) { @@ -636,6 +896,7 @@ export class WebSocketRuntimeBridge implements RuntimeBridge { clearTimeout(pending.timer); pending.reject(error); this.pendingBrowserCommands.delete(requestId); + this.cleanupBinaryTransfersForRequest(sessionId, requestId); } } } @@ -753,6 +1014,16 @@ function hasVisibleElementChanges( ); } +function isBinaryTransferMarker(value: unknown): value is { + transferId: string; +} { + return ( + isRecord(value) && + value.__webCapType === 'screenshot_transfer' && + typeof value.transferId === 'string' + ); +} + function shouldCollectExecutionEvidence( evidence: ExecutionEvidenceOption[] | undefined, option: Exclude, diff --git a/lib/server/tool-contracts.ts b/lib/server/tool-contracts.ts index 8b6dea0..6fdbc2c 100644 --- a/lib/server/tool-contracts.ts +++ b/lib/server/tool-contracts.ts @@ -23,6 +23,7 @@ export const executeScriptOptionsSchema = z export const toolInputSchemas = { session_status: z.object({}), + browser_screenshot: browserCommandRequestSchemas.browser_screenshot, browser_new_tab: browserCommandRequestSchemas.create_tab, script_execute: z.object({ script: z.string().min(1), @@ -40,6 +41,7 @@ export const rpcInputSchemas = { scriptRegistryList: z.object({}), browserWaitEvents: browserCommandRequestSchemas.wait_events, sessionStatus: toolInputSchemas.session_status, + browserScreenshot: browserCommandRequestSchemas.browser_screenshot, browserNewTab: browserCommandRequestSchemas.create_tab, scriptExecute: toolInputSchemas.script_execute, } as const; @@ -64,6 +66,34 @@ export const mcpToolDefinitions: { 'Return the current browser runtime connection status, including the last active tab, all known tabs for the active runtime, and connected runtime snapshots.', inputSchema: {}, }, + browser_screenshot: { + title: 'Capture Browser Screenshot', + description: + 'Capture a screenshot of the selected browser tab, save it under the Web Cap temporary screenshot directory, and return file metadata. Defaults to PNG visible viewport; set fullPage for a full-page screenshot when supported.', + inputSchema: { + tabId: z + .number() + .int() + .optional() + .describe('Target browser tab id. If omitted, the active tab is used.'), + type: z.enum(['png', 'jpeg']).optional().describe('Image format. Defaults to png.'), + quality: z + .number() + .int() + .min(0) + .max(100) + .optional() + .describe('JPEG quality from 0 to 100. Only used when type is jpeg.'), + fullPage: z + .boolean() + .optional() + .describe('Capture the full page instead of only the visible viewport when supported.'), + omitBackground: z + .boolean() + .optional() + .describe('Hide the default white background for pages with transparency when supported.'), + }, + }, browser_new_tab: { title: 'Create Browser Tab', description: @@ -76,12 +106,12 @@ export const mcpToolDefinitions: { script_execute: { title: 'Execute Script', description: - 'Execute script code directly in a specified tab of the connected browser. During execution, the script can call registered scripts through `cap.call("script-id", input)`, where the script id can be obtained via script search, for example: `(input) => cap.call("script-id", input)`. Inline executions receive a local script id in the execution result and local history. Set `register` to true to request permanent registration; the script is registered only after execution succeeds with a result object that includes `ok: true`.', + 'Execute script code directly in a specified tab of the connected browser. Scripts receive an input object, return a JSON-compatible result object, and can use the Playwright-style page API. Inline executions receive a local script id in the execution result and local history. Set `register` to true to request permanent registration; the script is registered only after execution succeeds with a result object that includes `ok: true`.', inputSchema: { script: z .string() .min(1) - .describe('Script source code to execute in the specified browser tab. The script can call registered scripts through cap.call(...).'), + .describe('Script source code to execute in the specified browser tab.'), input: z .record(z.string(), z.unknown()) .describe('Input object passed to the script at execution time.'), @@ -156,6 +186,10 @@ export async function executeCoreTool( const input = parseToolInput(toolName, rawInput); return (await app.browserNewTab(input)) as unknown as Record; } + case 'browser_screenshot': { + const input = parseToolInput(toolName, rawInput); + return (await app.browserScreenshot(input)) as unknown as Record; + } case 'script_execute': { const input = parseToolInput(toolName, rawInput); return (await app.scriptExecute(input)) as unknown as Record; diff --git a/shared/browser-command-contracts.ts b/shared/browser-command-contracts.ts index 0110919..e5f124d 100644 --- a/shared/browser-command-contracts.ts +++ b/shared/browser-command-contracts.ts @@ -9,6 +9,12 @@ export const DEFAULT_BROWSER_COMMAND_TIMEOUT_MS = 15_000; export const BROWSER_COMMAND_RESPONSE_GRACE_MS = 5_000; export const browserCommandInputSchemas = { + browser_screenshot: z.object({ + type: z.enum(['png', 'jpeg']).optional(), + quality: z.number().int().min(0).max(100).optional(), + fullPage: z.boolean().optional(), + omitBackground: z.boolean().optional(), + }), create_tab: z.object({ url: z.string().optional(), active: z.boolean().optional(), @@ -19,6 +25,9 @@ export const browserCommandInputSchemas = { } as const; export const browserCommandRequestSchemas = { + browser_screenshot: browserCommandInputSchemas.browser_screenshot.extend({ + tabId: z.number().int().optional(), + }), create_tab: browserCommandInputSchemas.create_tab, wait_events: browserCommandInputSchemas.wait_events.extend({ tabId: z.number().int().optional(), @@ -26,13 +35,14 @@ export const browserCommandRequestSchemas = { } as const; export type ContractedBrowserCommandName = keyof typeof browserCommandInputSchemas; +export type BrowserScreenshotInput = z.infer; export type CreateTabInput = z.infer; export type WaitEventsInput = z.infer; export function isContractedBrowserCommand( command: BrowserCommandName, ): command is ContractedBrowserCommandName { - return command === 'create_tab' || command === 'wait_events'; + return command === 'browser_screenshot' || command === 'create_tab' || command === 'wait_events'; } export function normalizeWaitEventsDurationMs(durationMs: number | undefined): number { diff --git a/shared/protocol.ts b/shared/protocol.ts index 08057e2..b32b1b3 100644 --- a/shared/protocol.ts +++ b/shared/protocol.ts @@ -20,6 +20,7 @@ export type BrowserCommandName = | 'click_element' | 'fill_input' | 'navigate' + | 'browser_screenshot' | 'create_tab' | 'wait_events'; @@ -64,6 +65,7 @@ export interface ExecuteScriptPayload { tabId?: number; activateTab?: boolean; evidence?: ExecutionEvidenceOption[]; + screenshotArtifactBasePath?: string; } export interface BrowserCommandPayload { @@ -115,6 +117,15 @@ export interface ExecutionResultPayload { result: Record; evidence: ExecutionEvidence; status?: 'succeeded' | 'interrupted'; + screenshotArtifacts?: RuntimeScreenshotArtifactPayload[]; +} + +export interface RuntimeScreenshotArtifactPayload { + kind: 'screenshot'; + path: string; + transferId: string; + mimeType: string; + type: 'png' | 'jpeg'; } export interface RuntimeErrorPayload { @@ -132,6 +143,16 @@ export interface BrowserCommandEventPayload { event: Record; } +export interface RuntimeBinaryPayloadStartPayload { + transferId: string; + kind: 'screenshot'; + mimeType: string; + type: 'png' | 'jpeg'; + byteLength: number; + resultShape: 'path' | 'metadata'; + path?: string; +} + export interface ScriptHistorySyncPayload { entries: ScriptExecutionHistoryEntry[]; } @@ -197,6 +218,13 @@ export type RuntimeEnvelope = timestamp: string; payload: BrowserCommandEventPayload; } + | { + type: 'binary_payload_start'; + sessionId: string; + requestId: string; + timestamp: string; + payload: RuntimeBinaryPayloadStartPayload; + } | { type: 'execution_result'; sessionId: string; @@ -270,6 +298,15 @@ export interface BrowserCommandResult { tab: RuntimeTabSnapshot; } +export interface BrowserScreenshotResult { + result: { + path: string; + sizeBytes?: number; + }; + timingMs: number; + tab: RuntimeTabSummary; +} + export interface RuntimeConnectionSnapshot { connected: boolean; sessionId: string; diff --git a/skills/web-cap/SKILL.md b/skills/web-cap/SKILL.md index 01c14f0..2ebf9e6 100644 --- a/skills/web-cap/SKILL.md +++ b/skills/web-cap/SKILL.md @@ -125,6 +125,14 @@ web-cap script-execute --tab-id --script "export default async function When a script operates on page content, prefer the Playwright-compatible runtime APIs exposed as global `page` and `cap.page`. Use `page.locator(...)`, role/text helpers, and locator actions instead of hand-rolled DOM clicks or form mutations. +If the target element cannot be determined confidently from roles, text, selectors, or DOM state, capture a screenshot before taking action. Use visual confirmation to disambiguate the target and avoid guessing, especially when controls have similar labels, icons, or repeated layouts. + +Use `browser-screenshot` with the tab id from `session-status`: + +```bash +web-cap browser-screenshot --tab-id --pretty +``` + Example form interaction: ```javascript diff --git a/tests/browser-command-contracts.test.ts b/tests/browser-command-contracts.test.ts index 354b7b2..7cfe69b 100644 --- a/tests/browser-command-contracts.test.ts +++ b/tests/browser-command-contracts.test.ts @@ -23,6 +23,27 @@ describe('browser command contracts', () => { }); }); + it('validates browser_screenshot options', () => { + expect(parseBrowserCommandRequest('browser_screenshot', { + tabId: 7, + type: 'jpeg', + quality: 80, + fullPage: true, + })).toEqual({ + tabId: 7, + type: 'jpeg', + quality: 80, + fullPage: true, + }); + + expect(() => parseBrowserCommandRequest('browser_screenshot', { type: 'webp' })).toThrow( + /Invalid browser_screenshot browser command input/, + ); + expect(() => parseBrowserCommandRequest('browser_screenshot', { quality: 101 })).toThrow( + /Invalid browser_screenshot browser command input/, + ); + }); + it('normalizes wait_events duration and derives command timeout', () => { expect(normalizeWaitEventsDurationMs(undefined)).toBe(30_000); expect(timeoutForBrowserCommand('wait_events', { durationMs: 250 })).toBe( diff --git a/tests/web-cap-app.test.ts b/tests/web-cap-app.test.ts index d0ca3fb..a076d9e 100644 --- a/tests/web-cap-app.test.ts +++ b/tests/web-cap-app.test.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, rm, stat, utimes, writeFile } from 'node:fs/promises'; import { createServer } from 'node:net'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; @@ -50,6 +50,7 @@ describe('WebCapAgentApp', () => { afterEach(async () => { vi.useRealTimers(); + delete process.env.WEB_CAP_STATE_DIR; for (const connectedClient of clients) { connectedClient.close(); } @@ -192,6 +193,75 @@ describe('WebCapAgentApp', () => { }); }); + it('stores page screenshot artifacts returned from script execution', async () => { + process.env.WEB_CAP_STATE_DIR = tempDir; + const screenshotBytes = Buffer.from('script screenshot bytes'); + await connectRuntime((envelope) => { + if (envelope.type !== 'execute_script') { + return; + } + + const screenshotPath = join( + String(envelope.payload.screenshotArtifactBasePath), + 's-abcdefghijk.png', + ); + const transferId = 'script-screenshot-transfer'; + client?.send(JSON.stringify(createRuntimeEnvelope( + 'binary_payload_start', + { + transferId, + kind: 'screenshot', + mimeType: 'image/png', + type: 'png', + byteLength: screenshotBytes.byteLength, + resultShape: 'metadata', + path: screenshotPath, + }, + { sessionId: 'runtime-session', requestId: envelope.requestId }, + ))); + client?.send(screenshotBytes); + client?.send( + JSON.stringify( + createRuntimeEnvelope( + 'execution_result', + { + result: { + ok: true, + screenshot: { + path: screenshotPath, + }, + }, + screenshotArtifacts: [ + { + kind: 'screenshot', + path: screenshotPath, + transferId, + mimeType: 'image/png', + type: 'png', + }, + ], + evidence: { + url: 'https://example.com/form', + events: [], + screenshots: [], + }, + }, + { sessionId: 'runtime-session', requestId: envelope.requestId }, + ), + ), + ); + }); + + const execution = await app.scriptExecute({ + script: 'export default async function () { return { ok: true }; }', + input: {}, + }); + + const screenshot = execution.result.screenshot as Record; + expect(screenshot.path).toEqual(expect.stringContaining(join(tempDir, 'temp-screenshots'))); + await expect(readFile(String(screenshot.path))).resolves.toEqual(screenshotBytes); + }); + it('summarizes consecutive managed mouse move evidence', async () => { await connectRuntime((envelope) => { if (envelope.type !== 'execute_script') { @@ -1196,6 +1266,9 @@ describe('WebCapAgentApp', () => { async scriptRegistryList() { return []; }, + async browserScreenshot() { + throw new Error('not used'); + }, async browserNewTab() { throw new Error('not used'); }, @@ -1284,6 +1357,90 @@ describe('WebCapAgentApp', () => { }); }); + it('captures a browser screenshot through the shared runtime bridge', async () => { + process.env.WEB_CAP_STATE_DIR = tempDir; + const screenshotDirectory = join(tempDir, 'temp-screenshots'); + await mkdir(screenshotDirectory, { recursive: true }); + const expiredScreenshot = join(screenshotDirectory, 'screenshot-old.png'); + const freshScreenshot = join(screenshotDirectory, 'screenshot-fresh.jpg'); + const nonScreenshotFile = join(screenshotDirectory, 'notes.txt'); + await writeFile(expiredScreenshot, 'old'); + await writeFile(freshScreenshot, 'fresh'); + await writeFile(nonScreenshotFile, 'notes'); + const expiredDate = new Date(Date.now() - 25 * 60 * 60 * 1000); + await utimes(expiredScreenshot, expiredDate, expiredDate); + + const screenshotBytes = Buffer.from('jpeg image bytes'); + await connectRuntime((envelope) => { + if (envelope.type !== 'browser_command') { + return; + } + + expect(envelope.payload.command).toBe('browser_screenshot'); + expect(envelope.payload.tabId).toBe(101); + expect(envelope.payload.input).toEqual({ + type: 'jpeg', + quality: 80, + fullPage: true, + }); + + const transferId = 'browser-screenshot-transfer'; + client?.send(JSON.stringify(createRuntimeEnvelope( + 'binary_payload_start', + { + transferId, + kind: 'screenshot', + mimeType: 'image/jpeg', + type: 'jpeg', + byteLength: screenshotBytes.byteLength, + resultShape: 'metadata', + }, + { sessionId: 'runtime-session', requestId: envelope.requestId }, + ))); + client?.send(screenshotBytes); + client?.send(JSON.stringify(createRuntimeEnvelope( + 'browser_command_result', + { + result: { + __webCapType: 'screenshot_transfer', + transferId, + resultShape: 'metadata', + }, + }, + { sessionId: 'runtime-session', requestId: envelope.requestId }, + ))); + }); + + const result = await app.browserScreenshot({ + tabId: 101, + type: 'jpeg', + quality: 80, + fullPage: true, + }); + + expect(result.result).toMatchObject({ + sizeBytes: screenshotBytes.byteLength, + }); + expect(result.result.path).toEqual(expect.stringContaining(screenshotDirectory)); + expect(result.result).not.toHaveProperty('data'); + expect(result).not.toHaveProperty('command'); + expect(result.tab).toEqual({ + tabId: 101, + url: 'https://example.com/form', + title: 'Example Form', + }); + + const storedPath = result.result.path; + if (typeof storedPath !== 'string') { + throw new Error('Expected screenshot path.'); + } + await expect(readFile(storedPath)).resolves.toEqual(screenshotBytes); + await expect(readFile(expiredScreenshot)).rejects.toMatchObject({ code: 'ENOENT' }); + await expect(readFile(freshScreenshot, 'utf8')).resolves.toBe('fresh'); + await expect(readFile(nonScreenshotFile, 'utf8')).resolves.toBe('notes'); + await expect(stat(storedPath)).resolves.toMatchObject({ size: screenshotBytes.byteLength }); + }); + it('waits for browser events with routed tab input and event streaming', async () => { const observedEvents: Record[] = []; await connectRuntime((envelope) => { diff --git a/tests/web-cap-cli.test.ts b/tests/web-cap-cli.test.ts index 97d701f..3701af5 100644 --- a/tests/web-cap-cli.test.ts +++ b/tests/web-cap-cli.test.ts @@ -33,6 +33,9 @@ describe('WEB_CAP CLI', () => { async scriptRegistryList() { return []; }, + async browserScreenshot() { + throw new Error('not used'); + }, async browserNewTab() { throw new Error('not used'); }, @@ -150,6 +153,22 @@ describe('WEB_CAP CLI', () => { active: false, }, }); + expect(parseCliArgs([ + 'browser-screenshot', + '--tab-id', + '7', + '--type', + 'jpeg', + '--quality', + '80', + ])).toEqual({ + name: 'browser-screenshot', + options: { + tabId: 7, + type: 'jpeg', + quality: 80, + }, + }); expect(parseCliArgs(['config', 'set', 'activateTabOnScriptExecute', 'true'])).toEqual({ name: 'config', options: { @@ -198,7 +217,7 @@ describe('WEB_CAP CLI', () => { expect(stdout).toContain('Script execution:'); expect(stdout).toContain('web-cap script-execute --tab-id --script '); expect(stdout).toContain('Runs JavaScript in the selected browser tab.'); - expect(stdout).toContain('use cap.call(...) inside the script'); + expect(stdout).not.toContain('use cap.call(...) inside the script'); expect(stdout).toContain('Playwright-style page API as global page and cap.page'); expect(stdout).toContain("await page.getByRole('button', { name: 'Login' }).click();"); expect(stdout).toContain("await page.locator('input[name=email]').fill(input.email);"); @@ -279,10 +298,26 @@ describe('WEB_CAP CLI', () => { }, }; }, + async browserScreenshot(input) { + calls.push(`browserScreenshot:${input.tabId ?? ''}:${input.type ?? ''}`); + return { + result: { + path: '/tmp/web-cap/temp-screenshots/s-Abc_123-xYz.png', + sizeBytes: 8, + }, + timingMs: 1, + tab: { + tabId: input.tabId ?? 2, + url: 'https://example.com', + title: 'Example', + }, + }; + }, }); const runs = [ ['session-status'], + ['browser-screenshot', '--tab-id', '2'], ['browser-new-tab', '--url', 'https://example.com', '--active', 'true'], ]; @@ -301,10 +336,16 @@ describe('WEB_CAP CLI', () => { expect(code).toBe(0); expect(stderr).toBe(''); expect(() => JSON.parse(stdout)).not.toThrow(); + if (argv[0] === 'browser-screenshot') { + const parsed = JSON.parse(stdout) as { result: Record }; + expect(parsed.result.path).toContain('temp-screenshots'); + expect(parsed.result).not.toHaveProperty('data'); + } } expect(calls).toEqual([ 'sessionStatus', + 'browserScreenshot:2:', 'browserNewTab:https://example.com:true', ]); }); @@ -428,6 +469,9 @@ describe('WEB_CAP CLI', () => { async scriptRegistryList() { return []; }, + async browserScreenshot() { + throw new Error('not used'); + }, async browserNewTab() { throw new Error('not used'); }, @@ -594,6 +638,9 @@ describe('WEB_CAP CLI', () => { async scriptRegistryList() { return []; }, + async browserScreenshot() { + throw new Error('not used'); + }, async browserNewTab() { throw new Error('not used'); }, From 172dc7d52dc78be580c445edbe0d453ec74409b7 Mon Sep 17 00:00:00 2001 From: EdgeStorage Date: Sun, 31 May 2026 15:56:18 +0800 Subject: [PATCH 2/3] feat: add page keyboard shim --- AGENTS.md | 26 +++ extension/entrypoints/background.ts | 26 ++- extension/runtime/chrome-debugger-client.ts | 43 ++++- .../injected/playwright-shim.injected.ts | 50 +++++- tests/debugger-executor.test.ts | 67 +++++++- tests/execution-helpers.test.ts | 158 ++++++++++++++++++ 6 files changed, 362 insertions(+), 8 deletions(-) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..b85935d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,26 @@ +# Agent Notes + +## Beta Release Flow + +When asked to publish a beta version: + +1. Do not change `package.json` version on the current branch. +2. Determine `BASE_VERSION` from the latest npm beta dist-tag, unless the user explicitly pins a different base version. Do not derive it from the current branch `package.json`. + - npm beta: `npm_config_cache=/private/tmp/web-cap-npm-cache npm view web-capability@beta version` + - Example: if npm beta is `0.0.7-beta.3`, then `BASE_VERSION=0.0.7` and the next beta is `0.0.7-beta.4`. +3. Check existing beta versions from both npm and the private git remote, then choose the next beta number for that `BASE_VERSION`. + - npm: `npm_config_cache=/private/tmp/web-cap-npm-cache npm view web-capability versions --json` + - private tags: `git ls-remote --tags private "v${BASE_VERSION}-beta.*"` +4. Commit any requested code changes on the current branch first, without changing the package version. +5. Create a temporary release worktree from the current `HEAD`. + - Example: `git worktree add /private/tmp/web-cap-tag-${BASE_VERSION}-beta.N HEAD` +6. In that temporary worktree only, update `package.json` to the beta version. +7. Commit the release version change in the temporary worktree. + - Example: `chore: release ${BASE_VERSION}-beta.N` +8. Create `v${BASE_VERSION}-beta.N` on that temporary release commit. +9. Push only the tag to the `private` remote. + - Example: `git push private v${BASE_VERSION}-beta.N` +10. Verify: + - `git show v${BASE_VERSION}-beta.N:package.json` reports the beta version. + - The current branch `package.json` still has its original version. + - The current branch worktree is clean. diff --git a/extension/entrypoints/background.ts b/extension/entrypoints/background.ts index 54ec332..48f32ab 100644 --- a/extension/entrypoints/background.ts +++ b/extension/entrypoints/background.ts @@ -208,7 +208,18 @@ class RuntimeClient { evidenceOptions: ExecutionEvidenceOption[] = ['common'], screenshotArtifactBasePath?: string, ): Promise { - const selectedTab = tabId ? await browser.tabs.get(tabId) : await this.getActiveTab(); + let selectedTab: BrowserTabLike | undefined; + try { + selectedTab = tabId ? await browser.tabs.get(tabId) : await this.getActiveTab(); + } catch (error) { + this.sendError( + requestId, + 'EXECUTION_FAILED', + error instanceof Error ? error.message : String(error), + { scriptId: scriptDefinition.id, tabId }, + ); + return; + } if (!selectedTab?.id || !selectedTab.url) { this.sendError(requestId, 'TAB_NOT_FOUND', 'No active browser tab is available.', { scriptId: scriptDefinition.id, @@ -438,7 +449,18 @@ class RuntimeClient { input: Record, tabId?: number, ): Promise { - const activeTab = tabId ? await browser.tabs.get(tabId) : await this.getActiveTab(); + let activeTab: BrowserTabLike | undefined; + try { + activeTab = tabId ? await browser.tabs.get(tabId) : await this.getActiveTab(); + } catch (error) { + this.sendError( + requestId, + 'EXECUTION_FAILED', + error instanceof Error ? error.message : String(error), + { command, tabId }, + ); + return; + } if (!activeTab?.id || !activeTab.url) { this.sendError(requestId, 'TAB_NOT_FOUND', 'No active browser tab is available.', { command, diff --git a/extension/runtime/chrome-debugger-client.ts b/extension/runtime/chrome-debugger-client.ts index c4fa7a5..000b7f5 100644 --- a/extension/runtime/chrome-debugger-client.ts +++ b/extension/runtime/chrome-debugger-client.ts @@ -8,6 +8,7 @@ interface AttachedSession { const DEBUGGER_VERSION = '1.3'; const DEFAULT_IDLE_DETACH_DELAY_MS = 60_000; +const DEFAULT_DEBUGGER_OPERATION_TIMEOUT_MS = 10_000; export class ChromeDebuggerClient { private readonly sessions = new Map(); @@ -15,6 +16,7 @@ export class ChromeDebuggerClient { constructor( private readonly idleDetachDelayMs = DEFAULT_IDLE_DETACH_DELAY_MS, private readonly onIdleDetach?: (tabId: number) => void, + private readonly operationTimeoutMs = DEFAULT_DEBUGGER_OPERATION_TIMEOUT_MS, ) {} isAvailable(): boolean { @@ -43,7 +45,7 @@ export class ChromeDebuggerClient { throw new Error('chrome.debugger is not available in this browser runtime.'); } - return await new Promise((resolve, reject) => { + return await this.withTimeout(`chrome.debugger.sendCommand(${method})`, (resolve, reject) => { chromeApi.debugger?.sendCommand( target, method, @@ -129,7 +131,7 @@ export class ChromeDebuggerClient { throw new Error('chrome.debugger is not available in this browser runtime.'); } - await new Promise((resolve, reject) => { + await this.withTimeout('chrome.debugger.attach', (resolve, reject) => { chromeApi.debugger?.attach(target, DEBUGGER_VERSION, () => { const error = chromeApi.runtime?.lastError; if (error) { @@ -147,7 +149,7 @@ export class ChromeDebuggerClient { return; } - await new Promise((resolve, reject) => { + await this.withTimeout('chrome.debugger.detach', (resolve, reject) => { chromeApi.debugger?.detach(target, () => { const error = chromeApi.runtime?.lastError; if (error) { @@ -158,5 +160,38 @@ export class ChromeDebuggerClient { }); }).catch(() => undefined); } -} + private async withTimeout( + label: string, + start: (resolve: (value: T) => void, reject: (error: Error) => void) => void, + ): Promise { + return await new Promise((resolve, reject) => { + let settled = false; + const timeout = setTimeout(() => { + if (settled) { + return; + } + settled = true; + reject(new Error(`${label} timed out after ${this.operationTimeoutMs}ms.`)); + }, this.operationTimeoutMs); + + const finish = (callback: () => void) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + callback(); + }; + + try { + start( + (value) => finish(() => resolve(value)), + (error) => finish(() => reject(error)), + ); + } catch (error) { + finish(() => reject(error instanceof Error ? error : new Error(String(error)))); + } + }); + } +} diff --git a/extension/runtime/injected/playwright-shim.injected.ts b/extension/runtime/injected/playwright-shim.injected.ts index c37933a..8eed21c 100644 --- a/extension/runtime/injected/playwright-shim.injected.ts +++ b/extension/runtime/injected/playwright-shim.injected.ts @@ -1,7 +1,7 @@ /* eslint-disable */ import type { PlaywrightShimDeps, RuntimeMethodTable, ScriptPlaywrightPage } from './playwright-shim-types.injected'; import { createLocator } from './playwright-locator.injected'; -import { cssEscape, hideHighlightOverlay, notImplemented, queryLocatorSelectorAll, timeoutFromOptions, waitForLocator } from './playwright-shim-helpers.injected'; +import { cssEscape, hideHighlightOverlay, notImplemented, pressKeyOnElement, queryLocatorSelectorAll, timeoutFromOptions, waitForLocator } from './playwright-shim-helpers.injected'; export function createPlaywrightPageApi(deps: PlaywrightShimDeps): ScriptPlaywrightPage { let defaultTimeoutMs = 5000; @@ -255,6 +255,53 @@ function createPageApi(): ScriptPlaywrightPage { }); }, }; + const activeKeyboardTarget = () => { + const element = document.activeElement; + if (element instanceof HTMLElement) { + return element; + } + if (document.body instanceof HTMLElement) { + return document.body; + } + throw new Error('page.keyboard requires an active HTMLElement target.'); + }; + const keyboardDelay = async (options: { delay?: unknown } = {}) => { + const delay = Math.max(Number(options.delay ?? 0), 0); + if (delay > 0) { + await deps.wait(delay); + } + }; + const dispatchKeyboardOnly = async (type: string, key: unknown) => { + const target = activeKeyboardTarget(); + target.dispatchEvent(new KeyboardEvent(type, { + key: String(key ?? ''), + bubbles: true, + cancelable: true, + })); + await deps.waitForManagedInput(); + }; + const keyboardApi: RuntimeMethodTable = { + async down(key: unknown) { + await dispatchKeyboardOnly('keydown', key); + }, + async up(key: unknown) { + await dispatchKeyboardOnly('keyup', key); + }, + async press(key: unknown, options: { delay?: unknown } = {}) { + await pressKeyOnElement(activeKeyboardTarget(), key, deps); + await keyboardDelay(options); + }, + async type(text: unknown, options: { delay?: unknown } = {}) { + const value = String(text ?? ''); + for (const char of value) { + await pressKeyOnElement(activeKeyboardTarget(), char, deps); + await keyboardDelay(options); + } + }, + async insertText(text: unknown, options: { delay?: unknown } = {}) { + await keyboardApi.type(text, options); + }, + }; const sameOriginFrameElements = () => [...document.querySelectorAll('iframe')] .filter((element): element is HTMLIFrameElement => element instanceof HTMLIFrameElement) @@ -666,6 +713,7 @@ function createPageApi(): ScriptPlaywrightPage { }; pageApi.__frameForElement = createFrameApi; Object.assign(pageApi, { + keyboard: keyboardApi, mouse: mouseApi, async $(selector: unknown) { return queryLocatorSelectorAll(String(selector))[0] ?? null; diff --git a/tests/debugger-executor.test.ts b/tests/debugger-executor.test.ts index 5416adb..f953e52 100644 --- a/tests/debugger-executor.test.ts +++ b/tests/debugger-executor.test.ts @@ -1,4 +1,5 @@ -import { afterEach, describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { ChromeDebuggerClient } from '../extension/runtime/chrome-debugger-client'; import { DebuggerScriptExecutor } from '../extension/runtime/debugger-executor'; import { createDebuggerActionScript, @@ -12,6 +13,8 @@ describe('DebuggerScriptExecutor', () => { const previousBrowser = (globalThis as typeof globalThis & { browser?: unknown }).browser; afterEach(() => { + vi.useRealTimers(); + if (previousChrome === undefined) { delete (globalThis as typeof globalThis & { chrome?: unknown }).chrome; } else { @@ -175,6 +178,68 @@ export default async function () { expect(commands.some(({ method }) => method === 'Runtime.removeBinding')).toBe(true); }); + it('times out debugger attach and sendCommand callbacks', async () => { + vi.useFakeTimers(); + + (globalThis as typeof globalThis & { chrome?: unknown }).chrome = { + debugger: { + attach: () => undefined, + detach: (_target: { tabId: number }, callback: () => void) => callback(), + sendCommand: () => undefined, + }, + runtime: { + lastError: undefined, + }, + }; + + const client = new ChromeDebuggerClient(60_000, undefined, 5); + const attached = client.withAttachedDebugger(7, async () => ({ ok: true })); + const attachedExpectation = expect(attached).rejects.toThrow( + 'chrome.debugger.attach timed out after 5ms.', + ); + await vi.advanceTimersByTimeAsync(5); + await attachedExpectation; + + const command = client.sendCommand({ tabId: 7 }, 'Runtime.evaluate'); + const commandExpectation = expect(command).rejects.toThrow( + 'chrome.debugger.sendCommand(Runtime.evaluate) timed out after 5ms.', + ); + await vi.advanceTimersByTimeAsync(5); + await commandExpectation; + }); + + it('does not wait forever for idle detach callbacks', async () => { + vi.useFakeTimers(); + let detached = false; + + (globalThis as typeof globalThis & { chrome?: unknown }).chrome = { + debugger: { + attach: (_target: { tabId: number }, _version: string, callback: () => void) => callback(), + detach: () => { + detached = true; + }, + sendCommand: ( + _target: { tabId: number }, + _method: string, + _params: Record, + callback: (result?: unknown) => void, + ) => callback({}), + }, + runtime: { + lastError: undefined, + }, + }; + + const client = new ChromeDebuggerClient(1, undefined, 5); + await expect(client.withAttachedDebugger(7, async () => ({ ok: true }))).resolves.toEqual({ + ok: true, + }); + + await vi.advanceTimersByTimeAsync(1); + await vi.advanceTimersByTimeAsync(5); + expect(detached).toBe(true); + }); + it('closes tabs through the managed window bridge', async () => { const commands: DebuggerCommand[] = []; const listeners = new Set(); diff --git a/tests/execution-helpers.test.ts b/tests/execution-helpers.test.ts index 9079282..9599cec 100644 --- a/tests/execution-helpers.test.ts +++ b/tests/execution-helpers.test.ts @@ -484,6 +484,164 @@ export default async function () { } }); + it('exposes Playwright keyboard actions on the page shim', async () => { + class FakeHTMLElement { + textContent = ''; + isContentEditable = true; + events: string[] = []; + + scrollIntoView() {} + + focus() { + fakeDocument.activeElement = this; + } + + dispatchEvent(event: { type: string }) { + this.events.push(event.type); + return true; + } + } + + class FakeInputElement extends FakeHTMLElement { + value = ''; + isContentEditable = false; + } + + class FakeKeyboardEvent { + type: string; + + constructor(type: string) { + this.type = type; + } + } + + class FakeInputEvent { + type: string; + + constructor(type: string) { + this.type = type; + } + } + + const target = new FakeHTMLElement(); + const fakeDocument = { + activeElement: target, + body: target, + documentElement: target, + querySelectorAll() { + return []; + }, + }; + const previousHTMLElement = Object.getOwnPropertyDescriptor(globalThis, 'HTMLElement'); + const previousHTMLInputElement = Object.getOwnPropertyDescriptor(globalThis, 'HTMLInputElement'); + const previousHTMLTextAreaElement = Object.getOwnPropertyDescriptor(globalThis, 'HTMLTextAreaElement'); + const previousKeyboardEvent = Object.getOwnPropertyDescriptor(globalThis, 'KeyboardEvent'); + const previousInputEvent = Object.getOwnPropertyDescriptor(globalThis, 'InputEvent'); + + Object.defineProperty(globalThis, 'document', { + value: fakeDocument, + configurable: true, + }); + Object.defineProperty(globalThis, 'HTMLElement', { + value: FakeHTMLElement, + configurable: true, + }); + Object.defineProperty(globalThis, 'HTMLInputElement', { + value: FakeInputElement, + configurable: true, + }); + Object.defineProperty(globalThis, 'HTMLTextAreaElement', { + value: FakeInputElement, + configurable: true, + }); + Object.defineProperty(globalThis, 'KeyboardEvent', { + value: FakeKeyboardEvent, + configurable: true, + }); + Object.defineProperty(globalThis, 'InputEvent', { + value: FakeInputEvent, + configurable: true, + }); + + try { + const script = scriptDefinitionSchema.parse({ + id: 'keyboard.type', + name: 'Keyboard Type', + version: '1.0.0', + status: 'active', + type: 'act', + summary: 'Types through the Playwright keyboard shim.', + target: { + site: 'generic-web', + urlPatterns: ['http://*', 'https://*'], + pageHints: [], + }, + tags: ['test'], + inputSchema: { + type: 'object', + properties: {}, + required: [], + additionalProperties: false, + }, + outputSchema: { + type: 'object', + properties: { + ok: { type: 'boolean' }, + text: { type: 'string' }, + eventCount: { type: 'number' }, + }, + required: ['ok', 'text', 'eventCount'], + additionalProperties: false, + }, + script: { + timeoutMs: 1_000, + code: ` +export default async function () { + await page.keyboard.type('hi'); + await page.keyboard.press('Enter'); + return { ok: true, text: document.activeElement.textContent, eventCount: document.activeElement.events.length }; +} + `.trim(), + }, + }); + + const expression = buildScriptExecutionExpression(script, {}, [], { evidence: ['events'] }); + const response = (await eval(expression)) as { + ok: boolean; + result?: Record; + }; + + expect(response.ok).toBe(true); + expect(response.result).toEqual({ ok: true, text: 'hi', eventCount: 11 }); + } finally { + if (previousHTMLElement) { + Object.defineProperty(globalThis, 'HTMLElement', previousHTMLElement); + } else { + delete (globalThis as { HTMLElement?: unknown }).HTMLElement; + } + if (previousHTMLInputElement) { + Object.defineProperty(globalThis, 'HTMLInputElement', previousHTMLInputElement); + } else { + delete (globalThis as { HTMLInputElement?: unknown }).HTMLInputElement; + } + if (previousHTMLTextAreaElement) { + Object.defineProperty(globalThis, 'HTMLTextAreaElement', previousHTMLTextAreaElement); + } else { + delete (globalThis as { HTMLTextAreaElement?: unknown }).HTMLTextAreaElement; + } + if (previousKeyboardEvent) { + Object.defineProperty(globalThis, 'KeyboardEvent', previousKeyboardEvent); + } else { + delete (globalThis as { KeyboardEvent?: unknown }).KeyboardEvent; + } + if (previousInputEvent) { + Object.defineProperty(globalThis, 'InputEvent', previousInputEvent); + } else { + delete (globalThis as { InputEvent?: unknown }).InputEvent; + } + } + }); + it('routes user script setTimeout through the managed timer bridge when provided', async () => { const originalSetTimeout = globalThis.setTimeout; const originalClearTimeout = globalThis.clearTimeout; From 9a6af05f365dc1b151f0b6724214e966809af62b Mon Sep 17 00:00:00 2001 From: EdgeStorage Date: Mon, 1 Jun 2026 00:35:06 +0800 Subject: [PATCH 3/3] fix: honor script execution timeout for debugger evaluate --- extension/runtime/chrome-debugger-client.ts | 40 ++++++++++++--------- extension/runtime/debugger-executor.ts | 2 ++ tests/debugger-executor.test.ts | 33 +++++++++++++++++ 3 files changed, 58 insertions(+), 17 deletions(-) diff --git a/extension/runtime/chrome-debugger-client.ts b/extension/runtime/chrome-debugger-client.ts index 000b7f5..8912ac2 100644 --- a/extension/runtime/chrome-debugger-client.ts +++ b/extension/runtime/chrome-debugger-client.ts @@ -39,27 +39,32 @@ export class ChromeDebuggerClient { target: DebuggeeTarget, method: string, commandParams?: Record, + options: { timeoutMs?: number } = {}, ): Promise { const chromeApi = this.getChromeApi(); if (!chromeApi?.debugger || !chromeApi.runtime) { throw new Error('chrome.debugger is not available in this browser runtime.'); } - return await this.withTimeout(`chrome.debugger.sendCommand(${method})`, (resolve, reject) => { - chromeApi.debugger?.sendCommand( - target, - method, - commandParams ?? {}, - (result?: unknown) => { - const error = chromeApi.runtime?.lastError; - if (error) { - reject(new Error(error.message)); - return; - } - resolve(result as T); - }, - ); - }); + return await this.withTimeout( + `chrome.debugger.sendCommand(${method})`, + (resolve, reject) => { + chromeApi.debugger?.sendCommand( + target, + method, + commandParams ?? {}, + (result?: unknown) => { + const error = chromeApi.runtime?.lastError; + if (error) { + reject(new Error(error.message)); + return; + } + resolve(result as T); + }, + ); + }, + options.timeoutMs, + ); } getChromeApi(): ChromeLike | undefined { @@ -164,6 +169,7 @@ export class ChromeDebuggerClient { private async withTimeout( label: string, start: (resolve: (value: T) => void, reject: (error: Error) => void) => void, + timeoutMs = this.operationTimeoutMs, ): Promise { return await new Promise((resolve, reject) => { let settled = false; @@ -172,8 +178,8 @@ export class ChromeDebuggerClient { return; } settled = true; - reject(new Error(`${label} timed out after ${this.operationTimeoutMs}ms.`)); - }, this.operationTimeoutMs); + reject(new Error(`${label} timed out after ${timeoutMs}ms.`)); + }, timeoutMs); const finish = (callback: () => void) => { if (settled) { diff --git a/extension/runtime/debugger-executor.ts b/extension/runtime/debugger-executor.ts index f0d18eb..f3db1ea 100644 --- a/extension/runtime/debugger-executor.ts +++ b/extension/runtime/debugger-executor.ts @@ -76,6 +76,8 @@ export class DebuggerScriptExecutor { returnByValue: true, userGesture: true, allowUnsafeEvalBlockedByCSP: true, + }, { + timeoutMs: scriptDefinition.script.timeoutMs, }); } finally { await browserBridge.dispose(); diff --git a/tests/debugger-executor.test.ts b/tests/debugger-executor.test.ts index f953e52..55a8db2 100644 --- a/tests/debugger-executor.test.ts +++ b/tests/debugger-executor.test.ts @@ -208,6 +208,39 @@ export default async function () { await commandExpectation; }); + it('allows long-running debugger commands to use a command-specific timeout', async () => { + vi.useFakeTimers(); + + (globalThis as typeof globalThis & { chrome?: unknown }).chrome = { + debugger: { + attach: () => undefined, + detach: (_target: { tabId: number }, callback: () => void) => callback(), + sendCommand: () => undefined, + }, + runtime: { + lastError: undefined, + }, + }; + + const client = new ChromeDebuggerClient(60_000, undefined, 10); + const command = client.sendCommand({ tabId: 7 }, 'Runtime.evaluate', undefined, { + timeoutMs: 15, + }); + let rejected = false; + void command.catch(() => { + rejected = true; + }); + + await vi.advanceTimersByTimeAsync(10); + expect(rejected).toBe(false); + + const commandExpectation = expect(command).rejects.toThrow( + 'chrome.debugger.sendCommand(Runtime.evaluate) timed out after 15ms.', + ); + await vi.advanceTimersByTimeAsync(5); + await commandExpectation; + }); + it('does not wait forever for idle detach callbacks', async () => { vi.useFakeTimers(); let detached = false;