From 13b4305a414617e57c1559c3f70d6d22a481b97f Mon Sep 17 00:00:00 2001 From: Octagon Date: Wed, 24 Jun 2026 22:17:32 +0100 Subject: [PATCH 1/9] feat: add Playwright runner support Recognise and run Playwright as a test runner alongside the unit runners. Adds @playwright/test detection, a parser for the JSON reporter (resolving spec paths against the report's rootDir, not cwd), best-effort config parsing for baseURL/testDir/webServer, and per-file run commands. Playwright is opt-in only (never auto-detected over Vitest/Jest); detectEnvironment gains type overloads so the no-config path never yields it. Co-Authored-By: Claude Opus 4.8 --- src/lib/detector.ts | 19 ++- src/lib/playwright.ts | 300 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 318 insertions(+), 1 deletion(-) create mode 100644 src/lib/playwright.ts diff --git a/src/lib/detector.ts b/src/lib/detector.ts index cb6d9ec..31062b0 100644 --- a/src/lib/detector.ts +++ b/src/lib/detector.ts @@ -1,7 +1,8 @@ import { readFile } from 'fs/promises' import { join } from 'path' +import { PLAYWRIGHT_DEFAULTS, playwrightRunCommand } from './playwright.js' -export type TestRunner = 'jest' | 'vitest' | 'pytest' | 'mocha' | 'go-test' | 'phpunit' | 'pest' | 'rspec' | 'cargo-test' | 'dotnet-test' | 'gradle-test' | 'maven-test' | 'swift-test' | 'unknown' +export type TestRunner = 'jest' | 'vitest' | 'pytest' | 'mocha' | 'go-test' | 'phpunit' | 'pest' | 'rspec' | 'cargo-test' | 'dotnet-test' | 'gradle-test' | 'maven-test' | 'swift-test' | 'playwright' | 'unknown' export type Language = 'typescript' | 'javascript' | 'python' | 'go' | 'php' | 'ruby' | 'rust' | 'csharp' | 'java' | 'swift' | 'unknown' export interface DetectedEnvironment { @@ -104,6 +105,7 @@ const RUNNER_DEFAULTS: Record, DetectedEnvironmen coverageCommand: 'swift test --enable-code-coverage', testCommand: 'swift test', }, + playwright: PLAYWRIGHT_DEFAULTS, } export function envForRunner(runner: string): DetectedEnvironment { @@ -159,6 +161,7 @@ export function fileTestCommand(env: DetectedEnvironment, testFilePath: string): case 'vitest': return `npx vitest run ${q}` case 'jest': return `npx jest --testPathPattern=${jestPath(testFilePath)}` case 'mocha': return `npx mocha ${q}` + case 'playwright': return playwrightRunCommand(testFilePath) case 'pytest': return `python -m pytest ${q} -v` case 'go-test': { const dir = testFilePath.includes('/') ? testFilePath.replace(/\/[^/]+$/, '') : '.' @@ -176,6 +179,17 @@ export function fileTestCommand(env: DetectedEnvironment, testFilePath: string): } } +// Overloads: with no configRunner, detection is purely auto and never yields +// 'playwright' (it is opt-in via --e2e), so the result narrows accordingly. +// With a configRunner the user may explicitly select playwright, so the wide +// DetectedEnvironment type applies. +export async function detectEnvironment( + cwd?: string, +): Promise }> +export async function detectEnvironment( + cwd: string | undefined, + configRunner: string | undefined, +): Promise export async function detectEnvironment( cwd: string = process.cwd(), configRunner?: string, @@ -196,6 +210,9 @@ export async function detectEnvironment( ...((pkg.devDependencies as Record) ?? {}), } + // Playwright is intentionally opt-in (selected via the --e2e flag, handled + // elsewhere) and is NOT auto-detected here: repos using Playwright for e2e + // usually also use Vitest/Jest for unit tests, which must keep winning here. if ('vitest' in deps) return RUNNER_DEFAULTS.vitest if ('jest' in deps || '@jest/core' in deps) return RUNNER_DEFAULTS.jest if ('mocha' in deps) return RUNNER_DEFAULTS.mocha diff --git a/src/lib/playwright.ts b/src/lib/playwright.ts new file mode 100644 index 0000000..35d421d --- /dev/null +++ b/src/lib/playwright.ts @@ -0,0 +1,300 @@ +// Playwright (end-to-end) runner support — PHASE 0 SEED. +// +// This module is intentionally self-contained and NOT yet wired into detector.ts / +// fix-loop.ts. It exists so the Playwright integration can be reviewed in isolation before +// we touch the unit-test path. The Phase 0 goal is the smallest useful slice: +// +// 1. Recognise a Playwright project (detectPlaywright). +// 2. Read the bits of playwright.config.ts the agent needs (loadPlaywrightConfig). +// 3. Produce the run commands for `lacuna fix --e2e` (playwrightTestCommand / RunCommand). +// 4. Turn a Playwright run into the structured failure summary the fix loop feeds back to +// the model (parsePlaywrightResults). +// +// Generation of brand-new specs from discovered "flows" is a LATER phase and deliberately +// not here — Phase 0 only repairs specs that already exist, which lets us reuse the entire +// existing fix loop (retry/oscillation/patch engine) and prove the run+parse plumbing first. +// +// Why Playwright does not fit the unit-test model (see project-map / design notes): there is +// no per-source-file coverage attribution, the app must be running, and "passing" means a +// green, non-flaky browser run rather than executed source lines. Everything below is shaped +// around that reality. + +import { readFile } from 'fs/promises' +import { join, isAbsolute } from 'path' +import type { DetectedEnvironment, TestRunner } from './detector.js' + +// --------------------------------------------------------------------------------------------- +// Runner defaults +// --------------------------------------------------------------------------------------------- + +// Drop-in addition for detector.ts's RUNNER_DEFAULTS once we wire it up. Note: there is no +// meaningful `coverageCommand` for E2E — coverage is not how Playwright suites are scoped — so +// it points at the plain test command. Selection is opt-in (a `--e2e` flag / `lacuna e2e` +// command), NEVER auto-detected over Vitest/Jest, because most repos have both and the unit +// path must keep winning. detectPlaywright() is only consulted when the user asks for E2E. +export const PLAYWRIGHT_DEFAULTS: DetectedEnvironment = { + testRunner: 'playwright' as TestRunner, // 'playwright' to be added to the TestRunner union + language: 'typescript', + // Playwright's own convention; the real testDir comes from playwright.config.ts at runtime. + testFilePattern: '**/*.{e2e,spec}.{ts,tsx,js,jsx,mjs}', + coverageCommand: 'npx playwright test', + testCommand: 'npx playwright test', +} + +// --------------------------------------------------------------------------------------------- +// Detection +// --------------------------------------------------------------------------------------------- + +// True when the project depends on @playwright/test. Kept separate from detector.ts's main +// resolution chain on purpose: Playwright coexists with a unit runner, so this is a capability +// check ("can we do E2E here?"), not a default-runner decision. +export async function detectPlaywright(cwd: string): Promise { + let pkg: { dependencies?: Record; devDependencies?: Record } = {} + try { + pkg = JSON.parse(await readFile(join(cwd, 'package.json'), 'utf-8')) + } catch { + return false + } + const deps = { ...(pkg.dependencies ?? {}), ...(pkg.devDependencies ?? {}) } + return '@playwright/test' in deps || 'playwright' in deps +} + +// --------------------------------------------------------------------------------------------- +// Config +// --------------------------------------------------------------------------------------------- + +export interface PlaywrightConfig { + // Base URL the specs navigate against (page.goto('/x') resolves against this). The model + // needs it to write correct goto() paths and we need it to know the app is reachable. + baseURL: string | null + // Directory Playwright loads specs from. Generated/repaired specs must live here, NOT in the + // co-located location context.ts computes for unit tests (that would fight Playwright). + testDir: string + // Command Playwright runs to bring the app up before the suite (config.webServer.command), + // and the URL it waits on. Null when the project expects an already-running app. + webServerCommand: string | null + webServerUrl: string | null + // Path to the resolved config file, for diagnostics. + configPath: string | null +} + +const CONFIG_FILENAMES = ['playwright.config.ts', 'playwright.config.js', 'playwright.config.mjs'] + +// Best-effort read of playwright.config.{ts,js,mjs}. The config is executable TS, not JSON, so +// we do NOT try to evaluate it — we extract the handful of fields the agent needs with narrow +// regexes and fall back to Playwright's documented defaults. This is the same pragmatic posture +// typecheck.ts takes with tsconfig (stripJsonc + tolerate failure): good enough to drive the +// flow, never throws. A later phase can shell out to `npx playwright test --list` for an exact, +// fully-resolved view if these heuristics prove too brittle. +export async function loadPlaywrightConfig(cwd: string): Promise { + const fallback: PlaywrightConfig = { + baseURL: null, + testDir: 'tests', // Playwright's default when testDir is unset + webServerCommand: null, + webServerUrl: null, + configPath: null, + } + + let raw: string | null = null + let configPath: string | null = null + for (const name of CONFIG_FILENAMES) { + try { + const p = join(cwd, name) + raw = await readFile(p, 'utf-8') + configPath = p + break + } catch { /* try next candidate */ } + } + if (raw == null) return fallback + + return { + baseURL: matchStringField(raw, 'baseURL') ?? fallback.baseURL, + testDir: matchStringField(raw, 'testDir') ?? fallback.testDir, + webServerCommand: matchWebServerField(raw, 'command'), + webServerUrl: matchWebServerField(raw, 'url'), + configPath, + } +} + +// Matches `key: 'value'` / `key: "value"` / `key: \`value\`` anywhere in the config source. +// Deliberately simple: it accepts the first literal assignment and ignores computed values +// (e.g. baseURL: process.env.BASE_URL) by returning null so the caller uses its fallback. +function matchStringField(src: string, key: string): string | null { + const re = new RegExp(`\\b${key}\\s*:\\s*(['"\`])([^'"\`]*)\\1`) + const m = src.match(re) + return m ? m[2] : null +} + +// Pulls a field out of the `webServer: { ... }` block specifically, so we don't accidentally +// grab a `command`/`url` from elsewhere. Returns null if there's no webServer block (the app +// is expected to be already running). +function matchWebServerField(src: string, key: string): string | null { + const block = src.match(/webServer\s*:\s*\{([\s\S]*?)\}/) + if (!block) return null + return matchStringField(block[1], key) +} + +// --------------------------------------------------------------------------------------------- +// Run commands +// --------------------------------------------------------------------------------------------- + +// Single-file run, the analogue of detector.ts's testFileCommand for the fix loop. Playwright +// takes a path filter as a positional arg. We delegate app start/stop entirely to Playwright's +// own webServer machinery (see design note 2) rather than orchestrating it ourselves. +export function playwrightRunCommand(testFile: string): string { + return `npx playwright test ${shellQuote(testFile)} --reporter=json` +} + +// Whole-suite run (selection phase / verification sweeps). +export function playwrightTestCommand(): string { + return 'npx playwright test --reporter=json' +} + +function shellQuote(s: string): string { + return `'${s.replace(/'/g, `'\\''`)}'` +} + +// --------------------------------------------------------------------------------------------- +// Result parsing +// --------------------------------------------------------------------------------------------- + +export interface PlaywrightFailure { + file: string + title: string // full test title path, e.g. "auth › user can log in" + message: string // trimmed error message + the failing step/location + attachments: string[] // trace.zip / screenshot paths, useful in a bug report and to the model +} + +export interface PlaywrightRunResult { + passed: number + failed: number + flaky: number + failures: PlaywrightFailure[] +} + +// Parses Playwright's JSON reporter output (`--reporter=json`) into a structured result the fix +// loop can act on. This is the E2E analogue of extract-error.ts: it answers "what failed and +// why" in a form small enough to put in a retry prompt. Returns null when the output isn't +// parseable JSON (e.g. the run crashed before producing a report) so the caller can fall back +// to raw stderr, exactly like the unit path does on an unrecognised runner. +export function parsePlaywrightResults(stdout: string): PlaywrightRunResult | null { + const json = extractJsonBlob(stdout) + if (!json) return null + + let report: PlaywrightJsonReport + try { + report = JSON.parse(json) as PlaywrightJsonReport + } catch { + return null + } + + const failures: PlaywrightFailure[] = [] + let passed = 0 + let failed = 0 + let flaky = 0 + + // Playwright reports spec/suite `file` paths RELATIVE TO config.rootDir (which is the + // testDir, not the project root), so we must resolve them against rootDir to get a path that + // actually exists. Resolving against cwd instead drops the testDir segment (e.g. looking for + // ./dashboard.spec.ts instead of ./e2e/dashboard.spec.ts), which makes the fix loop unable to + // read the file it was told is failing. + const rootDir = report.config?.rootDir + const resolveFile = (f: string | undefined): string => { + if (!f) return '(unknown)' + if (isAbsolute(f)) return f + return rootDir ? join(rootDir, f) : f + } + + // The JSON report is a tree: suites → (nested suites) → specs → tests → results. + const walkSuite = (suite: PwSuite, titlePath: string[]): void => { + const here = suite.title ? [...titlePath, suite.title] : titlePath + for (const spec of suite.specs ?? []) { + const specTitle = [...here, spec.title].filter(Boolean).join(' › ') + for (const test of spec.tests ?? []) { + const status = test.status ?? worstResultStatus(test.results) + if (status === 'flaky') flaky++ + else if (spec.ok) passed++ + else failed++ + + if (!spec.ok) { + const errs = (test.results ?? []).flatMap((r) => r.errors ?? []) + failures.push({ + file: resolveFile(suite.file ?? spec.file), + title: specTitle, + message: summariseErrors(errs), + attachments: collectAttachments(test.results), + }) + } + } + } + for (const child of suite.suites ?? []) walkSuite(child, here) + } + + for (const suite of report.suites ?? []) walkSuite(suite, []) + + return { passed, failed, flaky, failures } +} + +// Playwright prints non-JSON lines (webServer logs, the dot reporter) around the JSON blob when +// stdout is noisy. Grab the outermost {...} so JSON.parse has a clean payload. +function extractJsonBlob(stdout: string): string | null { + const start = stdout.indexOf('{') + const end = stdout.lastIndexOf('}') + if (start === -1 || end === -1 || end <= start) return null + return stdout.slice(start, end + 1) +} + +function worstResultStatus(results: PwResult[] | undefined): string { + if (!results || results.length === 0) return 'unknown' + if (results.some((r) => r.status === 'failed' || r.status === 'timedOut')) return 'failed' + return results[results.length - 1].status ?? 'unknown' +} + +function summariseErrors(errors: PwError[]): string { + if (errors.length === 0) return 'Test failed with no captured error message.' + // Strip ANSI colour codes Playwright embeds in messages; keep it compact for the prompt. + return errors + .map((e) => (e.message ?? '').replace(/\[[0-9;]*m/g, '').trim()) + .filter(Boolean) + .join('\n\n') + .slice(0, 4000) +} + +function collectAttachments(results: PwResult[] | undefined): string[] { + if (!results) return [] + return results + .flatMap((r) => r.attachments ?? []) + .map((a) => a.path) + .filter((p): p is string => typeof p === 'string') +} + +// --- Minimal shape of the Playwright JSON reporter we depend on. Intentionally partial. ------ + +interface PlaywrightJsonReport { + suites?: PwSuite[] + config?: { rootDir?: string } +} +interface PwSuite { + title?: string + file?: string + specs?: PwSpec[] + suites?: PwSuite[] +} +interface PwSpec { + title: string + ok: boolean + file?: string + tests?: PwTest[] +} +interface PwTest { + status?: string + results?: PwResult[] +} +interface PwResult { + status?: string + errors?: PwError[] + attachments?: { name?: string; path?: string }[] +} +interface PwError { + message?: string +} From da5baaf4248924e35408cd9e9927acc595299038 Mon Sep 17 00:00:00 2001 From: Octagon Date: Wed, 24 Jun 2026 22:17:32 +0100 Subject: [PATCH 2/9] feat: E2E test generation, repair, and data-testid injection Add `lacuna generate --e2e` and `lacuna fix --e2e`, a browser-layer counterpart to the unit/integration agent. generate --e2e: - Route discovery for Next.js (app + pages router) and React Router, dependency-gated so a Vite app with src/pages/ isn't misread as Next. - Per-route DOM snapshot via the project's own Playwright (the webServer starts the app once): aria tree + data-testids (with tag/role), redirect-aware. - DOM-aware spec generation with strict selector discipline (role/label/ testid only, no CSS/XPath, no arbitrary waits, no forced interactions, post-action validation, auth/isolation rules), priority-ordered truncation, parallel workers with the live panel, and flake confirmation. A spec that never goes green is deleted, never left broken. fix --e2e: - Repairs failing specs with a failure-analysis prompt (selector drift, strict-mode violations, sync, redirect) driven by a FRESH snapshot of the failing route, with an explicit repair-priority order that never weakens assertions to force a pass. --inject-testids (opt-in, the only path that edits app source): - Adds data-testid to page sources, then VERIFIES against a re-snapshot: a testid is kept only if it lands on an actual interactive element, else the source is reverted. So a non-forwarding or custom component leaves source untouched regardless of prompt guidance. - Library-aware (lib/flows/ui-libraries.ts + resolve-libraries.ts): detects the UI library by following the import chain, including barrel re-exports, so a custom component behind a barrel isn't mistaken for a library one; falls back to installed deps only when the chain can't be resolved. The agent loop, provider, retry/oscillation machinery, and result parser are shared with the unit path; target selection, context, and verification differ. Co-Authored-By: Claude Opus 4.8 --- src/agent/context.ts | 4 +- src/agent/e2e-loop.ts | 448 +++++++++++++++++++++++++++++ src/agent/fix-loop.ts | 182 ++++++++++-- src/agent/generator.ts | 70 ++++- src/agent/prompts/e2e.ts | 322 +++++++++++++++++++++ src/commands/fix.ts | 15 +- src/commands/generate.ts | 72 +++++ src/lib/flows/app-server.ts | 83 ++++++ src/lib/flows/discover.ts | 390 +++++++++++++++++++++++++ src/lib/flows/resolve-libraries.ts | 80 ++++++ src/lib/flows/snapshot.ts | 247 ++++++++++++++++ src/lib/flows/ui-libraries.ts | 234 +++++++++++++++ 12 files changed, 2112 insertions(+), 35 deletions(-) create mode 100644 src/agent/e2e-loop.ts create mode 100644 src/agent/prompts/e2e.ts create mode 100644 src/lib/flows/app-server.ts create mode 100644 src/lib/flows/discover.ts create mode 100644 src/lib/flows/resolve-libraries.ts create mode 100644 src/lib/flows/snapshot.ts create mode 100644 src/lib/flows/ui-libraries.ts diff --git a/src/agent/context.ts b/src/agent/context.ts index 4cd25a1..d059a73 100644 --- a/src/agent/context.ts +++ b/src/agent/context.ts @@ -231,7 +231,7 @@ function relativeMockPath(testFile: string, mockFile: string): string { // ─── Type definition collector ──────────────────────────────────────────────── -async function readTsconfigAliases(cwd: string): Promise> { +export async function readTsconfigAliases(cwd: string): Promise> { const candidates = ['tsconfig.json', 'tsconfig.app.json', 'tsconfig.base.json'] for (const name of candidates) { try { @@ -246,7 +246,7 @@ async function readTsconfigAliases(cwd: string): Promise void +} + +export interface E2ELoopResult { + flowsDiscovered: number + specsGenerated: number + specsFailed: number + skipped: number + errors: string[] +} + +const E2E_TIPS = [ + 'Specs target role/label locators (getByRole, getByLabel) from the captured page, not CSS.', + 'Use --route /path to (re)generate a single route; existing specs are skipped.', + 'Each spec is rerun to confirm it is not flaky before being kept.', + 'lacuna fix --e2e repairs failing Playwright specs the same way it fixes unit tests.', +] + +const FLAKE_CONFIRM_RUNS = 1 // extra green runs required after the first pass before we accept +const PER_RUN_TIMEOUT_MS = 120_000 + +const emptyResult = (): E2ELoopResult => ({ flowsDiscovered: 0, specsGenerated: 0, specsFailed: 0, skipped: 0, errors: [] }) + +export async function runE2ELoop(options: E2ELoopOptions): Promise { + const { config, cwd, dryRun, verbose, log } = options + + if (!(await detectPlaywright(cwd))) { + log(chalk.yellow('\n E2E generation needs @playwright/test in the project, but none was found.')) + return emptyResult() + } + + const pwConfig = await loadPlaywrightConfig(cwd) + const discovery = await discoverFlows(cwd, config.sourceDir) + + if (discovery.framework === 'unknown' || discovery.flows.length === 0) { + log(chalk.yellow('\n No routes discovered. E2E generation currently supports Next.js (app or pages router) and React Router.')) + return emptyResult() + } + log(chalk.dim(` Detected ${discovery.framework} with ${discovery.flows.length} route(s) under ${discovery.routeRoot}.`)) + + // Resolve which flows to process: a single --route, else all (capped), minus routes that + // already have a spec on disk (we don't regenerate over existing work). + const testDirAbs = join(cwd, pwConfig.testDir) + let flows = discovery.flows + if (options.targetRoute) { + flows = flows.filter((f) => f.route === options.targetRoute) + if (flows.length === 0) { + log(chalk.yellow(`\n Route ${options.targetRoute} not found among discovered routes.`)) + return emptyResult() + } + } + + const result = emptyResult() + result.flowsDiscovered = discovery.flows.length + + const pending: Flow[] = [] + for (const flow of flows) { + if (await specExists(testDirAbs, flow)) { + result.skipped++ + log(chalk.dim(` Skipping ${flow.route} — spec already exists.`)) + } else { + pending.push(flow) + } + } + + // No implicit ceiling — by default every discovered route is processed (like unit generate); + // re-runs are cheap because existing specs are skipped. --max-routes is an opt-in limiter for + // capping an expensive run on a large app. + if (options.maxRoutes != null && pending.length > options.maxRoutes) { + log(chalk.dim(` Limiting this run to ${options.maxRoutes} of ${pending.length} routes (--max-routes); re-run to do the rest.`)) + pending.length = options.maxRoutes + } + + if (pending.length === 0) { + log(chalk.green('\n Every discovered route already has a spec — nothing to generate.')) + return result + } + + // Dry-run is a preview: report which specs would be generated, without booting the app, + // snapshotting, or constructing the model provider (so it needs no API key). + if (dryRun) { + log(chalk.yellow(`\n [dry-run] would generate ${pending.length} spec(s):`)) + for (const flow of pending) { + log(chalk.dim(` ${flow.route} → ${join(pwConfig.testDir, specFileName(flow.route))}`)) + } + return result + } + + // Bring the app up once so parallel spec runs attach to one server instead of racing to bind + // the port. When we can't confirm a running server, fall back to sequential — each playwright + // invocation then safely manages its own webServer. + const server = await ensureAppServer(pwConfig, cwd, config.coverageTimeout * 1000) + let workerCount = Math.max(1, Math.min(options.workers ?? 1, 8)) + if (workerCount > 1 && !(server.managed || server.alreadyRunning)) { + log(chalk.yellow(` Running sequentially (parallel workers need one shared server): ${server.error ?? 'app server not confirmed up'}`)) + workerCount = 1 + } + + let display: WorkerDisplay | null = null + try { + // One browser run captures every pending route's DOM (reusing the server we just started). + log(chalk.dim(`\n Capturing the DOM for ${pending.length} route(s)...`)) + const snap = await snapshotRoutes(pending.map((f) => f.route), cwd, pwConfig, config.coverageTimeout * 1000) + if (!snap.ok) { + log(chalk.yellow(`\n Could not snapshot the app: ${snap.error}`)) + if (verbose && snap.rawOutput) log(chalk.dim(snap.rawOutput)) + log(chalk.dim(' Specs will be generated as conservative smoke tests without a DOM snapshot.')) + } + const snapshotByRoute = new Map() + for (const s of snap.snapshots) snapshotByRoute.set(s.route, s) + + const pwEnv = envForRunner('playwright') + + // Opt-in pre-pass: add data-testid attributes to page sources so specs can use stable + // getByTestId locators. Serial (it writes source and re-snapshots per route). Each injection + // is verified against a fresh snapshot and reverted if the testid never reached the DOM — so + // a component that doesn't forward props leaves the source unchanged. + if (options.injectTestIds) { + log(chalk.dim('\n --inject-testids: adding data-testid attributes to page sources...')) + const injector = new TestGenerator({ config, env: pwEnv }) + const depNames = await readProjectDepNames(cwd) // for barrel-proof UI-library detection + let added = 0 + for (const flow of pending) { + const enriched = await injectTestIdsForRoute(flow, snapshotByRoute.get(flow.route) ?? null, injector, cwd, pwConfig, depNames, options) + if (enriched) { snapshotByRoute.set(flow.route, enriched.snapshot); added += enriched.added } + } + log(added > 0 + ? chalk.green(` Injected ${added} data-testid attribute(s); reverted anything that did not reach the DOM.`) + : chalk.dim(' No testids were added (already covered, or components do not forward props).')) + } + + const exampleSpec = await findExampleSpec(testDirAbs) + const systemPrompt = buildE2ESystemPrompt() + + // In parallel mode, drive the same live worker panel the unit commands use (per-worker rows + // + progress bar). Sequential mode keeps the plain streamed logs. The display routes its own + // non-TTY/CI fallback to plain `[wN]` lines, so we suppress our per-route log() calls + // whenever a display is active to avoid double output. + display = workerCount > 1 + ? new WorkerDisplay(workerCount, pending.length, E2E_TIPS, 'generated') + : null + if (display) { log(''); display.start() } + + // Shared work queue: each worker owns its own TestGenerator (history is per-instance and + // must not be shared) and pulls the next route until the queue drains. + let nextIndex = 0 + const disp = display // const capture so the closure narrows past the null check + const runWorker = async (workerId: number): Promise => { + const generator = new TestGenerator({ config, env: pwEnv }) + const onStatus = disp ? (state: WorkerState) => disp.update(workerId, state) : undefined + while (true) { + const i = nextIndex++ + if (i >= pending.length) { onStatus?.({ phase: 'idle' }); return } + const flow = pending[i] + const specPath = join(testDirAbs, specFileName(flow.route)) + const relSpec = specPath.replace(cwd + '/', '') + const specName = relSpec.split('/').pop() ?? specFileName(flow.route) + if (!display) log(chalk.bold(`\n Generating: ${chalk.cyan(flow.route)} ${chalk.dim('→ ' + relSpec)}`)) + + const pageSource = await readFile(join(cwd, flow.sourceFile), 'utf-8').catch(() => null) + const userPrompt = buildE2EGeneratePrompt({ + route: flow.route, + specFilePath: relSpec, + baseURL: pwConfig.baseURL, + snapshot: snapshotByRoute.get(flow.route) ?? null, + pageSource, + dynamic: flow.dynamic, + existingSpecExample: exampleSpec, + }) + + const outcome = await generateAndVerifySpec(flow, specPath, specName, userPrompt, systemPrompt, generator, options, onStatus) + if (outcome.success) { + result.specsGenerated++ + onStatus?.({ phase: 'passed', file: relSpec }) + if (!display) log(chalk.green(` ${flow.route}: spec passes and is stable.`)) + } else { + result.specsFailed++ + if (outcome.error) result.errors.push(`${flow.route}: ${outcome.error}`) + onStatus?.({ phase: 'failed', file: relSpec }) + if (!display) log(chalk.red(` ${flow.route}: could not produce a passing spec.`)) + } + } + } + + await Promise.all(Array.from({ length: Math.min(workerCount, pending.length) }, (_, wi) => runWorker(wi))) + } finally { + display?.finish() + server.stop() + } + + return result +} + +// Generate → write → run → (on pass) flake-confirm, retrying on failure up to maxIterations. +// Leaves the spec on disk only if it ends green; otherwise removes it so we never commit a +// broken spec. +async function generateAndVerifySpec( + flow: Flow, + specPath: string, + debugName: string, // spec filename (e.g. login.spec.ts) — drives the per-file debug log slug + userPrompt: string, + systemPrompt: string, + generator: TestGenerator, + options: E2ELoopOptions, + onStatus?: (state: WorkerState) => void, // when set (parallel mode), drive the live worker panel instead of logging +): Promise<{ success: boolean; error?: string }> { + const { cwd, verbose, log, config } = options + const file = specPath.replace(cwd + '/', '') + let failureMsg = '' + + for (let attempt = 1; attempt <= config.maxIterations; attempt++) { + onStatus?.(attempt === 1 ? { phase: 'generating', file } : { phase: 'retrying', file, attempt, max: config.maxIterations }) + let code: string + try { + code = attempt === 1 + ? await generator.generateE2E(systemPrompt, userPrompt, debugName) + : await generator.retry(failureMsg) + } catch (err) { + if (err instanceof OscillationError) { return { success: false, error: 'Model looped on the same spec.' } } + if (err instanceof TruncatedOutputError) { failureMsg = 'Your spec was cut off before completion. Write a shorter, complete spec.'; continue } + return { success: false, error: err instanceof Error ? err.message : String(err) } + } + + if (!hasTestFunctions(code)) { + failureMsg = 'The spec contains no test() blocks. Write at least one test(...) with real actions and web-first assertions.' + if (!onStatus && verbose) log(chalk.yellow(' No test() found — retrying...')) + continue + } + + onStatus?.({ phase: 'writing', file }) + await mkdir(join(specPath, '..'), { recursive: true }).catch(() => {}) + await writeFile(specPath, code, 'utf-8') + + onStatus?.({ phase: 'running', file }) + const run = await runPlaywrightSpec(specPath, cwd) + if (!run.pass) { + failureMsg = run.failure + if (!onStatus) { + if (verbose) log(chalk.dim(failureMsg.split('\n').slice(0, 12).join('\n'))) + log(chalk.red(` Spec failed (attempt ${attempt}/${config.maxIterations}).`)) + } + continue + } + + // Passed once. Confirm it's not flaky before accepting. + let flaky = false + for (let c = 0; c < FLAKE_CONFIRM_RUNS; c++) { + const confirm = await runPlaywrightSpec(specPath, cwd) + if (!confirm.pass) { flaky = true; failureMsg = `The spec passed once then FAILED on rerun — it is FLAKY:\n${confirm.failure}\n\nRemove any race conditions and arbitrary waits; rely only on web-first auto-waiting assertions.`; break } + } + if (flaky) { + if (!onStatus) log(chalk.yellow(` Spec was flaky (attempt ${attempt}/${config.maxIterations}) — retrying for stability.`)) + continue + } + + return { success: true } + } + + // Exhausted: don't leave a broken/flaky spec behind. + await rm(specPath, { force: true }).catch(() => {}) + return { success: false, error: `No stable spec after ${config.maxIterations} attempts. Last failure:\n${failureMsg.slice(0, 800)}` } +} + +// Add data-testid attributes to one route's page source, then VERIFY they reached the DOM by +// re-snapshotting; revert the source if they didn't (component didn't forward props, or the edit +// broke render). Returns the enriched snapshot + count of testids that took, or null (no change). +async function injectTestIdsForRoute( + flow: Flow, + snapshot: RouteSnapshot | null, + generator: TestGenerator, + cwd: string, + pwConfig: PlaywrightConfig, + depNames: string[], + options: E2ELoopOptions, +): Promise<{ snapshot: RouteSnapshot; added: number } | null> { + const { log, verbose } = options + if (!snapshot || !snapshot.ok) return null + + // Map only named interactive elements; if everything interactive already has a testid, skip. + const interactives = snapshot.interactives.filter((e) => e.name) + if (interactives.length === 0 || snapshot.testIds.length >= interactives.length) return null + + const sourceAbs = join(cwd, flow.sourceFile) + const original = await readFile(sourceAbs, 'utf-8').catch(() => null) + if (!original) return null + + // Detect the UI library so the prompt can give its documented testid-forwarding convention (e.g. + // MUI inputProps). Follows the import chain (including barrel re-exports) so a custom component + // imported through a barrel is NOT mistaken for a library one; only falls back to installed deps + // when the chain genuinely can't be resolved. + const libs = await resolveComponentLibraries(original, sourceAbs, cwd, depNames) + const libraryGuidance = buildLibraryTestIdGuidance(libs) + if (libraryGuidance && verbose) log(chalk.dim(` ${flow.route}: detected ${libs.map((l) => l.name).join(', ')}.`)) + + let modified: string + try { + modified = await generator.injectTestIds( + buildTestIdInjectionSystemPrompt(), + buildTestIdInjectionPrompt({ + sourceFile: flow.sourceFile, + sourceCode: original, + interactives, + existingTestIds: snapshot.testIds.map((t) => t.testId), + libraryGuidance, + }), + 'inject-' + (flow.sourceFile.split('/').pop() ?? 'page'), + ) + } catch { return null } + + // Guard against truncated/garbage rewrites before touching source. + if (!isPlausibleTestIdEdit(original, modified)) { + if (verbose) log(chalk.dim(` ${flow.route}: testid edit looked implausible — skipped.`)) + return null + } + + const beforeIds = new Set(snapshot.testIds.map((t) => t.testId)) + await writeFile(sourceAbs, modified, 'utf-8') + + // Re-snapshot the single route. A testid only counts if it (a) is new and (b) landed on an + // actual INTERACTIVE element — a native control or an interactive ARIA role. This empirically + // enforces "testids go on controls, not wrappers": if library guidance was wrong for a custom + // component (e.g. a barrel-exported