diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 95a4b263b0..f4d3567a4f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -265,6 +265,13 @@ jobs: - name: Merge into HTML report + summary working-directory: e2e + env: + # Used by reporters/summary.ts to embed the exact local repro command + # in the agent-friendly failure report (agent-report.md). + SANITY_E2E_PROJECT_ID: ${{ vars.SANITY_E2E_PROJECT_ID }} + SANITY_E2E_BASE_URL: ${{ needs.deploy-preview.outputs.preview_url }} + SANITY_E2E_DATASET_CHROMIUM: ${{ env.CHROMIUM_DATASET }} + SANITY_E2E_DATASET_FIREFOX: ${{ env.FIREFOX_DATASET }} run: | BLOB_DIR=../all-blob-reports if [ -d ../all-blob-reports/blob-report ]; then @@ -273,7 +280,7 @@ jobs: if [ ! -d "$BLOB_DIR" ] || [ -z "$(ls -A "$BLOB_DIR" 2>/dev/null)" ]; then echo "No blob reports found to merge" mkdir -p playwright-report - echo '{"passed":0,"failed":0,"flaky":0,"skipped":0,"failedFiles":[],"failedFilesFormatted":""}' > test-summary.json + echo '{"passed":0,"failed":0,"flaky":0,"skipped":0,"hasFailures":false,"failedFiles":[],"failedFilesFormatted":""}' > test-summary.json exit 0 fi pnpm exec playwright merge-reports --reporter html,./reporters/summary.ts "$BLOB_DIR" @@ -303,6 +310,14 @@ jobs: } " >> "$GITHUB_OUTPUT" + - name: Append failure digest to job summary + if: ${{ !cancelled() && steps.summary.outputs.failed != '0' && steps.summary.outputs.failed != '' }} + run: | + if [ -f e2e/playwright-report/agent-report.md ]; then + # GITHUB_STEP_SUMMARY is limited to 1MB; keep a safety margin. + head -c 900000 e2e/playwright-report/agent-report.md >> "$GITHUB_STEP_SUMMARY" + fi + - name: Upload merged Playwright report if: ${{ !cancelled() }} uses: actions/upload-artifact@v7 @@ -381,6 +396,16 @@ jobs: if (s.skipped) parts.push('⚪ ' + s.skipped + ' skipped'); console.log('text=' + (parts.join(' • ') || '⚪ no tests')); } + if (u && s.failed) { + const block = [ + '🤖 **Share with an AI agent:**', + '', + '> Fix the failing e2e tests. The failure report with error messages and page snapshots is at ' + u + '/agent-report.md', + ].join('\n'); + process.stdout.write('agent_block</agent-report.md`**: a plain-markdown digest generated by [`reporters/summary.ts`](./reporters/summary.ts) during `merge-reports`, containing per failed/flaky test: + +- the full error message, stack, and failing code snippet (ANSI-stripped) +- Playwright's `error-context.md` attachment — an ARIA/YAML snapshot of the page at the moment of failure, designed for AI consumption +- the exact command (env vars included) to re-run the failed specs locally + +When tests fail, the PR comment includes a **Share with an AI agent** blockquote pointing at that URL — paste it into an agent chat to hand over everything needed to debug the run in a single request. The same digest is appended to the workflow run's job summary, and shipped in the `playwright-report-` GitHub artifact. + ### One-time setup 1. **Create a Vercel project** shared by the studio preview and report deploys (suggested name: `plugins-e2e-test-studio`), configured as described under _Vercel studio preview_. @@ -142,7 +152,7 @@ Helpers: Locally, Playwright starts `pnpm --filter e2e-studio dev` unless a server is already on port 3333 (set `SANITY_E2E_BASE_URL` to a deployed studio to skip the local server). In CI, tests run against the per-run Vercel preview deployment. -On pull requests, CI posts an **E2E Tests** status comment with pass/fail/flaky/skipped counts, a hosted HTML report URL, dataset names, and a link to the workflow run. +On pull requests, CI posts an **E2E Tests** status comment with pass/fail/flaky/skipped counts, a hosted HTML report URL, dataset names, and a link to the workflow run. On failure it also links the agent-friendly `agent-report.md` (see _Agent-friendly failure report_). ## Troubleshooting auth diff --git a/e2e/reporters/summary.ts b/e2e/reporters/summary.ts index b1080f5827..685270d34b 100644 --- a/e2e/reporters/summary.ts +++ b/e2e/reporters/summary.ts @@ -1,13 +1,107 @@ import fs from 'node:fs' import path from 'node:path' -import {type FullResult, type Reporter, type TestCase} from '@playwright/test/reporter' +import { + type FullResult, + type Reporter, + type TestCase, + type TestError, + type TestResult, +} from '@playwright/test/reporter' + +// Matches ANSI CSI color/style sequences that Playwright embeds in error output. +const ANSI_PATTERN = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, 'g') + +function stripAnsi(text: string): string { + return text.replace(ANSI_PATTERN, '') +} + +function formatError(error: TestError): string { + const parts: string[] = [] + // `stack` includes the message; fall back to `message` when there is no stack. + const body = error.stack || error.message + if (body) parts.push(stripAnsi(body)) + if (error.snippet) parts.push(stripAnsi(error.snippet)) + return parts.join('\n\n') +} /** - * Custom Playwright reporter that outputs a JSON summary of test results. + * Playwright attaches `error-context.md` (an ARIA/YAML snapshot of the page at + * the moment of failure, intended for AI agents) to every failed test. + */ +function readErrorContext(result: TestResult): string | undefined { + const attachment = result.attachments.find((a) => a.name === 'error-context') + if (!attachment) return undefined + if (attachment.body) return attachment.body.toString('utf8') + if (attachment.path && fs.existsSync(attachment.path)) { + return fs.readFileSync(attachment.path, 'utf8') + } + return undefined +} + +function formatFailedTest(test: TestCase, relativeFile: string): string { + const lines: string[] = [] + const title = test.titlePath().filter(Boolean).join(' › ') + const outcome = test.outcome() + const icon = outcome === 'flaky' ? '⚠️' : '❌' + lines.push(`## ${icon} ${title}`) + lines.push('') + lines.push(`- Location: \`${relativeFile}:${test.location.line}:${test.location.column}\``) + const projectName = test.parent.project()?.name + if (projectName) lines.push(`- Browser (Playwright project): ${projectName}`) + if (outcome === 'flaky') { + lines.push(`- Status: flaky — failed, then passed on retry (${test.results.length} attempts)`) + } else { + lines.push( + `- Status: failed (${test.results.length} attempt${test.results.length === 1 ? '' : 's'}, all failed)`, + ) + } + lines.push('') + + const lastFailed = [...test.results].reverse().find((r) => r.errors.length > 0) + if (lastFailed) { + const attemptLabel = + test.results.length > 1 ? ` (attempt ${lastFailed.retry + 1} of ${test.results.length})` : '' + lines.push(`### Error${attemptLabel}`) + lines.push('') + lines.push('```') + lines.push(lastFailed.errors.map(formatError).join('\n\n')) + lines.push('```') + lines.push('') + + const errorContext = readErrorContext(lastFailed) + if (errorContext) { + lines.push('### Page snapshot at the moment of failure') + lines.push('') + lines.push(errorContext.trim()) + lines.push('') + } + + const otherAttachments = lastFailed.attachments.filter( + (a) => a.name !== 'error-context' && !a.name.startsWith('_'), + ) + if (otherAttachments.length > 0) { + lines.push('### Other attachments (available in the HTML report at this deployment root)') + lines.push('') + for (const attachment of otherAttachments) { + lines.push(`- ${attachment.name} (${attachment.contentType})`) + } + lines.push('') + } + } + + return lines.join('\n') +} + +/** + * Custom Playwright reporter that outputs machine-readable test results. * - * Writes `test-summary.json` with counts and failed file paths. - * Used by the CI workflow to build PR comments with test summaries. + * Writes two files (used by the CI `report` job): + * - `test-summary.json` — counts and failed file paths, used to build the PR comment. + * - `playwright-report/agent-report.md` — a plain-markdown failure digest (error + * messages, code snippets, and Playwright's error-context page snapshots) meant + * to be fetched by AI agents in a single request. It is deployed alongside the + * HTML report, at `/agent-report.md`. */ export default class SummaryReporter implements Reporter { private tests: TestCase[] = [] @@ -20,6 +114,8 @@ export default class SummaryReporter implements Reporter { const cwd = process.cwd() const counts = {passed: 0, failed: 0, flaky: 0, skipped: 0} const failedFileSet = new Set() + const failedTests: TestCase[] = [] + const flakyTests: TestCase[] = [] // Deduplicate by test ID — onTestEnd is called per attempt (including retries), // so we only want the last attempt for each test. @@ -35,10 +131,12 @@ export default class SummaryReporter implements Reporter { break case 'unexpected': counts.failed++ + failedTests.push(test) if (test.location.file) failedFileSet.add(test.location.file) break case 'flaky': counts.flaky++ + flakyTests.push(test) break case 'skipped': counts.skipped++ @@ -49,10 +147,76 @@ export default class SummaryReporter implements Reporter { const failedFiles = [...failedFileSet].map((f) => path.relative(cwd, f)) const summary = { ...counts, + hasFailures: counts.failed > 0, failedFiles, // Pre-formatted for use in shell code blocks: each file on its own line with \ failedFilesFormatted: failedFiles.join(' \\\n '), } fs.writeFileSync('test-summary.json', JSON.stringify(summary)) + + this.writeAgentReport(cwd, counts, failedTests, flakyTests, failedFiles) + } + + private writeAgentReport( + cwd: string, + counts: {passed: number; failed: number; flaky: number; skipped: number}, + failedTests: TestCase[], + flakyTests: TestCase[], + failedFiles: string[], + ) { + const lines: string[] = [] + lines.push('# E2E test report (agent-friendly)') + lines.push('') + lines.push( + 'Plain-markdown digest of the Playwright e2e run, generated for AI agents. The interactive HTML report (screenshots, videos, traces) is served from this same deployment root.', + ) + lines.push('') + + const runUrl = + process.env.GITHUB_SERVER_URL && process.env.GITHUB_REPOSITORY && process.env.GITHUB_RUN_ID + ? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}` + : undefined + if (process.env.GITHUB_SHA) lines.push(`- Commit: ${process.env.GITHUB_SHA}`) + if (runUrl) lines.push(`- Workflow run: ${runUrl}`) + lines.push( + `- Summary: ${counts.passed} passed, ${counts.failed} failed, ${counts.flaky} flaky, ${counts.skipped} skipped`, + ) + lines.push('') + + if (failedTests.length === 0 && flakyTests.length === 0) { + lines.push('All tests passed — no failures to report.') + lines.push('') + } else { + for (const test of [...failedTests, ...flakyTests]) { + const relativeFile = path.relative(cwd, test.location.file) + lines.push(formatFailedTest(test, relativeFile)) + lines.push('') + } + } + + if (failedFiles.length > 0) { + const reproEnv = [ + 'SANITY_E2E_PROJECT_ID', + 'SANITY_E2E_BASE_URL', + 'SANITY_E2E_DATASET_CHROMIUM', + 'SANITY_E2E_DATASET_FIREFOX', + ] + .map((name) => (process.env[name] ? `${name}=${process.env[name]} \\` : undefined)) + .filter((line) => line !== undefined) + lines.push('## How to reproduce locally') + lines.push('') + lines.push('From the repository root:') + lines.push('') + lines.push('```sh') + lines.push(...reproEnv) + lines.push('pnpm test:e2e \\') + lines.push(` ${failedFiles.join(' \\\n ')}`) + lines.push('```') + lines.push('') + } + + const reportDir = path.join(cwd, 'playwright-report') + fs.mkdirSync(reportDir, {recursive: true}) + fs.writeFileSync(path.join(reportDir, 'agent-report.md'), lines.join('\n')) } }