Add agent-friendly e2e failure report hosted next to the HTML report

.github/workflows/e2e.yml

@@ -265,6 +265,13 @@ jobs:
 
       - name: Merge into HTML report + summary
         working-directory: e2e
+        env:
+          # Used by reporters/summary.ts to embed the exact local repro command
+          # in the agent-friendly failure report (agent-report.md).
+          SANITY_E2E_PROJECT_ID: ${{ vars.SANITY_E2E_PROJECT_ID }}
+          SANITY_E2E_BASE_URL: ${{ needs.deploy-preview.outputs.preview_url }}
+          SANITY_E2E_DATASET_CHROMIUM: ${{ env.CHROMIUM_DATASET }}
+          SANITY_E2E_DATASET_FIREFOX: ${{ env.FIREFOX_DATASET }}
         run: |
           BLOB_DIR=../all-blob-reports
           if [ -d ../all-blob-reports/blob-report ]; then
@@ -273,7 +280,7 @@ jobs:
           if [ ! -d "$BLOB_DIR" ] || [ -z "$(ls -A "$BLOB_DIR" 2>/dev/null)" ]; then
             echo "No blob reports found to merge"
             mkdir -p playwright-report
-            echo '{"passed":0,"failed":0,"flaky":0,"skipped":0,"failedFiles":[],"failedFilesFormatted":""}' > test-summary.json
+            echo '{"passed":0,"failed":0,"flaky":0,"skipped":0,"hasFailures":false,"failedFiles":[],"failedFilesFormatted":""}' > test-summary.json
             exit 0
           fi
           pnpm exec playwright merge-reports --reporter html,./reporters/summary.ts "$BLOB_DIR"
@@ -303,6 +310,14 @@ jobs:
             }
           " >> "$GITHUB_OUTPUT"
 
+      - name: Append failure digest to job summary
+        if: ${{ !cancelled() && steps.summary.outputs.failed != '0' && steps.summary.outputs.failed != '' }}
+        run: |
+          if [ -f e2e/playwright-report/agent-report.md ]; then
+            # GITHUB_STEP_SUMMARY is limited to 1MB; keep a safety margin.
+            head -c 900000 e2e/playwright-report/agent-report.md >> "$GITHUB_STEP_SUMMARY"
+          fi
+
       - name: Upload merged Playwright report
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v7
@@ -381,6 +396,16 @@ jobs:
               if (s.skipped) parts.push('⚪ ' + s.skipped + ' skipped');
               console.log('text=' + (parts.join(' • ') || '⚪ no tests'));
             }
+            if (u && s.failed) {
+              const block = [
+                '🤖 **Share with an AI agent:**',
+                '',
+                '> Fix the failing e2e tests. The failure report with error messages and page snapshots is at ' + u + '/agent-report.md',
+              ].join('\n');
+              process.stdout.write('agent_block<<ENDOFAGENTBLOCK\n' + block + '\nENDOFAGENTBLOCK\n');
+            } else {
+              console.log('agent_block=');
+            }
           " >> "$GITHUB_OUTPUT"
 
       - name: Update PR Comment (tests passed)
@@ -407,6 +432,8 @@ jobs:
 
             ${{ steps.linked-summary.outputs.text }} • [view run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
 
+            ${{ steps.linked-summary.outputs.agent_block }}
+
             **Studio:** ${{ needs.deploy-preview.outputs.preview_url }}
 
             Datasets: `${{ env.CHROMIUM_DATASET }}`, `${{ env.FIREFOX_DATASET }}`

e2e/README.md

@@ -107,6 +107,16 @@ The studio preview reuses the same Vercel project and secrets as report hosting
 
 CI deploys the Playwright HTML report to the **same Vercel project** as the studio preview (report deploys are `--prebuilt` static output, so the studio build settings don't affect them). PR comments link counts into the hosted report (`[🟢 N passed](…#?q=s%3Apassed)`, etc.).
 
+### Agent-friendly failure report
+
+The HTML report is a JS single-page app — useful for humans, but an AI agent fetching the URL only gets an empty shell. So the report deployment also hosts **`<report-url>/agent-report.md`**: a plain-markdown digest generated by [`reporters/summary.ts`](./reporters/summary.ts) during `merge-reports`, containing per failed/flaky test:
+
+- the full error message, stack, and failing code snippet (ANSI-stripped)
+- Playwright's `error-context.md` attachment — an ARIA/YAML snapshot of the page at the moment of failure, designed for AI consumption
+- the exact command (env vars included) to re-run the failed specs locally
+
+When tests fail, the PR comment includes a **Share with an AI agent** blockquote pointing at that URL — paste it into an agent chat to hand over everything needed to debug the run in a single request. The same digest is appended to the workflow run's job summary, and shipped in the `playwright-report-<run_id>` GitHub artifact.
+
 ### One-time setup
 
 1. **Create a Vercel project** shared by the studio preview and report deploys (suggested name: `plugins-e2e-test-studio`), configured as described under _Vercel studio preview_.
@@ -142,7 +152,7 @@ Helpers:
 
 Locally, Playwright starts `pnpm --filter e2e-studio dev` unless a server is already on port 3333 (set `SANITY_E2E_BASE_URL` to a deployed studio to skip the local server). In CI, tests run against the per-run Vercel preview deployment.
 
-On pull requests, CI posts an **E2E Tests** status comment with pass/fail/flaky/skipped counts, a hosted HTML report URL, dataset names, and a link to the workflow run.
+On pull requests, CI posts an **E2E Tests** status comment with pass/fail/flaky/skipped counts, a hosted HTML report URL, dataset names, and a link to the workflow run. On failure it also links the agent-friendly `agent-report.md` (see _Agent-friendly failure report_).
 
 ## Troubleshooting auth
 

e2e/reporters/summary.ts

@@ -1,13 +1,107 @@
 import fs from 'node:fs'
 import path from 'node:path'
 
-import {type FullResult, type Reporter, type TestCase} from '@playwright/test/reporter'
+import {
+  type FullResult,
+  type Reporter,
+  type TestCase,
+  type TestError,
+  type TestResult,
+} from '@playwright/test/reporter'
+
+// Matches ANSI CSI color/style sequences that Playwright embeds in error output.
+const ANSI_PATTERN = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, 'g')
+
+function stripAnsi(text: string): string {
+  return text.replace(ANSI_PATTERN, '')
+}
+
+function formatError(error: TestError): string {
+  const parts: string[] = []
+  // `stack` includes the message; fall back to `message` when there is no stack.
+  const body = error.stack || error.message
+  if (body) parts.push(stripAnsi(body))
+  if (error.snippet) parts.push(stripAnsi(error.snippet))
+  return parts.join('\n\n')
+}
 
 /**
- * Custom Playwright reporter that outputs a JSON summary of test results.
+ * Playwright attaches `error-context.md` (an ARIA/YAML snapshot of the page at
+ * the moment of failure, intended for AI agents) to every failed test.
+ */
+function readErrorContext(result: TestResult): string | undefined {
+  const attachment = result.attachments.find((a) => a.name === 'error-context')
+  if (!attachment) return undefined
+  if (attachment.body) return attachment.body.toString('utf8')
+  if (attachment.path && fs.existsSync(attachment.path)) {
+    return fs.readFileSync(attachment.path, 'utf8')
+  }
+  return undefined
+}
+
+function formatFailedTest(test: TestCase, relativeFile: string): string {
+  const lines: string[] = []
+  const title = test.titlePath().filter(Boolean).join(' › ')
+  const outcome = test.outcome()
+  const icon = outcome === 'flaky' ? '⚠️' : '❌'
+  lines.push(`## ${icon} ${title}`)
+  lines.push('')
+  lines.push(`- Location: \`${relativeFile}:${test.location.line}:${test.location.column}\``)
+  const projectName = test.parent.project()?.name
+  if (projectName) lines.push(`- Browser (Playwright project): ${projectName}`)
+  if (outcome === 'flaky') {
+    lines.push(`- Status: flaky — failed, then passed on retry (${test.results.length} attempts)`)
+  } else {
+    lines.push(
+      `- Status: failed (${test.results.length} attempt${test.results.length === 1 ? '' : 's'}, all failed)`,
+    )
+  }
+  lines.push('')
+
+  const lastFailed = [...test.results].reverse().find((r) => r.errors.length > 0)
+  if (lastFailed) {
+    const attemptLabel =
+      test.results.length > 1 ? ` (attempt ${lastFailed.retry + 1} of ${test.results.length})` : ''
+    lines.push(`### Error${attemptLabel}`)
+    lines.push('')
+    lines.push('```')
+    lines.push(lastFailed.errors.map(formatError).join('\n\n'))
+    lines.push('```')
+    lines.push('')
+
+    const errorContext = readErrorContext(lastFailed)
+    if (errorContext) {
+      lines.push('### Page snapshot at the moment of failure')
+      lines.push('')
+      lines.push(errorContext.trim())
+      lines.push('')
+    }
+
+    const otherAttachments = lastFailed.attachments.filter(
+      (a) => a.name !== 'error-context' && !a.name.startsWith('_'),
+    )
+    if (otherAttachments.length > 0) {
+      lines.push('### Other attachments (available in the HTML report at this deployment root)')
+      lines.push('')
+      for (const attachment of otherAttachments) {
+        lines.push(`- ${attachment.name} (${attachment.contentType})`)
+      }
+      lines.push('')
+    }
+  }
+
+  return lines.join('\n')
+}
+
+/**
+ * Custom Playwright reporter that outputs machine-readable test results.
  *
- * Writes `test-summary.json` with counts and failed file paths.
- * Used by the CI workflow to build PR comments with test summaries.
+ * Writes two files (used by the CI `report` job):
+ * - `test-summary.json` — counts and failed file paths, used to build the PR comment.
+ * - `playwright-report/agent-report.md` — a plain-markdown failure digest (error
+ *   messages, code snippets, and Playwright's error-context page snapshots) meant
+ *   to be fetched by AI agents in a single request. It is deployed alongside the
+ *   HTML report, at `<report-url>/agent-report.md`.
  */
 export default class SummaryReporter implements Reporter {
   private tests: TestCase[] = []
@@ -20,6 +114,8 @@ export default class SummaryReporter implements Reporter {
     const cwd = process.cwd()
     const counts = {passed: 0, failed: 0, flaky: 0, skipped: 0}
     const failedFileSet = new Set<string>()
+    const failedTests: TestCase[] = []
+    const flakyTests: TestCase[] = []
 
     // Deduplicate by test ID — onTestEnd is called per attempt (including retries),
     // so we only want the last attempt for each test.
@@ -35,10 +131,12 @@ export default class SummaryReporter implements Reporter {
           break
         case 'unexpected':
           counts.failed++
+          failedTests.push(test)
           if (test.location.file) failedFileSet.add(test.location.file)
           break
         case 'flaky':
           counts.flaky++
+          flakyTests.push(test)
           break
         case 'skipped':
           counts.skipped++
@@ -49,10 +147,76 @@ export default class SummaryReporter implements Reporter {
     const failedFiles = [...failedFileSet].map((f) => path.relative(cwd, f))
     const summary = {
       ...counts,
+      hasFailures: counts.failed > 0,
       failedFiles,
       // Pre-formatted for use in shell code blocks: each file on its own line with \
       failedFilesFormatted: failedFiles.join(' \\\n  '),
     }
     fs.writeFileSync('test-summary.json', JSON.stringify(summary))
+
+    this.writeAgentReport(cwd, counts, failedTests, flakyTests, failedFiles)
+  }
+
+  private writeAgentReport(
+    cwd: string,
+    counts: {passed: number; failed: number; flaky: number; skipped: number},
+    failedTests: TestCase[],
+    flakyTests: TestCase[],
+    failedFiles: string[],
+  ) {
+    const lines: string[] = []
+    lines.push('# E2E test report (agent-friendly)')
+    lines.push('')
+    lines.push(
+      'Plain-markdown digest of the Playwright e2e run, generated for AI agents. The interactive HTML report (screenshots, videos, traces) is served from this same deployment root.',
+    )
+    lines.push('')
+
+    const runUrl =
+      process.env.GITHUB_SERVER_URL && process.env.GITHUB_REPOSITORY && process.env.GITHUB_RUN_ID
+        ? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
+        : undefined
+    if (process.env.GITHUB_SHA) lines.push(`- Commit: ${process.env.GITHUB_SHA}`)
+    if (runUrl) lines.push(`- Workflow run: ${runUrl}`)
+    lines.push(
+      `- Summary: ${counts.passed} passed, ${counts.failed} failed, ${counts.flaky} flaky, ${counts.skipped} skipped`,
+    )
+    lines.push('')
+
+    if (failedTests.length === 0 && flakyTests.length === 0) {
+      lines.push('All tests passed — no failures to report.')
+      lines.push('')
+    } else {
+      for (const test of [...failedTests, ...flakyTests]) {
+        const relativeFile = path.relative(cwd, test.location.file)
+        lines.push(formatFailedTest(test, relativeFile))
+        lines.push('')
+      }
+    }
+
+    if (failedFiles.length > 0) {
+      const reproEnv = [
+        'SANITY_E2E_PROJECT_ID',
+        'SANITY_E2E_BASE_URL',
+        'SANITY_E2E_DATASET_CHROMIUM',
+        'SANITY_E2E_DATASET_FIREFOX',
+      ]
+        .map((name) => (process.env[name] ? `${name}=${process.env[name]} \\` : undefined))
+        .filter((line) => line !== undefined)
+      lines.push('## How to reproduce locally')
+      lines.push('')
+      lines.push('From the repository root:')
+      lines.push('')
+      lines.push('```sh')
+      lines.push(...reproEnv)
+      lines.push('pnpm test:e2e \\')
+      lines.push(`  ${failedFiles.join(' \\\n  ')}`)
+      lines.push('```')
+      lines.push('')
+    }
+
+    const reportDir = path.join(cwd, 'playwright-report')
+    fs.mkdirSync(reportDir, {recursive: true})
+    fs.writeFileSync(path.join(reportDir, 'agent-report.md'), lines.join('\n'))
   }
 }