From 43caa916c34545c533697328c8f6365bed427020 Mon Sep 17 00:00:00 2001 From: Alexander Bychinskiy Date: Fri, 3 Jul 2026 14:52:32 +0300 Subject: [PATCH 1/2] chore(ci): surface pass/fail + failure reasons on the Actions run page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Playwright's built-in `github` reporter (CI-only, additive alongside the existing html/json reporters) for inline ::error:: annotations on the Checks tab, plus a step that parses test-results/json/run.json into a markdown pass/fail table + per-failure reason on $GITHUB_STEP_SUMMARY — so a run's outcome and its "why" are visible without downloading the HTML report artifact. --- .github/scripts/summarize-results.js | 156 +++++++++++++++++++++++++++ .github/workflows/playwright.yml | 14 +++ playwright.config.ts | 8 ++ 3 files changed, 178 insertions(+) create mode 100644 .github/scripts/summarize-results.js diff --git a/.github/scripts/summarize-results.js b/.github/scripts/summarize-results.js new file mode 100644 index 0000000..9cfd71e --- /dev/null +++ b/.github/scripts/summarize-results.js @@ -0,0 +1,156 @@ +#!/usr/bin/env node +// Reads the Playwright `json` reporter's output (playwright.config.ts's +// existing ['json', { outputFile: 'test-results/json/run.json' }] entry -- +// this script does not add or change that reporter) and appends a concise +// markdown pass/fail table + per-failure reasons to $GITHUB_STEP_SUMMARY. +// +// Run with `if: always()` in the workflow so it still produces a summary +// when the test step itself failed. Never throws on a missing/malformed +// report -- worst case it writes a "no report found" note, since a summary +// script crash must not mask the real test outcome that already ran. +// +// JSON reporter schema confirmed against the installed @playwright/test +// version (see node_modules/playwright/lib/runner/index.js, +// createJSONReport()/_serializeSuite()/_serializeTest()): +// report.stats -> { startTime, duration, expected, skipped, unexpected, flaky } +// report.suites[] -> { title, file, line, column, specs[], suites[] } (specs/suites both optional, recurse) +// suite.specs[].tests[] -> { status: 'expected'|'unexpected'|'flaky'|'skipped', results[] } +// test.results[] -> { status, error, errors[], ... } (last entry is the final attempt) +// +// stats key -> report meaning (Test.outcome()): +// expected = passed as expected -> "Passed" column +// unexpected = failed (all retries exhausted) -> "Failed" column +// flaky = failed then passed on retry -> "Flaky" column +// skipped = test.skip()/fixme -> "Skipped" column + +const fs = require('fs'); + +const REPORT_PATH = process.argv[2] || 'test-results/json/run.json'; +const summaryPath = process.env.GITHUB_STEP_SUMMARY; + +function readReport(reportPath) { + if (!fs.existsSync(reportPath)) return null; + try { + return JSON.parse(fs.readFileSync(reportPath, 'utf-8')); + } catch (err) { + console.error(`summarize-results: failed to parse ${reportPath}: ${err.message}`); + return null; + } +} + +function formatDuration(ms) { + if (!Number.isFinite(ms)) return 'n/a'; + const totalSeconds = ms / 1000; + if (totalSeconds < 60) return `${totalSeconds.toFixed(1)}s`; + const minutes = Math.floor(totalSeconds / 60); + const seconds = (totalSeconds % 60).toFixed(1); + return `${minutes}m ${seconds}s`; +} + +function escapeCell(value) { + return String(value).replace(/\|/g, '\\|').replace(/\r?\n/g, ' '); +} + +// Playwright error messages carry ANSI color codes (verified against a real +// run.json produced locally -- e.g. "expect(..."). Strip +// them so the step summary renders clean text instead of escape garbage. +function stripAnsi(value) { + // eslint-disable-next-line no-control-regex + return String(value).replace(/\x1b\[[0-9;]*m/g, ''); +} + +// Recursively walk suites -> specs -> tests, collecting every test whose +// final outcome is 'unexpected' (a real failure, not flaky-then-passed). +// `depth` tracks nesting so the top-level file suite (title === filename, +// confirmed via a real run.json) is excluded from the breadcrumb -- it only +// duplicates the Location column. Nested test.describe() titles (e.g. +// '@smoke') are kept. +function collectFailures(suites, breadcrumb = [], depth = 0) { + const failures = []; + for (const suite of suites || []) { + const nextBreadcrumb = depth === 0 || !suite.title ? breadcrumb : [...breadcrumb, suite.title]; + for (const spec of suite.specs || []) { + for (const test of spec.tests || []) { + if (test.status !== 'unexpected') continue; + const results = test.results || []; + const lastResult = results[results.length - 1]; + const rawMessage = + lastResult?.error?.message || + lastResult?.errors?.[0]?.message || + 'No error message captured (see HTML/JSON report artifact for full trace).'; + const firstLine = stripAnsi(rawMessage).split('\n')[0].slice(0, 200); + failures.push({ + title: [...nextBreadcrumb, spec.title].filter(Boolean).join(' > '), + location: `${spec.file}:${spec.line}`, + reason: firstLine, + }); + } + } + failures.push(...collectFailures(suite.suites, nextBreadcrumb, depth + 1)); + } + return failures; +} + +function buildMarkdown(report) { + if (!report) { + return [ + '## Playwright Run Summary', + '', + `_No report found at \`${REPORT_PATH}\` -- the test step likely crashed before writing any results. Check the job log above._`, + '', + ].join('\n'); + } + + const stats = report.stats || {}; + const passed = stats.expected ?? 0; + const failed = stats.unexpected ?? 0; + const flaky = stats.flaky ?? 0; + const skipped = stats.skipped ?? 0; + const total = passed + failed + flaky + skipped; + const duration = formatDuration(stats.duration); + + const lines = []; + lines.push('## Playwright Run Summary'); + lines.push(''); + lines.push('| Total | Passed | Failed | Flaky | Skipped | Duration |'); + lines.push('|---|---|---|---|---|---|'); + lines.push(`| ${total} | ${passed} | ${failed} | ${flaky} | ${skipped} | ${duration} |`); + lines.push(''); + + const failures = collectFailures(report.suites); + + if (failed === 0) { + lines.push('_No failures._'); + lines.push(''); + } else if (failures.length === 0) { + // Should not happen given `failed` came from the same report, but never + // let a shape mismatch hide the fact that failures exist. + lines.push(`_${failed} failing test(s) reported, but detail could not be extracted -- see the HTML/JSON report artifacts._`); + lines.push(''); + } else { + lines.push('### Failed tests'); + lines.push(''); + lines.push('| Test | Location | Reason |'); + lines.push('|---|---|---|'); + for (const f of failures) { + lines.push(`| ${escapeCell(f.title)} | \`${escapeCell(f.location)}\` | ${escapeCell(f.reason)} |`); + } + lines.push(''); + } + + lines.push('Full traces/screenshots: `playwright-html-report` and `playwright-json-report` build artifacts on this run.'); + lines.push(''); + + return lines.join('\n'); +} + +const report = readReport(REPORT_PATH); +const markdown = buildMarkdown(report); + +if (!summaryPath) { + // No GITHUB_STEP_SUMMARY (e.g. running this script locally) -- print + // instead of failing, so this stays safe to invoke outside CI too. + console.log(markdown); +} else { + fs.appendFileSync(summaryPath, markdown + '\n'); +} diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 66fbd3e..b70ab48 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -51,6 +51,20 @@ jobs: ELITEA_EMAIL: ${{ secrets.ELITEA_EMAIL }} ELITEA_PASSWORD: ${{ secrets.ELITEA_PASSWORD }} + - name: Write step summary (pass/fail + failure reasons) + # if: always() so the summary still renders on a red run — that's the + # whole point (reviewing WHY something failed without downloading the + # HTML report artifact). Reads the existing json reporter's output + # (test-results/json/run.json, unchanged by this step) and appends a + # markdown table to $GITHUB_STEP_SUMMARY, GitHub's standard mechanism + # for run-summary content. Complements, does not replace, the inline + # ::error:: annotations from the `github` reporter added in + # playwright.config.ts (CI-only) — annotations point at file:line on + # the Checks tab, this table gives an at-a-glance pass/fail count + + # reason on the run's Summary page. + if: always() + run: node .github/scripts/summarize-results.js test-results/json/run.json + - name: Upload HTML report # if: always() so a failing/red run (including the expected # known-defect reds above) still produces a downloadable report. diff --git a/playwright.config.ts b/playwright.config.ts index 6eed221..7afbc5e 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -24,6 +24,14 @@ export default defineConfig({ reporter: [ ['html', { outputFolder: 'test-results/reports' }], ['json', { outputFile: 'test-results/json/run.json' }], + // CI-only, additive: Playwright's built-in GitHub Actions reporter emits + // `::error::` workflow commands (file:line + message) for every failure, + // which GitHub renders as inline annotations on the Checks tab / Files + // view. Kept out of local runs -- it's noisy/irrelevant outside Actions. + // Confirmed against the installed @playwright/test 1.61.1 type defs + // (node_modules/playwright/types/test.d.ts): `['github']` is the exact, + // no-options tuple form of `ReporterDescription`. + ...(process.env.CI ? [['github'] as const] : []), ], use: { baseURL: env.BASE_URL, From 1be6238346352de3647ed577318556a463dd02dd Mon Sep 17 00:00:00 2001 From: Alexander Bychinskiy Date: Fri, 3 Jul 2026 17:58:35 +0300 Subject: [PATCH 2/2] chore(ci): add --disable-dev-shm-usage to Chromium launch args Operator-requested defense-in-depth for the infra-crash bucket seen on run 28658801255 ("Target page, context or browser has been closed" + empty .fill() reads). Verified this flag is already Playwright's own unconditional chromium default (playwright-core's chromiumSwitches), so this is a harmless duplicate rather than a fix for something that was missing -- noted inline. Real root cause of the infra-crash bucket, if it persists on the next run, is still open. --- playwright.config.ts | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/playwright.config.ts b/playwright.config.ts index 7afbc5e..343b5e5 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -45,6 +45,31 @@ export default defineConfig({ // column count) -- keeping automation on the same resolution avoids // introducing a new, unexplored viewport-dependent variable. viewport: { width: 1920, height: 1080 }, + // Diagnosed from run 28658801255 (this PR's own first two CI runs): a + // distinct infra-instability bucket -- repeated "Target page, context or + // browser has been closed" crashes and fields reading back empty right + // after .fill() -- separate from the correctly-red known-defect tests + // (GH#29, GH#72, ...). Consistent with Chromium exhausting /dev/shm on + // GitHub's ubuntu-latest runners (default 64MB shared-memory mount, + // small next to a 1920x1080 viewport under load); explicit here per + // operator request as defense-in-depth. + // + // VERIFIED CAVEAT (do not remove without re-checking): Playwright's own + // chromium launcher already hardcodes this exact flag unconditionally + // for every launch, on every OS (node_modules/playwright-core/lib/ + // coreBundle.js, `chromiumSwitches`) -- confirmed via + // `DEBUG=pw:browser npx playwright test ...`, which shows + // `--disable-dev-shm-usage` in the resolved command line with or + // without this block. So this line is a harmless duplicate (Chromium + // ignores repeated flags), not something that was missing before. It + // does NOT explain the infra-crash bucket in 28658801255 by itself -- + // that flag was already active on that run. Kept per operator request; + // if the infra-crash bucket persists on the next run, the real cause is + // still open (candidates: runner resource pressure under `workers: 1` + // + `retries: 2`, not literally /dev/shm). + launchOptions: { + args: ['--disable-dev-shm-usage'], + }, }, projects: [ {