diff --git a/internal/vibe-tests/.gitignore b/internal/vibe-tests/.gitignore index f55dd97408b9..bc6499e5ee46 100644 --- a/internal/vibe-tests/.gitignore +++ b/internal/vibe-tests/.gitignore @@ -16,6 +16,10 @@ AGENTS.html.md # Test results (generated) results/ +# Trends dashboard (generated from GitHub issues by `trends-collect` / `trends-report`) +trends.json +trends-dashboard.html + # Preview build temp dirs .preview-tmp* # .baseline/ is committed (real shadcn components for type checking) diff --git a/internal/vibe-tests/README.md b/internal/vibe-tests/README.md index 2ec472f72eef..230ad6765979 100644 --- a/internal/vibe-tests/README.md +++ b/internal/vibe-tests/README.md @@ -118,6 +118,33 @@ For evaluation fairness, check `universal-eval.ts`: - Verify each branch is measuring the equivalent concept for its system - Watch for any target getting a skip/bonus the others don't +## Trends Dashboard + +Each nightly run is posted as a GitHub issue labeled `vibe-test`, with a scoreboard +table in the body. `trends-collect` reads every such issue, parses the scoreboard +across the historical table layouts, and writes a normalized `trends.json`. +`trends-report` injects that data into a single self-contained HTML dashboard — +overall-score trend, Astryx-vs-Baseline gap with a moving average, per-dimension +small multiples, and a sortable table linking back to each issue. No build step, +no external chart deps (inline SVG). + +```bash +# One shot: refresh data + rebuild the dashboard +pnpm -F @astryxdesign/lns trends # → trends-dashboard.html + +# Or run the stages separately +pnpm -F @astryxdesign/lns trends:collect # → trends.json +pnpm -F @astryxdesign/lns trends:report # trends.json → trends-dashboard.html + +# Point at a different repo/label +pnpm -F @astryxdesign/lns trends -- --repo facebook/astryx --label vibe-test +``` + +Both `trends.json` and `trends-dashboard.html` are generated (gitignored). Runs that +carry the label but aren't score reports — experiments, audits, failed runs — are +excluded automatically; runs scored on the old 0–10 rubric are rescaled to 0–100 so +the trend line stays consistent. + ## Known Accepted Asymmetries These are intentional and documented; they slightly favor baseline, making Astryx wins more credible: @@ -139,7 +166,10 @@ internal/vibe-tests/ │ ├── build-previews.ts # TSX → HTML compilation + tsc │ ├── screenshot-previews.ts # Playwright screenshots │ ├── build-report.ts # Vite HTML report -│ └── deploy-report.ts # gh-pages deployment +│ ├── deploy-report.ts # gh-pages deployment +│ ├── trends-collect.ts # GitHub issues → trends.json time series +│ ├── trends-report.ts # trends.json → self-contained dashboard HTML +│ └── trends-template/ # Dashboard HTML shell (data injected at build) ├── .baseline/ # Real shadcn/ui components for baseline tsc ├── results/ # Iteration results (gitignored) └── README.md # This file diff --git a/internal/vibe-tests/package.json b/internal/vibe-tests/package.json index 06ec73874c85..f3668b689b15 100644 --- a/internal/vibe-tests/package.json +++ b/internal/vibe-tests/package.json @@ -9,6 +9,9 @@ }, "scripts": { "harness": "tsx src/cli.ts", + "trends": "tsx src/cli.ts trends", + "trends:collect": "tsx src/trends-collect.ts", + "trends:report": "tsx src/trends-report.ts", "interactive": "tsx src/interactive.ts", "interactive:sample": "tsx src/interactive.ts --sample 5", "aggregate": "tsx src/universal-aggregate.ts", diff --git a/internal/vibe-tests/src/cli.ts b/internal/vibe-tests/src/cli.ts index 6ce334bf7642..ce9cd7f39765 100755 --- a/internal/vibe-tests/src/cli.ts +++ b/internal/vibe-tests/src/cli.ts @@ -12,6 +12,7 @@ import {Command} from 'commander'; import * as path from 'node:path'; import * as fs from 'node:fs'; +import {execFileSync} from 'node:child_process'; import type {Iteration} from './types.js'; import {readJson} from './utils.js'; @@ -66,4 +67,62 @@ program } }); +/** + * Trends command — collect nightly GitHub-issue results into a time series + * and build the self-contained dashboard. + */ +program + .command('trends') + .description( + 'Collect nightly vibe-test issues and build the trends dashboard', + ) + .option( + '--collect-only', + 'Only refresh trends.json; skip building the dashboard', + ) + .option('--repo ', 'GitHub repo to read issues from', 'facebook/astryx') + .option('--label