From 0ac4e0ad97d213a5638fe29d2fa77d092defc525 Mon Sep 17 00:00:00 2001 From: ernestt Date: Mon, 3 Aug 2026 23:51:58 +0000 Subject: [PATCH] feat(vibe-tests): add trends dashboard from nightly issue history Collect nightly vibe-test results from GitHub issues (label:vibe-test) into a normalized time series and render a self-contained HTML dashboard: overall score trend, Astryx-vs-Baseline gap with a moving average, per-dimension small multiples, and a sortable table linking back to each issue. - trends-collect.ts parses scoreboards across historical table layouts, excludes non-score issues, and rescales old 0-10 runs to 0-100 - trends-report.ts injects the data into an inline-SVG dashboard (no deps) - exposed via 'lns trends' plus trends:collect / trends:report scripts --- internal/vibe-tests/.gitignore | 4 + internal/vibe-tests/README.md | 32 +- internal/vibe-tests/package.json | 3 + internal/vibe-tests/src/cli.ts | 59 +++ internal/vibe-tests/src/trends-collect.ts | 443 ++++++++++++++++++ internal/vibe-tests/src/trends-report.ts | 78 +++ .../src/trends-template/dashboard.html | 325 +++++++++++++ 7 files changed, 943 insertions(+), 1 deletion(-) create mode 100755 internal/vibe-tests/src/trends-collect.ts create mode 100755 internal/vibe-tests/src/trends-report.ts create mode 100644 internal/vibe-tests/src/trends-template/dashboard.html diff --git a/internal/vibe-tests/.gitignore b/internal/vibe-tests/.gitignore index f55dd97408b9..bc6499e5ee46 100644 --- a/internal/vibe-tests/.gitignore +++ b/internal/vibe-tests/.gitignore @@ -16,6 +16,10 @@ AGENTS.html.md # Test results (generated) results/ +# Trends dashboard (generated from GitHub issues by `trends-collect` / `trends-report`) +trends.json +trends-dashboard.html + # Preview build temp dirs .preview-tmp* # .baseline/ is committed (real shadcn components for type checking) diff --git a/internal/vibe-tests/README.md b/internal/vibe-tests/README.md index 2ec472f72eef..230ad6765979 100644 --- a/internal/vibe-tests/README.md +++ b/internal/vibe-tests/README.md @@ -118,6 +118,33 @@ For evaluation fairness, check `universal-eval.ts`: - Verify each branch is measuring the equivalent concept for its system - Watch for any target getting a skip/bonus the others don't +## Trends Dashboard + +Each nightly run is posted as a GitHub issue labeled `vibe-test`, with a scoreboard +table in the body. `trends-collect` reads every such issue, parses the scoreboard +across the historical table layouts, and writes a normalized `trends.json`. +`trends-report` injects that data into a single self-contained HTML dashboard — +overall-score trend, Astryx-vs-Baseline gap with a moving average, per-dimension +small multiples, and a sortable table linking back to each issue. No build step, +no external chart deps (inline SVG). + +```bash +# One shot: refresh data + rebuild the dashboard +pnpm -F @astryxdesign/lns trends # → trends-dashboard.html + +# Or run the stages separately +pnpm -F @astryxdesign/lns trends:collect # → trends.json +pnpm -F @astryxdesign/lns trends:report # trends.json → trends-dashboard.html + +# Point at a different repo/label +pnpm -F @astryxdesign/lns trends -- --repo facebook/astryx --label vibe-test +``` + +Both `trends.json` and `trends-dashboard.html` are generated (gitignored). Runs that +carry the label but aren't score reports — experiments, audits, failed runs — are +excluded automatically; runs scored on the old 0–10 rubric are rescaled to 0–100 so +the trend line stays consistent. + ## Known Accepted Asymmetries These are intentional and documented; they slightly favor baseline, making Astryx wins more credible: @@ -139,7 +166,10 @@ internal/vibe-tests/ │ ├── build-previews.ts # TSX → HTML compilation + tsc │ ├── screenshot-previews.ts # Playwright screenshots │ ├── build-report.ts # Vite HTML report -│ └── deploy-report.ts # gh-pages deployment +│ ├── deploy-report.ts # gh-pages deployment +│ ├── trends-collect.ts # GitHub issues → trends.json time series +│ ├── trends-report.ts # trends.json → self-contained dashboard HTML +│ └── trends-template/ # Dashboard HTML shell (data injected at build) ├── .baseline/ # Real shadcn/ui components for baseline tsc ├── results/ # Iteration results (gitignored) └── README.md # This file diff --git a/internal/vibe-tests/package.json b/internal/vibe-tests/package.json index 06ec73874c85..f3668b689b15 100644 --- a/internal/vibe-tests/package.json +++ b/internal/vibe-tests/package.json @@ -9,6 +9,9 @@ }, "scripts": { "harness": "tsx src/cli.ts", + "trends": "tsx src/cli.ts trends", + "trends:collect": "tsx src/trends-collect.ts", + "trends:report": "tsx src/trends-report.ts", "interactive": "tsx src/interactive.ts", "interactive:sample": "tsx src/interactive.ts --sample 5", "aggregate": "tsx src/universal-aggregate.ts", diff --git a/internal/vibe-tests/src/cli.ts b/internal/vibe-tests/src/cli.ts index 6ce334bf7642..ce9cd7f39765 100755 --- a/internal/vibe-tests/src/cli.ts +++ b/internal/vibe-tests/src/cli.ts @@ -12,6 +12,7 @@ import {Command} from 'commander'; import * as path from 'node:path'; import * as fs from 'node:fs'; +import {execFileSync} from 'node:child_process'; import type {Iteration} from './types.js'; import {readJson} from './utils.js'; @@ -66,4 +67,62 @@ program } }); +/** + * Trends command — collect nightly GitHub-issue results into a time series + * and build the self-contained dashboard. + */ +program + .command('trends') + .description( + 'Collect nightly vibe-test issues and build the trends dashboard', + ) + .option( + '--collect-only', + 'Only refresh trends.json; skip building the dashboard', + ) + .option('--repo ', 'GitHub repo to read issues from', 'facebook/astryx') + .option('--label