Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions internal/vibe-tests/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ AGENTS.html.md
# Test results (generated)
results/

# Trends dashboard (generated from GitHub issues by `trends-collect` / `trends-report`)
trends.json
trends-dashboard.html

# Preview build temp dirs
.preview-tmp*
# .baseline/ is committed (real shadcn components for type checking)
32 changes: 31 additions & 1 deletion internal/vibe-tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,33 @@ For evaluation fairness, check `universal-eval.ts`:
- Verify each branch is measuring the equivalent concept for its system
- Watch for any target getting a skip/bonus the others don't

## Trends Dashboard

Each nightly run is posted as a GitHub issue labeled `vibe-test`, with a scoreboard
table in the body. `trends-collect` reads every such issue, parses the scoreboard
across the historical table layouts, and writes a normalized `trends.json`.
`trends-report` injects that data into a single self-contained HTML dashboard —
overall-score trend, Astryx-vs-Baseline gap with a moving average, per-dimension
small multiples, and a sortable table linking back to each issue. No build step,
no external chart deps (inline SVG).

```bash
# One shot: refresh data + rebuild the dashboard
pnpm -F @astryxdesign/lns trends # → trends-dashboard.html

# Or run the stages separately
pnpm -F @astryxdesign/lns trends:collect # → trends.json
pnpm -F @astryxdesign/lns trends:report # trends.json → trends-dashboard.html

# Point at a different repo/label
pnpm -F @astryxdesign/lns trends -- --repo facebook/astryx --label vibe-test
```

Both `trends.json` and `trends-dashboard.html` are generated (gitignored). Runs that
carry the label but aren't score reports — experiments, audits, failed runs — are
excluded automatically; runs scored on the old 0–10 rubric are rescaled to 0–100 so
the trend line stays consistent.

## Known Accepted Asymmetries

These are intentional and documented; they slightly favor baseline, making Astryx wins more credible:
Expand All @@ -139,7 +166,10 @@ internal/vibe-tests/
│ ├── build-previews.ts # TSX → HTML compilation + tsc
│ ├── screenshot-previews.ts # Playwright screenshots
│ ├── build-report.ts # Vite HTML report
│ └── deploy-report.ts # gh-pages deployment
│ ├── deploy-report.ts # gh-pages deployment
│ ├── trends-collect.ts # GitHub issues → trends.json time series
│ ├── trends-report.ts # trends.json → self-contained dashboard HTML
│ └── trends-template/ # Dashboard HTML shell (data injected at build)
├── .baseline/ # Real shadcn/ui components for baseline tsc
├── results/ # Iteration results (gitignored)
└── README.md # This file
Expand Down
3 changes: 3 additions & 0 deletions internal/vibe-tests/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
},
"scripts": {
"harness": "tsx src/cli.ts",
"trends": "tsx src/cli.ts trends",
"trends:collect": "tsx src/trends-collect.ts",
"trends:report": "tsx src/trends-report.ts",
"interactive": "tsx src/interactive.ts",
"interactive:sample": "tsx src/interactive.ts --sample 5",
"aggregate": "tsx src/universal-aggregate.ts",
Expand Down
59 changes: 59 additions & 0 deletions internal/vibe-tests/src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import {Command} from 'commander';
import * as path from 'node:path';
import * as fs from 'node:fs';
import {execFileSync} from 'node:child_process';
import type {Iteration} from './types.js';
import {readJson} from './utils.js';

Expand Down Expand Up @@ -66,4 +67,62 @@ program
}
});

/**
* Trends command — collect nightly GitHub-issue results into a time series
* and build the self-contained dashboard.
*/
program
.command('trends')
.description(
'Collect nightly vibe-test issues and build the trends dashboard',
)
.option(
'--collect-only',
'Only refresh trends.json; skip building the dashboard',
)
.option('--repo <slug>', 'GitHub repo to read issues from', 'facebook/astryx')
.option('--label <label>', 'Issue label to collect', 'vibe-test')
.option('--open', 'Print the dashboard path on completion')
.action(async options => {
const scriptDir = import.meta.dirname;
const trendsJson = path.join(scriptDir, '..', 'trends.json');
const dashboard = path.join(scriptDir, '..', 'trends-dashboard.html');

execFileSync(
'npx',
[
'tsx',
path.join(scriptDir, 'trends-collect.ts'),
'--repo',
options.repo,
'--label',
options.label,
'--out',
trendsJson,
],
{stdio: 'inherit'},
);

if (options.collectOnly) {
return;
}

execFileSync(
'npx',
[
'tsx',
path.join(scriptDir, 'trends-report.ts'),
'--in',
trendsJson,
'--out',
dashboard,
],
{stdio: 'inherit'},
);

if (options.open) {
console.log(`\nOpen: ${dashboard}`);
}
});

program.parse();
Loading