Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions apps/web/app/api/leaderboard/compare/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,24 @@ import {
csvCompareResponse,
modelCompareToCsv,
} from "../../../../lib/compareExport";
import { pickIndexFilterParams } from "../../../../lib/compareFilterParams";
import { loadAnalysisIndex } from "../../../../lib/data";
import { applyIndexFilters } from "../../../../lib/indexFilters";
import { buildModelComparePayload } from "../../../../lib/modelCompare";

export async function GET(request: Request) {
const url = new URL(request.url);
const format = url.searchParams.get("format") ?? "json";
const left = url.searchParams.get("left");
const right = url.searchParams.get("right");
const filterParams = pickIndexFilterParams({
q: url.searchParams.get("q") ?? undefined,
model: url.searchParams.get("model") ?? undefined,
preset: url.searchParams.get("preset") ?? undefined,
fast: url.searchParams.get("fast") ?? undefined,
from: url.searchParams.get("from") ?? undefined,
to: url.searchParams.get("to") ?? undefined,
});

if (!left || !right) {
return Response.json(
Expand All @@ -18,15 +28,18 @@ export async function GET(request: Request) {
);
}

const index = await loadAnalysisIndex();
if (!index) {
const rawIndex = await loadAnalysisIndex();
if (!rawIndex) {
return Response.json(
{ error: "analysis-index not found" },
{ status: 404 },
);
}

const compare = buildModelComparePayload(index, left, right);
const index = applyIndexFilters(rawIndex, filterParams);
const compare = buildModelComparePayload(index, left, right, {
linkFilters: filterParams,
});
if (!compare) {
return Response.json(
{ error: "one or both models not found in analysis index" },
Expand Down
26 changes: 16 additions & 10 deletions apps/web/app/api/presets/compare/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,24 @@ import {
csvCompareResponse,
presetCompareToCsv,
} from "../../../../lib/compareExport";
import { pickIndexFilterParams } from "../../../../lib/compareFilterParams";
import { loadAnalysisIndex } from "../../../../lib/data";
import { applyIndexFilters } from "../../../../lib/indexFilters";
import { buildPresetComparePayload } from "../../../../lib/presetCompare";

function resolveFastMode(value: string | null): boolean | undefined {
if (value === "true") return true;
if (value === "false") return false;
return undefined;
}

export async function GET(request: Request) {
const url = new URL(request.url);
const format = url.searchParams.get("format") ?? "json";
const left = url.searchParams.get("left");
const right = url.searchParams.get("right");
const fastMode = resolveFastMode(url.searchParams.get("fast"));
const filterParams = pickIndexFilterParams({
q: url.searchParams.get("q") ?? undefined,
model: url.searchParams.get("model") ?? undefined,
preset: url.searchParams.get("preset") ?? undefined,
fast: url.searchParams.get("fast") ?? undefined,
from: url.searchParams.get("from") ?? undefined,
to: url.searchParams.get("to") ?? undefined,
});

if (!left || !right) {
return Response.json(
Expand All @@ -25,15 +28,18 @@ export async function GET(request: Request) {
);
}

const index = await loadAnalysisIndex();
if (!index) {
const rawIndex = await loadAnalysisIndex();
if (!rawIndex) {
return Response.json(
{ error: "analysis-index not found" },
{ status: 404 },
);
}

const compare = buildPresetComparePayload(index, left, right, { fastMode });
const index = applyIndexFilters(rawIndex, filterParams);
const compare = buildPresetComparePayload(index, left, right, {
linkFilters: filterParams,
});
if (!compare) {
return Response.json(
{ error: "one or both presets not found in analysis index" },
Expand Down
2 changes: 2 additions & 0 deletions apps/web/app/counterfactual/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
ResponsiveTable,
TruncateText,
} from "../../components/ResponsiveTable";
import { StaleIndexBanner } from "../../components/StaleIndexBanner";
import { loadAnalysisIndex, loadRunArtifacts } from "../../lib/data";
import {
aggregateCounterfactualStrings,
Expand Down Expand Up @@ -81,6 +82,7 @@ export default async function CounterfactualExplorerPage({

return (
<section className="stack">
<StaleIndexBanner />
<div>
<h1 className="title">Counterfactual failure modes</h1>
<p className="subtitle">
Expand Down
2 changes: 2 additions & 0 deletions apps/web/app/evidence/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
ResponsiveTable,
TruncateText,
} from "../../components/ResponsiveTable";
import { StaleIndexBanner } from "../../components/StaleIndexBanner";
import { loadAnalysisIndex, loadRunArtifacts } from "../../lib/data";
import {
aggregateEvidenceStrings,
Expand Down Expand Up @@ -88,6 +89,7 @@ export default async function EvidenceExplorerPage({

return (
<section className="stack">
<StaleIndexBanner />
<div>
<h1 className="title">Evidence planning</h1>
<p className="subtitle">
Expand Down
16 changes: 16 additions & 0 deletions apps/web/app/globals.css
Original file line number Diff line number Diff line change
Expand Up @@ -1990,6 +1990,22 @@ pre {
line-height: 1.35;
}

.experiment-matrix-cell-wrap {
display: flex;
flex-direction: column;
gap: 0.35rem;
}

.experiment-matrix-compare {
color: var(--color-accent);
font-size: 0.75rem;
text-decoration: none;
}

.experiment-matrix-compare:hover {
text-decoration: underline;
}

/* Heatmap improvements */
.benchmark-heatmap-wrap {
overflow-x: auto;
Expand Down
49 changes: 41 additions & 8 deletions apps/web/app/leaderboard/compare/page.tsx
Original file line number Diff line number Diff line change
@@ -1,11 +1,18 @@
import type { Metadata } from "next";
import Link from "next/link";
import { ActiveIndexFiltersNotice } from "../../../components/ActiveIndexFiltersNotice";
import { CompareDeltaChart } from "../../../components/charts/CompareDeltaChart";
import { CompareExportLink } from "../../../components/CompareExportLink";
import { CompareFormFilterFields } from "../../../components/CompareFormFilterFields";
import { CompareSwapLink } from "../../../components/CompareSwapLink";
import { MetricCard } from "../../../components/MetricCard";
import { ModelFilterSelect } from "../../../components/ModelFilterSelect";
import { loadAnalysisIndex } from "../../../lib/data";
import {
indexFilterExtraParams,
pickIndexFilterParams,
} from "../../../lib/compareFilterParams";
import { applyIndexFilters } from "../../../lib/indexFilters";
import { buildModelLeaderboard } from "../../../lib/modelLeaderboard";
import { buildModelComparePayload } from "../../../lib/modelCompare";
import { buildLeaderboardCompareSuggestions } from "../../../lib/leaderboardCompareSuggestions";
Expand All @@ -17,6 +24,12 @@ export const metadata: Metadata = {
type ModelCompareSearchParams = {
left?: string;
right?: string;
q?: string;
model?: string;
preset?: string;
fast?: string;
from?: string;
to?: string;
};

function formatMetric(value: number | null, digits = 2) {
Expand All @@ -35,9 +48,9 @@ export default async function ModelComparePage({
searchParams: Promise<ModelCompareSearchParams>;
}) {
const params = await searchParams;
const index = await loadAnalysisIndex();
const rawIndex = await loadAnalysisIndex();

if (!index) {
if (!rawIndex) {
return (
<section className="stack">
<h1 className="title">Compare models</h1>
Expand All @@ -52,20 +65,30 @@ export default async function ModelComparePage({
);
}

const models = buildModelLeaderboard(index).map((row) => row.model);
const filterParams = pickIndexFilterParams(params);
const filterExtraParams = indexFilterExtraParams(filterParams);
const index = applyIndexFilters(rawIndex, filterParams);
const models = buildModelLeaderboard(index, {
linkFilters: filterParams,
}).map((row) => row.model);
const leftModel = (params.left ?? "").trim();
const rightModel = (params.right ?? "").trim();
const compare =
leftModel && rightModel
? buildModelComparePayload(index, leftModel, rightModel)
? buildModelComparePayload(index, leftModel, rightModel, {
linkFilters: filterParams,
})
: null;
const suggestions = buildLeaderboardCompareSuggestions(
buildModelLeaderboard(index).map((row) => ({
key: row.model,
runCount: row.runCount,
})),
buildModelLeaderboard(index, { linkFilters: filterParams }).map(
(row) => ({
key: row.model,
runCount: row.runCount,
}),
),
{ left: leftModel, right: rightModel },
"/leaderboard/compare",
filterExtraParams,
);

return (
Expand All @@ -89,7 +112,15 @@ export default async function ModelComparePage({
</div>
</div>

<ActiveIndexFiltersNotice
filters={filterParams}
filteredRunCount={index.totals.runs}
totalRunCount={rawIndex.totals.runs}
clearHref="/leaderboard/compare"
/>

<form className="card" method="get">
<CompareFormFilterFields filters={filterParams} />
<div className="filter-grid">
<ModelFilterSelect
name="left"
Expand All @@ -113,13 +144,15 @@ export default async function ModelComparePage({
basePath="/leaderboard/compare"
left={leftModel}
right={rightModel}
extraParams={filterExtraParams}
/>
) : null}
{compare ? (
<CompareExportLink
apiPath="/api/leaderboard/compare"
left={leftModel}
right={rightModel}
extraParams={filterExtraParams}
/>
) : null}
</div>
Expand Down
75 changes: 68 additions & 7 deletions apps/web/app/leaderboard/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { loadAnalysisIndex } from "../../lib/data";
import { applyIndexFilters, collectIndexFacets } from "../../lib/indexFilters";
import { buildModelLeaderboard } from "../../lib/modelLeaderboard";
import { buildQueryString } from "../../lib/listPagination";
import { buildLeaderboardSideCompareHref } from "../../lib/compareFilterParams";

export const metadata: Metadata = {
title: "Model leaderboard",
Expand Down Expand Up @@ -84,7 +85,7 @@ export default async function ModelLeaderboardPage({
Critique issues
</Link>
<Link
href="/leaderboard/compare"
href={`/leaderboard/compare${buildQueryString(params, {})}`}
className="button secondary"
>
Compare models
Expand Down Expand Up @@ -184,6 +185,40 @@ export default async function ModelLeaderboardPage({
</Link>
),
},
{
key: "compare",
label: "Compare",
cellClass: "cell-actions",
hideOnMobile: true,
render: (row) => {
const model = (row as { model: string })
.model;
return (
<span className="cell-compare-links">
<a
href={buildLeaderboardSideCompareHref(
"/leaderboard/compare",
"left",
model,
params,
)}
>
L
</a>
<a
href={buildLeaderboardSideCompareHref(
"/leaderboard/compare",
"right",
model,
params,
)}
>
R
</a>
</span>
);
},
},
]}
data={rows.map((row) => ({
...row,
Expand All @@ -203,12 +238,38 @@ export default async function ModelLeaderboardPage({
}))}
getRowId={(row) => (row as { model: string }).model}
renderCardActions={(row) => (
<Link
href={(row as { runsHref: string }).runsHref}
className="button"
>
View runs
</Link>
<>
<Link
href={
(row as { runsHref: string }).runsHref
}
className="button"
>
View runs
</Link>
<a
href={buildLeaderboardSideCompareHref(
"/leaderboard/compare",
"left",
(row as { model: string }).model,
params,
)}
className="button secondary"
>
Set left
</a>
<a
href={buildLeaderboardSideCompareHref(
"/leaderboard/compare",
"right",
(row as { model: string }).model,
params,
)}
className="button secondary"
>
Set right
</a>
</>
)}
/>
)}
Expand Down
Loading
Loading