diff --git a/apps/web/app/api/outliers/route.ts b/apps/web/app/api/outliers/route.ts index 82e7367..f324147 100644 --- a/apps/web/app/api/outliers/route.ts +++ b/apps/web/app/api/outliers/route.ts @@ -11,6 +11,7 @@ function readFilterParams(url: URL) { fast: url.searchParams.get("fast") ?? undefined, from: url.searchParams.get("from") ?? undefined, to: url.searchParams.get("to") ?? undefined, + benchmark: url.searchParams.get("benchmark") ?? undefined, }; } @@ -28,7 +29,10 @@ export async function GET(request: Request) { } const filteredIndex = applyIndexFilters(index, filters); - const rows = await buildOutlierExplorerRows(filteredIndex); + const benchmarkId = (filters.benchmark ?? "").trim(); + const rows = await buildOutlierExplorerRows(filteredIndex, { + benchmarkId: benchmarkId || undefined, + }); if (format === "csv") { const csv = outliersToCsv(rows); diff --git a/apps/web/app/benchmarks/[id]/page.tsx b/apps/web/app/benchmarks/[id]/page.tsx index f79dd98..0906521 100644 --- a/apps/web/app/benchmarks/[id]/page.tsx +++ b/apps/web/app/benchmarks/[id]/page.tsx @@ -22,7 +22,10 @@ import { buildBenchmarkIndexLookup, resolveIndexedModeLabel, } from "../../../lib/benchmarkIndexLookup"; -import { buildBenchmarkOutlierLookup } from "../../../lib/outlierLookup"; +import { + buildBenchmarkOutlierLookup, + outliersBenchmarkHref, +} from "../../../lib/outlierLookup"; import { questionHubHref } from "../../../lib/questionGroups"; import { extractBenchmarkSummaryDisplay, @@ -167,7 +170,7 @@ export default async function BenchmarkDetailPage({ {outlierCount > 0 ? ( Outlier runs ({outlierCount}) diff --git a/apps/web/app/outliers/page.tsx b/apps/web/app/outliers/page.tsx index b98759a..860cb84 100644 --- a/apps/web/app/outliers/page.tsx +++ b/apps/web/app/outliers/page.tsx @@ -19,6 +19,7 @@ type OutliersSearchParams = { fast?: string; from?: string; to?: string; + benchmark?: string; }; export default async function OutliersPage({ @@ -47,7 +48,10 @@ export default async function OutliersPage({ const { models, presets } = collectIndexFacets(rawIndex); const index = applyIndexFilters(rawIndex, params); - const rows = await buildOutlierExplorerRows(index); + const benchmarkId = (params.benchmark ?? "").trim(); + const rows = await buildOutlierExplorerRows(index, { + benchmarkId: benchmarkId || undefined, + }); return (
@@ -78,15 +82,33 @@ export default async function OutliersPage({ params={params} totalRuns={rawIndex.runs.length} filteredRuns={index.runs.length} + preserveKeys={benchmarkId ? ["benchmark"] : []} /> + {benchmarkId ? ( +
+

+ Showing outliers for benchmark{" "} + + {benchmarkId} + + .{" "} + + Clear benchmark filter + +

+
+ ) : null} + {rows.length > 0 ? (
diff --git a/apps/web/app/runs/[id]/page.tsx b/apps/web/app/runs/[id]/page.tsx index be48d6f..7195a64 100644 --- a/apps/web/app/runs/[id]/page.tsx +++ b/apps/web/app/runs/[id]/page.tsx @@ -16,15 +16,18 @@ import { RunMetricsSummary } from "../../../components/RunMetricsSummary"; import { ConsensusSummary } from "../../../components/ConsensusSummary"; import { CritiqueBreakdown } from "../../../components/CritiqueBreakdown"; import { CritiqueByAgent } from "../../../components/CritiqueByAgent"; +import { MetricCard } from "../../../components/MetricCard"; import { extractConsensusSummary } from "../../../lib/consensusSummary"; import { extractCritiqueByType } from "../../../lib/critiqueBreakdown"; import { extractCritiqueByAgent } from "../../../lib/critiqueByAgent"; +import { extractCritiqueNotesFromRun } from "../../../lib/critiqueNotes"; +import { buildDriftCompareHref } from "../../../lib/confidenceDrift"; import { DownloadArtifactLink } from "../../../components/DownloadArtifactLink"; import { CopyPageLink } from "../../../components/CopyPageLink"; import { CopyTextButton } from "../../../components/CopyTextButton"; import { summarizeRun } from "../../../lib/runCompare"; import { buildCompareSuggestions } from "../../../lib/compareSuggestions"; -import { questionHubHref } from "../../../lib/questionGroups"; +import { questionHubHref, listQuestionInsightLinks } from "../../../lib/questionGroups"; import { buildRunStepTiming, formatDurationMs, @@ -76,6 +79,10 @@ export default async function RunTracePage({ const consensusSummary = extractConsensusSummary(run); const critiqueByType = extractCritiqueByType(run); const critiqueByAgent = extractCritiqueByAgent(run); + const critiqueNotes = extractCritiqueNotesFromRun(run); + const indexedRun = index?.runs.find((entry) => entry.id === run.id); + const insightLinks = listQuestionInsightLinks(run.question); + const driftCompareHref = buildDriftCompareHref(allRuns, run.id); const { schemaVersion, pipelineVersion } = run.metadata; return ( @@ -209,6 +216,108 @@ export default async function RunTracePage({
+ {indexedRun ? ( +
+

Confidence drift

+

+ Stage-to-stage confidence deltas from the analysis index + — compare with peers on the same question. +

+
+ + + + +
+
+ + Drift explorer + + + Compare drift peer + +
+
+ ) : null} + +
+

Question insight shortcuts

+

+ Open filtered insight pages scoped to this research question. +

+
+ {insightLinks.map((link) => ( + + {link.label} + + ))} +
+
+ {stepTimingSummary.timedStepCount > 0 ? (

Step timing

@@ -368,6 +477,51 @@ export default async function RunTracePage({
+ {critiqueNotes.length > 0 ? ( +
+

Critique notes

+

+ Verbatim issue notes from skeptic and red team critique + steps in this trace. +

+ ( + + {(row as { type: string }).type} + + ), + }, + { + key: "note", + label: "Note", + cellClass: "cell-question", + }, + ]} + data={critiqueNotes.map((note, idx) => ({ + ...note, + rowKey: `${note.agentName}-${note.type}-${idx}`, + }))} + getRowId={(row) => + (row as { rowKey: string }).rowKey + } + /> +
+ ) : null} +
) => { + const value = ( + row as { evidenceRisk?: number } + ).evidenceRisk; + return value == null + ? "—" + : value.toFixed(1); + }, + }, + { + key: "solverDrift", + label: "Solver→rev Δ", + helpKey: "solverToRevisionDelta", + hideOnMobile: true, + render: (row: Record) => { + const value = ( + row as { solverDrift?: number } + ).solverDrift; + return value == null + ? "—" + : value.toFixed(2); + }, + }, { key: "coherence", label: "Coherence", @@ -428,6 +456,8 @@ export default async function RunsPage({ issues: indexed?.issueCount, avgSeverity: indexed?.avgSeverity, solverConf: indexed?.solverConfidence, + evidenceRisk: indexed?.evidenceRiskLevel, + solverDrift: indexed?.solverToRevisionDelta, coherence: indexed?.coherence, factualRisk: indexed?.factualRisk, }; diff --git a/apps/web/lib/indexRunLookup.test.ts b/apps/web/lib/indexRunLookup.test.ts index 2b8b0fc..83dc159 100644 --- a/apps/web/lib/indexRunLookup.test.ts +++ b/apps/web/lib/indexRunLookup.test.ts @@ -16,14 +16,21 @@ function sampleIndex(): AnalysisIndex { fastMode: false, finalAnswerPreview: "A", stepCount: 10, - confidence: { solver: 0.8 }, + confidence: { + solver: 0.8, + solverToRevisionDelta: -0.12, + revisionToSynthesizerDelta: 0.05, + }, critique: { issueCount: 5, maxSeverity: 4, avgSeverity: 3.2, byType: { factual: 2 }, }, - research: { evidenceRiskLevel: 3 }, + research: { + evidenceRiskLevel: 3, + topCounterfactualFailureMode: "missing evidence", + }, quality: { coherence: 4.1, factualRisk: 2.5 }, }, ], @@ -51,9 +58,12 @@ describe("buildIndexRunLookup", () => { maxSeverity: 4, avgSeverity: 3.2, solverConfidence: 0.8, + solverToRevisionDelta: -0.12, + revisionToSynthesizerDelta: 0.05, evidenceRiskLevel: 3, coherence: 4.1, factualRisk: 2.5, + topCounterfactualFailureMode: "missing evidence", }); expect(lookup.get("missing")).toBeUndefined(); }); diff --git a/apps/web/lib/indexRunLookup.ts b/apps/web/lib/indexRunLookup.ts index 6e1fb76..f91041e 100644 --- a/apps/web/lib/indexRunLookup.ts +++ b/apps/web/lib/indexRunLookup.ts @@ -6,9 +6,12 @@ export type IndexRunSnapshot = { maxSeverity?: number; avgSeverity?: number; solverConfidence?: number; + solverToRevisionDelta?: number; + revisionToSynthesizerDelta?: number; evidenceRiskLevel?: number; coherence?: number; factualRisk?: number; + topCounterfactualFailureMode?: string; }; export function buildIndexRunLookup( @@ -23,9 +26,14 @@ export function buildIndexRunLookup( maxSeverity: run.critique.maxSeverity, avgSeverity: run.critique.avgSeverity, solverConfidence: run.confidence.solver, + solverToRevisionDelta: run.confidence.solverToRevisionDelta, + revisionToSynthesizerDelta: + run.confidence.revisionToSynthesizerDelta, evidenceRiskLevel: run.research?.evidenceRiskLevel, coherence: run.quality?.coherence, factualRisk: run.quality?.factualRisk, + topCounterfactualFailureMode: + run.research?.topCounterfactualFailureMode, }); } diff --git a/apps/web/lib/outlierExplorer.test.ts b/apps/web/lib/outlierExplorer.test.ts index 1581779..609981b 100644 --- a/apps/web/lib/outlierExplorer.test.ts +++ b/apps/web/lib/outlierExplorer.test.ts @@ -49,4 +49,32 @@ describe("buildOutlierExplorerRows", () => { expect(rows[0].peerRunId).toBeNull(); expect(rows[0].peerCompareHref).toBeNull(); }); + + it("filters outliers to a single benchmark when requested", async () => { + const index: AnalysisIndex = { + ...sampleIndex, + aggregates: { + ...sampleIndex.aggregates, + outlierRuns: [ + { + benchmarkId: "bench_1", + runId: "run_outlier", + avgSimilarity: 0.42, + zScore: -1.8, + }, + { + benchmarkId: "bench_2", + runId: "run_other", + avgSimilarity: 0.5, + zScore: -1.2, + }, + ], + }, + }; + const rows = await buildOutlierExplorerRows(index, { + benchmarkId: "bench_1", + }); + expect(rows).toHaveLength(1); + expect(rows[0]?.runId).toBe("run_outlier"); + }); }); diff --git a/apps/web/lib/outlierExplorer.ts b/apps/web/lib/outlierExplorer.ts index 66836eb..99a95ea 100644 --- a/apps/web/lib/outlierExplorer.ts +++ b/apps/web/lib/outlierExplorer.ts @@ -13,8 +13,14 @@ export type OutlierExplorerRow = { export async function buildOutlierExplorerRows( index: AnalysisIndex, + options?: { benchmarkId?: string }, ): Promise { - const outliers = index.aggregates.outlierRuns ?? []; + const benchmarkId = options?.benchmarkId?.trim(); + const outliers = benchmarkId + ? (index.aggregates.outlierRuns ?? []).filter( + (row) => row.benchmarkId === benchmarkId, + ) + : (index.aggregates.outlierRuns ?? []); const sorted = [...outliers].sort( (a, b) => a.avgSimilarity - b.avgSimilarity, ); diff --git a/apps/web/lib/outlierLookup.test.ts b/apps/web/lib/outlierLookup.test.ts index bf1f499..72899f5 100644 --- a/apps/web/lib/outlierLookup.test.ts +++ b/apps/web/lib/outlierLookup.test.ts @@ -6,6 +6,7 @@ import { buildRunOutlierContext, findRunOutlierEntry, listOutliersForBenchmark, + outliersBenchmarkHref, } from "./outlierLookup"; vi.mock("./data", async (importOriginal) => { @@ -102,4 +103,10 @@ describe("outlierLookup", () => { expect(lookup.get("run_outlier")?.zScore).toBe(-2.1); expect(lookup.has("run_other")).toBe(false); }); + + it("builds benchmark-scoped outlier explorer href", () => { + expect(outliersBenchmarkHref("bench_1")).toBe( + "/outliers?benchmark=bench_1", + ); + }); }); diff --git a/apps/web/lib/outlierLookup.ts b/apps/web/lib/outlierLookup.ts index 574fc69..cc9bc37 100644 --- a/apps/web/lib/outlierLookup.ts +++ b/apps/web/lib/outlierLookup.ts @@ -63,6 +63,10 @@ export function buildBenchmarkOutlierLookup( return lookup; } +export function outliersBenchmarkHref(benchmarkId: string): string { + return `/outliers?${new URLSearchParams({ benchmark: benchmarkId }).toString()}`; +} + export async function buildRunOutlierContext( index: AnalysisIndex | null, runId: string, diff --git a/apps/web/lib/questionGroups.test.ts b/apps/web/lib/questionGroups.test.ts index c1963b5..63aef45 100644 --- a/apps/web/lib/questionGroups.test.ts +++ b/apps/web/lib/questionGroups.test.ts @@ -79,8 +79,14 @@ describe("questionInsightHref", () => { "counterfactual", "issues", "outliers", + "timing", + "agents", + "errors", ]); expect(links[0]?.href).toBe("/quality?q=Q%3F"); + expect(links.find((link) => link.page === "timing")?.href).toBe( + "/timing?q=Q%3F", + ); }); }); diff --git a/apps/web/lib/questionGroups.ts b/apps/web/lib/questionGroups.ts index 67ce289..a49b24e 100644 --- a/apps/web/lib/questionGroups.ts +++ b/apps/web/lib/questionGroups.ts @@ -30,7 +30,10 @@ export type QuestionInsightPage = | "evidence" | "counterfactual" | "issues" - | "outliers"; + | "outliers" + | "timing" + | "agents" + | "errors"; const QUESTION_INSIGHT_PAGES: Array<{ page: QuestionInsightPage; @@ -42,6 +45,9 @@ const QUESTION_INSIGHT_PAGES: Array<{ { page: "counterfactual", label: "Counterfactual modes" }, { page: "issues", label: "Critique issues" }, { page: "outliers", label: "Outlier runs" }, + { page: "timing", label: "Pipeline timing" }, + { page: "agents", label: "Agent stats" }, + { page: "errors", label: "Pipeline errors" }, ]; export function questionInsightHref(