diff --git a/ergon-dashboard/src/app/api/samples/[sampleId]/events/route.ts b/ergon-dashboard/src/app/api/samples/[sampleId]/events/route.ts index 0ca3621d5..7ecf31d56 100644 --- a/ergon-dashboard/src/app/api/samples/[sampleId]/events/route.ts +++ b/ergon-dashboard/src/app/api/samples/[sampleId]/events/route.ts @@ -18,7 +18,7 @@ export async function GET(_request: Request, context: RouteContext) { } catch (error) { return NextResponse.json( { - detail: `Ergon API is unavailable while loading runtime events for sample ${sampleId}.`, + detail: `Ergon API is unavailable while loading events for sample ${sampleId}.`, error: error instanceof Error ? error.message : "Unknown backend fetch failure", }, { status: 503 }, diff --git a/ergon-dashboard/src/app/experiments/[definitionId]/page.tsx b/ergon-dashboard/src/app/experiments/[definitionId]/page.tsx deleted file mode 100644 index 2138ed6a9..000000000 --- a/ergon-dashboard/src/app/experiments/[definitionId]/page.tsx +++ /dev/null @@ -1,228 +0,0 @@ -import Link from "next/link"; -import { notFound } from "next/navigation"; - -import { StatusBadge } from "@/components/common/StatusBadge"; -import { SampleRunMetricExplorer } from "@/components/experiments/SampleRunMetricExplorer"; -import { formatRunMetricValue, metricDescriptor } from "@/components/experiments/sampleRunMetricExplorerModel"; -import { formatDurationMs } from "@/lib/formatDuration"; -import { loadExperimentDetail, type ExperimentDetailWithRunMetrics } from "@/lib/server-data/experiments"; - -interface ExperimentPageProps { - params: Promise<{ definitionId: string }>; -} - -function formatNumber(value: number | null | undefined, fallback = "—") { - if (value === null || value === undefined) return fallback; - return Number.isInteger(value) ? value.toString() : value.toFixed(2); -} - -function formatDate(value: string | null | undefined) { - if (!value) return "—"; - return new Date(value).toLocaleString(); -} - -function workerTeamLabel(workerTeam: Record) { - const entries = Object.entries(workerTeam); - if (entries.length === 0) return "—"; - return entries.map(([key, value]) => `${key}: ${String(value)}`).join(", "); -} - -function runHref(sampleId: string) { - return `/samples/${sampleId}`; -} - -export default async function ExperimentPage({ params }: ExperimentPageProps) { - const { definitionId } = await params; - let detail: ExperimentDetailWithRunMetrics | null = null; - const result = await loadExperimentDetail(definitionId); - if (result.ok) { - detail = result.data; - } else { - if (result.status === 404) notFound(); - throw new Error(`Failed to load experiment ${definitionId}: ${result.status}`); - } - - const experiment = detail.experiment; - const analytics = detail.analytics; - const sampleSelection = detail.sample_selection ?? {}; - const scoreDescriptor = metricDescriptor("score"); - const durationDescriptor = metricDescriptor("duration_ms"); - const tasksDescriptor = metricDescriptor("total_tasks"); - const costDescriptor = metricDescriptor("total_cost_usd"); - const observedCostTotal = detail.runMetricPoints.reduce((total, point) => { - const cost = point.metrics.total_cost_usd; - return cost.available && cost.value != null ? total + cost.value : total; - }, 0); - const hasObservedCosts = detail.runMetricPoints.some((point) => point.metrics.total_cost_usd.available); - - return ( -
-
-
- - Experiments - -

- {experiment.name} -

-

- {experiment.benchmark_type} · {experiment.sample_count} samples ·{" "} - {experiment.run_count} runs · latest activity {formatDate(analytics.latest_activity_at)} -

-
- -
- -
-
-
Model
-
{experiment.default_model_target ?? "—"}
-
-
-
Evaluator
-
{experiment.default_evaluator_slug ?? "—"}
-
-
-
Worker team
-
- {workerTeamLabel(experiment.default_worker_team ?? {})} -
-
-
-
Samples
-
- {Array.isArray(sampleSelection.instance_keys) - ? sampleSelection.instance_keys.join(", ") - : experiment.sample_count} -
-
-
- -
-
-
Score
-
- {formatNumber(analytics.average_score)} -
-
average completed-run score
-
-
-
Runs
-
- {analytics.status_counts.completed}/{analytics.total_runs} -
-
- {analytics.status_counts.failed} failed ·{" "} - {analytics.status_counts.executing + analytics.status_counts.evaluating} active -
-
-
-
Runtime
-
- {formatDurationMs(analytics.average_duration_ms)} -
-
- {formatNumber(analytics.average_tasks)} avg tasks -
-
-
-
Cost
-
- {hasObservedCosts ? formatRunMetricValue(costDescriptor, observedCostTotal) : "Unavailable"} -
-
- {hasObservedCosts ? "observed run costs only" : "no observed cost instrumentation"} -
-
-
- -
- -
- -
- - - - - - - - - - - - - - - {detail.runMetricPoints.map((point) => ( - - - - - - - - - - - ))} - {detail.runMetricPoints.length === 0 ? ( - - - - ) : null} - -
RunSampleStatusScoreDurationTasksCostModel
- - {point.runName} - - - - {point.sampleLabel} - - - - - {point.errorSummary ?
{point.errorSummary}
: null} - - 		- - {formatRunMetricValue(scoreDescriptor, point.metrics.score)} - - - - {formatRunMetricValue(durationDescriptor, point.metrics.duration_ms)} - - - - {formatRunMetricValue(tasksDescriptor, point.metrics.total_tasks)} - - - - {formatRunMetricValue(costDescriptor, point.metrics.total_cost_usd)} - - - - {point.modelTarget ?? "—"} - -
- This experiment has not launched any runs yet. -
-
-
- ); -} diff --git a/ergon-dashboard/src/app/experiments/[experimentId]/page.tsx b/ergon-dashboard/src/app/experiments/[experimentId]/page.tsx new file mode 100644 index 000000000..f7321093f --- /dev/null +++ b/ergon-dashboard/src/app/experiments/[experimentId]/page.tsx @@ -0,0 +1,18 @@ +import { notFound } from "next/navigation"; + +import { ExperimentDetail } from "@/components/experiments/ExperimentDetail"; +import { loadExperimentDetail } from "@/lib/server-data/experiments"; + +interface ExperimentPageProps { + params: Promise<{ experimentId: string }>; +} + +export default async function ExperimentPage({ params }: ExperimentPageProps) { + const { experimentId } = await params; + const result = await loadExperimentDetail(experimentId); + if (!result.ok) { + if (result.status === 404) notFound(); + throw new Error(`Failed to load experiment ${experimentId}: ${result.status}`); + } + return ; +} diff --git a/ergon-dashboard/src/app/experiments/page.tsx b/ergon-dashboard/src/app/experiments/page.tsx index f2fcc7ba6..57583845f 100644 --- a/ergon-dashboard/src/app/experiments/page.tsx +++ b/ergon-dashboard/src/app/experiments/page.tsx @@ -15,14 +15,14 @@ export default async function ExperimentsPage() { const runningCount = experiments.reduce( (sum, experiment) => - sum + experiment.status_counts.executing + experiment.status_counts.evaluating, + sum + experiment.samples.filter((sample) => ["executing", "evaluating"].includes(sample.status)).length, 0, ); const failedCount = experiments.reduce( - (sum, experiment) => sum + experiment.failure_count, + (sum, experiment) => sum + experiment.samples.filter((sample) => sample.status === "failed").length, 0, ); - const totalRuns = experiments.reduce((sum, experiment) => sum + experiment.run_count, 0); + const totalSamples = experiments.reduce((sum, experiment) => sum + experiment.sampleCount, 0); return (
@@ -51,8 +51,8 @@ export default async function ExperimentsPage() {
{failedCount}
-
Runs
-
{totalRuns}
+
Samples
+
{totalSamples}
diff --git a/ergon-dashboard/src/app/samples/[sampleId]/detail/page.tsx b/ergon-dashboard/src/app/samples/[sampleId]/detail/page.tsx new file mode 100644 index 000000000..13ca16277 --- /dev/null +++ b/ergon-dashboard/src/app/samples/[sampleId]/detail/page.tsx @@ -0,0 +1,21 @@ +import { notFound } from "next/navigation"; + +import { SampleDetail } from "@/components/samples/SampleDetail"; +import { loadSampleState } from "@/lib/server-data/samples"; + +interface SampleDetailPageProps { + params: Promise<{ + sampleId: string; + }>; +} + +export default async function SampleDetailPage({ params }: SampleDetailPageProps) { + const { sampleId } = await params; + const result = await loadSampleState(sampleId); + if (!result.ok) { + if (result.status === 404) notFound(); + throw new Error(`Failed to load sample ${sampleId}: ${result.status}`); + } + + return ; +} diff --git a/ergon-dashboard/src/app/samples/[sampleId]/page.tsx b/ergon-dashboard/src/app/samples/[sampleId]/page.tsx index e0d081207..0b5e79c52 100644 --- a/ergon-dashboard/src/app/samples/[sampleId]/page.tsx +++ b/ergon-dashboard/src/app/samples/[sampleId]/page.tsx @@ -1,25 +1,12 @@ -import { SampleWorkspacePage } from "@/components/sample/SampleWorkspacePage"; -import { loadSampleSnapshot } from "@/lib/server-data/samples"; -import type { SerializedSampleWorkspaceState } from "@/lib/types"; +import { redirect } from "next/navigation"; -interface LegacyRunPageProps { +interface SamplePageProps { params: Promise<{ sampleId: string; }>; } -export default async function RunPage({ params }: LegacyRunPageProps) { +export default async function SamplePage({ params }: SamplePageProps) { const { sampleId } = await params; - let initialRunState: SerializedSampleWorkspaceState | null = null; - let ssrError: string | null = null; - - const result = await loadSampleSnapshot(sampleId); - if (result.ok) { - initialRunState = result.data; - } else { - const detail = (result.body as { detail?: string })?.detail; - ssrError = detail ?? `Run API returned ${result.status}`; - } - - return ; + redirect(`/samples/${sampleId}/detail`); } diff --git a/ergon-dashboard/src/components/common/StatusBadge.tsx b/ergon-dashboard/src/components/common/StatusBadge.tsx index 12df8b03a..8eb24d474 100644 --- a/ergon-dashboard/src/components/common/StatusBadge.tsx +++ b/ergon-dashboard/src/components/common/StatusBadge.tsx @@ -1,5 +1,7 @@ "use client"; +import React from "react"; + import { SampleLifecycleStatus, TaskStatus } from "@/lib/types"; type StatusType = TaskStatus | SampleLifecycleStatus | string; diff --git a/ergon-dashboard/src/components/experiments/ExperimentDetail.tsx b/ergon-dashboard/src/components/experiments/ExperimentDetail.tsx new file mode 100644 index 000000000..6671f6f03 --- /dev/null +++ b/ergon-dashboard/src/components/experiments/ExperimentDetail.tsx @@ -0,0 +1,80 @@ +import Link from "next/link"; +import React from "react"; + +import { SampleTable } from "@/components/experiments/SampleTable"; +import type { ExperimentDashboardState } from "@/lib/sample-state/dashboard"; + +function formatDate(value: string | null | undefined) { + if (!value) return "-"; + return new Date(value).toLocaleString(); +} + +export function ExperimentDetail({ state }: { state: ExperimentDashboardState }) { + const selectedCount = state.environments.reduce((sum, env) => sum + env.selectedCount, 0); + + return ( +
+
+
+ + Experiments + +

{state.name}

+

+ {state.environments.length} environments · {state.sampleCount} samples · {selectedCount} selected · created{" "} + {formatDate(state.createdAt)} +

+
+
+ +
+ {state.environments.map((environment) => ( +
+
+ Environment +
+
+ {environment.environmentName} +
+
+ {environment.sampleCount} samples · {environment.selectedCount} selected · {environment.sourceMode} +
+
+ ))} + {state.environments.length === 0 ? ( +
+ No environments recorded yet. +
+ ) : null} +
+ +
+
+ Sampler Invocations +
+
+ {state.samplerInvocations.map((invocation) => ( +
+ {invocation.samplerName} + + requested {invocation.requestedK} · pool {invocation.candidatePoolSize} · selected{" "} + {invocation.selectedCount} + +
+ ))} + {state.samplerInvocations.length === 0 ? ( +
No sampler invocations recorded yet.
+ ) : null} +
+
+ + +
+ ); +} diff --git a/ergon-dashboard/src/components/experiments/SampleTable.tsx b/ergon-dashboard/src/components/experiments/SampleTable.tsx new file mode 100644 index 000000000..fb0ca5ad2 --- /dev/null +++ b/ergon-dashboard/src/components/experiments/SampleTable.tsx @@ -0,0 +1,56 @@ +import Link from "next/link"; +import React from "react"; + +import { StatusBadge } from "@/components/common/StatusBadge"; +import type { ExperimentSampleSummaryView } from "@/lib/contracts/rest"; +import type { SampleLifecycleStatus } from "@/lib/types"; + +export function SampleTable({ samples }: { samples: ExperimentSampleSummaryView[] }) { + return ( +
+ + + + + + + + + + + {samples.map((sample) => ( + + + + + + + ))} + {samples.length === 0 ? ( + + + + ) : null} + +
SampleEnvironmentStatusCreated
+ + {sample.sampleKey} + +
{sample.sampleId}
+ 		{sample.environmentName} + + + {new Date(sample.createdAt).toLocaleString()} +
+ No samples have been materialized yet. +
+
+ ); +} diff --git a/ergon-dashboard/src/components/indexes/ExperimentIndexTable.tsx b/ergon-dashboard/src/components/indexes/ExperimentIndexTable.tsx index 94ea847bc..299048726 100644 --- a/ergon-dashboard/src/components/indexes/ExperimentIndexTable.tsx +++ b/ergon-dashboard/src/components/indexes/ExperimentIndexTable.tsx @@ -4,10 +4,21 @@ import Link from "next/link"; import { useMemo, useState } from "react"; import { StatusBadge } from "@/components/common/StatusBadge"; -import { formatDateTime, formatDurationSeconds, formatPercent } from "@/components/indexes/format"; +import { formatDateTime } from "@/components/indexes/format"; import type { ExperimentSummary } from "@/lib/server-data/experiments"; import type { SampleLifecycleStatus } from "@/lib/types"; +function experimentStatus(experiment: ExperimentSummary): string { + if (experiment.samples.some((sample) => sample.status === "failed")) return "failed"; + if (experiment.samples.some((sample) => ["executing", "evaluating"].includes(sample.status))) { + return "executing"; + } + if (experiment.samples.length > 0 && experiment.samples.every((sample) => sample.status === "completed")) { + return "completed"; + } + return "pending"; +} + export function ExperimentIndexTable({ experiments }: { experiments: ExperimentSummary[] }) { const [query, setQuery] = useState(""); const [status, setStatus] = useState("all"); @@ -15,13 +26,12 @@ export function ExperimentIndexTable({ experiments }: { experiments: ExperimentS const filtered = useMemo(() => { const needle = query.trim().toLowerCase(); return experiments.filter((experiment) => { - const matchesStatus = status === "all" || experiment.status === status; + const computedStatus = experimentStatus(experiment); + const matchesStatus = status === "all" || computedStatus === status; const text = [ experiment.name, experiment.description, - experiment.benchmark_type, - experiment.default_model_target, - experiment.default_evaluator_slug, + experiment.environments.map((env) => env.environmentName).join(" "), ] .filter(Boolean) .join(" ") @@ -45,9 +55,8 @@ export function ExperimentIndexTable({ experiments }: { experiments: ExperimentS className="h-9 rounded-md border border-[var(--line)] bg-[var(--card)] px-3 text-sm text-[var(--ink)] outline-none focus:border-[var(--ink-2)]" > - + - @@ -55,74 +64,57 @@ export function ExperimentIndexTable({ experiments }: { experiments: ExperimentS
- +
- - - + + + - - - - + - {filtered.map((experiment) => ( - - - - - - - - - - - - ))} + {filtered.map((experiment) => { + const selectedCount = experiment.environments.reduce((sum, env) => sum + env.selectedCount, 0); + return ( + + + + + + + + + ); + })} {filtered.length === 0 ? ( - diff --git a/ergon-dashboard/src/components/samples/SampleDetail.tsx b/ergon-dashboard/src/components/samples/SampleDetail.tsx new file mode 100644 index 000000000..a4186a468 --- /dev/null +++ b/ergon-dashboard/src/components/samples/SampleDetail.tsx @@ -0,0 +1,89 @@ +import Link from "next/link"; +import React from "react"; + +import { StatusBadge } from "@/components/common/StatusBadge"; +import { SampleEvents } from "@/components/samples/SampleEvents"; +import type { SampleDashboardState } from "@/lib/sample-state/dashboard"; +import type { SampleLifecycleStatus } from "@/lib/types"; + +function JsonBlock({ value }: { value: Record }) { + const entries = Object.entries(value); + if (entries.length === 0) return -; + return 
{JSON.stringify(value, null, 2)}
; +} + +export function SampleDetail({ state }: { state: SampleDashboardState }) { + return ( +
+
+
+ + Experiment + +

{state.sampleKey}

+

+ {state.environmentName} · {state.sampleId} +

+
+ +
+ +
+
+
Environment
+
{state.environmentName}
+
+
+
Tasks
+
+ {state.graph.nodes.length} nodes · {state.graph.edges.length} edges +
+
+
+
Events
+
{state.events.length}
+
+
+ +
+
+
Source
+ +
+
+
Sample Ref
+ +
+
+ +
+

Graph Projection

+
+
ExperimentBenchmarkRunsFailedEnvironmentsSamplesSelected StatusScoreDurationLatestModel / EvaluatorCreated
- - {experiment.name} - -
- {experiment.description ?? experiment.definition_id} -
- 		- {experiment.benchmark_type} -
{experiment.sample_count} samples
- 		- {experiment.run_count} - - {experiment.failure_count} - - - - {formatPercent(experiment.average_score)} - - {formatDurationSeconds( - experiment.average_duration_ms === null - ? null - : experiment.average_duration_ms / 1000, - )} - - {formatDateTime(experiment.latest_activity_at ?? experiment.created_at)} - -
{experiment.default_model_target ?? "-"}
-
- {experiment.default_evaluator_slug ?? "-"} -
-
+ + {experiment.name} + +
+ {experiment.description ?? experiment.experimentId} +
+ 		+ {experiment.environments.map((env) => env.environmentName).join(", ") || "-"} + + {experiment.sampleCount} + + {selectedCount} + + + + {formatDateTime(experiment.createdAt)} +
+ No experiments match the current filters.
+ + + + + + + + + {state.graph.nodes.map((node) => ( + + + + + + ))} + +
TaskStatusWorker
{node.taskSlug}{node.assignedWorkerSlug ?? "-"}
+
+ + + +
+ ); +} diff --git a/ergon-dashboard/src/components/samples/SampleEvents.tsx b/ergon-dashboard/src/components/samples/SampleEvents.tsx new file mode 100644 index 000000000..73bcdb495 --- /dev/null +++ b/ergon-dashboard/src/components/samples/SampleEvents.tsx @@ -0,0 +1,25 @@ +import type { SampleEventView } from "@/lib/contracts/rest"; +import React from "react"; + +export function SampleEvents({ events }: { events: SampleEventView[] }) { + return ( +
+

Event Stream

+
+ {events.map((event) => ( +
+ + {new Date(event.timestamp).toLocaleString()} + + {event.eventType} + {event.targetType} +
+ ))} + {events.length === 0 ?
No events recorded.
: null} +
+
+ ); +} diff --git a/ergon-dashboard/src/components/shell/Topbar.tsx b/ergon-dashboard/src/components/shell/Topbar.tsx index bc3bf0dff..a5aaf5e25 100644 --- a/ergon-dashboard/src/components/shell/Topbar.tsx +++ b/ergon-dashboard/src/components/shell/Topbar.tsx @@ -5,7 +5,7 @@ import { usePathname } from "next/navigation"; const NAV_ITEMS = [ { label: "Experiments", href: "/experiments" }, - { label: "Runs", href: "/samples" }, + { label: "Samples", href: "/samples" }, ] as const; function isActive(href: string, pathname: string): boolean { @@ -74,7 +74,7 @@ export function Topbar() { data-testid="topbar-search" > ⌕ - Search experiments, runs, tasks… + Search experiments, samples, tasks… ⌘K diff --git a/ergon-dashboard/src/lib/contracts/rest.ts b/ergon-dashboard/src/lib/contracts/rest.ts index c6127f590..b697c59ce 100644 --- a/ergon-dashboard/src/lib/contracts/rest.ts +++ b/ergon-dashboard/src/lib/contracts/rest.ts @@ -11,6 +11,111 @@ const restSchemas = schemas as typeof schemas & { }; export const ExperimentDetailSchema = restSchemas.ExperimentDetailDto ?? schemas.ExperimentDetailView; +export const SampleRuntimeEventViewSchema = schemas.SampleRuntimeEventView; +const JsonRecordSchema = z.record(z.string(), z.unknown()); + +export const SamplerInvocationViewSchema = z.object({ + samplerInvocationId: z.string(), + samplerName: z.string(), + requestedK: z.number(), + candidatePoolSize: z.number(), + selectedCount: z.number(), + samplerConfig: JsonRecordSchema.default({}), + createdAt: z.string(), +}); + +export const EnvironmentContributionViewSchema = z.object({ + environmentId: z.string(), + environmentName: z.string(), + sourceMode: z.string(), + sampleCount: z.number(), + selectedCount: z.number(), + sourceMetadata: JsonRecordSchema.default({}), +}); + +export const ExperimentSampleSummaryViewSchema = z.object({ + sampleId: z.string(), + experimentId: z.string(), + environmentId: z.string(), + environmentName: z.string(), + sampleKey: z.string(), + sampleRef: JsonRecordSchema.default({}), + sourceMetadata: JsonRecordSchema.default({}), + status: z.string(), + createdAt: z.string(), +}); + +export const ExperimentDetailViewSchema = z.object({ + experimentId: z.string(), + name: z.string(), + description: z.string().nullable().optional(), + environments: z.array(EnvironmentContributionViewSchema).default([]), + sampleCount: z.number(), + samples: z.array(ExperimentSampleSummaryViewSchema).default([]), + samplerInvocations: z.array(SamplerInvocationViewSchema).default([]), + metadata: JsonRecordSchema.default({}), + createdAt: z.string(), +}); + +export const ExperimentListViewSchema = z.object({ + items: z.array(ExperimentDetailViewSchema).default([]), +}); + +export const SampleDetailViewSchema = z.object({ + sampleId: z.string(), + experimentId: z.string(), + environmentId: z.string(), + environmentName: z.string(), + sampleKey: z.string(), + sampleRef: JsonRecordSchema.default({}), + sourceMetadata: JsonRecordSchema.default({}), + status: z.string(), + createdAt: z.string(), + startedAt: z.string().nullable().optional(), + completedAt: z.string().nullable().optional(), +}); + +export const SampleEventViewSchema = SampleRuntimeEventViewSchema; + +export const SampleGraphNodeViewSchema = z.object({ + taskId: z.string(), + taskSlug: z.string(), + description: z.string(), + status: z.string(), + parentTaskId: z.string().nullable().optional(), + level: z.number().default(0), + assignedWorkerSlug: z.string().nullable().optional(), + createdAt: z.string(), + updatedAt: z.string(), +}); + +export const SampleGraphEdgeViewSchema = z.object({ + edgeId: z.string(), + sourceTaskId: z.string(), + targetTaskId: z.string(), + status: z.string(), + createdAt: z.string(), + updatedAt: z.string(), +}); + +export const SampleGraphViewSchema = z.object({ + nodes: z.array(SampleGraphNodeViewSchema).default([]), + edges: z.array(SampleGraphEdgeViewSchema).default([]), +}); + +export const SampleEventsViewSchema = z.object({ + items: z.array(SampleEventViewSchema).default([]), +}); + +export const SampleStateViewSchema = z.object({ + sampleId: z.string(), + experimentId: z.string(), + environmentId: z.string(), + environmentName: z.string(), + detail: SampleDetailViewSchema, + events: z.array(SampleEventViewSchema).default([]), + graph: SampleGraphViewSchema.default({ nodes: [], edges: [] }), +}); export const SampleExecutionAttemptSchema = schemas.SampleExecutionAttemptDto; export const SampleResourceSchema = schemas.SampleResourceDto; @@ -21,7 +126,6 @@ export const SampleCommunicationMessageSchema = schemas.SampleCommunicationMessa export const SampleCommunicationThreadSchema = schemas.SampleCommunicationThreadDto; export const SampleTaskEvaluationSchema = schemas.SampleTaskEvaluationDto; export const SampleSnapshotSchema = schemas.SampleSnapshotDto; -export const SampleRuntimeEventViewSchema = schemas.SampleRuntimeEventView; type KnownKeys = { [K in keyof T as string extends K ? never : number extends K ? never : symbol extends K @@ -33,9 +137,54 @@ export type BenchmarkName = z.infer; export type SampleLifecycleStatus = z.infer; export type TaskStatusValue = z.infer; -type RawExperimentDetail = Record; -type RawExperimentRunRow = Record; -type RawExperimentSummary = Record; +interface RawExperimentSummary { + [key: string]: unknown; + average_duration_ms?: number | null; + average_score?: number | null; + average_tasks?: number | null; + default_evaluator_slug?: string | null; + default_model_target?: string | null; + description?: string | null; + failure_count?: number; + latest_activity_at?: string | null; + status_counts?: Partial; + total_cost_usd?: number | null; +} + +interface RawExperimentRunRow { + [key: string]: unknown; + sample_id: string; + status: string; + instance_key: string; + completed_at?: string | null; + error_message?: string | null; + evaluator_slug?: string | null; + final_score?: number | null; + model_target?: string | null; + metrics?: ExperimentRunMetrics | null; + running_time_ms?: number | null; + seed?: number | null; + started_at?: string | null; + total_cost_usd?: number | null; + total_tasks?: number | null; + worker_team?: Record | null; +} + +interface RawExperimentDetail { + [key: string]: unknown; + experiment?: RawExperimentSummary; + runs?: RawExperimentRunRow[]; + analytics?: { + total_runs?: number; + status_counts?: Partial; + average_score?: number | null; + average_duration_ms?: number | null; + average_tasks?: number | null; + total_cost_usd?: number | null; + latest_activity_at?: string | null; + error_count?: number; + }; +} type RawSampleExecutionAttempt = KnownKeys>; type RawSampleResource = KnownKeys>; type RawSampleSandboxCommand = KnownKeys>; @@ -54,6 +203,19 @@ export type RawSampleSandboxCommandType = RawSampleSandboxCommand; export type SampleSnapshotMetrics = RawSampleSnapshotMetrics; export type SampleRuntimeEventView = z.infer; +export type SamplerInvocationView = z.infer; +export type EnvironmentContributionView = z.infer; +export type ExperimentSampleSummaryView = z.infer; +export type ExperimentDetailView = z.infer; +export type ExperimentListView = z.infer; +export type SampleDetailView = z.infer; +export type SampleEventView = z.infer; +export type SampleGraphNodeView = z.infer; +export type SampleGraphEdgeView = z.infer; +export type SampleGraphView = z.infer; +export type SampleEventsView = z.infer; +export type SampleStateView = z.infer; + export interface ExperimentStatusCounts { pending: number; executing: number; @@ -110,22 +272,11 @@ export interface ExperimentSummaryDetail total_cost_usd?: number | null; } -export interface ExperimentRunRow - extends Omit< - RawExperimentRunRow, - | "completed_at" - | "error_message" - | "evaluator_slug" - | "final_score" - | "model_target" - | "metrics" - | "running_time_ms" - | "seed" - | "started_at" - | "total_cost_usd" - | "total_tasks" - | "worker_team" - > { +export interface ExperimentRunRow { + [key: string]: unknown; + sample_id: string; + status: string; + instance_key: string; completed_at: string | null; error_message: string | null; evaluator_slug: string | null; @@ -369,7 +520,14 @@ export function parseExperimentDetail(input: unknown): ExperimentDetail { evaluator_slug: run.evaluator_slug ?? null, final_score: run.final_score ?? null, model_target: run.model_target ?? null, - metrics: run.metrics ?? {}, + metrics: { + sample_id: run.sample_id, + status: run.status, + instance_key: run.instance_key, + tool_call_count: 0, + cost_observed: false, + ...(run.metrics ?? {}), + }, running_time_ms: run.running_time_ms ?? null, seed: run.seed ?? null, started_at: run.started_at ?? null, @@ -380,6 +538,30 @@ export function parseExperimentDetail(input: unknown): ExperimentDetail { }; } +export function parseExperimentState(input: unknown): ExperimentDetailView { + return ExperimentDetailViewSchema.parse(input); +} + +export function parseExperimentListState(input: unknown): ExperimentListView { + return ExperimentListViewSchema.parse(input); +} + +export function parseSampleDetail(input: unknown): SampleDetailView { + return SampleDetailViewSchema.parse(input); +} + +export function parseSampleEvents(input: unknown): SampleEventsView { + return SampleEventsViewSchema.parse(input); +} + +export function parseSampleGraph(input: unknown): SampleGraphView { + return SampleGraphViewSchema.parse(input); +} + +export function parseSampleState(input: unknown): SampleStateView { + return SampleStateViewSchema.parse(input); +} + export function parseSampleSandbox(input: unknown): SampleSandbox { return normalizeSampleSandbox(SampleSandboxSchema.parse(input)); } diff --git a/ergon-dashboard/src/lib/sample-state/dashboard.ts b/ergon-dashboard/src/lib/sample-state/dashboard.ts new file mode 100644 index 000000000..4f74917ec --- /dev/null +++ b/ergon-dashboard/src/lib/sample-state/dashboard.ts @@ -0,0 +1,73 @@ +import type { + ExperimentDetailView, + SampleDetailView, + SampleEventView, + SampleGraphView, + SamplerInvocationView, + EnvironmentContributionView, + ExperimentSampleSummaryView, +} from "@/lib/contracts/rest"; + +export interface ExperimentDashboardState { + experimentId: string; + name: string; + description: string | null; + environments: EnvironmentContributionView[]; + samples: ExperimentSampleSummaryView[]; + samplerInvocations: SamplerInvocationView[]; + sampleCount: number; + metadata: Record; + createdAt: string; +} + +export interface SampleDashboardState { + sampleId: string; + experimentId: string; + environmentId: string; + environmentName: string; + sampleKey: string; + sampleRef: Record; + sourceMetadata: Record; + status: string; + createdAt: string; + startedAt: string | null; + completedAt: string | null; + events: SampleEventView[]; + graph: SampleGraphView; +} + +export function buildExperimentState(payload: ExperimentDetailView): ExperimentDashboardState { + return { + experimentId: payload.experimentId, + name: payload.name, + description: payload.description ?? null, + environments: payload.environments, + samples: payload.samples, + samplerInvocations: payload.samplerInvocations, + sampleCount: payload.sampleCount, + metadata: payload.metadata, + createdAt: payload.createdAt, + }; +} + +export function buildSampleState(payload: { + detail: SampleDetailView; + events: SampleEventView[]; + graph: SampleGraphView; +}): SampleDashboardState { + return { + sampleId: payload.detail.sampleId, + experimentId: payload.detail.experimentId, + environmentId: payload.detail.environmentId, + environmentName: payload.detail.environmentName, + sampleKey: payload.detail.sampleKey, + sampleRef: payload.detail.sampleRef, + sourceMetadata: payload.detail.sourceMetadata, + status: payload.detail.status, + createdAt: payload.detail.createdAt, + startedAt: payload.detail.startedAt ?? null, + completedAt: payload.detail.completedAt ?? null, + events: payload.events, + graph: payload.graph, + }; +} diff --git a/ergon-dashboard/src/lib/server-data/experiments.ts b/ergon-dashboard/src/lib/server-data/experiments.ts index 7425333b6..9894bce32 100644 --- a/ergon-dashboard/src/lib/server-data/experiments.ts +++ b/ergon-dashboard/src/lib/server-data/experiments.ts @@ -1,96 +1,19 @@ import { config } from "@/lib/config"; -import { parseExperimentDetail, type ExperimentDetail } from "@/lib/contracts/rest"; +import { + parseExperimentListState, + parseExperimentState, + type ExperimentDetailView, +} from "@/lib/contracts/rest"; +import { buildExperimentState, type ExperimentDashboardState } from "@/lib/sample-state/dashboard"; import { fetchErgonApi } from "@/lib/serverApi"; import { getHarnessExperiment } from "@/lib/testing/dashboardHarness"; -import { - normalizeRunMetricPoints, - type RunMetricPoint, -} from "@/components/experiments/sampleRunMetricExplorerModel"; import { backendUnavailable, type ServerDataResult } from "./responses"; -export interface ExperimentSummary { - definition_id: string; - name: string; - description: string | null; - benchmark_type: string; - sample_count: number; - status: string; - default_model_target: string | null; - default_evaluator_slug: string | null; - created_at: string; - run_count: number; - status_counts: { - pending: number; - executing: number; - evaluating: number; - completed: number; - failed: number; - cancelled: number; - }; - failure_count: number; - latest_activity_at: string | null; - average_score: number | null; - average_duration_ms: number | null; - average_tasks: number | null; - total_cost_usd: number | null; -} - -export interface ExperimentDetailWithRunMetrics extends ExperimentDetail { - runMetricPoints: RunMetricPoint[]; -} - -export function normalizeExperimentDetail(detail: ExperimentDetail): ExperimentDetailWithRunMetrics { - return { - ...detail, - runMetricPoints: normalizeRunMetricPoints(detail.runs), - }; -} - -function parseExperimentList(input: unknown): ExperimentSummary[] { - if (!Array.isArray(input)) return []; - return input.map((item) => { - const record = typeof item === "object" && item !== null ? (item as Record) : {}; - return { - definition_id: String(record.definition_id ?? ""), - name: String(record.name ?? ""), - description: optionalString(record.description), - benchmark_type: String(record.benchmark_type ?? ""), - sample_count: Number(record.sample_count ?? 0), - status: String(record.status ?? ""), - default_model_target: optionalString(record.default_model_target), - default_evaluator_slug: optionalString(record.default_evaluator_slug), - created_at: String(record.created_at ?? ""), - run_count: Number(record.run_count ?? 0), - status_counts: parseStatusCounts(record.status_counts), - failure_count: Number(record.failure_count ?? 0), - latest_activity_at: optionalString(record.latest_activity_at), - average_score: optionalNumber(record.average_score), - average_duration_ms: optionalNumber(record.average_duration_ms), - average_tasks: optionalNumber(record.average_tasks), - total_cost_usd: optionalNumber(record.total_cost_usd), - }; - }); -} - -function parseStatusCounts(input: unknown): ExperimentSummary["status_counts"] { - const record = typeof input === "object" && input !== null ? (input as Record) : {}; - return { - pending: Number(record.pending ?? 0), - executing: Number(record.executing ?? 0), - evaluating: Number(record.evaluating ?? 0), - completed: Number(record.completed ?? 0), - failed: Number(record.failed ?? 0), - cancelled: Number(record.cancelled ?? 0), - }; -} - -function optionalString(input: unknown): string | null { - return typeof input === "string" && input.length > 0 ? input : null; -} +export type ExperimentSummary = ExperimentDashboardState; -function optionalNumber(input: unknown): number | null { - return typeof input === "number" && Number.isFinite(input) ? input : null; +function normalizeExperimentDetail(detail: ExperimentDetailView): ExperimentDashboardState { + return buildExperimentState(detail); } export async function loadExperimentList(): Promise> { @@ -98,9 +21,10 @@ export async function loadExperimentList(): Promise> { + experimentId: string, +): Promise> { if (config.enableTestHarness) { - const detail = getHarnessExperiment(definitionId); + const detail = getHarnessExperiment(experimentId); if (detail !== null) { return { ok: true, - data: normalizeExperimentDetail(parseExperimentDetail(detail)), + data: normalizeExperimentDetail(parseExperimentState(detail)), status: 200, source: "harness", }; @@ -127,18 +51,18 @@ export async function loadExperimentDetail( } try { - const response = await fetchErgonApi(`/experiments/${definitionId}`); + const response = await fetchErgonApi(`/experiments/${experimentId}`); const body = await response.json(); if (response.ok) { return { ok: true, - data: normalizeExperimentDetail(parseExperimentDetail(body)), + data: normalizeExperimentDetail(parseExperimentState(body)), status: response.status, source: "backend", }; } return { ok: false, body, status: response.status, source: "backend" }; } catch (error) { - return backendUnavailable(`Ergon API is unavailable while loading experiment ${definitionId}.`, error); + return backendUnavailable(`Ergon API is unavailable while loading experiment ${experimentId}.`, error); } } diff --git a/ergon-dashboard/src/lib/server-data/samples.ts b/ergon-dashboard/src/lib/server-data/samples.ts index 66b13b363..db328e384 100644 --- a/ergon-dashboard/src/lib/server-data/samples.ts +++ b/ergon-dashboard/src/lib/server-data/samples.ts @@ -1,7 +1,14 @@ import { config } from "@/lib/config"; -import { parseSampleSnapshot, type SampleSnapshot } from "@/lib/contracts/rest"; +import { + parseSampleDetail, + parseSampleEvents, + parseSampleGraph, + parseSampleSnapshot, + type SampleSnapshot, +} from "@/lib/contracts/rest"; +import { buildSampleState, type SampleDashboardState } from "@/lib/sample-state/dashboard"; import { fetchErgonApi } from "@/lib/serverApi"; -import { getHarnessSample } from "@/lib/testing/dashboardHarness"; +import { getHarnessSample, getHarnessSampleState } from "@/lib/testing/dashboardHarness"; import { backendUnavailable, type ServerDataResult } from "./responses"; @@ -66,20 +73,20 @@ export async function loadSampleList( } return { ok: false, body, status: response.status, source: "backend" }; } catch (error) { - return backendUnavailable("Ergon API is unavailable while loading runs.", error); + return backendUnavailable("Ergon API is unavailable while loading samples.", error); } } export async function loadSampleSnapshot(sampleId: string): Promise> { if (config.enableTestHarness) { - const run = getHarnessSample(sampleId); - if (run !== null) { - return { ok: true, data: parseSampleSnapshot(run), status: 200, source: "harness" }; + const sample = getHarnessSample(sampleId); + if (sample !== null) { + return { ok: true, data: parseSampleSnapshot(sample), status: 200, source: "harness" }; } } try { - const response = await fetchErgonApi(`/samples/${sampleId}`); + const response = await fetchErgonApi(`/samples/${sampleId}/workspace`); const body = await response.json(); if (response.ok) { return { @@ -91,7 +98,53 @@ export async function loadSampleSnapshot(sampleId: string): Promise> { + if (config.enableTestHarness) { + const sample = getHarnessSampleState(sampleId); + if (sample !== null) { + return { ok: true, data: sample, status: 200, source: "harness" }; + } + } + + try { + const [detailResponse, eventsResponse, graphResponse] = await Promise.all([ + fetchErgonApi(`/samples/${sampleId}`), + fetchErgonApi(`/samples/${sampleId}/events`), + fetchErgonApi(`/samples/${sampleId}/graph`), + ]); + + const [detailBody, eventsBody, graphBody] = await Promise.all([ + detailResponse.json(), + eventsResponse.json(), + graphResponse.json(), + ]); + + if (!detailResponse.ok) { + return { ok: false, body: detailBody, status: detailResponse.status, source: "backend" }; + } + if (!eventsResponse.ok) { + return { ok: false, body: eventsBody, status: eventsResponse.status, source: "backend" }; + } + if (!graphResponse.ok) { + return { ok: false, body: graphBody, status: graphResponse.status, source: "backend" }; + } + + return { + ok: true, + data: buildSampleState({ + detail: parseSampleDetail(detailBody), + events: parseSampleEvents(eventsBody).items, + graph: parseSampleGraph(graphBody), + }), + status: 200, + source: "backend", + }; + } catch (error) { + return backendUnavailable(`Ergon API is unavailable while loading sample ${sampleId}.`, error); } } diff --git a/ergon-dashboard/src/lib/testing/dashboardHarness.ts b/ergon-dashboard/src/lib/testing/dashboardHarness.ts index e8f303309..f3d502902 100644 --- a/ergon-dashboard/src/lib/testing/dashboardHarness.ts +++ b/ergon-dashboard/src/lib/testing/dashboardHarness.ts @@ -10,7 +10,7 @@ import { store } from "@/lib/state/store"; import { CommunicationThreadState, ContextEventState, - ExperimentDetail, + SampleDashboardState, SerializedSampleWorkspaceState, TaskEvaluationState, TaskStatus, @@ -21,7 +21,8 @@ declare global { // eslint-disable-next-line no-var var __dashboardHarness: | { - experimentDetails: Record; + experimentDetails: Record; + sampleStates: Record; mutationsBySample: Record; seededSampleIds: Set; } @@ -29,7 +30,8 @@ declare global { } export interface DashboardHarnessSeedPayload { - experimentDetails?: Record; + experimentDetails?: Record; + sampleStates?: Record; runs?: SerializedSampleWorkspaceState[]; mutations?: Record; } @@ -38,6 +40,7 @@ function getHarnessState() { if (!global.__dashboardHarness) { global.__dashboardHarness = { experimentDetails: {}, + sampleStates: {}, mutationsBySample: {}, seededSampleIds: new Set(), }; @@ -56,6 +59,7 @@ export function resetDashboardHarness(): void { store.reset(); const harness = getHarnessState(); harness.experimentDetails = {}; + harness.sampleStates = {}; harness.mutationsBySample = {}; harness.seededSampleIds.clear(); } @@ -66,6 +70,7 @@ export function seedDashboardHarness(payload: DashboardHarnessSeedPayload): void const harness = getHarnessState(); harness.experimentDetails = payload.experimentDetails ?? {}; + harness.sampleStates = payload.sampleStates ?? {}; harness.mutationsBySample = payload.mutations ?? {}; for (const run of payload.runs ?? []) { @@ -74,11 +79,16 @@ export function seedDashboardHarness(payload: DashboardHarnessSeedPayload): void } } -export function getHarnessExperiment(definitionId: string): ExperimentDetail | null { +export function getHarnessExperiment(definitionId: string): unknown | null { requireHarnessEnabled(); return getHarnessState().experimentDetails[definitionId] ?? null; } +export function getHarnessSampleState(sampleId: string): SampleDashboardState | null { + requireHarnessEnabled(); + return getHarnessState().sampleStates[sampleId] ?? null; +} + export function getHarnessSample(sampleId: string): SerializedSampleWorkspaceState | null { requireHarnessEnabled(); if (!getHarnessState().seededSampleIds.has(sampleId)) { diff --git a/ergon-dashboard/src/lib/types.ts b/ergon-dashboard/src/lib/types.ts index dcafc355e..6bcf3c049 100644 --- a/ergon-dashboard/src/lib/types.ts +++ b/ergon-dashboard/src/lib/types.ts @@ -11,6 +11,7 @@ import type { SampleSnapshotMetrics, SampleTaskEvaluation as RestSampleTaskEvaluation, } from "@/lib/contracts/rest"; +export type { SampleDashboardState } from "@/lib/sample-state/dashboard"; import type { DashboardGraphMutationData as GeneratedDashboardGraphMutationData, DashboardResourcePublishedData as GeneratedDashboardResourcePublishedData, diff --git a/ergon-dashboard/tests/components/ExperimentDetail.test.ts b/ergon-dashboard/tests/components/ExperimentDetail.test.ts new file mode 100644 index 000000000..82f8c6087 --- /dev/null +++ b/ergon-dashboard/tests/components/ExperimentDetail.test.ts @@ -0,0 +1,18 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; + +import { ExperimentDetail } from "../../src/components/experiments/ExperimentDetail"; +import { buildExperimentState } from "../../src/lib/sample-state/dashboard"; +import { fixtureExperimentDetail } from "../contracts/experiment-rest-contract.test"; + +test("experiment detail renders sample rows and environment contributions", () => { + const html = renderToStaticMarkup( + createElement(ExperimentDetail, { state: buildExperimentState(fixtureExperimentDetail) }), + ); + + assert.match(html, /mini-validation/); + assert.match(html, /href="\/samples\/sample-1\/detail"/); + assert.doesNotMatch(html, />Runs { + const html = renderToStaticMarkup( + createElement(SampleDetail, { + state: buildSampleState({ + detail: fixtureSampleDetail, + events: fixtureSampleEvents.items, + graph: fixtureSampleGraph, + }), + }), + ); + + assert.match(html, new RegExp(`href="/experiments/${fixtureSampleDetail.experimentId}"`)); + assert.match(html, /task.added/); + assert.doesNotMatch(html, /mutation/i); +}); diff --git a/ergon-dashboard/tests/contracts/contracts.test.ts b/ergon-dashboard/tests/contracts/contracts.test.ts index 23aba06bc..d28f2f7f8 100644 --- a/ergon-dashboard/tests/contracts/contracts.test.ts +++ b/ergon-dashboard/tests/contracts/contracts.test.ts @@ -151,7 +151,7 @@ test("generated dashboard event schemas cover graph and context live events", () test("frontend routes expose experiment grouping and no cohort surface", () => { const root = process.cwd(); - assert.equal(fs.existsSync(path.join(root, "src/app/experiments/[definitionId]/page.tsx")), true); + assert.equal(fs.existsSync(path.join(root, "src/app/experiments/[experimentId]/page.tsx")), true); assert.equal(fs.existsSync(path.join(root, "src/app/cohorts/page.tsx")), false); assert.equal(fs.existsSync(path.join(root, "src/app/api/cohorts/route.ts")), false); }); diff --git a/ergon-dashboard/tests/contracts/experiment-rest-contract.test.ts b/ergon-dashboard/tests/contracts/experiment-rest-contract.test.ts new file mode 100644 index 000000000..35943c638 --- /dev/null +++ b/ergon-dashboard/tests/contracts/experiment-rest-contract.test.ts @@ -0,0 +1,68 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + parseExperimentListState, + parseExperimentState, + type ExperimentDetailView, +} from "../../src/lib/contracts/rest"; + +const retiredIdentityPattern = new RegExp(["run" + "Id", "definitionId"].join("|"), "i"); + +export const fixtureExperimentDetail: ExperimentDetailView = { + experimentId: "exp-1", + name: "mixed-training", + description: "mixed environment validation", + environments: [ + { + environmentId: "env-1", + environmentName: "mini-validation", + sourceMode: "materialized", + sampleCount: 3, + selectedCount: 2, + sourceMetadata: { provider: "records" }, + }, + ], + sampleCount: 3, + samples: [ + { + sampleId: "sample-1", + experimentId: "exp-1", + environmentId: "env-1", + environmentName: "mini-validation", + sampleKey: "problem-1", + sampleRef: { id: "problem-1" }, + sourceMetadata: { provider: "records" }, + status: "completed", + createdAt: "2026-05-26T00:00:00Z", + }, + ], + samplerInvocations: [ + { + samplerInvocationId: "sampler-1", + samplerName: "RandomSampler", + requestedK: 2, + candidatePoolSize: 3, + selectedCount: 2, + samplerConfig: {}, + createdAt: "2026-05-26T00:00:00Z", + }, + ], + metadata: {}, + createdAt: "2026-05-26T00:00:00Z", +}; + +test("experiment contract has environments and sample ids", () => { + const parsed = parseExperimentState(fixtureExperimentDetail); + + assert.equal(parsed.environments[0].environmentName, "mini-validation"); + assert.equal(parsed.samples[0].sampleId, "sample-1"); + assert.doesNotMatch(JSON.stringify(parsed), retiredIdentityPattern); +}); + +test("experiment list contract wraps sample-centered experiment items", () => { + const parsed = parseExperimentListState({ items: [fixtureExperimentDetail] }); + + assert.equal(parsed.items[0].experimentId, "exp-1"); + assert.equal(parsed.items[0].samplerInvocations[0].selectedCount, 2); +}); diff --git a/ergon-dashboard/tests/contracts/frontend-regressions.contract.test.ts b/ergon-dashboard/tests/contracts/frontend-regressions.contract.test.ts index 9a53654b9..e5879183c 100644 --- a/ergon-dashboard/tests/contracts/frontend-regressions.contract.test.ts +++ b/ergon-dashboard/tests/contracts/frontend-regressions.contract.test.ts @@ -3,7 +3,7 @@ import { readFileSync } from "node:fs"; import test from "node:test"; test("experiment detail does not pass server functions into client components", () => { - const source = readFileSync("src/app/experiments/[definitionId]/page.tsx", "utf8"); + const source = readFileSync("src/app/experiments/[experimentId]/page.tsx", "utf8"); assert.doesNotMatch(source, /getRunHref=\{[a-zA-Z_$][\w$]*\}/); }); @@ -14,11 +14,11 @@ test("run display state keeps live mode separate from graph sequence zero", () = assert.doesNotMatch(source, /snapshotSequence\s*\?\?\s*0/); }); -test("experiment detail bottom run table exposes each row as run navigation", () => { - const source = readFileSync("src/app/experiments/[definitionId]/page.tsx", "utf8"); +test("experiment detail sample table exposes each row as sample navigation", () => { + const source = readFileSync("src/components/experiments/SampleTable.tsx", "utf8"); - assert.match(source, /data-testid=\{`experiment-run-row-\$\{point\.sampleId\}`\}/); - assert.match(source, /href=\{runHref\(point\.sampleId\)\}/); + assert.match(source, /data-testid=\{`experiment-sample-row-\$\{sample\.sampleId\}`\}/); + assert.match(source, /href=\{`\/samples\/\$\{sample\.sampleId\}\/detail`\}/); }); test("run workspace can collapse the bottom activity timeline", () => { diff --git a/ergon-dashboard/tests/contracts/nav.contract.test.ts b/ergon-dashboard/tests/contracts/nav.contract.test.ts index 3d05817ed..14c52c141 100644 --- a/ergon-dashboard/tests/contracts/nav.contract.test.ts +++ b/ergon-dashboard/tests/contracts/nav.contract.test.ts @@ -7,7 +7,7 @@ test("top navigation only exposes current product surfaces", () => { const source = fs.readFileSync(path.join(process.cwd(), "src/components/shell/Topbar.tsx"), "utf8"); assert.match(source, /label: "Experiments"/); - assert.match(source, /label: "Runs"/); + assert.match(source, /label: "Samples"/); assert.doesNotMatch(source, /label: "Models"/); assert.doesNotMatch(source, /label: "Settings"/); }); diff --git a/ergon-dashboard/tests/contracts/sample-rest-contract.test.ts b/ergon-dashboard/tests/contracts/sample-rest-contract.test.ts index 5bd1072f3..1d301b7e1 100644 --- a/ergon-dashboard/tests/contracts/sample-rest-contract.test.ts +++ b/ergon-dashboard/tests/contracts/sample-rest-contract.test.ts @@ -1,64 +1,101 @@ import assert from "node:assert/strict"; -import { existsSync, readFileSync } from "node:fs"; import test from "node:test"; -import { schemas } from "../../src/generated/rest/contracts"; +import { + parseSampleDetail, + parseSampleEvents, + parseSampleGraph, + type SampleDetailView, + type SampleEventsView, + type SampleGraphView, +} from "../../src/lib/contracts/rest"; + +const retiredSamplePattern = new RegExp(["run" + "Id", "definitionId"].join("|"), "i"); +const retiredEventPattern = new RegExp(["GraphMutation", "mutation", "run" + "Id"].join("|"), "i"); const sampleId = "00000000-0000-4000-8000-000000000001"; -const taskId = "00000000-0000-4000-8000-000000000002"; -const edgeId = "00000000-0000-4000-8000-000000000003"; +const experimentId = "00000000-0000-4000-8000-000000000002"; +const environmentId = "00000000-0000-4000-8000-000000000003"; +const statusEventId = "00000000-0000-4000-8000-000000000004"; +const taskEventId = "00000000-0000-4000-8000-000000000005"; +const taskId = "00000000-0000-4000-8000-000000000006"; -test("sample detail no longer exposes generic mutation proxy route", () => { - assert.equal(existsSync("src/app/api/samples/[sampleId]/mutations/route.ts"), false); -}); +export const fixtureSampleDetail: SampleDetailView = { + sampleId, + experimentId, + environmentId, + environmentName: "mini-validation", + sampleKey: "problem-1", + sampleRef: { id: "problem-1" }, + sourceMetadata: { provider: "records" }, + status: "completed", + createdAt: "2026-05-26T00:00:00Z", + startedAt: "2026-05-26T00:00:01Z", + completedAt: "2026-05-26T00:00:05Z", +}; -test("sample workspace loads typed runtime events instead of mutation DTOs", () => { - const source = readFileSync("src/components/sample/SampleWorkspacePage.tsx", "utf8"); +export const fixtureSampleEvents: SampleEventsView = { + items: [ + { + eventId: statusEventId, + sampleId, + eventType: "sample.status_changed", + targetType: "sample", + targetId: sampleId, + status: "pending", + timestamp: "2026-05-26T00:00:00Z", + payload: { status: "pending" }, + }, + { + eventId: taskEventId, + sampleId, + eventType: "task.added", + targetType: "task", + targetId: taskId, + taskSlug: "prove", + status: "pending", + timestamp: "2026-05-26T00:00:01Z", + payload: { task_slug: "prove" }, + }, + ], +}; - assert.match(source, /fetch\(`\/api\/samples\/\$\{sampleId\}\/events`\)/); - assert.match(source, /parseSampleRuntimeEvents/); - assert.doesNotMatch(source, /\/api\/samples\/\$\{sampleId\}\/mutations/); -}); +export const fixtureSampleGraph: SampleGraphView = { + nodes: [ + { + taskId, + taskSlug: "prove", + description: "Prove the theorem", + status: "pending", + parentTaskId: null, + level: 0, + assignedWorkerSlug: "lean-worker", + createdAt: "2026-05-26T00:00:00Z", + updatedAt: "2026-05-26T00:00:01Z", + }, + ], + edges: [], +}; -test("generated REST contract does not expose sample mutations", () => { - const openapi = readFileSync("src/generated/rest/openapi.json", "utf8"); - const contracts = readFileSync("src/generated/rest/contracts.ts", "utf8"); +test("sample detail contract exposes provenance", () => { + const parsed = parseSampleDetail(fixtureSampleDetail); - assert.doesNotMatch(openapi, /\/samples\/\{sample_id\}\/mutations/); - assert.doesNotMatch(openapi, /GraphMutationRecordDto/); - assert.doesNotMatch(contracts, /GraphMutationRecordDto/); + assert.equal(parsed.sampleId, sampleId); + assert.equal(parsed.environmentName, "mini-validation"); + assert.doesNotMatch(JSON.stringify(parsed), retiredSamplePattern); }); -test("generated REST contract exposes sample runtime events as a discriminated union", () => { - const contracts = readFileSync("src/generated/rest/contracts.ts", "utf8"); +test("sample contract has events not mutations", () => { + const parsed = parseSampleEvents(fixtureSampleEvents); - assert.match(contracts, /const SampleRuntimeEventView = z\.discriminatedUnion\("eventType"/); - assert.match(contracts, /eventType: z\.literal\("task\.added"\)/); - assert.match(contracts, /eventType: z\.literal\("edge\.added"\)/); + assert.equal(parsed.items[0].eventType, "sample.status_changed"); + assert.equal(parsed.items[1].eventType, "task.added"); + assert.doesNotMatch(JSON.stringify(parsed), retiredEventPattern); +}); - const parsed = schemas.SampleRuntimeEventView.parse({ - eventId: edgeId, - sampleId, - timestamp: "2026-05-27T12:00:00Z", - eventType: "edge.added", - targetType: "edge", - targetId: edgeId, - sourceTaskId: taskId, - targetTaskId: "00000000-0000-4000-8000-000000000004", - status: "pending", - payload: {}, - }); +test("sample graph contract exposes projected tasks", () => { + const parsed = parseSampleGraph(fixtureSampleGraph); - assert.equal(parsed.eventType, "edge.added"); - assert.throws(() => - schemas.SampleRuntimeEventView.parse({ - eventId: edgeId, - sampleId, - timestamp: "2026-05-27T12:00:00Z", - eventType: "not.real", - targetType: "edge", - targetId: edgeId, - payload: {}, - }), - ); + assert.equal(parsed.nodes[0].taskSlug, "prove"); + assert.equal(parsed.edges.length, 0); }); diff --git a/ergon-dashboard/tests/contracts/server-data.contract.test.ts b/ergon-dashboard/tests/contracts/server-data.contract.test.ts index 2e6cda9c1..48d63141d 100644 --- a/ergon-dashboard/tests/contracts/server-data.contract.test.ts +++ b/ergon-dashboard/tests/contracts/server-data.contract.test.ts @@ -10,27 +10,31 @@ test("harness miss for experiment is represented as null, not notFound policy", assert.equal(getHarnessExperiment("missing-experiment"), null); }); -test("experiment list server data keeps operational analytics fields", async () => { +test("experiment list server data keeps sample-centered state fields", async () => { const originalFetch = globalThis.fetch; globalThis.fetch = async () => new Response( JSON.stringify([ { - definition_id: "11111111-1111-1111-1111-111111111111", + experimentId: "11111111-1111-1111-1111-111111111111", name: "MiniWob comparison", - benchmark_type: "miniwob", - sample_count: 12, - status: "running", - default_model_target: "openai:gpt-4.1", - default_evaluator_slug: "judge-v1", - created_at: "2026-05-20T12:00:00Z", - run_count: 3, - failure_count: 1, - latest_activity_at: "2026-05-20T12:30:00Z", - average_score: 0.82, - average_duration_ms: 1234, + environments: [ + { + environmentId: "env-1", + environmentName: "mini-validation", + sourceMode: "materialized", + sampleCount: 12, + selectedCount: 3, + sourceMetadata: {}, + }, + ], + sampleCount: 12, + samples: [], + samplerInvocations: [], + metadata: {}, + createdAt: "2026-05-20T12:00:00Z", }, - ]), + ]).replace(/^\[/, "{\"items\":[").replace(/\]$/, "]}"), { status: 200, headers: { "content-type": "application/json" } }, ); @@ -38,10 +42,9 @@ test("experiment list server data keeps operational analytics fields", async () const result = await loadExperimentList(); assert.equal(result.ok, true); - assert.equal(result.ok && result.data[0].failure_count, 1); - assert.equal(result.ok && result.data[0].latest_activity_at, "2026-05-20T12:30:00Z"); - assert.equal(result.ok && result.data[0].average_score, 0.82); - assert.equal(result.ok && result.data[0].average_duration_ms, 1234); + assert.equal(result.ok && result.data[0].experimentId, "11111111-1111-1111-1111-111111111111"); + assert.equal(result.ok && result.data[0].environments[0].environmentName, "mini-validation"); + assert.equal(result.ok && result.data[0].sampleCount, 12); } finally { globalThis.fetch = originalFetch; } diff --git a/ergon-dashboard/tests/unit/experiment-state.test.ts b/ergon-dashboard/tests/unit/experiment-state.test.ts new file mode 100644 index 000000000..44c12a051 --- /dev/null +++ b/ergon-dashboard/tests/unit/experiment-state.test.ts @@ -0,0 +1,15 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { buildExperimentState } from "../../src/lib/sample-state/dashboard"; +import { fixtureExperimentDetail } from "../contracts/experiment-rest-contract.test"; + +const retiredIdentityPattern = new RegExp(["run" + "Id", "definitionId", "mutation"].join("|"), "i"); + +test("experiment state exposes environments and samples without run vocabulary", () => { + const state = buildExperimentState(fixtureExperimentDetail); + + assert.equal(state.environments[0].environmentName, "mini-validation"); + assert.equal(state.samples[0].sampleId, "sample-1"); + assert.doesNotMatch(JSON.stringify(state), retiredIdentityPattern); +}); diff --git a/ergon-dashboard/tests/unit/sample-state.test.ts b/ergon-dashboard/tests/unit/sample-state.test.ts new file mode 100644 index 000000000..1f8aa9ba3 --- /dev/null +++ b/ergon-dashboard/tests/unit/sample-state.test.ts @@ -0,0 +1,22 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { buildSampleState } from "../../src/lib/sample-state/dashboard"; +import { fixtureSampleDetail, fixtureSampleEvents, fixtureSampleGraph } from "../contracts/sample-rest-contract.test"; + +const retiredIdentityPattern = new RegExp(["GraphMutation", "run" + "Id", "definitionId"].join("|"), "i"); + +test("sample state stores typed WAL events and graph projection", () => { + const state = buildSampleState({ + detail: fixtureSampleDetail, + events: fixtureSampleEvents.items, + graph: fixtureSampleGraph, + }); + + assert.deepEqual( + state.events.map((event) => event.eventType), + ["sample.status_changed", "task.added"], + ); + assert.equal(state.graph.nodes[0].taskSlug, "prove"); + assert.doesNotMatch(JSON.stringify(state), retiredIdentityPattern); +}); diff --git a/ergon_core/tests/unit/architecture/test_definition_identity_naming.py b/ergon_core/tests/unit/architecture/test_definition_identity_naming.py index c0653ec83..3de9e4c39 100644 --- a/ergon_core/tests/unit/architecture/test_definition_identity_naming.py +++ b/ergon_core/tests/unit/architecture/test_definition_identity_naming.py @@ -33,6 +33,15 @@ ROOT / "ergon_cli" / "ergon_cli" / "domains" / "experiments", ROOT / "ergon_cli" / "ergon_cli" / "domains" / "samples", ROOT / "ergon_cli" / "tests" / "unit" / "cli", + ROOT / "ergon-dashboard" / "src" / "app" / "experiments", + ROOT / "ergon-dashboard" / "src" / "app" / "samples", + ROOT / "ergon-dashboard" / "src" / "components" / "experiments", + ROOT / "ergon-dashboard" / "src" / "components" / "indexes", + ROOT / "ergon-dashboard" / "src" / "components" / "samples", + ROOT / "ergon-dashboard" / "src" / "lib" / "sample-state", + ROOT / "ergon-dashboard" / "tests" / "components", + ROOT / "ergon-dashboard" / "tests" / "contracts", + ROOT / "ergon-dashboard" / "tests" / "unit", ROOT / "tests" / "examples", ) ALLOWED_EXPERIMENT_ID_FILES = { @@ -54,6 +63,9 @@ / "samples.py", ROOT / "ergon_core" / "ergon_core" / "core" / "persistence" / "telemetry" / "models.py", ROOT / "ergon_core" / "tests" / "unit" / "state" / "test_type_invariants.py", + ROOT / "ergon-dashboard" / "src" / "lib" / "contracts" / "rest.ts", + ROOT / "ergon-dashboard" / "src" / "lib" / "server-data" / "experiments.ts", + ROOT / "ergon-dashboard" / "src" / "lib" / "server-data" / "samples.ts", } ALLOWED_EXPERIMENT_ID_PATTERNS_BY_FILE = { ROOT / "ergon_core" / "ergon_core" / "api" / "experiment" / "experiment.py": (