From a3c65174c45455afc57cee735a3d1286f2775376 Mon Sep 17 00:00:00 2001 From: catoncat Date: Fri, 3 Jul 2026 13:53:21 +0800 Subject: [PATCH] =?UTF-8?q?feat(eval):=20=E6=94=AF=E6=8C=81=20dogfood=20an?= =?UTF-8?q?swer=20facets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eval/acceptance-gate.test.ts | 5 ++ eval/acceptance-gate.ts | 28 ++++---- eval/dogfood-eval-core.test.ts | 53 +++++++++++++++ eval/dogfood-eval-core.ts | 116 +++++++++++++++++++++++++++++++-- eval/dogfood-schema.ts | 49 +++++++++++++- eval/perf-bench.ts | 8 +++ eval/run-dogfood-eval.ts | 54 ++++++++------- 7 files changed, 271 insertions(+), 42 deletions(-) diff --git a/eval/acceptance-gate.test.ts b/eval/acceptance-gate.test.ts index 5375080..a247d08 100644 --- a/eval/acceptance-gate.test.ts +++ b/eval/acceptance-gate.test.ts @@ -13,6 +13,10 @@ describe("acceptance gate", () => { pass: 5, fail: 0, hardFail: 0, + assertionPass: 5, + assertionFail: 0, + facetPass: 1, + facetFail: 0, }); expect(result.rows.map((row) => row.id)).toEqual([ "message-hit-context", @@ -22,5 +26,6 @@ describe("acceptance gate", () => { "pi-session-page-context", ]); expect(result.rows.every((row) => row.predicates.length > 0)).toBe(true); + expect(result.rows.find((row) => row.id === "message-hit-context")?.facetMark).toBe("pass"); }); }); diff --git a/eval/acceptance-gate.ts b/eval/acceptance-gate.ts index d58954a..2d4d9d0 100644 --- a/eval/acceptance-gate.ts +++ b/eval/acceptance-gate.ts @@ -1,7 +1,7 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { desiredContextMode, evaluateDogfoodItem, type DogfoodEvaluation } from "./dogfood-eval-core"; +import { buildDogfoodScoreboard, desiredContextMode, evaluateDogfoodItem, type DogfoodEvaluation, type DogfoodScoreboard } from "./dogfood-eval-core"; import type { DogfoodGolden } from "./dogfood-schema"; import { syncSessions } from "../src/indexer"; import { findSessions, getMessagePage, getMessageRange } from "../src/query"; @@ -33,6 +33,9 @@ export interface AcceptanceGateRow { selectedMatchSource: FindResult["matchSource"] | null; selectedMatchSeq: number | null; contextKind?: "read-range" | "read-page"; + assertionMark: DogfoodEvaluation["assertionMark"]; + facetMark: DogfoodEvaluation["facetMark"]; + failureClasses: DogfoodEvaluation["failureClasses"]; predicates: DogfoodEvaluation["predicateResults"]; } @@ -42,7 +45,7 @@ export interface AcceptanceGateResult { dbPath: string; sync: SyncSummary; sourceSyncs: Record; - scoreboard: Record<"total" | "pass" | "fail" | "skip" | "hardFail" | "candidateFail", number>; + scoreboard: DogfoodScoreboard; rows: AcceptanceGateRow[]; } @@ -71,7 +74,7 @@ export async function runAcceptanceGate(options: AcceptanceGateOptions = {}): Pr dbPath, sync: sourceSyncs.codex, sourceSyncs, - scoreboard: buildScoreboard(rows), + scoreboard: buildDogfoodScoreboard(rows.map((row) => ({ status: row.status, evaluation: row }))), rows, }; } finally { @@ -108,6 +111,9 @@ function evaluateAcceptanceItems(dbPath: string, items: DogfoodGolden[]): Accept selectedMatchSource: evaluation.selected.hit?.matchSource ?? null, selectedMatchSeq: evaluation.selected.hit?.matchSeq ?? null, ...(context.kind ? { contextKind: context.kind } : {}), + assertionMark: evaluation.assertionMark, + facetMark: evaluation.facetMark, + failureClasses: evaluation.failureClasses, predicates: evaluation.predicateResults, }; }); @@ -149,16 +155,6 @@ function readContextIfNeeded( return { kind: "read-page", text: messagesText(page.messages) }; } -function buildScoreboard(rows: AcceptanceGateRow[]): AcceptanceGateResult["scoreboard"] { - const scoreboard = { total: rows.length, pass: 0, fail: 0, skip: 0, hardFail: 0, candidateFail: 0 }; - for (const row of rows) { - scoreboard[row.mark] += 1; - if (row.status === "hard" && row.mark === "fail") scoreboard.hardFail += 1; - if (row.status === "candidate" && row.mark === "fail") scoreboard.candidateFail += 1; - } - return scoreboard; -} - function acceptanceGoldens(roots: AcceptanceFixtureRoots): DogfoodGolden[] { return [ { @@ -180,6 +176,12 @@ function acceptanceGoldens(roots: AcceptanceFixtureRoots): DogfoodGolden[] { after: 1, mustContain: ["health check returned 500", "rollback plan includes readback verification"], }, + answerFacets: [ + { + label: "failure symptom and mitigation evidence", + mustContain: ["health check returned 500", "rollback plan includes readback verification"], + }, + ], }, }, { diff --git a/eval/dogfood-eval-core.test.ts b/eval/dogfood-eval-core.test.ts index 008e747..ad32dcc 100644 --- a/eval/dogfood-eval-core.test.ts +++ b/eval/dogfood-eval-core.test.ts @@ -44,6 +44,52 @@ describe("dogfood eval core", () => { expect(evaluation.mark).toBe("pass"); }); + test("passes answer facets against retrieved evidence separately from assertions", () => { + const evaluation = evaluateDogfoodItem({ + item: golden({ + acceptableSessionUuids: ["session-a"], + answerFacets: [{ + label: "install order", + mustContain: ["Install official Cursor", "install ccursor"], + }], + }), + results: [findResult({ sessionUuid: "session-a" })], + contextKind: "read-range", + contextText: "Install official Cursor, log in, then install ccursor.", + }); + + expect(evaluation.mark).toBe("pass"); + expect(evaluation.assertionMark).toBe("pass"); + expect(evaluation.facetMark).toBe("pass"); + expect(evaluation.predicateResults.map((predicate) => `${predicate.group}.${predicate.label}`)).toEqual([ + "assertion.session_uuid", + "answer_facet.answer_facet", + ]); + expect(evaluation.predicateResults.find((predicate) => predicate.group === "answer_facet")?.facetLabel).toBe("install order"); + }); + + test("fails answer facets with a failure classification while candidate remains non-blocking", () => { + const evaluation = evaluateDogfoodItem({ + item: golden({ + status: "candidate", + answerFacets: [{ + label: "decision", + mustContain: ["chosen sqlite dbstat path"], + failureClass: "skill_guidance", + }], + }), + results: [findResult({ sessionUuid: "session-a" })], + contextKind: "read-range", + contextText: "The evidence mentions latency only.", + }); + + expect(evaluation.mark).toBe("fail"); + expect(evaluation.blocking).toBe(false); + expect(evaluation.assertionMark).toBe("skip"); + expect(evaluation.facetMark).toBe("fail"); + expect(evaluation.failureClasses).toEqual(["skill_guidance"]); + }); + test("checks source id, session ref, and nullable match sequence", () => { const evaluation = evaluateDogfoodItem({ item: golden({ sourceId: "codex", sessionRef: "session-a", matchSource: "session", matchSeq: null }), @@ -91,6 +137,11 @@ describe("dogfood eval core", () => { acceptableSessionUuids: ["target-session"], sessionRef: "target-session", matchSeq: null, + answerFacets: [{ + label: "remembered file", + mustContain: ["SKILL.md"], + failureClass: "cli_recall_ranking_context", + }], }, }), "goldens.local.jsonl"); @@ -115,6 +166,7 @@ function golden( matchSeq: number | null; topK: number; contextMustContain: string[]; + answerFacets: Array<{ label: string; mustContain: string[]; failureClass?: "coverage_index" | "skill_guidance" | "cli_recall_ranking_context" | "stale_golden" | "unclear_case" }>; }> = {}, ): DogfoodGolden { return { @@ -130,6 +182,7 @@ function golden( matchSource: overrides.matchSource, matchSeq: overrides.matchSeq, context: overrides.contextMustContain ? { mustContain: overrides.contextMustContain } : undefined, + answerFacets: overrides.answerFacets, }, }; } diff --git a/eval/dogfood-eval-core.ts b/eval/dogfood-eval-core.ts index 22f5d7f..7e1cc40 100644 --- a/eval/dogfood-eval-core.ts +++ b/eval/dogfood-eval-core.ts @@ -1,13 +1,17 @@ import type { FindResult } from "../src/types"; -import type { DogfoodGolden } from "./dogfood-schema"; +import type { DogfoodFailureClass, DogfoodGolden, DogfoodStatus } from "./dogfood-schema"; export type DogfoodMark = "pass" | "fail" | "skip"; +export type DogfoodPredicateGroup = "assertion" | "answer_facet"; export interface DogfoodPredicateResult { - label: "source_id" | "session_uuid" | "session_ref" | "cwd" | "match_source" | "match_seq" | "context"; + label: "source_id" | "session_uuid" | "session_ref" | "cwd" | "match_source" | "match_seq" | "context" | "answer_facet"; + group: DogfoodPredicateGroup; + facetLabel?: string; expected: string; actual: string; matched: boolean; + failureClass?: DogfoodFailureClass; } export interface SelectedDogfoodHit { @@ -29,6 +33,22 @@ export interface DogfoodEvaluation { blocking: boolean; selected: SelectedDogfoodHit; predicateResults: DogfoodPredicateResult[]; + assertionMark: DogfoodMark; + facetMark: DogfoodMark; + failureClasses: DogfoodFailureClass[]; +} + +export interface DogfoodScoreboard { + total: number; + pass: number; + fail: number; + skip: number; + hardFail: number; + candidateFail: number; + assertionPass: number; + assertionFail: number; + facetPass: number; + facetFail: number; } export function evaluateDogfoodItem(input: DogfoodEvaluationInput): DogfoodEvaluation { @@ -38,18 +58,27 @@ export function evaluateDogfoodItem(input: DogfoodEvaluationInput): DogfoodEvalu blocking: false, selected: { hit: null, rank: null, topK: input.item.expected.topK ?? 5 }, predicateResults: [], + assertionMark: "skip", + facetMark: "skip", + failureClasses: ["stale_golden"], }; } const selected = selectDogfoodHit(input.item, input.results); const predicates = buildPredicates(input, selected); const mark = predicates.length > 0 && predicates.every((predicate) => predicate.matched) ? "pass" : "fail"; + const assertionMark = groupMark(predicates, "assertion"); + const facetMark = groupMark(predicates, "answer_facet"); + const failureClasses = uniqueFailureClasses(predicates); return { mark, blocking: input.item.status === "hard" && mark === "fail", selected, predicateResults: predicates, + assertionMark, + facetMark, + failureClasses, }; } @@ -67,8 +96,8 @@ export function selectDogfoodHit(item: DogfoodGolden, results: FindResult[]): Se export function desiredContextMode(item: DogfoodGolden, hit: FindResult | null): "read-range" | "read-page" | null { const context = item.expected.context; - if (!context?.mustContain?.length) return null; - const mode = context.mode ?? "auto"; + if (!context?.mustContain?.length && !item.expected.answerFacets?.length) return null; + const mode = context?.mode ?? "auto"; if (mode !== "auto") return mode; // Session-only hits now use read-range --query to locate the real anchor, // so auto mode always prefers read-range. @@ -77,7 +106,32 @@ export function desiredContextMode(item: DogfoodGolden, hit: FindResult | null): export function missingContextNeedles(item: DogfoodGolden, contextText: string): string[] { const haystack = contextText.toLowerCase(); - return (item.expected.context?.mustContain ?? []).filter((needle) => !haystack.includes(needle.toLowerCase())); + return expectedEvidenceNeedles(item).filter((needle) => !haystack.includes(needle.toLowerCase())); +} + +export function buildDogfoodScoreboard(rows: Array<{ status: DogfoodStatus; evaluation: Pick }>): DogfoodScoreboard { + const scoreboard: DogfoodScoreboard = { + total: rows.length, + pass: 0, + fail: 0, + skip: 0, + hardFail: 0, + candidateFail: 0, + assertionPass: 0, + assertionFail: 0, + facetPass: 0, + facetFail: 0, + }; + for (const row of rows) { + scoreboard[row.evaluation.mark] += 1; + if (row.status === "hard" && row.evaluation.mark === "fail") scoreboard.hardFail += 1; + if (row.status === "candidate" && row.evaluation.mark === "fail") scoreboard.candidateFail += 1; + if (row.evaluation.assertionMark === "pass") scoreboard.assertionPass += 1; + if (row.evaluation.assertionMark === "fail") scoreboard.assertionFail += 1; + if (row.evaluation.facetMark === "pass") scoreboard.facetPass += 1; + if (row.evaluation.facetMark === "fail") scoreboard.facetFail += 1; + } + return scoreboard; } function buildPredicates( @@ -92,27 +146,33 @@ function buildPredicates( if (item.expected.sourceId) { predicates.push({ label: "source_id", + group: "assertion", expected: item.expected.sourceId, actual: hit?.sourceId ?? "no selected hit", matched: hit?.sourceId === item.expected.sourceId, + failureClass: hit ? "cli_recall_ranking_context" : "coverage_index", }); } if (acceptable.length > 0) { predicates.push({ label: "session_uuid", + group: "assertion", expected: `one of ${acceptable.join(", ")} in top ${selected.topK}`, actual: hit ? `${hit.sessionUuid} at rank ${selected.rank}` : "no results", matched: Boolean(hit && acceptable.includes(hit.sessionUuid) && (selected.rank ?? Infinity) <= selected.topK), + failureClass: hit ? "cli_recall_ranking_context" : "coverage_index", }); } if (item.expected.sessionRef) { predicates.push({ label: "session_ref", + group: "assertion", expected: item.expected.sessionRef, actual: hit?.sessionRef ?? "no selected hit", matched: hit?.sessionRef === item.expected.sessionRef, + failureClass: hit ? "cli_recall_ranking_context" : "coverage_index", }); } @@ -120,27 +180,33 @@ function buildPredicates( const needle = item.expected.cwdContains.toLowerCase(); predicates.push({ label: "cwd", + group: "assertion", expected: item.expected.cwdContains, actual: hit?.cwd ?? "no selected hit", matched: Boolean(hit?.cwd.toLowerCase().includes(needle)), + failureClass: hit ? "cli_recall_ranking_context" : "coverage_index", }); } if (item.expected.matchSource) { predicates.push({ label: "match_source", + group: "assertion", expected: item.expected.matchSource, actual: hit?.matchSource ?? "no selected hit", matched: hit?.matchSource === item.expected.matchSource, + failureClass: hit ? "cli_recall_ranking_context" : "coverage_index", }); } if (item.expected.matchSeq !== undefined) { predicates.push({ label: "match_seq", + group: "assertion", expected: String(item.expected.matchSeq), actual: hit ? String(hit.matchSeq) : "no selected hit", matched: hit?.matchSeq === item.expected.matchSeq, + failureClass: hit ? "cli_recall_ranking_context" : "coverage_index", }); } @@ -148,15 +214,55 @@ function buildPredicates( const haystack = input.contextText ?? ""; predicates.push({ label: "context", + group: "assertion", expected: needle, actual: input.contextUnavailableReason ?? contextActual(input.contextKind, haystack), matched: haystack.toLowerCase().includes(needle.toLowerCase()), + failureClass: contextFailureClass(input), + }); + } + + for (const facet of item.expected.answerFacets ?? []) { + const haystack = input.contextText ?? ""; + const missing = facet.mustContain.filter((needle) => !haystack.toLowerCase().includes(needle.toLowerCase())); + predicates.push({ + label: "answer_facet", + group: "answer_facet", + facetLabel: facet.label, + expected: `${facet.label}: ${facet.mustContain.join(" + ")}`, + actual: input.contextUnavailableReason ?? (missing.length > 0 ? `missing: ${missing.join(", ")}` : contextActual(input.contextKind, haystack)), + matched: missing.length === 0, + failureClass: facet.failureClass ?? contextFailureClass(input), }); } return predicates; } +function expectedEvidenceNeedles(item: DogfoodGolden): string[] { + return [ + ...(item.expected.context?.mustContain ?? []), + ...(item.expected.answerFacets ?? []).flatMap((facet) => facet.mustContain), + ]; +} + +function groupMark(predicates: DogfoodPredicateResult[], group: DogfoodPredicateGroup): DogfoodMark { + const groupPredicates = predicates.filter((predicate) => predicate.group === group); + if (groupPredicates.length === 0) return "skip"; + return groupPredicates.every((predicate) => predicate.matched) ? "pass" : "fail"; +} + +function uniqueFailureClasses(predicates: DogfoodPredicateResult[]): DogfoodFailureClass[] { + return [...new Set(predicates.filter((predicate) => !predicate.matched).map((predicate) => predicate.failureClass ?? "unclear_case"))]; +} + +function contextFailureClass(input: DogfoodEvaluationInput): DogfoodFailureClass { + if (input.contextUnavailableReason?.includes("no selected hit") || input.contextUnavailableReason?.includes("no results")) { + return "coverage_index"; + } + return "cli_recall_ranking_context"; +} + function contextActual(kind: string | undefined, text: string): string { if (!text) return kind ? `${kind}: empty context` : "context not read"; return `${kind ?? "context"}: ${text.length} chars`; diff --git a/eval/dogfood-schema.ts b/eval/dogfood-schema.ts index a125210..61fde04 100644 --- a/eval/dogfood-schema.ts +++ b/eval/dogfood-schema.ts @@ -4,6 +4,12 @@ import { isSessionSourceId, type FindSort, type MatchSource, type Selector, type export type DogfoodStatus = "candidate" | "hard" | "stale"; export type DogfoodOriginKind = "observed-user-ask" | "evidence-backed-derived" | "manual"; export type DogfoodContextMode = "auto" | "read-range" | "read-page"; +export type DogfoodFailureClass = + | "coverage_index" + | "skill_guidance" + | "cli_recall_ranking_context" + | "stale_golden" + | "unclear_case"; export interface DogfoodOrigin { kind: DogfoodOriginKind; @@ -22,6 +28,12 @@ export interface DogfoodExpectedContext { mustContain?: string[]; } +export interface DogfoodExpectedAnswerFacet { + label: string; + mustContain: string[]; + failureClass?: DogfoodFailureClass; +} + export interface DogfoodExpected { topK?: number; sourceId?: SessionSourceId; @@ -31,6 +43,7 @@ export interface DogfoodExpected { matchSource?: MatchSource; matchSeq?: number | null; context?: DogfoodExpectedContext; + answerFacets?: DogfoodExpectedAnswerFacet[]; } export interface DogfoodFindOptions { @@ -152,6 +165,9 @@ function parseExpected(value: unknown): DogfoodExpected | null { const context = parseContext(value.context); if (context) expected.context = context; + const answerFacets = parseAnswerFacets(value.answerFacets); + if (answerFacets === "invalid") return null; + if (answerFacets) expected.answerFacets = answerFacets; return hasExpectedAssertion(expected) ? expected : null; } @@ -178,6 +194,28 @@ function parseContext(value: unknown): DogfoodExpectedContext | undefined { return Object.keys(context).length > 0 ? context : undefined; } +function parseAnswerFacets(value: unknown): DogfoodExpectedAnswerFacet[] | undefined | "invalid" { + if (value === undefined) return undefined; + if (!Array.isArray(value)) return "invalid"; + + const facets: DogfoodExpectedAnswerFacet[] = []; + for (const item of value) { + if (!isRecord(item)) return "invalid"; + const label = readNonEmptyString(item, "label"); + const mustContain = readStringArray(item.mustContain); + if (!label || !mustContain) return "invalid"; + + const facet: DogfoodExpectedAnswerFacet = { label, mustContain }; + if (item.failureClass !== undefined) { + if (!isDogfoodFailureClass(item.failureClass)) return "invalid"; + facet.failureClass = item.failureClass; + } + facets.push(facet); + } + + return facets.length > 0 ? facets : undefined; +} + function parseFindOptions(value: unknown): DogfoodFindOptions | undefined | "invalid" { if (value === undefined) return undefined; if (!isRecord(value)) return "invalid"; @@ -243,10 +281,19 @@ function hasExpectedAssertion(expected: DogfoodExpected): boolean { || Boolean(expected.cwdContains) || Boolean(expected.matchSource) || expected.matchSeq !== undefined - || expected.context?.mustContain?.length, + || expected.context?.mustContain?.length + || expected.answerFacets?.length ); } +function isDogfoodFailureClass(value: unknown): value is DogfoodFailureClass { + return value === "coverage_index" + || value === "skill_guidance" + || value === "cli_recall_ranking_context" + || value === "stale_golden" + || value === "unclear_case"; +} + function readNonEmptyString(record: Record, key: string): string | undefined { const value = record[key]; return typeof value === "string" && value.trim() ? value.trim() : undefined; diff --git a/eval/perf-bench.ts b/eval/perf-bench.ts index 3e9155c..9192814 100644 --- a/eval/perf-bench.ts +++ b/eval/perf-bench.ts @@ -76,6 +76,10 @@ interface DogfoodScoreboard { skip: number; hardFail: number; candidateFail: number; + assertionPass: number; + assertionFail: number; + facetPass: number; + facetFail: number; } interface DogfoodScorecardSummary { @@ -637,6 +641,10 @@ function parseDogfoodStdout(stdout: string): { outDir?: string; scorecard?: stri skip: Number(scoreboard.skip) || 0, hardFail: Number(scoreboard.hardFail) || 0, candidateFail: Number(scoreboard.candidateFail) || 0, + assertionPass: Number(scoreboard.assertionPass) || 0, + assertionFail: Number(scoreboard.assertionFail) || 0, + facetPass: Number(scoreboard.facetPass) || 0, + facetFail: Number(scoreboard.facetFail) || 0, }, }; } catch { diff --git a/eval/run-dogfood-eval.ts b/eval/run-dogfood-eval.ts index e416dd6..0f2bc31 100644 --- a/eval/run-dogfood-eval.ts +++ b/eval/run-dogfood-eval.ts @@ -3,7 +3,7 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { spawn as childSpawn } from "node:child_process"; import { basename, join, resolve } from "node:path"; -import { desiredContextMode, evaluateDogfoodItem, missingContextNeedles, type DogfoodEvaluation } from "./dogfood-eval-core"; +import { buildDogfoodScoreboard, desiredContextMode, evaluateDogfoodItem, missingContextNeedles, type DogfoodEvaluation, type DogfoodScoreboard } from "./dogfood-eval-core"; import { parseDogfoodJsonl, type DogfoodGolden } from "./dogfood-schema"; import { DEFAULT_CODEX_DIR } from "../src/env"; import type { FindResult, FindSort, Selector } from "../src/types"; @@ -88,7 +88,7 @@ for (const [index, item] of entries.entries()) { }); } -const scoreboard = buildScoreboard(rows); +const scoreboard = buildDogfoodScoreboard(rows.map((row) => ({ status: row.item.status, evaluation: row.evaluation }))); const readmePath = join(outDir, "README.md"); const scorecardPath = join(outDir, "scorecard.json"); writeFileSync(readmePath, renderReadme(args.goldenPath, scoreboard, rows)); @@ -181,6 +181,9 @@ function emptyEvaluation(item: DogfoodGolden): DogfoodEvaluation { blocking: item.status === "hard", selected: { hit: null, rank: null, topK: item.expected.topK ?? 5 }, predicateResults: [], + assertionMark: item.status === "stale" ? "skip" : "fail", + facetMark: item.expected.answerFacets?.length ? "fail" : "skip", + failureClasses: item.status === "stale" ? ["stale_golden"] : ["coverage_index"], }; } @@ -194,9 +197,6 @@ async function readContextIfNeeded( const mode = desiredContextMode(item, hit); if (!mode) return {}; if (!hit) return { unavailableReason: "no selected hit for context read" }; - if (mode === "read-range" && typeof hit.matchSeq !== "number") { - return { kind: "read-range", unavailableReason: "selected hit has no numeric matchSeq" }; - } let command = buildContextCommand(item, hit, mode); let contextJson = await runCommand([...command, "--json"]); @@ -231,10 +231,13 @@ function buildContextCommand( ): string[] { const context = item.expected.context ?? {}; if (mode === "read-range") { + const anchorArgs = typeof hit.matchSeq === "number" + ? ["--seq", String(hit.matchSeq)] + : ["--query", context.query ?? item.query]; return [ process.execPath, "--import", "tsx", CLI_ENTRY, - "read-range", hit.sessionUuid, - "--seq", String(hit.matchSeq), + "read-range", hit.sessionRef, + ...anchorArgs, "--before", String(context.before ?? defaultWindow.before), "--after", String(context.after ?? defaultWindow.after), ]; @@ -242,25 +245,15 @@ function buildContextCommand( return [ process.execPath, "--import", "tsx", CLI_ENTRY, - "read-page", hit.sessionUuid, + "read-page", hit.sessionRef, "--offset", String(context.offset ?? 0), "--limit", String(context.limit ?? 20), ]; } -function buildScoreboard(rows: Array<{ item: DogfoodGolden; evaluation: DogfoodEvaluation }>): Record { - const scoreboard = { total: rows.length, pass: 0, fail: 0, skip: 0, hardFail: 0, candidateFail: 0 }; - for (const row of rows) { - scoreboard[row.evaluation.mark] += 1; - if (row.item.status === "hard" && row.evaluation.mark === "fail") scoreboard.hardFail += 1; - if (row.item.status === "candidate" && row.evaluation.mark === "fail") scoreboard.candidateFail += 1; - } - return scoreboard; -} - function renderReadme( sourcePath: string, - scoreboard: Record, + scoreboard: DogfoodScoreboard, rows: DogfoodEvalRow[], ): string { const lines = [ @@ -278,13 +271,17 @@ function renderReadme( `- skip: ${scoreboard.skip}`, `- hard_fail: ${scoreboard.hardFail}`, `- candidate_fail: ${scoreboard.candidateFail}`, + `- assertion_pass: ${scoreboard.assertionPass}`, + `- assertion_fail: ${scoreboard.assertionFail}`, + `- facet_pass: ${scoreboard.facetPass}`, + `- facet_fail: ${scoreboard.facetFail}`, "", - "| id | status | mark | blocking | selected_rank | selected_title |", - "|----|--------|------|----------|---------------|----------------|", + "| id | status | mark | assertions | facets | failure_classes | blocking | selected_rank | selected_title |", + "|----|--------|------|------------|--------|-----------------|----------|---------------|----------------|", ]; for (const row of rows) { - lines.push(`| ${row.item.id} | ${row.item.status} | ${row.evaluation.mark} | ${row.evaluation.blocking} | ${row.evaluation.selected.rank ?? "-"} | ${row.selectedTitle.replaceAll("|", "¦").slice(0, 60)} |`); + lines.push(`| ${row.item.id} | ${row.item.status} | ${row.evaluation.mark} | ${row.evaluation.assertionMark} | ${row.evaluation.facetMark} | ${formatFailureClasses(row.evaluation.failureClasses)} | ${row.evaluation.blocking} | ${row.evaluation.selected.rank ?? "-"} | ${row.selectedTitle.replaceAll("|", "¦").slice(0, 60)} |`); } for (const row of rows) { @@ -295,6 +292,9 @@ function renderReadme( lines.push(`- selected_attempt: ${row.selectedAttemptOrdinal ?? "-"} / \`${row.selectedAttemptQuery}\``); lines.push(`- status: ${row.item.status}`); lines.push(`- mark: ${row.evaluation.mark}`); + lines.push(`- assertions: ${row.evaluation.assertionMark}`); + lines.push(`- answer_facets: ${row.evaluation.facetMark}`); + lines.push(`- failure_classes: ${formatFailureClasses(row.evaluation.failureClasses)}`); lines.push(`- top1_title: ${row.top1Title}`); lines.push(`- selected_title: ${row.selectedTitle}`); lines.push(`- find_json: \`${rel(row.findJsonPath)}\``); @@ -313,7 +313,15 @@ function renderReadme( function formatPredicates(predicates: DogfoodEvaluation["predicateResults"]): string { if (predicates.length === 0) return "(none)"; - return predicates.map((predicate) => `${predicate.label}=${predicate.matched ? "ok" : "miss"}(${predicate.expected})`).join(", "); + return predicates.map((predicate) => { + const status = predicate.matched ? "ok" : `miss:${predicate.failureClass ?? "unclear_case"}`; + const label = predicate.group === "answer_facet" && predicate.facetLabel ? `answer_facet:${predicate.facetLabel}` : `${predicate.group}.${predicate.label}`; + return `${label}=${status}(${predicate.expected})`; + }).join(", "); +} + +function formatFailureClasses(classes: DogfoodEvaluation["failureClasses"]): string { + return classes.length > 0 ? classes.join(",") : "-"; } function rel(path: string): string {