[codex] #88 支持 dogfood answer facets

eval/acceptance-gate.test.ts

@@ -13,6 +13,10 @@ describe("acceptance gate", () => {
       pass: 5,
       fail: 0,
       hardFail: 0,
+      assertionPass: 5,
+      assertionFail: 0,
+      facetPass: 1,
+      facetFail: 0,
     });
     expect(result.rows.map((row) => row.id)).toEqual([
       "message-hit-context",
@@ -22,5 +26,6 @@ describe("acceptance gate", () => {
       "pi-session-page-context",
     ]);
     expect(result.rows.every((row) => row.predicates.length > 0)).toBe(true);
+    expect(result.rows.find((row) => row.id === "message-hit-context")?.facetMark).toBe("pass");
   });
 });

eval/acceptance-gate.ts

@@ -1,7 +1,7 @@
 import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { desiredContextMode, evaluateDogfoodItem, type DogfoodEvaluation } from "./dogfood-eval-core";
+import { buildDogfoodScoreboard, desiredContextMode, evaluateDogfoodItem, type DogfoodEvaluation, type DogfoodScoreboard } from "./dogfood-eval-core";
 import type { DogfoodGolden } from "./dogfood-schema";
 import { syncSessions } from "../src/indexer";
 import { findSessions, getMessagePage, getMessageRange } from "../src/query";
@@ -33,6 +33,9 @@ export interface AcceptanceGateRow {
   selectedMatchSource: FindResult["matchSource"] | null;
   selectedMatchSeq: number | null;
   contextKind?: "read-range" | "read-page";
+  assertionMark: DogfoodEvaluation["assertionMark"];
+  facetMark: DogfoodEvaluation["facetMark"];
+  failureClasses: DogfoodEvaluation["failureClasses"];
   predicates: DogfoodEvaluation["predicateResults"];
 }
 
@@ -42,7 +45,7 @@ export interface AcceptanceGateResult {
   dbPath: string;
   sync: SyncSummary;
   sourceSyncs: Record<AcceptanceSourceId, SyncSummary>;
-  scoreboard: Record<"total" | "pass" | "fail" | "skip" | "hardFail" | "candidateFail", number>;
+  scoreboard: DogfoodScoreboard;
   rows: AcceptanceGateRow[];
 }
 
@@ -71,7 +74,7 @@ export async function runAcceptanceGate(options: AcceptanceGateOptions = {}): Pr
       dbPath,
       sync: sourceSyncs.codex,
       sourceSyncs,
-      scoreboard: buildScoreboard(rows),
+      scoreboard: buildDogfoodScoreboard(rows.map((row) => ({ status: row.status, evaluation: row }))),
       rows,
     };
   } finally {
@@ -108,6 +111,9 @@ function evaluateAcceptanceItems(dbPath: string, items: DogfoodGolden[]): Accept
       selectedMatchSource: evaluation.selected.hit?.matchSource ?? null,
       selectedMatchSeq: evaluation.selected.hit?.matchSeq ?? null,
       ...(context.kind ? { contextKind: context.kind } : {}),
+      assertionMark: evaluation.assertionMark,
+      facetMark: evaluation.facetMark,
+      failureClasses: evaluation.failureClasses,
       predicates: evaluation.predicateResults,
     };
   });
@@ -149,16 +155,6 @@ function readContextIfNeeded(
   return { kind: "read-page", text: messagesText(page.messages) };
 }
 
-function buildScoreboard(rows: AcceptanceGateRow[]): AcceptanceGateResult["scoreboard"] {
-  const scoreboard = { total: rows.length, pass: 0, fail: 0, skip: 0, hardFail: 0, candidateFail: 0 };
-  for (const row of rows) {
-    scoreboard[row.mark] += 1;
-    if (row.status === "hard" && row.mark === "fail") scoreboard.hardFail += 1;
-    if (row.status === "candidate" && row.mark === "fail") scoreboard.candidateFail += 1;
-  }
-  return scoreboard;
-}
-
 function acceptanceGoldens(roots: AcceptanceFixtureRoots): DogfoodGolden[] {
   return [
     {
@@ -180,6 +176,12 @@ function acceptanceGoldens(roots: AcceptanceFixtureRoots): DogfoodGolden[] {
           after: 1,
           mustContain: ["health check returned 500", "rollback plan includes readback verification"],
         },
+        answerFacets: [
+          {
+            label: "failure symptom and mitigation evidence",
+            mustContain: ["health check returned 500", "rollback plan includes readback verification"],
+          },
+        ],
       },
     },
     {

eval/dogfood-eval-core.test.ts

@@ -44,6 +44,52 @@ describe("dogfood eval core", () => {
     expect(evaluation.mark).toBe("pass");
   });
 
+  test("passes answer facets against retrieved evidence separately from assertions", () => {
+    const evaluation = evaluateDogfoodItem({
+      item: golden({
+        acceptableSessionUuids: ["session-a"],
+        answerFacets: [{
+          label: "install order",
+          mustContain: ["Install official Cursor", "install ccursor"],
+        }],
+      }),
+      results: [findResult({ sessionUuid: "session-a" })],
+      contextKind: "read-range",
+      contextText: "Install official Cursor, log in, then install ccursor.",
+    });
+
+    expect(evaluation.mark).toBe("pass");
+    expect(evaluation.assertionMark).toBe("pass");
+    expect(evaluation.facetMark).toBe("pass");
+    expect(evaluation.predicateResults.map((predicate) => `${predicate.group}.${predicate.label}`)).toEqual([
+      "assertion.session_uuid",
+      "answer_facet.answer_facet",
+    ]);
+    expect(evaluation.predicateResults.find((predicate) => predicate.group === "answer_facet")?.facetLabel).toBe("install order");
+  });
+
+  test("fails answer facets with a failure classification while candidate remains non-blocking", () => {
+    const evaluation = evaluateDogfoodItem({
+      item: golden({
+        status: "candidate",
+        answerFacets: [{
+          label: "decision",
+          mustContain: ["chosen sqlite dbstat path"],
+          failureClass: "skill_guidance",
+        }],
+      }),
+      results: [findResult({ sessionUuid: "session-a" })],
+      contextKind: "read-range",
+      contextText: "The evidence mentions latency only.",
+    });
+
+    expect(evaluation.mark).toBe("fail");
+    expect(evaluation.blocking).toBe(false);
+    expect(evaluation.assertionMark).toBe("skip");
+    expect(evaluation.facetMark).toBe("fail");
+    expect(evaluation.failureClasses).toEqual(["skill_guidance"]);
+  });
+
   test("checks source id, session ref, and nullable match sequence", () => {
     const evaluation = evaluateDogfoodItem({
       item: golden({ sourceId: "codex", sessionRef: "session-a", matchSource: "session", matchSeq: null }),
@@ -91,6 +137,11 @@ describe("dogfood eval core", () => {
         acceptableSessionUuids: ["target-session"],
         sessionRef: "target-session",
         matchSeq: null,
+        answerFacets: [{
+          label: "remembered file",
+          mustContain: ["SKILL.md"],
+          failureClass: "cli_recall_ranking_context",
+        }],
       },
     }), "goldens.local.jsonl");
 
@@ -115,6 +166,7 @@ function golden(
     matchSeq: number | null;
     topK: number;
     contextMustContain: string[];
+    answerFacets: Array<{ label: string; mustContain: string[]; failureClass?: "coverage_index" | "skill_guidance" | "cli_recall_ranking_context" | "stale_golden" | "unclear_case" }>;
   }> = {},
 ): DogfoodGolden {
   return {
@@ -130,6 +182,7 @@ function golden(
       matchSource: overrides.matchSource,
       matchSeq: overrides.matchSeq,
       context: overrides.contextMustContain ? { mustContain: overrides.contextMustContain } : undefined,
+      answerFacets: overrides.answerFacets,
     },
   };
 }