diff --git a/eval/acceptance-gate.test.ts b/eval/acceptance-gate.test.ts index 61926b0..a4ed8de 100644 --- a/eval/acceptance-gate.test.ts +++ b/eval/acceptance-gate.test.ts @@ -6,9 +6,11 @@ describe("acceptance gate", () => { const result = await runAcceptanceGate(); expect(result.sync.added).toBe(6); + expect(result.sourceSyncs["claude-code"].added).toBe(1); + expect(result.sourceSyncs.pi.added).toBe(1); expect(result.scoreboard).toMatchObject({ - total: 4, - pass: 4, + total: 6, + pass: 6, fail: 0, hardFail: 0, }); @@ -17,6 +19,8 @@ describe("acceptance gate", () => { "session-only-compact-context", "cjk-message-hit", "exact-query-profile-phrase", + "claude-code-message-range-context", + "pi-session-page-context", ]); expect(result.rows.every((row) => row.predicates.length > 0)).toBe(true); }); diff --git a/eval/acceptance-gate.ts b/eval/acceptance-gate.ts index 3858fb4..b978eb8 100644 --- a/eval/acceptance-gate.ts +++ b/eval/acceptance-gate.ts @@ -13,6 +13,12 @@ const CJK_HIT_SESSION = "33333333-3333-4333-8333-333333333333"; const NOISE_SESSION = "44444444-4444-4444-8444-444444444444"; const EXACT_PHRASE_SESSION = "55555555-5555-4555-8555-555555555555"; const SPLIT_METADATA_SESSION = "66666666-6666-4666-8666-666666666666"; +const CLAUDE_CODE_HIT_SESSION = "claude-code:claude-eval-session"; +const PI_HIT_SESSION = "pi:pi-eval-session"; + +type AcceptanceSourceId = "codex" | "claude-code" | "pi"; + +export type AcceptanceFixtureRoots = Record; export interface AcceptanceGateOptions { keepTemp?: boolean; @@ -34,8 +40,10 @@ export interface AcceptanceGateRow { export interface AcceptanceGateResult { fixtureRoot: string; + sourceRoots: AcceptanceFixtureRoots; dbPath: string; sync: SyncSummary; + sourceSyncs: Record; scoreboard: Record<"total" | "pass" | "fail" | "skip" | "hardFail" | "candidateFail", number>; rows: AcceptanceGateRow[]; } @@ -43,12 +51,31 @@ export interface AcceptanceGateResult { export async function runAcceptanceGate(options: AcceptanceGateOptions = {}): Promise { const base = mkdtempSync(join(tmpdir(), "sherlog-acceptance-")); try { - const fixtureRoot = join(base, "sessions"); const dbPath = join(base, "index.sqlite"); - writeAcceptanceFixtures(fixtureRoot); - const sync = await syncSessions({ dbPath, rootDir: fixtureRoot }); - const rows = evaluateAcceptanceItems(dbPath, acceptanceGoldens()); - return { fixtureRoot, dbPath, sync, scoreboard: buildScoreboard(rows), rows }; + const sourceRoots = writeAcceptanceFixtures(base); + const sourceSyncs = { + codex: await syncSessions({ dbPath, rootDir: sourceRoots.codex }), + "claude-code": await syncSessions({ + dbPath, + sourceId: "claude-code", + selector: { source: "claude-code", kind: "all", root: sourceRoots["claude-code"] }, + }), + pi: await syncSessions({ + dbPath, + sourceId: "pi", + selector: { source: "pi", kind: "all", root: sourceRoots.pi }, + }), + }; + const rows = evaluateAcceptanceItems(dbPath, acceptanceGoldens(sourceRoots)); + return { + fixtureRoot: sourceRoots.codex, + sourceRoots, + dbPath, + sync: sourceSyncs.codex, + sourceSyncs, + scoreboard: buildScoreboard(rows), + rows, + }; } finally { if (!options.keepTemp) rmSync(base, { recursive: true, force: true }); } @@ -123,7 +150,7 @@ function buildScoreboard(rows: AcceptanceGateRow[]): AcceptanceGateResult["score return scoreboard; } -function acceptanceGoldens(): DogfoodGolden[] { +function acceptanceGoldens(roots: AcceptanceFixtureRoots): DogfoodGolden[] { return [ { id: "message-hit-context", @@ -209,10 +236,67 @@ function acceptanceGoldens(): DogfoodGolden[] { }, }, }, + { + id: "claude-code-message-range-context", + query: "claude adapter needle", + intent: "Claude Code source recall should preserve source-qualified session refs and readable range context", + status: "hard", + find: { selector: { source: "claude-code", kind: "all", root: roots["claude-code"] } }, + expected: { + topK: 1, + sourceId: "claude-code", + acceptableSessionUuids: [CLAUDE_CODE_HIT_SESSION], + sessionRef: CLAUDE_CODE_HIT_SESSION, + cwdContains: "/tmp/sherlog-acceptance/claude", + matchSource: "message", + matchSeq: 0, + context: { + mode: "read-range", + before: 0, + after: 1, + mustContain: ["claude adapter needle", "claude range evidence"], + }, + }, + }, + { + id: "pi-session-page-context", + query: "pi compact queue", + intent: "Pi compaction recall should preserve source-qualified session refs and readable page context", + status: "hard", + find: { selector: { source: "pi", kind: "all", root: roots.pi } }, + expected: { + topK: 1, + sourceId: "pi", + acceptableSessionUuids: [PI_HIT_SESSION], + sessionRef: PI_HIT_SESSION, + cwdContains: "/tmp/sherlog-acceptance/pi", + matchSource: "session", + matchSeq: null, + context: { + mode: "read-page", + offset: 0, + limit: 10, + mustContain: ["pi accepted user prompt", "pi accepted assistant reply"], + }, + }, + }, ]; } -function writeAcceptanceFixtures(root: string): void { +function writeAcceptanceFixtures(base: string): AcceptanceFixtureRoots { + const roots: AcceptanceFixtureRoots = { + codex: join(base, "sessions"), + "claude-code": join(base, "claude-projects", "synthetic-project"), + pi: join(base, "pi-sessions"), + }; + + writeCodexAcceptanceFixtures(roots.codex); + writeClaudeCodeAcceptanceFixture(roots["claude-code"]); + writePiAcceptanceFixture(roots.pi); + return roots; +} + +function writeCodexAcceptanceFixtures(root: string): void { const day = join(root, "2026", "06", "26"); mkdirSync(day, { recursive: true }); writeCodexSession(day, MESSAGE_HIT_SESSION, "/tmp/sherlog-acceptance/deploy", [ @@ -243,6 +327,51 @@ function writeAcceptanceFixtures(root: string): void { ]); } +function writeClaudeCodeAcceptanceFixture(root: string): void { + mkdirSync(root, { recursive: true }); + writeFileSync( + join(root, "claude-eval.jsonl"), + `${[ + claudeLine({ + type: "user", + sessionId: "claude-eval-session", + cwd: "/tmp/sherlog-acceptance/claude", + timestamp: "2026-06-26T05:01:00.000Z", + message: { content: "claude adapter needle should rank this session first" }, + }), + claudeLine({ + type: "assistant", + sessionId: "claude-eval-session", + cwd: "/tmp/sherlog-acceptance/claude", + timestamp: "2026-06-26T05:01:01.000Z", + message: { content: [{ type: "text", text: "claude range evidence remains readable after source-qualified find" }] }, + }), + ].join("\n")}\n`, + ); +} + +function writePiAcceptanceFixture(root: string): void { + const projectDir = join(root, "--tmp-pi-project--"); + mkdirSync(projectDir, { recursive: true }); + writeFileSync( + join(projectDir, "pi-eval.jsonl"), + `${[ + piLine({ type: "session", id: "pi-eval-session", cwd: "/tmp/sherlog-acceptance/pi", timestamp: "2026-06-26T05:02:00.000Z" }), + piLine({ + type: "message", + timestamp: "2026-06-26T05:02:01.000Z", + message: { role: "user", content: [{ type: "text", text: "pi accepted user prompt" }], timestamp: "2026-06-26T05:02:01.000Z" }, + }), + piLine({ + type: "message", + timestamp: "2026-06-26T05:02:02.000Z", + message: { role: "assistant", content: [{ type: "text", text: "pi accepted assistant reply" }], timestamp: "2026-06-26T05:02:02.000Z" }, + }), + piLine({ type: "compaction", id: "c1", timestamp: "2026-06-26T05:02:03.000Z", summary: "pi compact queue handoff survives as session recall" }), + ].join("\n")}\n`, + ); +} + function writeCodexSession(day: string, uuid: string, cwd: string, records: Record[]): void { const filePath = join(day, `rollout-2026-06-26T05-00-00-${uuid}.jsonl`); const content = [line("session_meta", { id: uuid, cwd }), line("turn_context", { model: "gpt-5.4" }), ...records] @@ -267,6 +396,14 @@ function line(type: string, payload: Record): Record): string { + return JSON.stringify(record); +} + +function piLine(record: Record): string { + return JSON.stringify(record); +} + function messagesText(messages: Array<{ role: string; contentText: string }>): string { return messages.map((message) => `${message.role}: ${message.contentText}`).join("\n"); }