Skip to content

Add source-aware acceptance eval fixtures #80

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
catoncat wants to merge 1 commit into from
Closed

Add source-aware acceptance eval fixtures #80

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions eval/acceptance-gate.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@ describe("acceptance gate", () => {
const result = await runAcceptanceGate();

expect(result.sync.added).toBe(6);
expect(result.sourceSyncs["claude-code"].added).toBe(1);
expect(result.sourceSyncs.pi.added).toBe(1);
expect(result.scoreboard).toMatchObject({
total: 4,
pass: 4,
total: 6,
pass: 6,
fail: 0,
hardFail: 0,
});
Expand All @@ -17,6 +19,8 @@ describe("acceptance gate", () => {
"session-only-compact-context",
"cjk-message-hit",
"exact-query-profile-phrase",
"claude-code-message-range-context",
"pi-session-page-context",
]);
expect(result.rows.every((row) => row.predicates.length > 0)).toBe(true);
});
Expand Down
151 changes: 144 additions & 7 deletions eval/acceptance-gate.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,12 @@ const CJK_HIT_SESSION = "33333333-3333-4333-8333-333333333333";
const NOISE_SESSION = "44444444-4444-4444-8444-444444444444";
const EXACT_PHRASE_SESSION = "55555555-5555-4555-8555-555555555555";
const SPLIT_METADATA_SESSION = "66666666-6666-4666-8666-666666666666";
const CLAUDE_CODE_HIT_SESSION = "claude-code:claude-eval-session";
const PI_HIT_SESSION = "pi:pi-eval-session";

type AcceptanceSourceId = "codex" | "claude-code" | "pi";

export type AcceptanceFixtureRoots = Record<AcceptanceSourceId, string>;

export interface AcceptanceGateOptions {
keepTemp?: boolean;
Expand All @@ -34,21 +40,42 @@ export interface AcceptanceGateRow {

export interface AcceptanceGateResult {
fixtureRoot: string;
sourceRoots: AcceptanceFixtureRoots;
dbPath: string;
sync: SyncSummary;
sourceSyncs: Record<AcceptanceSourceId, SyncSummary>;
scoreboard: Record<"total" | "pass" | "fail" | "skip" | "hardFail" | "candidateFail", number>;
rows: AcceptanceGateRow[];
}

export async function runAcceptanceGate(options: AcceptanceGateOptions = {}): Promise<AcceptanceGateResult> {
const base = mkdtempSync(join(tmpdir(), "sherlog-acceptance-"));
try {
const fixtureRoot = join(base, "sessions");
const dbPath = join(base, "index.sqlite");
writeAcceptanceFixtures(fixtureRoot);
const sync = await syncSessions({ dbPath, rootDir: fixtureRoot });
const rows = evaluateAcceptanceItems(dbPath, acceptanceGoldens());
return { fixtureRoot, dbPath, sync, scoreboard: buildScoreboard(rows), rows };
const sourceRoots = writeAcceptanceFixtures(base);
const sourceSyncs = {
codex: await syncSessions({ dbPath, rootDir: sourceRoots.codex }),
"claude-code": await syncSessions({
dbPath,
sourceId: "claude-code",
selector: { source: "claude-code", kind: "all", root: sourceRoots["claude-code"] },
}),
pi: await syncSessions({
dbPath,
sourceId: "pi",
selector: { source: "pi", kind: "all", root: sourceRoots.pi },
}),
};
const rows = evaluateAcceptanceItems(dbPath, acceptanceGoldens(sourceRoots));
return {
fixtureRoot: sourceRoots.codex,
sourceRoots,
dbPath,
sync: sourceSyncs.codex,
sourceSyncs,
scoreboard: buildScoreboard(rows),
rows,
};
} finally {
if (!options.keepTemp) rmSync(base, { recursive: true, force: true });
}
Expand Down Expand Up @@ -123,7 +150,7 @@ function buildScoreboard(rows: AcceptanceGateRow[]): AcceptanceGateResult["score
return scoreboard;
}

function acceptanceGoldens(): DogfoodGolden[] {
function acceptanceGoldens(roots: AcceptanceFixtureRoots): DogfoodGolden[] {
return [
{
id: "message-hit-context",
Expand Down Expand Up @@ -209,10 +236,67 @@ function acceptanceGoldens(): DogfoodGolden[] {
},
},
},
{
id: "claude-code-message-range-context",
query: "claude adapter needle",
intent: "Claude Code source recall should preserve source-qualified session refs and readable range context",
status: "hard",
find: { selector: { source: "claude-code", kind: "all", root: roots["claude-code"] } },
expected: {
topK: 1,
sourceId: "claude-code",
acceptableSessionUuids: [CLAUDE_CODE_HIT_SESSION],
sessionRef: CLAUDE_CODE_HIT_SESSION,
cwdContains: "/tmp/sherlog-acceptance/claude",
matchSource: "message",
matchSeq: 0,
context: {
mode: "read-range",
before: 0,
after: 1,
mustContain: ["claude adapter needle", "claude range evidence"],
},
},
},
{
id: "pi-session-page-context",
query: "pi compact queue",
intent: "Pi compaction recall should preserve source-qualified session refs and readable page context",
status: "hard",
find: { selector: { source: "pi", kind: "all", root: roots.pi } },
expected: {
topK: 1,
sourceId: "pi",
acceptableSessionUuids: [PI_HIT_SESSION],
sessionRef: PI_HIT_SESSION,
cwdContains: "/tmp/sherlog-acceptance/pi",
matchSource: "session",
matchSeq: null,
context: {
mode: "read-page",
offset: 0,
limit: 10,
mustContain: ["pi accepted user prompt", "pi accepted assistant reply"],
},
},
},
];
}

function writeAcceptanceFixtures(root: string): void {
function writeAcceptanceFixtures(base: string): AcceptanceFixtureRoots {
const roots: AcceptanceFixtureRoots = {
codex: join(base, "sessions"),
"claude-code": join(base, "claude-projects", "synthetic-project"),
pi: join(base, "pi-sessions"),
};

writeCodexAcceptanceFixtures(roots.codex);
writeClaudeCodeAcceptanceFixture(roots["claude-code"]);
writePiAcceptanceFixture(roots.pi);
return roots;
}

function writeCodexAcceptanceFixtures(root: string): void {
const day = join(root, "2026", "06", "26");
mkdirSync(day, { recursive: true });
writeCodexSession(day, MESSAGE_HIT_SESSION, "/tmp/sherlog-acceptance/deploy", [
Expand Down Expand Up @@ -243,6 +327,51 @@ function writeAcceptanceFixtures(root: string): void {
]);
}

function writeClaudeCodeAcceptanceFixture(root: string): void {
mkdirSync(root, { recursive: true });
writeFileSync(
join(root, "claude-eval.jsonl"),
`${[
claudeLine({
type: "user",
sessionId: "claude-eval-session",
cwd: "/tmp/sherlog-acceptance/claude",
timestamp: "2026-06-26T05:01:00.000Z",
message: { content: "claude adapter needle should rank this session first" },
}),
claudeLine({
type: "assistant",
sessionId: "claude-eval-session",
cwd: "/tmp/sherlog-acceptance/claude",
timestamp: "2026-06-26T05:01:01.000Z",
message: { content: [{ type: "text", text: "claude range evidence remains readable after source-qualified find" }] },
}),
].join("\n")}\n`,
);
}

function writePiAcceptanceFixture(root: string): void {
const projectDir = join(root, "--tmp-pi-project--");
mkdirSync(projectDir, { recursive: true });
writeFileSync(
join(projectDir, "pi-eval.jsonl"),
`${[
piLine({ type: "session", id: "pi-eval-session", cwd: "/tmp/sherlog-acceptance/pi", timestamp: "2026-06-26T05:02:00.000Z" }),
piLine({
type: "message",
timestamp: "2026-06-26T05:02:01.000Z",
message: { role: "user", content: [{ type: "text", text: "pi accepted user prompt" }], timestamp: "2026-06-26T05:02:01.000Z" },
}),
piLine({
type: "message",
timestamp: "2026-06-26T05:02:02.000Z",
message: { role: "assistant", content: [{ type: "text", text: "pi accepted assistant reply" }], timestamp: "2026-06-26T05:02:02.000Z" },
}),
piLine({ type: "compaction", id: "c1", timestamp: "2026-06-26T05:02:03.000Z", summary: "pi compact queue handoff survives as session recall" }),
].join("\n")}\n`,
);
}

function writeCodexSession(day: string, uuid: string, cwd: string, records: Record<string, unknown>[]): void {
const filePath = join(day, `rollout-2026-06-26T05-00-00-${uuid}.jsonl`);
const content = [line("session_meta", { id: uuid, cwd }), line("turn_context", { model: "gpt-5.4" }), ...records]
Expand All @@ -267,6 +396,14 @@ function line(type: string, payload: Record<string, unknown>): Record<string, un
};
}

function claudeLine(record: Record<string, unknown>): string {
return JSON.stringify(record);
}

function piLine(record: Record<string, unknown>): string {
return JSON.stringify(record);
}

function messagesText(messages: Array<{ role: string; contentText: string }>): string {
return messages.map((message) => `${message.role}: ${message.contentText}`).join("\n");
}