diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index bc136e0..9f8f315 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -34,6 +34,8 @@ - 核心层负责 selector、coverage、DB、query/read/list/stats。 - `find` 默认跨 public source fanout 并合并结果；切换 `--source claude-code` 或 `--source pi` 时走同一组命令但只查目标 source。当前非 Codex 语义仍限于 allowlisted transcript text。 +Accepted/rejected projection boundaries live in [SOURCE_CONTRACTS.md](SOURCE_CONTRACTS.md). Treat that file as the code-facing contract for parser privacy fixtures; it is not an upstream raw-format stability promise. + Codex adapter 会把原有 `sessionUuid` 映射为 source-aware identity： - `sourceId = "codex"` @@ -144,7 +146,7 @@ CLI 默认 `find` 会对 public sources 执行单源 `findSessions()` fanout， 下面这些不要误写成现状： -- 真正按 broad/exact query profile 分权的 scoring +- 真正按 broad/exact query profile 分权的 scoring（真实 A/B 未证明有效，当前仍不落地） - richer projection / event replay / range cache - duplicate collapse / diversity control - 强约束 gold set / rubric / error taxonomy diff --git a/docs/RANKING_WEIGHTS.md b/docs/RANKING_WEIGHTS.md index 2c1131e..42f97fa 100644 --- a/docs/RANKING_WEIGHTS.md +++ b/docs/RANKING_WEIGHTS.md @@ -46,12 +46,13 @@ ``` normalizedBm25 + (contentPhrase ? 8 : 0) + + commandSequenceBonus + termCoverage * 2 + (matchSource === "message" ? 4 : 0) + (matchRole === "user" ? 2 : 0) ``` -四项的设计意图是:**bm25 + 三个 token-free 的硬证据**。bm25 处理“词义相关性”,这三项处理 bm25 看不到的元信息。 +这些项的设计意图是:**bm25 + token-free 的硬证据**。bm25 处理“词义相关性”,其余项处理 bm25 看不到的短语邻接、命令 token 邻近、消息可回读性与作者角色。 ### `contentPhrase ? 8 : 0` diff --git a/docs/SOURCE_CONTRACTS.md b/docs/SOURCE_CONTRACTS.md new file mode 100644 index 0000000..2b5abdb --- /dev/null +++ b/docs/SOURCE_CONTRACTS.md @@ -0,0 +1,68 @@ +# Source Adapter Contracts + +This document describes the searchable projection contract for Sherlog public source adapters. It is not a promise that upstream raw transcript formats are stable. `claude-code` and `pi` remain experimental transcript-reader support until upstream formats and fixtures are stronger. + +## Shared Invariants + +- Source adapters implement the boundary in `src/sources/types.ts`: root resolution, file collection, inventory, snapshot, and `parseFile`. +- Indexing stores only accepted transcript projections: session metadata, raw user/assistant text where allowed, and a small set of session-level handoff fields. +- Unsupported/private records are filtered by default. Tool outputs, attachments, diagnostics, sidechain/meta records, and thinking blocks must not enter `messages_fts` or `sessions_fts` unless a future issue adds an explicit privacy design. +- Read isolation is source-aware. `sessionRef` values returned by `find` can be passed back to `read-range` or `read-page` without guessing the source. +- Malformed JSONL records are skipped instead of failing the whole file. + +## Codex + +Accepted projection: + +- `session_meta` id and cwd metadata. +- `turn_context` model and cwd metadata. +- `event_msg` records whose payload type is `user_message` or `agent_message` and whose `message` is non-empty text. +- `compacted` payload text as session-level `compactText`. +- `response_item` reasoning summaries as session-level `reasoningSummaryText`. + +Rejected projection: + +- malformed JSONL lines. +- unrelated record types. +- `event_msg` records with other payload types, including tool-result-like records. +- empty user/assistant messages. +- internal approval/evaluation transcript markers filtered by `looksInternal()`. + +## Claude Code + +Accepted projection: + +- policy-approved user and assistant text records from `claude-code-policy`. +- accepted session id, cwd, and timestamp metadata only from accepted records. + +Rejected projection: + +- meta records. +- sidechain records. +- tool results. +- thinking blocks. +- attachment-like non-text content. +- skipped records when deriving inventory grouping and snapshots. + +## Pi + +Accepted projection: + +- `session` metadata with id, cwd, and timestamp. +- latest `model_change` model id. +- user/assistant text content from `message` records. +- `compaction` summary text as session-level `compactText`. + +Rejected projection: + +- tool-result roles. +- tool-call content. +- thinking content. +- unsupported content parts. +- malformed or incomplete records. + +## Current Contract Limits + +- The contract covers what Sherlog indexes and reads from synthetic fixtures, not every upstream raw variant. +- Do not broaden accepted record classes to improve recall without adding privacy-specific tests first. +- Keep source-specific parser changes paired with negative fixtures that prove private or diagnostic data is still filtered. diff --git a/eval/acceptance-gate.test.ts b/eval/acceptance-gate.test.ts new file mode 100644 index 0000000..5375080 --- /dev/null +++ b/eval/acceptance-gate.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, test } from "vitest"; +import { runAcceptanceGate } from "./acceptance-gate"; + +describe("acceptance gate", () => { + test("passes synthetic evidence-level retrieval fixtures", async () => { + const result = await runAcceptanceGate(); + + expect(result.sync.added).toBe(4); + expect(result.sourceSyncs["claude-code"].added).toBe(1); + expect(result.sourceSyncs.pi.added).toBe(1); + expect(result.scoreboard).toMatchObject({ + total: 5, + pass: 5, + fail: 0, + hardFail: 0, + }); + expect(result.rows.map((row) => row.id)).toEqual([ + "message-hit-context", + "session-only-compact-context", + "cjk-message-hit", + "claude-code-message-range-context", + "pi-session-page-context", + ]); + expect(result.rows.every((row) => row.predicates.length > 0)).toBe(true); + }); +}); diff --git a/eval/acceptance-gate.ts b/eval/acceptance-gate.ts new file mode 100644 index 0000000..d58954a --- /dev/null +++ b/eval/acceptance-gate.ts @@ -0,0 +1,387 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { desiredContextMode, evaluateDogfoodItem, type DogfoodEvaluation } from "./dogfood-eval-core"; +import type { DogfoodGolden } from "./dogfood-schema"; +import { syncSessions } from "../src/indexer"; +import { findSessions, getMessagePage, getMessageRange } from "../src/query"; +import type { FindResult, SyncSummary } from "../src/types"; + +const MESSAGE_HIT_SESSION = "11111111-1111-4111-8111-111111111111"; +const SESSION_HIT_SESSION = "22222222-2222-4222-8222-222222222222"; +const CJK_HIT_SESSION = "33333333-3333-4333-8333-333333333333"; +const NOISE_SESSION = "44444444-4444-4444-8444-444444444444"; +const CLAUDE_CODE_HIT_SESSION = "claude-code:claude-eval-session"; +const PI_HIT_SESSION = "pi:pi-eval-session"; + +type AcceptanceSourceId = "codex" | "claude-code" | "pi"; + +export type AcceptanceFixtureRoots = Record; + +export interface AcceptanceGateOptions { + keepTemp?: boolean; +} + +export interface AcceptanceGateRow { + id: string; + query: string; + status: DogfoodGolden["status"]; + mark: DogfoodEvaluation["mark"]; + blocking: boolean; + selectedRank: number | null; + selectedSessionRef: string | null; + selectedMatchSource: FindResult["matchSource"] | null; + selectedMatchSeq: number | null; + contextKind?: "read-range" | "read-page"; + predicates: DogfoodEvaluation["predicateResults"]; +} + +export interface AcceptanceGateResult { + fixtureRoot: string; + sourceRoots: AcceptanceFixtureRoots; + dbPath: string; + sync: SyncSummary; + sourceSyncs: Record; + scoreboard: Record<"total" | "pass" | "fail" | "skip" | "hardFail" | "candidateFail", number>; + rows: AcceptanceGateRow[]; +} + +export async function runAcceptanceGate(options: AcceptanceGateOptions = {}): Promise { + const base = mkdtempSync(join(tmpdir(), "sherlog-acceptance-")); + try { + const dbPath = join(base, "index.sqlite"); + const sourceRoots = writeAcceptanceFixtures(base); + const sourceSyncs = { + codex: await syncSessions({ dbPath, rootDir: sourceRoots.codex }), + "claude-code": await syncSessions({ + dbPath, + sourceId: "claude-code", + selector: { source: "claude-code", kind: "all", root: sourceRoots["claude-code"] }, + }), + pi: await syncSessions({ + dbPath, + sourceId: "pi", + selector: { source: "pi", kind: "all", root: sourceRoots.pi }, + }), + }; + const rows = evaluateAcceptanceItems(dbPath, acceptanceGoldens(sourceRoots)); + return { + fixtureRoot: sourceRoots.codex, + sourceRoots, + dbPath, + sync: sourceSyncs.codex, + sourceSyncs, + scoreboard: buildScoreboard(rows), + rows, + }; + } finally { + if (!options.keepTemp) rmSync(base, { recursive: true, force: true }); + } +} + +function evaluateAcceptanceItems(dbPath: string, items: DogfoodGolden[]): AcceptanceGateRow[] { + return items.map((item) => { + const limit = Math.max(item.expected.topK ?? 5, item.find?.limit ?? 0, 5); + const summary = findSessions(dbPath, item.query, limit, item.find?.selector ?? null, { + sort: item.find?.sort, + excludeSessions: item.find?.excludeSessionUuids, + sourceId: item.expected.sourceId, + }); + const preselected = evaluateDogfoodItem({ item, results: summary.results }).selected; + const context = readContextIfNeeded(dbPath, item, preselected.hit); + const evaluation = evaluateDogfoodItem({ + item, + results: summary.results, + contextText: context.text, + contextKind: context.kind, + contextUnavailableReason: context.unavailableReason, + }); + + return { + id: item.id, + query: item.query, + status: item.status, + mark: evaluation.mark, + blocking: evaluation.blocking, + selectedRank: evaluation.selected.rank, + selectedSessionRef: evaluation.selected.hit?.sessionRef ?? null, + selectedMatchSource: evaluation.selected.hit?.matchSource ?? null, + selectedMatchSeq: evaluation.selected.hit?.matchSeq ?? null, + ...(context.kind ? { contextKind: context.kind } : {}), + predicates: evaluation.predicateResults, + }; + }); +} + +function readContextIfNeeded( + dbPath: string, + item: DogfoodGolden, + hit: FindResult | null, +): { kind?: "read-range" | "read-page"; text?: string; unavailableReason?: string } { + const mode = desiredContextMode(item, hit); + if (!mode) return {}; + if (!hit) return { unavailableReason: "no selected hit for context read" }; + const context = item.expected.context ?? {}; + if (mode === "read-range") { + if (typeof hit.matchSeq !== "number") { + // Session-only hit: use the query to locate the real anchor inside the + // session transcript, mirroring buildEvidenceReadAction's read-range --query path. + const query = context.query ?? item.query; + if (!query) { + return { kind: "read-range", unavailableReason: "selected hit has no numeric matchSeq and no query available" }; + } + const range = getMessageRange(dbPath, hit.sessionRef, { + query, + before: context.before ?? 2, + after: context.after ?? 2, + }); + return { kind: "read-range", text: messagesText(range.messages) }; + } + const range = getMessageRange(dbPath, hit.sessionRef, { + seq: hit.matchSeq, + before: context.before ?? 2, + after: context.after ?? 2, + }); + return { kind: "read-range", text: messagesText(range.messages) }; + } + + const page = getMessagePage(dbPath, hit.sessionRef, context.offset ?? 0, context.limit ?? 20); + return { kind: "read-page", text: messagesText(page.messages) }; +} + +function buildScoreboard(rows: AcceptanceGateRow[]): AcceptanceGateResult["scoreboard"] { + const scoreboard = { total: rows.length, pass: 0, fail: 0, skip: 0, hardFail: 0, candidateFail: 0 }; + for (const row of rows) { + scoreboard[row.mark] += 1; + if (row.status === "hard" && row.mark === "fail") scoreboard.hardFail += 1; + if (row.status === "candidate" && row.mark === "fail") scoreboard.candidateFail += 1; + } + return scoreboard; +} + +function acceptanceGoldens(roots: AcceptanceFixtureRoots): DogfoodGolden[] { + return [ + { + id: "message-hit-context", + query: "health check returned 500", + intent: "message hits should identify the exact session and readable range context", + status: "hard", + expected: { + topK: 1, + sourceId: "codex", + acceptableSessionUuids: [MESSAGE_HIT_SESSION], + sessionRef: MESSAGE_HIT_SESSION, + cwdContains: "/tmp/sherlog-acceptance/deploy", + matchSource: "message", + matchSeq: 1, + context: { + mode: "read-range", + before: 1, + after: 1, + mustContain: ["health check returned 500", "rollback plan includes readback verification"], + }, + }, + }, + { + id: "session-only-compact-context", + query: "durable output queue", + intent: "session-level compact recall should still lead to raw transcript context", + status: "hard", + expected: { + topK: 1, + sourceId: "codex", + acceptableSessionUuids: [SESSION_HIT_SESSION], + sessionRef: SESSION_HIT_SESSION, + cwdContains: "/tmp/sherlog-acceptance/handoff", + matchSource: "session", + matchSeq: null, + context: { + mode: "read-range", + mustContain: ["Prepare release notes", "Use the existing checklist"], + }, + }, + }, + { + id: "cjk-message-hit", + query: "回滚预案", + intent: "CJK message recall should preserve evidence identity and range context", + status: "hard", + expected: { + topK: 1, + sourceId: "codex", + acceptableSessionUuids: [CJK_HIT_SESSION], + sessionRef: CJK_HIT_SESSION, + cwdContains: "/tmp/sherlog-acceptance/cjk", + matchSource: "message", + matchSeq: 0, + context: { + mode: "read-range", + before: 0, + after: 1, + mustContain: ["回滚预案", "健康检查恢复"], + }, + }, + }, + { + id: "claude-code-message-range-context", + query: "claude adapter needle", + intent: "Claude Code source recall should preserve source-qualified session refs and readable range context", + status: "hard", + find: { selector: { source: "claude-code", kind: "all", root: roots["claude-code"] } }, + expected: { + topK: 1, + sourceId: "claude-code", + acceptableSessionUuids: [CLAUDE_CODE_HIT_SESSION], + sessionRef: CLAUDE_CODE_HIT_SESSION, + cwdContains: "/tmp/sherlog-acceptance/claude", + matchSource: "message", + matchSeq: 0, + context: { + mode: "read-range", + before: 0, + after: 1, + mustContain: ["claude adapter needle", "claude range evidence"], + }, + }, + }, + { + id: "pi-session-page-context", + query: "pi compact queue", + intent: "Pi compaction recall should preserve source-qualified session refs and readable page context", + status: "hard", + find: { selector: { source: "pi", kind: "all", root: roots.pi } }, + expected: { + topK: 1, + sourceId: "pi", + acceptableSessionUuids: [PI_HIT_SESSION], + sessionRef: PI_HIT_SESSION, + cwdContains: "/tmp/sherlog-acceptance/pi", + matchSource: "session", + matchSeq: null, + context: { + mode: "read-page", + offset: 0, + limit: 10, + mustContain: ["pi accepted user prompt", "pi accepted assistant reply"], + }, + }, + }, + ]; +} + +function writeAcceptanceFixtures(base: string): AcceptanceFixtureRoots { + const roots: AcceptanceFixtureRoots = { + codex: join(base, "sessions"), + "claude-code": join(base, "claude-projects", "synthetic-project"), + pi: join(base, "pi-sessions"), + }; + + writeCodexAcceptanceFixtures(roots.codex); + writeClaudeCodeAcceptanceFixture(roots["claude-code"]); + writePiAcceptanceFixture(roots.pi); + return roots; +} + +function writeCodexAcceptanceFixtures(root: string): void { + const day = join(root, "2026", "06", "26"); + mkdirSync(day, { recursive: true }); + writeCodexSession(day, MESSAGE_HIT_SESSION, "/tmp/sherlog-acceptance/deploy", [ + event("user_message", "Investigate deploy failure"), + event("agent_message", "The health check returned 500 after deploy."), + event("user_message", "The rollback plan includes readback verification."), + ]); + writeCodexSession(day, SESSION_HIT_SESSION, "/tmp/sherlog-acceptance/handoff", [ + event("user_message", "Prepare release notes"), + compacted("handoff says durable output queue needs final verification"), + event("agent_message", "Use the existing checklist before publishing."), + ]); + writeCodexSession(day, CJK_HIT_SESSION, "/tmp/sherlog-acceptance/cjk", [ + event("user_message", "准备回滚预案"), + event("agent_message", "先确认健康检查恢复，再继续发布。"), + ]); + writeCodexSession(day, NOISE_SESSION, "/tmp/sherlog-acceptance/noise", [ + event("user_message", "Refactor parser docs"), + event("agent_message", "No deploy or handoff evidence here."), + ]); +} + +function writeClaudeCodeAcceptanceFixture(root: string): void { + mkdirSync(root, { recursive: true }); + writeFileSync( + join(root, "claude-eval.jsonl"), + `${[ + claudeLine({ + type: "user", + sessionId: "claude-eval-session", + cwd: "/tmp/sherlog-acceptance/claude", + timestamp: "2026-06-26T05:01:00.000Z", + message: { content: "claude adapter needle should rank this session first" }, + }), + claudeLine({ + type: "assistant", + sessionId: "claude-eval-session", + cwd: "/tmp/sherlog-acceptance/claude", + timestamp: "2026-06-26T05:01:01.000Z", + message: { content: [{ type: "text", text: "claude range evidence remains readable after source-qualified find" }] }, + }), + ].join("\n")}\n`, + ); +} + +function writePiAcceptanceFixture(root: string): void { + const projectDir = join(root, "--tmp-pi-project--"); + mkdirSync(projectDir, { recursive: true }); + writeFileSync( + join(projectDir, "pi-eval.jsonl"), + `${[ + piLine({ type: "session", id: "pi-eval-session", cwd: "/tmp/sherlog-acceptance/pi", timestamp: "2026-06-26T05:02:00.000Z" }), + piLine({ + type: "message", + timestamp: "2026-06-26T05:02:01.000Z", + message: { role: "user", content: [{ type: "text", text: "pi accepted user prompt" }], timestamp: "2026-06-26T05:02:01.000Z" }, + }), + piLine({ + type: "message", + timestamp: "2026-06-26T05:02:02.000Z", + message: { role: "assistant", content: [{ type: "text", text: "pi accepted assistant reply" }], timestamp: "2026-06-26T05:02:02.000Z" }, + }), + piLine({ type: "compaction", id: "c1", timestamp: "2026-06-26T05:02:03.000Z", summary: "pi compact queue handoff survives as session recall" }), + ].join("\n")}\n`, + ); +} + +function writeCodexSession(day: string, uuid: string, cwd: string, records: Record[]): void { + const filePath = join(day, `rollout-2026-06-26T05-00-00-${uuid}.jsonl`); + const content = [line("session_meta", { id: uuid, cwd }), line("turn_context", { model: "gpt-5.4" }), ...records] + .map((record) => JSON.stringify(record)) + .join("\n"); + writeFileSync(filePath, content); +} + +function event(type: "user_message" | "agent_message", message: string): Record { + return line("event_msg", { type, message }); +} + +function compacted(message: string): Record { + return line("compacted", { message }); +} + +function line(type: string, payload: Record): Record { + return { + timestamp: "2026-06-26T05:00:00.000Z", + type, + payload, + }; +} + +function claudeLine(record: Record): string { + return JSON.stringify(record); +} + +function piLine(record: Record): string { + return JSON.stringify(record); +} + +function messagesText(messages: Array<{ role: string; contentText: string }>): string { + return messages.map((message) => `${message.role}: ${message.contentText}`).join("\n"); +} diff --git a/eval/dogfood-eval-core.test.ts b/eval/dogfood-eval-core.test.ts index 30630a0..008e747 100644 --- a/eval/dogfood-eval-core.test.ts +++ b/eval/dogfood-eval-core.test.ts @@ -44,11 +44,26 @@ describe("dogfood eval core", () => { expect(evaluation.mark).toBe("pass"); }); - test("uses read-range for message hits and read-page for session-only hits in auto mode", () => { + test("checks source id, session ref, and nullable match sequence", () => { + const evaluation = evaluateDogfoodItem({ + item: golden({ sourceId: "codex", sessionRef: "session-a", matchSource: "session", matchSeq: null }), + results: [findResult({ matchSource: "session", matchSeq: null })], + }); + + expect(evaluation.mark).toBe("pass"); + expect(evaluation.predicateResults.map((predicate) => predicate.label)).toEqual([ + "source_id", + "session_ref", + "match_source", + "match_seq", + ]); + }); + + test("uses read-range for both message and session-only hits in auto mode", () => { const item = golden({ contextMustContain: ["decision"] }); expect(desiredContextMode(item, findResult({ matchSource: "message", matchSeq: 7 }))).toBe("read-range"); - expect(desiredContextMode(item, findResult({ matchSource: "session", matchSeq: null }))).toBe("read-page"); + expect(desiredContextMode(item, findResult({ matchSource: "session", matchSeq: null }))).toBe("read-range"); }); test("reports missing context needles case-insensitively", () => { @@ -72,7 +87,10 @@ describe("dogfood eval core", () => { }, expected: { topK: 5, + sourceId: "codex", acceptableSessionUuids: ["target-session"], + sessionRef: "target-session", + matchSeq: null, }, }), "goldens.local.jsonl"); @@ -90,7 +108,11 @@ describe("dogfood eval core", () => { function golden( overrides: Partial<{ status: "candidate" | "hard" | "stale"; + sourceId: "codex" | "claude-code" | "pi"; acceptableSessionUuids: string[]; + sessionRef: string; + matchSource: "message" | "session"; + matchSeq: number | null; topK: number; contextMustContain: string[]; }> = {}, @@ -102,7 +124,11 @@ function golden( status: overrides.status ?? "hard", expected: { topK: overrides.topK, + sourceId: overrides.sourceId, acceptableSessionUuids: overrides.acceptableSessionUuids, + sessionRef: overrides.sessionRef, + matchSource: overrides.matchSource, + matchSeq: overrides.matchSeq, context: overrides.contextMustContain ? { mustContain: overrides.contextMustContain } : undefined, }, }; diff --git a/eval/dogfood-eval-core.ts b/eval/dogfood-eval-core.ts index 107293f..22f5d7f 100644 --- a/eval/dogfood-eval-core.ts +++ b/eval/dogfood-eval-core.ts @@ -4,7 +4,7 @@ import type { DogfoodGolden } from "./dogfood-schema"; export type DogfoodMark = "pass" | "fail" | "skip"; export interface DogfoodPredicateResult { - label: "session_uuid" | "cwd" | "match_source" | "context"; + label: "source_id" | "session_uuid" | "session_ref" | "cwd" | "match_source" | "match_seq" | "context"; expected: string; actual: string; matched: boolean; @@ -70,7 +70,9 @@ export function desiredContextMode(item: DogfoodGolden, hit: FindResult | null): if (!context?.mustContain?.length) return null; const mode = context.mode ?? "auto"; if (mode !== "auto") return mode; - return typeof hit?.matchSeq === "number" ? "read-range" : "read-page"; + // Session-only hits now use read-range --query to locate the real anchor, + // so auto mode always prefers read-range. + return "read-range"; } export function missingContextNeedles(item: DogfoodGolden, contextText: string): string[] { @@ -87,6 +89,15 @@ function buildPredicates( const predicates: DogfoodPredicateResult[] = []; const acceptable = item.expected.acceptableSessionUuids ?? []; + if (item.expected.sourceId) { + predicates.push({ + label: "source_id", + expected: item.expected.sourceId, + actual: hit?.sourceId ?? "no selected hit", + matched: hit?.sourceId === item.expected.sourceId, + }); + } + if (acceptable.length > 0) { predicates.push({ label: "session_uuid", @@ -96,6 +107,15 @@ function buildPredicates( }); } + if (item.expected.sessionRef) { + predicates.push({ + label: "session_ref", + expected: item.expected.sessionRef, + actual: hit?.sessionRef ?? "no selected hit", + matched: hit?.sessionRef === item.expected.sessionRef, + }); + } + if (item.expected.cwdContains) { const needle = item.expected.cwdContains.toLowerCase(); predicates.push({ @@ -115,6 +135,15 @@ function buildPredicates( }); } + if (item.expected.matchSeq !== undefined) { + predicates.push({ + label: "match_seq", + expected: String(item.expected.matchSeq), + actual: hit ? String(hit.matchSeq) : "no selected hit", + matched: hit?.matchSeq === item.expected.matchSeq, + }); + } + for (const needle of item.expected.context?.mustContain ?? []) { const haystack = input.contextText ?? ""; predicates.push({ diff --git a/eval/dogfood-schema.ts b/eval/dogfood-schema.ts index cbbd4cf..a125210 100644 --- a/eval/dogfood-schema.ts +++ b/eval/dogfood-schema.ts @@ -1,5 +1,5 @@ import { canonicalizeSelector } from "../src/selector"; -import type { FindSort, MatchSource, Selector } from "../src/types"; +import { isSessionSourceId, type FindSort, type MatchSource, type Selector, type SessionSourceId } from "../src/types"; export type DogfoodStatus = "candidate" | "hard" | "stale"; export type DogfoodOriginKind = "observed-user-ask" | "evidence-backed-derived" | "manual"; @@ -18,14 +18,18 @@ export interface DogfoodExpectedContext { after?: number; offset?: number; limit?: number; + query?: string; mustContain?: string[]; } export interface DogfoodExpected { topK?: number; + sourceId?: SessionSourceId; acceptableSessionUuids?: string[]; + sessionRef?: string; cwdContains?: string; matchSource?: MatchSource; + matchSeq?: number | null; context?: DogfoodExpectedContext; } @@ -120,9 +124,16 @@ function parseExpected(value: unknown): DogfoodExpected | null { const topK = readPositiveInteger(value.topK); if (topK) expected.topK = topK; + if (typeof value.sourceId === "string" && isSessionSourceId(value.sourceId)) { + expected.sourceId = value.sourceId; + } + const acceptableSessionUuids = readStringArray(value.acceptableSessionUuids); if (acceptableSessionUuids) expected.acceptableSessionUuids = acceptableSessionUuids; + const sessionRef = readNonEmptyString(value, "sessionRef"); + if (sessionRef) expected.sessionRef = sessionRef; + const cwdContains = readNonEmptyString(value, "cwdContains"); if (cwdContains) expected.cwdContains = cwdContains; @@ -130,6 +141,15 @@ function parseExpected(value: unknown): DogfoodExpected | null { expected.matchSource = value.matchSource; } + if (Object.hasOwn(value, "matchSeq")) { + if (value.matchSeq === null) { + expected.matchSeq = null; + } else { + const matchSeq = readNonNegativeInteger(value.matchSeq); + if (typeof matchSeq === "number") expected.matchSeq = matchSeq; + } + } + const context = parseContext(value.context); if (context) expected.context = context; @@ -153,6 +173,7 @@ function parseContext(value: unknown): DogfoodExpectedContext | undefined { if (limit) context.limit = limit; const mustContain = readStringArray(value.mustContain); if (mustContain) context.mustContain = mustContain; + if (typeof value.query === "string" && value.query.length > 0) context.query = value.query; return Object.keys(context).length > 0 ? context : undefined; } @@ -216,9 +237,12 @@ function parseOrigin(value: unknown, _lineNumber: number): DogfoodOrigin | undef function hasExpectedAssertion(expected: DogfoodExpected): boolean { return Boolean( - expected.acceptableSessionUuids?.length + Boolean(expected.sourceId) + || expected.acceptableSessionUuids?.length + || Boolean(expected.sessionRef) || Boolean(expected.cwdContains) || Boolean(expected.matchSource) + || expected.matchSeq !== undefined || expected.context?.mustContain?.length, ); } diff --git a/eval/perf-bench.ts b/eval/perf-bench.ts index f837a3c..06e504a 100644 --- a/eval/perf-bench.ts +++ b/eval/perf-bench.ts @@ -1,29 +1,107 @@ #!/usr/bin/env -S node --import tsx -import { mkdirSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { spawn as childSpawn } from "node:child_process"; import { homedir, tmpdir } from "node:os"; import { join, resolve } from "node:path"; -import { existsSync } from "node:fs"; -import { spawn as childSpawn } from "node:child_process"; -interface PerQueryRecord { - query: string; +interface LatencyStats { runs: number; samplesMs: number[]; p50Ms: number; p95Ms: number; } +interface TopHitRecord { + sourceId: string; + sessionRef: string; + matchSource: string; + matchSeq: number | null; +} + +interface ReadProbeRecord extends LatencyStats { + kind: "read-range" | "read-page"; + sourceId: string; + sessionRef: string; + argv: string[]; + messagesReturned: number; + anchorSeq?: number; + totalCount?: number; +} + +interface PerQueryRecord extends LatencyStats { + query: string; + resultCount: number; + scannedMessageCount: number; + topHit: TopHitRecord | null; + readRange: ReadProbeRecord | null; + readPage: ReadProbeRecord | null; +} + +interface CoverageCostSummary { + statusMs: number; + coverageCount: number; + freshness: Record; + staleReasons: Record; + requestedCoverage: { + freshness: string; + staleReason: string; + sourceFileCount: number; + recommendedAction: string; + } | null; +} + interface Report { generatedAt: string; + sourceId: string; dbPath: string; rootDir: string; sessionCount: number; + messageCount: number; syncMs: number; dbSizeBytes: number; + runsPerQuery: number; + readRunsPerProbe: number; + coverage: CoverageCostSummary; perQuery: PerQueryRecord[]; } +interface FindJsonPayload { + scannedMessageCount?: number; + results?: Array<{ + sourceId?: string; + sessionRef?: string; + matchSource?: string; + matchSeq?: number | null; + }>; +} + +interface ReadJsonPayload { + anchorSeq?: number; + totalCount?: number; + messages?: unknown[]; +} + +interface StatsJsonPayload { + sessionCount?: number; + messageCount?: number; + dbSizeBytes?: number; +} + +interface StatusJsonPayload { + coverage?: Array<{ + freshness?: string; + sourceFileSetFingerprint?: string; + currentSourceFileSetFingerprint?: string; + }>; + requestedCoverage?: { + freshness?: string; + staleReason?: string; + sourceFileCount?: number; + recommendedAction?: string; + }; +} + // Bench query 选取原则: // - 单 token 高频(hammerspoon/envchain): 检验最常见广义 fts 命中 // - 短 token(sb): 检验 trigram fallback 路径 @@ -40,7 +118,8 @@ const BENCH_QUERIES: string[] = [ "部署 health check", ]; -const RUNS_PER_QUERY = 5; // 第 1 次作为 warmup,统计后 4 次 +const DEFAULT_RUNS_PER_QUERY = 5; // 第 1 次作为 warmup,统计后续样本 +const DEFAULT_READ_RUNS_PER_PROBE = 3; // 第 1 次作为 warmup,统计后续样本 const ROOT = resolve(import.meta.dirname, ".."); const CLI_ENTRY = resolve(ROOT, "src", "cli.ts"); const OUT_BASE = resolve(ROOT, "data", "shlog-perf"); @@ -48,27 +127,44 @@ const OUT_BASE = resolve(ROOT, "data", "shlog-perf"); interface CliArgs { root: string; db: string; + source: string; jsonOnly: boolean; + runsPerQuery: number; + readRunsPerProbe: number; } function parseArgs(argv: string[]): CliArgs { let root = join(homedir(), ".codex", "sessions"); let db = join(tmpdir(), `shlog-perf-${Date.now()}.db`); + let source = "codex"; let jsonOnly = false; + let runsPerQuery = DEFAULT_RUNS_PER_QUERY; + let readRunsPerProbe = DEFAULT_READ_RUNS_PER_PROBE; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === "--root") { root = resolve(argv[++i] ?? root); } else if (a === "--db") { db = resolve(argv[++i] ?? db); + } else if (a === "--source") { + source = argv[++i] ?? source; + } else if (a === "--runs") { + runsPerQuery = parsePositiveInt(argv[++i], DEFAULT_RUNS_PER_QUERY); + } else if (a === "--read-runs") { + readRunsPerProbe = parsePositiveInt(argv[++i], DEFAULT_READ_RUNS_PER_PROBE); } else if (a === "--json-only") { jsonOnly = true; } else if (a === "--help" || a === "-h") { - console.log("Usage: npm run eval:perf -- [--root ] [--db ] [--json-only]"); + console.log("Usage: npm run eval:perf -- [--source ] [--root ] [--db ] [--runs ] [--read-runs ] [--json-only]"); process.exit(0); } } - return { root, db, jsonOnly }; + return { root, db, source, jsonOnly, runsPerQuery, readRunsPerProbe }; +} + +function parsePositiveInt(value: string | undefined, fallback: number): number { + const parsed = Number.parseInt(value ?? "", 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; } const args = parseArgs(process.argv.slice(2)); @@ -111,11 +207,43 @@ function spawnAndCapture(cmd: string[], cwd: string): Promise<{ stdout: string; async function runOrThrow(cmd: string[]): Promise { const r = await run(cmd); if (r.exitCode !== 0) { - throw new Error(`command failed (exit ${r.exitCode}): ${cmd.join(" ")}\n${r.stderr}`); + throw new Error(`command failed (exit ${r.exitCode}): ${cmd.join(" ")}\n${r.stderr || r.stdout}`); } return r; } +async function runJsonOrThrow(cmd: string[]): Promise<{ run: RunResult; payload: T }> { + const result = await runOrThrow(cmd); + try { + return { run: result, payload: JSON.parse(result.stdout) as T }; + } catch (error) { + throw new Error(`command did not emit JSON: ${cmd.join(" ")}\n${String(error)}\n${result.stdout}`); + } +} + +async function benchJsonCommand(cmd: string[], runs: number): Promise<{ latency: LatencyStats; payload: T }> { + const samplesAll: number[] = []; + let payload: T | null = null; + for (let i = 0; i < runs; i++) { + const result = await runJsonOrThrow(cmd); + samplesAll.push(result.run.ms); + payload = result.payload; + } + if (!payload) throw new Error(`no payload produced for command: ${cmd.join(" ")}`); + return { latency: latencyStats(samplesAll), payload }; +} + +function latencyStats(samplesAll: number[]): LatencyStats { + const samples = samplesAll.length > 1 ? samplesAll.slice(1) : samplesAll; + const sorted = [...samples].sort((a, b) => a - b); + return { + runs: samples.length, + samplesMs: samplesAll.map((x) => Number(x.toFixed(2))), + p50Ms: Number(median(sorted).toFixed(2)), + p95Ms: Number(percentile(samples, 0.95).toFixed(2)), + }; +} + function median(sorted: number[]): number { const n = sorted.length; if (n === 0) return 0; @@ -125,9 +253,9 @@ function median(sorted: number[]): number { } function percentile(samplesMs: number[], p: number): number { - // 4 个样本下 p95 数学意义薄弱: 直接取 max 作为 worst-case 近似 + // 小样本下 p95 数学意义薄弱: 直接取 max 作为 worst-case 近似 if (samplesMs.length === 0) return 0; - if (p >= 0.99) return Math.max(...samplesMs); + if (p >= 0.95) return Math.max(...samplesMs); const sorted = [...samplesMs].sort((a, b) => a - b); const idx = Math.min(sorted.length - 1, Math.floor(p * sorted.length)); return sorted[idx]!; @@ -144,6 +272,14 @@ function fmtBytes(n: number): string { return `${(n / 1024 / 1024 / 1024).toFixed(2)} GB`; } +function cliCommand(...command: string[]): string[] { + return [process.execPath, "--import", "tsx", CLI_ENTRY, ...command]; +} + +function publicArgv(cmd: string[]): string[] { + return ["shlog", ...cmd.slice(4)]; +} + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); const outDir = args.jsonOnly ? "" : join(OUT_BASE, stamp); if (!args.jsonOnly) { @@ -151,7 +287,7 @@ if (!args.jsonOnly) { } // 1. sync -const syncRun = await runOrThrow([process.execPath, "--import", "tsx", CLI_ENTRY, "sync", "--db", args.db, "--root", args.root, "--json"]); +const syncRun = await runOrThrow(cliCommand("sync", "--source", args.source, "--db", args.db, "--root", args.root, "--json")); const syncMs = syncRun.ms; let sessionCount = 0; try { @@ -161,46 +297,63 @@ try { // 解析失败保持 0 } -// 2. find x N runs per query +// 2. coverage/freshness cost +const statusSelector = JSON.stringify({ source: args.source, kind: "all", root: args.root }); +const statusResult = await runJsonOrThrow(cliCommand( + "status", + "--source", + args.source, + "--root", + args.root, + "--selector", + statusSelector, + "--db", + args.db, + "--json", +)); +const coverage = coverageCostSummary(statusResult.run, statusResult.payload); + +// 3. find + raw-read probes const perQuery: PerQueryRecord[] = []; for (const q of BENCH_QUERIES) { - const samplesAll: number[] = []; - for (let i = 0; i < RUNS_PER_QUERY; i++) { - const r = await runOrThrow([process.execPath, "--import", "tsx", CLI_ENTRY, "find", q, "--db", args.db, "--limit", "10", "--json"]); - samplesAll.push(r.ms); - } - // 丢弃首次 warmup - const samples = samplesAll.slice(1); - const sorted = [...samples].sort((a, b) => a - b); + const findCommand = cliCommand("find", q, "--source", args.source, "--db", args.db, "--limit", "10", "--json"); + const { latency, payload } = await benchJsonCommand(findCommand, args.runsPerQuery); + const topHit = topHitFromFind(payload); perQuery.push({ query: q, - runs: samples.length, - samplesMs: samplesAll.map((x) => Number(x.toFixed(2))), - p50Ms: Number(median(sorted).toFixed(2)), - p95Ms: Number(percentile(samples, 0.95).toFixed(2)), + ...latency, + resultCount: Array.isArray(payload.results) ? payload.results.length : 0, + scannedMessageCount: typeof payload.scannedMessageCount === "number" ? payload.scannedMessageCount : 0, + topHit, + readRange: topHit ? await measureReadRange(topHit) : null, + readPage: topHit ? await measureReadPage(topHit) : null, }); } -// 3. stats -> dbSizeBytes -const statsRun = await runOrThrow([process.execPath, "--import", "tsx", CLI_ENTRY, "stats", "--db", args.db, "--json"]); +// 4. stats -> db size and indexed counts +const statsRun = await runJsonOrThrow(cliCommand("stats", "--source", args.source, "--db", args.db, "--json")); let dbSizeBytes = 0; -try { - const parsed = JSON.parse(statsRun.stdout) as { dbSizeBytes?: number; sessionCount?: number }; - if (typeof parsed.dbSizeBytes === "number") dbSizeBytes = parsed.dbSizeBytes; - if (typeof parsed.sessionCount === "number" && parsed.sessionCount > 0) { - sessionCount = parsed.sessionCount; - } -} catch { - // 忽略 +let messageCount = 0; +if (typeof statsRun.payload.dbSizeBytes === "number") dbSizeBytes = statsRun.payload.dbSizeBytes; +if (typeof statsRun.payload.sessionCount === "number" && statsRun.payload.sessionCount > 0) { + sessionCount = statsRun.payload.sessionCount; +} +if (typeof statsRun.payload.messageCount === "number") { + messageCount = statsRun.payload.messageCount; } const report: Report = { generatedAt: new Date().toISOString(), + sourceId: args.source, dbPath: args.db, rootDir: args.root, sessionCount, + messageCount, syncMs: Number(syncMs.toFixed(2)), dbSizeBytes, + runsPerQuery: args.runsPerQuery, + readRunsPerProbe: args.readRunsPerProbe, + coverage, perQuery, }; @@ -209,39 +362,169 @@ if (!args.jsonOnly) { writeFileSync(join(outDir, "report.md"), buildMarkdown(report)); } +const readProbes = perQuery.flatMap((row) => [row.readRange, row.readPage].filter((probe): probe is ReadProbeRecord => probe !== null)); const slowest = [...perQuery].sort((a, b) => b.p95Ms - a.p95Ms)[0]; -console.log(JSON.stringify({ +const slowestRead = [...readProbes].sort((a, b) => b.p95Ms - a.p95Ms)[0]; +const summary = { outDir: outDir || null, + sourceId: report.sourceId, sessionCount, - syncMs: Number(syncMs.toFixed(2)), + messageCount, + syncMs: report.syncMs, dbSizeBytes, + coverage: report.coverage, queryCount: perQuery.length, + readProbeCount: readProbes.length, slowestQuery: slowest ? { query: slowest.query, p95Ms: slowest.p95Ms } : null, -}, null, 2)); + slowestRead: slowestRead ? { kind: slowestRead.kind, p95Ms: slowestRead.p95Ms } : null, +}; +console.log(JSON.stringify(args.jsonOnly ? report : summary, null, 2)); + +function topHitFromFind(payload: FindJsonPayload): TopHitRecord | null { + const first = payload.results?.[0]; + if (!first || typeof first.sourceId !== "string" || typeof first.sessionRef !== "string" || typeof first.matchSource !== "string") { + return null; + } + const matchSeq = typeof first.matchSeq === "number" ? first.matchSeq : null; + return { + sourceId: first.sourceId, + sessionRef: first.sessionRef, + matchSource: first.matchSource, + matchSeq, + }; +} + +async function measureReadRange(hit: TopHitRecord): Promise { + if (hit.matchSeq === null) return null; + const cmd = cliCommand( + "read-range", + hit.sessionRef, + "--source", + args.source, + "--seq", + String(hit.matchSeq), + "--before", + "2", + "--after", + "2", + "--db", + args.db, + "--json", + ); + const { latency, payload } = await benchJsonCommand(cmd, args.readRunsPerProbe); + return { + kind: "read-range", + sourceId: hit.sourceId, + sessionRef: hit.sessionRef, + argv: publicArgv(cmd), + messagesReturned: Array.isArray(payload.messages) ? payload.messages.length : 0, + anchorSeq: typeof payload.anchorSeq === "number" ? payload.anchorSeq : undefined, + ...latency, + }; +} + +async function measureReadPage(hit: TopHitRecord): Promise { + const cmd = cliCommand( + "read-page", + hit.sessionRef, + "--source", + args.source, + "--offset", + "0", + "--limit", + "40", + "--db", + args.db, + "--json", + ); + const { latency, payload } = await benchJsonCommand(cmd, args.readRunsPerProbe); + return { + kind: "read-page", + sourceId: hit.sourceId, + sessionRef: hit.sessionRef, + argv: publicArgv(cmd), + messagesReturned: Array.isArray(payload.messages) ? payload.messages.length : 0, + totalCount: typeof payload.totalCount === "number" ? payload.totalCount : undefined, + ...latency, + }; +} + +function coverageCostSummary(run: RunResult, payload: StatusJsonPayload): CoverageCostSummary { + const coverageRows = Array.isArray(payload.coverage) ? payload.coverage : []; + const freshness = countBy(coverageRows.map((row) => row.freshness ?? "unknown")); + const staleReasons = countBy(coverageRows.map((row) => staleReasonForCoverage(row))); + const requested = payload.requestedCoverage; + return { + statusMs: Number(run.ms.toFixed(2)), + coverageCount: coverageRows.length, + freshness, + staleReasons, + requestedCoverage: requested ? { + freshness: requested.freshness ?? "unknown", + staleReason: requested.staleReason ?? "unknown", + sourceFileCount: requested.sourceFileCount ?? 0, + recommendedAction: requested.recommendedAction ?? "unknown", + } : null, + }; +} + +function staleReasonForCoverage(row: NonNullable[number]): string { + if (row.freshness !== "stale") return "none"; + return row.sourceFileSetFingerprint && row.currentSourceFileSetFingerprint === row.sourceFileSetFingerprint + ? "source_content_changed" + : "source_set_changed"; +} + +function countBy(values: string[]): Record { + return values.reduce>((acc, value) => { + acc[value] = (acc[value] ?? 0) + 1; + return acc; + }, {}); +} function buildMarkdown(r: Report): string { const lines: string[] = []; lines.push("# shlog 性能基准报告"); lines.push(""); lines.push(`- generated_at: ${r.generatedAt}`); + lines.push(`- source: \`${r.sourceId}\``); lines.push(`- root: \`${r.rootDir}\``); lines.push(`- db: \`${r.dbPath}\``); lines.push(`- session_count: ${r.sessionCount}`); + lines.push(`- message_count: ${r.messageCount}`); lines.push(`- sync_ms: ${r.syncMs.toFixed(1)}`); lines.push(`- db_size: ${fmtBytes(r.dbSizeBytes)} (${r.dbSizeBytes} bytes)`); + lines.push(`- find_runs_per_query: ${r.runsPerQuery} (first run is warmup when runs > 1)`); + lines.push(`- read_runs_per_probe: ${r.readRunsPerProbe} (first run is warmup when runs > 1)`); lines.push(""); - lines.push("## per-query find latency"); + lines.push("## coverage and freshness cost"); + lines.push(""); + lines.push(`- status_ms: ${r.coverage.statusMs.toFixed(1)}`); + lines.push(`- coverage_count: ${r.coverage.coverageCount}`); + lines.push(`- freshness: \`${JSON.stringify(r.coverage.freshness)}\``); + lines.push(`- stale_reasons: \`${JSON.stringify(r.coverage.staleReasons)}\``); + if (r.coverage.requestedCoverage) { + lines.push(`- requested_coverage: \`${JSON.stringify(r.coverage.requestedCoverage)}\``); + } lines.push(""); - lines.push(`运行配置: 每个 query ${RUNS_PER_QUERY} 次,丢弃首次 warmup,统计后 ${RUNS_PER_QUERY - 1} 次。`); + lines.push("## per-query latency and raw-read probes"); lines.push(""); - lines.push("| query | runs | p50 ms | p95 ms | samples (incl. warmup) ms |"); - lines.push("|-------|-----:|-------:|-------:|---------------------------|"); + lines.push("| query | results | scanned msgs | top hit | find p50 ms | find p95 ms | read-range p95 ms | read-page p95 ms |"); + lines.push("|-------|--------:|-------------:|---------|------------:|------------:|------------------:|----------------:|"); for (const row of r.perQuery) { - const samples = row.samplesMs.map((x) => x.toFixed(1)).join(", "); - lines.push(`| \`${row.query}\` | ${row.runs} |${fmtMs(row.p50Ms)} |${fmtMs(row.p95Ms)} | ${samples} |`); + lines.push([ + `| \`${row.query}\``, + row.resultCount.toString(), + row.scannedMessageCount.toString(), + row.topHit ? `\`${row.topHit.sourceId}/${row.topHit.matchSource}\`` : "-", + fmtMs(row.p50Ms), + fmtMs(row.p95Ms), + row.readRange ? fmtMs(row.readRange.p95Ms) : "-", + `${row.readPage ? fmtMs(row.readPage.p95Ms) : "-"} |`, + ].join(" | ")); } lines.push(""); - lines.push("> 注: 4 个有效样本下 p95 取最大值作为 worst-case 近似,统计意义有限。"); + lines.push("> 注: 小样本下 p95 取最大值作为 worst-case 近似;报告只包含计数、耗时、source/session ref 和命令参数,不包含 transcript 内容。"); lines.push(""); return lines.join("\n"); } diff --git a/eval/run-acceptance-eval.ts b/eval/run-acceptance-eval.ts new file mode 100644 index 0000000..48bdfdc --- /dev/null +++ b/eval/run-acceptance-eval.ts @@ -0,0 +1,9 @@ +#!/usr/bin/env -S node --import tsx + +import { runAcceptanceGate } from "./acceptance-gate"; + +const keepTemp = process.argv.includes("--keep-temp"); +const result = await runAcceptanceGate({ keepTemp }); + +console.log(JSON.stringify(result, null, 2)); +if (result.scoreboard.hardFail > 0) process.exitCode = 1; diff --git a/package.json b/package.json index fce30f7..8d1cd3d 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "shlog": "tsx ./src/cli.ts", "cxs": "npm run shlog --", "eval:manual": "tsx ./eval/run-manual-eval.ts", + "eval:acceptance": "tsx ./eval/run-acceptance-eval.ts", "eval:compare": "tsx ./eval/compare-eval-batches.ts", "eval:perf": "tsx ./eval/perf-bench.ts", "eval:dogfood": "tsx ./eval/run-dogfood-eval.ts" diff --git a/src/cli.test.ts b/src/cli.test.ts index 420ac89..f55025b 100644 --- a/src/cli.test.ts +++ b/src/cli.test.ts @@ -4,7 +4,7 @@ import { chmodSync, existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } import { tmpdir } from "node:os"; import { join } from "node:path"; import Database from "better-sqlite3"; -import { openWriteDb } from "./db"; +import { openWriteDb, replaceSession } from "./db"; import { INDEX_VERSION } from "./env"; import { syncSessions } from "./indexer"; @@ -345,6 +345,92 @@ describe("shlog cli", { timeout: 20_000 }, () => { expect(findPayload.results[0]?.sessionUuid).toBe("12121212-1212-4212-8212-121212121212"); }); + test("find --json gives session-only hits a read-range --query action to locate the real anchor", async () => { + const base = mkdtempSync(join(tmpdir(), "cxs-cli-session-evidence-")); + tempDirs.push(base); + const dbPath = join(base, "index.sqlite"); + const sessionUuid = "82828282-8282-4828-8828-828282828282"; + const db = openWriteDb(dbPath); + + replaceSession(db, { + sessionUuid, + filePath: join(base, "session-only.jsonl"), + title: "payloadbeacon postmortem outline", + summaryText: "", + compactText: "", + reasoningSummaryText: "", + cwd: "/tmp/session-evidence", + model: "gpt-5.4", + startedAt: "2026-04-24T01:00:00.000Z", + endedAt: "2026-04-24T01:00:00.000Z", + messages: [ + { + role: "user", + contentText: "first raw transcript line without the query token", + timestamp: "2026-04-24T01:00:00.000Z", + seq: 0, + sourceKind: "event_msg", + }, + { + role: "assistant", + contentText: "second raw transcript line for follow-up evidence", + timestamp: "2026-04-24T01:00:30.000Z", + seq: 1, + sourceKind: "event_msg", + }, + ], + }, 1, 1, INDEX_VERSION, ""); + db.close(); + + const found = await runCli(["find", "payloadbeacon", "--db", dbPath, "--json"]); + expect(found.exitCode).toBe(0); + const payload = JSON.parse(found.stdout) as { + results: Array<{ + sessionUuid: string; + matchSource: string; + matchSeq: number | null; + evidenceRead: { + kind: "read-range"; + reason: string; + sourceId: string; + sessionRef: string; + query: string; + before: number; + after: number; + argv: string[]; + }; + }>; + }; + const result = payload.results[0]; + + expect(result?.sessionUuid).toBe(sessionUuid); + expect(result?.matchSource).toBe("session"); + expect(result?.matchSeq).toBeNull(); + expect(result?.evidenceRead).toMatchObject({ + kind: "read-range", + reason: "session_level_match", + sourceId: "codex", + sessionRef: sessionUuid, + query: "payloadbeacon", + before: 2, + after: 2, + }); + expect(result?.evidenceRead.argv).toEqual(["shlog", "read-range", sessionUuid, "--query", "payloadbeacon", "--before", "2", "--after", "2"]); + + const page = await runCli([ + ...result?.evidenceRead.argv.slice(1), // drop leading "shlog" + "--db", + dbPath, + "--json", + ]); + expect(page.exitCode).toBe(0); + const pagePayload = JSON.parse(page.stdout) as { messages: Array<{ seq: number; contentText: string }> }; + expect(pagePayload.messages[0]).toMatchObject({ + seq: 0, + contentText: "first raw transcript line without the query token", + }); + }); + test("fixed commands reject unsupported --source values before other work", async () => { const base = mkdtempSync(join(tmpdir(), "cxs-cli-source-unsupported-")); tempDirs.push(base); diff --git a/src/cli.ts b/src/cli.ts index 6ea5b57..aadccc0 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -10,6 +10,7 @@ import { statsReadoutEnabled, } from "./env"; import { IndexSchemaUpgradeRequiredError, IndexUnavailableError, listCoverageRecords, withReadDb } from "./db"; +import { buildEvidenceReadAction } from "./evidence-read"; import { getSessionSourceAdapter, listSessionSourceAdapters } from "./sources"; // One-shot migration from legacy cxs data dirs to the current shlog state dir. @@ -178,7 +179,14 @@ program // 含 better-sqlite3 模块加载;shlog 是一次性进程,所以这就是诚实的端到端。 const elapsedMs = Math.round(performance.now()); if (options.json) { - console.log(JSON.stringify({ ...result, elapsedMs }, null, 2)); + console.log(JSON.stringify({ + ...result, + results: result.results.map((findResult) => ({ + ...findResult, + evidenceRead: buildEvidenceReadAction({ ...findResult, query: result.query }), + })), + elapsedMs, + }, null, 2)); return; } printFindResults(result.query, result.results, result.scannedMessageCount, elapsedMs, statsReadoutEnabled(), result.nextAction); diff --git a/src/evidence-read.test.ts b/src/evidence-read.test.ts new file mode 100644 index 0000000..d811da6 --- /dev/null +++ b/src/evidence-read.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, test } from "vitest"; +import { buildEvidenceReadAction } from "./evidence-read"; + +describe("buildEvidenceReadAction", () => { + test("message hits resolve to a bounded read-range command", () => { + expect(buildEvidenceReadAction({ + sourceId: "codex", + sessionRef: "11111111-1111-4111-8111-111111111111", + matchSeq: 7, + query: "ranking weights", + })).toEqual({ + kind: "read-range", + reason: "message_match", + sourceId: "codex", + sessionRef: "11111111-1111-4111-8111-111111111111", + seq: 7, + before: 2, + after: 2, + argv: [ + "shlog", + "read-range", + "11111111-1111-4111-8111-111111111111", + "--seq", + "7", + "--before", + "2", + "--after", + "2", + ], + }); + }); + + test("session-only hits with query resolve to read-range --query (finds real anchor, not offset=0)", () => { + expect(buildEvidenceReadAction({ + sourceId: "claude-code", + sessionRef: "claude-code:session-abc", + matchSeq: null, + query: "durable output queue", + })).toEqual({ + kind: "read-range", + reason: "session_level_match", + sourceId: "claude-code", + sessionRef: "claude-code:session-abc", + query: "durable output queue", + before: 2, + after: 2, + argv: [ + "shlog", + "read-range", + "claude-code:session-abc", + "--query", + "durable output queue", + "--before", + "2", + "--after", + "2", + ], + }); + }); + + test("session-only hits without query fall back to read-page offset=0", () => { + expect(buildEvidenceReadAction({ + sourceId: "claude-code", + sessionRef: "claude-code:session-abc", + matchSeq: null, + })).toEqual({ + kind: "read-page", + reason: "session_level_match", + sourceId: "claude-code", + sessionRef: "claude-code:session-abc", + offset: 0, + limit: 40, + argv: ["shlog", "read-page", "claude-code:session-abc", "--offset", "0", "--limit", "40"], + }); + }); +}); diff --git a/src/evidence-read.ts b/src/evidence-read.ts new file mode 100644 index 0000000..aa193f8 --- /dev/null +++ b/src/evidence-read.ts @@ -0,0 +1,110 @@ +import { PROGRAM_NAME } from "./env"; +import type { FindResult, SessionSourceId } from "./types"; + +const DEFAULT_READ_RANGE_BEFORE = 2; +const DEFAULT_READ_RANGE_AFTER = 2; +const DEFAULT_SESSION_PAGE_OFFSET = 0; +const DEFAULT_SESSION_PAGE_LIMIT = 40; + +export type EvidenceReadAction = + | { + kind: "read-range"; + reason: "message_match" | "session_level_match"; + sourceId: SessionSourceId; + sessionRef: string; + seq: number; + before: number; + after: number; + argv: string[]; + } + | { + kind: "read-range"; + reason: "session_level_match"; + sourceId: SessionSourceId; + sessionRef: string; + query: string; + before: number; + after: number; + argv: string[]; + } + | { + kind: "read-page"; + reason: "session_level_match"; + sourceId: SessionSourceId; + sessionRef: string; + offset: number; + limit: number; + argv: string[]; + }; + +export function buildEvidenceReadAction( + result: Pick & { query?: string }, +): EvidenceReadAction { + if (result.matchSeq === null) { + // Session-level hit: the match came from session metadata/compact, not a + // specific message. When we have the query, point read-range at it so + // resolveAnchorSeq can locate the real evidence anchor inside the session + // transcript. Without a query we fall back to read-page from the start. + if (result.query) { + return { + kind: "read-range", + reason: "session_level_match", + sourceId: result.sourceId, + sessionRef: result.sessionRef, + query: result.query, + before: DEFAULT_READ_RANGE_BEFORE, + after: DEFAULT_READ_RANGE_AFTER, + argv: [ + PROGRAM_NAME, + "read-range", + result.sessionRef, + "--query", + result.query, + "--before", + String(DEFAULT_READ_RANGE_BEFORE), + "--after", + String(DEFAULT_READ_RANGE_AFTER), + ], + }; + } + + return { + kind: "read-page", + reason: "session_level_match", + sourceId: result.sourceId, + sessionRef: result.sessionRef, + offset: DEFAULT_SESSION_PAGE_OFFSET, + limit: DEFAULT_SESSION_PAGE_LIMIT, + argv: [ + PROGRAM_NAME, + "read-page", + result.sessionRef, + "--offset", + String(DEFAULT_SESSION_PAGE_OFFSET), + "--limit", + String(DEFAULT_SESSION_PAGE_LIMIT), + ], + }; + } + + return { + kind: "read-range", + reason: "message_match", + sourceId: result.sourceId, + sessionRef: result.sessionRef, + seq: result.matchSeq, + before: DEFAULT_READ_RANGE_BEFORE, + after: DEFAULT_READ_RANGE_AFTER, + argv: [ + PROGRAM_NAME, + "read-range", + result.sessionRef, + "--seq", + String(result.matchSeq), + "--before", + String(DEFAULT_READ_RANGE_BEFORE), + "--after", + String(DEFAULT_READ_RANGE_AFTER), + ], + }; +} diff --git a/src/query/read.ts b/src/query/read.ts index 7e3c968..447e50e 100644 --- a/src/query/read.ts +++ b/src/query/read.ts @@ -95,6 +95,10 @@ function resolveAnchorSeq( if (query) { const best = searchTopHitInSession(db, session, query); if (best && typeof best.matchSeq === "number") return best.matchSeq; + // Query found no message-level hit (e.g. session-level match from title or + // compact). Fall back to seq=0 so read-range still returns a usable window + // instead of throwing — the caller can page forward if needed. + return 0; } throw new Error("read-range requires explicit session_uuid plus either --seq or --query"); diff --git a/src/ranking.ts b/src/ranking.ts index 0832a08..cee2bc1 100644 --- a/src/ranking.ts +++ b/src/ranking.ts @@ -23,6 +23,7 @@ export interface RawHitRow { } export interface QueryProfile { + kind: "broad" | "exact"; normalizedQuery: string; terms: string[]; isMultiTerm: boolean; @@ -30,12 +31,12 @@ export interface QueryProfile { } /** - * Kept for backward compat with callers that still read `kind`. The v3 - * scoring pipeline does not branch on kind anymore; it treats `isMultiTerm` - * as a continuous signal instead. Single-term queries are labelled "broad", - * everything else "exact" to preserve existing external tests. + * Kept for backward compat with callers that still read `kind`. The current + * scoring pipeline does not branch on broad/exact because real-data A/B showed + * no useful ranking impact; phrase, term, path-like, and metadata signals are + * applied uniformly instead. */ -export function classifyQueryProfile(query: string): QueryProfile & { kind: "broad" | "exact" } { +export function classifyQueryProfile(query: string): QueryProfile { const normalizedQuery = query.trim().toLowerCase(); const terms = queryTerms(query); diff --git a/src/sources/claude-code.test.ts b/src/sources/claude-code.test.ts index 6b26615..fcea16c 100644 --- a/src/sources/claude-code.test.ts +++ b/src/sources/claude-code.test.ts @@ -134,6 +134,66 @@ describe("claude-code source adapter", () => { expect(searchableProjection).not.toContain("attachment text must not leak"); }); + test("sync skips malformed and unsupported records without leaking format-drift text", async () => { + const { root } = writeClaudeFixture("format-drift", [ + "{this is not json", + claudeLine({ + type: "system", + sessionId: "unsupported-session-must-not-win", + cwd: "/tmp/unsupported-cwd-must-not-win", + timestamp: "1999-01-01T00:00:00.000Z", + message: { content: "unsupported claude text must not leak" }, + }), + claudeLine({ + type: "user", + sessionId: "format-drift-session", + cwd: "/tmp/claude-format-cwd", + timestamp: "2026-06-10T00:00:00.000Z", + message: { content: "accepted claude format drift needle" }, + }), + "{\"type\":\"assistant\",\"message\":", + claudeLine({ + type: "assistant", + sessionId: "format-drift-session", + cwd: "/tmp/claude-format-cwd", + timestamp: "2026-06-10T00:00:01.000Z", + message: { + content: [ + { type: "text", text: "accepted claude format drift answer" }, + { type: "attachment", text: "format drift attachment must not leak" }, + ], + }, + }), + ]); + const dbPath = join(root, "index.sqlite"); + + const summary = await syncSessions({ + dbPath, + sourceId: "claude-code", + selector: { source: "claude-code", kind: "all", root }, + }); + + expect(summary.errors).toBe(0); + expect(summary.added).toBe(1); + expect(summary.coverage.sourceFileCount).toBe(1); + expect(summary.coverage.indexedSessionCount).toBe(1); + + const foundAccepted = findSessions(dbPath, "accepted claude format drift needle", 10, { source: "claude-code", kind: "all", root }, { sourceId: "claude-code" }); + expect(foundAccepted.results.map((result) => result.sessionUuid)).toEqual(["claude-code:format-drift-session"]); + + const foundUnsupported = findSessions(dbPath, "unsupported claude text", 10, { source: "claude-code", kind: "all", root }, { sourceId: "claude-code" }); + expect(foundUnsupported.results).toEqual([]); + + const page = getMessagePage(dbPath, "claude-code:format-drift-session", 0, 10); + expect(page.session.cwd).toBe("/tmp/claude-format-cwd"); + expect(page.messages.map((message) => message.contentText)).toEqual([ + "accepted claude format drift needle", + "accepted claude format drift answer", + ]); + expect(JSON.stringify(page)).not.toContain("unsupported claude text must not leak"); + expect(JSON.stringify(page)).not.toContain("format drift attachment must not leak"); + }); + test("ignores skipped records when deriving inventory grouping dates and snapshot file metadata", async () => { const { root, filePath } = writeClaudeFixture("inventory-policy", [ claudeLine({ diff --git a/src/sources/codex.test.ts b/src/sources/codex.test.ts index 31348cf..d25f897 100644 --- a/src/sources/codex.test.ts +++ b/src/sources/codex.test.ts @@ -60,6 +60,54 @@ describe("codex source adapter", () => { expect(parsed.session.sessionUuid).toBe("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"); expect(parsed.session.messages[0]?.contentText).toBe("adapter session"); }); + + test("filters unsupported, internal, and malformed Codex records from searchable projection", async () => { + const base = mkdtempSync(join(tmpdir(), "cxs-codex-contract-")); + tempDirs.push(base); + const root = join(base, "sessions"); + const day = join(root, "2026", "04", "23"); + mkdirSync(day, { recursive: true }); + const filePath = join( + day, + "rollout-2026-04-23T12-00-00-bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb.jsonl", + ); + writeFileSync( + filePath, + [ + "{not json", + line("session_meta", { id: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", cwd: "/tmp/codex-contract" }), + line("event_msg", { type: "tool_result", message: "tool result must not leak" }), + line("event_msg", { type: "user_message", message: "accepted codex user text" }), + line("event_msg", { + type: "user_message", + message: "The following is the Codex agent history whose request action you are assessing\ninternal marker must not leak", + }), + line("event_msg", { type: "agent_message", message: "accepted codex assistant text" }), + line("other_record", { message: "unrelated record must not leak" }), + ].join("\n"), + ); + + const parsed = await codexSourceAdapter.parseFile({ + filePath, + cwd: "/tmp/fallback", + pathDate: "2026-04-23", + mtimeMs: 0, + size: 0, + }); + + expect(parsed.kind).toBe("parsed"); + if (parsed.kind !== "parsed") return; + expect(parsed.session.messages.map((message) => message.contentText)).toEqual([ + "accepted codex user text", + "accepted codex assistant text", + ]); + + const searchableProjection = JSON.stringify(parsed.session); + expect(searchableProjection).not.toContain("tool result must not leak"); + expect(searchableProjection).not.toContain("internal marker must not leak"); + expect(searchableProjection).not.toContain("unrelated record must not leak"); + expect(searchableProjection).not.toContain("{not json"); + }); }); function line(type: string, payload: Record): string { diff --git a/src/sources/pi.test.ts b/src/sources/pi.test.ts index a7059cb..557ea3a 100644 --- a/src/sources/pi.test.ts +++ b/src/sources/pi.test.ts @@ -141,6 +141,64 @@ describe("pi source adapter", () => { expect(searchableProjection).not.toContain("tool call must not leak"); }); + test("sync skips malformed and unsupported records without leaking format-drift text", async () => { + const { root } = writePiFixture("format-drift", [ + "{this is not json", + piLine({ type: "session", id: "pi-format-drift-session", cwd: "/tmp/pi-format-cwd", timestamp: "2026-06-10T00:00:00.000Z" }), + piLine({ type: "future_event", timestamp: "1999-01-01T00:00:00.000Z", text: "unsupported pi text must not leak" }), + piLine({ + type: "message", + timestamp: "2026-06-10T00:00:01.000Z", + message: { role: "user", content: [{ type: "text", text: "accepted pi format drift needle" }], timestamp: "2026-06-10T00:00:01.000Z" }, + }), + "{\"type\":\"message\",\"message\":", + piLine({ + type: "message", + timestamp: "2026-06-10T00:00:02.000Z", + message: { + role: "assistant", + content: [ + { type: "text", text: "accepted pi format drift answer" }, + { type: "toolCall", name: "bash", arguments: { command: "format drift tool call must not leak" } }, + ], + timestamp: "2026-06-10T00:00:02.000Z", + }, + }), + piLine({ type: "compaction", id: "c1", timestamp: "2026-06-10T00:00:03.000Z", summary: "accepted pi format drift summary" }), + ]); + const dbPath = join(root, "index.sqlite"); + + const summary = await syncSessions({ + dbPath, + sourceId: "pi", + selector: { source: "pi", kind: "all", root }, + }); + + expect(summary.errors).toBe(0); + expect(summary.added).toBe(1); + expect(summary.coverage.sourceFileCount).toBe(1); + expect(summary.coverage.indexedSessionCount).toBe(1); + + const foundAccepted = findSessions(dbPath, "accepted pi format drift needle", 10, { source: "pi", kind: "all", root }, { sourceId: "pi" }); + expect(foundAccepted.results.map((result) => result.sessionUuid)).toEqual(["pi:pi-format-drift-session"]); + + const foundCompaction = findSessions(dbPath, "accepted pi format drift summary", 10, { source: "pi", kind: "all", root }, { sourceId: "pi" }); + expect(foundCompaction.results.map((result) => result.sessionUuid)).toEqual(["pi:pi-format-drift-session"]); + expect(foundCompaction.results[0]?.matchSource).toBe("session"); + + const foundUnsupported = findSessions(dbPath, "unsupported pi text", 10, { source: "pi", kind: "all", root }, { sourceId: "pi" }); + expect(foundUnsupported.results).toEqual([]); + + const page = getMessagePage(dbPath, "pi:pi-format-drift-session", 0, 10); + expect(page.session.cwd).toBe("/tmp/pi-format-cwd"); + expect(page.messages.map((message) => message.contentText)).toEqual([ + "accepted pi format drift needle", + "accepted pi format drift answer", + ]); + expect(JSON.stringify(page)).not.toContain("unsupported pi text must not leak"); + expect(JSON.stringify(page)).not.toContain("format drift tool call must not leak"); + }); + test("uses the latest Pi model_change as session model", async () => { const { filePath } = writePiFixture("latest-model", [ piLine({ type: "session", id: "pi-model-session", cwd: "/tmp/pi-model-cwd", timestamp: "2026-06-06T00:00:00.000Z" }),