From ee242b2c31025d40f32b54bace3799f4daf8b09b Mon Sep 17 00:00:00 2001 From: catoncat Date: Mon, 13 Jul 2026 10:04:29 +0800 Subject: [PATCH] =?UTF-8?q?feat(read):=20=E4=B8=BA=E5=A4=A7=E6=B6=88?= =?UTF-8?q?=E6=81=AF=E5=A2=9E=E5=8A=A0=E7=A1=AE=E5=AE=9A=E6=80=A7=E7=9C=81?= =?UTF-8?q?=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 实现 #89：read-range/read-page 默认省略超大单条消息，保留 seq/role/timestamp 与 query 证据 span，并输出结构化 elision metadata。 --- skill-packages/sherlog/SKILL.md | 2 +- .../sherlog/references/cli-surface.md | 7 ++ .../sherlog/references/json-schema.md | 57 ++++++++- .../references/progressive-workflow.md | 8 +- src/cli.test.ts | 119 ++++++++++++++++++ src/cli.ts | 15 ++- src/evidence-read.test.ts | 3 + src/evidence-read.ts | 27 ++-- src/format.ts | 28 ++++- src/query/message-elision.ts | 94 ++++++++++++++ src/query/read.ts | 14 ++- src/types.ts | 10 ++ 12 files changed, 359 insertions(+), 25 deletions(-) create mode 100644 src/query/message-elision.ts diff --git a/skill-packages/sherlog/SKILL.md b/skill-packages/sherlog/SKILL.md index 6f447e3..e7be77e 100644 --- a/skill-packages/sherlog/SKILL.md +++ b/skill-packages/sherlog/SKILL.md @@ -29,7 +29,7 @@ description: "Use proactively for local Codex history and personal setup archaeo - 回答"当时说了什么/决定了什么"前必须用 `read-range` / `read-page` 读内容；`find`、title、summary 只能定位候选。 - `find` 默认跨 public indexed sources 搜索；只有用户指定、缩小范围或诊断时才加 `--source codex|claude-code|pi`。 - 后续读取优先使用 `find --json` 返回的 `sessionRef`；不要从 uuid 自己猜 source。 -- `matchSource = "session"` 时 `matchSeq = null`;先 `read-page`,不要伪造 `read-range --seq`。 +- `find --json` 结果优先跟随 `evidenceRead.argv` 读取证据；`matchSource = "session"` 时 `matchSeq = null`,不要伪造 `read-range --seq`。 - `find` 默认按 relevance 排序；"最新/最近 + 关键词"用 `--sort ended`,必要时 `--exclude-session ` 排除 self-hit。 - 只读 SQLite 只允许查 Sherlog index 的稳定 metadata；内容判断仍回到 `read-*`。 - `sync` 只更新 index/coverage。coverage 缺失或确实 stale 时才同步同一范围；`sync --prune` 只在用户明确要丢弃已消失 source 的旧索引记录时使用。 diff --git a/skill-packages/sherlog/references/cli-surface.md b/skill-packages/sherlog/references/cli-surface.md index 1f8f0bd..220bd10 100644 --- a/skill-packages/sherlog/references/cli-surface.md +++ b/skill-packages/sherlog/references/cli-surface.md @@ -102,6 +102,8 @@ Purpose: 搜索相关 session，返回最小必要命中。用于 semantic recal text header 带效率回述:`shlog find "q" · 检索 ~N 条 · 结果 R · Xms`(`检索 ~N` = 范围内语料规模诚实分母,`--json` 里是 `scannedMessageCount` / `elapsedMs`)。`read-range` / `read-page` 的 header 带「读取 K 条 / 本 session 共 T 条 · Xms」和 `total=… · hasMore=… · Xms`。大规模调查时可用这些真实数字做一句简短尾注,**不要据此编造「省 X%」**。 +`find --json` 的每个结果会带 `evidenceRead`，优先执行里面的 `argv` 去读取内容证据。message-level 命中通常是 `read-range --seq ... --query ...`，这样后续超大消息省略时仍能围绕 query term 保留证据 span；session-level 命中可能是 `read-range --query ...` 或 fallback `read-page`。 + 效率回述默认开,环境变量 `SHLOG_STATS=0`(或 `off`/`false`/`no`)可关闭文本 header 里的注解(`检索 ~N 条 / 读取 K 条 / Xms`);`--json` 的 `scannedMessageCount` / `elapsedMs` 与 `read-page` 的 `total/hasMore` 等功能字段始终保留。关闭时文本里没有可锚的数字,直接省掉效率尾注、别硬编。 零结果不是结束条件。`find --json` 会在同一次调用中评估 raw source freshness，因此 `coverage.complete` / `freshness` / `staleReason` 与 `nextAction` 使用同一 snapshot；纯 SQLite query facade 无法检查 raw 时则诚实返回 `complete=false` / `freshness=not_checked`，但保留 `coveringSelectors`。`--json` 下如果返回 `nextAction`,按它选择/检查同一 selector；text 输出也会打印 `next:` 步骤。Codex `find` 对非空结果会忽略 `source_content_changed` 软 stale,避免当前会话尾部变化反复逼 agent 同步；此时 `complete=false` 表示最新尾部未获证明，不表示已有索引不可查询。若非空结果仍返回 `nextAction.reason=stale_or_missing_coverage`,通常是 coverage 缺失、source file 集合变化或非 Codex source 保守同步；需要完整结论时按 `nextAction.commands` 同步并重试。fresh coverage 下仍无结果,才可以说没找到。 @@ -135,6 +137,7 @@ Notes: - 必须显式传 `` 或 `find` 返回的 `` - 必须二选一提供 `--seq` 或 `--query` +- 默认会确定性省略超大单条消息；`--max-message-chars ` 控制单条消息保留预算，`0` 表示不省略。JSON 会在被省略的 message 上返回 `elision` metadata。 - 可选 `--source codex|claude-code|pi`;省略等价于 Codex。`:` qualifier 会直接决定读取 source；若同时传 `--source`,两者必须匹配。 Example: @@ -142,12 +145,15 @@ Example: ```bash "${SHLOG_BIN:-${CXS_BIN:-shlog}}" read-range --seq 12 --before 4 --after 8 --json "${SHLOG_BIN:-${CXS_BIN:-shlog}}" read-range --query "IME" --before 4 --after 8 --json +"${SHLOG_BIN:-${CXS_BIN:-shlog}}" read-range --seq 12 --query "exact clue" --max-message-chars 2000 --json ``` ## read-page Purpose: 顺序分页读取某个 session 的消息。metadata projection 只能给候选;要确认"当时说了什么/是否有意义",用 `read-page` 或 `read-range`。 +默认会确定性省略超大单条消息；`--max-message-chars ` 控制单条消息保留预算，`0` 表示不省略。JSON 会在被省略的 message 上返回 `elision` metadata。 + 可选 `--source codex|claude-code|pi`;省略等价于 Codex。 如果传入 `claude-code:` 或 `pi:` 这类 `sessionRef`,无需再传对应 `--source`。 @@ -155,6 +161,7 @@ Example: ```bash "${SHLOG_BIN:-${CXS_BIN:-shlog}}" read-page --offset 0 --limit 40 --json +"${SHLOG_BIN:-${CXS_BIN:-shlog}}" read-page --offset 0 --limit 40 --max-message-chars 2000 --json ``` ## list diff --git a/skill-packages/sherlog/references/json-schema.md b/skill-packages/sherlog/references/json-schema.md index 1db4cb6..c9b50f1 100644 --- a/skill-packages/sherlog/references/json-schema.md +++ b/skill-packages/sherlog/references/json-schema.md @@ -41,6 +41,7 @@ Top-level shape: matchTimestamp: string | null; score: number; snippet: string; + evidenceRead: EvidenceReadAction; // 仅 CLI JSON 输出注入；优先执行它读取内容证据 } ``` @@ -49,7 +50,45 @@ Top-level shape: as the read command input; for Codex it is usually the bare UUID, while Claude Code and Pi refs are source-qualified such as `claude-code:` or `pi:`. -`matchSource = "session"` means the hit came from session-level fields such as title, derived summary, compact handoff, or reasoning summary rather than a concrete message. In that case `matchSeq` is `null`; use `read-page` first instead of fabricating a `read-range --seq` anchor. +`matchSource = "session"` means the hit came from session-level fields such as title, derived summary, compact handoff, or reasoning summary rather than a concrete message. In that case `matchSeq` is `null`; follow `evidenceRead.argv` instead of fabricating a `read-range --seq` anchor. + +`EvidenceReadAction`: + +```ts +type EvidenceReadAction = + | { + kind: "read-range"; + reason: "message_match"; + sourceId: "codex" | "claude-code" | "pi"; + sessionRef: string; + seq: number; + query?: string; + before: number; + after: number; + argv: string[]; + } + | { + kind: "read-range"; + reason: "session_level_match"; + sourceId: "codex" | "claude-code" | "pi"; + sessionRef: string; + query: string; + before: number; + after: number; + argv: string[]; + } + | { + kind: "read-page"; + reason: "session_level_match"; + sourceId: "codex" | "claude-code" | "pi"; + sessionRef: string; + offset: number; + limit: number; + argv: string[]; + }; +``` + +优先执行 `evidenceRead.argv`。message-level `read-range` 在有原 query 时会同时带 `--seq` 和 `--query`，用于保持稳定 seq anchor 并让 read command 在超大消息省略时围绕 query term 保留证据 span。session-level hit 有 query 时会用 `read-range --query` 重新定位真实 message anchor；没有 query 时才 fallback `read-page`。 `QueryNextAction` appears on `find` / `list` when the command cannot prove the target coverage is fresh enough for the requested conclusion. For Codex `find`, non-empty results no longer get a retry gate for `source_content_changed` soft stale, which commonly means the current session file is still being appended. If non-empty `find` still returns `stale_or_missing_coverage`, treat it as a harder risk such as missing coverage, a changed source file set, or a non-Codex source staying conservative unless a follow-up `status` says otherwise. @@ -272,6 +311,22 @@ errors in `--json` mode for expected index setup and read failures: contentText: string; timestamp: string; sourceKind: string; + elision?: MessageElision; +} +``` + +`read-range` / `read-page` 默认会对超大单条消息做确定性省略。被省略的 message 保留 `sessionUuid`、`seq`、`role`、`timestamp`、`sourceKind`，并把 `contentText` 替换成保留片段加显式省略 marker；未省略的小消息没有 `elision` 字段。传 `--max-message-chars 0` 可关闭单条消息省略。 + +`MessageElision`: + +```ts +{ + originalCharCount: number; + displayedCharCount: number; + omittedCharCount: number; + strategy: "head_tail" | "around_query"; + query?: string; + hint: string; // 例如 rerun read with --max-message-chars N } ``` diff --git a/skill-packages/sherlog/references/progressive-workflow.md b/skill-packages/sherlog/references/progressive-workflow.md index 3d319d9..bfc4141 100644 --- a/skill-packages/sherlog/references/progressive-workflow.md +++ b/skill-packages/sherlog/references/progressive-workflow.md @@ -18,7 +18,7 @@ Hard rules: - `sync` only updates index/coverage; normal retrieval does not need `sync` unless coverage is missing or stale. Bare `sync` is only a first-install default Codex bootstrap; scoped agent work should still use `sync --cwd` / `sync --root` / `sync --selector`. - Do not use `sync --prune` for normal retrieval. - `find` default sort is relevance; use `--sort ended` only when the user's question is time-oriented. -- `matchSource = "session"` means `matchSeq = null`; use `read-page` instead of inventing a seq. +- `find --json` results include `evidenceRead`; follow `evidenceRead.argv` for content verification. If `matchSource = "session"`, `matchSeq = null`; do not invent a `read-range --seq`. - Current public sources are `codex`, experimental `claude-code`, and experimental `pi`; `find` omits `--source` to search all public indexed sources by default. Pass `--source codex`, `--source claude-code`, or `--source pi` only to narrow or diagnose. Other source-scoped commands still omit `--source` as Codex-compatible default. Claude Code and Pi are part of the normal CLI surface now, but they are still not stable raw-format promises. ## Scenario 1: Metadata Projection @@ -63,13 +63,13 @@ sqlite3 -readonly "$DB_PATH" \ "${SHLOG_BIN:-${CXS_BIN:-shlog}}" sync --root /Users/me/.codex/sessions --json ``` -候选出来后读内容: +候选出来后优先执行结果里的 `evidenceRead.argv` 读内容；message-level 命中通常会同时带 `--seq` 和 `--query`，session-level 命中可能用 `read-range --query` 重新定位: ```bash -"${SHLOG_BIN:-${CXS_BIN:-shlog}}" read-range --seq --before 4 --after 8 --json +"${SHLOG_BIN:-${CXS_BIN:-shlog}}" read-range --seq --query "cf tunnel" --before 2 --after 2 --json ``` -如果 `matchSeq` 是 `null`,改用: +如果没有 `evidenceRead` 或需要手动 fallback,`matchSeq = null` 时用: ```bash "${SHLOG_BIN:-${CXS_BIN:-shlog}}" read-page --offset 0 --limit 40 --json diff --git a/src/cli.test.ts b/src/cli.test.ts index f093f0b..3359d31 100644 --- a/src/cli.test.ts +++ b/src/cli.test.ts @@ -1334,6 +1334,7 @@ describe("shlog cli", { timeout: 20_000 }, () => { const result = await runCli(["find", "health check", "--db", dbPath]); expect(result.exitCode).toBe(0); expect(result.stdout).toContain("next: shlog read-range 44444444-4444-4444-8444-444444444444 --seq 0"); + expect(result.stdout).toContain("--query 'health check'"); expect(result.stdout).not.toContain("next: cxs window"); }); @@ -1767,6 +1768,124 @@ describe("shlog cli", { timeout: 20_000 }, () => { expect(payload2.hasMore).toBe(false); }); + test("read-range JSON elides huge matched messages while preserving query evidence", async () => { + const base = mkdtempSync(join(tmpdir(), "cxs-cli-elide-range-")); + tempDirs.push(base); + const sessionsRoot = join(base, "sessions", "2026", "04", "21"); + mkdirSync(sessionsRoot, { recursive: true }); + + const huge = `${"A".repeat(1_200)} alpha ${"gap ".repeat(300)} evidence ${"B".repeat(1_200)}`; + writeFileSync( + join(sessionsRoot, "rollout-2026-04-21T10-00-00-99999999-9999-4999-8999-999999999999.jsonl"), + [ + line("session_meta", { id: "99999999-9999-4999-8999-999999999999", cwd: "/tmp/elide-range" }), + line("turn_context", { model: "gpt-5.4" }), + line("event_msg", { type: "user_message", message: huge }), + ].join("\n"), + ); + + const dbPath = join(base, "index.sqlite"); + await syncSessions({ dbPath, rootDir: join(base, "sessions") }); + + const result = await runCli([ + "read-range", + "99999999-9999-4999-8999-999999999999", + "--query", + "alpha evidence", + "--before", + "0", + "--after", + "0", + "--json", + "--db", + dbPath, + ]); + + expect(result.exitCode).toBe(0); + const payload = JSON.parse(result.stdout) as { + anchorSeq: number; + messages: Array<{ seq: number; role: string; timestamp: string; sessionUuid: string; contentText: string; elision?: { originalCharCount: number; omittedCharCount: number; strategy: string; query?: string; hint: string } }>; + }; + const message = payload.messages[0]!; + expect(payload.anchorSeq).toBe(0); + expect(message.seq).toBe(0); + expect(message.role).toBe("user"); + expect(message.timestamp).toBe("2026-04-21T00:00:00.000Z"); + expect(message.sessionUuid).toBe("99999999-9999-4999-8999-999999999999"); + expect(message.contentText).toContain("evidence"); + expect(message.contentText.length).toBeLessThan(1_000); + expect(message.elision).toMatchObject({ + originalCharCount: huge.length, + strategy: "around_query", + query: "alpha evidence", + }); + expect(message.elision!.omittedCharCount).toBeGreaterThan(0); + expect(message.elision!.hint).toContain("--max-message-chars"); + }); + + test("read-page elides huge no-match messages and leaves small messages unchanged", async () => { + const base = mkdtempSync(join(tmpdir(), "cxs-cli-elide-page-")); + tempDirs.push(base); + const sessionsRoot = join(base, "sessions", "2026", "04", "21"); + mkdirSync(sessionsRoot, { recursive: true }); + + const huge = `${"START".repeat(200)} ${"middle".repeat(200)} ${"END".repeat(200)}`; + writeFileSync( + join(sessionsRoot, "rollout-2026-04-21T10-00-00-10101010-1010-4010-8010-101010101010.jsonl"), + [ + line("session_meta", { id: "10101010-1010-4010-8010-101010101010", cwd: "/tmp/elide-page" }), + line("turn_context", { model: "gpt-5.4" }), + line("event_msg", { type: "user_message", message: huge }), + line("event_msg", { type: "agent_message", message: "small message stays whole" }), + ].join("\n"), + ); + + const dbPath = join(base, "index.sqlite"); + await syncSessions({ dbPath, rootDir: join(base, "sessions") }); + + const jsonResult = await runCli([ + "read-page", + "10101010-1010-4010-8010-101010101010", + "--offset", + "0", + "--limit", + "2", + "--json", + "--db", + dbPath, + ]); + + expect(jsonResult.exitCode).toBe(0); + const payload = JSON.parse(jsonResult.stdout) as { + messages: Array<{ contentText: string; elision?: { originalCharCount: number; omittedCharCount: number; strategy: string; hint: string } }>; + }; + expect(payload.messages[0]?.contentText).toContain("START"); + expect(payload.messages[0]?.contentText).toContain("END"); + expect(payload.messages[0]?.contentText).not.toContain("middlemiddlemiddlemiddlemiddlemiddle"); + expect(payload.messages[0]?.elision).toMatchObject({ originalCharCount: huge.length, strategy: "head_tail" }); + expect(payload.messages[1]?.contentText).toBe("small message stays whole"); + expect(payload.messages[1]?.elision).toBeUndefined(); + + const textResult = await runCli([ + "read-page", + "10101010-1010-4010-8010-101010101010", + "--offset", + "0", + "--limit", + "1", + "--max-message-chars", + "1500", + "--db", + dbPath, + ]); + + expect(textResult.exitCode).toBe(0); + expect(textResult.stdout).toContain("elided"); + expect(textResult.stdout).toContain("--max-message-chars"); + expect(textResult.stdout).toContain("2026-04-21T00:00:00.000Z"); + expect(textResult.stdout).toContain("END".repeat(200)); + }); + test("sync exits non-zero by default when per-file indexing fails", async () => { const base = mkdtempSync(join(tmpdir(), "cxs-cli-sync-")); tempDirs.push(base); diff --git a/src/cli.ts b/src/cli.ts index f27febc..1c93e2e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -35,6 +35,7 @@ import { listSessionSummaries, SessionNotFoundError, } from "./query"; +import { DEFAULT_MAX_MESSAGE_CHARS } from "./query/message-elision"; import { canonicalizeSelector, parseSelectorJson, SelectorParseError, selectorSource } from "./selector"; import { collectStatus } from "./status"; import { SyncLockTimeoutError } from "./sync-lock"; @@ -213,6 +214,7 @@ program .option("--query ", "用 query 在该 session 内重新定位命中点") .option("--before ", "前文条数", "2") .option("--after ", "后文条数", "2") + .option("--max-message-chars ", "单条超大消息最多保留的字符数；0 表示不省略", String(DEFAULT_MAX_MESSAGE_CHARS)) .option("--db ", "覆盖默认数据库路径", DEFAULT_DB_PATH) .option("--json", "输出 JSON") .action((sessionUuid, options) => { @@ -223,6 +225,7 @@ program query: options.query, before: parsePositiveInt(options.before, 2), after: parsePositiveInt(options.after, 2), + maxMessageChars: parseNonNegativeInt(options.maxMessageChars, DEFAULT_MAX_MESSAGE_CHARS), }); const elapsedMs = Math.round(performance.now()); if (options.json) { @@ -250,6 +253,7 @@ program .option("--source ", `session source (public: ${publicSourceLabel()})`) .option("--offset ", "起始 offset", "0") .option("--limit ", "页大小", "20") + .option("--max-message-chars ", "单条超大消息最多保留的字符数；0 表示不省略", String(DEFAULT_MAX_MESSAGE_CHARS)) .option("--db ", "覆盖默认数据库路径", DEFAULT_DB_PATH) .option("--json", "输出 JSON") .action((sessionUuid, options) => { @@ -260,6 +264,7 @@ program sessionRefForSource(sessionUuid, sourceId), parseNonNegativeInt(options.offset, 0), parsePositiveInt(options.limit, 20), + { maxMessageChars: parseNonNegativeInt(options.maxMessageChars, DEFAULT_MAX_MESSAGE_CHARS) }, ); const elapsedMs = Math.round(performance.now()); if (options.json) { @@ -846,28 +851,34 @@ function buildSessionNotFoundNextAction( function buildReadRangeRetryArgv( sessionRef: string, - options: { seq?: string; query?: string; before?: string; after?: string; db?: string }, + options: { seq?: string; query?: string; before?: string; after?: string; maxMessageChars?: string; db?: string }, ): string[] { const argv = [PROGRAM_NAME, "read-range", sessionRef]; if (options.seq !== undefined) argv.push("--seq", options.seq); if (options.query !== undefined) argv.push("--query", options.query); if (options.before !== undefined) argv.push("--before", options.before); if (options.after !== undefined) argv.push("--after", options.after); + if (isNonDefaultMaxMessageChars(options.maxMessageChars)) argv.push("--max-message-chars", options.maxMessageChars!); if (options.db) argv.push("--db", options.db); return argv; } function buildReadPageRetryArgv( sessionRef: string, - options: { offset?: string; limit?: string; db?: string }, + options: { offset?: string; limit?: string; maxMessageChars?: string; db?: string }, ): string[] { const argv = [PROGRAM_NAME, "read-page", sessionRef]; if (options.offset !== undefined) argv.push("--offset", options.offset); if (options.limit !== undefined) argv.push("--limit", options.limit); + if (isNonDefaultMaxMessageChars(options.maxMessageChars)) argv.push("--max-message-chars", options.maxMessageChars!); if (options.db) argv.push("--db", options.db); return argv; } +function isNonDefaultMaxMessageChars(value: string | undefined): boolean { + return value !== undefined && value !== String(DEFAULT_MAX_MESSAGE_CHARS); +} + function emitSelectorError(error: SelectorParseError, jsonMode: boolean): void { if (jsonMode) { console.log( diff --git a/src/evidence-read.test.ts b/src/evidence-read.test.ts index d811da6..a3893e3 100644 --- a/src/evidence-read.test.ts +++ b/src/evidence-read.test.ts @@ -14,6 +14,7 @@ describe("buildEvidenceReadAction", () => { sourceId: "codex", sessionRef: "11111111-1111-4111-8111-111111111111", seq: 7, + query: "ranking weights", before: 2, after: 2, argv: [ @@ -26,6 +27,8 @@ describe("buildEvidenceReadAction", () => { "2", "--after", "2", + "--query", + "ranking weights", ], }); }); diff --git a/src/evidence-read.ts b/src/evidence-read.ts index aa193f8..44bfc1a 100644 --- a/src/evidence-read.ts +++ b/src/evidence-read.ts @@ -13,6 +13,7 @@ export type EvidenceReadAction = sourceId: SessionSourceId; sessionRef: string; seq: number; + query?: string; before: number; after: number; argv: string[]; @@ -87,24 +88,28 @@ export function buildEvidenceReadAction( }; } + const argv = [ + PROGRAM_NAME, + "read-range", + result.sessionRef, + "--seq", + String(result.matchSeq), + "--before", + String(DEFAULT_READ_RANGE_BEFORE), + "--after", + String(DEFAULT_READ_RANGE_AFTER), + ]; + if (result.query) argv.push("--query", result.query); + return { kind: "read-range", reason: "message_match", sourceId: result.sourceId, sessionRef: result.sessionRef, seq: result.matchSeq, + ...(result.query ? { query: result.query } : {}), before: DEFAULT_READ_RANGE_BEFORE, after: DEFAULT_READ_RANGE_AFTER, - argv: [ - PROGRAM_NAME, - "read-range", - result.sessionRef, - "--seq", - String(result.matchSeq), - "--before", - String(DEFAULT_READ_RANGE_BEFORE), - "--after", - String(DEFAULT_READ_RANGE_AFTER), - ], + argv, }; } diff --git a/src/format.ts b/src/format.ts index f30127f..c0bb8ac 100644 --- a/src/format.ts +++ b/src/format.ts @@ -75,7 +75,7 @@ export function printFindResults( if (result.matchSeq === null) { console.log(chalk.gray(`next: ${PROGRAM_NAME} read-page ${result.sessionRef} --offset 0 --limit 40`)); } else { - console.log(chalk.gray(`next: ${PROGRAM_NAME} read-range ${result.sessionRef} --seq ${result.matchSeq}`)); + console.log(chalk.gray(`next: ${PROGRAM_NAME} read-range ${result.sessionRef} --seq ${result.matchSeq} --query ${shellArg(query)}`)); } } printNextAction(nextAction); @@ -103,7 +103,8 @@ export function printReadRangeResult( for (const message of messages) { const marker = message.seq === anchorSeq ? chalk.green(">>") : " "; const role = message.role === "user" ? chalk.blue("U") : chalk.white("A"); - console.log(`${marker} [${message.seq}] ${role} ${trimTranscriptMessage(message.contentText)}`); + console.log(`${marker} [${message.seq}] ${role}${message.elision ? ` ${message.timestamp}` : ""} ${formatTranscriptMessage(message)}`); + printElisionMetadata(message); } } @@ -126,7 +127,8 @@ export function printReadPage( for (const message of messages) { const role = message.role === "user" ? chalk.blue("U") : chalk.white("A"); - console.log(`[${message.seq}] ${role} ${trimTranscriptMessage(message.contentText)}`); + console.log(`[${message.seq}] ${role}${message.elision ? ` ${message.timestamp}` : ""} ${formatTranscriptMessage(message)}`); + printElisionMetadata(message); } } @@ -209,6 +211,26 @@ function trimTranscriptMessage(text: string): string { return trimText(text, TRANSCRIPT_TEXT_BUDGET); } +function formatTranscriptMessage(message: MessageRecord): string { + if (message.elision) return collapseWhitespace(message.contentText); + return trimTranscriptMessage(message.contentText); +} + +function printElisionMetadata(message: MessageRecord): void { + if (!message.elision) return; + const elision = message.elision; + console.log(chalk.gray(` elided ${elision.omittedCharCount}/${elision.originalCharCount} chars (${elision.strategy}); ${elision.hint}`)); +} + +function shellArg(value: string): string { + if (/^[A-Za-z0-9_./:@=-]+$/.test(value)) return value; + return `'${value.replaceAll("'", "'\\''")}'`; +} + +function collapseWhitespace(text: string): string { + return trimText(text, Number.MAX_SAFE_INTEGER); +} + function trimText(text: string, limit: number): string { // OPTIMIZATION: Avoid expensive regex replace operations (text.replace(/\s+/g, " ")) // on large strings. Use a single-pass loop with charCodeAt to track spaces diff --git a/src/query/message-elision.ts b/src/query/message-elision.ts new file mode 100644 index 0000000..f2ad1d9 --- /dev/null +++ b/src/query/message-elision.ts @@ -0,0 +1,94 @@ +import type { MessageElision, MessageRecord } from "../types"; +import { queryTerms } from "../tokenize"; + +export const DEFAULT_MAX_MESSAGE_CHARS = 800; + +export function elideMessages( + messages: MessageRecord[], + options: { maxMessageChars?: number; anchorSeq?: number; query?: string } = {}, +): MessageRecord[] { + const maxMessageChars = options.maxMessageChars ?? DEFAULT_MAX_MESSAGE_CHARS; + if (maxMessageChars <= 0) return messages; + + return messages.map((message) => { + const query = message.seq === options.anchorSeq ? options.query : undefined; + return elideMessage(message, maxMessageChars, query); + }); +} + +function elideMessage(message: MessageRecord, maxMessageChars: number, query?: string): MessageRecord { + if (message.contentText.length <= maxMessageChars) return message; + + const anchor = query ? findQueryAnchor(message.contentText, query) : null; + const strategy: MessageElision["strategy"] = anchor ? "around_query" : "head_tail"; + const preserved = strategy === "around_query" + ? preserveAroundQuery(message.contentText, maxMessageChars, anchor!.index, anchor!.length) + : preserveHeadTail(message.contentText, maxMessageChars); + const omittedCharCount = message.contentText.length - preserved.visibleCharCount; + const hint = `Rerun this read with --max-message-chars ${message.contentText.length} to inspect the full message.`; + const elision: MessageElision = { + originalCharCount: message.contentText.length, + displayedCharCount: preserved.text.length, + omittedCharCount, + strategy, + ...(strategy === "around_query" && query ? { query } : {}), + hint, + }; + return { + ...message, + contentText: preserved.text, + elision, + }; +} + +function findQueryAnchor(text: string, query: string): { index: number; length: number } | null { + const textLower = text.toLowerCase(); + const queryLower = query.toLowerCase(); + const phraseIndex = textLower.indexOf(queryLower); + if (phraseIndex >= 0) return { index: phraseIndex, length: query.length }; + + const terms = queryTerms(query).sort((left, right) => right.length - left.length); + for (const term of terms) { + const termIndex = textLower.indexOf(term.toLowerCase()); + if (termIndex >= 0) return { index: termIndex, length: term.length }; + } + + return null; +} + +function preserveAroundQuery( + text: string, + maxMessageChars: number, + queryIndex: number, + queryLength: number, +): { text: string; visibleCharCount: number } { + const budget = Math.max(maxMessageChars, queryLength); + const start = Math.max(0, queryIndex - Math.floor((budget - queryLength) / 2)); + const end = Math.min(text.length, start + budget); + const adjustedStart = Math.max(0, end - budget); + return markElision(text, adjustedStart, end); +} + +function preserveHeadTail(text: string, maxMessageChars: number): { text: string; visibleCharCount: number } { + const headCount = Math.min(Math.ceil(maxMessageChars / 2), text.length); + const tailCount = Math.min(Math.floor(maxMessageChars / 2), Math.max(0, text.length - headCount)); + return markElision(text, 0, headCount, text.length - tailCount, text.length); +} + +function markElision( + text: string, + firstStart: number, + firstEnd: number, + secondStart?: number, + secondEnd?: number, +): { text: string; visibleCharCount: number } { + const visible = secondStart === undefined + ? text.slice(firstStart, firstEnd) + : `${text.slice(firstStart, firstEnd)}\n[... shlog elided middle ...]\n${text.slice(secondStart, secondEnd)}`; + const prefix = firstStart > 0 ? "[... shlog elided prefix ...]\n" : ""; + const suffix = secondEnd === undefined && firstEnd < text.length ? "\n[... shlog elided suffix ...]" : ""; + const visibleCharCount = secondStart === undefined + ? firstEnd - firstStart + : (firstEnd - firstStart) + (secondEnd! - secondStart); + return { text: `${prefix}${visible}${suffix}`, visibleCharCount }; +} diff --git a/src/query/read.ts b/src/query/read.ts index 447e50e..08caf35 100644 --- a/src/query/read.ts +++ b/src/query/read.ts @@ -2,6 +2,7 @@ import { coverageEntriesForSession, getMessagesForPage, getMessagesForRange, get import { rerankHits } from "../ranking"; import { DEFAULT_SESSION_SOURCE_ID, isSessionSourceId, type FindResult, type SessionRecord, type SessionSourceId } from "../types"; import type { Db } from "../db"; +import { elideMessages } from "./message-elision"; import { searchMessageHits } from "./search"; export class SessionNotFoundError extends Error { @@ -22,7 +23,7 @@ export class SessionNotFoundError extends Error { export function getMessageRange( dbPath: string, sessionUuid: string, - options: { seq?: number; query?: string; before: number; after: number }, + options: { seq?: number; query?: string; before: number; after: number; maxMessageChars?: number }, ): { session: SessionRecord; anchorSeq: number; @@ -38,7 +39,11 @@ export function getMessageRange( const rangeStartSeq = Math.max(0, anchorSeq - options.before); const rangeEndSeq = anchorSeq + options.after; - const messages = getMessagesForRange(db, session.id, rangeStartSeq, rangeEndSeq); + const messages = elideMessages(getMessagesForRange(db, session.id, rangeStartSeq, rangeEndSeq), { + anchorSeq, + query: options.query, + maxMessageChars: options.maxMessageChars, + }); return { session, anchorSeq, @@ -55,6 +60,7 @@ export function getMessagePage( sessionUuid: string, offset: number, limit: number, + options: { maxMessageChars?: number } = {}, ): { session: SessionRecord; offset: number; @@ -67,7 +73,9 @@ export function getMessagePage( return withSourceAwareReadDb(dbPath, (db) => { const session = getSessionRecord(db, sessionUuid); if (!session) throw new SessionNotFoundError(sessionUuid); - const messages = getMessagesForPage(db, session.id, offset, limit); + const messages = elideMessages(getMessagesForPage(db, session.id, offset, limit), { + maxMessageChars: options.maxMessageChars, + }); const totalCount = session.messageCount; const hasMore = offset + messages.length < totalCount; return { diff --git a/src/types.ts b/src/types.ts index 6a6e9e5..cb16b62 100644 --- a/src/types.ts +++ b/src/types.ts @@ -190,6 +190,16 @@ export interface MessageRecord { contentText: string; timestamp: string; sourceKind: string; + elision?: MessageElision; +} + +export interface MessageElision { + originalCharCount: number; + displayedCharCount: number; + omittedCharCount: number; + strategy: "head_tail" | "around_query"; + query?: string; + hint: string; } export interface FindResult {