From 814b9336fc0ac0523f8e5b14f3db981981b8fe7f Mon Sep 17 00:00:00 2001 From: hanityx Date: Mon, 18 May 2026 02:23:42 +0900 Subject: [PATCH 1/4] refactor(api): establish provider extension groundwork --- .../src/domains/providers/adapters.test.ts | 60 ++++++ apps/api-ts/src/domains/providers/adapters.ts | 46 +++++ .../src/domains/providers/path-safety.test.ts | 27 +++ .../src/domains/providers/path-safety.ts | 51 ++--- apps/api-ts/src/lib/providers.test.ts | 22 +++ apps/api-ts/src/lib/providers.ts | 9 +- docs/ADDING_PROVIDER.md | 183 ++++++++++++++++++ docs/PROVIDER_SUPPORT.md | 4 +- package.json | 2 +- .../generate-provider-support.mjs | 8 +- 10 files changed, 380 insertions(+), 32 deletions(-) create mode 100644 apps/api-ts/src/domains/providers/adapters.test.ts create mode 100644 apps/api-ts/src/domains/providers/adapters.ts create mode 100644 docs/ADDING_PROVIDER.md rename {tools => packages/shared-contracts/provider-support}/generate-provider-support.mjs (92%) diff --git a/apps/api-ts/src/domains/providers/adapters.test.ts b/apps/api-ts/src/domains/providers/adapters.test.ts new file mode 100644 index 0000000..dec8e42 --- /dev/null +++ b/apps/api-ts/src/domains/providers/adapters.test.ts @@ -0,0 +1,60 @@ +import { + getProviderCapability, + PROVIDER_IDS, +} from "@threadlens/shared-contracts"; +import { describe, expect, it } from "vitest"; +import { + type ProviderAdapter, + type ProviderSessionLocator, + getProviderAdapter, + listProviderAdapters, + PROVIDER_ADAPTERS, +} from "./adapters.js"; +import { providerRootSpecs } from "./path-safety.js"; + +describe("provider adapters", () => { + it("registers one internal adapter for each provider contract id", () => { + expect(listProviderAdapters().map((adapter) => adapter.id)).toEqual(PROVIDER_IDS); + expect(Object.keys(PROVIDER_ADAPTERS)).toEqual([...PROVIDER_IDS]); + }); + + it("keeps adapter roots equivalent to current provider root specs", () => { + for (const provider of PROVIDER_IDS) { + const adapter = getProviderAdapter(provider); + expect(adapter).toBeDefined(); + expect(adapter?.roots()).toEqual(providerRootSpecs(provider)); + } + }); + + it("fails closed for unknown provider ids passed through internal casts", () => { + expect(getProviderAdapter("unknown" as ProviderAdapter["id"])).toBeUndefined(); + }); + + it("does not duplicate provider capabilities inside adapters", () => { + for (const provider of PROVIDER_IDS) { + const adapter = getProviderAdapter(provider); + expect(adapter).toBeDefined(); + if (!adapter) throw new Error(`missing adapter for ${provider}`); + expect("capabilities" in adapter).toBe(false); + expect(getProviderCapability(adapter.id).label).toBe(adapter.label); + } + }); + + it("defines session locators without forcing DB-backed providers into fake paths", () => { + const fileLocator: ProviderSessionLocator = { + kind: "file", + file_path: "/tmp/session.jsonl", + }; + const sqliteLocator: ProviderSessionLocator = { + kind: "sqlite", + db_path: "/tmp/provider.db", + session_id: "session-1", + }; + + expect(fileLocator.kind).toBe("file"); + expect(sqliteLocator).toMatchObject({ + kind: "sqlite", + session_id: "session-1", + }); + }); +}); diff --git a/apps/api-ts/src/domains/providers/adapters.ts b/apps/api-ts/src/domains/providers/adapters.ts new file mode 100644 index 0000000..6ac16b8 --- /dev/null +++ b/apps/api-ts/src/domains/providers/adapters.ts @@ -0,0 +1,46 @@ +import { + PROVIDER_IDS, + PROVIDER_LABELS, + type ProviderId, +} from "@threadlens/shared-contracts"; +import { providerRootSpecs, providerScanRootSpecs } from "./path-safety.js"; +import type { ProviderRootSpec } from "./types.js"; + +export type ProviderSessionLocator = + | { kind: "file"; file_path: string } + | { kind: "sqlite"; db_path: string; session_id: string }; + +export type ProviderAdapter = { + id: ProviderId; + label: string; + roots(): ProviderRootSpec[]; + scanRoots?(): Promise; +}; + +export const PROVIDER_ADAPTERS = Object.freeze( + Object.fromEntries( + PROVIDER_IDS.map((id) => [ + id, + Object.freeze({ + id, + label: PROVIDER_LABELS[id], + roots: () => providerRootSpecs(id), + scanRoots: () => providerScanRootSpecs(id), + } satisfies ProviderAdapter), + ]), + ), +) as Readonly>; + +const PROVIDER_ADAPTERS_BY_ID: Readonly< + Partial> +> = PROVIDER_ADAPTERS; + +export function getProviderAdapter( + provider: ProviderId, +): ProviderAdapter | undefined { + return PROVIDER_ADAPTERS_BY_ID[provider]; +} + +export function listProviderAdapters(): ProviderAdapter[] { + return PROVIDER_IDS.map((id) => PROVIDER_ADAPTERS[id]); +} diff --git a/apps/api-ts/src/domains/providers/path-safety.test.ts b/apps/api-ts/src/domains/providers/path-safety.test.ts index 73bb9ba..86e6901 100644 --- a/apps/api-ts/src/domains/providers/path-safety.test.ts +++ b/apps/api-ts/src/domains/providers/path-safety.test.ts @@ -2,6 +2,11 @@ import { mkdtemp, mkdir, realpath, rm, symlink, writeFile } from "node:fs/promis import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { + getProviderCapability, + PROVIDER_IDS, + type ProviderId, +} from "@threadlens/shared-contracts"; import { APP_DATA_DIR, CHAT_DIR, CODEX_HOME } from "../../lib/constants.js"; import { codexTranscriptSearchRoots, @@ -51,6 +56,28 @@ describe("provider path safety", () => { ).toBe(true); }); + it("fails closed for unknown provider ids passed across internal boundaries", () => { + const roots = providerRootSpecs("unknown-provider" as ProviderId); + expect(roots).toEqual([]); + }); + + it("keeps cleanup capability source of truth explicit for current providers", () => { + expect( + Object.fromEntries( + PROVIDER_IDS.map((provider) => [ + provider, + getProviderCapability(provider).safe_cleanup, + ]), + ), + ).toEqual({ + codex: true, + chatgpt: false, + claude: true, + gemini: true, + copilot: true, + }); + }); + it("keeps Codex and dot-home providers separate from desktop app-data cache roots", () => { const codexRoots = providerRootSpecs("codex"); const claudeRoots = providerRootSpecs("claude"); diff --git a/apps/api-ts/src/domains/providers/path-safety.ts b/apps/api-ts/src/domains/providers/path-safety.ts index 874d1bc..b28b27e 100644 --- a/apps/api-ts/src/domains/providers/path-safety.ts +++ b/apps/api-ts/src/domains/providers/path-safety.ts @@ -233,30 +233,33 @@ export function providerRootSpecs(provider: ProviderId): ProviderRootSpec[] { ...providerBackupRootSpecs(provider), ]; } - return [ - { - source: "vscode_global", - root: COPILOT_VSCODE_GLOBAL, - exts: [".jsonl", ".json"], - }, - { - source: "cursor_global", - root: COPILOT_CURSOR_GLOBAL, - exts: [".jsonl", ".json"], - }, - { - source: "vscode_workspace_chats", - root: COPILOT_VSCODE_WORKSPACE_STORAGE, - exts: [".json"], - }, - { - source: "cursor_workspace_chats", - root: COPILOT_CURSOR_WORKSPACE_STORAGE, - exts: [".json"], - }, - ...(archivedSpec ? [archivedSpec] : []), - ...providerBackupRootSpecs(provider), - ]; + if (provider === "copilot") { + return [ + { + source: "vscode_global", + root: COPILOT_VSCODE_GLOBAL, + exts: [".jsonl", ".json"], + }, + { + source: "cursor_global", + root: COPILOT_CURSOR_GLOBAL, + exts: [".jsonl", ".json"], + }, + { + source: "vscode_workspace_chats", + root: COPILOT_VSCODE_WORKSPACE_STORAGE, + exts: [".json"], + }, + { + source: "cursor_workspace_chats", + root: COPILOT_CURSOR_WORKSPACE_STORAGE, + exts: [".json"], + }, + ...(archivedSpec ? [archivedSpec] : []), + ...providerBackupRootSpecs(provider), + ]; + } + return []; } export function codexTranscriptSearchRoots(): ProviderRootSpec[] { diff --git a/apps/api-ts/src/lib/providers.test.ts b/apps/api-ts/src/lib/providers.test.ts index d9e6c60..8b17503 100644 --- a/apps/api-ts/src/lib/providers.test.ts +++ b/apps/api-ts/src/lib/providers.test.ts @@ -2,14 +2,17 @@ import { mkdtemp, mkdir, realpath, symlink, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import type { ProviderId } from "@threadlens/shared-contracts"; import { APP_DATA_DIR, CHAT_DIR } from "./constants.js"; import { buildProviderActionToken, codexTranscriptSearchRoots, + listProviderAdapters, listProviderIds, parseProviderId, providerRootSpecs, resolveSafePathWithinRoots, + runProviderSessionAction, } from "./providers"; describe("buildProviderActionToken", () => { @@ -56,6 +59,12 @@ describe("buildProviderActionToken", () => { }); describe("provider registry", () => { + it("exposes provider adapters through the compatibility facade", () => { + expect(listProviderAdapters().map((adapter) => adapter.id)).toEqual( + listProviderIds(), + ); + }); + it("includes chatgpt provider in dynamic list", () => { expect(listProviderIds()).toContain("chatgpt"); }); @@ -112,6 +121,19 @@ describe("provider registry", () => { expect(copilotRoots.some((spec) => spec.root.endsWith(path.join("Code", "User", "workspaceStorage")))).toBe(true); expect(copilotRoots.some((spec) => spec.root.endsWith(path.join("Cursor", "User", "workspaceStorage")))).toBe(true); }); + + it("fails closed for cleanup actions with unknown provider ids across internal boundaries", async () => { + const result = await runProviderSessionAction( + "unknown-provider" as ProviderId, + "delete_local", + ["/tmp/threadlens-unknown-provider.jsonl"], + true, + "", + ); + + expect(result.ok).toBe(false); + expect(result.error).toBe("cleanup-disabled-provider"); + }); }); describe("resolveSafePathWithinRoots", () => { diff --git a/apps/api-ts/src/lib/providers.ts b/apps/api-ts/src/lib/providers.ts index 50f08a7..a86189e 100644 --- a/apps/api-ts/src/lib/providers.ts +++ b/apps/api-ts/src/lib/providers.ts @@ -13,8 +13,15 @@ import { invalidateProviderSearchCaches } from "../domains/providers/search.js"; import { invalidateProviderMatrixCache } from "../domains/providers/matrix.js"; import { resolveAllowedProviderFilePath } from "../domains/providers/path-safety.js"; import type { ProviderId } from "../domains/providers/types.js"; +import { findProviderCapability } from "@threadlens/shared-contracts"; export type * from "../domains/providers/types.js"; +export type { ProviderAdapter } from "../domains/providers/adapters.js"; + +export { + getProviderAdapter, + listProviderAdapters, +} from "../domains/providers/adapters.js"; export { listProviderIds, @@ -60,7 +67,7 @@ export { export { buildProviderActionToken, buildSessionTranscript }; function supportsProviderCleanup(provider: ProviderId): boolean { - return provider !== "chatgpt"; + return findProviderCapability(provider)?.safe_cleanup === true; } export async function runProviderSessionAction( diff --git a/docs/ADDING_PROVIDER.md b/docs/ADDING_PROVIDER.md new file mode 100644 index 0000000..6b1cfbd --- /dev/null +++ b/docs/ADDING_PROVIDER.md @@ -0,0 +1,183 @@ +# Adding a Provider + +ThreadLens provider support is intentionally local-first. A provider can expose +session discovery, transcript parsing, search, diagnostics, archive, backup, and +delete behavior, so new providers need both capability metadata and path-safety +coverage. + +This guide describes the current internal extension path. It is not an external +plugin API, and new provider code should stay in reviewed source until a separate +security model exists. + +## 1. Add Provider Metadata + +Update `packages/shared-contracts/src/index.ts`. + +Add the provider to `PROVIDER_REGISTRY` with: + +- `id` +- `label` +- `docs_visibility` +- `search_scope_visibility` +- `provider_tab_group` +- `thread_review` +- `read_sessions` +- `read_transcript` +- `analyze_context` +- `safe_cleanup` +- `hard_delete` + +`docs/PROVIDER_SUPPORT.md` is generated. Do not edit it by hand. The capability +table comes from the shared contracts registry, while provider-specific workflow +notes are maintained by the generator. After changing provider metadata or +generator notes, run: + +```sh +pnpm docs:provider-support +``` + +## 2. Add Local Roots + +Provider roots define where ThreadLens is allowed to read provider files from. +Update the provider adapter/root definitions in +`apps/api-ts/src/domains/providers/`. + +Add roots for: + +- live session files +- archived session files, if supported +- provider action backups, if supported +- provider-specific cache locations, if needed + +Path safety must fail closed. Unknown provider ids must not fall through to +another provider's roots. + +The current first step toward cleaner extension is the internal adapter registry. +Adapters identify the provider and expose root discovery. Keep this boundary +small at first: + +```ts +export type ProviderAdapter = { + id: ProviderId; + label: string; + roots(): ProviderRootSpec[]; + scanRoots?(): Promise; +}; +``` + +Do not duplicate capabilities in adapters. Capabilities must remain sourced from +`getProviderCapability(adapter.id)`. + +Providers that do not store sessions as standalone files should use a session +locator model in later PRs instead of encoding provider-specific ids into fake +file paths: + +```ts +export type ProviderSessionLocator = + | { kind: "file"; file_path: string } + | { kind: "sqlite"; db_path: string; session_id: string }; +``` + +## 3. Add Search and Transcript Support + +Most providers can use the existing JSON/JSONL transcript flow. In the current +code, custom storage shapes still touch these areas: + +- `apps/api-ts/src/domains/providers/search.ts` +- `apps/api-ts/src/domains/providers/transcript.ts` +- `apps/api-ts/src/domains/providers/probe.ts` + +Add fixtures for the real local file shape. Do not infer a provider format from +docs alone; use sample session files. + +The intended direction is to move provider-specific session discovery and +transcript building behind adapters over follow-up PRs. For database-backed +providers, start read-only: + +- open the database in read-only mode +- use a busy timeout or retry path for live provider databases +- emit `ProviderSessionRow` values from provider session records +- build `TranscriptPayload` from provider message/part records +- keep destructive actions disabled until backup/delete semantics are designed + +## 4. Add Provider Diagnostics + +Update `apps/api-ts/src/domains/providers/matrix.ts` so the provider appears in +the provider health matrix with useful evidence. + +The matrix should show: + +- whether roots were detected +- whether session files were found +- whether safe cleanup and hard delete are actually available +- the roots or evidence used for that decision + +## 5. Add Archive, Backup, and Delete Behavior + +If the provider supports destructive file actions, update +`apps/api-ts/src/domains/providers/actions.ts`. + +Destructive actions must keep the existing safety rules: + +- dry-run before apply +- confirm token before apply +- path-safety resolution before file mutation +- backup behavior must not be weakened + +If the provider should be read-only, set `safe_cleanup: false` and +`hard_delete: false`. + +## 6. Add Tests + +At minimum, cover: + +- provider id parsing and listing +- provider root specs +- path-safety allow/reject behavior +- transcript parsing from fixture files +- search result extraction +- matrix capability output +- archive/delete behavior if supported + +Useful existing tests: + +- `apps/api-ts/src/domains/providers/path-safety.test.ts` +- `apps/api-ts/src/domains/providers/parser-fixtures.test.ts` +- `apps/api-ts/src/domains/providers/search.test.ts` +- `apps/api-ts/src/domains/providers/transcript.test.ts` +- `apps/api-ts/src/domains/providers/matrix.test.ts` +- `apps/api-ts/src/domains/providers/actions.test.ts` + +## Adapter Direction + +The current code still has provider behavior spread across registry, path +safety, search, transcript, matrix, and actions. The intended direction is an +internal provider adapter registry, not an external plugin system. + +The adapter boundary starts with provider identity and root discovery. Later PRs +can move provider-specific behavior behind that boundary, such as: + +- health evidence +- session discovery +- transcript parsing +- provider-specific archive or restore targets + +Capabilities should remain in `packages/shared-contracts/src/index.ts`; adapter +code should read them through `getProviderCapability(adapter.id)` instead of +storing a second copy. + +Do not add dynamic loading of third-party provider code. ThreadLens reads and can +mutate local AI session files, so provider support should stay in reviewed source +code unless a separate security model is designed. + +## Checklist + +Before opening a provider PR: + +```sh +pnpm docs:provider-support +pnpm --filter @threadlens/shared-contracts test +pnpm --filter @threadlens/api test +``` + +If provider UI copy changes are included, also run the relevant web or TUI tests. diff --git a/docs/PROVIDER_SUPPORT.md b/docs/PROVIDER_SUPPORT.md index 51c54f0..e216fdc 100644 --- a/docs/PROVIDER_SUPPORT.md +++ b/docs/PROVIDER_SUPPORT.md @@ -1,6 +1,6 @@ # Provider Support -_Generated from `packages/shared-contracts/src/index.ts`. Do not hand-edit this file; run `pnpm docs:provider-support`._ +_Generated from `packages/shared-contracts/src/index.ts` and provider notes in `packages/shared-contracts/provider-support/generate-provider-support.mjs`. Do not hand-edit this file; run `pnpm docs:provider-support`._ ThreadLens reads local conversation data from multiple provider-specific stores. This document distinguishes between: @@ -33,7 +33,7 @@ The primary search/session workflow currently covers `Codex`, `Claude`, `Gemini` ## Local Path Notes -- `Codex` reads session logs from `CODEX_HOME` plus detected Codex home mirrors such as `~/.codex` and `~/.codex-cli`; it is not tied to macOS app-data paths. +- `Codex` indexes session logs from `CODEX_HOME` plus Codex-managed archive, recovery, cwd-backup, and cleanup-backup roots; thread-id transcript lookup can also fall back to Codex home mirrors such as `~/.codex` and `~/.codex-cli`. - `ChatGPT` reads the local desktop cache for the installed app; this provider remains read-only and stays outside the default search and cleanup flow. - `Claude` reads local session stores from dot-home roots such as `~/.claude`. - `Gemini` reads local session stores from dot-home roots such as `~/.gemini`. diff --git a/package.json b/package.json index aa5be64..1a000ce 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "dev:web": "pnpm --filter @threadlens/web dev", "dev:desktop": "ELECTRON_RENDERER_URL=http://127.0.0.1:5174 pnpm --filter @threadlens/desktop-electron dev", "dev": "pnpm --filter @threadlens/shared-contracts build && pnpm -r --parallel --filter @threadlens/api --filter @threadlens/web dev", - "docs:provider-support": "pnpm --filter @threadlens/shared-contracts build && node tools/generate-provider-support.mjs", + "docs:provider-support": "pnpm --filter @threadlens/shared-contracts build && node packages/shared-contracts/provider-support/generate-provider-support.mjs", "build": "pnpm --filter @threadlens/shared-contracts build && pnpm --filter @threadlens/api build && pnpm --filter @threadlens/web build", "build:tui": "pnpm --filter @threadlens/tui build", "start:tui": "pnpm --filter @threadlens/shared-contracts build && pnpm --filter @threadlens/tui exec tsx src/cli.tsx", diff --git a/tools/generate-provider-support.mjs b/packages/shared-contracts/provider-support/generate-provider-support.mjs similarity index 92% rename from tools/generate-provider-support.mjs rename to packages/shared-contracts/provider-support/generate-provider-support.mjs index 91557a4..46c1324 100644 --- a/tools/generate-provider-support.mjs +++ b/packages/shared-contracts/provider-support/generate-provider-support.mjs @@ -6,15 +6,15 @@ import { INTERNAL_PROVIDER_IDS, PROVIDER_REGISTRY, SEARCHABLE_PROVIDER_IDS, -} from "../packages/shared-contracts/dist/index.js"; +} from "../dist/index.js"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const repoRoot = path.resolve(__dirname, ".."); +const repoRoot = path.resolve(__dirname, "../../.."); const outputPath = path.join(repoRoot, "docs", "PROVIDER_SUPPORT.md"); const PROVIDER_PATH_NOTES = { codex: - "reads session logs from `CODEX_HOME` plus detected Codex home mirrors such as `~/.codex` and `~/.codex-cli`; it is not tied to macOS app-data paths.", + "indexes session logs from `CODEX_HOME` plus Codex-managed archive, recovery, cwd-backup, and cleanup-backup roots; thread-id transcript lookup can also fall back to Codex home mirrors such as `~/.codex` and `~/.codex-cli`.", claude: "reads local session stores from dot-home roots such as `~/.claude`.", gemini: "reads local session stores from dot-home roots such as `~/.gemini`.", copilot: @@ -136,7 +136,7 @@ async function main() { const { publicScope, internalScope } = buildGuideList(); const content = `# Provider Support -_Generated from \`packages/shared-contracts/src/index.ts\`. Do not hand-edit this file; run \`pnpm docs:provider-support\`._ +_Generated from \`packages/shared-contracts/src/index.ts\` and provider notes in \`packages/shared-contracts/provider-support/generate-provider-support.mjs\`. Do not hand-edit this file; run \`pnpm docs:provider-support\`._ ThreadLens reads local conversation data from multiple provider-specific stores. This document distinguishes between: From 2884d107fcd7d45d215fa465dffe3fd78d915783 Mon Sep 17 00:00:00 2001 From: hanityx Date: Mon, 18 May 2026 02:23:42 +0900 Subject: [PATCH 2/4] refactor(api): split provider search engine --- .../src/domains/providers/search-helpers.ts | 8 + .../src/domains/providers/search.test.ts | 406 +++- apps/api-ts/src/domains/providers/search.ts | 2095 +---------------- .../src/domains/providers/search/abort.ts | 37 + .../src/domains/providers/search/cache.ts | 76 + .../providers/search/codex-session-path.ts | 30 + .../domains/providers/search/concurrency.ts | 23 + .../src/domains/providers/search/constants.ts | 36 + .../src/domains/providers/search/limits.ts | 84 + .../domains/providers/search/local-search.ts | 153 ++ .../providers/search/manifest-store.ts | 113 + .../providers/search/openable-threads.ts | 54 + .../search/provider-session-reports.ts | 110 + .../providers/search/provider-session-scan.ts | 171 ++ .../src/domains/providers/search/raw-files.ts | 191 ++ .../domains/providers/search/row-search.ts | 179 ++ .../providers/search/session-accumulator.ts | 86 + .../providers/search/session-dedupe.ts | 72 + .../providers/search/session-hit-resolver.ts | 84 + .../providers/search/session-hit-search.ts | 119 + .../search/session-manifest-files.ts | 146 ++ .../providers/search/session-manifest-rows.ts | 144 ++ .../providers/search/session-manifest.ts | 102 + .../search/session-metadata-results.ts | 111 + .../providers/search/session-raw-search.ts | 53 + .../providers/search/session-results.ts | 50 + .../providers/search/session-row-order.ts | 30 + .../providers/search/session-search.ts | 149 ++ .../search/transcript-loader-cache.ts | 33 + .../providers/search/transcript-matches.ts | 151 ++ .../search/transcript-message-matches.ts | 66 + .../providers/search/transcript-policy.ts | 10 + .../search/transcript-raw-matches.ts | 52 + .../src/domains/providers/search/types.ts | 57 + docs/ADDING_PROVIDER.md | 5 + 35 files changed, 3211 insertions(+), 2075 deletions(-) create mode 100644 apps/api-ts/src/domains/providers/search/abort.ts create mode 100644 apps/api-ts/src/domains/providers/search/cache.ts create mode 100644 apps/api-ts/src/domains/providers/search/codex-session-path.ts create mode 100644 apps/api-ts/src/domains/providers/search/concurrency.ts create mode 100644 apps/api-ts/src/domains/providers/search/constants.ts create mode 100644 apps/api-ts/src/domains/providers/search/limits.ts create mode 100644 apps/api-ts/src/domains/providers/search/local-search.ts create mode 100644 apps/api-ts/src/domains/providers/search/manifest-store.ts create mode 100644 apps/api-ts/src/domains/providers/search/openable-threads.ts create mode 100644 apps/api-ts/src/domains/providers/search/provider-session-reports.ts create mode 100644 apps/api-ts/src/domains/providers/search/provider-session-scan.ts create mode 100644 apps/api-ts/src/domains/providers/search/raw-files.ts create mode 100644 apps/api-ts/src/domains/providers/search/row-search.ts create mode 100644 apps/api-ts/src/domains/providers/search/session-accumulator.ts create mode 100644 apps/api-ts/src/domains/providers/search/session-dedupe.ts create mode 100644 apps/api-ts/src/domains/providers/search/session-hit-resolver.ts create mode 100644 apps/api-ts/src/domains/providers/search/session-hit-search.ts create mode 100644 apps/api-ts/src/domains/providers/search/session-manifest-files.ts create mode 100644 apps/api-ts/src/domains/providers/search/session-manifest-rows.ts create mode 100644 apps/api-ts/src/domains/providers/search/session-manifest.ts create mode 100644 apps/api-ts/src/domains/providers/search/session-metadata-results.ts create mode 100644 apps/api-ts/src/domains/providers/search/session-raw-search.ts create mode 100644 apps/api-ts/src/domains/providers/search/session-results.ts create mode 100644 apps/api-ts/src/domains/providers/search/session-row-order.ts create mode 100644 apps/api-ts/src/domains/providers/search/session-search.ts create mode 100644 apps/api-ts/src/domains/providers/search/transcript-loader-cache.ts create mode 100644 apps/api-ts/src/domains/providers/search/transcript-matches.ts create mode 100644 apps/api-ts/src/domains/providers/search/transcript-message-matches.ts create mode 100644 apps/api-ts/src/domains/providers/search/transcript-policy.ts create mode 100644 apps/api-ts/src/domains/providers/search/transcript-raw-matches.ts create mode 100644 apps/api-ts/src/domains/providers/search/types.ts diff --git a/apps/api-ts/src/domains/providers/search-helpers.ts b/apps/api-ts/src/domains/providers/search-helpers.ts index 0ee6499..02faa0e 100644 --- a/apps/api-ts/src/domains/providers/search-helpers.ts +++ b/apps/api-ts/src/domains/providers/search-helpers.ts @@ -49,6 +49,14 @@ export function matchesConversationSearch( return tokens.every((token) => normalizedText.includes(token)); } +export function isExactPhraseSearchMatch( + text: string, + normalizedQuery: string, +): boolean { + if (!normalizedQuery) return false; + return normalizeSearchQuery(text).includes(normalizedQuery); +} + export function buildSearchSnippet( text: string, normalizedQuery: string, diff --git a/apps/api-ts/src/domains/providers/search.test.ts b/apps/api-ts/src/domains/providers/search.test.ts index d9e7cb2..d02f696 100644 --- a/apps/api-ts/src/domains/providers/search.test.ts +++ b/apps/api-ts/src/domains/providers/search.test.ts @@ -1,5 +1,6 @@ +import { createHash } from "node:crypto"; import path from "node:path"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import { SEARCHABLE_PROVIDER_IDS } from "@threadlens/shared-contracts"; import { describe, expect, it, vi } from "vitest"; @@ -427,10 +428,22 @@ describe("searchConversationSessions", () => { ], ]), ); + const transcriptLoader = vi.fn(async () => + makeTranscript(row, [ + { + idx: 0, + role: "assistant", + text: "token from parsed transcript", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + ]), + ); const result = await searchConversationSessions([row], "token", { pageSize: 10, previewHitsPerSession: 3, + transcriptLoader, rawFileSearchLoader, }); @@ -524,6 +537,20 @@ describe("searchConversationSessions", () => { ], ]), ); + const transcriptLoader = vi.fn(async (_provider, filePath) => { + const row = + [newerWeakRow, middleWeakRow, olderStrongRow].find((item) => item.file_path === filePath) ?? + newerWeakRow; + return makeTranscript(row, [ + { + idx: 0, + role: "assistant", + text: `token transcript hit for ${row.session_id}`, + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + ]); + }); const page1 = await searchConversationSessions( [newerWeakRow, middleWeakRow, olderStrongRow], @@ -531,6 +558,7 @@ describe("searchConversationSessions", () => { { pageSize: 1, previewHitsPerSession: 1, + transcriptLoader, rawFileSearchLoader, }, ); @@ -541,6 +569,7 @@ describe("searchConversationSessions", () => { pageSize: 1, cursor: page1.next_cursor ?? undefined, previewHitsPerSession: 1, + transcriptLoader, rawFileSearchLoader, }, ); @@ -550,7 +579,7 @@ describe("searchConversationSessions", () => { expect(page2.sessions[0]?.session_id).not.toBe("search-older-strong"); }); - it("uses raw file previews for session pages without loading full transcripts", async () => { + it("uses raw file hits as a prefilter but builds previews from parsed transcripts", async () => { const row = makeRow({ session_id: "rollout-2026-03-25T10-00-00-019d-raw-preview", display_title: "Raw preview token session", @@ -562,7 +591,7 @@ describe("searchConversationSessions", () => { { idx: 0, role: "assistant", - text: "token hidden in transcript parser", + text: "token from parsed transcript", ts: "2026-03-25T10:00:00.000Z", source_type: "response_item.message", }, @@ -596,15 +625,65 @@ describe("searchConversationSessions", () => { expect(result.sessions).toHaveLength(1); expect(result.sessions[0]).toMatchObject({ session_id: "search-raw-preview", - match_count: 4, - has_more_hits: true, + match_count: 2, + has_more_hits: false, }); expect(result.sessions[0]?.preview_matches.map((match) => match.snippet)).toEqual([ "Raw preview token session", - "token preview one", - "token preview two", + "token from parsed transcript", ]); - expect(transcriptLoader).not.toHaveBeenCalled(); + expect(transcriptLoader).toHaveBeenCalledTimes(1); + expect(rawFileSearchLoader).toHaveBeenCalledTimes(1); + }); + + it("does not expose raw matches from system or policy transcript messages", async () => { + const row = makeRow({ + session_id: "rollout-2026-03-25T10-00-00-019d-policy-only", + display_title: "Policy only session", + file_path: "/tmp/search-policy-only.jsonl", + mtime: "2026-03-25T10:00:00.000Z", + }); + const transcriptLoader = vi.fn(async () => + makeTranscript(row, [ + { + idx: 0, + role: "system", + text: "token from system instructions", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + { + idx: 1, + role: "assistant", + text: "# AGENTS.md instructions for token handling", + ts: "2026-03-25T10:00:01.000Z", + source_type: "response_item.message", + }, + ]), + ); + const rawFileSearchLoader = vi.fn(async () => + new Map([ + [ + row.file_path, + { + snippets: ["token raw system preview"], + match_count: 1, + has_more_hits: false, + exact_phrase_count: 0, + }, + ], + ]), + ); + + const result = await searchConversationSessions([row], "token", { + pageSize: 10, + previewHitsPerSession: 3, + transcriptLoader, + rawFileSearchLoader, + }); + + expect(result.sessions).toHaveLength(0); + expect(transcriptLoader).toHaveBeenCalledTimes(1); expect(rawFileSearchLoader).toHaveBeenCalledTimes(1); }); @@ -749,7 +828,7 @@ describe("searchConversationSessionHits", () => { expect(page2.has_more).toBe(false); }); - it("uses the raw file search path for session detail expansion before transcript fallback", async () => { + it("uses raw file hits as a prefilter for session detail expansion", async () => { const row = makeRow({ session_id: "rollout-2026-03-25T10-00-00-019d-session-hits-raw", display_title: "Unrelated title", @@ -760,7 +839,28 @@ describe("searchConversationSessionHits", () => { { idx: 0, role: "assistant", - text: "token transcript fallback one", + text: "token transcript one", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + { + idx: 1, + role: "assistant", + text: "token transcript two", + ts: "2026-03-25T10:00:01.000Z", + source_type: "response_item.message", + }, + { + idx: 2, + role: "assistant", + text: "token transcript three", + ts: "2026-03-25T10:00:02.000Z", + source_type: "response_item.message", + }, + { + idx: 3, + role: "assistant", + text: "token transcript four", ts: "2026-03-25T10:00:00.000Z", source_type: "response_item.message", }, @@ -799,16 +899,16 @@ describe("searchConversationSessionHits", () => { expect(page1.total_hits).toBe(4); expect(page1.hits.map((hit) => hit.snippet)).toEqual([ - "token preview one", - "token preview two", + "token transcript one", + "token transcript two", ]); expect(page1.has_more).toBe(true); expect(page2.hits.map((hit) => hit.snippet)).toEqual([ - "token preview three", - "token preview four", + "token transcript three", + "token transcript four", ]); expect(page2.has_more).toBe(false); - expect(transcriptLoader).not.toHaveBeenCalled(); + expect(transcriptLoader).toHaveBeenCalledTimes(1); expect(rawFileSearchLoader).toHaveBeenCalledTimes(2); }); @@ -856,7 +956,7 @@ describe("searchConversationSessionHits", () => { rawFileSearchLoader, }); - expect(page1.hits).toHaveLength(21); + expect(page1.hits).toHaveLength(40); expect(page1.has_more).toBe(true); expect(page2.hits.length).toBeGreaterThan(0); expect(page2.hits[0]?.snippet).toContain("token transcript hit"); @@ -1027,6 +1127,7 @@ describe("provider manifest cache behavior", () => { size: 128, mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), })); + const realpathMock = vi.fn(async (target: string) => path.resolve(target)); const probeSessionFile = vi.fn(async () => ({ ok: true, format: "jsonl", @@ -1046,6 +1147,7 @@ describe("provider manifest cache behavior", () => { const actual = await importOriginal(); return { ...actual, + realpath: realpathMock, stat: statMock, }; }); @@ -1118,6 +1220,7 @@ describe("provider manifest cache behavior", () => { size: 128, mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), })); + const realpathMock = vi.fn(async (target: string) => path.resolve(target)); const probeSessionFile = vi.fn(async () => ({ ok: true, format: "jsonl", @@ -1136,6 +1239,7 @@ describe("provider manifest cache behavior", () => { const actual = await importOriginal(); return { ...actual, + realpath: realpathMock, stat: statMock, }; }); @@ -1176,8 +1280,182 @@ describe("provider manifest cache behavior", () => { const secondWalk = vi.fn(async () => ["/virtual/claude/session-b.jsonl"]); installMocks(secondWalk); const secondMod = await import("./search.js"); - await secondMod.getProviderSessionScan("claude", 1); + const secondScan = await secondMod.getProviderSessionScan("claude", 1); expect(secondWalk).not.toHaveBeenCalled(); + expect(secondScan.rows[0]?.source).toBe("projects"); + } finally { + vi.useRealTimers(); + vi.unstubAllEnvs(); + vi.resetModules(); + vi.doUnmock("../../lib/utils.js"); + vi.doUnmock("node:fs/promises"); + vi.doUnmock("./path-safety.js"); + vi.doUnmock("./title-detection.js"); + vi.doUnmock("./probe.js"); + vi.doUnmock("./matrix.js"); + await rm(cacheDir, { recursive: true, force: true }); + } + }); + + it("reclassifies persisted manifest candidate source from the current matching root", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-03-25T10:00:00.000Z")); + vi.resetModules(); + const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); + vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); + + const version = createHash("sha1").update("claude").digest("hex").slice(0, 8); + const cacheFile = path.join(cacheDir, `manifest-claude-${version}.json`); + const sessionPath = "/virtual/claude/session-a.jsonl"; + await mkdir(path.dirname(cacheFile), { recursive: true }); + await writeFile( + cacheFile, + JSON.stringify({ + expires_at: Date.now() + 60_000, + manifest: { + provider: "claude", + name: "Claude", + root_exists: true, + candidates: [ + { + source: "stale-cache-source", + file_path: sessionPath, + size_bytes: 128, + mtime: "2026-03-25T10:00:00.000Z", + mtime_ms: Date.parse("2026-03-25T10:00:00.000Z"), + }, + ], + total_bytes: 128, + }, + }), + "utf8", + ); + + vi.doMock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + realpath: vi.fn(async (target: string) => path.resolve(target)), + }; + }); + vi.doMock("./path-safety.js", () => ({ + providerScanRootSpecs: async () => [ + { root: "/virtual/claude", source: "projects", exts: [".jsonl"] }, + ], + })); + + try { + const { readPersistedProviderManifest } = await import("./search/manifest-store.js"); + const persisted = await readPersistedProviderManifest("claude"); + + expect(persisted?.manifest.candidates).toHaveLength(1); + expect(persisted?.manifest.candidates[0]).toMatchObject({ + file_path: path.resolve(sessionPath), + source: "projects", + }); + } finally { + vi.useRealTimers(); + vi.unstubAllEnvs(); + vi.resetModules(); + vi.doUnmock("node:fs/promises"); + vi.doUnmock("./path-safety.js"); + await rm(cacheDir, { recursive: true, force: true }); + } + }); + + it("discards persisted manifest candidates outside current provider roots", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-03-25T10:00:00.000Z")); + vi.resetModules(); + const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); + vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); + + const version = createHash("sha1").update("claude").digest("hex").slice(0, 8); + const cacheFile = path.join(cacheDir, `manifest-claude-${version}.json`); + await mkdir(path.dirname(cacheFile), { recursive: true }); + await writeFile( + cacheFile, + JSON.stringify({ + expires_at: Date.now() + 60_000, + manifest: { + provider: "claude", + name: "Claude", + root_exists: true, + candidates: [ + { + source: "projects", + file_path: "/outside/session-a.jsonl", + size_bytes: 128, + mtime: "2026-03-25T10:00:00.000Z", + mtime_ms: Date.parse("2026-03-25T10:00:00.000Z"), + }, + ], + total_bytes: 128, + }, + }), + "utf8", + ); + + const walkFilesByExt = vi.fn(async () => ["/virtual/claude/session-b.jsonl"]); + const statMock = vi.fn(async () => ({ + size: 256, + mtimeMs: Date.parse("2026-03-25T10:05:00.000Z"), + })); + const realpathMock = vi.fn(async (target: string) => path.resolve(target)); + const probeSessionFile = vi.fn(async (filePath: string) => ({ + ok: true, + format: "jsonl", + error: null, + detected_title: path.basename(filePath, ".jsonl"), + title_source: "fixture", + })); + + vi.doMock("../../lib/utils.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + walkFilesByExt, + }; + }); + vi.doMock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + realpath: realpathMock, + stat: statMock, + }; + }); + vi.doMock("./path-safety.js", () => ({ + providerScanRootSpecs: async () => [ + { root: "/virtual/claude", source: "projects", exts: [".jsonl"] }, + ], + providerName: () => "Claude", + codexTranscriptSearchRoots: async () => [], + })); + vi.doMock("./title-detection.js", () => ({ + getCodexThreadTitleMap: vi.fn(async () => new Map()), + invalidateCodexThreadTitleMapCache: vi.fn(), + extractCodexThreadIdFromSessionName: vi.fn(() => ""), + })); + vi.doMock("./probe.js", () => ({ + inferSessionId: vi.fn((filePath: string) => + filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, + ), + isCopilotGlobalSessionLikeFile: vi.fn(() => false), + isWorkspaceChatSessionPath: vi.fn(() => false), + probeSessionFile, + })); + vi.doMock("./matrix.js", () => ({ + providerStatus: vi.fn(() => "ready"), + })); + + try { + const mod = await import("./search.js"); + const scan = await mod.getProviderSessionScan("claude", 1); + + expect(walkFilesByExt).toHaveBeenCalledTimes(1); + expect(scan.rows[0]?.file_path).toBe("/virtual/claude/session-b.jsonl"); + expect(probeSessionFile).toHaveBeenCalledWith("/virtual/claude/session-b.jsonl"); } finally { vi.useRealTimers(); vi.unstubAllEnvs(); @@ -1588,6 +1866,100 @@ describe("abort propagation", () => { await rm(cacheDir, { recursive: true, force: true }); } }); + + it("keeps shared manifest work alive when the first caller aborts", async () => { + vi.resetModules(); + const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); + vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); + const firstWalk = deferred(); + const walkFilesByExt = vi + .fn() + .mockImplementationOnce(async () => firstWalk.promise); + const statMock = vi.fn(async () => ({ + size: 128, + mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), + })); + + vi.doMock("../../lib/utils.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + walkFilesByExt, + }; + }); + vi.doMock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + realpath: vi.fn(async (target: string) => path.resolve(target)), + stat: statMock, + }; + }); + vi.doMock("./path-safety.js", () => ({ + providerScanRootSpecs: async () => [ + { + root: "/virtual/claude", + source: "projects", + exts: [".jsonl"], + }, + ], + providerName: () => "Claude", + codexTranscriptSearchRoots: async () => [], + })); + vi.doMock("./title-detection.js", () => ({ + getCodexThreadTitleMap: vi.fn(async () => new Map()), + invalidateCodexThreadTitleMapCache: vi.fn(), + extractCodexThreadIdFromSessionName: vi.fn(() => ""), + })); + vi.doMock("./probe.js", () => ({ + inferSessionId: vi.fn((filePath: string) => + filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, + ), + isCopilotGlobalSessionLikeFile: vi.fn(() => false), + isWorkspaceChatSessionPath: vi.fn(() => false), + probeSessionFile: vi.fn(async () => ({ + ok: true, + format: "jsonl", + error: null, + detected_title: "", + title_source: null, + })), + })); + vi.doMock("./matrix.js", () => ({ + providerStatus: vi.fn(() => "ready"), + })); + + try { + const mod = await import("./search.js"); + const controller = new AbortController(); + const firstRequest = mod.getProviderSessionScan("claude", 1, { + signal: controller.signal, + }); + await waitForCondition( + () => walkFilesByExt.mock.calls.length === 1, + "the shared manifest rebuild to start", + ); + const secondRequest = mod.getProviderSessionScan("claude", 1); + + controller.abort(); + await expect(firstRequest).rejects.toMatchObject({ name: "AbortError" }); + + firstWalk.resolve(["/virtual/claude/session-a.jsonl"]); + const secondScan = await secondRequest; + expect(secondScan.rows[0]?.file_path).toContain("session-a.jsonl"); + expect(walkFilesByExt).toHaveBeenCalledTimes(1); + } finally { + vi.unstubAllEnvs(); + vi.resetModules(); + vi.doUnmock("../../lib/utils.js"); + vi.doUnmock("node:fs/promises"); + vi.doUnmock("./path-safety.js"); + vi.doUnmock("./title-detection.js"); + vi.doUnmock("./probe.js"); + vi.doUnmock("./matrix.js"); + await rm(cacheDir, { recursive: true, force: true }); + } + }); }); describe("searchConversationSessionHitsTs", () => { diff --git a/apps/api-ts/src/domains/providers/search.ts b/apps/api-ts/src/domains/providers/search.ts index 78a0e52..75a4358 100644 --- a/apps/api-ts/src/domains/providers/search.ts +++ b/apps/api-ts/src/domains/providers/search.ts @@ -1,2058 +1,37 @@ -import { spawn } from "node:child_process"; -import { createHash } from "node:crypto"; -import { mkdir, readFile, rm, stat, writeFile } from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { SEARCHABLE_PROVIDER_IDS } from "@threadlens/shared-contracts"; - -import { buildSessionTranscript } from "./transcript.js"; -import type { - ConversationSearchPayload, - ConversationSearchSessionHitsPayload, - ConversationSearchSessionResult, - ConversationSearchResult, - ProviderId, - ProviderSessionRow, - ProviderSessionScan, - TranscriptPayload, -} from "./types.js"; -import { - buildSearchSnippet, - buildSearchTokens, - fallbackDisplayTitle, - matchesConversationSearch, - normalizeSearchQuery, - normalizeSearchText, -} from "./search-helpers.js"; -import { - codexTranscriptSearchRoots, - providerName, - providerScanRootSpecs, -} from "./path-safety.js"; -import { - extractCodexThreadIdFromSessionName, - getCodexThreadTitleMap, - invalidateCodexThreadTitleMapCache, -} from "./title-detection.js"; -import { - inferSessionId, - isCopilotGlobalSessionLikeFile, - isWorkspaceChatSessionPath, - probeSessionFile, -} from "./probe.js"; -import { - providerStatus, -} from "./matrix.js"; -import { - nowIsoUtc, - walkFilesByExt, -} from "../../lib/utils.js"; - -const PROVIDER_SCAN_CACHE_TTL_MS = 60_000; -const PROVIDER_MANIFEST_CACHE_TTL_MS = 5 * 60_000; -const OPENABLE_THREAD_IDS_CACHE_TTL_MS = 15_000; -const CONVERSATION_SEARCH_RESPONSE_CACHE_TTL_MS = 15_000; -const DEFAULT_CONVERSATION_SEARCH_PROVIDERS: ProviderId[] = [...SEARCHABLE_PROVIDER_IDS]; -const DEFAULT_CONVERSATION_SEARCH_LIMIT = 40; -const MAX_CONVERSATION_SEARCH_LIMIT = 200; -const DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT = 10_000; -const MAX_CONVERSATION_SEARCH_SCAN_LIMIT = 1_200; -const SEARCH_TRANSCRIPT_CONCURRENCY = 4; -const CONVERSATION_SEARCH_SCAN_CONCURRENCY = 8; -const SEARCH_TRANSCRIPT_CACHE_MAX_ENTRIES = 2_000; -const DEFAULT_CONVERSATION_SEARCH_SCAN_MULTIPLIER = 4; -const DEFAULT_CONVERSATION_SEARCH_SCAN_FLOOR = 160; -const PROVIDER_SCAN_FILE_STAT_CONCURRENCY = 32; -const PROVIDER_SESSION_MANIFEST_FILE_LIMIT = 8_000; -const RAW_CONVERSATION_FILE_MAX_MATCHES = 10_000; -const PROVIDER_SCAN_BUDGET_WEIGHTS: Record = { - codex: 1.35, - chatgpt: 0.6, - claude: 1.35, - gemini: 1, - copilot: 0.85, -}; -type ProviderScanCacheEntry = { - expires_at: number; - scan: ProviderSessionScan; -}; - -type ProviderSessionCandidate = { - source: string; - file_path: string; - size_bytes: number; - mtime: string; - mtime_ms: number; -}; - -type ProviderSessionManifest = { - provider: ProviderId; - name: string; - root_exists: boolean; - candidates: ProviderSessionCandidate[]; - total_bytes: number; -}; - -type ProviderManifestCacheEntry = { - expires_at: number; - manifest: ProviderSessionManifest; -}; - -type PersistedProviderManifestCacheEntry = { - expires_at: number; - manifest: ProviderSessionManifest; -}; - -type ConversationSearchResponseCacheEntry = { - expires_at: number; - payload: ConversationSearchPayload; -}; - -type ConversationTranscriptLoader = ( - provider: ProviderId, - filePath: string, -) => Promise; - -type CachedConversationTranscriptLoader = ( - row: ProviderSessionRow, -) => Promise; - -type TranscriptSearchCacheEntry = { - mtime: string; - transcript: TranscriptPayload | null; -}; - -type SearchMatchMeta = { - exactPhrase: boolean; -}; - -type SearchSessionAccumulator = { - session: ConversationSearchSessionResult; - exact_phrase_count: number; - seen_hits: Set; -}; - -type RawConversationFileMatch = { - snippets: string[]; - match_count: number; - has_more_hits: boolean; - exact_phrase_count: number; -}; - -type RawConversationFileSearchLoader = ( - rows: ProviderSessionRow[], - q: string, - options?: { - previewHitsPerSession?: number; - maxHitsPerSession?: number; - signal?: AbortSignal; - }, -) => Promise>; - -const RAW_CONVERSATION_FILE_EXT_PATTERN = /\.(jsonl|json|md|txt|data)$/i; -const RAW_FILE_SEARCH_CHUNK_SIZE = 160; -const EMPTY_PROVIDER_SESSION_PROBE = { - ok: true, - format: "unknown" as const, - error: null, - detected_title: "", - title_source: null, -}; - -function providerSessionSourcePriority(source: string): number { - const normalized = normalizeSearchText(source).toLowerCase(); - if (normalized === "sessions") return 500; - if (normalized === "projects") return 480; - if (normalized === "transcripts") return 470; - if (normalized === "history") return 460; - if (normalized === "tmp") return 450; - if (normalized.includes("workspace")) return 440; - if (normalized.includes("global")) return 430; - if (normalized.includes("cleanup_backups")) return 100; - return 300; -} - -function searchLogicalSessionKey(row: ProviderSessionRow): string { - const inferred = row.session_id || inferSessionId(row.file_path) || path.resolve(row.file_path); - return `${row.provider}:${inferred}`; -} - -function dedupeConversationSearchRows(rows: ProviderSessionRow[]): ProviderSessionRow[] { - const uniqueRows = new Map(); - for (const row of rows) { - const key = searchLogicalSessionKey(row); - const existing = uniqueRows.get(key); - if (!existing) { - uniqueRows.set(key, row); - continue; - } - const existingPriority = providerSessionSourcePriority(existing.source); - const nextPriority = providerSessionSourcePriority(row.source); - if (nextPriority > existingPriority) { - uniqueRows.set(key, row); - continue; - } - if ( - nextPriority === existingPriority && - Date.parse(String(row.mtime || "")) > Date.parse(String(existing.mtime || "")) - ) { - uniqueRows.set(key, row); - } - } - return [...uniqueRows.values()]; -} - -// Detects injected policy/system-prompt messages that appear as user role in Codex sessions. -// These are AGENTS.md instructions, permission blocks, or assistant persona declarations -// injected at conversation start — not actual user input. -function isPolicyInjectionMessage(text: string): boolean { - const t = text.trimStart(); - return ( - t.startsWith("# AGENTS.md instructions for") || - t.startsWith("") || - t.startsWith("") || - /^You are (Codex|Assist|Claude|Gemini|GPT|ChatGPT|Copilot)\b/i.test(t) || - (t.startsWith("# ") && t.includes("")) - ); -} - -const providerScanCache = new Map(); -const providerScanInflight = new Map>(); -const providerManifestCache = new Map(); -const providerManifestInflight = new Map>(); -const invalidatedProviderManifests = new Set(); -const providerSearchGeneration = new Map(); -const conversationSearchResponseCache = new Map(); -const transcriptSearchCache = new Map(); -let openableThreadIdsCache: - | { - expires_at: number; - ids: Set; - } - | null = null; - -function createAbortError(): Error { - const error = new Error("conversation-search-aborted"); - error.name = "AbortError"; - return error; -} - -function throwIfAborted(signal?: AbortSignal): void { - if (signal?.aborted) throw createAbortError(); -} - -function awaitAbortable(promise: Promise, signal?: AbortSignal): Promise { - if (!signal) return promise; - return new Promise((resolve, reject) => { - const onAbort = () => { - signal.removeEventListener("abort", onAbort); - reject(createAbortError()); - }; - if (signal.aborted) { - onAbort(); - } else { - signal.addEventListener("abort", onAbort, { once: true }); - } - promise.then( - (value) => { - signal.removeEventListener("abort", onAbort); - resolve(value); - }, - (error) => { - signal.removeEventListener("abort", onAbort); - reject(error); - }, - ); - }); -} - -function getProviderSearchGeneration(provider: ProviderId): number { - return providerSearchGeneration.get(provider) ?? 0; -} - -function bumpProviderSearchGeneration(provider: ProviderId): number { - const next = getProviderSearchGeneration(provider) + 1; - providerSearchGeneration.set(provider, next); - return next; -} - -function resolveSearchCacheDirectory(): string { - const override = String(process.env.THREADLENS_SEARCH_CACHE_DIR || "").trim(); - return override || path.join(os.tmpdir(), "threadlens-search-cache"); -} - -function providerManifestCacheFilePath(provider: ProviderId): string { - const version = createHash("sha1").update(provider).digest("hex").slice(0, 8); - return path.join(resolveSearchCacheDirectory(), `manifest-${provider}-${version}.json`); -} - -async function readPersistedProviderManifest( - provider: ProviderId, -): Promise { - try { - const raw = await readFile(providerManifestCacheFilePath(provider), "utf8"); - const parsed = JSON.parse(raw) as PersistedProviderManifestCacheEntry; - if (!parsed || typeof parsed !== "object") return null; - if (!parsed.manifest || !Array.isArray(parsed.manifest.candidates)) return null; - return parsed; - } catch { - return null; - } -} - -async function writePersistedProviderManifest( - provider: ProviderId, - entry: PersistedProviderManifestCacheEntry, -): Promise { - const filePath = providerManifestCacheFilePath(provider); - await mkdir(path.dirname(filePath), { recursive: true }); - await writeFile(filePath, JSON.stringify(entry), "utf8"); -} - -async function deletePersistedProviderManifest(provider: ProviderId): Promise { - await rm(providerManifestCacheFilePath(provider), { force: true }); -} -let openableThreadIdsInflight: Promise> | null = null; - -export function invalidateProviderSearchCaches(provider: ProviderId) { - bumpProviderSearchGeneration(provider); - for (const key of providerScanCache.keys()) { - if (key.startsWith(`${provider}:`)) { - providerScanCache.delete(key); - } - } - for (const key of providerScanInflight.keys()) { - if (key.startsWith(`${provider}:`)) { - providerScanInflight.delete(key); - } - } - providerManifestCache.delete(provider); - providerManifestInflight.delete(provider); - invalidatedProviderManifests.add(provider); - conversationSearchResponseCache.clear(); - void deletePersistedProviderManifest(provider); -} - -function providerScanCacheKey(provider: ProviderId, limit: number): string { - return `${provider}:${limit}`; -} - -function transcriptSearchCacheKey(row: ProviderSessionRow): string { - return `${row.provider}:${path.resolve(row.file_path)}`; -} - -function conversationSearchResponseCacheKey(options: { - q: string; - providers: ProviderId[]; - pageSize: number; - cursor?: string; - transcriptLimit?: number; - previewHitsPerSession?: number; - sessionLimitPerProvider?: number; -}): string { - return [ - options.q, - [...options.providers].sort().join(","), - options.pageSize, - options.cursor || "", - options.transcriptLimit || "", - options.previewHitsPerSession || "", - options.sessionLimitPerProvider || "", - ].join("::"); -} - -function shouldIncludeProviderSessionFile( - provider: ProviderId, - source: string, - filePath: string, -): boolean { - const baseName = path.basename(filePath); - if (source === "cleanup_backups" && baseName === "_manifest.json") { - return false; - } - if ( - provider === "copilot" && - (source === "vscode_workspace_chats" || source === "cursor_workspace_chats") - ) { - return isWorkspaceChatSessionPath(filePath); - } - if ( - provider === "copilot" && - (source === "vscode_global" || source === "cursor_global") - ) { - return isCopilotGlobalSessionLikeFile(filePath); - } - return true; -} - -function trimTranscriptSearchCache() { - while (transcriptSearchCache.size > SEARCH_TRANSCRIPT_CACHE_MAX_ENTRIES) { - const oldestKey = transcriptSearchCache.keys().next().value; - if (!oldestKey) return; - transcriptSearchCache.delete(oldestKey); - } -} - -export function resolveConversationSearchLimits(options?: { - limit?: number; - sessionLimitPerProvider?: number; -}) { - const resultLimit = Math.max( - 1, - Math.min( - MAX_CONVERSATION_SEARCH_LIMIT, - Number(options?.limit) || DEFAULT_CONVERSATION_SEARCH_LIMIT, - ), - ); - const scanLimit = Math.max( - DEFAULT_CONVERSATION_SEARCH_SCAN_FLOOR, - Math.min( - MAX_CONVERSATION_SEARCH_SCAN_LIMIT, - Number(options?.sessionLimitPerProvider) || - Math.max( - resultLimit * DEFAULT_CONVERSATION_SEARCH_SCAN_MULTIPLIER, - DEFAULT_CONVERSATION_SEARCH_SCAN_FLOOR, - ), - ), - ); - return { - resultLimit, - scanLimit, - }; -} - -export function buildConversationSearchProviderBudgets( - providers: ProviderId[], - totalBudget: number, -): Array<{ provider: ProviderId; limit: number }> { - const uniqueProviders = Array.from(new Set(providers)); - const safeBudget = Math.max(1, Math.floor(totalBudget)); - if (!uniqueProviders.length) return []; - if (uniqueProviders.length === 1) { - return [{ provider: uniqueProviders[0], limit: safeBudget }]; - } - - const weightedProviders = uniqueProviders.map((provider) => ({ - provider, - weight: PROVIDER_SCAN_BUDGET_WEIGHTS[provider] ?? 1, - })); - const totalWeight = weightedProviders.reduce( - (sum, entry) => sum + entry.weight, - 0, - ); - - const provisional = weightedProviders.map((entry) => { - const exact = (safeBudget * entry.weight) / totalWeight; - return { - provider: entry.provider, - exact, - limit: Math.max(1, Math.floor(exact)), - }; - }); - - let remaining = safeBudget - provisional.reduce((sum, entry) => sum + entry.limit, 0); - provisional - .sort((a, b) => (b.exact - b.limit) - (a.exact - a.limit)) - .forEach((entry) => { - if (remaining <= 0) return; - entry.limit += 1; - remaining -= 1; - }); - - const limitByProvider = new Map( - provisional.map((entry) => [entry.provider, entry.limit] as const), - ); - return uniqueProviders.map((provider) => ({ - provider, - limit: limitByProvider.get(provider) ?? 1, - })); -} - -export function createCachedConversationTranscriptLoader( - baseLoader: ConversationTranscriptLoader, -): CachedConversationTranscriptLoader { - return async (row) => { - const key = transcriptSearchCacheKey(row); - const cached = transcriptSearchCache.get(key); - if (cached && cached.mtime === row.mtime) { - transcriptSearchCache.delete(key); - transcriptSearchCache.set(key, cached); - return cached.transcript; - } - - const transcript = await baseLoader(row.provider, row.file_path).catch(() => null); - if (transcript) { - transcriptSearchCache.set(key, { - mtime: row.mtime, - transcript, - }); - trimTranscriptSearchCache(); - } - return transcript; - }; -} - -function conversationSearchResultDedupKey(result: ConversationSearchResult): string { - return [ - result.provider, - result.session_id, - result.file_path, - result.match_kind, - result.role || "", - normalizeSearchText(result.snippet).toLowerCase(), - ].join("::"); -} - -function dedupeConversationSearchResults( - results: ConversationSearchResult[], -): ConversationSearchResult[] { - const deduped: ConversationSearchResult[] = []; - const seen = new Set(); - for (const result of results) { - const key = conversationSearchResultDedupKey(result); - if (seen.has(key)) continue; - seen.add(key); - deduped.push(result); - } - return deduped; -} - -function decodeConversationSearchCursor(cursor?: string): number { - const offset = Math.max(0, Math.floor(Number(cursor) || 0)); - return Number.isFinite(offset) ? offset : 0; -} - -function encodeConversationSearchCursor(offset: number): string { - return String(Math.max(0, Math.floor(offset))); -} - -function isExactPhraseSearchMatch( - text: string, - normalizedQuery: string, -): boolean { - if (!normalizedQuery) return false; - return normalizeSearchQuery(text).includes(normalizedQuery); -} - -function createSessionAccumulator( - row: ProviderSessionRow, - identity: ReturnType, -): SearchSessionAccumulator { - return { - session: { - provider: row.provider, - session_id: identity.sessionId, - ...(identity.threadId ? { thread_id: identity.threadId } : {}), - title: identity.title, - ...(row.display_title ? { display_title: row.display_title } : {}), - file_path: row.file_path, - source: row.source, - mtime: row.mtime, - match_count: 0, - title_match_count: 0, - best_match_kind: "message", - preview_matches: [], - has_more_hits: false, - }, - exact_phrase_count: 0, - seen_hits: new Set(), - }; -} - -function addSessionSearchHit( - session: SearchSessionAccumulator, - result: ConversationSearchResult, - meta: SearchMatchMeta, - previewLimit: number, -): void { - const dedupeKey = conversationSearchResultDedupKey(result); - if (session.seen_hits.has(dedupeKey)) return; - session.seen_hits.add(dedupeKey); - session.session.match_count += 1; - if (result.match_kind === "title") { - session.session.title_match_count += 1; - session.session.best_match_kind = "title"; - } - if (meta.exactPhrase) { - session.exact_phrase_count += 1; - } - if (session.session.preview_matches.length < previewLimit) { - session.session.preview_matches.push(result); - } - session.session.has_more_hits = - session.session.match_count > session.session.preview_matches.length; -} - -function compareConversationSearchSessions( - left: SearchSessionAccumulator, - right: SearchSessionAccumulator, -): number { - const leftTitle = left.session.title_match_count > 0 ? 1 : 0; - const rightTitle = right.session.title_match_count > 0 ? 1 : 0; - if (leftTitle !== rightTitle) return rightTitle - leftTitle; - if (left.exact_phrase_count !== right.exact_phrase_count) { - return right.exact_phrase_count - left.exact_phrase_count; - } - if (left.session.match_count !== right.session.match_count) { - return right.session.match_count - left.session.match_count; - } - return ( - Date.parse(String(right.session.mtime || "")) - - Date.parse(String(left.session.mtime || "")) - ); -} - -async function mapWithConcurrency( - items: T[], - concurrency: number, - worker: (item: T, index: number) => Promise, -): Promise { - if (items.length === 0) return []; - const results = new Array(items.length); - let nextIndex = 0; - const workerCount = Math.max(1, Math.min(concurrency, items.length)); - - await Promise.all( - Array.from({ length: workerCount }, async () => { - while (true) { - const currentIndex = nextIndex; - nextIndex += 1; - if (currentIndex >= items.length) return; - results[currentIndex] = await worker(items[currentIndex], currentIndex); - } - }), - ); - - return results; -} - -async function buildProviderSessionManifest( - provider: ProviderId, - options?: { signal?: AbortSignal }, -): Promise { - throwIfAborted(options?.signal); - const roots = await providerScanRootSpecs(provider); - throwIfAborted(options?.signal); - const rootExists = - provider === "chatgpt" - ? roots.length > 0 - : (await Promise.all(roots.map((r) => walkRootExists(r.root)))).some(Boolean); - - const candidates: ProviderSessionCandidate[] = []; - - const rootFiles = await Promise.all( - roots.map((spec) => { - throwIfAborted(options?.signal); - return walkFilesByExt(spec.root, spec.exts, PROVIDER_SESSION_MANIFEST_FILE_LIMIT); - }), - ); - - for (let i = 0; i < roots.length; i += 1) { - throwIfAborted(options?.signal); - const spec = roots[i]; - const files = rootFiles[i] ?? []; - const filteredFiles = files.filter((filePath) => - shouldIncludeProviderSessionFile(provider, spec.source, filePath), - ); - - const scannedCandidates = await mapWithConcurrency( - filteredFiles, - PROVIDER_SCAN_FILE_STAT_CONCURRENCY, - async (file) => { - try { - throwIfAborted(options?.signal); - const st = await stat(file); - throwIfAborted(options?.signal); - return { - source: spec.source, - file_path: file, - size_bytes: Number(st.size), - mtime: new Date(Number(st.mtimeMs)).toISOString(), - mtime_ms: Number(st.mtimeMs), - }; - } catch { - return null; - } - }, - ); - candidates.push( - ...scannedCandidates.filter( - ( - candidate, - ): candidate is { - source: string; - file_path: string; - size_bytes: number; - mtime: string; - mtime_ms: number; - } => Boolean(candidate), - ), - ); - } - - candidates.sort((a, b) => b.mtime_ms - a.mtime_ms); - return { - provider, - name: providerName(provider), - root_exists: rootExists, - candidates, - total_bytes: candidates.reduce( - (sum, candidate) => sum + Number(candidate.size_bytes || 0), - 0, - ), - }; -} - -async function materializeProviderSessionScan( - manifest: ProviderSessionManifest, - limit = 80, - options?: { maxLimit?: number; signal?: AbortSignal }, -): Promise { - throwIfAborted(options?.signal); - const startedAt = Date.now(); - const hardLimit = Math.max( - 1, - Number(options?.maxLimit) || MAX_CONVERSATION_SEARCH_SCAN_LIMIT, - ); - const safeLimit = Math.max(1, Math.min(hardLimit, Number(limit) || 80)); - const selected = manifest.candidates.slice(0, safeLimit); - const codexTitleMap = - manifest.provider === "codex" ? await getCodexThreadTitleMap() : null; - throwIfAborted(options?.signal); - const rows: ProviderSessionRow[] = await Promise.all( - selected.map((candidate) => - materializeProviderSessionRow(manifest, candidate, codexTitleMap, { - signal: options?.signal, - }), - ), - ); - - return { - provider: manifest.provider, - name: manifest.name, - status: providerStatus(manifest.root_exists, manifest.candidates.length), - rows, - scanned: rows.length, - truncated: manifest.candidates.length > safeLimit, - scan_ms: Math.max(0, Date.now() - startedAt), - total_bytes: manifest.total_bytes, - }; -} - -function materializeProviderSessionMetadataScan( - manifest: ProviderSessionManifest, - limit = 80, - options?: { maxLimit?: number }, -): ProviderSessionScan { - const startedAt = Date.now(); - const hardLimit = Math.max( - 1, - Number(options?.maxLimit) || MAX_CONVERSATION_SEARCH_SCAN_LIMIT, - ); - const safeLimit = Math.max(1, Math.min(hardLimit, Number(limit) || 80)); - const rows: ProviderSessionRow[] = manifest.candidates.slice(0, safeLimit).map((candidate) => ({ - provider: manifest.provider, - source: candidate.source, - session_id: inferSessionId(candidate.file_path), - display_title: "", - file_path: candidate.file_path, - size_bytes: candidate.size_bytes, - mtime: candidate.mtime, - probe: EMPTY_PROVIDER_SESSION_PROBE, - })); - - return { - provider: manifest.provider, - name: manifest.name, - status: providerStatus(manifest.root_exists, manifest.candidates.length), - rows, - scanned: rows.length, - truncated: manifest.candidates.length > safeLimit, - scan_ms: Math.max(0, Date.now() - startedAt), - total_bytes: manifest.total_bytes, - }; -} - -async function materializeProviderSessionRow( - manifest: ProviderSessionManifest, - candidate: ProviderSessionCandidate, - codexTitleMap?: Map | null, - options?: { signal?: AbortSignal }, -): Promise { - throwIfAborted(options?.signal); - const rawSessionId = inferSessionId(candidate.file_path); - const codexThreadId = - manifest.provider === "codex" - ? extractCodexThreadIdFromSessionName(rawSessionId) - : ""; - const sessionId = codexThreadId || rawSessionId; - const probe = await probeSessionFile(candidate.file_path); - throwIfAborted(options?.signal); - return { - provider: manifest.provider, - source: candidate.source, - session_id: sessionId, - display_title: - (codexTitleMap && codexThreadId ? codexTitleMap.get(codexThreadId) : "") || - probe.detected_title || - fallbackDisplayTitle(manifest.provider, sessionId, candidate.source), - file_path: candidate.file_path, - size_bytes: candidate.size_bytes, - mtime: candidate.mtime, - probe, - }; -} - -function dedupeProviderSessionCandidates( - provider: ProviderId, - candidates: ProviderSessionCandidate[], -): ProviderSessionCandidate[] { - const uniqueCandidates = new Map(); - for (const candidate of candidates) { - const key = `${provider}:${inferSessionId(candidate.file_path) || path.resolve(candidate.file_path)}`; - const existing = uniqueCandidates.get(key); - if (!existing) { - uniqueCandidates.set(key, candidate); - continue; - } - const existingPriority = providerSessionSourcePriority(existing.source); - const nextPriority = providerSessionSourcePriority(candidate.source); - if (nextPriority > existingPriority || (nextPriority === existingPriority && candidate.mtime_ms > existing.mtime_ms)) { - uniqueCandidates.set(key, candidate); - } - } - return [...uniqueCandidates.values()]; -} - -function selectProviderSessionManifestCandidate( - manifest: ProviderSessionManifest, - options: { sessionId: string; filePath?: string }, -): ProviderSessionCandidate | null { - const normalizedFilePath = options.filePath ? path.resolve(options.filePath) : ""; - const dedupedCandidates = dedupeProviderSessionCandidates(manifest.provider, manifest.candidates); - const matchesRequestedSession = (candidate: ProviderSessionCandidate) => - inferSessionId(candidate.file_path) === options.sessionId; - const filePathMatch = normalizedFilePath - ? dedupedCandidates.find((candidate) => path.resolve(candidate.file_path) === normalizedFilePath) - : null; - if (filePathMatch && matchesRequestedSession(filePathMatch)) { - return filePathMatch; - } - return dedupedCandidates.find(matchesRequestedSession) ?? null; -} - -async function walkRootExists(root: string): Promise { - try { - await stat(root); - return true; - } catch { - return false; - } -} - -export async function getProviderSessionScan( - provider: ProviderId, - limit = 80, - options?: { forceRefresh?: boolean; signal?: AbortSignal }, -): Promise { - const safeLimit = Math.max( - 1, - Math.min(MAX_CONVERSATION_SEARCH_SCAN_LIMIT, Number(limit) || 80), - ); - const key = providerScanCacheKey(provider, safeLimit); - const forceRefresh = Boolean(options?.forceRefresh); - const now = Date.now(); - - if (!forceRefresh) { - const cached = providerScanCache.get(key); - if (cached && cached.expires_at > now) return cached.scan; - } else { - invalidateProviderSearchCaches(provider); - if (provider === "codex") { - invalidateCodexThreadTitleMapCache(); - } - } - const generation = getProviderSearchGeneration(provider); - - const inflight = providerScanInflight.get(key); - if (inflight) return awaitAbortable(inflight, options?.signal); - - const task = getProviderSessionManifest(provider, forceRefresh, options?.signal) - .then((manifest) => - materializeProviderSessionScan(manifest, safeLimit, { - signal: options?.signal, - }), - ) - .then((scan) => { - if (getProviderSearchGeneration(provider) === generation) { - providerScanCache.set(key, { - expires_at: Date.now() + PROVIDER_SCAN_CACHE_TTL_MS, - scan, - }); - } - return scan; - }) - .finally(() => { - if (providerScanInflight.get(key) === task) { - providerScanInflight.delete(key); - } - }); - - providerScanInflight.set(key, task); - return awaitAbortable(task, options?.signal); -} - -async function getProviderSessionManifest( - provider: ProviderId, - forceRefresh = false, - signal?: AbortSignal, -): Promise { - const now = Date.now(); - const skipPersistedManifest = forceRefresh || invalidatedProviderManifests.has(provider); - const generation = getProviderSearchGeneration(provider); - throwIfAborted(signal); - if (!skipPersistedManifest) { - const cached = providerManifestCache.get(provider); - if (cached && cached.expires_at > now) return cached.manifest; - const persisted = await readPersistedProviderManifest(provider); - if (persisted && persisted.expires_at > now) { - providerManifestCache.set(provider, persisted); - return persisted.manifest; - } - } - - const inflight = providerManifestInflight.get(provider); - if (inflight) return awaitAbortable(inflight, signal); - - const task = buildProviderSessionManifest(provider, { signal }) - .then((manifest) => { - throwIfAborted(signal); - return manifest; - }) - .then((manifest) => { - if (getProviderSearchGeneration(provider) !== generation) { - return manifest; - } - const entry = { - expires_at: Date.now() + PROVIDER_MANIFEST_CACHE_TTL_MS, - manifest, - }; - providerManifestCache.set(provider, entry); - invalidatedProviderManifests.delete(provider); - return writePersistedProviderManifest(provider, entry) - .catch(() => undefined) - .then(() => manifest); - }) - .finally(() => { - if (providerManifestInflight.get(provider) === task) { - providerManifestInflight.delete(provider); - } - }); - - providerManifestInflight.set(provider, task); - return awaitAbortable(task, signal); -} - -export async function primeConversationSearchCaches( - providers: ProviderId[] = defaultConversationSearchProviders(), -): Promise { - const targets = Array.from(new Set(providers)); - await Promise.allSettled([ - ...targets.map((provider) => - getProviderSessionManifest(provider, false), - ), - resolveOpenableThreadIds(false), - ]); -} - -export function defaultConversationSearchProviders(): ProviderId[] { - return [...DEFAULT_CONVERSATION_SEARCH_PROVIDERS]; -} - -function buildSearchIdentity( - row: ProviderSessionRow, - openableThreadIds?: Set, -): { - sessionId: string; - threadId: string | null; - title: string; -} { - const rawSessionId = inferSessionId(row.file_path) || row.session_id; - const threadIdCandidate = - row.provider === "codex" - ? extractCodexThreadIdFromSessionName(rawSessionId) || - extractCodexThreadIdFromSessionName(row.session_id) || - row.session_id || - "" - : ""; - const threadId = - threadIdCandidate && openableThreadIds - ? openableThreadIds.has(threadIdCandidate) - ? threadIdCandidate - : "" - : threadIdCandidate; - - return { - sessionId: rawSessionId, - threadId: threadId || null, - title: - row.display_title || - row.probe.detected_title || - fallbackDisplayTitle(row.provider, row.session_id, row.source), - }; -} - -export function isMetadataOnlyConversationQuery(query: string): boolean { - const normalized = normalizeSearchText(query); - if (!normalized) return false; - if (/[\\/]/.test(normalized)) return true; - if (/\.(jsonl|json|data|md|txt)\b/i.test(normalized)) return true; - if (/^rollout-\d{4}-\d{2}-\d{2}/i.test(normalized)) return true; - if ( - /^[0-9a-f]{8}-[0-9a-f]{4,}(?:-[0-9a-f]{4,}){2,}$/i.test(normalized) - ) { - return true; - } - return false; -} - -function buildBaseSearchResult( - row: ProviderSessionRow, - identity: ReturnType, -) { - return { - provider: row.provider, - session_id: identity.sessionId, - title: identity.title, - file_path: row.file_path, - mtime: row.mtime, - ...(identity.threadId ? { thread_id: identity.threadId } : {}), - }; -} - -function buildMetadataSearchResult( - row: ProviderSessionRow, - identity: ReturnType, - normalizedQuery: string, - tokens: string[], -): { result: ConversationSearchResult; meta: SearchMatchMeta } | null { - const metadataCandidates = [ - identity.title, - identity.sessionId, - row.source, - path.basename(row.file_path), - providerName(row.provider), - row.file_path, - ] - .map((value) => normalizeSearchText(value)) - .filter(Boolean); - - for (const candidate of metadataCandidates) { - if (!matchesConversationSearch(candidate, normalizedQuery, tokens)) continue; - return { - result: { - ...buildBaseSearchResult(row, identity), - ...(row.display_title ? { display_title: row.display_title } : {}), - source: row.source, - match_kind: "title", - snippet: buildSearchSnippet(candidate, normalizedQuery, tokens, 140), - }, - meta: { - exactPhrase: isExactPhraseSearchMatch(candidate, normalizedQuery), - }, - }; - } - - return null; -} - -function isRawConversationFileSearchEligible(filePath: string): boolean { - return RAW_CONVERSATION_FILE_EXT_PATTERN.test(filePath); -} - -function buildRipgrepPattern(query: string): string | null { - const trimmedQuery = normalizeSearchText(query); - const normalizedQuery = normalizeSearchQuery(trimmedQuery); - const tokens = buildSearchTokens(trimmedQuery); - if (!normalizedQuery && !tokens.length) return null; - if (normalizedQuery && tokens.length <= 1) { - return normalizedQuery; - } - if (tokens.length > 0) { - return tokens[0]; - } - return normalizedQuery; -} - -async function searchConversationFilesWithRipgrep( - rows: ProviderSessionRow[], - q: string, - options?: { - previewHitsPerSession?: number; - maxHitsPerSession?: number; - signal?: AbortSignal; - }, -): Promise> { - const pattern = buildRipgrepPattern(q); - if (!pattern) return new Map(); - const previewLimit = Math.max( - 1, - Math.min( - RAW_CONVERSATION_FILE_MAX_MATCHES, - Number(options?.previewHitsPerSession) || 3, - ), - ); - const maxHitsPerSession = Math.max( - previewLimit, - Math.min( - RAW_CONVERSATION_FILE_MAX_MATCHES, - Number(options?.maxHitsPerSession) || previewLimit, - ), - ); - const maxPerFile = maxHitsPerSession + 1; - const normalizedQuery = normalizeSearchQuery(q); - const tokens = buildSearchTokens(q); - const textRows = rows.filter((row) => isRawConversationFileSearchEligible(row.file_path)); - if (!textRows.length) return new Map(); - - const files = textRows.map((row) => row.file_path); - const matches = new Map(); - for (let index = 0; index < files.length; index += RAW_FILE_SEARCH_CHUNK_SIZE) { - throwIfAborted(options?.signal); - const chunk = files.slice(index, index + RAW_FILE_SEARCH_CHUNK_SIZE); - try { - const stdout = await new Promise((resolve, reject) => { - const child = spawn( - "rg", - [ - "--json", - "-F", - "-i", - "-m", - String(maxPerFile), - pattern, - "--", - ...chunk, - ], - { - stdio: ["ignore", "pipe", "pipe"], - }, - ); - let out = ""; - let err = ""; - const abortHandler = () => { - child.kill("SIGTERM"); - reject(createAbortError()); - }; - options?.signal?.addEventListener("abort", abortHandler, { once: true }); - child.stdout.setEncoding("utf8"); - child.stderr.setEncoding("utf8"); - child.stdout.on("data", (data) => { - out += data; - }); - child.stderr.on("data", (data) => { - err += data; - }); - child.on("error", (error) => { - options?.signal?.removeEventListener("abort", abortHandler); - reject(error); - }); - child.on("close", (code) => { - options?.signal?.removeEventListener("abort", abortHandler); - if (code === 0 || code === 1) { - resolve(out); - return; - } - reject(new Error(err || `rg exited with code ${String(code)}`)); - }); - }); - - for (const line of stdout.split(/\r?\n/)) { - throwIfAborted(options?.signal); - if (!line.trim()) continue; - let event: unknown; - try { - event = JSON.parse(line); - } catch { - continue; - } - if (!event || typeof event !== "object") continue; - const parsed = event as { - type?: string; - data?: { - path?: { text?: string }; - lines?: { text?: string }; - }; - }; - if (parsed.type !== "match") continue; - const filePath = parsed.data?.path?.text; - const rawLine = parsed.data?.lines?.text; - if (!filePath || !rawLine) continue; - if (!matchesConversationSearch(rawLine, normalizedQuery, tokens)) continue; - const current = matches.get(filePath) ?? { - snippets: [], - match_count: 0, - has_more_hits: false, - exact_phrase_count: 0, - }; - current.match_count += 1; - if (isExactPhraseSearchMatch(rawLine, normalizedQuery)) { - current.exact_phrase_count += 1; - } - if (current.snippets.length < maxHitsPerSession) { - const snippet = buildSearchSnippet(rawLine, normalizedQuery, tokens); - if (snippet && !current.snippets.includes(snippet)) { - current.snippets.push(snippet); - } - } - if (current.match_count > maxHitsPerSession) { - current.has_more_hits = true; - } - matches.set(filePath, current); - } - } catch (error) { - if ((error as Error)?.name === "AbortError") throw error; - const failure = error as { code?: number }; - if (failure.code === 1) continue; - throw error; - } - } - - return matches; -} - -async function collectConversationSessionMatches( - row: ProviderSessionRow, - q: string, - options?: { - transcriptLoader?: CachedConversationTranscriptLoader; - transcriptLimit?: number; - openableThreadIds?: Set; - previewHitsPerSession?: number; - exhaustiveHits?: boolean; - rawFileMatch?: RawConversationFileMatch | null; - rawFileSearchComplete?: boolean; - signal?: AbortSignal; - }, -): Promise { - throwIfAborted(options?.signal); - const trimmedQuery = normalizeSearchText(q); - const normalizedQuery = normalizeSearchQuery(trimmedQuery); - const tokens = buildSearchTokens(trimmedQuery); - const transcriptLimit = Math.max( - 100, - Math.min( - DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, - Number(options?.transcriptLimit) || DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, - ), - ); - const metadataOnlyQuery = isMetadataOnlyConversationQuery(trimmedQuery); - const transcriptLoader = - options?.transcriptLoader ?? - createCachedConversationTranscriptLoader((provider: ProviderId, filePath: string) => - buildSessionTranscript(provider, filePath, transcriptLimit), - ); - const identity = buildSearchIdentity(row, options?.openableThreadIds); - const exhaustiveHits = Boolean(options?.exhaustiveHits); - const previewHitsPerSession = Math.max( - 1, - Math.min( - exhaustiveHits ? RAW_CONVERSATION_FILE_MAX_MATCHES : 20, - Number(options?.previewHitsPerSession) || 3, - ), - ); - const accumulator = createSessionAccumulator(row, identity); - const metadataResult = buildMetadataSearchResult( - row, - identity, - normalizedQuery, - tokens, - ); - - if (metadataResult) { - addSessionSearchHit( - accumulator, - metadataResult.result, - metadataResult.meta, - previewHitsPerSession, - ); - } - - if (options?.rawFileMatch) { - for (const snippet of options.rawFileMatch.snippets) { - addSessionSearchHit( - accumulator, - { - ...buildBaseSearchResult(row, identity), - ...(row.display_title ? { display_title: row.display_title } : {}), - source: row.source, - match_kind: "message", - snippet, - }, - { - exactPhrase: isExactPhraseSearchMatch(snippet, normalizedQuery), - }, - previewHitsPerSession, - ); - } - accumulator.session.match_count = Math.max( - accumulator.session.match_count, - options.rawFileMatch.match_count, - ); - accumulator.session.has_more_hits = - options.rawFileMatch.has_more_hits || - accumulator.session.match_count > accumulator.session.preview_matches.length; - accumulator.exact_phrase_count = Math.max( - accumulator.exact_phrase_count, - options.rawFileMatch.exact_phrase_count, - ); - return accumulator.session.match_count > 0 ? accumulator : null; - } - - if (options?.rawFileSearchComplete && isRawConversationFileSearchEligible(row.file_path)) { - return accumulator.session.match_count > 0 ? accumulator : null; - } - - if (!metadataOnlyQuery) { - throwIfAborted(options?.signal); - const transcript = await transcriptLoader(row); - if (transcript?.messages?.length) { - for (let i = 0; i < transcript.messages.length; i += 1) { - throwIfAborted(options?.signal); - const message = transcript.messages[i]; - if (message.role === "system" || message.role === "tool") continue; - if (isPolicyInjectionMessage(message.text)) continue; - if (!matchesConversationSearch(message.text, normalizedQuery, tokens)) continue; - addSessionSearchHit( - accumulator, - { - ...buildBaseSearchResult(row, identity), - ...(row.display_title ? { display_title: row.display_title } : {}), - source: row.source, - match_kind: "message", - snippet: buildSearchSnippet(message.text, normalizedQuery, tokens), - role: message.role, - }, - { - exactPhrase: isExactPhraseSearchMatch(message.text, normalizedQuery), - }, - previewHitsPerSession, - ); - if (!exhaustiveHits && accumulator.session.match_count >= previewHitsPerSession + 1) { - break; - } - } - } - } - - if (accumulator.session.match_count === 0) return null; - accumulator.session.has_more_hits = - accumulator.session.match_count > accumulator.session.preview_matches.length; - return accumulator; -} - -export async function searchConversationSessions( - rows: ProviderSessionRow[], - q: string, - options?: { - pageSize?: number; - cursor?: string; - transcriptLoader?: ConversationTranscriptLoader; - transcriptLimit?: number; - openableThreadIds?: Set; - previewHitsPerSession?: number; - rawFileSearchLoader?: RawConversationFileSearchLoader; - signal?: AbortSignal; - }, -): Promise<{ - searched_sessions: number; - available_sessions: number; - truncated: boolean; - total_matching_sessions: number | null; - total_matching_hits: number | null; - has_more: boolean; - next_cursor: string | null; - sessions: ConversationSearchSessionResult[]; - results: ConversationSearchResult[]; -}> { - const pageSize = Math.max(1, Math.min(200, Number(options?.pageSize) || 40)); - const offset = decodeConversationSearchCursor(options?.cursor); - const metadataOnlyQuery = isMetadataOnlyConversationQuery(q); - const transcriptLimit = Math.max( - 100, - Math.min( - DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, - Number(options?.transcriptLimit) || DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, - ), - ); - const cachedTranscriptLoader = - options?.transcriptLoader - ? createCachedConversationTranscriptLoader(options.transcriptLoader) - : undefined; - const seenPaths = new Set(); - const sortedRows = dedupeConversationSearchRows( - [...rows] - .sort( - (a, b) => - Date.parse(String(b.mtime || "")) - Date.parse(String(a.mtime || "")), - ) - .filter((row) => { - const key = `${row.provider}:${path.resolve(row.file_path)}`; - if (seenPaths.has(key)) return false; - seenPaths.add(key); - return true; - }), - ).sort( - (a, b) => - Date.parse(String(b.mtime || "")) - Date.parse(String(a.mtime || "")), - ); - - const sessionMatches: SearchSessionAccumulator[] = []; - let scannedRows = 0; - let rawFileMatches = new Map(); - let rawFileSearchComplete = false; - const shouldUseRawFileSearch = - !metadataOnlyQuery && (Boolean(options?.rawFileSearchLoader) || !options?.transcriptLoader); - if (shouldUseRawFileSearch) { - try { - rawFileMatches = await ( - options?.rawFileSearchLoader ?? searchConversationFilesWithRipgrep - )(sortedRows, q, { - previewHitsPerSession: options?.previewHitsPerSession, - signal: options?.signal, - }); - rawFileSearchComplete = true; - } catch (error) { - if ((error as Error)?.name === "AbortError") throw error; - rawFileMatches = new Map(); - rawFileSearchComplete = false; - } - } - - for ( - let index = 0; - index < sortedRows.length; - index += CONVERSATION_SEARCH_SCAN_CONCURRENCY - ) { - throwIfAborted(options?.signal); - const batchRows = sortedRows.slice( - index, - index + CONVERSATION_SEARCH_SCAN_CONCURRENCY, - ); - const batchMatches = await mapWithConcurrency( - batchRows, - CONVERSATION_SEARCH_SCAN_CONCURRENCY, - async (row) => - collectConversationSessionMatches(row, q, { - transcriptLoader: cachedTranscriptLoader, - transcriptLimit, - openableThreadIds: options?.openableThreadIds, - previewHitsPerSession: options?.previewHitsPerSession, - rawFileMatch: rawFileMatches.get(row.file_path) ?? null, - rawFileSearchComplete, - signal: options?.signal, - }), - ); - - for (let batchOffset = 0; batchOffset < batchMatches.length; batchOffset += 1) { - scannedRows = index + batchOffset + 1; - const session = batchMatches[batchOffset]; - if (!session) continue; - sessionMatches.push(session); - } - } - - const orderedMatches = sessionMatches.sort(compareConversationSearchSessions); - const totalMatchingSessions = orderedMatches.length; - const hasMore = offset + pageSize < totalMatchingSessions; - const pageSessions = orderedMatches - .slice(offset, offset + pageSize) - .sort(compareConversationSearchSessions) - .map((item) => item.session); - const nextCursor = hasMore - ? encodeConversationSearchCursor(offset + pageSize) - : null; - const totalMatchingHits = null; - - return { - searched_sessions: scannedRows, - available_sessions: sortedRows.length, - truncated: scannedRows < sortedRows.length, - total_matching_sessions: totalMatchingSessions, - total_matching_hits: totalMatchingHits, - has_more: hasMore, - next_cursor: nextCursor, - sessions: pageSessions, - results: pageSessions.flatMap((session) => session.preview_matches), - }; -} - -export async function searchConversationSessionHits( - row: ProviderSessionRow, - q: string, - options?: { - pageSize?: number; - cursor?: string; - transcriptLoader?: ConversationTranscriptLoader; - transcriptLimit?: number; - openableThreadIds?: Set; - rawFileSearchLoader?: RawConversationFileSearchLoader; - signal?: AbortSignal; - }, -): Promise { - throwIfAborted(options?.signal); - const pageSize = Math.max(1, Math.min(200, Number(options?.pageSize) || 40)); - const offset = decodeConversationSearchCursor(options?.cursor); - const requestedHitWindow = Math.max( - pageSize, - Math.min(RAW_CONVERSATION_FILE_MAX_MATCHES, offset + pageSize + 1), - ); - const transcriptLimit = Math.max( - 100, - Math.min( - DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, - Number(options?.transcriptLimit) || DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, - ), - ); - const cachedTranscriptLoader = - options?.transcriptLoader - ? createCachedConversationTranscriptLoader(options.transcriptLoader) - : undefined; - let rawFileMatch: RawConversationFileMatch | null = null; - let rawFileSearchComplete = false; - const shouldUseRawFileSearch = - Boolean(options?.rawFileSearchLoader) || !options?.transcriptLoader; - if (shouldUseRawFileSearch) { - try { - rawFileMatch = - ( - await (options?.rawFileSearchLoader ?? searchConversationFilesWithRipgrep)([row], q, { - previewHitsPerSession: requestedHitWindow, - maxHitsPerSession: requestedHitWindow, - signal: options?.signal, - }) - ).get(row.file_path) ?? null; - rawFileSearchComplete = true; - } catch (error) { - if ((error as Error)?.name === "AbortError") throw error; - rawFileMatch = null; - rawFileSearchComplete = false; - } - } - const canSatisfyCursorFromRawFile = - !rawFileMatch || - offset < rawFileMatch.snippets.length || - !rawFileMatch.has_more_hits; - const session = await collectConversationSessionMatches(row, q, { - transcriptLoader: cachedTranscriptLoader, - transcriptLimit, - openableThreadIds: options?.openableThreadIds, - previewHitsPerSession: requestedHitWindow, - exhaustiveHits: true, - rawFileMatch: canSatisfyCursorFromRawFile ? rawFileMatch : null, - rawFileSearchComplete: canSatisfyCursorFromRawFile ? rawFileSearchComplete : false, - signal: options?.signal, - }); - - const hits = session?.session.preview_matches ?? []; - const totalHits = Math.max(hits.length, session?.session.match_count ?? 0); - const hasMore = offset + pageSize < totalHits; - const nextCursor = hasMore - ? encodeConversationSearchCursor(offset + pageSize) - : null; - - return { - generated_at: nowIsoUtc(), - q: normalizeSearchText(q), - provider: row.provider, - session_id: row.session_id, - file_path: row.file_path, - page_size: pageSize, - total_hits: totalHits, - has_more: hasMore, - next_cursor: nextCursor, - hits: hits.slice(offset, offset + pageSize), - }; -} - -export async function searchConversationRows( - rows: ProviderSessionRow[], - q: string, - options?: { - limit?: number; - transcriptLoader?: ConversationTranscriptLoader; - transcriptLimit?: number; - openableThreadIds?: Set; - }, -): Promise<{ - searched_sessions: number; - available_sessions: number; - truncated: boolean; - results: ConversationSearchResult[]; -}> { - const trimmedQuery = normalizeSearchText(q); - const normalizedQuery = normalizeSearchQuery(trimmedQuery); - const tokens = buildSearchTokens(trimmedQuery); - const { resultLimit: safeLimit } = resolveConversationSearchLimits({ - limit: options?.limit, - }); - const transcriptLimit = Math.max( - 100, - Math.min( - DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, - Number(options?.transcriptLimit) || - DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, - ), - ); - const transcriptLoader = - options?.transcriptLoader ?? - ((provider: ProviderId, filePath: string) => - buildSessionTranscript(provider, filePath, transcriptLimit)); - const metadataOnlyQuery = isMetadataOnlyConversationQuery(trimmedQuery); - const cachedTranscriptLoader = - createCachedConversationTranscriptLoader(transcriptLoader); - const seenPaths = new Set(); - const sortedRows = [...rows] - .sort( - (a, b) => - Date.parse(String(b.mtime || "")) - Date.parse(String(a.mtime || "")), - ) - .filter((row) => { - const key = `${row.provider}:${path.resolve(row.file_path)}`; - if (seenPaths.has(key)) return false; - seenPaths.add(key); - return true; - }); - const metadataResults: ConversationSearchResult[] = []; - const transcriptRows: Array<{ - row: ProviderSessionRow; - identity: ReturnType; - }> = []; - - for (const row of sortedRows) { - const identity = buildSearchIdentity(row, options?.openableThreadIds); - const metadataResult = buildMetadataSearchResult( - row, - identity, - normalizedQuery, - tokens, - ); - if (metadataResult) { - metadataResults.push(metadataResult.result); - continue; - } - transcriptRows.push({ row, identity }); - } - - let results = dedupeConversationSearchResults(metadataResults); - const metadataSatisfiesLimit = results.length >= safeLimit; - const metadataOnlyTruncated = - results.length > safeLimit || - (metadataSatisfiesLimit && transcriptRows.length > 0); - - if (!metadataSatisfiesLimit && !metadataOnlyQuery) { - let transcriptStoppedEarly = false; - for ( - let offset = 0; - offset < transcriptRows.length && results.length < safeLimit; - offset += SEARCH_TRANSCRIPT_CONCURRENCY - ) { - const chunk = transcriptRows.slice( - offset, - offset + SEARCH_TRANSCRIPT_CONCURRENCY, - ); - const chunkResults = await mapWithConcurrency( - chunk, - SEARCH_TRANSCRIPT_CONCURRENCY, - async ({ row, identity }) => { - const rowResults: ConversationSearchResult[] = []; - const baseResult = buildBaseSearchResult(row, identity); - const transcript = await cachedTranscriptLoader(row); - if (!transcript?.messages?.length) return rowResults; - - for (let i = 0; i < transcript.messages.length; i += 1) { - const message = transcript.messages[i]; - if (message.role === "system" || message.role === "tool") continue; - if (isPolicyInjectionMessage(message.text)) continue; - if ( - !matchesConversationSearch(message.text, normalizedQuery, tokens) - ) { - continue; - } - rowResults.push({ - ...baseResult, - match_kind: "message", - snippet: buildSearchSnippet(message.text, normalizedQuery, tokens), - role: message.role, - }); - } - - return rowResults; - }, - ); - - results = dedupeConversationSearchResults([ - ...results, - ...chunkResults.flat(), - ]); - if ( - results.length >= safeLimit && - offset + chunk.length < transcriptRows.length - ) { - transcriptStoppedEarly = true; - } - - if (results.length >= safeLimit) { - return { - searched_sessions: sortedRows.length, - available_sessions: sortedRows.length, - truncated: results.length > safeLimit || transcriptStoppedEarly, - results: results.slice(0, safeLimit), - }; - } - } - } - - return { - searched_sessions: sortedRows.length, - available_sessions: sortedRows.length, - truncated: metadataOnlyTruncated || results.length > safeLimit, - results: results.slice(0, safeLimit), - }; -} - -async function resolveOpenableThreadIds( - forceRefresh: boolean, -): Promise> { - const now = Date.now(); - if (!forceRefresh && openableThreadIdsCache && openableThreadIdsCache.expires_at > now) { - return new Set(openableThreadIdsCache.ids); - } - if (!forceRefresh && openableThreadIdsInflight) { - return openableThreadIdsInflight.then((ids) => new Set(ids)); - } - - const loadIds = (async () => { - const { getThreadsTs } = await import("../threads/query.js"); - const threads = await getThreadsTs({ - offset: "0", - limit: "240", - q: "", - sort: "updated_desc", - ...(forceRefresh ? { refresh: "1" } : {}), - }); - const ids = new Set( - (threads.rows ?? []) - .map((row) => String(row.thread_id || "").trim()) - .filter(Boolean), - ); - openableThreadIdsCache = { - expires_at: Date.now() + OPENABLE_THREAD_IDS_CACHE_TTL_MS, - ids, - }; - return ids; - })(); - - if (!forceRefresh) { - openableThreadIdsInflight = loadIds; - } - - try { - return new Set(await loadIds); - } finally { - if (openableThreadIdsInflight === loadIds) { - openableThreadIdsInflight = null; - } - } -} - -export async function searchLocalConversationsTs( - q: string, - options?: { - providers?: ProviderId[]; - limit?: number; - pageSize?: number; - cursor?: string; - forceRefresh?: boolean; - transcriptLimit?: number; - previewHitsPerSession?: number; - sessionLimitPerProvider?: number; - signal?: AbortSignal; - }, -): Promise { - const trimmedQuery = normalizeSearchText(q); - const metadataOnlyQuery = isMetadataOnlyConversationQuery(trimmedQuery); - const providers = - options?.providers?.length ? Array.from(new Set(options.providers)) : defaultConversationSearchProviders(); - const safePageSize = Math.max( - 1, - Math.min( - MAX_CONVERSATION_SEARCH_LIMIT, - Number(options?.pageSize) || Number(options?.limit) || DEFAULT_CONVERSATION_SEARCH_LIMIT, - ), - ); - const forceRefresh = Boolean(options?.forceRefresh); - const cacheKey = conversationSearchResponseCacheKey({ - q: trimmedQuery, - providers, - pageSize: safePageSize, - cursor: options?.cursor, - transcriptLimit: options?.transcriptLimit, - previewHitsPerSession: options?.previewHitsPerSession, - sessionLimitPerProvider: options?.sessionLimitPerProvider, - }); - if (!forceRefresh) { - const cached = conversationSearchResponseCache.get(cacheKey); - if (cached && cached.expires_at > Date.now()) { - return cached.payload; - } - } - const explicitPerProviderLimit = Math.max( - 0, - Math.floor(Number(options?.sessionLimitPerProvider) || 0), - ); - const providerBudgets = - explicitPerProviderLimit > 0 - ? providers.map((provider) => ({ - provider, - limit: explicitPerProviderLimit, - })) - : buildConversationSearchProviderBudgets( - providers, - resolveConversationSearchLimits({ limit: safePageSize }).scanLimit, - ); - const scans = await Promise.all( - providerBudgets.map(async ({ provider, limit }) => { - throwIfAborted(options?.signal); - if (!metadataOnlyQuery) { - return getProviderSessionScan(provider, limit, { - forceRefresh, - signal: options?.signal, - }); - } - const manifest = await getProviderSessionManifest( - provider, - forceRefresh, - options?.signal, - ); - return materializeProviderSessionMetadataScan(manifest, limit); - }), - ); - const rows = scans.flatMap((scan) => scan.rows); - const openableThreadIds = await resolveOpenableThreadIds(forceRefresh); - const result = await searchConversationSessions(rows, trimmedQuery, { - pageSize: safePageSize, - cursor: options?.cursor, - transcriptLimit: options?.transcriptLimit, - openableThreadIds, - previewHitsPerSession: options?.previewHitsPerSession, - signal: options?.signal, - }); - - const payload = { - generated_at: nowIsoUtc(), - q: trimmedQuery, - providers, - limit: safePageSize, - page_size: safePageSize, - searched_sessions: result.searched_sessions, - available_sessions: result.available_sessions, - truncated: scans.some((scan) => scan.truncated) || result.truncated, - total_matching_sessions: result.total_matching_sessions, - total_matching_hits: result.total_matching_hits, - has_more: result.has_more, - next_cursor: result.next_cursor, - preview_hits_per_session: Math.max( - 1, - Math.min(20, Number(options?.previewHitsPerSession) || 3), - ), - sessions: result.sessions, - results: result.results, - }; - if (!forceRefresh) { - conversationSearchResponseCache.set(cacheKey, { - expires_at: Date.now() + CONVERSATION_SEARCH_RESPONSE_CACHE_TTL_MS, - payload, - }); - } - return payload; -} - -export async function searchConversationSessionHitsTs( - q: string, - options: { - provider: ProviderId; - sessionId: string; - filePath?: string; - pageSize?: number; - cursor?: string; - forceRefresh?: boolean; - transcriptLimit?: number; - signal?: AbortSignal; - }, -): Promise { - const forceRefresh = Boolean(options.forceRefresh); - const manifest = await getProviderSessionManifest( - options.provider, - forceRefresh, - options.signal, - ); - const candidate = selectProviderSessionManifestCandidate(manifest, { - sessionId: options.sessionId, - filePath: options.filePath, - }); - if (!candidate) return null; - const codexTitleMap = - manifest.provider === "codex" ? await getCodexThreadTitleMap() : null; - const targetRow = await materializeProviderSessionRow( - manifest, - candidate, - codexTitleMap, - { signal: options.signal }, - ); - if (!targetRow) return null; - if (!targetRow.probe.ok) return null; - const openableThreadIds = await resolveOpenableThreadIds(forceRefresh); - return searchConversationSessionHits(targetRow, q, { - pageSize: options.pageSize, - cursor: options.cursor, - transcriptLimit: options.transcriptLimit, - openableThreadIds, - signal: options.signal, - }); -} - -export function selectConversationSessionHitsRow( - rows: ProviderSessionRow[], - options: { sessionId: string; filePath?: string }, -): ProviderSessionRow | null { - const normalizedFilePath = options.filePath ? path.resolve(options.filePath) : ""; - const dedupedRows = dedupeConversationSearchRows(rows); - const matchesRequestedSession = (row: ProviderSessionRow) => - row.session_id === options.sessionId || inferSessionId(row.file_path) === options.sessionId; - const filePathMatch = normalizedFilePath - ? dedupedRows.find((row) => path.resolve(row.file_path) === normalizedFilePath) - : null; - if (filePathMatch && matchesRequestedSession(filePathMatch)) { - return filePathMatch; - } - return dedupedRows.find(matchesRequestedSession) ?? null; -} - -export async function getProviderSessionsTs( - provider?: ProviderId, - limit = 80, - options?: { forceRefresh?: boolean }, -) { - const targets: ProviderId[] = provider - ? [provider] - : ["codex", "chatgpt", "claude", "gemini", "copilot"]; - const scans = await Promise.all( - targets.map((p) => getProviderSessionScan(p, limit, options)), - ); - - const rows = scans.flatMap((scan) => scan.rows); - return { - generated_at: nowIsoUtc(), - summary: { - providers: scans.length, - rows: rows.length, - parse_ok: rows.filter((row) => row.probe.ok).length, - parse_fail: rows.filter((row) => !row.probe.ok).length, - }, - providers: scans.map((scan) => ({ - provider: scan.provider, - name: scan.name, - status: scan.status, - scanned: scan.scanned, - truncated: scan.truncated, - scan_ms: scan.scan_ms, - total_bytes: scan.total_bytes, - })), - rows, - }; -} - -export async function getProviderParserHealthTs( - provider?: ProviderId, - limitPerProvider = 80, - options?: { forceRefresh?: boolean }, -) { - const targets: ProviderId[] = provider - ? [provider] - : ["codex", "chatgpt", "claude", "gemini", "copilot"]; - const scans = await Promise.all( - targets.map((item) => getProviderSessionScan(item, limitPerProvider, options)), - ); - const reports: Array> = scans.map((scan) => { - const parseOk = scan.rows.filter((row) => row.probe.ok).length; - const parseFail = scan.rows.length - parseOk; - const score = scan.rows.length - ? Number(((parseOk / scan.rows.length) * 100).toFixed(1)) - : null; - return { - provider: scan.provider, - name: scan.name, - status: scan.status, - scanned: scan.rows.length, - parse_ok: parseOk, - parse_fail: parseFail, - parse_score: score, - truncated: scan.truncated, - scan_ms: scan.scan_ms, - sample_errors: scan.rows - .filter((row) => !row.probe.ok) - .slice(0, 8) - .map((row) => ({ - session_id: row.session_id, - path: row.file_path, - format: row.probe.format, - error: row.probe.error, - })), - }; - }); - const totalScanned = reports.reduce( - (sum, row) => sum + Number((row.scanned as number) || 0), - 0, - ); - const totalFail = reports.reduce( - (sum, row) => sum + Number((row.parse_fail as number) || 0), - 0, - ); - const totalOk = reports.reduce( - (sum, row) => sum + Number((row.parse_ok as number) || 0), - 0, - ); - return { - generated_at: nowIsoUtc(), - summary: { - providers: reports.length, - scanned: totalScanned, - parse_ok: totalOk, - parse_fail: totalFail, - parse_score: totalScanned - ? Number(((totalOk / totalScanned) * 100).toFixed(1)) - : null, - }, - reports, - }; -} - -export async function resolveCodexSessionPathByThreadId( - threadId: string, -): Promise { - const normalized = String(threadId || "").trim(); - if (!normalized) return null; - - const recent = await getProviderSessionScan("codex", 240); - const inRecent = recent.rows.find((row) => row.session_id === normalized); - if (inRecent) return inRecent.file_path; - - const roots = codexTranscriptSearchRoots(); - for (const spec of roots) { - const files = await walkFilesByExt(spec.root, spec.exts, 8000); - const hit = files.find((file) => path.basename(file).includes(normalized)); - if (hit) return hit; - } - return null; -} +export { + isMetadataOnlyConversationQuery, +} from "./search/session-results.js"; +export { + buildConversationSearchProviderBudgets, + resolveConversationSearchLimits, +} from "./search/limits.js"; +export { + searchConversationRows, +} from "./search/row-search.js"; +export { + createCachedConversationTranscriptLoader, +} from "./search/transcript-matches.js"; +export { + searchConversationSessionHits, + searchConversationSessions, +} from "./search/session-search.js"; +export { + getProviderParserHealthTs, + getProviderSessionsTs, +} from "./search/provider-session-reports.js"; +export { + searchLocalConversationsTs, +} from "./search/local-search.js"; +export { + searchConversationSessionHitsTs, + selectConversationSessionHitsRow, +} from "./search/session-hit-resolver.js"; +export { + resolveCodexSessionPathByThreadId, +} from "./search/codex-session-path.js"; +export { + defaultConversationSearchProviders, + getProviderSessionScan, + invalidateProviderSearchCaches, + primeConversationSearchCaches, +} from "./search/provider-session-scan.js"; diff --git a/apps/api-ts/src/domains/providers/search/abort.ts b/apps/api-ts/src/domains/providers/search/abort.ts new file mode 100644 index 0000000..ab8b2a1 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/abort.ts @@ -0,0 +1,37 @@ +export function createAbortError(): Error { + const error = new Error("conversation-search-aborted"); + error.name = "AbortError"; + return error; +} + +export function throwIfAborted(signal?: AbortSignal): void { + if (signal?.aborted) throw createAbortError(); +} + +export function awaitAbortable( + promise: Promise, + signal?: AbortSignal, +): Promise { + if (!signal) return promise; + return new Promise((resolve, reject) => { + const onAbort = () => { + signal.removeEventListener("abort", onAbort); + reject(createAbortError()); + }; + if (signal.aborted) { + onAbort(); + } else { + signal.addEventListener("abort", onAbort, { once: true }); + } + promise.then( + (value) => { + signal.removeEventListener("abort", onAbort); + resolve(value); + }, + (error) => { + signal.removeEventListener("abort", onAbort); + reject(error); + }, + ); + }); +} diff --git a/apps/api-ts/src/domains/providers/search/cache.ts b/apps/api-ts/src/domains/providers/search/cache.ts new file mode 100644 index 0000000..c8d0786 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/cache.ts @@ -0,0 +1,76 @@ +import path from "node:path"; + +import type { ProviderId, ProviderSessionRow, ProviderSessionScan } from "../types.js"; +import { SEARCH_TRANSCRIPT_CACHE_MAX_ENTRIES } from "./constants.js"; +import type { + ConversationSearchResponseCacheEntry, + ProviderManifestCacheEntry, + ProviderSessionManifest, + ProviderScanCacheEntry, + TranscriptSearchCacheEntry, +} from "./types.js"; + +export const providerScanCache = new Map(); +export const providerScanInflight = new Map>(); +export const providerManifestCache = new Map(); +export const providerManifestInflight = new Map< + ProviderId, + Promise +>(); +export const invalidatedProviderManifests = new Set(); +export const conversationSearchResponseCache = new Map< + string, + ConversationSearchResponseCacheEntry +>(); +export const transcriptSearchCache = new Map(); + +const providerSearchGeneration = new Map(); + +export function getProviderSearchGeneration(provider: ProviderId): number { + return providerSearchGeneration.get(provider) ?? 0; +} + +export function bumpProviderSearchGeneration(provider: ProviderId): number { + const next = getProviderSearchGeneration(provider) + 1; + providerSearchGeneration.set(provider, next); + return next; +} + +export function providerScanCacheKey( + provider: ProviderId, + limit: number, +): string { + return `${provider}:${limit}`; +} + +export function transcriptSearchCacheKey(row: ProviderSessionRow): string { + return `${row.provider}:${path.resolve(row.file_path)}`; +} + +export function conversationSearchResponseCacheKey(options: { + q: string; + providers: ProviderId[]; + pageSize: number; + cursor?: string; + transcriptLimit?: number; + previewHitsPerSession?: number; + sessionLimitPerProvider?: number; +}): string { + return [ + options.q, + [...options.providers].sort().join(","), + options.pageSize, + options.cursor || "", + options.transcriptLimit || "", + options.previewHitsPerSession || "", + options.sessionLimitPerProvider || "", + ].join("::"); +} + +export function trimTranscriptSearchCache() { + while (transcriptSearchCache.size > SEARCH_TRANSCRIPT_CACHE_MAX_ENTRIES) { + const oldestKey = transcriptSearchCache.keys().next().value; + if (!oldestKey) return; + transcriptSearchCache.delete(oldestKey); + } +} diff --git a/apps/api-ts/src/domains/providers/search/codex-session-path.ts b/apps/api-ts/src/domains/providers/search/codex-session-path.ts new file mode 100644 index 0000000..a7173ca --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/codex-session-path.ts @@ -0,0 +1,30 @@ +import path from "node:path"; + +import { + codexTranscriptSearchRoots, +} from "../path-safety.js"; +import { + walkFilesByExt, +} from "../../../lib/utils.js"; +import { + getProviderSessionScan, +} from "./provider-session-scan.js"; + +export async function resolveCodexSessionPathByThreadId( + threadId: string, +): Promise { + const normalized = String(threadId || "").trim(); + if (!normalized) return null; + + const recent = await getProviderSessionScan("codex", 240); + const inRecent = recent.rows.find((row) => row.session_id === normalized); + if (inRecent) return inRecent.file_path; + + const roots = codexTranscriptSearchRoots(); + for (const spec of roots) { + const files = await walkFilesByExt(spec.root, spec.exts, 8000); + const hit = files.find((file) => path.basename(file).includes(normalized)); + if (hit) return hit; + } + return null; +} diff --git a/apps/api-ts/src/domains/providers/search/concurrency.ts b/apps/api-ts/src/domains/providers/search/concurrency.ts new file mode 100644 index 0000000..bca879e --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/concurrency.ts @@ -0,0 +1,23 @@ +export async function mapWithConcurrency( + items: T[], + concurrency: number, + worker: (item: T, index: number) => Promise, +): Promise { + if (items.length === 0) return []; + const results = new Array(items.length); + let nextIndex = 0; + const workerCount = Math.max(1, Math.min(concurrency, items.length)); + + await Promise.all( + Array.from({ length: workerCount }, async () => { + while (true) { + const currentIndex = nextIndex; + nextIndex += 1; + if (currentIndex >= items.length) return; + results[currentIndex] = await worker(items[currentIndex], currentIndex); + } + }), + ); + + return results; +} diff --git a/apps/api-ts/src/domains/providers/search/constants.ts b/apps/api-ts/src/domains/providers/search/constants.ts new file mode 100644 index 0000000..6110c80 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/constants.ts @@ -0,0 +1,36 @@ +import { SEARCHABLE_PROVIDER_IDS } from "@threadlens/shared-contracts"; + +import type { ProviderId } from "../types.js"; + +export const PROVIDER_SCAN_CACHE_TTL_MS = 60_000; +export const PROVIDER_MANIFEST_CACHE_TTL_MS = 5 * 60_000; +export const OPENABLE_THREAD_IDS_CACHE_TTL_MS = 15_000; +export const CONVERSATION_SEARCH_RESPONSE_CACHE_TTL_MS = 15_000; +export const DEFAULT_CONVERSATION_SEARCH_PROVIDERS: ProviderId[] = [ + ...SEARCHABLE_PROVIDER_IDS, +]; +export const DEFAULT_CONVERSATION_SEARCH_LIMIT = 40; +export const MAX_CONVERSATION_SEARCH_LIMIT = 200; +export const DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT = 10_000; +export const MAX_CONVERSATION_SEARCH_SCAN_LIMIT = 1_200; +export const SEARCH_TRANSCRIPT_CONCURRENCY = 4; +export const CONVERSATION_SEARCH_SCAN_CONCURRENCY = 8; +export const SEARCH_TRANSCRIPT_CACHE_MAX_ENTRIES = 2_000; +export const DEFAULT_CONVERSATION_SEARCH_SCAN_MULTIPLIER = 4; +export const DEFAULT_CONVERSATION_SEARCH_SCAN_FLOOR = 160; +export const PROVIDER_SCAN_FILE_STAT_CONCURRENCY = 32; +export const PROVIDER_SESSION_MANIFEST_FILE_LIMIT = 8_000; +export const PROVIDER_SCAN_BUDGET_WEIGHTS: Partial> = { + codex: 1.35, + chatgpt: 0.6, + claude: 1.35, + gemini: 1, + copilot: 0.85, +}; +export const EMPTY_PROVIDER_SESSION_PROBE = { + ok: true, + format: "unknown" as const, + error: null, + detected_title: "", + title_source: null, +}; diff --git a/apps/api-ts/src/domains/providers/search/limits.ts b/apps/api-ts/src/domains/providers/search/limits.ts new file mode 100644 index 0000000..86f82bb --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/limits.ts @@ -0,0 +1,84 @@ +import type { ProviderId } from "../types.js"; +import { + DEFAULT_CONVERSATION_SEARCH_LIMIT, + DEFAULT_CONVERSATION_SEARCH_SCAN_FLOOR, + DEFAULT_CONVERSATION_SEARCH_SCAN_MULTIPLIER, + MAX_CONVERSATION_SEARCH_LIMIT, + MAX_CONVERSATION_SEARCH_SCAN_LIMIT, + PROVIDER_SCAN_BUDGET_WEIGHTS, +} from "./constants.js"; + +export function resolveConversationSearchLimits(options?: { + limit?: number; + sessionLimitPerProvider?: number; +}) { + const resultLimit = Math.max( + 1, + Math.min( + MAX_CONVERSATION_SEARCH_LIMIT, + Number(options?.limit) || DEFAULT_CONVERSATION_SEARCH_LIMIT, + ), + ); + const scanLimit = Math.max( + DEFAULT_CONVERSATION_SEARCH_SCAN_FLOOR, + Math.min( + MAX_CONVERSATION_SEARCH_SCAN_LIMIT, + Number(options?.sessionLimitPerProvider) || + Math.max( + resultLimit * DEFAULT_CONVERSATION_SEARCH_SCAN_MULTIPLIER, + DEFAULT_CONVERSATION_SEARCH_SCAN_FLOOR, + ), + ), + ); + return { + resultLimit, + scanLimit, + }; +} + +export function buildConversationSearchProviderBudgets( + providers: ProviderId[], + totalBudget: number, +): Array<{ provider: ProviderId; limit: number }> { + const uniqueProviders = Array.from(new Set(providers)); + const safeBudget = Math.max(1, Math.floor(totalBudget)); + if (!uniqueProviders.length) return []; + if (uniqueProviders.length === 1) { + return [{ provider: uniqueProviders[0], limit: safeBudget }]; + } + + const weightedProviders = uniqueProviders.map((provider) => ({ + provider, + weight: PROVIDER_SCAN_BUDGET_WEIGHTS[provider] ?? 1, + })); + const totalWeight = weightedProviders.reduce( + (sum, entry) => sum + entry.weight, + 0, + ); + + const provisional = weightedProviders.map((entry) => { + const exact = (safeBudget * entry.weight) / totalWeight; + return { + provider: entry.provider, + exact, + limit: Math.max(1, Math.floor(exact)), + }; + }); + + let remaining = safeBudget - provisional.reduce((sum, entry) => sum + entry.limit, 0); + provisional + .sort((a, b) => (b.exact - b.limit) - (a.exact - a.limit)) + .forEach((entry) => { + if (remaining <= 0) return; + entry.limit += 1; + remaining -= 1; + }); + + const limitByProvider = new Map( + provisional.map((entry) => [entry.provider, entry.limit] as const), + ); + return uniqueProviders.map((provider) => ({ + provider, + limit: limitByProvider.get(provider) ?? 1, + })); +} diff --git a/apps/api-ts/src/domains/providers/search/local-search.ts b/apps/api-ts/src/domains/providers/search/local-search.ts new file mode 100644 index 0000000..74a73c1 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/local-search.ts @@ -0,0 +1,153 @@ +import type { + ConversationSearchPayload, + ProviderId, +} from "../types.js"; +import { + normalizeSearchText, +} from "../search-helpers.js"; +import { nowIsoUtc } from "../../../lib/utils.js"; +import { + throwIfAborted, +} from "./abort.js"; +import { + conversationSearchResponseCache, + conversationSearchResponseCacheKey, +} from "./cache.js"; +import { + CONVERSATION_SEARCH_RESPONSE_CACHE_TTL_MS, + DEFAULT_CONVERSATION_SEARCH_LIMIT, + MAX_CONVERSATION_SEARCH_LIMIT, +} from "./constants.js"; +import { + buildConversationSearchProviderBudgets, + resolveConversationSearchLimits, +} from "./limits.js"; +import { + materializeProviderSessionMetadataScan, +} from "./session-manifest.js"; +import { + resolveOpenableThreadIds, +} from "./openable-threads.js"; +import { + defaultConversationSearchProviders, + getProviderSessionManifest, + getProviderSessionScan, +} from "./provider-session-scan.js"; +import { + isMetadataOnlyConversationQuery, +} from "./session-results.js"; +import { + searchConversationSessions, +} from "./session-search.js"; + +export async function searchLocalConversationsTs( + q: string, + options?: { + providers?: ProviderId[]; + limit?: number; + pageSize?: number; + cursor?: string; + forceRefresh?: boolean; + transcriptLimit?: number; + previewHitsPerSession?: number; + sessionLimitPerProvider?: number; + signal?: AbortSignal; + }, +): Promise { + const trimmedQuery = normalizeSearchText(q); + const metadataOnlyQuery = isMetadataOnlyConversationQuery(trimmedQuery); + const providers = + options?.providers?.length ? Array.from(new Set(options.providers)) : defaultConversationSearchProviders(); + const safePageSize = Math.max( + 1, + Math.min( + MAX_CONVERSATION_SEARCH_LIMIT, + Number(options?.pageSize) || Number(options?.limit) || DEFAULT_CONVERSATION_SEARCH_LIMIT, + ), + ); + const forceRefresh = Boolean(options?.forceRefresh); + const cacheKey = conversationSearchResponseCacheKey({ + q: trimmedQuery, + providers, + pageSize: safePageSize, + cursor: options?.cursor, + transcriptLimit: options?.transcriptLimit, + previewHitsPerSession: options?.previewHitsPerSession, + sessionLimitPerProvider: options?.sessionLimitPerProvider, + }); + if (!forceRefresh) { + const cached = conversationSearchResponseCache.get(cacheKey); + if (cached && cached.expires_at > Date.now()) { + return cached.payload; + } + } + const explicitPerProviderLimit = Math.max( + 0, + Math.floor(Number(options?.sessionLimitPerProvider) || 0), + ); + const providerBudgets = + explicitPerProviderLimit > 0 + ? providers.map((provider) => ({ + provider, + limit: explicitPerProviderLimit, + })) + : buildConversationSearchProviderBudgets( + providers, + resolveConversationSearchLimits({ limit: safePageSize }).scanLimit, + ); + const scans = await Promise.all( + providerBudgets.map(async ({ provider, limit }) => { + throwIfAborted(options?.signal); + if (!metadataOnlyQuery) { + return getProviderSessionScan(provider, limit, { + forceRefresh, + signal: options?.signal, + }); + } + const manifest = await getProviderSessionManifest( + provider, + forceRefresh, + options?.signal, + ); + return materializeProviderSessionMetadataScan(manifest, limit); + }), + ); + const rows = scans.flatMap((scan) => scan.rows); + const openableThreadIds = await resolveOpenableThreadIds(forceRefresh); + const result = await searchConversationSessions(rows, trimmedQuery, { + pageSize: safePageSize, + cursor: options?.cursor, + transcriptLimit: options?.transcriptLimit, + openableThreadIds, + previewHitsPerSession: options?.previewHitsPerSession, + signal: options?.signal, + }); + + const payload = { + generated_at: nowIsoUtc(), + q: trimmedQuery, + providers, + limit: safePageSize, + page_size: safePageSize, + searched_sessions: result.searched_sessions, + available_sessions: result.available_sessions, + truncated: scans.some((scan) => scan.truncated) || result.truncated, + total_matching_sessions: result.total_matching_sessions, + total_matching_hits: result.total_matching_hits, + has_more: result.has_more, + next_cursor: result.next_cursor, + preview_hits_per_session: Math.max( + 1, + Math.min(20, Number(options?.previewHitsPerSession) || 3), + ), + sessions: result.sessions, + results: result.results, + }; + if (!forceRefresh) { + conversationSearchResponseCache.set(cacheKey, { + expires_at: Date.now() + CONVERSATION_SEARCH_RESPONSE_CACHE_TTL_MS, + payload, + }); + } + return payload; +} diff --git a/apps/api-ts/src/domains/providers/search/manifest-store.ts b/apps/api-ts/src/domains/providers/search/manifest-store.ts new file mode 100644 index 0000000..04c79fd --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/manifest-store.ts @@ -0,0 +1,113 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import type { ProviderId } from "../types.js"; +import { providerScanRootSpecs } from "../path-safety.js"; +import type { + PersistedProviderManifestCacheEntry, + ProviderSessionCandidate, +} from "./types.js"; + +export function resolveSearchCacheDirectory(): string { + const override = String(process.env.THREADLENS_SEARCH_CACHE_DIR || "").trim(); + return override || path.join(os.tmpdir(), "threadlens-search-cache"); +} + +export function providerManifestCacheFilePath(provider: ProviderId): string { + const version = createHash("sha1").update(provider).digest("hex").slice(0, 8); + return path.join(resolveSearchCacheDirectory(), `manifest-${provider}-${version}.json`); +} + +export async function readPersistedProviderManifest( + provider: ProviderId, +): Promise { + try { + const raw = await readFile(providerManifestCacheFilePath(provider), "utf8"); + const parsed = JSON.parse(raw) as PersistedProviderManifestCacheEntry; + if (!parsed || typeof parsed !== "object") return null; + if (!parsed.manifest || !Array.isArray(parsed.manifest.candidates)) return null; + if (parsed.manifest.provider !== provider) return null; + const candidates = await validatePersistedProviderCandidates( + provider, + parsed.manifest.candidates, + ); + if (parsed.manifest.candidates.length > 0 && candidates.length === 0) { + return null; + } + return { + ...parsed, + manifest: { + ...parsed.manifest, + candidates, + total_bytes: candidates.reduce( + (sum, candidate) => sum + Number(candidate.size_bytes || 0), + 0, + ), + }, + }; + } catch { + return null; + } +} + +async function validatePersistedProviderCandidates( + provider: ProviderId, + candidates: ProviderSessionCandidate[], +): Promise { + const specs = await providerScanRootSpecs(provider); + const rootRealPaths = await Promise.all( + specs.map(async (spec) => { + try { + return { + ...spec, + realRoot: await realpath(spec.root), + }; + } catch { + return null; + } + }), + ); + const usableRoots = rootRealPaths.filter( + (entry): entry is NonNullable<(typeof rootRealPaths)[number]> => Boolean(entry), + ); + if (!usableRoots.length) return []; + + const safeCandidates: ProviderSessionCandidate[] = []; + for (const candidate of candidates) { + const ext = path.extname(candidate.file_path).toLowerCase(); + let realCandidate = ""; + try { + realCandidate = await realpath(candidate.file_path); + } catch { + continue; + } + const matchingRoot = usableRoots.find( + (spec) => + spec.exts.includes(ext) && + (realCandidate === spec.realRoot || + realCandidate.startsWith(`${spec.realRoot}${path.sep}`)), + ); + if (!matchingRoot) continue; + safeCandidates.push({ + ...candidate, + file_path: realCandidate, + source: matchingRoot.source, + }); + } + return safeCandidates; +} + +export async function writePersistedProviderManifest( + provider: ProviderId, + entry: PersistedProviderManifestCacheEntry, +): Promise { + const filePath = providerManifestCacheFilePath(provider); + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile(filePath, JSON.stringify(entry), "utf8"); +} + +export async function deletePersistedProviderManifest(provider: ProviderId): Promise { + await rm(providerManifestCacheFilePath(provider), { force: true }); +} diff --git a/apps/api-ts/src/domains/providers/search/openable-threads.ts b/apps/api-ts/src/domains/providers/search/openable-threads.ts new file mode 100644 index 0000000..75a1edf --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/openable-threads.ts @@ -0,0 +1,54 @@ +import { OPENABLE_THREAD_IDS_CACHE_TTL_MS } from "./constants.js"; + +let openableThreadIdsCache: + | { + expires_at: number; + ids: Set; + } + | null = null; +let openableThreadIdsInflight: Promise> | null = null; + +export async function resolveOpenableThreadIds( + forceRefresh: boolean, +): Promise> { + const now = Date.now(); + if (!forceRefresh && openableThreadIdsCache && openableThreadIdsCache.expires_at > now) { + return new Set(openableThreadIdsCache.ids); + } + if (!forceRefresh && openableThreadIdsInflight) { + return openableThreadIdsInflight.then((ids) => new Set(ids)); + } + + const loadIds = (async () => { + const { getThreadsTs } = await import("../../threads/query.js"); + const threads = await getThreadsTs({ + offset: "0", + limit: "240", + q: "", + sort: "updated_desc", + ...(forceRefresh ? { refresh: "1" } : {}), + }); + const ids = new Set( + (threads.rows ?? []) + .map((row) => String(row.thread_id || "").trim()) + .filter(Boolean), + ); + openableThreadIdsCache = { + expires_at: Date.now() + OPENABLE_THREAD_IDS_CACHE_TTL_MS, + ids, + }; + return ids; + })(); + + if (!forceRefresh) { + openableThreadIdsInflight = loadIds; + } + + try { + return new Set(await loadIds); + } finally { + if (openableThreadIdsInflight === loadIds) { + openableThreadIdsInflight = null; + } + } +} diff --git a/apps/api-ts/src/domains/providers/search/provider-session-reports.ts b/apps/api-ts/src/domains/providers/search/provider-session-reports.ts new file mode 100644 index 0000000..e6502f9 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/provider-session-reports.ts @@ -0,0 +1,110 @@ +import type { ProviderId } from "../types.js"; +import { nowIsoUtc } from "../../../lib/utils.js"; +import { getProviderSessionScan } from "./provider-session-scan.js"; + +const PROVIDER_SESSION_REPORT_TARGETS: ProviderId[] = [ + "codex", + "chatgpt", + "claude", + "gemini", + "copilot", +]; + +export async function getProviderSessionsTs( + provider?: ProviderId, + limit = 80, + options?: { forceRefresh?: boolean }, +) { + const targets: ProviderId[] = provider + ? [provider] + : PROVIDER_SESSION_REPORT_TARGETS; + const scans = await Promise.all( + targets.map((p) => getProviderSessionScan(p, limit, options)), + ); + + const rows = scans.flatMap((scan) => scan.rows); + return { + generated_at: nowIsoUtc(), + summary: { + providers: scans.length, + rows: rows.length, + parse_ok: rows.filter((row) => row.probe.ok).length, + parse_fail: rows.filter((row) => !row.probe.ok).length, + }, + providers: scans.map((scan) => ({ + provider: scan.provider, + name: scan.name, + status: scan.status, + scanned: scan.scanned, + truncated: scan.truncated, + scan_ms: scan.scan_ms, + total_bytes: scan.total_bytes, + })), + rows, + }; +} + +export async function getProviderParserHealthTs( + provider?: ProviderId, + limitPerProvider = 80, + options?: { forceRefresh?: boolean }, +) { + const targets: ProviderId[] = provider + ? [provider] + : PROVIDER_SESSION_REPORT_TARGETS; + const scans = await Promise.all( + targets.map((item) => getProviderSessionScan(item, limitPerProvider, options)), + ); + const reports: Array> = scans.map((scan) => { + const parseOk = scan.rows.filter((row) => row.probe.ok).length; + const parseFail = scan.rows.length - parseOk; + const score = scan.rows.length + ? Number(((parseOk / scan.rows.length) * 100).toFixed(1)) + : null; + return { + provider: scan.provider, + name: scan.name, + status: scan.status, + scanned: scan.rows.length, + parse_ok: parseOk, + parse_fail: parseFail, + parse_score: score, + truncated: scan.truncated, + scan_ms: scan.scan_ms, + sample_errors: scan.rows + .filter((row) => !row.probe.ok) + .slice(0, 8) + .map((row) => ({ + session_id: row.session_id, + path: row.file_path, + format: row.probe.format, + error: row.probe.error, + })), + }; + }); + const totalScanned = reports.reduce( + (sum, row) => sum + Number((row.scanned as number) || 0), + 0, + ); + const totalFail = reports.reduce( + (sum, row) => sum + Number((row.parse_fail as number) || 0), + 0, + ); + const totalOk = reports.reduce( + (sum, row) => sum + Number((row.parse_ok as number) || 0), + 0, + ); + return { + generated_at: nowIsoUtc(), + summary: { + providers: reports.length, + scanned: totalScanned, + parse_ok: totalOk, + parse_fail: totalFail, + parse_score: totalScanned + ? Number(((totalOk / totalScanned) * 100).toFixed(1)) + : null, + }, + reports, + }; +} diff --git a/apps/api-ts/src/domains/providers/search/provider-session-scan.ts b/apps/api-ts/src/domains/providers/search/provider-session-scan.ts new file mode 100644 index 0000000..a29e6d2 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/provider-session-scan.ts @@ -0,0 +1,171 @@ +import type { + ProviderId, + ProviderSessionScan, +} from "../types.js"; +import { + invalidateCodexThreadTitleMapCache, +} from "../title-detection.js"; +import { + awaitAbortable, + throwIfAborted, +} from "./abort.js"; +import { + bumpProviderSearchGeneration, + conversationSearchResponseCache, + getProviderSearchGeneration, + invalidatedProviderManifests, + providerManifestCache, + providerManifestInflight, + providerScanCache, + providerScanCacheKey, + providerScanInflight, +} from "./cache.js"; +import { + DEFAULT_CONVERSATION_SEARCH_PROVIDERS, + MAX_CONVERSATION_SEARCH_SCAN_LIMIT, + PROVIDER_MANIFEST_CACHE_TTL_MS, + PROVIDER_SCAN_CACHE_TTL_MS, +} from "./constants.js"; +import { + deletePersistedProviderManifest, + readPersistedProviderManifest, + writePersistedProviderManifest, +} from "./manifest-store.js"; +import { resolveOpenableThreadIds } from "./openable-threads.js"; +import { + buildProviderSessionManifest, + materializeProviderSessionScan, +} from "./session-manifest.js"; +import type { ProviderSessionManifest } from "./types.js"; + +export function invalidateProviderSearchCaches(provider: ProviderId) { + bumpProviderSearchGeneration(provider); + for (const key of providerScanCache.keys()) { + if (key.startsWith(`${provider}:`)) { + providerScanCache.delete(key); + } + } + for (const key of providerScanInflight.keys()) { + if (key.startsWith(`${provider}:`)) { + providerScanInflight.delete(key); + } + } + providerManifestCache.delete(provider); + providerManifestInflight.delete(provider); + invalidatedProviderManifests.add(provider); + conversationSearchResponseCache.clear(); + void deletePersistedProviderManifest(provider); +} + +export async function getProviderSessionScan( + provider: ProviderId, + limit = 80, + options?: { forceRefresh?: boolean; signal?: AbortSignal }, +): Promise { + const safeLimit = Math.max( + 1, + Math.min(MAX_CONVERSATION_SEARCH_SCAN_LIMIT, Number(limit) || 80), + ); + const key = providerScanCacheKey(provider, safeLimit); + const forceRefresh = Boolean(options?.forceRefresh); + const now = Date.now(); + + if (!forceRefresh) { + const cached = providerScanCache.get(key); + if (cached && cached.expires_at > now) return cached.scan; + } else { + invalidateProviderSearchCaches(provider); + if (provider === "codex") { + invalidateCodexThreadTitleMapCache(); + } + } + const generation = getProviderSearchGeneration(provider); + + const inflight = providerScanInflight.get(key); + if (inflight) return awaitAbortable(inflight, options?.signal); + + const task = getProviderSessionManifest(provider, forceRefresh) + .then((manifest) => + materializeProviderSessionScan(manifest, safeLimit), + ) + .then((scan) => { + if (getProviderSearchGeneration(provider) === generation) { + providerScanCache.set(key, { + expires_at: Date.now() + PROVIDER_SCAN_CACHE_TTL_MS, + scan, + }); + } + return scan; + }) + .finally(() => { + if (providerScanInflight.get(key) === task) { + providerScanInflight.delete(key); + } + }); + + providerScanInflight.set(key, task); + return awaitAbortable(task, options?.signal); +} + +export async function getProviderSessionManifest( + provider: ProviderId, + forceRefresh = false, + signal?: AbortSignal, +): Promise { + const now = Date.now(); + const skipPersistedManifest = forceRefresh || invalidatedProviderManifests.has(provider); + const generation = getProviderSearchGeneration(provider); + throwIfAborted(signal); + if (!skipPersistedManifest) { + const cached = providerManifestCache.get(provider); + if (cached && cached.expires_at > now) return cached.manifest; + const persisted = await readPersistedProviderManifest(provider); + if (persisted && persisted.expires_at > now) { + providerManifestCache.set(provider, persisted); + return persisted.manifest; + } + } + + const inflight = providerManifestInflight.get(provider); + if (inflight) return awaitAbortable(inflight, signal); + + const task = buildProviderSessionManifest(provider) + .then((manifest) => { + if (getProviderSearchGeneration(provider) !== generation) { + return manifest; + } + const entry = { + expires_at: Date.now() + PROVIDER_MANIFEST_CACHE_TTL_MS, + manifest, + }; + providerManifestCache.set(provider, entry); + invalidatedProviderManifests.delete(provider); + return writePersistedProviderManifest(provider, entry) + .catch(() => undefined) + .then(() => manifest); + }) + .finally(() => { + if (providerManifestInflight.get(provider) === task) { + providerManifestInflight.delete(provider); + } + }); + + providerManifestInflight.set(provider, task); + return awaitAbortable(task, signal); +} + +export async function primeConversationSearchCaches( + providers: ProviderId[] = defaultConversationSearchProviders(), +): Promise { + const targets = Array.from(new Set(providers)); + await Promise.allSettled([ + ...targets.map((provider) => + getProviderSessionManifest(provider, false), + ), + resolveOpenableThreadIds(false), + ]); +} + +export function defaultConversationSearchProviders(): ProviderId[] { + return [...DEFAULT_CONVERSATION_SEARCH_PROVIDERS]; +} diff --git a/apps/api-ts/src/domains/providers/search/raw-files.ts b/apps/api-ts/src/domains/providers/search/raw-files.ts new file mode 100644 index 0000000..30ef6ea --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/raw-files.ts @@ -0,0 +1,191 @@ +import { spawn } from "node:child_process"; + +import type { ProviderSessionRow } from "../types.js"; +import { + buildSearchSnippet, + buildSearchTokens, + isExactPhraseSearchMatch, + matchesConversationSearch, + normalizeSearchQuery, + normalizeSearchText, +} from "../search-helpers.js"; +import { + createAbortError, + throwIfAborted, +} from "./abort.js"; + +const RAW_CONVERSATION_FILE_EXT_PATTERN = /\.(jsonl|json|md|txt|data)$/i; +export const RAW_CONVERSATION_FILE_MAX_MATCHES = 10_000; +const RAW_FILE_SEARCH_CHUNK_SIZE = 160; + +export type RawConversationFileMatch = { + snippets: string[]; + match_count: number; + has_more_hits: boolean; + exact_phrase_count: number; +}; + +export type RawConversationFileSearchLoader = ( + rows: ProviderSessionRow[], + q: string, + options?: { + previewHitsPerSession?: number; + maxHitsPerSession?: number; + signal?: AbortSignal; + }, +) => Promise>; + +export function isRawConversationFileSearchEligible(filePath: string): boolean { + return RAW_CONVERSATION_FILE_EXT_PATTERN.test(filePath); +} + +function buildRipgrepPattern(query: string): string | null { + const trimmedQuery = normalizeSearchText(query); + const normalizedQuery = normalizeSearchQuery(trimmedQuery); + const tokens = buildSearchTokens(trimmedQuery); + if (!normalizedQuery && !tokens.length) return null; + if (normalizedQuery && tokens.length <= 1) { + return normalizedQuery; + } + if (tokens.length > 0) { + return tokens[0]; + } + return normalizedQuery; +} + +export async function searchConversationFilesWithRipgrep( + rows: ProviderSessionRow[], + q: string, + options?: { + previewHitsPerSession?: number; + maxHitsPerSession?: number; + signal?: AbortSignal; + }, +): Promise> { + const pattern = buildRipgrepPattern(q); + if (!pattern) return new Map(); + const previewLimit = Math.max( + 1, + Math.min( + RAW_CONVERSATION_FILE_MAX_MATCHES, + Number(options?.previewHitsPerSession) || 3, + ), + ); + const maxHitsPerSession = Math.max( + previewLimit, + Math.min( + RAW_CONVERSATION_FILE_MAX_MATCHES, + Number(options?.maxHitsPerSession) || previewLimit, + ), + ); + const maxPerFile = maxHitsPerSession + 1; + const normalizedQuery = normalizeSearchQuery(q); + const tokens = buildSearchTokens(q); + const textRows = rows.filter((row) => isRawConversationFileSearchEligible(row.file_path)); + if (!textRows.length) return new Map(); + + const files = textRows.map((row) => row.file_path); + const matches = new Map(); + for (let index = 0; index < files.length; index += RAW_FILE_SEARCH_CHUNK_SIZE) { + throwIfAborted(options?.signal); + const chunk = files.slice(index, index + RAW_FILE_SEARCH_CHUNK_SIZE); + try { + const stdout = await new Promise((resolve, reject) => { + const child = spawn( + "rg", + [ + "--json", + "-F", + "-i", + "-m", + String(maxPerFile), + pattern, + "--", + ...chunk, + ], + { + stdio: ["ignore", "pipe", "pipe"], + }, + ); + let out = ""; + let err = ""; + const abortHandler = () => { + child.kill("SIGTERM"); + reject(createAbortError()); + }; + options?.signal?.addEventListener("abort", abortHandler, { once: true }); + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (data) => { + out += data; + }); + child.stderr.on("data", (data) => { + err += data; + }); + child.on("error", (error) => { + options?.signal?.removeEventListener("abort", abortHandler); + reject(error); + }); + child.on("close", (code) => { + options?.signal?.removeEventListener("abort", abortHandler); + if (code === 0 || code === 1) { + resolve(out); + return; + } + reject(new Error(err || `rg exited with code ${String(code)}`)); + }); + }); + + for (const line of stdout.split(/\r?\n/)) { + throwIfAborted(options?.signal); + if (!line.trim()) continue; + let event: unknown; + try { + event = JSON.parse(line); + } catch { + continue; + } + if (!event || typeof event !== "object") continue; + const parsed = event as { + type?: string; + data?: { + path?: { text?: string }; + lines?: { text?: string }; + }; + }; + if (parsed.type !== "match") continue; + const filePath = parsed.data?.path?.text; + const rawLine = parsed.data?.lines?.text; + if (!filePath || !rawLine) continue; + if (!matchesConversationSearch(rawLine, normalizedQuery, tokens)) continue; + const current = matches.get(filePath) ?? { + snippets: [], + match_count: 0, + has_more_hits: false, + exact_phrase_count: 0, + }; + current.match_count += 1; + if (isExactPhraseSearchMatch(rawLine, normalizedQuery)) { + current.exact_phrase_count += 1; + } + if (current.snippets.length < maxHitsPerSession) { + const snippet = buildSearchSnippet(rawLine, normalizedQuery, tokens); + if (snippet && !current.snippets.includes(snippet)) { + current.snippets.push(snippet); + } + } + if (current.match_count > maxHitsPerSession) { + current.has_more_hits = true; + } + matches.set(filePath, current); + } + } catch (error) { + if ((error as Error)?.name === "AbortError") throw error; + const failure = error as { code?: number }; + if (failure.code === 1) continue; + throw error; + } + } + + return matches; +} diff --git a/apps/api-ts/src/domains/providers/search/row-search.ts b/apps/api-ts/src/domains/providers/search/row-search.ts new file mode 100644 index 0000000..68dad0e --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/row-search.ts @@ -0,0 +1,179 @@ +import path from "node:path"; +import { buildSessionTranscript } from "../transcript.js"; +import type { + ConversationSearchResult, + ProviderId, + ProviderSessionRow, +} from "../types.js"; +import { + buildSearchSnippet, + buildSearchTokens, + matchesConversationSearch, + normalizeSearchQuery, + normalizeSearchText, +} from "../search-helpers.js"; +import { + DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, + SEARCH_TRANSCRIPT_CONCURRENCY, +} from "./constants.js"; +import { mapWithConcurrency } from "./concurrency.js"; +import { resolveConversationSearchLimits } from "./limits.js"; +import { + buildBaseSearchResult, + buildMetadataSearchResult, + buildSearchIdentity, + dedupeConversationSearchResults, + isMetadataOnlyConversationQuery, + type SearchIdentity, +} from "./session-results.js"; +import { + createCachedConversationTranscriptLoader, + isPolicyInjectionMessage, +} from "./transcript-matches.js"; +import type { ConversationTranscriptLoader } from "./types.js"; + +export async function searchConversationRows( + rows: ProviderSessionRow[], + q: string, + options?: { + limit?: number; + transcriptLoader?: ConversationTranscriptLoader; + transcriptLimit?: number; + openableThreadIds?: Set; + }, +): Promise<{ + searched_sessions: number; + available_sessions: number; + truncated: boolean; + results: ConversationSearchResult[]; +}> { + const trimmedQuery = normalizeSearchText(q); + const normalizedQuery = normalizeSearchQuery(trimmedQuery); + const tokens = buildSearchTokens(trimmedQuery); + const { resultLimit: safeLimit } = resolveConversationSearchLimits({ + limit: options?.limit, + }); + const transcriptLimit = Math.max( + 100, + Math.min( + DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, + Number(options?.transcriptLimit) || + DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, + ), + ); + const transcriptLoader = + options?.transcriptLoader ?? + ((provider: ProviderId, filePath: string) => + buildSessionTranscript(provider, filePath, transcriptLimit)); + const metadataOnlyQuery = isMetadataOnlyConversationQuery(trimmedQuery); + const cachedTranscriptLoader = + createCachedConversationTranscriptLoader(transcriptLoader); + const seenPaths = new Set(); + const sortedRows = [...rows] + .sort( + (a, b) => + Date.parse(String(b.mtime || "")) - Date.parse(String(a.mtime || "")), + ) + .filter((row) => { + const key = `${row.provider}:${path.resolve(row.file_path)}`; + if (seenPaths.has(key)) return false; + seenPaths.add(key); + return true; + }); + const metadataResults: ConversationSearchResult[] = []; + const transcriptRows: Array<{ + row: ProviderSessionRow; + identity: SearchIdentity; + }> = []; + + for (const row of sortedRows) { + const identity = buildSearchIdentity(row, options?.openableThreadIds); + const metadataResult = buildMetadataSearchResult( + row, + identity, + normalizedQuery, + tokens, + ); + if (metadataResult) { + metadataResults.push(metadataResult.result); + continue; + } + transcriptRows.push({ row, identity }); + } + + let results = dedupeConversationSearchResults(metadataResults); + const metadataSatisfiesLimit = results.length >= safeLimit; + const metadataOnlyTruncated = + results.length > safeLimit || + (metadataSatisfiesLimit && transcriptRows.length > 0); + + if (!metadataSatisfiesLimit && !metadataOnlyQuery) { + let transcriptStoppedEarly = false; + for ( + let offset = 0; + offset < transcriptRows.length && results.length < safeLimit; + offset += SEARCH_TRANSCRIPT_CONCURRENCY + ) { + const chunk = transcriptRows.slice( + offset, + offset + SEARCH_TRANSCRIPT_CONCURRENCY, + ); + const chunkResults = await mapWithConcurrency( + chunk, + SEARCH_TRANSCRIPT_CONCURRENCY, + async ({ row, identity }) => { + const rowResults: ConversationSearchResult[] = []; + const baseResult = buildBaseSearchResult(row, identity); + const transcript = await cachedTranscriptLoader(row); + if (!transcript?.messages?.length) return rowResults; + + for (let i = 0; i < transcript.messages.length; i += 1) { + const message = transcript.messages[i]; + if (message.role === "system" || message.role === "tool") continue; + if (isPolicyInjectionMessage(message.text)) continue; + if ( + !matchesConversationSearch(message.text, normalizedQuery, tokens) + ) { + continue; + } + rowResults.push({ + ...baseResult, + match_kind: "message", + snippet: buildSearchSnippet(message.text, normalizedQuery, tokens), + role: message.role, + }); + } + + return rowResults; + }, + ); + + results = dedupeConversationSearchResults([ + ...results, + ...chunkResults.flat(), + ]); + if ( + results.length >= safeLimit && + offset + chunk.length < transcriptRows.length + ) { + transcriptStoppedEarly = true; + } + + if (results.length >= safeLimit) { + return { + searched_sessions: sortedRows.length, + available_sessions: sortedRows.length, + truncated: results.length > safeLimit || transcriptStoppedEarly, + results: results.slice(0, safeLimit), + }; + } + } + } + + return { + searched_sessions: sortedRows.length, + available_sessions: sortedRows.length, + truncated: metadataOnlyTruncated || results.length > safeLimit, + results: results.slice(0, safeLimit), + }; +} diff --git a/apps/api-ts/src/domains/providers/search/session-accumulator.ts b/apps/api-ts/src/domains/providers/search/session-accumulator.ts new file mode 100644 index 0000000..e471713 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/session-accumulator.ts @@ -0,0 +1,86 @@ +import type { + ConversationSearchResult, + ConversationSearchSessionResult, + ProviderSessionRow, +} from "../types.js"; +import { + conversationSearchResultDedupKey, +} from "./session-dedupe.js"; +import type { + SearchIdentity, + SearchMatchMeta, +} from "./session-metadata-results.js"; + +export type SearchSessionAccumulator = { + session: ConversationSearchSessionResult; + exact_phrase_count: number; + seen_hits: Set; +}; + +export function createSessionAccumulator( + row: ProviderSessionRow, + identity: SearchIdentity, +): SearchSessionAccumulator { + return { + session: { + provider: row.provider, + session_id: identity.sessionId, + ...(identity.threadId ? { thread_id: identity.threadId } : {}), + title: identity.title, + ...(row.display_title ? { display_title: row.display_title } : {}), + file_path: row.file_path, + source: row.source, + mtime: row.mtime, + match_count: 0, + title_match_count: 0, + best_match_kind: "message", + preview_matches: [], + has_more_hits: false, + }, + exact_phrase_count: 0, + seen_hits: new Set(), + }; +} + +export function addSessionSearchHit( + session: SearchSessionAccumulator, + result: ConversationSearchResult, + meta: SearchMatchMeta, + previewLimit: number, +): void { + const dedupeKey = conversationSearchResultDedupKey(result); + if (session.seen_hits.has(dedupeKey)) return; + session.seen_hits.add(dedupeKey); + session.session.match_count += 1; + if (result.match_kind === "title") { + session.session.title_match_count += 1; + session.session.best_match_kind = "title"; + } + if (meta.exactPhrase) { + session.exact_phrase_count += 1; + } + if (session.session.preview_matches.length < previewLimit) { + session.session.preview_matches.push(result); + } + session.session.has_more_hits = + session.session.match_count > session.session.preview_matches.length; +} + +export function compareConversationSearchSessions( + left: SearchSessionAccumulator, + right: SearchSessionAccumulator, +): number { + const leftTitle = left.session.title_match_count > 0 ? 1 : 0; + const rightTitle = right.session.title_match_count > 0 ? 1 : 0; + if (leftTitle !== rightTitle) return rightTitle - leftTitle; + if (left.exact_phrase_count !== right.exact_phrase_count) { + return right.exact_phrase_count - left.exact_phrase_count; + } + if (left.session.match_count !== right.session.match_count) { + return right.session.match_count - left.session.match_count; + } + return ( + Date.parse(String(right.session.mtime || "")) - + Date.parse(String(left.session.mtime || "")) + ); +} diff --git a/apps/api-ts/src/domains/providers/search/session-dedupe.ts b/apps/api-ts/src/domains/providers/search/session-dedupe.ts new file mode 100644 index 0000000..c32d659 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/session-dedupe.ts @@ -0,0 +1,72 @@ +import path from "node:path"; + +import type { + ConversationSearchResult, + ProviderSessionRow, +} from "../types.js"; +import { + normalizeSearchText, +} from "../search-helpers.js"; +import { + inferSessionId, +} from "../probe.js"; +import { + providerSessionSourcePriority, +} from "./session-manifest.js"; + +function searchLogicalSessionKey(row: ProviderSessionRow): string { + const inferred = row.session_id || inferSessionId(row.file_path) || path.resolve(row.file_path); + return `${row.provider}:${inferred}`; +} + +export function dedupeConversationSearchRows( + rows: ProviderSessionRow[], +): ProviderSessionRow[] { + const uniqueRows = new Map(); + for (const row of rows) { + const key = searchLogicalSessionKey(row); + const existing = uniqueRows.get(key); + if (!existing) { + uniqueRows.set(key, row); + continue; + } + const existingPriority = providerSessionSourcePriority(existing.source); + const nextPriority = providerSessionSourcePriority(row.source); + if (nextPriority > existingPriority) { + uniqueRows.set(key, row); + continue; + } + if ( + nextPriority === existingPriority && + Date.parse(String(row.mtime || "")) > Date.parse(String(existing.mtime || "")) + ) { + uniqueRows.set(key, row); + } + } + return [...uniqueRows.values()]; +} + +export function conversationSearchResultDedupKey(result: ConversationSearchResult): string { + return [ + result.provider, + result.session_id, + result.file_path, + result.match_kind, + result.role || "", + normalizeSearchText(result.snippet).toLowerCase(), + ].join("::"); +} + +export function dedupeConversationSearchResults( + results: ConversationSearchResult[], +): ConversationSearchResult[] { + const deduped: ConversationSearchResult[] = []; + const seen = new Set(); + for (const result of results) { + const key = conversationSearchResultDedupKey(result); + if (seen.has(key)) continue; + seen.add(key); + deduped.push(result); + } + return deduped; +} diff --git a/apps/api-ts/src/domains/providers/search/session-hit-resolver.ts b/apps/api-ts/src/domains/providers/search/session-hit-resolver.ts new file mode 100644 index 0000000..9b699be --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/session-hit-resolver.ts @@ -0,0 +1,84 @@ +import path from "node:path"; + +import { + inferSessionId, +} from "../probe.js"; +import type { + ConversationSearchSessionHitsPayload, + ProviderId, + ProviderSessionRow, +} from "../types.js"; +import { + materializeProviderSessionRow, + selectProviderSessionManifestCandidate, +} from "./session-manifest.js"; +import { + resolveOpenableThreadIds, +} from "./openable-threads.js"; +import { + getProviderSessionManifest, +} from "./provider-session-scan.js"; +import { + dedupeConversationSearchRows, +} from "./session-results.js"; +import { + searchConversationSessionHits, +} from "./session-search.js"; + +export async function searchConversationSessionHitsTs( + q: string, + options: { + provider: ProviderId; + sessionId: string; + filePath?: string; + pageSize?: number; + cursor?: string; + forceRefresh?: boolean; + transcriptLimit?: number; + signal?: AbortSignal; + }, +): Promise { + const forceRefresh = Boolean(options.forceRefresh); + const manifest = await getProviderSessionManifest( + options.provider, + forceRefresh, + options.signal, + ); + const candidate = selectProviderSessionManifestCandidate(manifest, { + sessionId: options.sessionId, + filePath: options.filePath, + }); + if (!candidate) return null; + const targetRow = await materializeProviderSessionRow( + manifest, + candidate, + { signal: options.signal }, + ); + if (!targetRow) return null; + if (!targetRow.probe.ok) return null; + const openableThreadIds = await resolveOpenableThreadIds(forceRefresh); + return searchConversationSessionHits(targetRow, q, { + pageSize: options.pageSize, + cursor: options.cursor, + transcriptLimit: options.transcriptLimit, + openableThreadIds, + signal: options.signal, + }); +} + +export function selectConversationSessionHitsRow( + rows: ProviderSessionRow[], + options: { sessionId: string; filePath?: string }, +): ProviderSessionRow | null { + const normalizedFilePath = options.filePath ? path.resolve(options.filePath) : ""; + const dedupedRows = dedupeConversationSearchRows(rows); + const matchesRequestedSession = (row: ProviderSessionRow) => + row.session_id === options.sessionId || inferSessionId(row.file_path) === options.sessionId; + const filePathMatch = normalizedFilePath + ? dedupedRows.find((row) => path.resolve(row.file_path) === normalizedFilePath) + : null; + if (filePathMatch && matchesRequestedSession(filePathMatch)) { + return filePathMatch; + } + return dedupedRows.find(matchesRequestedSession) ?? null; +} diff --git a/apps/api-ts/src/domains/providers/search/session-hit-search.ts b/apps/api-ts/src/domains/providers/search/session-hit-search.ts new file mode 100644 index 0000000..e38fac5 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/session-hit-search.ts @@ -0,0 +1,119 @@ +import type { + ConversationSearchSessionHitsPayload, + ProviderSessionRow, +} from "../types.js"; +import { + normalizeSearchText, +} from "../search-helpers.js"; +import { nowIsoUtc } from "../../../lib/utils.js"; +import { + throwIfAborted, +} from "./abort.js"; +import { + DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, +} from "./constants.js"; +import { + RAW_CONVERSATION_FILE_MAX_MATCHES, + type RawConversationFileMatch, + type RawConversationFileSearchLoader, + searchConversationFilesWithRipgrep, +} from "./raw-files.js"; +import { + decodeConversationSearchCursor, + encodeConversationSearchCursor, +} from "./session-results.js"; +import { + collectConversationSessionMatches, + createCachedConversationTranscriptLoader, +} from "./transcript-matches.js"; +import type { + ConversationTranscriptLoader, +} from "./types.js"; + +export async function searchConversationSessionHits( + row: ProviderSessionRow, + q: string, + options?: { + pageSize?: number; + cursor?: string; + transcriptLoader?: ConversationTranscriptLoader; + transcriptLimit?: number; + openableThreadIds?: Set; + rawFileSearchLoader?: RawConversationFileSearchLoader; + signal?: AbortSignal; + }, +): Promise { + throwIfAborted(options?.signal); + const pageSize = Math.max(1, Math.min(200, Number(options?.pageSize) || 40)); + const offset = decodeConversationSearchCursor(options?.cursor); + const requestedHitWindow = Math.max( + pageSize, + Math.min(RAW_CONVERSATION_FILE_MAX_MATCHES, offset + pageSize + 1), + ); + const transcriptLimit = Math.max( + 100, + Math.min( + DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, + Number(options?.transcriptLimit) || DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, + ), + ); + const cachedTranscriptLoader = + options?.transcriptLoader + ? createCachedConversationTranscriptLoader(options.transcriptLoader) + : undefined; + let rawFileMatch: RawConversationFileMatch | null = null; + let rawFileSearchComplete = false; + const shouldUseRawFileSearch = + Boolean(options?.rawFileSearchLoader) || !options?.transcriptLoader; + if (shouldUseRawFileSearch) { + try { + rawFileMatch = + ( + await (options?.rawFileSearchLoader ?? searchConversationFilesWithRipgrep)([row], q, { + previewHitsPerSession: requestedHitWindow, + maxHitsPerSession: requestedHitWindow, + signal: options?.signal, + }) + ).get(row.file_path) ?? null; + rawFileSearchComplete = true; + } catch (error) { + if ((error as Error)?.name === "AbortError") throw error; + rawFileMatch = null; + rawFileSearchComplete = false; + } + } + const canSatisfyCursorFromRawFile = + !rawFileMatch || + offset < rawFileMatch.snippets.length || + !rawFileMatch.has_more_hits; + const session = await collectConversationSessionMatches(row, q, { + transcriptLoader: cachedTranscriptLoader, + transcriptLimit, + openableThreadIds: options?.openableThreadIds, + previewHitsPerSession: requestedHitWindow, + exhaustiveHits: true, + rawFileMatch: canSatisfyCursorFromRawFile ? rawFileMatch : null, + rawFileSearchComplete: canSatisfyCursorFromRawFile ? rawFileSearchComplete : false, + signal: options?.signal, + }); + + const hits = session?.session.preview_matches ?? []; + const totalHits = Math.max(hits.length, session?.session.match_count ?? 0); + const hasMore = offset + pageSize < totalHits; + const nextCursor = hasMore + ? encodeConversationSearchCursor(offset + pageSize) + : null; + + return { + generated_at: nowIsoUtc(), + q: normalizeSearchText(q), + provider: row.provider, + session_id: row.session_id, + file_path: row.file_path, + page_size: pageSize, + total_hits: totalHits, + has_more: hasMore, + next_cursor: nextCursor, + hits: hits.slice(offset, offset + pageSize), + }; +} diff --git a/apps/api-ts/src/domains/providers/search/session-manifest-files.ts b/apps/api-ts/src/domains/providers/search/session-manifest-files.ts new file mode 100644 index 0000000..be4a169 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/session-manifest-files.ts @@ -0,0 +1,146 @@ +import { stat } from "node:fs/promises"; +import path from "node:path"; + +import { + normalizeSearchText, +} from "../search-helpers.js"; +import { + isCopilotGlobalSessionLikeFile, + isWorkspaceChatSessionPath, +} from "../probe.js"; +import type { + ProviderId, + ProviderRootSpec, +} from "../types.js"; +import { + walkFilesByExt, +} from "../../../lib/utils.js"; +import { + throwIfAborted, +} from "./abort.js"; +import { + PROVIDER_SCAN_FILE_STAT_CONCURRENCY, + PROVIDER_SESSION_MANIFEST_FILE_LIMIT, +} from "./constants.js"; +import { + mapWithConcurrency, +} from "./concurrency.js"; +import type { + ProviderSessionCandidate, +} from "./types.js"; + +export function providerSessionSourcePriority(source: string): number { + const normalized = normalizeSearchText(source).toLowerCase(); + if (normalized === "sessions") return 500; + if (normalized === "projects") return 480; + if (normalized === "transcripts") return 470; + if (normalized === "history") return 460; + if (normalized === "tmp") return 450; + if (normalized.includes("workspace")) return 440; + if (normalized.includes("global")) return 430; + if (normalized.includes("cleanup_backups")) return 100; + return 300; +} + +function shouldIncludeProviderSessionFile( + provider: ProviderId, + source: string, + filePath: string, +): boolean { + const baseName = path.basename(filePath); + if (source === "cleanup_backups" && baseName === "_manifest.json") { + return false; + } + if ( + provider === "copilot" && + (source === "vscode_workspace_chats" || source === "cursor_workspace_chats") + ) { + return isWorkspaceChatSessionPath(filePath); + } + if ( + provider === "copilot" && + (source === "vscode_global" || source === "cursor_global") + ) { + return isCopilotGlobalSessionLikeFile(filePath); + } + return true; +} + +async function walkRootExists(root: string): Promise { + try { + await stat(root); + return true; + } catch { + return false; + } +} + +export async function providerSessionRootExists( + provider: ProviderId, + roots: ProviderRootSpec[], +): Promise { + return provider === "chatgpt" + ? roots.length > 0 + : (await Promise.all(roots.map((r) => walkRootExists(r.root)))).some(Boolean); +} + +export async function collectProviderSessionCandidates( + provider: ProviderId, + roots: ProviderRootSpec[], + options?: { signal?: AbortSignal }, +): Promise { + const candidates: ProviderSessionCandidate[] = []; + + const rootFiles = await Promise.all( + roots.map((spec) => { + throwIfAborted(options?.signal); + return walkFilesByExt(spec.root, spec.exts, PROVIDER_SESSION_MANIFEST_FILE_LIMIT); + }), + ); + + for (let i = 0; i < roots.length; i += 1) { + throwIfAborted(options?.signal); + const spec = roots[i]; + const files = rootFiles[i] ?? []; + const filteredFiles = files.filter((filePath) => + shouldIncludeProviderSessionFile(provider, spec.source, filePath), + ); + + const scannedCandidates = await mapWithConcurrency( + filteredFiles, + PROVIDER_SCAN_FILE_STAT_CONCURRENCY, + async (file) => { + try { + throwIfAborted(options?.signal); + const st = await stat(file); + throwIfAborted(options?.signal); + return { + source: spec.source, + file_path: file, + size_bytes: Number(st.size), + mtime: new Date(Number(st.mtimeMs)).toISOString(), + mtime_ms: Number(st.mtimeMs), + }; + } catch { + return null; + } + }, + ); + candidates.push( + ...scannedCandidates.filter( + ( + candidate, + ): candidate is { + source: string; + file_path: string; + size_bytes: number; + mtime: string; + mtime_ms: number; + } => Boolean(candidate), + ), + ); + } + + candidates.sort((a, b) => b.mtime_ms - a.mtime_ms); + return candidates; +} diff --git a/apps/api-ts/src/domains/providers/search/session-manifest-rows.ts b/apps/api-ts/src/domains/providers/search/session-manifest-rows.ts new file mode 100644 index 0000000..5e39901 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/session-manifest-rows.ts @@ -0,0 +1,144 @@ +import { + fallbackDisplayTitle, +} from "../search-helpers.js"; +import { + extractCodexThreadIdFromSessionName, + getCodexThreadTitleMap, +} from "../title-detection.js"; +import { + inferSessionId, + probeSessionFile, +} from "../probe.js"; +import { + providerStatus, +} from "../matrix.js"; +import type { + ProviderSessionRow, + ProviderSessionScan, +} from "../types.js"; +import { + throwIfAborted, +} from "./abort.js"; +import { + EMPTY_PROVIDER_SESSION_PROBE, + MAX_CONVERSATION_SEARCH_SCAN_LIMIT, +} from "./constants.js"; +import type { + ProviderSessionCandidate, + ProviderSessionManifest, +} from "./types.js"; + +async function materializeProviderSessionRowWithTitles( + manifest: ProviderSessionManifest, + candidate: ProviderSessionCandidate, + codexTitleMap?: Map | null, + options?: { signal?: AbortSignal }, +): Promise { + throwIfAborted(options?.signal); + const rawSessionId = inferSessionId(candidate.file_path); + const codexThreadId = + manifest.provider === "codex" + ? extractCodexThreadIdFromSessionName(rawSessionId) + : ""; + const sessionId = codexThreadId || rawSessionId; + const probe = await probeSessionFile(candidate.file_path); + throwIfAborted(options?.signal); + return { + provider: manifest.provider, + source: candidate.source, + session_id: sessionId, + display_title: + (codexTitleMap && codexThreadId ? codexTitleMap.get(codexThreadId) : "") || + probe.detected_title || + fallbackDisplayTitle(manifest.provider, sessionId, candidate.source), + file_path: candidate.file_path, + size_bytes: candidate.size_bytes, + mtime: candidate.mtime, + probe, + }; +} + +export async function materializeProviderSessionRow( + manifest: ProviderSessionManifest, + candidate: ProviderSessionCandidate, + options?: { signal?: AbortSignal }, +): Promise { + const codexTitleMap = + manifest.provider === "codex" ? await getCodexThreadTitleMap() : null; + return materializeProviderSessionRowWithTitles( + manifest, + candidate, + codexTitleMap, + options, + ); +} + +export async function materializeProviderSessionScan( + manifest: ProviderSessionManifest, + limit = 80, + options?: { maxLimit?: number; signal?: AbortSignal }, +): Promise { + throwIfAborted(options?.signal); + const startedAt = Date.now(); + const hardLimit = Math.max( + 1, + Number(options?.maxLimit) || MAX_CONVERSATION_SEARCH_SCAN_LIMIT, + ); + const safeLimit = Math.max(1, Math.min(hardLimit, Number(limit) || 80)); + const selected = manifest.candidates.slice(0, safeLimit); + const codexTitleMap = + manifest.provider === "codex" ? await getCodexThreadTitleMap() : null; + throwIfAborted(options?.signal); + const rows: ProviderSessionRow[] = await Promise.all( + selected.map((candidate) => + materializeProviderSessionRowWithTitles(manifest, candidate, codexTitleMap, { + signal: options?.signal, + }), + ), + ); + + return { + provider: manifest.provider, + name: manifest.name, + status: providerStatus(manifest.root_exists, manifest.candidates.length), + rows, + scanned: rows.length, + truncated: manifest.candidates.length > safeLimit, + scan_ms: Math.max(0, Date.now() - startedAt), + total_bytes: manifest.total_bytes, + }; +} + +export function materializeProviderSessionMetadataScan( + manifest: ProviderSessionManifest, + limit = 80, + options?: { maxLimit?: number }, +): ProviderSessionScan { + const startedAt = Date.now(); + const hardLimit = Math.max( + 1, + Number(options?.maxLimit) || MAX_CONVERSATION_SEARCH_SCAN_LIMIT, + ); + const safeLimit = Math.max(1, Math.min(hardLimit, Number(limit) || 80)); + const rows: ProviderSessionRow[] = manifest.candidates.slice(0, safeLimit).map((candidate) => ({ + provider: manifest.provider, + source: candidate.source, + session_id: inferSessionId(candidate.file_path), + display_title: "", + file_path: candidate.file_path, + size_bytes: candidate.size_bytes, + mtime: candidate.mtime, + probe: EMPTY_PROVIDER_SESSION_PROBE, + })); + + return { + provider: manifest.provider, + name: manifest.name, + status: providerStatus(manifest.root_exists, manifest.candidates.length), + rows, + scanned: rows.length, + truncated: manifest.candidates.length > safeLimit, + scan_ms: Math.max(0, Date.now() - startedAt), + total_bytes: manifest.total_bytes, + }; +} diff --git a/apps/api-ts/src/domains/providers/search/session-manifest.ts b/apps/api-ts/src/domains/providers/search/session-manifest.ts new file mode 100644 index 0000000..2b41507 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/session-manifest.ts @@ -0,0 +1,102 @@ +import path from "node:path"; + +import { + providerName, + providerScanRootSpecs, +} from "../path-safety.js"; +import { + inferSessionId, +} from "../probe.js"; +import type { + ProviderId, +} from "../types.js"; +import { + throwIfAborted, +} from "./abort.js"; +import { + collectProviderSessionCandidates, + providerSessionRootExists, + providerSessionSourcePriority, +} from "./session-manifest-files.js"; +import type { + ProviderSessionCandidate, + ProviderSessionManifest, +} from "./types.js"; + +export { + providerSessionSourcePriority, +} from "./session-manifest-files.js"; +export { + materializeProviderSessionMetadataScan, + materializeProviderSessionRow, + materializeProviderSessionScan, +} from "./session-manifest-rows.js"; + +export async function buildProviderSessionManifest( + provider: ProviderId, + options?: { signal?: AbortSignal }, +): Promise { + throwIfAborted(options?.signal); + const roots = await providerScanRootSpecs(provider); + throwIfAborted(options?.signal); + const rootExists = await providerSessionRootExists(provider, roots); + const candidates = await collectProviderSessionCandidates(provider, roots, { + signal: options?.signal, + }); + return { + provider, + name: providerName(provider), + root_exists: rootExists, + candidates, + total_bytes: candidates.reduce( + (sum, candidate) => sum + Number(candidate.size_bytes || 0), + 0, + ), + }; +} + +function dedupeProviderSessionCandidates( + provider: ProviderId, + candidates: ProviderSessionCandidate[], +): ProviderSessionCandidate[] { + const uniqueCandidates = new Map(); + for (const candidate of candidates) { + const key = `${provider}:${inferSessionId(candidate.file_path) || path.resolve(candidate.file_path)}`; + const existing = uniqueCandidates.get(key); + if (!existing) { + uniqueCandidates.set(key, candidate); + continue; + } + const existingPriority = providerSessionSourcePriority(existing.source); + const nextPriority = providerSessionSourcePriority(candidate.source); + if ( + nextPriority > existingPriority || + (nextPriority === existingPriority && candidate.mtime_ms > existing.mtime_ms) + ) { + uniqueCandidates.set(key, candidate); + } + } + return [...uniqueCandidates.values()]; +} + +export function selectProviderSessionManifestCandidate( + manifest: ProviderSessionManifest, + options: { sessionId: string; filePath?: string }, +): ProviderSessionCandidate | null { + const normalizedFilePath = options.filePath ? path.resolve(options.filePath) : ""; + const dedupedCandidates = dedupeProviderSessionCandidates( + manifest.provider, + manifest.candidates, + ); + const matchesRequestedSession = (candidate: ProviderSessionCandidate) => + inferSessionId(candidate.file_path) === options.sessionId; + const filePathMatch = normalizedFilePath + ? dedupedCandidates.find( + (candidate) => path.resolve(candidate.file_path) === normalizedFilePath, + ) + : null; + if (filePathMatch && matchesRequestedSession(filePathMatch)) { + return filePathMatch; + } + return dedupedCandidates.find(matchesRequestedSession) ?? null; +} diff --git a/apps/api-ts/src/domains/providers/search/session-metadata-results.ts b/apps/api-ts/src/domains/providers/search/session-metadata-results.ts new file mode 100644 index 0000000..8266454 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/session-metadata-results.ts @@ -0,0 +1,111 @@ +import path from "node:path"; + +import type { + ConversationSearchResult, + ProviderSessionRow, +} from "../types.js"; +import { + buildSearchSnippet, + fallbackDisplayTitle, + isExactPhraseSearchMatch, + matchesConversationSearch, + normalizeSearchText, +} from "../search-helpers.js"; +import { + providerName, +} from "../path-safety.js"; +import { + extractCodexThreadIdFromSessionName, +} from "../title-detection.js"; +import { + inferSessionId, +} from "../probe.js"; + +export type SearchIdentity = { + sessionId: string; + threadId: string | null; + title: string; +}; + +export type SearchMatchMeta = { + exactPhrase: boolean; +}; + +export function buildSearchIdentity( + row: ProviderSessionRow, + openableThreadIds?: Set, +): SearchIdentity { + const rawSessionId = inferSessionId(row.file_path) || row.session_id; + const threadIdCandidate = + row.provider === "codex" + ? extractCodexThreadIdFromSessionName(rawSessionId) || + extractCodexThreadIdFromSessionName(row.session_id) || + row.session_id || + "" + : ""; + const threadId = + threadIdCandidate && openableThreadIds + ? openableThreadIds.has(threadIdCandidate) + ? threadIdCandidate + : "" + : threadIdCandidate; + + return { + sessionId: rawSessionId, + threadId: threadId || null, + title: + row.display_title || + row.probe.detected_title || + fallbackDisplayTitle(row.provider, row.session_id, row.source), + }; +} + +export function buildBaseSearchResult( + row: ProviderSessionRow, + identity: SearchIdentity, +) { + return { + provider: row.provider, + session_id: identity.sessionId, + title: identity.title, + file_path: row.file_path, + mtime: row.mtime, + ...(identity.threadId ? { thread_id: identity.threadId } : {}), + }; +} + +export function buildMetadataSearchResult( + row: ProviderSessionRow, + identity: SearchIdentity, + normalizedQuery: string, + tokens: string[], +): { result: ConversationSearchResult; meta: SearchMatchMeta } | null { + const metadataCandidates = [ + identity.title, + identity.sessionId, + row.source, + path.basename(row.file_path), + providerName(row.provider), + row.file_path, + ] + .map((value) => normalizeSearchText(value)) + .filter(Boolean); + + for (const candidate of metadataCandidates) { + if (!matchesConversationSearch(candidate, normalizedQuery, tokens)) continue; + return { + result: { + ...buildBaseSearchResult(row, identity), + ...(row.display_title ? { display_title: row.display_title } : {}), + source: row.source, + match_kind: "title", + snippet: buildSearchSnippet(candidate, normalizedQuery, tokens, 140), + }, + meta: { + exactPhrase: isExactPhraseSearchMatch(candidate, normalizedQuery), + }, + }; + } + + return null; +} diff --git a/apps/api-ts/src/domains/providers/search/session-raw-search.ts b/apps/api-ts/src/domains/providers/search/session-raw-search.ts new file mode 100644 index 0000000..480dd2e --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/session-raw-search.ts @@ -0,0 +1,53 @@ +import type { ProviderSessionRow } from "../types.js"; +import { + type RawConversationFileMatch, + type RawConversationFileSearchLoader, + searchConversationFilesWithRipgrep, +} from "./raw-files.js"; +import type { ConversationTranscriptLoader } from "./types.js"; + +export type ConversationRawFileSearchState = { + rawFileMatches: Map; + rawFileSearchComplete: boolean; +}; + +export async function loadConversationRawFileMatches( + rows: ProviderSessionRow[], + q: string, + options: { + metadataOnlyQuery: boolean; + transcriptLoader?: ConversationTranscriptLoader; + previewHitsPerSession?: number; + rawFileSearchLoader?: RawConversationFileSearchLoader; + signal?: AbortSignal; + }, +): Promise { + const shouldUseRawFileSearch = + !options.metadataOnlyQuery && + (Boolean(options.rawFileSearchLoader) || !options.transcriptLoader); + if (!shouldUseRawFileSearch) { + return { + rawFileMatches: new Map(), + rawFileSearchComplete: false, + }; + } + + try { + const rawFileMatches = await ( + options.rawFileSearchLoader ?? searchConversationFilesWithRipgrep + )(rows, q, { + previewHitsPerSession: options.previewHitsPerSession, + signal: options.signal, + }); + return { + rawFileMatches, + rawFileSearchComplete: true, + }; + } catch (error) { + if ((error as Error)?.name === "AbortError") throw error; + return { + rawFileMatches: new Map(), + rawFileSearchComplete: false, + }; + } +} diff --git a/apps/api-ts/src/domains/providers/search/session-results.ts b/apps/api-ts/src/domains/providers/search/session-results.ts new file mode 100644 index 0000000..b0b85d7 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/session-results.ts @@ -0,0 +1,50 @@ +import { + normalizeSearchQuery, + normalizeSearchText, +} from "../search-helpers.js"; + +export type { + SearchIdentity, + SearchMatchMeta, +} from "./session-metadata-results.js"; +export type { + SearchSessionAccumulator, +} from "./session-accumulator.js"; + +export { + dedupeConversationSearchResults, + dedupeConversationSearchRows, +} from "./session-dedupe.js"; +export { + buildBaseSearchResult, + buildMetadataSearchResult, + buildSearchIdentity, +} from "./session-metadata-results.js"; +export { + addSessionSearchHit, + compareConversationSearchSessions, + createSessionAccumulator, +} from "./session-accumulator.js"; + +export function decodeConversationSearchCursor(cursor?: string): number { + const offset = Math.max(0, Math.floor(Number(cursor) || 0)); + return Number.isFinite(offset) ? offset : 0; +} + +export function encodeConversationSearchCursor(offset: number): string { + return String(Math.max(0, Math.floor(offset))); +} + +export function isMetadataOnlyConversationQuery(query: string): boolean { + const normalized = normalizeSearchText(query); + if (!normalized) return false; + if (/[\\/]/.test(normalized)) return true; + if (/\.(jsonl|json|data|md|txt)\b/i.test(normalized)) return true; + if (/^rollout-\d{4}-\d{2}-\d{2}/i.test(normalized)) return true; + if ( + /^[0-9a-f]{8}-[0-9a-f]{4,}(?:-[0-9a-f]{4,}){2,}$/i.test(normalized) + ) { + return true; + } + return false; +} diff --git a/apps/api-ts/src/domains/providers/search/session-row-order.ts b/apps/api-ts/src/domains/providers/search/session-row-order.ts new file mode 100644 index 0000000..779ee4a --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/session-row-order.ts @@ -0,0 +1,30 @@ +import path from "node:path"; +import type { + ProviderSessionRow, +} from "../types.js"; +import { + dedupeConversationSearchRows, +} from "./session-results.js"; + +function compareProviderSessionRowsByMtime( + a: ProviderSessionRow, + b: ProviderSessionRow, +): number { + return Date.parse(String(b.mtime || "")) - Date.parse(String(a.mtime || "")); +} + +export function prepareConversationSearchRows( + rows: ProviderSessionRow[], +): ProviderSessionRow[] { + const seenPaths = new Set(); + return dedupeConversationSearchRows( + [...rows] + .sort(compareProviderSessionRowsByMtime) + .filter((row) => { + const key = `${row.provider}:${path.resolve(row.file_path)}`; + if (seenPaths.has(key)) return false; + seenPaths.add(key); + return true; + }), + ).sort(compareProviderSessionRowsByMtime); +} diff --git a/apps/api-ts/src/domains/providers/search/session-search.ts b/apps/api-ts/src/domains/providers/search/session-search.ts new file mode 100644 index 0000000..2067f70 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/session-search.ts @@ -0,0 +1,149 @@ +import type { + ConversationSearchResult, + ConversationSearchSessionResult, + ProviderSessionRow, +} from "../types.js"; +import { + throwIfAborted, +} from "./abort.js"; +import { + mapWithConcurrency, +} from "./concurrency.js"; +import { + CONVERSATION_SEARCH_SCAN_CONCURRENCY, + DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, +} from "./constants.js"; +import { + type RawConversationFileSearchLoader, +} from "./raw-files.js"; +import { loadConversationRawFileMatches } from "./session-raw-search.js"; +import { + compareConversationSearchSessions, + decodeConversationSearchCursor, + encodeConversationSearchCursor, + isMetadataOnlyConversationQuery, + type SearchSessionAccumulator, +} from "./session-results.js"; +import { + prepareConversationSearchRows, +} from "./session-row-order.js"; +import { + collectConversationSessionMatches, + createCachedConversationTranscriptLoader, +} from "./transcript-matches.js"; +import type { + ConversationTranscriptLoader, +} from "./types.js"; + +export { + searchConversationSessionHits, +} from "./session-hit-search.js"; + +export async function searchConversationSessions( + rows: ProviderSessionRow[], + q: string, + options?: { + pageSize?: number; + cursor?: string; + transcriptLoader?: ConversationTranscriptLoader; + transcriptLimit?: number; + openableThreadIds?: Set; + previewHitsPerSession?: number; + rawFileSearchLoader?: RawConversationFileSearchLoader; + signal?: AbortSignal; + }, +): Promise<{ + searched_sessions: number; + available_sessions: number; + truncated: boolean; + total_matching_sessions: number | null; + total_matching_hits: number | null; + has_more: boolean; + next_cursor: string | null; + sessions: ConversationSearchSessionResult[]; + results: ConversationSearchResult[]; +}> { + const pageSize = Math.max(1, Math.min(200, Number(options?.pageSize) || 40)); + const offset = decodeConversationSearchCursor(options?.cursor); + const metadataOnlyQuery = isMetadataOnlyConversationQuery(q); + const transcriptLimit = Math.max( + 100, + Math.min( + DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, + Number(options?.transcriptLimit) || DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, + ), + ); + const cachedTranscriptLoader = + options?.transcriptLoader + ? createCachedConversationTranscriptLoader(options.transcriptLoader) + : undefined; + const sortedRows = prepareConversationSearchRows(rows); + + const sessionMatches: SearchSessionAccumulator[] = []; + let scannedRows = 0; + const { rawFileMatches, rawFileSearchComplete } = + await loadConversationRawFileMatches(sortedRows, q, { + metadataOnlyQuery, + transcriptLoader: options?.transcriptLoader, + previewHitsPerSession: options?.previewHitsPerSession, + rawFileSearchLoader: options?.rawFileSearchLoader, + signal: options?.signal, + }); + + for ( + let index = 0; + index < sortedRows.length; + index += CONVERSATION_SEARCH_SCAN_CONCURRENCY + ) { + throwIfAborted(options?.signal); + const batchRows = sortedRows.slice( + index, + index + CONVERSATION_SEARCH_SCAN_CONCURRENCY, + ); + const batchMatches = await mapWithConcurrency( + batchRows, + CONVERSATION_SEARCH_SCAN_CONCURRENCY, + async (row) => + collectConversationSessionMatches(row, q, { + transcriptLoader: cachedTranscriptLoader, + transcriptLimit, + openableThreadIds: options?.openableThreadIds, + previewHitsPerSession: options?.previewHitsPerSession, + rawFileMatch: rawFileMatches.get(row.file_path) ?? null, + rawFileSearchComplete, + signal: options?.signal, + }), + ); + + for (let batchOffset = 0; batchOffset < batchMatches.length; batchOffset += 1) { + scannedRows = index + batchOffset + 1; + const session = batchMatches[batchOffset]; + if (!session) continue; + sessionMatches.push(session); + } + } + + const orderedMatches = sessionMatches.sort(compareConversationSearchSessions); + const totalMatchingSessions = orderedMatches.length; + const hasMore = offset + pageSize < totalMatchingSessions; + const pageSessions = orderedMatches + .slice(offset, offset + pageSize) + .sort(compareConversationSearchSessions) + .map((item) => item.session); + const nextCursor = hasMore + ? encodeConversationSearchCursor(offset + pageSize) + : null; + const totalMatchingHits = null; + + return { + searched_sessions: scannedRows, + available_sessions: sortedRows.length, + truncated: scannedRows < sortedRows.length, + total_matching_sessions: totalMatchingSessions, + total_matching_hits: totalMatchingHits, + has_more: hasMore, + next_cursor: nextCursor, + sessions: pageSessions, + results: pageSessions.flatMap((session) => session.preview_matches), + }; +} diff --git a/apps/api-ts/src/domains/providers/search/transcript-loader-cache.ts b/apps/api-ts/src/domains/providers/search/transcript-loader-cache.ts new file mode 100644 index 0000000..be7dea0 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/transcript-loader-cache.ts @@ -0,0 +1,33 @@ +import { + transcriptSearchCache, + transcriptSearchCacheKey, + trimTranscriptSearchCache, +} from "./cache.js"; +import type { + CachedConversationTranscriptLoader, + ConversationTranscriptLoader, +} from "./types.js"; + +export function createCachedConversationTranscriptLoader( + baseLoader: ConversationTranscriptLoader, +): CachedConversationTranscriptLoader { + return async (row) => { + const key = transcriptSearchCacheKey(row); + const cached = transcriptSearchCache.get(key); + if (cached && cached.mtime === row.mtime) { + transcriptSearchCache.delete(key); + transcriptSearchCache.set(key, cached); + return cached.transcript; + } + + const transcript = await baseLoader(row.provider, row.file_path).catch(() => null); + if (transcript) { + transcriptSearchCache.set(key, { + mtime: row.mtime, + transcript, + }); + trimTranscriptSearchCache(); + } + return transcript; + }; +} diff --git a/apps/api-ts/src/domains/providers/search/transcript-matches.ts b/apps/api-ts/src/domains/providers/search/transcript-matches.ts new file mode 100644 index 0000000..671ddf9 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/transcript-matches.ts @@ -0,0 +1,151 @@ +import { buildSessionTranscript } from "../transcript.js"; +import type { + ProviderId, + ProviderSessionRow, +} from "../types.js"; +import { + buildSearchTokens, + normalizeSearchQuery, + normalizeSearchText, +} from "../search-helpers.js"; +import { + throwIfAborted, +} from "./abort.js"; +import { + DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, +} from "./constants.js"; +import { + isRawConversationFileSearchEligible, + RAW_CONVERSATION_FILE_MAX_MATCHES, + type RawConversationFileMatch, +} from "./raw-files.js"; +import { + addSessionSearchHit, + buildMetadataSearchResult, + buildSearchIdentity, + createSessionAccumulator, + isMetadataOnlyConversationQuery, + type SearchSessionAccumulator, +} from "./session-results.js"; +import { + createCachedConversationTranscriptLoader, +} from "./transcript-loader-cache.js"; +import { applyTranscriptMessageMatches } from "./transcript-message-matches.js"; +import { + applyRawConversationFileMatch, +} from "./transcript-raw-matches.js"; +import type { + CachedConversationTranscriptLoader, +} from "./types.js"; + +export { createCachedConversationTranscriptLoader } from "./transcript-loader-cache.js"; +export { isPolicyInjectionMessage } from "./transcript-policy.js"; + +function isStructuredTranscriptFilePath(filePath: string): boolean { + return /\.(jsonl|json)$/i.test(filePath); +} + +export async function collectConversationSessionMatches( + row: ProviderSessionRow, + q: string, + options?: { + transcriptLoader?: CachedConversationTranscriptLoader; + transcriptLimit?: number; + openableThreadIds?: Set; + previewHitsPerSession?: number; + exhaustiveHits?: boolean; + rawFileMatch?: RawConversationFileMatch | null; + rawFileSearchComplete?: boolean; + signal?: AbortSignal; + }, +): Promise { + throwIfAborted(options?.signal); + const trimmedQuery = normalizeSearchText(q); + const normalizedQuery = normalizeSearchQuery(trimmedQuery); + const tokens = buildSearchTokens(trimmedQuery); + const transcriptLimit = Math.max( + 100, + Math.min( + DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, + Number(options?.transcriptLimit) || DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, + ), + ); + const metadataOnlyQuery = isMetadataOnlyConversationQuery(trimmedQuery); + const transcriptLoader = + options?.transcriptLoader ?? + createCachedConversationTranscriptLoader((provider: ProviderId, filePath: string) => + buildSessionTranscript(provider, filePath, transcriptLimit), + ); + const identity = buildSearchIdentity(row, options?.openableThreadIds); + const exhaustiveHits = Boolean(options?.exhaustiveHits); + const previewHitsPerSession = Math.max( + 1, + Math.min( + exhaustiveHits ? RAW_CONVERSATION_FILE_MAX_MATCHES : 20, + Number(options?.previewHitsPerSession) || 3, + ), + ); + const accumulator = createSessionAccumulator(row, identity); + const metadataResult = buildMetadataSearchResult( + row, + identity, + normalizedQuery, + tokens, + ); + + if (metadataResult) { + addSessionSearchHit( + accumulator, + metadataResult.result, + metadataResult.meta, + previewHitsPerSession, + ); + } + + if (options?.rawFileMatch) { + throwIfAborted(options?.signal); + const transcript = await transcriptLoader(row); + if (transcript?.messages?.length) { + applyTranscriptMessageMatches(accumulator, row, identity, transcript.messages, { + normalizedQuery, + tokens, + previewHitsPerSession, + exhaustiveHits, + signal: options?.signal, + }); + } else if (!isStructuredTranscriptFilePath(row.file_path)) { + applyRawConversationFileMatch( + accumulator, + row, + identity, + options.rawFileMatch, + normalizedQuery, + previewHitsPerSession, + ); + } + return accumulator.session.match_count > 0 ? accumulator : null; + } + + if (options?.rawFileSearchComplete && isRawConversationFileSearchEligible(row.file_path)) { + return accumulator.session.match_count > 0 ? accumulator : null; + } + + if (!metadataOnlyQuery) { + throwIfAborted(options?.signal); + const transcript = await transcriptLoader(row); + if (transcript?.messages?.length) { + applyTranscriptMessageMatches(accumulator, row, identity, transcript.messages, { + normalizedQuery, + tokens, + previewHitsPerSession, + exhaustiveHits, + signal: options?.signal, + }); + } + } + + if (accumulator.session.match_count === 0) return null; + accumulator.session.has_more_hits = + accumulator.session.match_count > accumulator.session.preview_matches.length; + return accumulator; +} diff --git a/apps/api-ts/src/domains/providers/search/transcript-message-matches.ts b/apps/api-ts/src/domains/providers/search/transcript-message-matches.ts new file mode 100644 index 0000000..61814c7 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/transcript-message-matches.ts @@ -0,0 +1,66 @@ +import type { + ProviderSessionRow, + TranscriptMessage, +} from "../types.js"; +import { + buildSearchSnippet, + isExactPhraseSearchMatch, + matchesConversationSearch, +} from "../search-helpers.js"; +import { + throwIfAborted, +} from "./abort.js"; +import { + addSessionSearchHit, + buildBaseSearchResult, + type SearchIdentity, + type SearchSessionAccumulator, +} from "./session-results.js"; +import { + isPolicyInjectionMessage, +} from "./transcript-policy.js"; + +export function applyTranscriptMessageMatches( + accumulator: SearchSessionAccumulator, + row: ProviderSessionRow, + identity: SearchIdentity, + messages: readonly TranscriptMessage[], + options: { + normalizedQuery: string; + tokens: string[]; + previewHitsPerSession: number; + exhaustiveHits: boolean; + signal?: AbortSignal; + }, +): void { + for (let i = 0; i < messages.length; i += 1) { + throwIfAborted(options.signal); + const message = messages[i]; + if (message.role === "system" || message.role === "tool") continue; + if (isPolicyInjectionMessage(message.text)) continue; + if (!matchesConversationSearch(message.text, options.normalizedQuery, options.tokens)) { + continue; + } + addSessionSearchHit( + accumulator, + { + ...buildBaseSearchResult(row, identity), + ...(row.display_title ? { display_title: row.display_title } : {}), + source: row.source, + match_kind: "message", + snippet: buildSearchSnippet(message.text, options.normalizedQuery, options.tokens), + role: message.role, + }, + { + exactPhrase: isExactPhraseSearchMatch(message.text, options.normalizedQuery), + }, + options.previewHitsPerSession, + ); + if ( + !options.exhaustiveHits && + accumulator.session.match_count >= options.previewHitsPerSession + 1 + ) { + break; + } + } +} diff --git a/apps/api-ts/src/domains/providers/search/transcript-policy.ts b/apps/api-ts/src/domains/providers/search/transcript-policy.ts new file mode 100644 index 0000000..c562f3a --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/transcript-policy.ts @@ -0,0 +1,10 @@ +export function isPolicyInjectionMessage(text: string): boolean { + const t = text.trimStart(); + return ( + t.startsWith("# AGENTS.md instructions for") || + t.startsWith("") || + t.startsWith("") || + /^You are (Codex|Assist|Claude|Gemini|GPT|ChatGPT|Copilot)\b/i.test(t) || + (t.startsWith("# ") && t.includes("")) + ); +} diff --git a/apps/api-ts/src/domains/providers/search/transcript-raw-matches.ts b/apps/api-ts/src/domains/providers/search/transcript-raw-matches.ts new file mode 100644 index 0000000..45552d5 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/transcript-raw-matches.ts @@ -0,0 +1,52 @@ +import type { + ProviderSessionRow, +} from "../types.js"; +import { + isExactPhraseSearchMatch, +} from "../search-helpers.js"; +import type { + RawConversationFileMatch, +} from "./raw-files.js"; +import { + addSessionSearchHit, + buildBaseSearchResult, + type SearchIdentity, + type SearchSessionAccumulator, +} from "./session-results.js"; + +export function applyRawConversationFileMatch( + accumulator: SearchSessionAccumulator, + row: ProviderSessionRow, + identity: SearchIdentity, + rawFileMatch: RawConversationFileMatch, + normalizedQuery: string, + previewHitsPerSession: number, +): void { + for (const snippet of rawFileMatch.snippets) { + addSessionSearchHit( + accumulator, + { + ...buildBaseSearchResult(row, identity), + ...(row.display_title ? { display_title: row.display_title } : {}), + source: row.source, + match_kind: "message", + snippet, + }, + { + exactPhrase: isExactPhraseSearchMatch(snippet, normalizedQuery), + }, + previewHitsPerSession, + ); + } + accumulator.session.match_count = Math.max( + accumulator.session.match_count, + rawFileMatch.match_count, + ); + accumulator.session.has_more_hits = + rawFileMatch.has_more_hits || + accumulator.session.match_count > accumulator.session.preview_matches.length; + accumulator.exact_phrase_count = Math.max( + accumulator.exact_phrase_count, + rawFileMatch.exact_phrase_count, + ); +} diff --git a/apps/api-ts/src/domains/providers/search/types.ts b/apps/api-ts/src/domains/providers/search/types.ts new file mode 100644 index 0000000..161749f --- /dev/null +++ b/apps/api-ts/src/domains/providers/search/types.ts @@ -0,0 +1,57 @@ +import type { + ConversationSearchPayload, + ProviderId, + ProviderSessionRow, + ProviderSessionScan, + TranscriptPayload, +} from "../types.js"; + +export type ProviderScanCacheEntry = { + expires_at: number; + scan: ProviderSessionScan; +}; + +export type ProviderSessionCandidate = { + source: string; + file_path: string; + size_bytes: number; + mtime: string; + mtime_ms: number; +}; + +export type ProviderSessionManifest = { + provider: ProviderId; + name: string; + root_exists: boolean; + candidates: ProviderSessionCandidate[]; + total_bytes: number; +}; + +export type ProviderManifestCacheEntry = { + expires_at: number; + manifest: ProviderSessionManifest; +}; + +export type PersistedProviderManifestCacheEntry = { + expires_at: number; + manifest: ProviderSessionManifest; +}; + +export type ConversationSearchResponseCacheEntry = { + expires_at: number; + payload: ConversationSearchPayload; +}; + +export type ConversationTranscriptLoader = ( + provider: ProviderId, + filePath: string, +) => Promise; + +export type CachedConversationTranscriptLoader = ( + row: ProviderSessionRow, +) => Promise; + +export type TranscriptSearchCacheEntry = { + mtime: string; + transcript: TranscriptPayload | null; +}; diff --git a/docs/ADDING_PROVIDER.md b/docs/ADDING_PROVIDER.md index 6b1cfbd..ec3a839 100644 --- a/docs/ADDING_PROVIDER.md +++ b/docs/ADDING_PROVIDER.md @@ -83,10 +83,15 @@ export type ProviderSessionLocator = Most providers can use the existing JSON/JSONL transcript flow. In the current code, custom storage shapes still touch these areas: +- `apps/api-ts/src/domains/providers/search/constants.ts` - `apps/api-ts/src/domains/providers/search.ts` - `apps/api-ts/src/domains/providers/transcript.ts` - `apps/api-ts/src/domains/providers/probe.ts` +`search/constants.ts` contains the default searchable providers and optional +scan budget weights. New providers use a default weight of `1` when omitted, so +only add a custom weight when the provider needs a different scan budget. + Add fixtures for the real local file shape. Do not infer a provider format from docs alone; use sample session files. From c808b51361410fb04923de111e88afa880414c8a Mon Sep 17 00:00:00 2001 From: hanityx Date: Mon, 18 May 2026 02:23:43 +0900 Subject: [PATCH 3/4] refactor(api): split provider runtime modules --- .../src/app/routes/platform-recovery.ts | 269 +++++ apps/api-ts/src/app/routes/platform.ts | 259 +---- apps/api-ts/src/app/routes/providers.ts | 23 +- .../src/domains/providers/actions.test.ts | 277 ++++- apps/api-ts/src/domains/providers/actions.ts | 750 +++--------- .../src/domains/providers/actions/archives.ts | 113 ++ .../src/domains/providers/actions/backups.ts | 254 +++++ .../providers/actions/confirm-tokens.ts | 81 ++ .../domains/providers/actions/execution.ts | 134 +++ .../domains/providers/actions/responses.ts | 159 +++ .../domains/providers/actions/selection.ts | 63 + .../src/domains/providers/actions/targets.ts | 55 + .../src/domains/providers/adapter-health.ts | 28 + .../src/domains/providers/adapters.test.ts | 7 + apps/api-ts/src/domains/providers/adapters.ts | 29 +- .../providers/claude-title-detection.ts | 41 + .../src/domains/providers/codex-title-map.ts | 80 ++ .../src/domains/providers/matrix.test.ts | 51 +- apps/api-ts/src/domains/providers/matrix.ts | 275 +---- .../providers/matrix/adapter-access.ts | 35 + .../src/domains/providers/matrix/copilot.ts | 45 + .../providers/matrix/provider-entries.ts | 148 +++ .../src/domains/providers/matrix/signals.ts | 103 ++ .../src/domains/providers/matrix/summary.ts | 41 + .../src/domains/providers/path-safety.ts | 291 +---- .../src/domains/providers/probe.test.ts | 3 + apps/api-ts/src/domains/providers/probe.ts | 5 +- .../src/domains/providers/provider-roots.ts | 112 ++ .../src/domains/providers/roots/backups.ts | 78 ++ .../src/domains/providers/roots/chatgpt.ts | 55 + .../src/domains/providers/roots/codex.ts | 76 ++ .../session-transcript-service.test.ts | 95 ++ .../providers/session-transcript-service.ts | 53 + .../src/domains/providers/title-detection.ts | 132 +-- .../domains/providers/title-normalization.ts | 7 + .../src/domains/providers/transcript.test.ts | 7 + .../src/domains/providers/transcript.ts | 349 +----- .../src/domains/providers/transcripts/json.ts | 141 +++ .../domains/providers/transcripts/jsonl.ts | 100 ++ .../providers/transcripts/normalizers.ts | 125 ++ apps/api-ts/src/domains/providers/types.ts | 7 + apps/api-ts/src/lib/providers.ts | 23 +- apps/api-ts/src/lib/recovery-backups.ts | 131 +++ apps/api-ts/src/lib/recovery-checklist.ts | 64 ++ apps/api-ts/src/lib/recovery-export.ts | 186 +++ apps/api-ts/src/lib/recovery-portable-zip.ts | 159 +++ apps/api-ts/src/lib/recovery-smoke.ts | 477 ++++++++ apps/api-ts/src/lib/recovery.ts | 1010 +---------------- .../shared-contracts/src/api-client.test.ts | 23 + packages/shared-contracts/src/api-client.ts | 20 +- 50 files changed, 4126 insertions(+), 2923 deletions(-) create mode 100644 apps/api-ts/src/app/routes/platform-recovery.ts create mode 100644 apps/api-ts/src/domains/providers/actions/archives.ts create mode 100644 apps/api-ts/src/domains/providers/actions/backups.ts create mode 100644 apps/api-ts/src/domains/providers/actions/confirm-tokens.ts create mode 100644 apps/api-ts/src/domains/providers/actions/execution.ts create mode 100644 apps/api-ts/src/domains/providers/actions/responses.ts create mode 100644 apps/api-ts/src/domains/providers/actions/selection.ts create mode 100644 apps/api-ts/src/domains/providers/actions/targets.ts create mode 100644 apps/api-ts/src/domains/providers/adapter-health.ts create mode 100644 apps/api-ts/src/domains/providers/claude-title-detection.ts create mode 100644 apps/api-ts/src/domains/providers/codex-title-map.ts create mode 100644 apps/api-ts/src/domains/providers/matrix/adapter-access.ts create mode 100644 apps/api-ts/src/domains/providers/matrix/copilot.ts create mode 100644 apps/api-ts/src/domains/providers/matrix/provider-entries.ts create mode 100644 apps/api-ts/src/domains/providers/matrix/signals.ts create mode 100644 apps/api-ts/src/domains/providers/matrix/summary.ts create mode 100644 apps/api-ts/src/domains/providers/provider-roots.ts create mode 100644 apps/api-ts/src/domains/providers/roots/backups.ts create mode 100644 apps/api-ts/src/domains/providers/roots/chatgpt.ts create mode 100644 apps/api-ts/src/domains/providers/roots/codex.ts create mode 100644 apps/api-ts/src/domains/providers/session-transcript-service.test.ts create mode 100644 apps/api-ts/src/domains/providers/session-transcript-service.ts create mode 100644 apps/api-ts/src/domains/providers/title-normalization.ts create mode 100644 apps/api-ts/src/domains/providers/transcripts/json.ts create mode 100644 apps/api-ts/src/domains/providers/transcripts/jsonl.ts create mode 100644 apps/api-ts/src/domains/providers/transcripts/normalizers.ts create mode 100644 apps/api-ts/src/lib/recovery-backups.ts create mode 100644 apps/api-ts/src/lib/recovery-checklist.ts create mode 100644 apps/api-ts/src/lib/recovery-export.ts create mode 100644 apps/api-ts/src/lib/recovery-portable-zip.ts create mode 100644 apps/api-ts/src/lib/recovery-smoke.ts diff --git a/apps/api-ts/src/app/routes/platform-recovery.ts b/apps/api-ts/src/app/routes/platform-recovery.ts new file mode 100644 index 0000000..641a309 --- /dev/null +++ b/apps/api-ts/src/app/routes/platform-recovery.ts @@ -0,0 +1,269 @@ +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import path from "node:path"; +import type { FastifyInstance } from "fastify"; +import { z } from "zod"; +import { + exportRecoveryBackupsTs, + getRecoveryCenterDataTs, + openRecoveryBackupArchiveReadStream, + runRecoveryDrillTs, + updateRecoveryChecklistItem, +} from "../../lib/recovery.js"; +import { + envelope, + pathExists, + type QueryMap, + withSchemaVersion, +} from "../../lib/utils.js"; + +const RECOVERY_BACKUP_DOWNLOAD_TOKEN_TTL_MS = 10 * 60 * 1000; + +type RecoveryBackupDownloadTokenRecord = { + archivePath: string; + exportRoot?: string; + expiresAt: number; +}; + +const recoveryBackupDownloadTokens = new Map(); + +function pruneExpiredRecoveryBackupDownloadTokens(now = Date.now()) { + for (const [token, record] of recoveryBackupDownloadTokens.entries()) { + if (record.expiresAt <= now) { + recoveryBackupDownloadTokens.delete(token); + } + } +} + +function issueRecoveryBackupDownloadToken(archivePath: string, exportRoot?: string) { + pruneExpiredRecoveryBackupDownloadTokens(); + const token = randomUUID(); + recoveryBackupDownloadTokens.set(token, { + archivePath, + exportRoot, + expiresAt: Date.now() + RECOVERY_BACKUP_DOWNLOAD_TOKEN_TTL_MS, + }); + return token; +} + +export function issueRecoveryBackupDownloadTokenForTests( + archivePath: string, + exportRoot?: string, +) { + return issueRecoveryBackupDownloadToken(archivePath, exportRoot); +} + +function consumeRecoveryBackupDownloadToken(token: string) { + pruneExpiredRecoveryBackupDownloadTokens(); + const record = recoveryBackupDownloadTokens.get(token); + if (!record) return null; + recoveryBackupDownloadTokens.delete(token); + return record; +} + +const recoveryChecklistSchema = z.object({ + item_id: z.string().min(1), + done: z.boolean(), +}); + +const recoveryBackupExportSchema = z.object({ + backup_ids: z.array(z.string().min(1)).max(200).optional().default([]), + backup_root: z.string().optional().default(""), + export_root: z.string().optional().default(""), +}); + +const recoveryBackupDownloadSchema = z.object({ + archive_path: z.string().min(1), + export_root: z.string().optional().default(""), +}); + +const recoveryBackupDownloadTokenSchema = z.object({ + token: z.string().uuid(), +}); + +const recoveryOpenFolderSchema = z.object({ + folder_path: z.string().min(1), + root_path: z.string().optional().default(""), +}); + +async function openDirectoryInOs(directoryPath: string): Promise { + const [command, args] = + process.platform === "darwin" + ? ["open", [directoryPath]] + : process.platform === "win32" + ? ["explorer", [directoryPath]] + : ["xdg-open", [directoryPath]]; + + await new Promise((resolve, reject) => { + execFile(command, args, (error) => { + if (error) { + reject(error); + return; + } + resolve(); + }); + }); +} + +function resolveScopedFolderPath(folderPath: string, rootPath?: string) { + const resolvedFolder = path.resolve(folderPath); + const resolvedRoot = String(rootPath || "").trim() + ? path.resolve(String(rootPath || "").trim()) + : ""; + if (!resolvedRoot) return resolvedFolder; + const relative = path.relative(resolvedRoot, resolvedFolder); + if (relative.startsWith("..") || path.isAbsolute(relative)) return null; + return resolvedFolder; +} + +export async function registerRecoveryRoutes(app: FastifyInstance): Promise { + app.get<{ Querystring: QueryMap }>("/api/recovery-center", async (req, reply) => { + try { + const backupRootRaw = Array.isArray(req.query.backup_root) + ? req.query.backup_root[0] + : req.query.backup_root; + return reply.code(200).send( + withSchemaVersion( + await getRecoveryCenterDataTs({ + backupRoot: String(backupRootRaw || "").trim() || undefined, + }), + ), + ); + } catch (error) { + return reply.code(500).send(envelope(null, `recovery-center-error: ${String(error)}`)); + } + }); + + app.post<{ Body: unknown }>("/api/recovery-checklist", async (req, reply) => { + const parsed = recoveryChecklistSchema.safeParse(req.body); + if (!parsed.success) return reply.code(400).send(envelope(null, parsed.error.message)); + try { + const result = await updateRecoveryChecklistItem(parsed.data.item_id, parsed.data.done); + if (!result.ok) return reply.code(400).send(withSchemaVersion(result)); + const data = await getRecoveryCenterDataTs(); + return reply.code(200).send(withSchemaVersion({ ok: true, data })); + } catch (error) { + return reply.code(500).send(envelope(null, `recovery-checklist-error: ${String(error)}`)); + } + }); + + app.post("/api/recovery-drill", async (_req, reply) => { + try { + const drill = await runRecoveryDrillTs(); + const center = await getRecoveryCenterDataTs(); + return reply.code(drill.ok ? 200 : 400).send( + withSchemaVersion({ + ok: Boolean(drill.ok), + data: { ...center, drill: drill.drill }, + drill: drill.drill, + error: drill.error ?? "", + }), + ); + } catch (error) { + return reply.code(500).send(envelope(null, `recovery-drill-error: ${String(error)}`)); + } + }); + + app.post<{ Body: unknown }>("/api/recovery-backup-export", async (req, reply) => { + const parsed = recoveryBackupExportSchema.safeParse(req.body ?? {}); + if (!parsed.success) return reply.code(400).send(envelope(null, parsed.error.message)); + try { + const result = await exportRecoveryBackupsTs({ + backup_ids: parsed.data.backup_ids, + roots: { + backup_root: String(parsed.data.backup_root || "").trim() || undefined, + export_root: String(parsed.data.export_root || "").trim() || undefined, + }, + }); + if (result.ok && result.archive_path) { + const archivePath = String(result.archive_path || "").trim(); + const exportRoot = String(result.export_root || "").trim() || undefined; + return reply.code(200).send( + withSchemaVersion({ + ...result, + download_token: issueRecoveryBackupDownloadToken(archivePath, exportRoot), + }), + ); + } + return reply.code(result.ok ? 200 : 400).send(withSchemaVersion(result)); + } catch (error) { + return reply.code(500).send(envelope(null, `recovery-backup-export-error: ${String(error)}`)); + } + }); + + app.post<{ Body: unknown }>("/api/recovery-open-folder", async (req, reply) => { + const parsed = recoveryOpenFolderSchema.safeParse(req.body ?? {}); + if (!parsed.success) return reply.code(400).send(envelope(null, parsed.error.message)); + try { + const safeFolderPath = resolveScopedFolderPath( + parsed.data.folder_path, + String(parsed.data.root_path || "").trim() || undefined, + ); + if (!safeFolderPath) { + return reply.code(400).send(envelope(null, "recovery-open-folder-outside-root")); + } + const exists = await pathExists(safeFolderPath); + if (!exists) { + return reply.code(404).send(envelope(null, "recovery-open-folder-not-found")); + } + await openDirectoryInOs(safeFolderPath); + return reply.code(200).send( + withSchemaVersion({ + ok: true, + folder_path: safeFolderPath, + }), + ); + } catch (error) { + return reply.code(500).send(envelope(null, `recovery-open-folder-error: ${String(error)}`)); + } + }); + + app.get<{ Querystring: QueryMap }>("/api/recovery-backup-export/download", async (req, reply) => { + const tokenRaw = Array.isArray(req.query.token) ? req.query.token[0] : req.query.token; + const parsedToken = recoveryBackupDownloadTokenSchema.safeParse({ + token: tokenRaw ?? "", + }); + if (parsedToken.success) { + try { + const tokenRecord = consumeRecoveryBackupDownloadToken(parsedToken.data.token); + if (!tokenRecord) { + return reply.code(400).send(envelope(null, "recovery-backup-export-download-not-found")); + } + const opened = await openRecoveryBackupArchiveReadStream( + tokenRecord.archivePath, + tokenRecord.exportRoot, + ); + if (!opened) { + return reply.code(400).send(envelope(null, "recovery-backup-export-download-not-found")); + } + reply.header("content-type", "application/zip"); + reply.header("content-disposition", `attachment; filename=\"${path.basename(opened.archivePath)}\"`); + return reply.send(opened.stream); + } catch (error) { + return reply.code(500).send(envelope(null, `recovery-backup-export-download-error: ${String(error)}`)); + } + } + + const archivePathRaw = Array.isArray(req.query.archive_path) ? req.query.archive_path[0] : req.query.archive_path; + const exportRootRaw = Array.isArray(req.query.export_root) ? req.query.export_root[0] : req.query.export_root; + const parsed = recoveryBackupDownloadSchema.safeParse({ + archive_path: archivePathRaw ?? "", + export_root: exportRootRaw ?? "", + }); + if (!parsed.success) return reply.code(400).send(envelope(null, parsed.error.message)); + try { + const opened = await openRecoveryBackupArchiveReadStream( + parsed.data.archive_path, + String(parsed.data.export_root || "").trim() || undefined, + ); + if (!opened) { + return reply.code(400).send(envelope(null, "recovery-backup-export-download-not-found")); + } + reply.header("content-type", "application/zip"); + reply.header("content-disposition", `attachment; filename=\"${path.basename(opened.archivePath)}\"`); + return reply.send(opened.stream); + } catch (error) { + return reply.code(500).send(envelope(null, `recovery-backup-export-download-error: ${String(error)}`)); + } + }); +} diff --git a/apps/api-ts/src/app/routes/platform.ts b/apps/api-ts/src/app/routes/platform.ts index 8479969..a19a552 100644 --- a/apps/api-ts/src/app/routes/platform.ts +++ b/apps/api-ts/src/app/routes/platform.ts @@ -1,6 +1,3 @@ -import { execFile } from "node:child_process"; -import path from "node:path"; -import { randomUUID } from "node:crypto"; import type { FastifyInstance, FastifyRequest } from "fastify"; import { z } from "zod"; import { @@ -22,17 +19,11 @@ import { withSchemaVersion, } from "../../lib/utils.js"; import { - exportRecoveryBackupsTs, getCompareAppsStatusTs, getLatestSmokeStatusTs, - openRecoveryBackupArchiveReadStream, - getRecoveryCenterDataTs, getRelatedToolsStatusTs, getRuntimeHealthTs, - runRecoveryDrillTs, - updateRecoveryChecklistItem, } from "../../lib/recovery.js"; -import { pathExists } from "../../lib/utils.js"; import { getDataSourceInventoryTs } from "../../domains/recovery/inventory.js"; import { appendRoadmapCheckinTs, @@ -49,6 +40,9 @@ import { getAgentLoopsStatusTs, runAgentLoopActionTs, } from "../../domains/ops/agent-loops.js"; +import { registerRecoveryRoutes } from "./platform-recovery.js"; + +export { issueRecoveryBackupDownloadTokenForTests } from "./platform-recovery.js"; export type ProxyRequest = FastifyRequest<{ Params: { "*": string }; @@ -56,50 +50,6 @@ export type ProxyRequest = FastifyRequest<{ Body: unknown; }>; -const RECOVERY_BACKUP_DOWNLOAD_TOKEN_TTL_MS = 10 * 60 * 1000; - -type RecoveryBackupDownloadTokenRecord = { - archivePath: string; - exportRoot?: string; - expiresAt: number; -}; - -const recoveryBackupDownloadTokens = new Map(); - -function pruneExpiredRecoveryBackupDownloadTokens(now = Date.now()) { - for (const [token, record] of recoveryBackupDownloadTokens.entries()) { - if (record.expiresAt <= now) { - recoveryBackupDownloadTokens.delete(token); - } - } -} - -function issueRecoveryBackupDownloadToken(archivePath: string, exportRoot?: string) { - pruneExpiredRecoveryBackupDownloadTokens(); - const token = randomUUID(); - recoveryBackupDownloadTokens.set(token, { - archivePath, - exportRoot, - expiresAt: Date.now() + RECOVERY_BACKUP_DOWNLOAD_TOKEN_TTL_MS, - }); - return token; -} - -export function issueRecoveryBackupDownloadTokenForTests( - archivePath: string, - exportRoot?: string, -) { - return issueRecoveryBackupDownloadToken(archivePath, exportRoot); -} - -function consumeRecoveryBackupDownloadToken(token: string) { - pruneExpiredRecoveryBackupDownloadTokens(); - const record = recoveryBackupDownloadTokens.get(token); - if (!record) return null; - recoveryBackupDownloadTokens.delete(token); - return record; -} - export async function registerPlatformRoutes( app: FastifyInstance, deps: { @@ -107,59 +57,6 @@ export async function registerPlatformRoutes( getCachedDataSources: (forceRefresh: boolean) => Promise; }, ): Promise { - const recoveryChecklistSchema = z.object({ - item_id: z.string().min(1), - done: z.boolean(), - }); - - const recoveryBackupExportSchema = z.object({ - backup_ids: z.array(z.string().min(1)).max(200).optional().default([]), - backup_root: z.string().optional().default(""), - export_root: z.string().optional().default(""), - }); - - const recoveryBackupDownloadSchema = z.object({ - archive_path: z.string().min(1), - export_root: z.string().optional().default(""), - }); - -const recoveryBackupDownloadTokenSchema = z.object({ - token: z.string().uuid(), -}); - -const recoveryOpenFolderSchema = z.object({ - folder_path: z.string().min(1), - root_path: z.string().optional().default(""), -}); - -async function openDirectoryInOs(directoryPath: string): Promise { - const [command, args] = - process.platform === "darwin" - ? ["open", [directoryPath]] - : process.platform === "win32" - ? ["explorer", [directoryPath]] - : ["xdg-open", [directoryPath]]; - - await new Promise((resolve, reject) => { - execFile(command, args, (error) => { - if (error) { - reject(error); - return; - } - resolve(); - }); - }); -} - -function resolveScopedFolderPath(folderPath: string, rootPath?: string) { - const resolvedFolder = path.resolve(folderPath); - const resolvedRoot = String(rootPath || "").trim() ? path.resolve(String(rootPath || "").trim()) : ""; - if (!resolvedRoot) return resolvedFolder; - const relative = path.relative(resolvedRoot, resolvedFolder); - if (relative.startsWith("..") || path.isAbsolute(relative)) return null; - return resolvedFolder; -} - const alertConfigSchema = z.object({ desktop_notify: z.boolean(), }); @@ -237,155 +134,7 @@ function resolveScopedFolderPath(folderPath: string, rootPath?: string) { }, ); - app.get<{ Querystring: QueryMap }>("/api/recovery-center", async (req, reply) => { - try { - const backupRootRaw = Array.isArray(req.query.backup_root) - ? req.query.backup_root[0] - : req.query.backup_root; - return reply.code(200).send( - withSchemaVersion( - await getRecoveryCenterDataTs({ - backupRoot: String(backupRootRaw || "").trim() || undefined, - }), - ), - ); - } catch (error) { - return reply.code(500).send(envelope(null, `recovery-center-error: ${String(error)}`)); - } - }); - - app.post<{ Body: unknown }>("/api/recovery-checklist", async (req, reply) => { - const parsed = recoveryChecklistSchema.safeParse(req.body); - if (!parsed.success) return reply.code(400).send(envelope(null, parsed.error.message)); - try { - const result = await updateRecoveryChecklistItem(parsed.data.item_id, parsed.data.done); - if (!result.ok) return reply.code(400).send(withSchemaVersion(result)); - const data = await getRecoveryCenterDataTs(); - return reply.code(200).send(withSchemaVersion({ ok: true, data })); - } catch (error) { - return reply.code(500).send(envelope(null, `recovery-checklist-error: ${String(error)}`)); - } - }); - - app.post("/api/recovery-drill", async (_req, reply) => { - try { - const drill = await runRecoveryDrillTs(); - const center = await getRecoveryCenterDataTs(); - return reply.code(drill.ok ? 200 : 400).send( - withSchemaVersion({ - ok: Boolean(drill.ok), - data: { ...center, drill: drill.drill }, - drill: drill.drill, - error: drill.error ?? "", - }), - ); - } catch (error) { - return reply.code(500).send(envelope(null, `recovery-drill-error: ${String(error)}`)); - } - }); - - app.post<{ Body: unknown }>("/api/recovery-backup-export", async (req, reply) => { - const parsed = recoveryBackupExportSchema.safeParse(req.body ?? {}); - if (!parsed.success) return reply.code(400).send(envelope(null, parsed.error.message)); - try { - const result = await exportRecoveryBackupsTs({ - backup_ids: parsed.data.backup_ids, - roots: { - backup_root: String(parsed.data.backup_root || "").trim() || undefined, - export_root: String(parsed.data.export_root || "").trim() || undefined, - }, - }); - if (result.ok && result.archive_path) { - const archivePath = String(result.archive_path || "").trim(); - const exportRoot = String(result.export_root || "").trim() || undefined; - return reply.code(200).send( - withSchemaVersion({ - ...result, - download_token: issueRecoveryBackupDownloadToken(archivePath, exportRoot), - }), - ); - } - return reply.code(result.ok ? 200 : 400).send(withSchemaVersion(result)); - } catch (error) { - return reply.code(500).send(envelope(null, `recovery-backup-export-error: ${String(error)}`)); - } - }); - - app.post<{ Body: unknown }>("/api/recovery-open-folder", async (req, reply) => { - const parsed = recoveryOpenFolderSchema.safeParse(req.body ?? {}); - if (!parsed.success) return reply.code(400).send(envelope(null, parsed.error.message)); - try { - const safeFolderPath = resolveScopedFolderPath( - parsed.data.folder_path, - String(parsed.data.root_path || "").trim() || undefined, - ); - if (!safeFolderPath) { - return reply.code(400).send(envelope(null, "recovery-open-folder-outside-root")); - } - const exists = await pathExists(safeFolderPath); - if (!exists) { - return reply.code(404).send(envelope(null, "recovery-open-folder-not-found")); - } - await openDirectoryInOs(safeFolderPath); - return reply.code(200).send( - withSchemaVersion({ - ok: true, - folder_path: safeFolderPath, - }), - ); - } catch (error) { - return reply.code(500).send(envelope(null, `recovery-open-folder-error: ${String(error)}`)); - } - }); - - app.get<{ Querystring: QueryMap }>("/api/recovery-backup-export/download", async (req, reply) => { - const tokenRaw = Array.isArray(req.query.token) ? req.query.token[0] : req.query.token; - const parsedToken = recoveryBackupDownloadTokenSchema.safeParse({ - token: tokenRaw ?? "", - }); - if (parsedToken.success) { - try { - const tokenRecord = consumeRecoveryBackupDownloadToken(parsedToken.data.token); - if (!tokenRecord) { - return reply.code(400).send(envelope(null, "recovery-backup-export-download-not-found")); - } - const opened = await openRecoveryBackupArchiveReadStream( - tokenRecord.archivePath, - tokenRecord.exportRoot, - ); - if (!opened) { - return reply.code(400).send(envelope(null, "recovery-backup-export-download-not-found")); - } - reply.header("content-type", "application/zip"); - reply.header("content-disposition", `attachment; filename=\"${path.basename(opened.archivePath)}\"`); - return reply.send(opened.stream); - } catch (error) { - return reply.code(500).send(envelope(null, `recovery-backup-export-download-error: ${String(error)}`)); - } - } - - const archivePathRaw = Array.isArray(req.query.archive_path) ? req.query.archive_path[0] : req.query.archive_path; - const exportRootRaw = Array.isArray(req.query.export_root) ? req.query.export_root[0] : req.query.export_root; - const parsed = recoveryBackupDownloadSchema.safeParse({ - archive_path: archivePathRaw ?? "", - export_root: exportRootRaw ?? "", - }); - if (!parsed.success) return reply.code(400).send(envelope(null, parsed.error.message)); - try { - const opened = await openRecoveryBackupArchiveReadStream( - parsed.data.archive_path, - String(parsed.data.export_root || "").trim() || undefined, - ); - if (!opened) { - return reply.code(400).send(envelope(null, "recovery-backup-export-download-not-found")); - } - reply.header("content-type", "application/zip"); - reply.header("content-disposition", `attachment; filename=\"${path.basename(opened.archivePath)}\"`); - return reply.send(opened.stream); - } catch (error) { - return reply.code(500).send(envelope(null, `recovery-backup-export-download-error: ${String(error)}`)); - } - }); + await registerRecoveryRoutes(app); app.get("/api/related-tools", async (_req, reply) => { try { diff --git a/apps/api-ts/src/app/routes/providers.ts b/apps/api-ts/src/app/routes/providers.ts index 84127c7..84e83fa 100644 --- a/apps/api-ts/src/app/routes/providers.ts +++ b/apps/api-ts/src/app/routes/providers.ts @@ -9,7 +9,7 @@ import { resolveAllowedProviderFilePath, } from "../../domains/providers/path-safety.js"; import { getProviderMatrixTs } from "../../domains/providers/matrix.js"; -import { buildSessionTranscript } from "../../domains/providers/transcript.js"; +import { getProviderSessionTranscript } from "../../domains/providers/session-transcript-service.js"; import { runProviderSessionAction } from "../../lib/providers.js"; import { getProviderParserHealthTs, @@ -372,22 +372,17 @@ export async function registerProviderRoutes( if (!filePath) { return reply.code(400).send(envelope(null, "file_path required")); } - const safeFilePath = await resolveAllowedProviderFilePath(provider, filePath); - if (!safeFilePath) { - return reply - .code(400) - .send(envelope(null, "file_path outside provider roots")); - } - const exists = await pathExists(safeFilePath); - if (!exists) { - return reply.code(404).send(envelope(null, "session file not found")); - } - const data = await buildSessionTranscript( + const result = await getProviderSessionTranscript( provider, - safeFilePath, + filePath, Number(limitRaw) || 300, ); - return reply.code(200).send(withSchemaVersion(data)); + if (!result.ok) { + return reply + .code(result.statusCode) + .send(envelope(null, result.message)); + } + return reply.code(200).send(withSchemaVersion(result.data)); } catch (error) { return reply .code(500) diff --git a/apps/api-ts/src/domains/providers/actions.test.ts b/apps/api-ts/src/domains/providers/actions.test.ts index fdca570..351ea66 100644 --- a/apps/api-ts/src/domains/providers/actions.test.ts +++ b/apps/api-ts/src/domains/providers/actions.test.ts @@ -1,6 +1,6 @@ import os from "node:os"; import path from "node:path"; -import { mkdir, mkdtemp, readFile, realpath, rm, stat, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from "node:fs/promises"; import { describe, expect, it, vi } from "vitest"; import { CLAUDE_PROJECTS_DIR, @@ -11,6 +11,7 @@ import { HOME_DIR, } from "../../lib/constants.js"; import { deriveProviderBackupRelativePath, runProviderSessionAction } from "./actions.js"; +import type { ProviderId, ProviderSessionAction } from "./types.js"; describe("deriveProviderBackupRelativePath", () => { it("stores Claude project backups under the provider source instead of the filesystem root", () => { @@ -55,6 +56,50 @@ describe("deriveProviderBackupRelativePath", () => { }); describe("runProviderSessionAction", () => { + it("uses action-aware provider support before resolving targets", async () => { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-provider-action-")); + const filePath = path.join(tempDir, "session.jsonl"); + try { + await writeFile(filePath, "{}", "utf8"); + const resolveAllowedProviderFilePath = vi.fn(async () => filePath); + const deps = { + resolveAllowedProviderFilePath, + supportsProviderAction: ( + _provider: ProviderId, + action: ProviderSessionAction, + ) => action === "archive_local", + invalidateProviderCaches: () => undefined, + }; + + const archivePreview = await runProviderSessionAction( + deps, + "codex", + "archive_local", + [filePath], + true, + "", + ); + expect(archivePreview.ok).toBe(true); + expect(archivePreview.confirm_token_expected).toMatch(/^PROVIDER-/); + expect(resolveAllowedProviderFilePath).toHaveBeenCalledTimes(1); + + const deletePreview = await runProviderSessionAction( + deps, + "codex", + "delete_local", + [filePath], + true, + "", + ); + expect(deletePreview.ok).toBe(false); + expect(deletePreview.error).toBe("cleanup-disabled-provider"); + expect(deletePreview.confirm_token_expected).toBe(""); + expect(resolveAllowedProviderFilePath).toHaveBeenCalledTimes(1); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }); + it("rejects custom backup roots outside the user home during preview", async () => { const tempDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-provider-action-")); const filePath = path.join(tempDir, "session.jsonl"); @@ -64,7 +109,7 @@ describe("runProviderSessionAction", () => { const result = await runProviderSessionAction( { resolveAllowedProviderFilePath: async () => filePath, - supportsProviderCleanup: () => true, + supportsProviderAction: () => true, invalidateProviderCaches: () => undefined, }, "codex", @@ -93,7 +138,7 @@ describe("runProviderSessionAction", () => { const result = await runProviderSessionAction( { resolveAllowedProviderFilePath: async () => filePath, - supportsProviderCleanup: () => true, + supportsProviderAction: () => true, invalidateProviderCaches: () => undefined, }, "codex", @@ -112,6 +157,136 @@ describe("runProviderSessionAction", () => { } }); + it("rejects custom backup roots that resolve outside the user home", async () => { + const previousHome = process.env.HOME; + const previousCodexHome = process.env.CODEX_HOME; + const previousStateDir = process.env.THREADLENS_STATE_DIR; + const root = await mkdtemp(path.join(os.tmpdir(), "threadlens-provider-backup-root-")); + + try { + const homeDir = path.join(root, "home"); + const outsideDir = path.join(root, "outside"); + process.env.HOME = homeDir; + process.env.CODEX_HOME = path.join(homeDir, ".codex"); + process.env.THREADLENS_STATE_DIR = path.join(root, "state"); + vi.resetModules(); + + const constants = await import("../../lib/constants.js"); + const { runProviderSessionAction: runAction } = await import("./actions.js"); + const sourceDir = path.join(constants.CODEX_HOME, "sessions", "backup-root-smoke"); + const sourcePath = path.join(sourceDir, "session.jsonl"); + const backupRootLink = path.join(constants.HOME_DIR, "ThreadLens Backups Link"); + + await mkdir(sourceDir, { recursive: true }); + await mkdir(outsideDir, { recursive: true }); + await writeFile(sourcePath, "{\"type\":\"session\"}\n", "utf8"); + await symlink(outsideDir, backupRootLink, "dir"); + + const result = await runAction( + { + resolveAllowedProviderFilePath: async () => sourcePath, + supportsProviderAction: () => true, + invalidateProviderCaches: () => undefined, + }, + "codex", + "backup_local", + [sourcePath], + true, + "", + { backup_root: backupRootLink }, + ); + + expect(result.ok).toBe(false); + expect(result.error).toBe("backup_root_outside_home"); + expect(result.confirm_token_expected).toBe(""); + } finally { + if (previousHome === undefined) delete process.env.HOME; + else process.env.HOME = previousHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; + if (previousStateDir === undefined) delete process.env.THREADLENS_STATE_DIR; + else process.env.THREADLENS_STATE_DIR = previousStateDir; + vi.resetModules(); + await rm(root, { recursive: true, force: true }); + } + }); + + it("does not delete provider files when backup destination resolves outside the user home", async () => { + const previousHome = process.env.HOME; + const previousCodexHome = process.env.CODEX_HOME; + const previousStateDir = process.env.THREADLENS_STATE_DIR; + const root = await mkdtemp(path.join(os.tmpdir(), "threadlens-provider-backup-dest-")); + + try { + const homeDir = path.join(root, "home"); + const outsideDir = path.join(root, "outside"); + process.env.HOME = homeDir; + process.env.CODEX_HOME = path.join(homeDir, ".codex"); + process.env.THREADLENS_STATE_DIR = path.join(root, "state"); + vi.resetModules(); + + const constants = await import("../../lib/constants.js"); + const { runProviderSessionAction: runAction } = await import("./actions.js"); + const sourceDir = path.join(constants.CODEX_HOME, "sessions", "backup-dest-smoke"); + const sourcePath = path.join(sourceDir, "session.jsonl"); + const backupRoot = path.join(constants.HOME_DIR, "ThreadLens Test Backups"); + const providerActionsLink = path.join(backupRoot, "provider_actions"); + const payload = "{\"type\":\"session\",\"id\":\"backup-dest-smoke\"}\n"; + const deps = { + resolveAllowedProviderFilePath: async () => sourcePath, + supportsProviderAction: () => true, + invalidateProviderCaches: () => undefined, + }; + + await mkdir(sourceDir, { recursive: true }); + await mkdir(backupRoot, { recursive: true }); + await mkdir(outsideDir, { recursive: true }); + await writeFile(sourcePath, payload, "utf8"); + await symlink(outsideDir, providerActionsLink, "dir"); + + const preview = await runAction( + deps, + "codex", + "delete_local", + [sourcePath], + true, + "", + { backup_before_delete: true, backup_root: backupRoot }, + ); + expect(preview.ok).toBe(true); + expect(preview.confirm_token_expected).toMatch(/^PROVIDER-/); + + const result = await runAction( + deps, + "codex", + "delete_local", + [sourcePath], + false, + preview.confirm_token_expected, + { backup_before_delete: true, backup_root: backupRoot }, + ); + + expect(result.ok).toBe(false); + expect(result.mode).toBe("failed"); + expect(result.applied_count).toBe(0); + expect(result.failed).toEqual( + expect.arrayContaining([ + expect.objectContaining({ step: "backup_root", error: "backup_root_outside_home" }), + ]), + ); + await expect(readFile(sourcePath, "utf8")).resolves.toBe(payload); + } finally { + if (previousHome === undefined) delete process.env.HOME; + else process.env.HOME = previousHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; + if (previousStateDir === undefined) delete process.env.THREADLENS_STATE_DIR; + else process.env.THREADLENS_STATE_DIR = previousStateDir; + vi.resetModules(); + await rm(root, { recursive: true, force: true }); + } + }); + it("rejects custom backup roots outside the user home", async () => { const tempDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-provider-action-")); const filePath = path.join(tempDir, "session.jsonl"); @@ -121,7 +296,7 @@ describe("runProviderSessionAction", () => { const result = await runProviderSessionAction( { resolveAllowedProviderFilePath: async () => filePath, - supportsProviderCleanup: () => true, + supportsProviderAction: () => true, invalidateProviderCaches: () => undefined, }, "codex", @@ -139,6 +314,92 @@ describe("runProviderSessionAction", () => { } }); + it("does not delete provider files when backup manifest writing fails", async () => { + const previousHome = process.env.HOME; + const previousCodexHome = process.env.CODEX_HOME; + const previousStateDir = process.env.THREADLENS_STATE_DIR; + const root = await mkdtemp(path.join(os.tmpdir(), "threadlens-provider-manifest-")); + const fixedNow = "2026-05-02T00:00:00.000Z"; + const folderName = "2026-05-02T00-00-00-000Z-delete_local"; + + try { + const homeDir = path.join(root, "home"); + process.env.HOME = homeDir; + process.env.CODEX_HOME = path.join(homeDir, ".codex"); + process.env.THREADLENS_STATE_DIR = path.join(root, "state"); + vi.resetModules(); + vi.doMock("../../lib/utils.js", async (importOriginal) => ({ + ...(await importOriginal()), + nowIsoUtc: () => fixedNow, + })); + + const constants = await import("../../lib/constants.js"); + const { runProviderSessionAction: runAction } = await import("./actions.js"); + const sourceDir = path.join(constants.CODEX_HOME, "sessions", "manifest-smoke"); + const sourcePath = path.join(sourceDir, "session.jsonl"); + const backupRoot = path.join(constants.HOME_DIR, "ThreadLens Test Backups"); + const manifestPath = path.join( + backupRoot, + "provider_actions", + "codex", + folderName, + "_manifest.json", + ); + const payload = "{\"type\":\"session\",\"id\":\"manifest-smoke\"}\n"; + const deps = { + resolveAllowedProviderFilePath: async () => sourcePath, + supportsProviderAction: () => true, + invalidateProviderCaches: () => undefined, + }; + + await mkdir(sourceDir, { recursive: true }); + await mkdir(manifestPath, { recursive: true }); + await writeFile(sourcePath, payload, "utf8"); + + const preview = await runAction( + deps, + "codex", + "delete_local", + [sourcePath], + true, + "", + { backup_before_delete: true, backup_root: backupRoot }, + ); + expect(preview.ok).toBe(true); + expect(preview.confirm_token_expected).toMatch(/^PROVIDER-/); + + const result = await runAction( + deps, + "codex", + "delete_local", + [sourcePath], + false, + preview.confirm_token_expected, + { backup_before_delete: true, backup_root: backupRoot }, + ); + + expect(result.ok).toBe(false); + expect(result.mode).toBe("failed"); + expect(result.applied_count).toBe(0); + expect(result.failed).toEqual( + expect.arrayContaining([ + expect.objectContaining({ file_path: manifestPath, step: "manifest_write" }), + ]), + ); + await expect(readFile(sourcePath, "utf8")).resolves.toBe(payload); + } finally { + if (previousHome === undefined) delete process.env.HOME; + else process.env.HOME = previousHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; + if (previousStateDir === undefined) delete process.env.THREADLENS_STATE_DIR; + else process.env.THREADLENS_STATE_DIR = previousStateDir; + vi.doUnmock("../../lib/utils.js"); + vi.resetModules(); + await rm(root, { recursive: true, force: true }); + } + }); + it("restores archived Codex sessions back to the source sessions directory", async () => { const testRunId = `threadlens-vitest-${Date.now()}-${Math.random().toString(16).slice(2)}`; const archivedDir = path.join(CODEX_HOME, "archived_sessions", "__threadlens-vitest__", testRunId); @@ -148,7 +409,7 @@ describe("runProviderSessionAction", () => { const payload = `{"type":"session","id":"${testRunId}"}\n`; const deps = { resolveAllowedProviderFilePath: async () => archivedPath, - supportsProviderCleanup: () => true, + supportsProviderAction: () => true, invalidateProviderCaches: () => undefined, }; @@ -201,7 +462,7 @@ describe("runProviderSessionAction", () => { const payload = `{"type":"session","id":"${testRunId}"}\n`; const deps = { resolveAllowedProviderFilePath: async () => sourcePath, - supportsProviderCleanup: () => true, + supportsProviderAction: () => true, invalidateProviderCaches: () => undefined, }; @@ -269,7 +530,7 @@ describe("runProviderSessionAction", () => { const payload = "{\"type\":\"session\",\"id\":\"realpath-smoke\"}\n"; const deps = { resolveAllowedProviderFilePath: async (_provider: "codex", filePath: string) => realpath(filePath), - supportsProviderCleanup: () => true, + supportsProviderAction: () => true, invalidateProviderCaches: () => undefined, }; @@ -365,7 +626,7 @@ describe("runProviderSessionAction", () => { const archivedPath = path.join(archivedDir, fileName); const deps = { resolveAllowedProviderFilePath: async (_provider: typeof provider, filePath: string) => filePath, - supportsProviderCleanup: () => true, + supportsProviderAction: () => true, invalidateProviderCaches: () => undefined, }; diff --git a/apps/api-ts/src/domains/providers/actions.ts b/apps/api-ts/src/domains/providers/actions.ts index faf9b20..5ee37cf 100644 --- a/apps/api-ts/src/domains/providers/actions.ts +++ b/apps/api-ts/src/domains/providers/actions.ts @@ -1,400 +1,53 @@ -import path from "node:path"; -import { createHash, randomUUID } from "node:crypto"; -import { copyFile, mkdir, realpath, stat, unlink, writeFile } from "node:fs/promises"; -import { BACKUP_ROOT, HOME_DIR } from "../../lib/constants.js"; -import { nowIsoUtc } from "../../lib/utils.js"; -import { isPathInsideRoot, providerRootSpecs } from "./path-safety.js"; +import { BACKUP_ROOT } from "../../lib/constants.js"; +import { + deriveProviderBackupId, + resolveProviderActionBackupRoot, + stageProviderActionBackup, +} from "./actions/backups.js"; +import { + consumeProviderActionConfirmToken, + issueProviderActionConfirmToken, + pruneProviderActionTokens, +} from "./actions/confirm-tokens.js"; +import { + buildProviderActionFingerprint, + normalizeProviderActionOptions, +} from "./actions/selection.js"; +import { + buildProviderActionAppliedResponse, + buildProviderActionFailureResponse, + buildProviderActionPreviewResponse, +} from "./actions/responses.js"; +import { + type ProviderActionExecutionFailure, + runArchiveProviderAction, + runDeleteProviderAction, + runUnarchiveProviderAction, +} from "./actions/execution.js"; +import { + normalizeProviderActionTargetPaths, + resolveProviderActionTargets, +} from "./actions/targets.js"; import type { ProviderId, - ProviderRootSpec, ProviderSessionAction, ProviderSessionActionOptions, } from "./types.js"; -type ProviderActionTokenEntry = { - provider: ProviderId; - action: ProviderSessionAction; - paths: string[]; - backup_before_delete: boolean; - backup_root: string; - expires_at: number; -}; - -type ProviderBackupManifestItem = { - source_path: string; - backup_rel_path: string; - backup_abs_path: string; -}; - -type ProviderBackupFailure = { - file_path: string; - step: string; - error: string; -}; - -type ProviderBackupStageResult = { - destination: string; - manifest_path: string; - items: ProviderBackupManifestItem[]; - failed: ProviderBackupFailure[]; -}; - type ProviderActionDeps = { resolveAllowedProviderFilePath: ( provider: ProviderId, filePath: string, ) => Promise; - supportsProviderCleanup: (provider: ProviderId) => boolean; + supportsProviderAction: ( + provider: ProviderId, + action: ProviderSessionAction, + ) => boolean; invalidateProviderCaches: (provider: ProviderId) => void; }; -const PROVIDER_ACTION_TOKEN_TTL_MS = 10 * 60_000; -const providerActionTokenCache = new Map(); - -function normalizeProviderActionPaths(filePaths: string[]): string[] { - return Array.from( - new Set( - filePaths - .map((item) => path.resolve(String(item || "").trim())) - .filter(Boolean), - ), - ).sort(); -} - -function normalizeProviderActionOptions( - options?: ProviderSessionActionOptions, -): { backup_before_delete: boolean; backup_root: string } { - return { - backup_before_delete: Boolean(options?.backup_before_delete), - backup_root: String(options?.backup_root ?? "").trim(), - }; -} - -function resolveProviderActionBackupRoot( - backupRootOverride: string, -): { ok: true; backupRoot: string } | { ok: false; error: string } { - const backupRoot = backupRootOverride - ? path.resolve(backupRootOverride) - : BACKUP_ROOT; - if (!isPathInsideRoot(backupRoot, HOME_DIR)) { - return { ok: false, error: "backup_root_outside_home" }; - } - if (path.relative(HOME_DIR, backupRoot).split(path.sep).some((segment) => segment.startsWith("."))) { - return { ok: false, error: "backup_root_hidden" }; - } - return { ok: true, backupRoot }; -} - -function deriveProviderBackupId(backupRoot: string, backupTo: string | null): string | null { - if (!backupTo) return null; - const relativePath = path.relative(path.resolve(backupRoot), path.resolve(backupTo)); - if (!relativePath || relativePath.startsWith("..") || path.isAbsolute(relativePath)) return null; - return relativePath.split(path.sep).join("/"); -} - -function requiresProviderCleanupPrivilege(action: ProviderSessionAction): boolean { - return action !== "backup_local"; -} - -function isSafeProviderRelativePath(relativePath: string): boolean { - return Boolean(relativePath) && - relativePath !== "." && - relativePath !== ".." && - !relativePath.startsWith(`..${path.sep}`) && - !path.isAbsolute(relativePath); -} - -async function relativePathWithinProviderRoot( - filePath: string, - spec: ProviderRootSpec, -): Promise { - const resolvedPath = path.resolve(filePath); - if (isPathInsideRoot(resolvedPath, spec.root)) { - const relativePath = path.relative(spec.root, resolvedPath); - return isSafeProviderRelativePath(relativePath) ? relativePath : null; - } - - try { - const [realTarget, realRoot] = await Promise.all([ - realpath(resolvedPath), - realpath(spec.root), - ]); - if (isPathInsideRoot(realTarget, realRoot)) { - const relativePath = path.relative(realRoot, realTarget); - return isSafeProviderRelativePath(relativePath) ? relativePath : null; - } - } catch { - // Fall back to the resolved path comparison above when a root cannot be realpathed. - } - - return null; -} - -async function resolveArchivedSessionRestoreTarget( - provider: ProviderId, - filePath: string, -): Promise { - const specs = providerRootSpecs(provider); - const archivedSpecs = specs - .filter((spec) => spec.source === "archived_sessions") - .sort((left, right) => right.root.length - left.root.length); - const sessionsSpec = specs.find((spec) => spec.source === "sessions"); - let relativePath: string | null = null; - for (const spec of archivedSpecs) { - relativePath = await relativePathWithinProviderRoot(filePath, spec); - if (relativePath) break; - } - if (!relativePath) return null; - - if (sessionsSpec && !relativePath.includes(path.sep)) { - return path.join(sessionsSpec.root, relativePath); - } - const [sourceName = "", ...sourceRelativeParts] = relativePath.split(path.sep); - const sourceRelativePath = sourceRelativeParts.join(path.sep); - const sourceSpec = specs.find( - (spec) => - spec.source === sourceName && - spec.source !== "cleanup_backups" && - spec.source !== "archived_sessions", - ); - if (sourceSpec && sourceRelativePath) { - return path.join(sourceSpec.root, sourceRelativePath); - } - if (sessionsSpec) { - return path.join(sessionsSpec.root, relativePath); - } - return null; -} - -async function resolveArchivedSessionStoreTarget( - provider: ProviderId, - filePath: string, -): Promise { - const resolvedPath = path.resolve(filePath); - const specs = providerRootSpecs(provider); - const archivedSpec = specs.find((spec) => spec.source === "archived_sessions"); - if (!archivedSpec) return null; - const sourceSpecs = specs - .filter( - (spec) => - spec.source !== "cleanup_backups" && - spec.source !== "archived_sessions" && - spec.exts.includes(path.extname(resolvedPath).toLowerCase()), - ) - .sort((left, right) => right.root.length - left.root.length); - let sourceSpec: ProviderRootSpec | null = null; - let relativePath: string | null = null; - for (const spec of sourceSpecs) { - relativePath = await relativePathWithinProviderRoot(resolvedPath, spec); - if (relativePath) { - sourceSpec = spec; - break; - } - } - if (!sourceSpec || !relativePath) return null; - - const archiveRelativePath = - provider === "codex" && sourceSpec.source === "sessions" - ? relativePath - : path.join(sourceSpec.source, relativePath); - return path.join(archivedSpec.root, archiveRelativePath); -} - -function resolveArchivedSessionRoot(provider: ProviderId): string | null { - return providerRootSpecs(provider).find((spec) => spec.source === "archived_sessions") - ?.root ?? null; -} - -function pruneProviderActionTokens(now = Date.now()) { - for (const [token, entry] of providerActionTokenCache.entries()) { - if (entry.expires_at <= now) providerActionTokenCache.delete(token); - } -} - -function issueProviderActionConfirmToken( - provider: ProviderId, - action: ProviderSessionAction, - filePaths: string[], - options?: ProviderSessionActionOptions, -): string { - const normalized = normalizeProviderActionPaths(filePaths); - const normalizedOptions = normalizeProviderActionOptions(options); - const token = `PROVIDER-${randomUUID().replace(/-/g, "").slice(0, 12).toUpperCase()}`; - providerActionTokenCache.set(token, { - provider, - action, - paths: normalized, - backup_before_delete: normalizedOptions.backup_before_delete, - backup_root: normalizedOptions.backup_root, - expires_at: Date.now() + PROVIDER_ACTION_TOKEN_TTL_MS, - }); - return token; -} - -function consumeProviderActionConfirmToken( - token: string, - provider: ProviderId, - action: ProviderSessionAction, - filePaths: string[], - options?: ProviderSessionActionOptions, -): { ok: boolean; reason: string } { - pruneProviderActionTokens(); - const normalizedOptions = normalizeProviderActionOptions(options); - const key = String(token || "").trim(); - if (!key) return { ok: false, reason: "missing-confirm-token" }; - const entry = providerActionTokenCache.get(key); - if (!entry) return { ok: false, reason: "invalid-confirm-token" }; - if (entry.expires_at <= Date.now()) { - providerActionTokenCache.delete(key); - return { ok: false, reason: "expired-confirm-token" }; - } - const normalized = normalizeProviderActionPaths(filePaths); - const sameProvider = entry.provider === provider; - const sameAction = entry.action === action; - const sameOptions = - entry.backup_before_delete === normalizedOptions.backup_before_delete && - entry.backup_root === normalizedOptions.backup_root; - const samePaths = - entry.paths.length === normalized.length && - entry.paths.every((item, idx) => item === normalized[idx]); - if (!sameProvider || !sameAction || !sameOptions || !samePaths) { - return { ok: false, reason: "confirm-token-scope-mismatch" }; - } - providerActionTokenCache.delete(key); - return { ok: true, reason: "" }; -} - -export function deriveProviderBackupRelativePath( - provider: ProviderId, - filePath: string, -): string { - const resolvedPath = path.resolve(filePath); - const ext = path.extname(resolvedPath).toLowerCase(); - const matchingSpecs = providerRootSpecs(provider) - .filter((spec) => spec.exts.includes(ext) && isPathInsideRoot(resolvedPath, spec.root)) - .sort((left, right) => right.root.length - left.root.length); - - for (const spec of matchingSpecs) { - const relativePath = path.relative(spec.root, resolvedPath); - if ( - !relativePath || - relativePath === "." || - relativePath === ".." || - relativePath.startsWith(`..${path.sep}`) || - path.isAbsolute(relativePath) - ) { - continue; - } - return path.join(spec.source, relativePath); - } - - return path.join("misc", path.basename(resolvedPath)); -} - -async function stageProviderActionBackup( - provider: ProviderId, - action: ProviderSessionAction, - filePaths: string[], - backupRoot = BACKUP_ROOT, -): Promise { - const folderName = `${nowIsoUtc().replace(/[:.]/g, "-")}-${action}`; - const destination = path.join( - backupRoot, - "provider_actions", - provider, - folderName, - ); - await mkdir(destination, { recursive: true }); - - const items: ProviderBackupManifestItem[] = []; - const failed: ProviderBackupFailure[] = []; - - for (const rawSourcePath of filePaths) { - const sourcePath = path.resolve(rawSourcePath); - const backupRelativePath = deriveProviderBackupRelativePath(provider, sourcePath); - const targetPath = path.join(destination, backupRelativePath); - try { - await mkdir(path.dirname(targetPath), { recursive: true }); - await copyFile(sourcePath, targetPath); - items.push({ - source_path: sourcePath, - backup_rel_path: backupRelativePath, - backup_abs_path: targetPath, - }); - } catch (error) { - failed.push({ - file_path: sourcePath, - step: `${action}:backup_copy`, - error: String(error), - }); - } - } - - const manifestPath = path.join(destination, "_manifest.json"); - try { - await writeFile( - manifestPath, - JSON.stringify( - { - generated_at: nowIsoUtc(), - provider, - action, - item_count: items.length, - items, - }, - null, - 2, - ), - "utf-8", - ); - } catch (error) { - failed.push({ - file_path: manifestPath, - step: "manifest_write", - error: String(error), - }); - } - - return { - destination, - manifest_path: manifestPath, - items, - failed, - }; -} - -export function buildProviderActionToken( - provider: ProviderId, - action: ProviderSessionAction, - filePaths: string[], - options?: ProviderSessionActionOptions, -): string { - const digest = buildProviderActionFingerprint(provider, action, filePaths, options); - return `PROVIDER-${digest}`; -} - -export function buildProviderActionFingerprint( - provider: ProviderId, - action: ProviderSessionAction, - filePaths: string[], - options?: ProviderSessionActionOptions, -): string { - const normalizedOptions = normalizeProviderActionOptions(options); - const normalized = normalizeProviderActionPaths(filePaths); - const raw = JSON.stringify({ - provider, - action, - paths: normalized, - backup_before_delete: normalizedOptions.backup_before_delete, - backup_root: normalizedOptions.backup_root, - }); - const digest = createHash("sha256") - .update(raw, "utf-8") - .digest("hex") - .slice(0, 12) - .toUpperCase(); - return digest; -} +export { buildProviderActionFingerprint, buildProviderActionToken } from "./actions/selection.js"; +export { deriveProviderBackupRelativePath } from "./actions/backups.js"; export async function runProviderSessionAction( deps: ProviderActionDeps, @@ -407,79 +60,37 @@ export async function runProviderSessionAction( ) { pruneProviderActionTokens(); const normalizedOptions = normalizeProviderActionOptions(options); - const uniquePaths = Array.from( - new Set(filePaths.map((item) => String(item || "").trim()).filter(Boolean)), - ); - if ( - requiresProviderCleanupPrivilege(action) && - !deps.supportsProviderCleanup(provider) - ) { - return { - ok: false, + const uniquePaths = normalizeProviderActionTargetPaths(filePaths); + if (!deps.supportsProviderAction(provider, action)) { + return buildProviderActionFailureResponse({ provider, action, - dry_run: dryRun, - target_count: uniquePaths.length, - valid_count: 0, - applied_count: 0, - confirm_token_expected: "", - confirm_token_accepted: false, - selection_fingerprint: "", - backup_before_delete: normalizedOptions.backup_before_delete, - failure_summary: { - skipped_count: 0, - failed_count: 0, - partial_failure: false, - }, + dryRun, + targetCount: uniquePaths.length, + validCount: 0, + selectionFingerprint: "", + options: normalizedOptions, skipped: [], error: "cleanup-disabled-provider", - }; - } - const skipped: Array<{ file_path: string; reason: string }> = []; - const valid: string[] = []; - - for (const candidate of uniquePaths) { - const safePath = await deps.resolveAllowedProviderFilePath(provider, candidate); - if (!safePath) { - skipped.push({ - file_path: candidate, - reason: "outside-provider-root-extension-or-realpath", - }); - continue; - } - try { - const st = await stat(safePath); - if (!st.isFile()) { - skipped.push({ file_path: candidate, reason: "not-a-file" }); - continue; - } - valid.push(safePath); - } catch { - skipped.push({ file_path: candidate, reason: "not-found" }); - } + }); } - + const { skipped, valid } = await resolveProviderActionTargets( + provider, + uniquePaths, + deps.resolveAllowedProviderFilePath, + ); if (!valid.length && !dryRun) { - return { - ok: false, + return buildProviderActionFailureResponse({ provider, action, - dry_run: false, - target_count: uniquePaths.length, - valid_count: valid.length, - applied_count: 0, - confirm_token_expected: "", - confirm_token_accepted: false, - selection_fingerprint: "", - backup_before_delete: normalizedOptions.backup_before_delete, - failure_summary: { - skipped_count: skipped.length, - failed_count: 0, - partial_failure: false, - }, + dryRun: false, + targetCount: uniquePaths.length, + validCount: valid.length, + selectionFingerprint: "", + options: normalizedOptions, skipped, error: "no-valid-targets", - }; + }); } const selectionFingerprint = valid.length @@ -488,56 +99,38 @@ export async function runProviderSessionAction( const shouldBackup = action === "backup_local" || normalizedOptions.backup_before_delete; - const backupRootResult = resolveProviderActionBackupRoot( + const backupRootResult = await resolveProviderActionBackupRoot( normalizedOptions.backup_root, ); if (shouldBackup && !backupRootResult.ok) { - return { - ok: false, + return buildProviderActionFailureResponse({ provider, action, - dry_run: dryRun, - target_count: uniquePaths.length, - valid_count: valid.length, - applied_count: 0, - confirm_token_expected: "", - confirm_token_accepted: false, - selection_fingerprint: selectionFingerprint, - backup_before_delete: normalizedOptions.backup_before_delete, - failure_summary: { - skipped_count: skipped.length, - failed_count: 0, - partial_failure: false, - }, + dryRun, + targetCount: uniquePaths.length, + validCount: valid.length, + selectionFingerprint, + options: normalizedOptions, skipped, error: backupRootResult.error, - }; + }); } if (dryRun) { const expectedToken = valid.length ? issueProviderActionConfirmToken(provider, action, valid, normalizedOptions) : ""; - return { - ok: true, + return buildProviderActionPreviewResponse({ provider, action, - dry_run: true, - target_count: uniquePaths.length, - valid_count: valid.length, - applied_count: 0, - confirm_token_expected: expectedToken, - confirm_token_accepted: false, - selection_fingerprint: selectionFingerprint, - backup_before_delete: normalizedOptions.backup_before_delete, - failure_summary: { - skipped_count: skipped.length, - failed_count: 0, - partial_failure: false, - }, + dryRun: true, + targetCount: uniquePaths.length, + validCount: valid.length, + selectionFingerprint, + options: normalizedOptions, skipped, - mode: "preview", - }; + confirmTokenExpected: expectedToken, + }); } if (action !== "backup_local") { @@ -552,27 +145,19 @@ export async function runProviderSessionAction( const expectedToken = valid.length ? issueProviderActionConfirmToken(provider, action, valid, normalizedOptions) : ""; - return { - ok: false, + return buildProviderActionFailureResponse({ provider, action, - dry_run: false, - target_count: uniquePaths.length, - valid_count: valid.length, - applied_count: 0, - confirm_token_expected: expectedToken, - confirm_token_accepted: false, - selection_fingerprint: selectionFingerprint, - backup_before_delete: normalizedOptions.backup_before_delete, - failure_summary: { - skipped_count: skipped.length, - failed_count: 0, - partial_failure: false, - }, - skipped, - error: consume.reason, - }; - } + dryRun: false, + targetCount: uniquePaths.length, + validCount: valid.length, + selectionFingerprint, + options: normalizedOptions, + skipped, + confirmTokenExpected: expectedToken, + error: consume.reason, + }); + } } let applied = 0; @@ -580,8 +165,14 @@ export async function runProviderSessionAction( let backupTo: string | null = null; let backupManifestPath: string | null = null; let backedUpCount = 0; - const failed: Array<{ file_path: string; step: string; error: string }> = []; - const backupRoot = backupRootResult.ok ? backupRootResult.backupRoot : BACKUP_ROOT; + const failed: ProviderActionExecutionFailure[] = []; + const backupRoot = shouldBackup + ? backupRootResult.ok + ? backupRootResult.backupRoot + : (() => { + throw new Error(`unreachable invalid backup root: ${backupRootResult.error}`); + })() + : BACKUP_ROOT; const backupStage = shouldBackup ? await stageProviderActionBackup(provider, action, valid, backupRoot) : null; @@ -591,130 +182,65 @@ export async function runProviderSessionAction( backupManifestPath = backupStage.manifest_path; backedUpCount = backupStage.items.length; } + const backupManifestWriteFailed = + backupStage?.failed.some((failure) => failure.step === "manifest_write") ?? false; + if (backupManifestWriteFailed && action !== "backup_local") { + return buildProviderActionAppliedResponse({ + provider, + action, + targetCount: uniquePaths.length, + validCount: valid.length, + selectionFingerprint, + options: normalizedOptions, + skipped, + failed, + applied: 0, + backedUpCount, + backupId: deriveProviderBackupId(backupRoot, backupTo), + backupTo, + backupManifestPath, + shouldBackup, + backupStageFailedCount: backupStage?.failed.length ?? 0, + archivedTo, + }); + } if (action === "backup_local") { applied = backupStage?.items.length ?? 0; } else if (action === "archive_local") { - archivedTo = resolveArchivedSessionRoot(provider); - for (const sourcePath of valid) { - const targetPath = await resolveArchivedSessionStoreTarget(provider, sourcePath); - if (!targetPath) { - failed.push({ - file_path: sourcePath, - step: "archive_local", - error: "no-archived-session-target", - }); - continue; - } - try { - await stat(targetPath); - failed.push({ - file_path: sourcePath, - step: "archive_local", - error: "target-already-exists", - }); - continue; - } catch { - // Missing target is expected before archive. - } - try { - await mkdir(path.dirname(targetPath), { recursive: true }); - await copyFile(sourcePath, targetPath); - await unlink(sourcePath); - applied += 1; - } catch (error) { - failed.push({ - file_path: sourcePath, - step: "archive_local", - error: String(error), - }); - } - } + const result = await runArchiveProviderAction(provider, valid); + applied = result.applied; + archivedTo = result.archivedTo; + failed.push(...result.failed); } else if (action === "unarchive_local") { - for (const sourcePath of valid) { - const targetPath = await resolveArchivedSessionRestoreTarget(provider, sourcePath); - if (!targetPath) { - failed.push({ - file_path: sourcePath, - step: "unarchive_local", - error: "not-an-archived-session", - }); - continue; - } - try { - await stat(targetPath); - failed.push({ - file_path: sourcePath, - step: "unarchive_local", - error: "target-already-exists", - }); - continue; - } catch { - // Missing target is expected before restore. - } - try { - await mkdir(path.dirname(targetPath), { recursive: true }); - await copyFile(sourcePath, targetPath); - await unlink(sourcePath); - applied += 1; - } catch (error) { - failed.push({ - file_path: sourcePath, - step: "unarchive_local", - error: String(error), - }); - } - } + const result = await runUnarchiveProviderAction(provider, valid); + applied = result.applied; + failed.push(...result.failed); } else { const deleteTargets = backupStage?.items.map((item) => item.source_path) ?? valid; - for (const sourcePath of deleteTargets) { - try { - await unlink(sourcePath); - applied += 1; - } catch (error) { - failed.push({ - file_path: sourcePath, - step: "delete_local", - error: String(error), - }); - } - } + const result = await runDeleteProviderAction(deleteTargets); + applied = result.applied; + failed.push(...result.failed); } deps.invalidateProviderCaches(provider); - return { - ok: failed.length === 0, + return buildProviderActionAppliedResponse({ provider, action, - dry_run: false, - target_count: uniquePaths.length, - valid_count: valid.length, - applied_count: applied, - confirm_token_expected: "", - confirm_token_accepted: action === "backup_local" ? false : true, - selection_fingerprint: selectionFingerprint, - backup_before_delete: normalizedOptions.backup_before_delete, - backed_up_count: backedUpCount, - backup_id: deriveProviderBackupId(backupRoot, backupTo), - backup_to: backupTo, - backup_manifest_path: backupManifestPath, - backup_summary: shouldBackup - ? { - destination: backupTo, - manifest_path: backupManifestPath, - copied_count: backedUpCount, - failed_count: backupStage?.failed.length ?? 0, - } - : null, - failure_summary: { - skipped_count: skipped.length, - failed_count: failed.length, - partial_failure: failed.length > 0 && applied > 0, - }, + targetCount: uniquePaths.length, + validCount: valid.length, + selectionFingerprint, + options: normalizedOptions, skipped, failed, - archived_to: archivedTo, - mode: failed.length === 0 ? "applied" : applied > 0 ? "partial" : "failed", - }; + applied, + backedUpCount, + backupId: deriveProviderBackupId(backupRoot, backupTo), + backupTo, + backupManifestPath, + shouldBackup, + backupStageFailedCount: backupStage?.failed.length ?? 0, + archivedTo, + }); } diff --git a/apps/api-ts/src/domains/providers/actions/archives.ts b/apps/api-ts/src/domains/providers/actions/archives.ts new file mode 100644 index 0000000..7bccc69 --- /dev/null +++ b/apps/api-ts/src/domains/providers/actions/archives.ts @@ -0,0 +1,113 @@ +import path from "node:path"; +import { realpath } from "node:fs/promises"; +import { isPathInsideRoot, providerRootSpecs } from "../path-safety.js"; +import type { ProviderId, ProviderRootSpec } from "../types.js"; + +function isSafeProviderRelativePath(relativePath: string): boolean { + return Boolean(relativePath) && + relativePath !== "." && + relativePath !== ".." && + !relativePath.startsWith(`..${path.sep}`) && + !path.isAbsolute(relativePath); +} + +async function relativePathWithinProviderRoot( + filePath: string, + spec: ProviderRootSpec, +): Promise { + const resolvedPath = path.resolve(filePath); + if (isPathInsideRoot(resolvedPath, spec.root)) { + const relativePath = path.relative(spec.root, resolvedPath); + return isSafeProviderRelativePath(relativePath) ? relativePath : null; + } + + try { + const [realTarget, realRoot] = await Promise.all([ + realpath(resolvedPath), + realpath(spec.root), + ]); + if (isPathInsideRoot(realTarget, realRoot)) { + const relativePath = path.relative(realRoot, realTarget); + return isSafeProviderRelativePath(relativePath) ? relativePath : null; + } + } catch { + // Fall back to the resolved path comparison above when a root cannot be realpathed. + } + + return null; +} + +export async function resolveArchivedSessionRestoreTarget( + provider: ProviderId, + filePath: string, +): Promise { + const specs = providerRootSpecs(provider); + const archivedSpecs = specs + .filter((spec) => spec.source === "archived_sessions") + .sort((left, right) => right.root.length - left.root.length); + const sessionsSpec = specs.find((spec) => spec.source === "sessions"); + let relativePath: string | null = null; + for (const spec of archivedSpecs) { + relativePath = await relativePathWithinProviderRoot(filePath, spec); + if (relativePath) break; + } + if (!relativePath) return null; + + if (sessionsSpec && !relativePath.includes(path.sep)) { + return path.join(sessionsSpec.root, relativePath); + } + const [sourceName = "", ...sourceRelativeParts] = relativePath.split(path.sep); + const sourceRelativePath = sourceRelativeParts.join(path.sep); + const sourceSpec = specs.find( + (spec) => + spec.source === sourceName && + spec.source !== "cleanup_backups" && + spec.source !== "archived_sessions", + ); + if (sourceSpec && sourceRelativePath) { + return path.join(sourceSpec.root, sourceRelativePath); + } + if (sessionsSpec) { + return path.join(sessionsSpec.root, relativePath); + } + return null; +} + +export async function resolveArchivedSessionStoreTarget( + provider: ProviderId, + filePath: string, +): Promise { + const resolvedPath = path.resolve(filePath); + const specs = providerRootSpecs(provider); + const archivedSpec = specs.find((spec) => spec.source === "archived_sessions"); + if (!archivedSpec) return null; + const sourceSpecs = specs + .filter( + (spec) => + spec.source !== "cleanup_backups" && + spec.source !== "archived_sessions" && + spec.exts.includes(path.extname(resolvedPath).toLowerCase()), + ) + .sort((left, right) => right.root.length - left.root.length); + let sourceSpec: ProviderRootSpec | null = null; + let relativePath: string | null = null; + for (const spec of sourceSpecs) { + relativePath = await relativePathWithinProviderRoot(resolvedPath, spec); + if (relativePath) { + sourceSpec = spec; + break; + } + } + if (!sourceSpec || !relativePath) return null; + + const archiveRelativePath = + provider === "codex" && sourceSpec.source === "sessions" + ? relativePath + : path.join(sourceSpec.source, relativePath); + return path.join(archivedSpec.root, archiveRelativePath); +} + +export function resolveArchivedSessionRoot(provider: ProviderId): string | null { + return providerRootSpecs(provider).find((spec) => spec.source === "archived_sessions") + ?.root ?? null; +} diff --git a/apps/api-ts/src/domains/providers/actions/backups.ts b/apps/api-ts/src/domains/providers/actions/backups.ts new file mode 100644 index 0000000..4af58ab --- /dev/null +++ b/apps/api-ts/src/domains/providers/actions/backups.ts @@ -0,0 +1,254 @@ +import path from "node:path"; +import { copyFile, mkdir, realpath, writeFile } from "node:fs/promises"; +import { BACKUP_ROOT, HOME_DIR } from "../../../lib/constants.js"; +import { nowIsoUtc } from "../../../lib/utils.js"; +import { + isPathInsideRoot, + providerRootSpecs, + providerScanRootSpecs, +} from "../path-safety.js"; +import type { + ProviderId, + ProviderSessionAction, +} from "../types.js"; + +type ProviderBackupManifestItem = { + source_path: string; + backup_rel_path: string; + backup_abs_path: string; +}; + +type ProviderBackupFailure = { + file_path: string; + step: string; + error: string; +}; + +export type ProviderBackupStageResult = { + destination: string; + manifest_path: string; + items: ProviderBackupManifestItem[]; + failed: ProviderBackupFailure[]; +}; + +export async function resolveProviderActionBackupRoot( + backupRootOverride: string, +): Promise<{ ok: true; backupRoot: string } | { ok: false; error: string }> { + const backupRoot = backupRootOverride + ? path.resolve(backupRootOverride) + : BACKUP_ROOT; + if (!isPathInsideRoot(backupRoot, HOME_DIR)) { + return { ok: false, error: "backup_root_outside_home" }; + } + if (path.relative(HOME_DIR, backupRoot).split(path.sep).some((segment) => segment.startsWith("."))) { + return { ok: false, error: "backup_root_hidden" }; + } + const realHome = await realpath(HOME_DIR).catch(() => path.resolve(HOME_DIR)); + const realBackupAncestor = await resolveExistingAncestorRealpath(backupRoot); + if (realBackupAncestor && !isPathInsideRoot(realBackupAncestor, realHome)) { + return { ok: false, error: "backup_root_outside_home" }; + } + return { ok: true, backupRoot }; +} + +async function resolveExistingAncestorRealpath(filePath: string): Promise { + let currentPath = path.resolve(filePath); + while (true) { + try { + return await realpath(currentPath); + } catch { + const parentPath = path.dirname(currentPath); + if (parentPath === currentPath) return null; + currentPath = parentPath; + } + } +} + +export function deriveProviderBackupId( + backupRoot: string, + backupTo: string | null, +): string | null { + if (!backupTo) return null; + const relativePath = path.relative(path.resolve(backupRoot), path.resolve(backupTo)); + if (!relativePath || relativePath.startsWith("..") || path.isAbsolute(relativePath)) return null; + return relativePath.split(path.sep).join("/"); +} + +export function deriveProviderBackupRelativePath( + provider: ProviderId, + filePath: string, +): string { + const resolvedPath = path.resolve(filePath); + const ext = path.extname(resolvedPath).toLowerCase(); + const matchingSpecs = providerRootSpecs(provider) + .filter((spec) => spec.exts.includes(ext) && isPathInsideRoot(resolvedPath, spec.root)) + .sort((left, right) => right.root.length - left.root.length); + + for (const spec of matchingSpecs) { + const relativePath = path.relative(spec.root, resolvedPath); + if (!isSafeProviderBackupRelativePath(relativePath)) continue; + return path.join(spec.source, relativePath); + } + + return path.join("misc", path.basename(resolvedPath)); +} + +async function deriveProviderBackupRelativePathForSafePath( + provider: ProviderId, + filePath: string, +): Promise { + const resolvedPath = path.resolve(filePath); + let realSourcePath = resolvedPath; + try { + realSourcePath = await realpath(resolvedPath); + } catch { + return deriveProviderBackupRelativePath(provider, resolvedPath); + } + + const ext = path.extname(realSourcePath).toLowerCase(); + const matchingSpecs: Array<{ source: string; root: string }> = []; + for (const spec of await providerScanRootSpecs(provider)) { + if (!spec.exts.includes(ext)) continue; + try { + const realRoot = await realpath(spec.root); + if ( + realSourcePath === realRoot || + realSourcePath.startsWith(`${realRoot}${path.sep}`) + ) { + matchingSpecs.push({ source: spec.source, root: realRoot }); + } + } catch { + continue; + } + } + + matchingSpecs.sort((left, right) => right.root.length - left.root.length); + for (const spec of matchingSpecs) { + const relativePath = path.relative(spec.root, realSourcePath); + if (!isSafeProviderBackupRelativePath(relativePath)) continue; + return path.join(spec.source, relativePath); + } + + return deriveProviderBackupRelativePath(provider, realSourcePath); +} + +export async function stageProviderActionBackup( + provider: ProviderId, + action: ProviderSessionAction, + filePaths: string[], + backupRoot = BACKUP_ROOT, +): Promise { + const folderName = `${nowIsoUtc().replace(/[:.]/g, "-")}-${action}`; + const destination = path.join( + backupRoot, + "provider_actions", + provider, + folderName, + ); + const manifestPath = path.join(destination, "_manifest.json"); + const destinationSafety = await validateProviderBackupDestination(destination); + if (!destinationSafety.ok) { + return { + destination, + manifest_path: manifestPath, + items: [], + failed: [ + { + file_path: destination, + step: "backup_root", + error: destinationSafety.error, + }, + ], + }; + } + await mkdir(destination, { recursive: true }); + const realDestination = await realpath(destination).catch(() => null); + if (!realDestination || !isPathInsideRoot(realDestination, destinationSafety.realHome)) { + return { + destination, + manifest_path: manifestPath, + items: [], + failed: [ + { + file_path: destination, + step: "backup_root", + error: "backup_root_outside_home", + }, + ], + }; + } + + const items: ProviderBackupManifestItem[] = []; + const failed: ProviderBackupFailure[] = []; + + for (const rawSourcePath of filePaths) { + const sourcePath = path.resolve(rawSourcePath); + const backupRelativePath = await deriveProviderBackupRelativePathForSafePath(provider, sourcePath); + const targetPath = path.join(destination, backupRelativePath); + try { + await mkdir(path.dirname(targetPath), { recursive: true }); + await copyFile(sourcePath, targetPath); + items.push({ + source_path: sourcePath, + backup_rel_path: backupRelativePath, + backup_abs_path: targetPath, + }); + } catch (error) { + failed.push({ + file_path: sourcePath, + step: `${action}:backup_copy`, + error: String(error), + }); + } + } + + try { + await writeFile( + manifestPath, + JSON.stringify( + { + generated_at: nowIsoUtc(), + provider, + action, + item_count: items.length, + items, + }, + null, + 2, + ), + "utf-8", + ); + } catch (error) { + failed.push({ + file_path: manifestPath, + step: "manifest_write", + error: String(error), + }); + } + + return { + destination, + manifest_path: manifestPath, + items, + failed, + }; +} + +async function validateProviderBackupDestination( + destination: string, +): Promise<{ ok: true; realHome: string } | { ok: false; error: string }> { + const realHome = await realpath(HOME_DIR).catch(() => path.resolve(HOME_DIR)); + const realDestinationAncestor = await resolveExistingAncestorRealpath(destination); + if (realDestinationAncestor && !isPathInsideRoot(realDestinationAncestor, realHome)) { + return { ok: false, error: "backup_root_outside_home" }; + } + return { ok: true, realHome }; +} + +function isSafeProviderBackupRelativePath(relativePath: string): boolean { + return Boolean(relativePath) && + relativePath !== "." && + relativePath !== ".." && + !relativePath.startsWith(`..${path.sep}`) && + !path.isAbsolute(relativePath); +} diff --git a/apps/api-ts/src/domains/providers/actions/confirm-tokens.ts b/apps/api-ts/src/domains/providers/actions/confirm-tokens.ts new file mode 100644 index 0000000..c08ea2f --- /dev/null +++ b/apps/api-ts/src/domains/providers/actions/confirm-tokens.ts @@ -0,0 +1,81 @@ +import { randomUUID } from "node:crypto"; +import { + normalizeProviderActionOptions, + normalizeProviderActionPaths, +} from "./selection.js"; +import type { + ProviderId, + ProviderSessionAction, + ProviderSessionActionOptions, +} from "../types.js"; + +type ProviderActionTokenEntry = { + provider: ProviderId; + action: ProviderSessionAction; + paths: string[]; + backup_before_delete: boolean; + backup_root: string; + expires_at: number; +}; + +const PROVIDER_ACTION_TOKEN_TTL_MS = 10 * 60_000; +const providerActionTokenCache = new Map(); + +export function pruneProviderActionTokens(now = Date.now()) { + for (const [token, entry] of providerActionTokenCache.entries()) { + if (entry.expires_at <= now) providerActionTokenCache.delete(token); + } +} + +export function issueProviderActionConfirmToken( + provider: ProviderId, + action: ProviderSessionAction, + filePaths: string[], + options?: ProviderSessionActionOptions, +): string { + const normalized = normalizeProviderActionPaths(filePaths); + const normalizedOptions = normalizeProviderActionOptions(options); + const token = `PROVIDER-${randomUUID().replace(/-/g, "").slice(0, 12).toUpperCase()}`; + providerActionTokenCache.set(token, { + provider, + action, + paths: normalized, + backup_before_delete: normalizedOptions.backup_before_delete, + backup_root: normalizedOptions.backup_root, + expires_at: Date.now() + PROVIDER_ACTION_TOKEN_TTL_MS, + }); + return token; +} + +export function consumeProviderActionConfirmToken( + token: string, + provider: ProviderId, + action: ProviderSessionAction, + filePaths: string[], + options?: ProviderSessionActionOptions, +): { ok: boolean; reason: string } { + pruneProviderActionTokens(); + const normalizedOptions = normalizeProviderActionOptions(options); + const key = String(token || "").trim(); + if (!key) return { ok: false, reason: "missing-confirm-token" }; + const entry = providerActionTokenCache.get(key); + if (!entry) return { ok: false, reason: "invalid-confirm-token" }; + if (entry.expires_at <= Date.now()) { + providerActionTokenCache.delete(key); + return { ok: false, reason: "expired-confirm-token" }; + } + const normalized = normalizeProviderActionPaths(filePaths); + const sameProvider = entry.provider === provider; + const sameAction = entry.action === action; + const sameOptions = + entry.backup_before_delete === normalizedOptions.backup_before_delete && + entry.backup_root === normalizedOptions.backup_root; + const samePaths = + entry.paths.length === normalized.length && + entry.paths.every((item, idx) => item === normalized[idx]); + if (!sameProvider || !sameAction || !sameOptions || !samePaths) { + return { ok: false, reason: "confirm-token-scope-mismatch" }; + } + providerActionTokenCache.delete(key); + return { ok: true, reason: "" }; +} diff --git a/apps/api-ts/src/domains/providers/actions/execution.ts b/apps/api-ts/src/domains/providers/actions/execution.ts new file mode 100644 index 0000000..b5d875b --- /dev/null +++ b/apps/api-ts/src/domains/providers/actions/execution.ts @@ -0,0 +1,134 @@ +import path from "node:path"; +import { copyFile, mkdir, stat, unlink } from "node:fs/promises"; + +import { + resolveArchivedSessionRestoreTarget, + resolveArchivedSessionRoot, + resolveArchivedSessionStoreTarget, +} from "./archives.js"; +import type { + ProviderId, +} from "../types.js"; + +export type ProviderActionExecutionFailure = { + file_path: string; + step: string; + error: string; +}; + +export async function runArchiveProviderAction( + provider: ProviderId, + valid: string[], +): Promise<{ + applied: number; + archivedTo: string | null; + failed: ProviderActionExecutionFailure[]; +}> { + let applied = 0; + const failed: ProviderActionExecutionFailure[] = []; + const archivedTo = resolveArchivedSessionRoot(provider); + for (const sourcePath of valid) { + const targetPath = await resolveArchivedSessionStoreTarget(provider, sourcePath); + if (!targetPath) { + failed.push({ + file_path: sourcePath, + step: "archive_local", + error: "no-archived-session-target", + }); + continue; + } + try { + await stat(targetPath); + failed.push({ + file_path: sourcePath, + step: "archive_local", + error: "target-already-exists", + }); + continue; + } catch { + // Missing target is expected before archive. + } + try { + await mkdir(path.dirname(targetPath), { recursive: true }); + await copyFile(sourcePath, targetPath); + await unlink(sourcePath); + applied += 1; + } catch (error) { + failed.push({ + file_path: sourcePath, + step: "archive_local", + error: String(error), + }); + } + } + return { applied, archivedTo, failed }; +} + +export async function runUnarchiveProviderAction( + provider: ProviderId, + valid: string[], +): Promise<{ + applied: number; + failed: ProviderActionExecutionFailure[]; +}> { + let applied = 0; + const failed: ProviderActionExecutionFailure[] = []; + for (const sourcePath of valid) { + const targetPath = await resolveArchivedSessionRestoreTarget(provider, sourcePath); + if (!targetPath) { + failed.push({ + file_path: sourcePath, + step: "unarchive_local", + error: "not-an-archived-session", + }); + continue; + } + try { + await stat(targetPath); + failed.push({ + file_path: sourcePath, + step: "unarchive_local", + error: "target-already-exists", + }); + continue; + } catch { + // Missing target is expected before restore. + } + try { + await mkdir(path.dirname(targetPath), { recursive: true }); + await copyFile(sourcePath, targetPath); + await unlink(sourcePath); + applied += 1; + } catch (error) { + failed.push({ + file_path: sourcePath, + step: "unarchive_local", + error: String(error), + }); + } + } + return { applied, failed }; +} + +export async function runDeleteProviderAction( + deleteTargets: string[], +): Promise<{ + applied: number; + failed: ProviderActionExecutionFailure[]; +}> { + let applied = 0; + const failed: ProviderActionExecutionFailure[] = []; + for (const sourcePath of deleteTargets) { + try { + await unlink(sourcePath); + applied += 1; + } catch (error) { + failed.push({ + file_path: sourcePath, + step: "delete_local", + error: String(error), + }); + } + } + return { applied, failed }; +} diff --git a/apps/api-ts/src/domains/providers/actions/responses.ts b/apps/api-ts/src/domains/providers/actions/responses.ts new file mode 100644 index 0000000..0121b31 --- /dev/null +++ b/apps/api-ts/src/domains/providers/actions/responses.ts @@ -0,0 +1,159 @@ +import type { ProviderId, ProviderSessionAction } from "../types.js"; +import type { NormalizedProviderActionOptions } from "./selection.js"; +import type { ProviderActionSkippedTarget } from "./targets.js"; +import type { ProviderActionExecutionFailure } from "./execution.js"; + +type ProviderActionResponseBase = { + provider: ProviderId; + action: ProviderSessionAction; + dryRun: boolean; + targetCount: number; + validCount: number; + selectionFingerprint: string; + options: NormalizedProviderActionOptions; + skipped: ProviderActionSkippedTarget[]; +}; + +type ProviderActionFailureResponseInput = ProviderActionResponseBase & { + error: string; + confirmTokenExpected?: string; +}; + +export function buildProviderActionFailureResponse({ + provider, + action, + dryRun, + targetCount, + validCount, + selectionFingerprint, + options, + skipped, + error, + confirmTokenExpected = "", +}: ProviderActionFailureResponseInput) { + return { + ok: false, + provider, + action, + dry_run: dryRun, + target_count: targetCount, + valid_count: validCount, + applied_count: 0, + confirm_token_expected: confirmTokenExpected, + confirm_token_accepted: false, + selection_fingerprint: selectionFingerprint, + backup_before_delete: options.backup_before_delete, + failure_summary: { + skipped_count: skipped.length, + failed_count: 0, + partial_failure: false, + }, + skipped, + error, + }; +} + +export function buildProviderActionPreviewResponse({ + provider, + action, + targetCount, + validCount, + selectionFingerprint, + options, + skipped, + confirmTokenExpected, +}: ProviderActionResponseBase & { + confirmTokenExpected: string; +}) { + return { + ok: true, + provider, + action, + dry_run: true, + target_count: targetCount, + valid_count: validCount, + applied_count: 0, + confirm_token_expected: confirmTokenExpected, + confirm_token_accepted: false, + selection_fingerprint: selectionFingerprint, + backup_before_delete: options.backup_before_delete, + failure_summary: { + skipped_count: skipped.length, + failed_count: 0, + partial_failure: false, + }, + skipped, + mode: "preview", + }; +} + +export function buildProviderActionAppliedResponse({ + provider, + action, + targetCount, + validCount, + selectionFingerprint, + options, + skipped, + failed, + applied, + backedUpCount, + backupId, + backupTo, + backupManifestPath, + shouldBackup, + backupStageFailedCount, + archivedTo, +}: { + provider: ProviderId; + action: ProviderSessionAction; + targetCount: number; + validCount: number; + selectionFingerprint: string; + options: NormalizedProviderActionOptions; + skipped: ProviderActionSkippedTarget[]; + failed: ProviderActionExecutionFailure[]; + applied: number; + backedUpCount: number; + backupId: string | null; + backupTo: string | null; + backupManifestPath: string | null; + shouldBackup: boolean; + backupStageFailedCount: number; + archivedTo: string | null; +}) { + return { + ok: failed.length === 0, + provider, + action, + dry_run: false, + target_count: targetCount, + valid_count: validCount, + applied_count: applied, + confirm_token_expected: "", + confirm_token_accepted: action === "backup_local" ? false : true, + selection_fingerprint: selectionFingerprint, + backup_before_delete: options.backup_before_delete, + backed_up_count: backedUpCount, + backup_id: backupId, + backup_to: backupTo, + backup_manifest_path: backupManifestPath, + backup_summary: shouldBackup + ? { + destination: backupTo, + manifest_path: backupManifestPath, + copied_count: backedUpCount, + failed_count: backupStageFailedCount, + } + : null, + failure_summary: { + skipped_count: skipped.length, + failed_count: failed.length, + partial_failure: failed.length > 0 && applied > 0, + }, + skipped, + failed, + archived_to: archivedTo, + mode: failed.length === 0 ? "applied" : applied > 0 ? "partial" : "failed", + }; +} diff --git a/apps/api-ts/src/domains/providers/actions/selection.ts b/apps/api-ts/src/domains/providers/actions/selection.ts new file mode 100644 index 0000000..60cda11 --- /dev/null +++ b/apps/api-ts/src/domains/providers/actions/selection.ts @@ -0,0 +1,63 @@ +import path from "node:path"; +import { createHash } from "node:crypto"; +import type { + ProviderId, + ProviderSessionAction, + ProviderSessionActionOptions, +} from "../types.js"; + +export type NormalizedProviderActionOptions = { + backup_before_delete: boolean; + backup_root: string; +}; + +export function normalizeProviderActionPaths(filePaths: string[]): string[] { + return Array.from( + new Set( + filePaths + .map((item) => path.resolve(String(item || "").trim())) + .filter(Boolean), + ), + ).sort(); +} + +export function normalizeProviderActionOptions( + options?: ProviderSessionActionOptions, +): NormalizedProviderActionOptions { + return { + backup_before_delete: Boolean(options?.backup_before_delete), + backup_root: String(options?.backup_root ?? "").trim(), + }; +} + +export function buildProviderActionToken( + provider: ProviderId, + action: ProviderSessionAction, + filePaths: string[], + options?: ProviderSessionActionOptions, +): string { + const digest = buildProviderActionFingerprint(provider, action, filePaths, options); + return `PROVIDER-${digest}`; +} + +export function buildProviderActionFingerprint( + provider: ProviderId, + action: ProviderSessionAction, + filePaths: string[], + options?: ProviderSessionActionOptions, +): string { + const normalizedOptions = normalizeProviderActionOptions(options); + const normalized = normalizeProviderActionPaths(filePaths); + const raw = JSON.stringify({ + provider, + action, + paths: normalized, + backup_before_delete: normalizedOptions.backup_before_delete, + backup_root: normalizedOptions.backup_root, + }); + return createHash("sha256") + .update(raw, "utf-8") + .digest("hex") + .slice(0, 12) + .toUpperCase(); +} diff --git a/apps/api-ts/src/domains/providers/actions/targets.ts b/apps/api-ts/src/domains/providers/actions/targets.ts new file mode 100644 index 0000000..24e1b4d --- /dev/null +++ b/apps/api-ts/src/domains/providers/actions/targets.ts @@ -0,0 +1,55 @@ +import { stat } from "node:fs/promises"; +import type { + ProviderId, +} from "../types.js"; + +export type ProviderActionSkippedTarget = { + file_path: string; + reason: string; +}; + +export type ProviderActionPathResolver = ( + provider: ProviderId, + filePath: string, +) => Promise; + +export function normalizeProviderActionTargetPaths(filePaths: string[]): string[] { + return Array.from( + new Set(filePaths.map((item) => String(item || "").trim()).filter(Boolean)), + ); +} + +export async function resolveProviderActionTargets( + provider: ProviderId, + uniquePaths: string[], + resolveAllowedProviderFilePath: ProviderActionPathResolver, +): Promise<{ + skipped: ProviderActionSkippedTarget[]; + valid: string[]; +}> { + const skipped: ProviderActionSkippedTarget[] = []; + const valid: string[] = []; + + for (const candidate of uniquePaths) { + const safePath = await resolveAllowedProviderFilePath(provider, candidate); + if (!safePath) { + skipped.push({ + file_path: candidate, + reason: "outside-provider-root-extension-or-realpath", + }); + continue; + } + try { + const st = await stat(safePath); + if (!st.isFile()) { + skipped.push({ file_path: candidate, reason: "not-a-file" }); + continue; + } + valid.push(safePath); + } catch { + skipped.push({ file_path: candidate, reason: "not-found" }); + } + } + + return { skipped, valid }; +} diff --git a/apps/api-ts/src/domains/providers/adapter-health.ts b/apps/api-ts/src/domains/providers/adapter-health.ts new file mode 100644 index 0000000..8c7521c --- /dev/null +++ b/apps/api-ts/src/domains/providers/adapter-health.ts @@ -0,0 +1,28 @@ +import { + GEMINI_ANTIGRAVITY_CONVERSATIONS_DIR, + GEMINI_HISTORY_DIR, + GEMINI_HOME, + GEMINI_TMP_DIR, +} from "../../lib/constants.js"; +import { countFilesRecursiveByExt, pathExists } from "../../lib/utils.js"; +import type { ProviderHealthEvidence } from "./types.js"; + +export async function getGeminiProviderHealth(): Promise { + const rootExists = await pathExists(GEMINI_HOME); + const sessionLogCount = + (await countFilesRecursiveByExt(GEMINI_TMP_DIR, [".jsonl", ".json"])) + + (await countFilesRecursiveByExt(GEMINI_HISTORY_DIR, [".jsonl", ".json"])) + + (await countFilesRecursiveByExt(GEMINI_ANTIGRAVITY_CONVERSATIONS_DIR, [".pb"])); + + return { + root_exists: rootExists, + session_log_count: sessionLogCount, + roots: [ + GEMINI_HOME, + GEMINI_TMP_DIR, + GEMINI_HISTORY_DIR, + GEMINI_ANTIGRAVITY_CONVERSATIONS_DIR, + ], + notes: "History, tmp, and checkpoint files.", + }; +} diff --git a/apps/api-ts/src/domains/providers/adapters.test.ts b/apps/api-ts/src/domains/providers/adapters.test.ts index dec8e42..19e2517 100644 --- a/apps/api-ts/src/domains/providers/adapters.test.ts +++ b/apps/api-ts/src/domains/providers/adapters.test.ts @@ -40,6 +40,13 @@ describe("provider adapters", () => { } }); + it("keeps provider health optional and separate from capabilities", () => { + expect(getProviderAdapter("gemini")?.health).toEqual(expect.any(Function)); + expect(getProviderAdapter("codex")?.health).toBeUndefined(); + expect("health" in (getProviderAdapter("codex") ?? {})).toBe(false); + expect("capabilities" in (getProviderAdapter("gemini") ?? {})).toBe(false); + }); + it("defines session locators without forcing DB-backed providers into fake paths", () => { const fileLocator: ProviderSessionLocator = { kind: "file", diff --git a/apps/api-ts/src/domains/providers/adapters.ts b/apps/api-ts/src/domains/providers/adapters.ts index 6ac16b8..c09ff1d 100644 --- a/apps/api-ts/src/domains/providers/adapters.ts +++ b/apps/api-ts/src/domains/providers/adapters.ts @@ -3,8 +3,9 @@ import { PROVIDER_LABELS, type ProviderId, } from "@threadlens/shared-contracts"; +import { getGeminiProviderHealth } from "./adapter-health.js"; import { providerRootSpecs, providerScanRootSpecs } from "./path-safety.js"; -import type { ProviderRootSpec } from "./types.js"; +import type { ProviderHealthEvidence, ProviderRootSpec } from "./types.js"; export type ProviderSessionLocator = | { kind: "file"; file_path: string } @@ -15,19 +16,29 @@ export type ProviderAdapter = { label: string; roots(): ProviderRootSpec[]; scanRoots?(): Promise; + health?(): Promise; }; +function providerHealth(provider: ProviderId): ProviderAdapter["health"] { + if (provider === "gemini") return getGeminiProviderHealth; + return undefined; +} + export const PROVIDER_ADAPTERS = Object.freeze( Object.fromEntries( - PROVIDER_IDS.map((id) => [ - id, - Object.freeze({ + PROVIDER_IDS.map((id) => { + const health = providerHealth(id); + return [ id, - label: PROVIDER_LABELS[id], - roots: () => providerRootSpecs(id), - scanRoots: () => providerScanRootSpecs(id), - } satisfies ProviderAdapter), - ]), + Object.freeze({ + id, + label: PROVIDER_LABELS[id], + roots: () => providerRootSpecs(id), + scanRoots: () => providerScanRootSpecs(id), + ...(health ? { health } : {}), + } satisfies ProviderAdapter), + ]; + }), ), ) as Readonly>; diff --git a/apps/api-ts/src/domains/providers/claude-title-detection.ts b/apps/api-ts/src/domains/providers/claude-title-detection.ts new file mode 100644 index 0000000..36d49fb --- /dev/null +++ b/apps/api-ts/src/domains/providers/claude-title-detection.ts @@ -0,0 +1,41 @@ +import { CLAUDE_PROJECTS_DIR } from "../../lib/constants.js"; +import { isRecord, readFileTail, safeJsonParse } from "../../lib/utils.js"; +import { isPathInsideRoot } from "./path-safety.js"; +import { normalizeDetectedTitle } from "./title-normalization.js"; + +export async function detectClaudeRenamedTitle( + filePath: string, + format: "jsonl" | "json" | "unknown", +): Promise<{ title: string; source: string | null } | null> { + if ( + format !== "jsonl" || + !isPathInsideRoot(filePath, CLAUDE_PROJECTS_DIR) + ) { + return null; + } + const tail = await readFileTail(filePath, 262_144); + if (!tail.text.trim()) return null; + let customTitle = ""; + let agentName = ""; + for (const line of tail.text.split(/\r?\n/)) { + const parsed = safeJsonParse(line); + if (!isRecord(parsed)) continue; + const type = String(parsed.type ?? ""); + if (type === "custom-title") { + const value = normalizeDetectedTitle(String(parsed.customTitle ?? "")); + if (value) customTitle = value; + continue; + } + if (type === "agent-name") { + const value = normalizeDetectedTitle(String(parsed.agentName ?? "")); + if (value) agentName = value; + } + } + if (customTitle) { + return { title: customTitle, source: "claude-custom-title" }; + } + if (agentName) { + return { title: agentName, source: "claude-agent-name" }; + } + return null; +} diff --git a/apps/api-ts/src/domains/providers/codex-title-map.ts b/apps/api-ts/src/domains/providers/codex-title-map.ts new file mode 100644 index 0000000..ef88290 --- /dev/null +++ b/apps/api-ts/src/domains/providers/codex-title-map.ts @@ -0,0 +1,80 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { CODEX_HOME } from "../../lib/constants.js"; +import { isRecord, safeJsonParse } from "../../lib/utils.js"; +import { normalizeDetectedTitle } from "./title-normalization.js"; + +type CodexTitleMapCacheEntry = { + expires_at: number; + map: Map; +}; + +let codexTitleMapCache: CodexTitleMapCacheEntry | null = null; + +export function invalidateCodexThreadTitleMapCache() { + codexTitleMapCache = null; +} + +function extractUuidFromText(text: string): string { + const match = String(text || "").match(/[0-9a-f]{8}-[0-9a-f-]{27,}/i); + return match ? match[0] : ""; +} + +export function extractCodexThreadIdFromSessionName(name: string): string { + return extractUuidFromText(name); +} + +export async function getCodexThreadTitleMap(): Promise> { + const now = Date.now(); + if (codexTitleMapCache && codexTitleMapCache.expires_at > now) { + return codexTitleMapCache.map; + } + const titleMap = new Map(); + const globalStateTitleIds = new Set(); + try { + const raw = await readFile( + path.join(CODEX_HOME, ".codex-global-state.json"), + "utf-8", + ); + const parsed = safeJsonParse(raw); + const blob = + isRecord(parsed) && isRecord(parsed["thread-titles"]) + ? (parsed["thread-titles"] as Record) + : null; + const titles = + blob && isRecord(blob.titles) + ? (blob.titles as Record) + : null; + if (titles) { + for (const [id, title] of Object.entries(titles)) { + const tid = extractUuidFromText(id); + const txt = normalizeDetectedTitle(String(title ?? "")); + if (tid && txt) { + titleMap.set(tid, txt); + globalStateTitleIds.add(tid); + } + } + } + } catch { + // no-op + } + try { + const raw = await readFile(path.join(CODEX_HOME, "session_index.jsonl"), "utf-8"); + for (const line of raw.split(/\r?\n/)) { + const parsed = safeJsonParse(line); + if (!isRecord(parsed)) continue; + const tid = extractUuidFromText(String(parsed.id ?? "")); + const txt = normalizeDetectedTitle(String(parsed.thread_name ?? "")); + if (tid && txt && !globalStateTitleIds.has(tid)) { + titleMap.set(tid, txt); + } + } + } catch { + // no-op + } + codexTitleMapCache = { + expires_at: now + 60_000, + map: titleMap, + }; + return titleMap; +} diff --git a/apps/api-ts/src/domains/providers/matrix.test.ts b/apps/api-ts/src/domains/providers/matrix.test.ts index 11fe914..8e18b77 100644 --- a/apps/api-ts/src/domains/providers/matrix.test.ts +++ b/apps/api-ts/src/domains/providers/matrix.test.ts @@ -1,8 +1,13 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@threadlens/shared-contracts", () => ({ - getProviderCapability: () => ({ safe_cleanup: true, hard_delete: true }), -})); +vi.mock("@threadlens/shared-contracts", async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + getProviderCapability: () => ({ safe_cleanup: true, hard_delete: true }), + }; +}); vi.mock("../../lib/constants.js", () => ({ CHAT_DIR: "/mock/chat", @@ -31,14 +36,6 @@ vi.mock("../../lib/utils.js", () => ({ vi.mock("./path-safety.js", () => ({ codexTranscriptSearchRoots: () => [{ root: "/mock/codex/sessions" }], - providerName: (provider: string) => - ({ - chatgpt: "ChatGPT", - claude: "Claude", - gemini: "Gemini", - copilot: "Copilot", - codex: "Codex", - })[provider] ?? provider, providerRootSpecs: (provider: string) => provider === "copilot" ? [ @@ -56,6 +53,7 @@ vi.mock("./probe.js", () => ({ isWorkspaceChatSessionPath: () => true, })); +import { listProviderAdapters } from "./adapters.js"; import { getProviderMatrixTs, invalidateProviderMatrixCache } from "./matrix.js"; describe("provider matrix notes", () => { @@ -102,6 +100,17 @@ describe("provider matrix notes", () => { ); }); + it("keeps matrix rows aligned with the provider adapter registry", async () => { + const data = await getProviderMatrixTs({ forceRefresh: true }); + + expect(data.providers.map((provider) => provider.provider)).toEqual( + listProviderAdapters().map((adapter) => adapter.id), + ); + expect(data.providers.map((provider) => provider.name)).toEqual( + listProviderAdapters().map((adapter) => adapter.label), + ); + }); + it("counts Copilot matrix sessions with the same roots used by provider sessions", async () => { const data = await getProviderMatrixTs({ forceRefresh: true }); const copilot = data.providers.find((provider) => provider.provider === "copilot"); @@ -109,4 +118,24 @@ describe("provider matrix notes", () => { expect(copilot?.evidence.session_log_count).toBe(3); expect(copilot?.evidence.roots).toContain("/mock/backups/copilot"); }); + + it("uses adapter health evidence for Gemini without changing the matrix shape", async () => { + const data = await getProviderMatrixTs({ forceRefresh: true }); + const gemini = data.providers.find((provider) => provider.provider === "gemini"); + + expect(gemini).toMatchObject({ + provider: "gemini", + status: "active", + evidence: { + roots: [ + "/mock/gemini", + "/mock/gemini/tmp", + "/mock/gemini/history", + "/mock/gemini/checkpoints", + ], + session_log_count: 3, + notes: "History, tmp, and checkpoint files.", + }, + }); + }); }); diff --git a/apps/api-ts/src/domains/providers/matrix.ts b/apps/api-ts/src/domains/providers/matrix.ts index 2fcd2f6..bf712f4 100644 --- a/apps/api-ts/src/domains/providers/matrix.ts +++ b/apps/api-ts/src/domains/providers/matrix.ts @@ -1,39 +1,14 @@ -import path from "node:path"; -import { getProviderCapability, type ProviderId } from "@threadlens/shared-contracts"; import { - CHAT_DIR, - CODEX_HOME, - CLAUDE_HOME, - CLAUDE_PROJECTS_DIR, - CLAUDE_TRANSCRIPTS_DIR, - COPILOT_CURSOR_GLOBAL, - COPILOT_CURSOR_WORKSPACE_STORAGE, - COPILOT_VSCODE_GLOBAL, - COPILOT_VSCODE_WORKSPACE_STORAGE, - GEMINI_ANTIGRAVITY_CONVERSATIONS_DIR, - GEMINI_HISTORY_DIR, - GEMINI_HOME, - GEMINI_TMP_DIR, -} from "../../lib/constants.js"; -import { - countFilesRecursiveByExt, - countJsonlFilesRecursive, nowIsoUtc, - pathExists, - quickFileCount, - walkFilesByExt, } from "../../lib/utils.js"; import { - codexTranscriptSearchRoots, - providerName, - providerRootSpecs, - providerScanRootSpecs, -} from "./path-safety.js"; + buildProviderMatrixProviders, +} from "./matrix/provider-entries.js"; +import { readProviderMatrixSignals } from "./matrix/signals.js"; import { - isCopilotGlobalSessionLikeFile, - isWorkspaceChatSessionPath, -} from "./probe.js"; -import type { ProviderMatrixData, ProviderStatus } from "./types.js"; + buildProviderMatrixSummary, +} from "./matrix/summary.js"; +import type { ProviderMatrixData } from "./types.js"; type ProviderMatrixCacheEntry = { expires_at: number; @@ -41,250 +16,20 @@ type ProviderMatrixCacheEntry = { }; const PROVIDER_MATRIX_CACHE_TTL_MS = 30_000; -const COPILOT_MATRIX_SESSION_SCAN_LIMIT = 5_000; let providerMatrixCache: ProviderMatrixCacheEntry | null = null; let providerMatrixInflight: Promise | null = null; -function supportsProviderCleanup(provider: ProviderId): boolean { - return getProviderCapability(provider).safe_cleanup; -} - -function supportsProviderHardDelete(provider: ProviderId): boolean { - return getProviderCapability(provider).hard_delete; -} - -export function providerStatus( - rootExists: boolean, - sessionLogs: number, -): ProviderStatus { - if (sessionLogs > 0) return "active"; - if (rootExists) return "detected"; - return "missing"; -} - -export function capabilityLevel( - status: ProviderStatus, - safeCleanup: boolean, -): "full" | "read-only" | "unavailable" { - if (safeCleanup) return "full"; - if (status !== "missing") return "read-only"; - return "unavailable"; -} +export { capabilityLevel, providerStatus } from "./matrix/summary.js"; export function invalidateProviderMatrixCache() { providerMatrixCache = null; } -function shouldIncludeCopilotMatrixSessionFile(source: string, filePath: string): boolean { - if (source === "cleanup_backups" && path.basename(filePath) === "_manifest.json") { - return false; - } - if (source === "vscode_workspace_chats" || source === "cursor_workspace_chats") { - return isWorkspaceChatSessionPath(filePath); - } - if (source === "vscode_global" || source === "cursor_global") { - return isCopilotGlobalSessionLikeFile(filePath); - } - return true; -} - -async function countCopilotMatrixSessionFiles() { - const roots = providerRootSpecs("copilot"); - const counts = await Promise.all( - roots.map(async (spec) => { - const files = await walkFilesByExt( - spec.root, - spec.exts, - COPILOT_MATRIX_SESSION_SCAN_LIMIT, - ); - return files.filter((filePath) => - shouldIncludeCopilotMatrixSessionFile(spec.source, filePath), - ).length; - }), - ); - return counts.reduce((sum, count) => sum + count, 0); -} - async function buildProviderMatrixData(): Promise { - const codexHomes = Array.from( - new Set(codexTranscriptSearchRoots().map((spec) => path.dirname(spec.root))), - ); - const codexRootExists = await pathExists(CODEX_HOME); - const chatGptRootExists = await pathExists(CHAT_DIR); - const claudeRootExists = await pathExists(CLAUDE_HOME); - const geminiRootExists = await pathExists(GEMINI_HOME); - const copilotVsCodeExists = await pathExists(COPILOT_VSCODE_GLOBAL); - const copilotCursorExists = await pathExists(COPILOT_CURSOR_GLOBAL); - const copilotVsCodeWorkspaceExists = await pathExists( - COPILOT_VSCODE_WORKSPACE_STORAGE, - ); - const copilotCursorWorkspaceExists = await pathExists( - COPILOT_CURSOR_WORKSPACE_STORAGE, - ); - const chatGptConversationRoots = await providerScanRootSpecs("chatgpt"); - - const codexSessionLogs = - (await countJsonlFilesRecursive(path.join(CODEX_HOME, "sessions"))) + - (await countJsonlFilesRecursive(path.join(CODEX_HOME, "archived_sessions"))); - const claudeSessionLogs = - (await countFilesRecursiveByExt(CLAUDE_PROJECTS_DIR, [".jsonl", ".json"])) + - (await countFilesRecursiveByExt(CLAUDE_TRANSCRIPTS_DIR, [".jsonl", ".json"])); - const geminiSessionLogs = - (await countFilesRecursiveByExt(GEMINI_TMP_DIR, [".jsonl", ".json"])) + - (await countFilesRecursiveByExt(GEMINI_HISTORY_DIR, [".jsonl", ".json"])) + - (await countFilesRecursiveByExt(GEMINI_ANTIGRAVITY_CONVERSATIONS_DIR, [".pb"])); - const chatGptSessionLogs = ( - await Promise.all( - chatGptConversationRoots.map((spec) => quickFileCount(spec.root)), - ) - ).reduce((sum, count) => sum + count, 0); - const copilotSignalFiles = await countCopilotMatrixSessionFiles(); - - const codexStatus = providerStatus(codexRootExists, codexSessionLogs); - const chatGptStatus = providerStatus(chatGptRootExists, chatGptSessionLogs); - const claudeStatus = providerStatus(claudeRootExists, claudeSessionLogs); - const geminiStatus = providerStatus(geminiRootExists, geminiSessionLogs); - const copilotStatus = providerStatus( - copilotVsCodeExists || - copilotCursorExists || - copilotVsCodeWorkspaceExists || - copilotCursorWorkspaceExists, - copilotSignalFiles, - ); - - const providers = [ - { - provider: "codex" as ProviderId, - name: "Codex", - status: codexStatus, - capability_level: capabilityLevel(codexStatus, supportsProviderCleanup("codex")), - capabilities: { - read_sessions: true, - analyze_context: true, - safe_cleanup: supportsProviderCleanup("codex"), - hard_delete: supportsProviderHardDelete("codex"), - }, - evidence: { - roots: codexHomes, - session_log_count: codexSessionLogs, - notes: "Thread logs, pinned state, and global state.", - }, - }, - { - provider: "chatgpt" as ProviderId, - name: providerName("chatgpt"), - status: chatGptStatus, - capability_level: capabilityLevel( - chatGptStatus, - supportsProviderCleanup("chatgpt"), - ), - capabilities: { - read_sessions: chatGptRootExists, - analyze_context: chatGptSessionLogs > 0, - safe_cleanup: supportsProviderCleanup("chatgpt"), - hard_delete: supportsProviderHardDelete("chatgpt"), - }, - evidence: { - roots: [CHAT_DIR], - session_log_count: chatGptSessionLogs, - notes: "Desktop cache and conversation files.", - }, - }, - { - provider: "claude" as ProviderId, - name: providerName("claude"), - status: claudeStatus, - capability_level: capabilityLevel( - claudeStatus, - supportsProviderCleanup("claude") && claudeStatus !== "missing", - ), - capabilities: { - read_sessions: claudeRootExists, - analyze_context: claudeSessionLogs > 0, - safe_cleanup: supportsProviderCleanup("claude") && claudeStatus !== "missing", - hard_delete: - supportsProviderHardDelete("claude") && claudeStatus !== "missing", - }, - evidence: { - roots: [CLAUDE_HOME, CLAUDE_PROJECTS_DIR, CLAUDE_TRANSCRIPTS_DIR], - session_log_count: claudeSessionLogs, - notes: "Session and transcript files.", - }, - }, - { - provider: "gemini" as ProviderId, - name: providerName("gemini"), - status: geminiStatus, - capability_level: capabilityLevel( - geminiStatus, - supportsProviderCleanup("gemini") && geminiStatus !== "missing", - ), - capabilities: { - read_sessions: geminiRootExists, - analyze_context: geminiSessionLogs > 0, - safe_cleanup: supportsProviderCleanup("gemini") && geminiStatus !== "missing", - hard_delete: - supportsProviderHardDelete("gemini") && geminiStatus !== "missing", - }, - evidence: { - roots: [ - GEMINI_HOME, - GEMINI_TMP_DIR, - GEMINI_HISTORY_DIR, - GEMINI_ANTIGRAVITY_CONVERSATIONS_DIR, - ], - session_log_count: geminiSessionLogs, - notes: "History, tmp, and checkpoint files.", - }, - }, - { - provider: "copilot" as ProviderId, - name: providerName("copilot"), - status: copilotStatus, - capability_level: capabilityLevel( - copilotStatus, - supportsProviderCleanup("copilot") && copilotStatus !== "missing", - ), - capabilities: { - read_sessions: - copilotVsCodeExists || - copilotCursorExists || - copilotVsCodeWorkspaceExists || - copilotCursorWorkspaceExists, - analyze_context: copilotSignalFiles > 0, - safe_cleanup: - supportsProviderCleanup("copilot") && copilotStatus !== "missing", - hard_delete: - supportsProviderHardDelete("copilot") && copilotStatus !== "missing", - }, - evidence: { - roots: [ - COPILOT_VSCODE_GLOBAL, - COPILOT_CURSOR_GLOBAL, - COPILOT_VSCODE_WORKSPACE_STORAGE, - COPILOT_CURSOR_WORKSPACE_STORAGE, - ...providerRootSpecs("copilot") - .filter((spec) => spec.source === "cleanup_backups") - .map((spec) => spec.root), - ], - session_log_count: copilotSignalFiles, - notes: "Workspace chat files and editor traces.", - }, - }, - ]; + const signals = await readProviderMatrixSignals(); + const providers = buildProviderMatrixProviders(signals); - const summary = { - total: providers.length, - active: providers.filter((x) => x.status === "active").length, - detected: providers.filter((x) => x.status !== "missing").length, - read_analyze_ready: providers.filter( - (x) => x.capabilities.read_sessions && x.capabilities.analyze_context, - ).length, - safe_cleanup_ready: providers.filter((x) => x.capabilities.safe_cleanup) - .length, - hard_delete_ready: providers.filter((x) => x.capabilities.hard_delete) - .length, - }; + const summary = buildProviderMatrixSummary(providers); return { generated_at: nowIsoUtc(), diff --git a/apps/api-ts/src/domains/providers/matrix/adapter-access.ts b/apps/api-ts/src/domains/providers/matrix/adapter-access.ts new file mode 100644 index 0000000..7e73401 --- /dev/null +++ b/apps/api-ts/src/domains/providers/matrix/adapter-access.ts @@ -0,0 +1,35 @@ +import { + getProviderCapability, + type ProviderId, +} from "@threadlens/shared-contracts"; + +import { + getProviderAdapter, +} from "../adapters.js"; + +export function supportsProviderCleanup(provider: ProviderId): boolean { + return getProviderCapability(provider).safe_cleanup; +} + +export function supportsProviderHardDelete(provider: ProviderId): boolean { + return getProviderCapability(provider).hard_delete; +} + +export function providerLabel(provider: ProviderId): string { + return getProviderAdapter(provider)?.label ?? provider; +} + +export function providerRoots(provider: ProviderId) { + return getProviderAdapter(provider)?.roots() ?? []; +} + +export async function providerScanRoots(provider: ProviderId) { + const adapter = getProviderAdapter(provider); + if (!adapter) return []; + return adapter.scanRoots ? adapter.scanRoots() : adapter.roots(); +} + +export async function providerHealth(provider: ProviderId) { + const adapter = getProviderAdapter(provider); + return adapter?.health ? adapter.health() : null; +} diff --git a/apps/api-ts/src/domains/providers/matrix/copilot.ts b/apps/api-ts/src/domains/providers/matrix/copilot.ts new file mode 100644 index 0000000..5926757 --- /dev/null +++ b/apps/api-ts/src/domains/providers/matrix/copilot.ts @@ -0,0 +1,45 @@ +import path from "node:path"; + +import { + walkFilesByExt, +} from "../../../lib/utils.js"; +import { + isCopilotGlobalSessionLikeFile, + isWorkspaceChatSessionPath, +} from "../probe.js"; +import type { + ProviderRootSpec, +} from "../types.js"; + +const COPILOT_MATRIX_SESSION_SCAN_LIMIT = 5_000; + +function shouldIncludeCopilotMatrixSessionFile(source: string, filePath: string): boolean { + if (source === "cleanup_backups" && path.basename(filePath) === "_manifest.json") { + return false; + } + if (source === "vscode_workspace_chats" || source === "cursor_workspace_chats") { + return isWorkspaceChatSessionPath(filePath); + } + if (source === "vscode_global" || source === "cursor_global") { + return isCopilotGlobalSessionLikeFile(filePath); + } + return true; +} + +export async function countCopilotMatrixSessionFiles( + roots: ProviderRootSpec[], +): Promise { + const counts = await Promise.all( + roots.map(async (spec) => { + const files = await walkFilesByExt( + spec.root, + spec.exts, + COPILOT_MATRIX_SESSION_SCAN_LIMIT, + ); + return files.filter((filePath) => + shouldIncludeCopilotMatrixSessionFile(spec.source, filePath), + ).length; + }), + ); + return counts.reduce((sum, count) => sum + count, 0); +} diff --git a/apps/api-ts/src/domains/providers/matrix/provider-entries.ts b/apps/api-ts/src/domains/providers/matrix/provider-entries.ts new file mode 100644 index 0000000..3c16b1c --- /dev/null +++ b/apps/api-ts/src/domains/providers/matrix/provider-entries.ts @@ -0,0 +1,148 @@ +import type { ProviderId } from "@threadlens/shared-contracts"; +import { + CHAT_DIR, + CLAUDE_HOME, + CLAUDE_PROJECTS_DIR, + CLAUDE_TRANSCRIPTS_DIR, + COPILOT_CURSOR_GLOBAL, + COPILOT_CURSOR_WORKSPACE_STORAGE, + COPILOT_VSCODE_GLOBAL, + COPILOT_VSCODE_WORKSPACE_STORAGE, +} from "../../../lib/constants.js"; +import { + providerLabel, + supportsProviderCleanup, + supportsProviderHardDelete, +} from "./adapter-access.js"; +import { + capabilityLevel, + providerStatus, +} from "./summary.js"; +import type { + ProviderMatrixSignals, +} from "./signals.js"; +import type { + ProviderMatrixData, +} from "../types.js"; + +export function buildProviderMatrixProviders( + signals: ProviderMatrixSignals, +): ProviderMatrixData["providers"] { + const codexStatus = providerStatus(signals.codexRootExists, signals.codexSessionLogs); + const chatGptStatus = providerStatus(signals.chatGptRootExists, signals.chatGptSessionLogs); + const claudeStatus = providerStatus(signals.claudeRootExists, signals.claudeSessionLogs); + const geminiStatus = providerStatus(signals.geminiRootExists, signals.geminiSessionLogs); + const copilotStatus = providerStatus(signals.copilotRootExists, signals.copilotSignalFiles); + + return [ + { + provider: "codex" as ProviderId, + name: providerLabel("codex"), + status: codexStatus, + capability_level: capabilityLevel(codexStatus, supportsProviderCleanup("codex")), + capabilities: { + read_sessions: true, + analyze_context: true, + safe_cleanup: supportsProviderCleanup("codex"), + hard_delete: supportsProviderHardDelete("codex"), + }, + evidence: { + roots: signals.codexHomes, + session_log_count: signals.codexSessionLogs, + notes: "Thread logs, pinned state, and global state.", + }, + }, + { + provider: "chatgpt" as ProviderId, + name: providerLabel("chatgpt"), + status: chatGptStatus, + capability_level: capabilityLevel( + chatGptStatus, + supportsProviderCleanup("chatgpt"), + ), + capabilities: { + read_sessions: signals.chatGptRootExists, + analyze_context: signals.chatGptSessionLogs > 0, + safe_cleanup: supportsProviderCleanup("chatgpt"), + hard_delete: supportsProviderHardDelete("chatgpt"), + }, + evidence: { + roots: [CHAT_DIR], + session_log_count: signals.chatGptSessionLogs, + notes: "Desktop cache and conversation files.", + }, + }, + { + provider: "claude" as ProviderId, + name: providerLabel("claude"), + status: claudeStatus, + capability_level: capabilityLevel( + claudeStatus, + supportsProviderCleanup("claude") && claudeStatus !== "missing", + ), + capabilities: { + read_sessions: signals.claudeRootExists, + analyze_context: signals.claudeSessionLogs > 0, + safe_cleanup: supportsProviderCleanup("claude") && claudeStatus !== "missing", + hard_delete: + supportsProviderHardDelete("claude") && claudeStatus !== "missing", + }, + evidence: { + roots: [CLAUDE_HOME, CLAUDE_PROJECTS_DIR, CLAUDE_TRANSCRIPTS_DIR], + session_log_count: signals.claudeSessionLogs, + notes: "Session and transcript files.", + }, + }, + { + provider: "gemini" as ProviderId, + name: providerLabel("gemini"), + status: geminiStatus, + capability_level: capabilityLevel( + geminiStatus, + supportsProviderCleanup("gemini") && geminiStatus !== "missing", + ), + capabilities: { + read_sessions: signals.geminiRootExists, + analyze_context: signals.geminiSessionLogs > 0, + safe_cleanup: supportsProviderCleanup("gemini") && geminiStatus !== "missing", + hard_delete: + supportsProviderHardDelete("gemini") && geminiStatus !== "missing", + }, + evidence: { + roots: signals.geminiRoots, + session_log_count: signals.geminiSessionLogs, + notes: signals.geminiNotes, + }, + }, + { + provider: "copilot" as ProviderId, + name: providerLabel("copilot"), + status: copilotStatus, + capability_level: capabilityLevel( + copilotStatus, + supportsProviderCleanup("copilot") && copilotStatus !== "missing", + ), + capabilities: { + read_sessions: signals.copilotRootExists, + analyze_context: signals.copilotSignalFiles > 0, + safe_cleanup: + supportsProviderCleanup("copilot") && copilotStatus !== "missing", + hard_delete: + supportsProviderHardDelete("copilot") && copilotStatus !== "missing", + }, + evidence: { + roots: [ + COPILOT_VSCODE_GLOBAL, + COPILOT_CURSOR_GLOBAL, + COPILOT_VSCODE_WORKSPACE_STORAGE, + COPILOT_CURSOR_WORKSPACE_STORAGE, + ...signals.copilotProviderRoots + .filter((spec) => spec.source === "cleanup_backups") + .map((spec) => spec.root), + ], + session_log_count: signals.copilotSignalFiles, + notes: "Workspace chat files and editor traces.", + }, + }, + ]; +} diff --git a/apps/api-ts/src/domains/providers/matrix/signals.ts b/apps/api-ts/src/domains/providers/matrix/signals.ts new file mode 100644 index 0000000..03f1ae9 --- /dev/null +++ b/apps/api-ts/src/domains/providers/matrix/signals.ts @@ -0,0 +1,103 @@ +import path from "node:path"; +import { + CHAT_DIR, + CODEX_HOME, + CLAUDE_HOME, + CLAUDE_PROJECTS_DIR, + CLAUDE_TRANSCRIPTS_DIR, + COPILOT_CURSOR_GLOBAL, + COPILOT_CURSOR_WORKSPACE_STORAGE, + COPILOT_VSCODE_GLOBAL, + COPILOT_VSCODE_WORKSPACE_STORAGE, +} from "../../../lib/constants.js"; +import { + countFilesRecursiveByExt, + countJsonlFilesRecursive, + pathExists, + quickFileCount, +} from "../../../lib/utils.js"; +import { + providerHealth, + providerRoots, + providerScanRoots, +} from "./adapter-access.js"; +import { + countCopilotMatrixSessionFiles, +} from "./copilot.js"; +import { codexTranscriptSearchRoots } from "../path-safety.js"; +import type { + ProviderRootSpec, +} from "../types.js"; + +export type ProviderMatrixSignals = { + codexHomes: string[]; + codexRootExists: boolean; + chatGptRootExists: boolean; + claudeRootExists: boolean; + geminiRootExists: boolean; + geminiSessionLogs: number; + geminiRoots: string[]; + geminiNotes: string; + copilotProviderRoots: ProviderRootSpec[]; + copilotRootExists: boolean; + codexSessionLogs: number; + chatGptSessionLogs: number; + claudeSessionLogs: number; + copilotSignalFiles: number; +}; + +export async function readProviderMatrixSignals(): Promise { + const codexHomes = Array.from( + new Set(codexTranscriptSearchRoots().map((spec) => path.dirname(spec.root))), + ); + const codexRootExists = await pathExists(CODEX_HOME); + const chatGptRootExists = await pathExists(CHAT_DIR); + const claudeRootExists = await pathExists(CLAUDE_HOME); + const copilotVsCodeExists = await pathExists(COPILOT_VSCODE_GLOBAL); + const copilotCursorExists = await pathExists(COPILOT_CURSOR_GLOBAL); + const copilotVsCodeWorkspaceExists = await pathExists( + COPILOT_VSCODE_WORKSPACE_STORAGE, + ); + const copilotCursorWorkspaceExists = await pathExists( + COPILOT_CURSOR_WORKSPACE_STORAGE, + ); + const chatGptConversationRoots = await providerScanRoots("chatgpt"); + const geminiHealth = await providerHealth("gemini"); + const copilotProviderRoots = providerRoots("copilot"); + + const codexSessionLogs = + (await countJsonlFilesRecursive(path.join(CODEX_HOME, "sessions"))) + + (await countJsonlFilesRecursive(path.join(CODEX_HOME, "archived_sessions"))); + const claudeSessionLogs = + (await countFilesRecursiveByExt(CLAUDE_PROJECTS_DIR, [".jsonl", ".json"])) + + (await countFilesRecursiveByExt(CLAUDE_TRANSCRIPTS_DIR, [".jsonl", ".json"])); + const geminiRootExists = geminiHealth?.root_exists ?? false; + const geminiSessionLogs = geminiHealth?.session_log_count ?? 0; + const chatGptSessionLogs = ( + await Promise.all( + chatGptConversationRoots.map((spec) => quickFileCount(spec.root)), + ) + ).reduce((sum, count) => sum + count, 0); + const copilotSignalFiles = await countCopilotMatrixSessionFiles(copilotProviderRoots); + + return { + codexHomes, + codexRootExists, + chatGptRootExists, + claudeRootExists, + geminiRootExists, + geminiSessionLogs, + geminiRoots: geminiHealth?.roots ?? [], + geminiNotes: geminiHealth?.notes ?? "History, tmp, and checkpoint files.", + copilotProviderRoots, + copilotRootExists: + copilotVsCodeExists || + copilotCursorExists || + copilotVsCodeWorkspaceExists || + copilotCursorWorkspaceExists, + codexSessionLogs, + chatGptSessionLogs, + claudeSessionLogs, + copilotSignalFiles, + }; +} diff --git a/apps/api-ts/src/domains/providers/matrix/summary.ts b/apps/api-ts/src/domains/providers/matrix/summary.ts new file mode 100644 index 0000000..ee72e70 --- /dev/null +++ b/apps/api-ts/src/domains/providers/matrix/summary.ts @@ -0,0 +1,41 @@ +import type { + ProviderMatrixData, + ProviderStatus, +} from "../types.js"; + +type ProviderMatrixProvider = ProviderMatrixData["providers"][number]; + +export function providerStatus( + rootExists: boolean, + sessionLogs: number, +): ProviderStatus { + if (sessionLogs > 0) return "active"; + if (rootExists) return "detected"; + return "missing"; +} + +export function capabilityLevel( + status: ProviderStatus, + safeCleanup: boolean, +): "full" | "read-only" | "unavailable" { + if (safeCleanup) return "full"; + if (status !== "missing") return "read-only"; + return "unavailable"; +} + +export function buildProviderMatrixSummary( + providers: ProviderMatrixProvider[], +): ProviderMatrixData["summary"] { + return { + total: providers.length, + active: providers.filter((x) => x.status === "active").length, + detected: providers.filter((x) => x.status !== "missing").length, + read_analyze_ready: providers.filter( + (x) => x.capabilities.read_sessions && x.capabilities.analyze_context, + ).length, + safe_cleanup_ready: providers.filter((x) => x.capabilities.safe_cleanup) + .length, + hard_delete_ready: providers.filter((x) => x.capabilities.hard_delete) + .length, + }; +} diff --git a/apps/api-ts/src/domains/providers/path-safety.ts b/apps/api-ts/src/domains/providers/path-safety.ts index b28b27e..7fdf403 100644 --- a/apps/api-ts/src/domains/providers/path-safety.ts +++ b/apps/api-ts/src/domains/providers/path-safety.ts @@ -1,4 +1,4 @@ -import { lstat, realpath, readdir } from "node:fs/promises"; +import { lstat, realpath } from "node:fs/promises"; import path from "node:path"; import { PROVIDER_IDS, @@ -6,39 +6,19 @@ import { type ProviderId, } from "@threadlens/shared-contracts"; import { - BACKUP_ROOT, CHAT_DIR, CODEX_HOME, - CLAUDE_HOME, - CLAUDE_PROJECTS_DIR, - CLAUDE_TRANSCRIPTS_DIR, - COPILOT_CURSOR_GLOBAL, - COPILOT_CURSOR_WORKSPACE_STORAGE, - COPILOT_VSCODE_GLOBAL, - COPILOT_VSCODE_WORKSPACE_STORAGE, - GEMINI_ANTIGRAVITY_CONVERSATIONS_DIR, - GEMINI_HISTORY_DIR, - GEMINI_HOME, - GEMINI_TMP_DIR, - HOME_DIR, } from "../../lib/constants.js"; -import { pathExists } from "../../lib/utils.js"; -import type { ProviderRootSpec } from "./types.js"; - -type ChatGptRootsCacheEntry = { - expires_at: number; - roots: ProviderRootSpec[]; -}; - -type CodexCwdBackupRootsCacheEntry = { - expires_at: number; - roots: ProviderRootSpec[]; -}; +import { + providerRootSpecs, + providerScanRootSpecs, +} from "./provider-roots.js"; -const CHATGPT_ROOT_DISCOVERY_TTL_MS = 45_000; -const CODEX_CWD_BACKUP_ROOT_DISCOVERY_TTL_MS = 60_000; -let chatGptRootsCache: ChatGptRootsCacheEntry | null = null; -let codexCwdBackupRootsCache: CodexCwdBackupRootsCacheEntry | null = null; +export { + codexTranscriptSearchRoots, + providerRootSpecs, + providerScanRootSpecs, +} from "./provider-roots.js"; export function providerName(provider: ProviderId): string { return PROVIDER_LABELS[provider] ?? provider; @@ -82,257 +62,6 @@ function isCodexCwdBackupPath(filePath: string): boolean { ); } -async function discoverChatGptConversationRoots(): Promise { - const now = Date.now(); - if (chatGptRootsCache && chatGptRootsCache.expires_at > now) { - return chatGptRootsCache.roots; - } - if (!(await pathExists(CHAT_DIR))) return []; - const out = new Map(); - const push = (source: string, root: string) => { - if (!out.has(root)) out.set(root, { source, root, exts: [".data"] }); - }; - - const topLevel = await readdir(CHAT_DIR, { withFileTypes: true }).catch( - () => [], - ); - - for (const entry of topLevel) { - if (!entry.isDirectory()) continue; - if (entry.name.startsWith("conversations-v3-")) { - push("conversations", path.join(CHAT_DIR, entry.name)); - continue; - } - const nestedRoot = path.join(CHAT_DIR, entry.name); - const nested = await readdir(nestedRoot, { withFileTypes: true }).catch( - () => [], - ); - for (const child of nested) { - if (!child.isDirectory()) continue; - if (!child.name.startsWith("conversations-v3-")) continue; - push(entry.name, path.join(nestedRoot, child.name)); - } - } - - const roots = Array.from(out.values()); - chatGptRootsCache = { - expires_at: now + CHATGPT_ROOT_DISCOVERY_TTL_MS, - roots, - }; - return roots; -} - -function providerActionBackupRoots(provider: ProviderId): string[] { - return Array.from( - new Set([ - path.join(BACKUP_ROOT, "provider_actions", provider), - path.join(CODEX_HOME, "local_cleanup_backups", "provider_actions", provider), - ]), - ); -} - -function providerBackupRootSpecs(provider: ProviderId): ProviderRootSpec[] { - const cleanupRoots = () => - providerActionBackupRoots(provider).map((root) => ({ - source: "cleanup_backups", - root, - exts: [".jsonl", ".json"], - })); - if (provider === "codex") { - return [ - { - source: "recovered_sessions", - root: path.join(CODEX_HOME, "recovered-sessions"), - exts: [".jsonl"], - }, - ...cleanupRoots().map((spec) => ({ ...spec, exts: [".jsonl"] })), - ]; - } - if (provider === "claude") { - return cleanupRoots(); - } - if (provider === "gemini") { - return cleanupRoots().map((spec) => ({ ...spec, exts: [".jsonl", ".json", ".pb"] })); - } - if (provider === "copilot") { - return cleanupRoots(); - } - return []; -} - -function providerArchivedRootSpec(provider: ProviderId): ProviderRootSpec | null { - if (provider === "codex") { - return { - source: "archived_sessions", - root: path.join(CODEX_HOME, "archived_sessions"), - exts: [".jsonl"], - }; - } - if (provider === "claude") { - return { - source: "archived_sessions", - root: path.join(CODEX_HOME, "archived_sessions", "claude"), - exts: [".jsonl", ".json"], - }; - } - if (provider === "gemini") { - return { - source: "archived_sessions", - root: path.join(CODEX_HOME, "archived_sessions", "gemini"), - exts: [".jsonl", ".json", ".pb"], - }; - } - if (provider === "copilot") { - return { - source: "archived_sessions", - root: path.join(CODEX_HOME, "archived_sessions", "copilot"), - exts: [".jsonl", ".json"], - }; - } - return null; -} - -export function providerRootSpecs(provider: ProviderId): ProviderRootSpec[] { - const archivedSpec = providerArchivedRootSpec(provider); - if (provider === "codex") { - return [ - { - source: "sessions", - root: path.join(CODEX_HOME, "sessions"), - exts: [".jsonl"], - }, - ...(archivedSpec ? [archivedSpec] : []), - ...providerBackupRootSpecs(provider), - ]; - } - if (provider === "chatgpt") { - return [{ source: "chat_cache", root: CHAT_DIR, exts: [".data"] }]; - } - if (provider === "claude") { - return [ - { source: "projects", root: CLAUDE_PROJECTS_DIR, exts: [".jsonl"] }, - { - source: "transcripts", - root: CLAUDE_TRANSCRIPTS_DIR, - exts: [".jsonl", ".json"], - }, - ...(archivedSpec ? [archivedSpec] : []), - ...providerBackupRootSpecs(provider), - ]; - } - if (provider === "gemini") { - return [ - { source: "tmp", root: GEMINI_TMP_DIR, exts: [".jsonl", ".json"] }, - { source: "history", root: GEMINI_HISTORY_DIR, exts: [".jsonl", ".json"] }, - { - source: "antigravity_conversations", - root: GEMINI_ANTIGRAVITY_CONVERSATIONS_DIR, - exts: [".pb"], - }, - ...(archivedSpec ? [archivedSpec] : []), - ...providerBackupRootSpecs(provider), - ]; - } - if (provider === "copilot") { - return [ - { - source: "vscode_global", - root: COPILOT_VSCODE_GLOBAL, - exts: [".jsonl", ".json"], - }, - { - source: "cursor_global", - root: COPILOT_CURSOR_GLOBAL, - exts: [".jsonl", ".json"], - }, - { - source: "vscode_workspace_chats", - root: COPILOT_VSCODE_WORKSPACE_STORAGE, - exts: [".json"], - }, - { - source: "cursor_workspace_chats", - root: COPILOT_CURSOR_WORKSPACE_STORAGE, - exts: [".json"], - }, - ...(archivedSpec ? [archivedSpec] : []), - ...providerBackupRootSpecs(provider), - ]; - } - return []; -} - -export function codexTranscriptSearchRoots(): ProviderRootSpec[] { - const homes = Array.from( - new Set([ - CODEX_HOME, - path.join(HOME_DIR, ".codex-cli"), - path.join(HOME_DIR, ".codex"), - ].filter(Boolean)), - ); - const roots: ProviderRootSpec[] = []; - roots.push({ - source: "cleanup_backups", - root: path.join(BACKUP_ROOT, "provider_actions", "codex"), - exts: [".jsonl"], - }); - for (const home of homes) { - roots.push({ - source: "sessions", - root: path.join(home, "sessions"), - exts: [".jsonl"], - }); - roots.push({ - source: "archived_sessions", - root: path.join(home, "archived_sessions"), - exts: [".jsonl"], - }); - roots.push({ - source: "recovered_sessions", - root: path.join(home, "recovered-sessions"), - exts: [".jsonl"], - }); - roots.push({ - source: "cleanup_backups", - root: path.join(home, "local_cleanup_backups", "provider_actions", "codex"), - exts: [".jsonl"], - }); - } - return roots; -} - -async function discoverCodexCwdBackupRoots(): Promise { - if (codexCwdBackupRootsCache && codexCwdBackupRootsCache.expires_at > Date.now()) { - return codexCwdBackupRootsCache.roots; - } - const entries = await readdir(CODEX_HOME, { withFileTypes: true }).catch(() => []); - const roots = entries - .filter((entry) => entry.isDirectory() && entry.name.startsWith("jsonl-cwd-backups-")) - .map((entry) => ({ - source: "cwd_backups", - root: path.join(CODEX_HOME, entry.name), - exts: [".jsonl"], - })); - codexCwdBackupRootsCache = { - expires_at: Date.now() + CODEX_CWD_BACKUP_ROOT_DISCOVERY_TTL_MS, - roots, - }; - return roots; -} - -export async function providerScanRootSpecs( - provider: ProviderId, -): Promise { - if (provider === "chatgpt") { - const discovered = await discoverChatGptConversationRoots(); - if (discovered.length > 0) return discovered; - return providerRootSpecs(provider); - } - if (provider !== "codex") return providerRootSpecs(provider); - const extraRoots = await discoverCodexCwdBackupRoots(); - return [...providerRootSpecs(provider), ...extraRoots]; -} - export function isAllowedProviderFilePath( provider: ProviderId, filePath: string, diff --git a/apps/api-ts/src/domains/providers/probe.test.ts b/apps/api-ts/src/domains/providers/probe.test.ts index 16afb8f..b1de920 100644 --- a/apps/api-ts/src/domains/providers/probe.test.ts +++ b/apps/api-ts/src/domains/providers/probe.test.ts @@ -12,6 +12,9 @@ describe("provider probe helpers", () => { expect(inferSessionId("/tmp/12345678-1234-1234-1234-1234567890ab.json")).toBe( "12345678-1234-1234-1234-1234567890ab", ); + expect(inferSessionId("C:\\Users\\hwan\\.codex\\sessions\\win-session.jsonl")).toBe( + "win-session", + ); }); it("treats binary provider caches as readable id hints", async () => { diff --git a/apps/api-ts/src/domains/providers/probe.ts b/apps/api-ts/src/domains/providers/probe.ts index 4e0dfe0..3468072 100644 --- a/apps/api-ts/src/domains/providers/probe.ts +++ b/apps/api-ts/src/domains/providers/probe.ts @@ -15,8 +15,9 @@ function inferFormat(filePath: string): "jsonl" | "json" | "unknown" { } export function inferSessionId(filePath: string): string { - const base = path.basename(filePath); - const ext = path.extname(base); + const normalizedPath = filePath.replace(/\\/g, "/"); + const base = path.posix.basename(normalizedPath); + const ext = path.posix.extname(base); if (!ext) return base; return base.slice(0, -ext.length); } diff --git a/apps/api-ts/src/domains/providers/provider-roots.ts b/apps/api-ts/src/domains/providers/provider-roots.ts new file mode 100644 index 0000000..02c0d0e --- /dev/null +++ b/apps/api-ts/src/domains/providers/provider-roots.ts @@ -0,0 +1,112 @@ +import path from "node:path"; +import type { + ProviderId, +} from "@threadlens/shared-contracts"; + +import { + CHAT_DIR, + CODEX_HOME, + CLAUDE_PROJECTS_DIR, + CLAUDE_TRANSCRIPTS_DIR, + COPILOT_CURSOR_GLOBAL, + COPILOT_CURSOR_WORKSPACE_STORAGE, + COPILOT_VSCODE_GLOBAL, + COPILOT_VSCODE_WORKSPACE_STORAGE, + GEMINI_ANTIGRAVITY_CONVERSATIONS_DIR, + GEMINI_HISTORY_DIR, + GEMINI_TMP_DIR, +} from "../../lib/constants.js"; +import { discoverChatGptConversationRoots } from "./roots/chatgpt.js"; +import { discoverCodexCwdBackupRoots } from "./roots/codex.js"; +import { + providerArchivedRootSpec, + providerBackupRootSpecs, +} from "./roots/backups.js"; +import type { + ProviderRootSpec, +} from "./types.js"; + +export { codexTranscriptSearchRoots } from "./roots/codex.js"; + +export function providerRootSpecs(provider: ProviderId): ProviderRootSpec[] { + const archivedSpec = providerArchivedRootSpec(provider); + if (provider === "codex") { + return [ + { + source: "sessions", + root: path.join(CODEX_HOME, "sessions"), + exts: [".jsonl"], + }, + ...(archivedSpec ? [archivedSpec] : []), + ...providerBackupRootSpecs(provider), + ]; + } + if (provider === "chatgpt") { + return [{ source: "chat_cache", root: CHAT_DIR, exts: [".data"] }]; + } + if (provider === "claude") { + return [ + { source: "projects", root: CLAUDE_PROJECTS_DIR, exts: [".jsonl"] }, + { + source: "transcripts", + root: CLAUDE_TRANSCRIPTS_DIR, + exts: [".jsonl", ".json"], + }, + ...(archivedSpec ? [archivedSpec] : []), + ...providerBackupRootSpecs(provider), + ]; + } + if (provider === "gemini") { + return [ + { source: "tmp", root: GEMINI_TMP_DIR, exts: [".jsonl", ".json"] }, + { source: "history", root: GEMINI_HISTORY_DIR, exts: [".jsonl", ".json"] }, + { + source: "antigravity_conversations", + root: GEMINI_ANTIGRAVITY_CONVERSATIONS_DIR, + exts: [".pb"], + }, + ...(archivedSpec ? [archivedSpec] : []), + ...providerBackupRootSpecs(provider), + ]; + } + if (provider === "copilot") { + return [ + { + source: "vscode_global", + root: COPILOT_VSCODE_GLOBAL, + exts: [".jsonl", ".json"], + }, + { + source: "cursor_global", + root: COPILOT_CURSOR_GLOBAL, + exts: [".jsonl", ".json"], + }, + { + source: "vscode_workspace_chats", + root: COPILOT_VSCODE_WORKSPACE_STORAGE, + exts: [".json"], + }, + { + source: "cursor_workspace_chats", + root: COPILOT_CURSOR_WORKSPACE_STORAGE, + exts: [".json"], + }, + ...(archivedSpec ? [archivedSpec] : []), + ...providerBackupRootSpecs(provider), + ]; + } + return []; +} + +export async function providerScanRootSpecs( + provider: ProviderId, +): Promise { + if (provider === "chatgpt") { + const discovered = await discoverChatGptConversationRoots(); + if (discovered.length > 0) return discovered; + return providerRootSpecs(provider); + } + if (provider !== "codex") return providerRootSpecs(provider); + const extraRoots = await discoverCodexCwdBackupRoots(); + return [...providerRootSpecs(provider), ...extraRoots]; +} diff --git a/apps/api-ts/src/domains/providers/roots/backups.ts b/apps/api-ts/src/domains/providers/roots/backups.ts new file mode 100644 index 0000000..eb0c3fc --- /dev/null +++ b/apps/api-ts/src/domains/providers/roots/backups.ts @@ -0,0 +1,78 @@ +import path from "node:path"; +import type { ProviderId } from "@threadlens/shared-contracts"; + +import { + BACKUP_ROOT, + CODEX_HOME, +} from "../../../lib/constants.js"; +import type { ProviderRootSpec } from "../types.js"; + +function providerActionBackupRoots(provider: ProviderId): string[] { + return Array.from( + new Set([ + path.join(BACKUP_ROOT, "provider_actions", provider), + path.join(CODEX_HOME, "local_cleanup_backups", "provider_actions", provider), + ]), + ); +} + +export function providerBackupRootSpecs(provider: ProviderId): ProviderRootSpec[] { + const cleanupRoots = () => + providerActionBackupRoots(provider).map((root) => ({ + source: "cleanup_backups", + root, + exts: [".jsonl", ".json"], + })); + if (provider === "codex") { + return [ + { + source: "recovered_sessions", + root: path.join(CODEX_HOME, "recovered-sessions"), + exts: [".jsonl"], + }, + ...cleanupRoots().map((spec) => ({ ...spec, exts: [".jsonl"] })), + ]; + } + if (provider === "claude") { + return cleanupRoots(); + } + if (provider === "gemini") { + return cleanupRoots().map((spec) => ({ ...spec, exts: [".jsonl", ".json", ".pb"] })); + } + if (provider === "copilot") { + return cleanupRoots(); + } + return []; +} + +export function providerArchivedRootSpec(provider: ProviderId): ProviderRootSpec | null { + if (provider === "codex") { + return { + source: "archived_sessions", + root: path.join(CODEX_HOME, "archived_sessions"), + exts: [".jsonl"], + }; + } + if (provider === "claude") { + return { + source: "archived_sessions", + root: path.join(CODEX_HOME, "archived_sessions", "claude"), + exts: [".jsonl", ".json"], + }; + } + if (provider === "gemini") { + return { + source: "archived_sessions", + root: path.join(CODEX_HOME, "archived_sessions", "gemini"), + exts: [".jsonl", ".json", ".pb"], + }; + } + if (provider === "copilot") { + return { + source: "archived_sessions", + root: path.join(CODEX_HOME, "archived_sessions", "copilot"), + exts: [".jsonl", ".json"], + }; + } + return null; +} diff --git a/apps/api-ts/src/domains/providers/roots/chatgpt.ts b/apps/api-ts/src/domains/providers/roots/chatgpt.ts new file mode 100644 index 0000000..d5eeb0f --- /dev/null +++ b/apps/api-ts/src/domains/providers/roots/chatgpt.ts @@ -0,0 +1,55 @@ +import { readdir } from "node:fs/promises"; +import path from "node:path"; +import { CHAT_DIR } from "../../../lib/constants.js"; +import { pathExists } from "../../../lib/utils.js"; +import type { + ProviderRootSpec, +} from "../types.js"; + +type ChatGptRootsCacheEntry = { + expires_at: number; + roots: ProviderRootSpec[]; +}; + +const CHATGPT_ROOT_DISCOVERY_TTL_MS = 45_000; +let chatGptRootsCache: ChatGptRootsCacheEntry | null = null; + +export async function discoverChatGptConversationRoots(): Promise { + const now = Date.now(); + if (chatGptRootsCache && chatGptRootsCache.expires_at > now) { + return chatGptRootsCache.roots; + } + if (!(await pathExists(CHAT_DIR))) return []; + const out = new Map(); + const push = (source: string, root: string) => { + if (!out.has(root)) out.set(root, { source, root, exts: [".data"] }); + }; + + const topLevel = await readdir(CHAT_DIR, { withFileTypes: true }).catch( + () => [], + ); + + for (const entry of topLevel) { + if (!entry.isDirectory()) continue; + if (entry.name.startsWith("conversations-v3-")) { + push("conversations", path.join(CHAT_DIR, entry.name)); + continue; + } + const nestedRoot = path.join(CHAT_DIR, entry.name); + const nested = await readdir(nestedRoot, { withFileTypes: true }).catch( + () => [], + ); + for (const child of nested) { + if (!child.isDirectory()) continue; + if (!child.name.startsWith("conversations-v3-")) continue; + push(entry.name, path.join(nestedRoot, child.name)); + } + } + + const roots = Array.from(out.values()); + chatGptRootsCache = { + expires_at: now + CHATGPT_ROOT_DISCOVERY_TTL_MS, + roots, + }; + return roots; +} diff --git a/apps/api-ts/src/domains/providers/roots/codex.ts b/apps/api-ts/src/domains/providers/roots/codex.ts new file mode 100644 index 0000000..b0b6ce8 --- /dev/null +++ b/apps/api-ts/src/domains/providers/roots/codex.ts @@ -0,0 +1,76 @@ +import { readdir } from "node:fs/promises"; +import path from "node:path"; +import { + BACKUP_ROOT, + CODEX_HOME, + HOME_DIR, +} from "../../../lib/constants.js"; +import type { + ProviderRootSpec, +} from "../types.js"; + +type CodexCwdBackupRootsCacheEntry = { + expires_at: number; + roots: ProviderRootSpec[]; +}; + +const CODEX_CWD_BACKUP_ROOT_DISCOVERY_TTL_MS = 60_000; +let codexCwdBackupRootsCache: CodexCwdBackupRootsCacheEntry | null = null; + +export function codexTranscriptSearchRoots(): ProviderRootSpec[] { + const homes = Array.from( + new Set([ + CODEX_HOME, + path.join(HOME_DIR, ".codex-cli"), + path.join(HOME_DIR, ".codex"), + ].filter(Boolean)), + ); + const roots: ProviderRootSpec[] = []; + roots.push({ + source: "cleanup_backups", + root: path.join(BACKUP_ROOT, "provider_actions", "codex"), + exts: [".jsonl"], + }); + for (const home of homes) { + roots.push({ + source: "sessions", + root: path.join(home, "sessions"), + exts: [".jsonl"], + }); + roots.push({ + source: "archived_sessions", + root: path.join(home, "archived_sessions"), + exts: [".jsonl"], + }); + roots.push({ + source: "recovered_sessions", + root: path.join(home, "recovered-sessions"), + exts: [".jsonl"], + }); + roots.push({ + source: "cleanup_backups", + root: path.join(home, "local_cleanup_backups", "provider_actions", "codex"), + exts: [".jsonl"], + }); + } + return roots; +} + +export async function discoverCodexCwdBackupRoots(): Promise { + if (codexCwdBackupRootsCache && codexCwdBackupRootsCache.expires_at > Date.now()) { + return codexCwdBackupRootsCache.roots; + } + const entries = await readdir(CODEX_HOME, { withFileTypes: true }).catch(() => []); + const roots = entries + .filter((entry) => entry.isDirectory() && entry.name.startsWith("jsonl-cwd-backups-")) + .map((entry) => ({ + source: "cwd_backups", + root: path.join(CODEX_HOME, entry.name), + exts: [".jsonl"], + })); + codexCwdBackupRootsCache = { + expires_at: Date.now() + CODEX_CWD_BACKUP_ROOT_DISCOVERY_TTL_MS, + roots, + }; + return roots; +} diff --git a/apps/api-ts/src/domains/providers/session-transcript-service.test.ts b/apps/api-ts/src/domains/providers/session-transcript-service.test.ts new file mode 100644 index 0000000..9f1b2a0 --- /dev/null +++ b/apps/api-ts/src/domains/providers/session-transcript-service.test.ts @@ -0,0 +1,95 @@ +import { mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +const tmpDirs: string[] = []; +const originalCodexHome = process.env.CODEX_HOME; + +afterEach(async () => { + process.env.CODEX_HOME = originalCodexHome; + vi.resetModules(); + await Promise.all(tmpDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +async function setupCodexSessionFixture() { + const codexHome = await mkdtemp(path.join(os.tmpdir(), "threadlens-codex-home-")); + tmpDirs.push(codexHome); + process.env.CODEX_HOME = codexHome; + vi.resetModules(); + const sessionsDir = path.join(codexHome, "sessions"); + await import("node:fs/promises").then(({ mkdir }) => mkdir(sessionsDir, { recursive: true })); + const filePath = path.join(sessionsDir, "session.jsonl"); + await writeFile( + filePath, + JSON.stringify({ + timestamp: "2026-05-01T00:00:00.000Z", + type: "response_item", + payload: { + type: "message", + role: "user", + content: [{ type: "input_text", text: "hello transcript" }], + }, + }), + "utf8", + ); + const service = await import("./session-transcript-service.js"); + return { filePath, service }; +} + +describe("getProviderSessionTranscript", () => { + it("loads an allowed provider transcript", async () => { + const { filePath, service } = await setupCodexSessionFixture(); + + const result = await service.getProviderSessionTranscript("codex", filePath, 50); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data.file_path).toBe(await realpath(filePath)); + expect(result.data.messages[0]?.text).toBe("hello transcript"); + }); + + it("fails closed for a path outside provider roots", async () => { + const { service } = await setupCodexSessionFixture(); + const outsideDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-outside-")); + tmpDirs.push(outsideDir); + const outsideFile = path.join(outsideDir, "session.jsonl"); + await writeFile(outsideFile, "{}", "utf8"); + + const result = await service.getProviderSessionTranscript("codex", outsideFile, 50); + + expect(result).toEqual({ + ok: false, + statusCode: 400, + message: "file_path outside provider roots", + }); + }); + + it("fails closed when the provider does not support transcript reads", async () => { + const { filePath, service } = await setupCodexSessionFixture(); + + const result = await service.getProviderSessionTranscript("chatgpt", filePath, 50); + + expect(result).toEqual({ + ok: false, + statusCode: 400, + message: "provider does not support transcript reads", + }); + }); + + it("fails closed for unknown provider ids at the capability gate", async () => { + const { filePath, service } = await setupCodexSessionFixture(); + + const result = await service.getProviderSessionTranscript( + "unknown" as Parameters[0], + filePath, + 50, + ); + + expect(result).toEqual({ + ok: false, + statusCode: 400, + message: "provider does not support transcript reads", + }); + }); +}); diff --git a/apps/api-ts/src/domains/providers/session-transcript-service.ts b/apps/api-ts/src/domains/providers/session-transcript-service.ts new file mode 100644 index 0000000..8116a7e --- /dev/null +++ b/apps/api-ts/src/domains/providers/session-transcript-service.ts @@ -0,0 +1,53 @@ +import { findProviderCapability } from "@threadlens/shared-contracts"; +import type { ProviderId, TranscriptPayload } from "./types.js"; +import { resolveAllowedProviderFilePath } from "./path-safety.js"; +import { buildSessionTranscript } from "./transcript.js"; +import { pathExists } from "../../lib/utils.js"; + +export type ProviderSessionTranscriptServiceResult = + | { + ok: true; + data: TranscriptPayload; + } + | { + ok: false; + statusCode: 400 | 404; + message: string; + }; + +export async function getProviderSessionTranscript( + provider: ProviderId, + filePath: string, + limit?: number, +): Promise { + if (findProviderCapability(provider)?.read_transcript !== true) { + return { + ok: false, + statusCode: 400, + message: "provider does not support transcript reads", + }; + } + + const safeFilePath = await resolveAllowedProviderFilePath(provider, filePath); + if (!safeFilePath) { + return { + ok: false, + statusCode: 400, + message: "file_path outside provider roots", + }; + } + + const exists = await pathExists(safeFilePath); + if (!exists) { + return { + ok: false, + statusCode: 404, + message: "session file not found", + }; + } + + return { + ok: true, + data: await buildSessionTranscript(provider, safeFilePath, Number(limit) || 300), + }; +} diff --git a/apps/api-ts/src/domains/providers/title-detection.ts b/apps/api-ts/src/domains/providers/title-detection.ts index d05d979..0e70e26 100644 --- a/apps/api-ts/src/domains/providers/title-detection.ts +++ b/apps/api-ts/src/domains/providers/title-detection.ts @@ -1,19 +1,14 @@ -import { readFile } from "node:fs/promises"; import path from "node:path"; -import { CODEX_HOME, CLAUDE_PROJECTS_DIR } from "../../lib/constants.js"; -import { isRecord, readFileTail, safeJsonParse } from "../../lib/utils.js"; -import { isPathInsideRoot } from "./path-safety.js"; +import { isRecord } from "../../lib/utils.js"; +import { normalizeDetectedTitle } from "./title-normalization.js"; -type CodexTitleMapCacheEntry = { - expires_at: number; - map: Map; -}; - -let codexTitleMapCache: CodexTitleMapCacheEntry | null = null; - -export function invalidateCodexThreadTitleMapCache() { - codexTitleMapCache = null; -} +export { + extractCodexThreadIdFromSessionName, + getCodexThreadTitleMap, + invalidateCodexThreadTitleMapCache, +} from "./codex-title-map.js"; +export { detectClaudeRenamedTitle } from "./claude-title-detection.js"; +export { normalizeDetectedTitle } from "./title-normalization.js"; export function fallbackDisplayTitle( detectedTitle: string, @@ -30,23 +25,6 @@ export function fallbackDisplayTitle( ); } -function extractUuidFromText(text: string): string { - const match = String(text || "").match(/[0-9a-f]{8}-[0-9a-f-]{27,}/i); - return match ? match[0] : ""; -} - -export function extractCodexThreadIdFromSessionName(name: string): string { - return extractUuidFromText(name); -} - -export function normalizeDetectedTitle(text: string, maxLen = 96): string { - const singleLine = String(text || "").replace(/\s+/g, " ").trim(); - if (!singleLine) return ""; - return singleLine.length > maxLen - ? `${singleLine.slice(0, maxLen - 1).trimEnd()}…` - : singleLine; -} - function isBoilerplateTitle(text: string): boolean { const normalized = String(text || "").trim().toLowerCase(); return ( @@ -168,95 +146,3 @@ export function detectSessionTitleFromHead( return { title: "", source: null }; } - -export async function detectClaudeRenamedTitle( - filePath: string, - format: "jsonl" | "json" | "unknown", -): Promise<{ title: string; source: string | null } | null> { - if ( - format !== "jsonl" || - !isPathInsideRoot(filePath, CLAUDE_PROJECTS_DIR) - ) { - return null; - } - const tail = await readFileTail(filePath, 262_144); - if (!tail.text.trim()) return null; - let customTitle = ""; - let agentName = ""; - for (const line of tail.text.split(/\r?\n/)) { - const parsed = safeJsonParse(line); - if (!isRecord(parsed)) continue; - const type = String(parsed.type ?? ""); - if (type === "custom-title") { - const value = normalizeDetectedTitle(String(parsed.customTitle ?? "")); - if (value) customTitle = value; - continue; - } - if (type === "agent-name") { - const value = normalizeDetectedTitle(String(parsed.agentName ?? "")); - if (value) agentName = value; - } - } - if (customTitle) { - return { title: customTitle, source: "claude-custom-title" }; - } - if (agentName) { - return { title: agentName, source: "claude-agent-name" }; - } - return null; -} - -export async function getCodexThreadTitleMap(): Promise> { - const now = Date.now(); - if (codexTitleMapCache && codexTitleMapCache.expires_at > now) { - return codexTitleMapCache.map; - } - const titleMap = new Map(); - const globalStateTitleIds = new Set(); - try { - const raw = await readFile( - path.join(CODEX_HOME, ".codex-global-state.json"), - "utf-8", - ); - const parsed = safeJsonParse(raw); - const blob = - isRecord(parsed) && isRecord(parsed["thread-titles"]) - ? (parsed["thread-titles"] as Record) - : null; - const titles = - blob && isRecord(blob.titles) - ? (blob.titles as Record) - : null; - if (titles) { - for (const [id, title] of Object.entries(titles)) { - const tid = extractUuidFromText(id); - const txt = normalizeDetectedTitle(String(title ?? "")); - if (tid && txt) { - titleMap.set(tid, txt); - globalStateTitleIds.add(tid); - } - } - } - } catch { - // no-op - } - try { - const raw = await readFile(path.join(CODEX_HOME, "session_index.jsonl"), "utf-8"); - for (const line of raw.split(/\r?\n/)) { - const parsed = safeJsonParse(line); - if (!isRecord(parsed)) continue; - const tid = extractUuidFromText(String(parsed.id ?? "")); - const txt = normalizeDetectedTitle(String(parsed.thread_name ?? "")); - if (tid && txt && !globalStateTitleIds.has(tid)) { - titleMap.set(tid, txt); - } - } - } catch { - // no-op - } - codexTitleMapCache = { - expires_at: now + 60_000, - map: titleMap, - }; - return titleMap; -} diff --git a/apps/api-ts/src/domains/providers/title-normalization.ts b/apps/api-ts/src/domains/providers/title-normalization.ts new file mode 100644 index 0000000..f49359d --- /dev/null +++ b/apps/api-ts/src/domains/providers/title-normalization.ts @@ -0,0 +1,7 @@ +export function normalizeDetectedTitle(text: string, maxLen = 96): string { + const singleLine = String(text || "").replace(/\s+/g, " ").trim(); + if (!singleLine) return ""; + return singleLine.length > maxLen + ? `${singleLine.slice(0, maxLen - 1).trimEnd()}…` + : singleLine; +} diff --git a/apps/api-ts/src/domains/providers/transcript.test.ts b/apps/api-ts/src/domains/providers/transcript.test.ts index 180cb98..8d9de9e 100644 --- a/apps/api-ts/src/domains/providers/transcript.test.ts +++ b/apps/api-ts/src/domains/providers/transcript.test.ts @@ -3,6 +3,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { buildSessionTranscript } from "./transcript"; +import { inferSessionId } from "./transcripts/normalizers.js"; const tmpDirs: string[] = []; @@ -17,6 +18,12 @@ afterEach(async () => { }); describe("buildSessionTranscript", () => { + it("infers transcript session ids from Windows-style paths on POSIX runtimes", () => { + expect(inferSessionId("C:\\Users\\hwan\\.codex\\sessions\\win-transcript.jsonl")).toBe( + "win-transcript", + ); + }); + it("ignores duplicated event_msg user_message entries when response_item already exists", async () => { const tmpDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-transcript-")); tmpDirs.push(tmpDir); diff --git a/apps/api-ts/src/domains/providers/transcript.ts b/apps/api-ts/src/domains/providers/transcript.ts index f53c864..e6a3826 100644 --- a/apps/api-ts/src/domains/providers/transcript.ts +++ b/apps/api-ts/src/domains/providers/transcript.ts @@ -1,331 +1,20 @@ import { readFile } from "node:fs/promises"; -import { readFileTail, isRecord } from "../../lib/utils.js"; +import { readFileTail } from "../../lib/utils.js"; import type { ProviderId, - TranscriptMessage, TranscriptPayload, } from "./types.js"; - -function inferFormat(filePath: string): "jsonl" | "json" | "unknown" { - const lower = filePath.toLowerCase(); - if (lower.endsWith(".jsonl")) return "jsonl"; - if (lower.endsWith(".json")) return "json"; - return "unknown"; -} - -function inferSessionId(filePath: string): string { - const match = - filePath.match(/\/([0-9a-f]{8}-[0-9a-f-]{27,})\.jsonl?$/i) || - filePath.match(/\/([^/]+)\.jsonl?$/i); - return match ? match[1] : filePath.split("/").pop() || filePath; -} - -function extractCodexThreadIdFromSessionName(name: string): string { - const match = String(name || "").match(/[0-9a-f]{8}-[0-9a-f-]{27,}/i); - return match ? match[0] : ""; -} - -function normalizeDetectedTitle(text: string, maxLen = 96): string { - const singleLine = String(text || "").replace(/\s+/g, " ").trim(); - if (!singleLine) return ""; - return singleLine.length > maxLen - ? `${singleLine.slice(0, maxLen - 1).trimEnd()}…` - : singleLine; -} - -function isBoilerplateTitle(text: string): boolean { - const normalized = String(text || "").trim().toLowerCase(); - return ( - normalized === "assistant" || - normalized === "user" || - normalized === "system" || - normalized === "developer" || - normalized === "tool" - ); -} - -function looksLikeIdOnly(text: string): boolean { - const normalized = String(text || "").trim(); - return /^[0-9a-f]{8}-[0-9a-f-]{27,}$/i.test(normalized); -} - -function parseTranscriptRole(raw: unknown): TranscriptMessage["role"] { - const role = String(raw ?? "").toLowerCase(); - if ( - role === "user" || - role === "assistant" || - role === "developer" || - role === "system" || - role === "tool" - ) { - return role; - } - return "unknown"; -} - -function parseProviderLikeRole( - raw: unknown, - provider?: ProviderId, -): TranscriptMessage["role"] { - const direct = parseTranscriptRole(raw); - if (direct !== "unknown") return direct; - const role = String(raw ?? "").trim().toLowerCase(); - if (role === "human" || role === "prompt" || role === "request" || role === "question") { - return "user"; - } - if ( - role === "model" || - role === "reply" || - role === "response" || - role === "thinking" || - role === "gemini" || - role === "claude" || - role === "chatgpt" || - role === "copilot" - ) { - return "assistant"; - } - if (role.includes("tool")) return "tool"; - if (!role && provider && provider !== "codex") return "assistant"; - return "unknown"; -} - -function normalizeTranscriptTimestamp(raw: unknown): string | null { - if (typeof raw === "string" && raw.trim()) return raw; - if (typeof raw === "number" && Number.isFinite(raw)) { - try { - return new Date(raw).toISOString(); - } catch { - return null; - } - } - return null; -} - -function extractTranscriptText(value: unknown, depth = 0): string { - if (depth > 5 || value === null || value === undefined) return ""; - if (typeof value === "string") return normalizeDetectedTitle(value, 2000); - if (Array.isArray(value)) { - const chunks = value - .slice(0, 30) - .map((item) => extractTranscriptText(item, depth + 1)) - .filter(Boolean); - return normalizeDetectedTitle(chunks.join(" "), 2000); - } - if (typeof value !== "object") return ""; - const obj = value as Record; - for (const key of ["text", "input_text", "output_text", "content", "message", "body", "value"]) { - if (!Object.prototype.hasOwnProperty.call(obj, key)) continue; - const hit = extractTranscriptText(obj[key], depth + 1); - if (hit) return hit; - } - const fallback = Object.values(obj) - .slice(0, 12) - .map((item) => extractTranscriptText(item, depth + 1)) - .find(Boolean); - return fallback ? normalizeDetectedTitle(fallback, 2000) : ""; -} - -function parseJsonlTranscriptLine( - line: string, -): Omit | null { - let obj: unknown; - try { - obj = JSON.parse(line); - } catch { - return null; - } - if (!obj || typeof obj !== "object") return null; - const row = obj as Record; - const type = String(row.type ?? "unknown"); - const ts = normalizeTranscriptTimestamp(row.timestamp ?? row.ts ?? row.time); - - if (type === "queue-operation" || type === "last-prompt") return null; - - if (type === "response_item") { - const payload = row.payload; - if (!payload || typeof payload !== "object") return null; - const payloadObj = payload as Record; - if (String(payloadObj.type ?? "") === "message") { - const text = extractTranscriptText(payloadObj.content); - if (!text || looksLikeIdOnly(text) || isBoilerplateTitle(text)) return null; - return { - role: parseProviderLikeRole(payloadObj.role), - text, - ts, - source_type: "response_item.message", - }; - } - return null; - } - - if (type === "event_msg") { - const payload = row.payload; - if (!payload || typeof payload !== "object") return null; - const payloadObj = payload as Record; - const eventType = String(payloadObj.type ?? ""); - if (eventType === "user_message" || eventType === "agent_message") return null; - const text = extractTranscriptText(payloadObj.message ?? payloadObj.text); - if (!text || looksLikeIdOnly(text) || isBoilerplateTitle(text)) return null; - return { - role: parseProviderLikeRole(payloadObj.role ?? "assistant"), - text, - ts, - source_type: `event_msg.${String(payloadObj.type ?? "event")}`, - }; - } - - if (isRecord(row.message)) { - const message = row.message as Record; - const text = extractTranscriptText( - message.content ?? message.parts ?? message.text ?? row.content ?? row.text, - ); - if (!text || looksLikeIdOnly(text) || isBoilerplateTitle(text)) return null; - return { - role: parseProviderLikeRole(message.role ?? row.role ?? row.type), - text, - ts, - source_type: type, - }; - } - - const role = parseTranscriptRole(row.role); - const text = extractTranscriptText(row.text ?? row.message ?? row.content ?? row.payload); - if (!text || looksLikeIdOnly(text) || isBoilerplateTitle(text)) return null; - return { - role: role === "unknown" ? parseProviderLikeRole(row.type) : role, - text, - ts, - source_type: type, - }; -} - -function pushTranscriptMessage( - out: Omit[], - provider: ProviderId, - roleRaw: unknown, - textValue: unknown, - tsRaw: unknown, - sourceType: string, -): void { - const text = extractTranscriptText(textValue); - if (!text || looksLikeIdOnly(text) || isBoilerplateTitle(text)) return; - out.push({ - role: parseProviderLikeRole(roleRaw, provider), - text, - ts: normalizeTranscriptTimestamp(tsRaw), - source_type: sourceType, - }); -} - -function collectJsonTranscriptMessages( - value: unknown, - provider: ProviderId, - out: Omit[], - depth = 0, - sourceType = "json", -): void { - if (depth > 6 || value === null || value === undefined) return; - if (Array.isArray(value)) { - for (const item of value.slice(0, 400)) { - collectJsonTranscriptMessages(item, provider, out, depth + 1, sourceType); - } - return; - } - if (!isRecord(value)) return; - - const obj = value as Record; - const ts = - obj.timestamp ?? - obj.ts ?? - obj.time ?? - obj.createdAt ?? - obj.creationDate ?? - obj.lastMessageDate ?? - null; - - if (isRecord(obj.message)) { - const message = obj.message as Record; - pushTranscriptMessage( - out, - provider, - message.role ?? obj.role ?? obj.type ?? obj.kind ?? "user", - message.content ?? message.parts ?? message.text ?? message, - message.timestamp ?? ts, - `${sourceType}.message`, - ); - } else if (typeof obj.message === "string") { - pushTranscriptMessage( - out, - provider, - obj.role ?? obj.type ?? obj.kind ?? "user", - obj.message, - ts, - `${sourceType}.message`, - ); - } - - const invocationValue = isRecord(obj.invocationMessage) - ? (obj.invocationMessage as Record).value - : undefined; - const directValue = obj.content ?? obj.text ?? obj.body ?? obj.value ?? invocationValue; - if (directValue !== undefined) { - pushTranscriptMessage( - out, - provider, - obj.role ?? obj.type ?? obj.kind ?? obj.author ?? provider, - directValue, - ts, - `${sourceType}.${String(obj.type ?? obj.kind ?? "entry")}`, - ); - } - - for (const [key, nested] of [ - ["messages", obj.messages], - ["requests", obj.requests], - ["response", obj.response], - ["responses", obj.responses], - ["items", obj.items], - ["events", obj.events], - ["turns", obj.turns], - ["conversation", obj.conversation], - ] as const) { - if (nested === undefined) continue; - collectJsonTranscriptMessages(nested, provider, out, depth + 1, `${sourceType}.${key}`); - } -} - -function parseJsonTranscriptPayload( - raw: string, - provider: ProviderId, -): TranscriptMessage[] { - const trimmed = String(raw ?? "").trim(); - if (!trimmed) return []; - let parsed: unknown; - try { - parsed = JSON.parse(trimmed); - } catch { - return []; - } - - const out: Omit[] = []; - collectJsonTranscriptMessages(parsed, provider, out); - - const deduped = out.filter((item, index) => { - if (index === 0) return true; - const prev = out[index - 1]; - return !( - prev.role === item.role && - prev.text === item.text && - prev.source_type === item.source_type - ); - }); - - return deduped.map((item, index) => ({ - ...item, - idx: index + 1, - })); -} +import { + parseJsonTranscriptPayload, +} from "./transcripts/json.js"; +import { + parseJsonlTranscriptLines, +} from "./transcripts/jsonl.js"; +import { + extractCodexThreadIdFromSessionName, + inferFormat, + inferSessionId, +} from "./transcripts/normalizers.js"; export async function buildSessionTranscript( provider: ProviderId, @@ -369,21 +58,11 @@ export async function buildSessionTranscript( } const tail = await readFileTail(filePath, 2_621_440); - const parseTranscriptLines = (rawLines: string[]): TranscriptMessage[] => { - const out: TranscriptMessage[] = []; - for (const line of rawLines) { - const parsed = parseJsonlTranscriptLine(line); - if (!parsed) continue; - out.push({ ...parsed, idx: out.length + 1 }); - } - return out; - }; - let lines = tail.text .split("\n") .map((line) => line.trim()) .filter(Boolean); - let messages = parseTranscriptLines(lines); + let messages = parseJsonlTranscriptLines(lines); if (tail.truncated && safeLimit > messages.length) { const fullRaw = await readFile(filePath, "utf-8").catch(() => ""); @@ -392,7 +71,7 @@ export async function buildSessionTranscript( .split("\n") .map((line) => line.trim()) .filter(Boolean); - messages = parseTranscriptLines(lines); + messages = parseJsonlTranscriptLines(lines); } } diff --git a/apps/api-ts/src/domains/providers/transcripts/json.ts b/apps/api-ts/src/domains/providers/transcripts/json.ts new file mode 100644 index 0000000..d74d6e5 --- /dev/null +++ b/apps/api-ts/src/domains/providers/transcripts/json.ts @@ -0,0 +1,141 @@ +import { + isRecord, +} from "../../../lib/utils.js"; +import type { + ProviderId, + TranscriptMessage, +} from "../types.js"; +import { + extractTranscriptText, + isBoilerplateTitle, + looksLikeIdOnly, + normalizeTranscriptTimestamp, + parseProviderLikeRole, +} from "./normalizers.js"; + +function pushTranscriptMessage( + out: Omit[], + provider: ProviderId, + roleRaw: unknown, + textValue: unknown, + tsRaw: unknown, + sourceType: string, +): void { + const text = extractTranscriptText(textValue); + if (!text || looksLikeIdOnly(text) || isBoilerplateTitle(text)) return; + out.push({ + role: parseProviderLikeRole(roleRaw, provider), + text, + ts: normalizeTranscriptTimestamp(tsRaw), + source_type: sourceType, + }); +} + +function collectJsonTranscriptMessages( + value: unknown, + provider: ProviderId, + out: Omit[], + depth = 0, + sourceType = "json", +): void { + if (depth > 6 || value === null || value === undefined) return; + if (Array.isArray(value)) { + for (const item of value.slice(0, 400)) { + collectJsonTranscriptMessages(item, provider, out, depth + 1, sourceType); + } + return; + } + if (!isRecord(value)) return; + + const obj = value as Record; + const ts = + obj.timestamp ?? + obj.ts ?? + obj.time ?? + obj.createdAt ?? + obj.creationDate ?? + obj.lastMessageDate ?? + null; + + if (isRecord(obj.message)) { + const message = obj.message as Record; + pushTranscriptMessage( + out, + provider, + message.role ?? obj.role ?? obj.type ?? obj.kind ?? "user", + message.content ?? message.parts ?? message.text ?? message, + message.timestamp ?? ts, + `${sourceType}.message`, + ); + } else if (typeof obj.message === "string") { + pushTranscriptMessage( + out, + provider, + obj.role ?? obj.type ?? obj.kind ?? "user", + obj.message, + ts, + `${sourceType}.message`, + ); + } + + const invocationValue = isRecord(obj.invocationMessage) + ? (obj.invocationMessage as Record).value + : undefined; + const directValue = obj.content ?? obj.text ?? obj.body ?? obj.value ?? invocationValue; + if (directValue !== undefined) { + pushTranscriptMessage( + out, + provider, + obj.role ?? obj.type ?? obj.kind ?? obj.author ?? provider, + directValue, + ts, + `${sourceType}.${String(obj.type ?? obj.kind ?? "entry")}`, + ); + } + + for (const [key, nested] of [ + ["messages", obj.messages], + ["requests", obj.requests], + ["response", obj.response], + ["responses", obj.responses], + ["items", obj.items], + ["events", obj.events], + ["turns", obj.turns], + ["conversation", obj.conversation], + ] as const) { + if (nested === undefined) continue; + collectJsonTranscriptMessages(nested, provider, out, depth + 1, `${sourceType}.${key}`); + } +} + +export function parseJsonTranscriptPayload( + raw: string, + provider: ProviderId, +): TranscriptMessage[] { + const trimmed = String(raw ?? "").trim(); + if (!trimmed) return []; + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + return []; + } + + const out: Omit[] = []; + collectJsonTranscriptMessages(parsed, provider, out); + + const deduped = out.filter((item, index) => { + if (index === 0) return true; + const prev = out[index - 1]; + return !( + prev.role === item.role && + prev.text === item.text && + prev.source_type === item.source_type + ); + }); + + return deduped.map((item, index) => ({ + ...item, + idx: index + 1, + })); +} diff --git a/apps/api-ts/src/domains/providers/transcripts/jsonl.ts b/apps/api-ts/src/domains/providers/transcripts/jsonl.ts new file mode 100644 index 0000000..f49ffb9 --- /dev/null +++ b/apps/api-ts/src/domains/providers/transcripts/jsonl.ts @@ -0,0 +1,100 @@ +import { + isRecord, +} from "../../../lib/utils.js"; +import type { + TranscriptMessage, +} from "../types.js"; +import { + extractTranscriptText, + isBoilerplateTitle, + looksLikeIdOnly, + normalizeTranscriptTimestamp, + parseProviderLikeRole, + parseTranscriptRole, +} from "./normalizers.js"; + +function parseJsonlTranscriptLine( + line: string, +): Omit | null { + let obj: unknown; + try { + obj = JSON.parse(line); + } catch { + return null; + } + if (!obj || typeof obj !== "object") return null; + const row = obj as Record; + const type = String(row.type ?? "unknown"); + const ts = normalizeTranscriptTimestamp(row.timestamp ?? row.ts ?? row.time); + + if (type === "queue-operation" || type === "last-prompt") return null; + + if (type === "response_item") { + const payload = row.payload; + if (!payload || typeof payload !== "object") return null; + const payloadObj = payload as Record; + if (String(payloadObj.type ?? "") === "message") { + const text = extractTranscriptText(payloadObj.content); + if (!text || looksLikeIdOnly(text) || isBoilerplateTitle(text)) return null; + return { + role: parseProviderLikeRole(payloadObj.role), + text, + ts, + source_type: "response_item.message", + }; + } + return null; + } + + if (type === "event_msg") { + const payload = row.payload; + if (!payload || typeof payload !== "object") return null; + const payloadObj = payload as Record; + const eventType = String(payloadObj.type ?? ""); + if (eventType === "user_message" || eventType === "agent_message") return null; + const text = extractTranscriptText(payloadObj.message ?? payloadObj.text); + if (!text || looksLikeIdOnly(text) || isBoilerplateTitle(text)) return null; + return { + role: parseProviderLikeRole(payloadObj.role ?? "assistant"), + text, + ts, + source_type: `event_msg.${String(payloadObj.type ?? "event")}`, + }; + } + + if (isRecord(row.message)) { + const message = row.message as Record; + const text = extractTranscriptText( + message.content ?? message.parts ?? message.text ?? row.content ?? row.text, + ); + if (!text || looksLikeIdOnly(text) || isBoilerplateTitle(text)) return null; + return { + role: parseProviderLikeRole(message.role ?? row.role ?? row.type), + text, + ts, + source_type: type, + }; + } + + const role = parseTranscriptRole(row.role); + const text = extractTranscriptText(row.text ?? row.message ?? row.content ?? row.payload); + if (!text || looksLikeIdOnly(text) || isBoilerplateTitle(text)) return null; + return { + role: role === "unknown" ? parseProviderLikeRole(row.type) : role, + text, + ts, + source_type: type, + }; +} + +export function parseJsonlTranscriptLines( + rawLines: string[], +): TranscriptMessage[] { + const out: TranscriptMessage[] = []; + for (const line of rawLines) { + const parsed = parseJsonlTranscriptLine(line); + if (!parsed) continue; + out.push({ ...parsed, idx: out.length + 1 }); + } + return out; +} diff --git a/apps/api-ts/src/domains/providers/transcripts/normalizers.ts b/apps/api-ts/src/domains/providers/transcripts/normalizers.ts new file mode 100644 index 0000000..d6a0cf3 --- /dev/null +++ b/apps/api-ts/src/domains/providers/transcripts/normalizers.ts @@ -0,0 +1,125 @@ +import type { + ProviderId, + TranscriptMessage, +} from "../types.js"; + +export function inferFormat(filePath: string): "jsonl" | "json" | "unknown" { + const lower = filePath.toLowerCase(); + if (lower.endsWith(".jsonl")) return "jsonl"; + if (lower.endsWith(".json")) return "json"; + return "unknown"; +} + +export function inferSessionId(filePath: string): string { + const normalizedPath = filePath.replace(/\\/g, "/"); + const match = + normalizedPath.match(/\/([0-9a-f]{8}-[0-9a-f-]{27,})\.jsonl?$/i) || + normalizedPath.match(/\/([^/]+)\.jsonl?$/i); + return match ? match[1] : normalizedPath.split("/").pop() || normalizedPath; +} + +export function extractCodexThreadIdFromSessionName(name: string): string { + const match = String(name || "").match(/[0-9a-f]{8}-[0-9a-f-]{27,}/i); + return match ? match[0] : ""; +} + +export function normalizeDetectedTitle(text: string, maxLen = 96): string { + const singleLine = String(text || "").replace(/\s+/g, " ").trim(); + if (!singleLine) return ""; + return singleLine.length > maxLen + ? `${singleLine.slice(0, maxLen - 1).trimEnd()}…` + : singleLine; +} + +export function isBoilerplateTitle(text: string): boolean { + const normalized = String(text || "").trim().toLowerCase(); + return ( + normalized === "assistant" || + normalized === "user" || + normalized === "system" || + normalized === "developer" || + normalized === "tool" + ); +} + +export function looksLikeIdOnly(text: string): boolean { + const normalized = String(text || "").trim(); + return /^[0-9a-f]{8}-[0-9a-f-]{27,}$/i.test(normalized); +} + +export function parseTranscriptRole(raw: unknown): TranscriptMessage["role"] { + const role = String(raw ?? "").toLowerCase(); + if ( + role === "user" || + role === "assistant" || + role === "developer" || + role === "system" || + role === "tool" + ) { + return role; + } + return "unknown"; +} + +export function parseProviderLikeRole( + raw: unknown, + provider?: ProviderId, +): TranscriptMessage["role"] { + const direct = parseTranscriptRole(raw); + if (direct !== "unknown") return direct; + const role = String(raw ?? "").trim().toLowerCase(); + if (role === "human" || role === "prompt" || role === "request" || role === "question") { + return "user"; + } + if ( + role === "model" || + role === "reply" || + role === "response" || + role === "thinking" || + role === "gemini" || + role === "claude" || + role === "chatgpt" || + role === "copilot" + ) { + return "assistant"; + } + if (role.includes("tool")) return "tool"; + if (!role && provider && provider !== "codex") return "assistant"; + return "unknown"; +} + +export function normalizeTranscriptTimestamp(raw: unknown): string | null { + if (typeof raw === "string" && raw.trim()) return raw; + if (typeof raw === "number" && Number.isFinite(raw)) { + try { + return new Date(raw).toISOString(); + } catch { + return null; + } + } + return null; +} + +export function extractTranscriptText(value: unknown, depth = 0): string { + if (depth > 5 || value === null || value === undefined) return ""; + if (typeof value === "string") return normalizeDetectedTitle(value, 2000); + if (Array.isArray(value)) { + const chunks = value + .slice(0, 30) + .map((item) => extractTranscriptText(item, depth + 1)) + .filter(Boolean); + return normalizeDetectedTitle(chunks.join(" "), 2000); + } + if (typeof value !== "object") return ""; + const obj = value as Record; + for (const key of ["text", "input_text", "output_text", "content", "message", "body", "value"]) { + if (!Object.prototype.hasOwnProperty.call(obj, key)) continue; + const hit = extractTranscriptText(obj[key], depth + 1); + if (hit) return hit; + } + const fallback = Object.values(obj) + .slice(0, 12) + .map((item) => extractTranscriptText(item, depth + 1)) + .find(Boolean); + return fallback ? normalizeDetectedTitle(fallback, 2000) : ""; +} diff --git a/apps/api-ts/src/domains/providers/types.ts b/apps/api-ts/src/domains/providers/types.ts index 11f05b2..842874a 100644 --- a/apps/api-ts/src/domains/providers/types.ts +++ b/apps/api-ts/src/domains/providers/types.ts @@ -131,6 +131,13 @@ export type ProviderRootSpec = { exts: string[]; }; +export type ProviderHealthEvidence = { + root_exists: boolean; + session_log_count: number; + roots: string[]; + notes: string; +}; + export type ProviderMatrixData = { generated_at: string; mode: string; diff --git a/apps/api-ts/src/lib/providers.ts b/apps/api-ts/src/lib/providers.ts index a86189e..a7f26a8 100644 --- a/apps/api-ts/src/lib/providers.ts +++ b/apps/api-ts/src/lib/providers.ts @@ -13,6 +13,7 @@ import { invalidateProviderSearchCaches } from "../domains/providers/search.js"; import { invalidateProviderMatrixCache } from "../domains/providers/matrix.js"; import { resolveAllowedProviderFilePath } from "../domains/providers/path-safety.js"; import type { ProviderId } from "../domains/providers/types.js"; +import type { ProviderSessionAction } from "../domains/providers/types.js"; import { findProviderCapability } from "@threadlens/shared-contracts"; export type * from "../domains/providers/types.js"; @@ -66,8 +67,24 @@ export { } from "../domains/providers/search-helpers.js"; export { buildProviderActionToken, buildSessionTranscript }; -function supportsProviderCleanup(provider: ProviderId): boolean { - return findProviderCapability(provider)?.safe_cleanup === true; +function supportsProviderAction( + provider: ProviderId, + action: ProviderSessionAction, +): boolean { + const capability = findProviderCapability(provider); + if (!capability) { + return false; + } + if (action === "backup_local") { + return capability.read_sessions === true; + } + if (action === "delete_local") { + return ( + capability.safe_cleanup === true && + capability.hard_delete === true + ); + } + return capability.safe_cleanup === true; } export async function runProviderSessionAction( @@ -81,7 +98,7 @@ export async function runProviderSessionAction( return runProviderSessionActionInternal( { resolveAllowedProviderFilePath, - supportsProviderCleanup, + supportsProviderAction, invalidateProviderCaches: (targetProvider) => { invalidateProviderSearchCaches(targetProvider); invalidateProviderMatrixCache(); diff --git a/apps/api-ts/src/lib/recovery-backups.ts b/apps/api-ts/src/lib/recovery-backups.ts new file mode 100644 index 0000000..fab7716 --- /dev/null +++ b/apps/api-ts/src/lib/recovery-backups.ts @@ -0,0 +1,131 @@ +import { readdir, stat } from "node:fs/promises"; +import path from "node:path"; +import { BACKUP_ROOT } from "./constants.js"; +import { walkFiles } from "./utils.js"; + +export type RecoveryBackupSet = { + backup_id: string; + path: string; + file_count: number; + total_bytes: number; + latest_mtime: string; + sample_files: string[]; +}; + +type RecoveryBackupCandidate = { + backup_id: string; + path: string; + rank: number; + mtime_ms: number; +}; + +export function isCleanupBackupId(name: string): boolean { + return /^\d{8}T\d{6}Z$/.test(String(name || "").trim()); +} + +async function listBackupCandidates( + options?: { backupRoot?: string }, +): Promise { + const backupRoot = options?.backupRoot ?? BACKUP_ROOT; + const entries = await readdir(backupRoot, { withFileTypes: true }).catch( + () => [], + ); + const out: RecoveryBackupCandidate[] = []; + + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const backupId = entry.name; + const fullPath = path.join(backupRoot, backupId); + const dirStat = await stat(fullPath).catch(() => null); + const dirMtimeMs = Number(dirStat?.mtimeMs ?? 0); + + if (isCleanupBackupId(backupId)) { + out.push({ + backup_id: backupId, + path: fullPath, + rank: 0, + mtime_ms: dirMtimeMs, + }); + continue; + } + + if (backupId !== "provider_actions") { + out.push({ + backup_id: backupId, + path: fullPath, + rank: 2, + mtime_ms: dirMtimeMs, + }); + continue; + } + + // Provider action archives are nested: provider_actions//. + const providers = await readdir(fullPath, { withFileTypes: true }).catch( + () => [], + ); + for (const providerEntry of providers) { + if (!providerEntry.isDirectory()) continue; + const providerDir = path.join(fullPath, providerEntry.name); + const runs = await readdir(providerDir, { withFileTypes: true }).catch( + () => [], + ); + for (const runEntry of runs) { + if (!runEntry.isDirectory()) continue; + const runDir = path.join(providerDir, runEntry.name); + const runStat = await stat(runDir).catch(() => null); + out.push({ + backup_id: `provider_actions/${providerEntry.name}/${runEntry.name}`, + path: runDir, + rank: 1, + mtime_ms: Number(runStat?.mtimeMs ?? dirMtimeMs), + }); + } + } + } + + return out; +} + +export async function scanBackupSets( + limit = 20, + options?: { backupRoot?: string }, +): Promise { + try { + const candidates = await listBackupCandidates(options); + const dirs = candidates + .sort((a, b) => { + if (a.rank !== b.rank) return a.rank - b.rank; + if (a.mtime_ms !== b.mtime_ms) return b.mtime_ms - a.mtime_ms; + return b.backup_id.localeCompare(a.backup_id); + }) + .slice(0, Math.max(1, limit)); + + const result: RecoveryBackupSet[] = []; + for (const candidate of dirs) { + const root = candidate.path; + const files = await walkFiles(root, 20_000); + let totalBytes = 0; + let latestMtime = 0; + for (const file of files) { + try { + const st = await stat(file); + totalBytes += Number(st.size); + latestMtime = Math.max(latestMtime, Number(st.mtimeMs)); + } catch { + // no-op + } + } + result.push({ + backup_id: candidate.backup_id, + path: root, + file_count: files.length, + total_bytes: totalBytes, + latest_mtime: latestMtime ? new Date(latestMtime).toISOString() : "", + sample_files: files.slice(0, 20), + }); + } + return result; + } catch { + return []; + } +} diff --git a/apps/api-ts/src/lib/recovery-checklist.ts b/apps/api-ts/src/lib/recovery-checklist.ts new file mode 100644 index 0000000..bb426c3 --- /dev/null +++ b/apps/api-ts/src/lib/recovery-checklist.ts @@ -0,0 +1,64 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { RECOVERY_CHECKLIST_FILE } from "./constants.js"; +import { isRecord, readJsonFile } from "./utils.js"; + +export type RecoveryChecklistItem = { + id: string; + label: string; + done: boolean; +}; + +function defaultRecoveryChecklist(): RecoveryChecklistItem[] { + return [ + { id: "backup_exists", label: "Confirm the latest backup set exists", done: false }, + { id: "dry_run_ok", label: "Review the cleanup dry-run result", done: false }, + { id: "token_verified", label: "Verify the execution token", done: false }, + { id: "drill_run", label: "Run and review the recovery drill", done: false }, + { id: "post_verify", label: "Verify state after execution", done: false }, + ]; +} + +export async function loadRecoveryChecklist(): Promise { + const data = await readJsonFile(RECOVERY_CHECKLIST_FILE); + if (isRecord(data) && Array.isArray(data.items)) { + return data.items + .filter((item) => isRecord(item)) + .map((item) => ({ + id: String(item.id ?? ""), + label: String(item.label ?? ""), + done: Boolean(item.done), + })) + .filter((item) => item.id && item.label); + } + const defaults = defaultRecoveryChecklist(); + await saveRecoveryChecklist(defaults); + return defaults; +} + +async function saveRecoveryChecklist(items: RecoveryChecklistItem[]) { + await mkdir(path.dirname(RECOVERY_CHECKLIST_FILE), { recursive: true }); + await writeFile( + RECOVERY_CHECKLIST_FILE, + JSON.stringify({ items }, null, 2), + "utf-8", + ); +} + +export async function updateRecoveryChecklistItem( + itemId: string, + done: boolean, +) { + const id = String(itemId ?? "").trim(); + if (!id) return { ok: false, error: "item_id is required" }; + const items = await loadRecoveryChecklist(); + let changed = false; + const next = items.map((item) => { + if (item.id !== id) return item; + changed = true; + return { ...item, done: Boolean(done) }; + }); + if (!changed) return { ok: false, error: "checklist item not found" }; + await saveRecoveryChecklist(next); + return { ok: true, items: next }; +} diff --git a/apps/api-ts/src/lib/recovery-export.ts b/apps/api-ts/src/lib/recovery-export.ts new file mode 100644 index 0000000..11ee53c --- /dev/null +++ b/apps/api-ts/src/lib/recovery-export.ts @@ -0,0 +1,186 @@ +import { createReadStream } from "node:fs"; +import { cp, mkdir, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { BACKUP_ROOT, RECOVERY_EXPORT_ROOT } from "./constants.js"; +import { scanBackupSets } from "./recovery-backups.js"; +import { writePortableZipArchive } from "./recovery-portable-zip.js"; +import { nowIsoUtc } from "./utils.js"; + +type RecoveryRootsOverride = { + backup_root?: string; + export_root?: string; +}; + +type RecoveryBackupExportOptions = { + backup_ids?: string[]; + roots?: RecoveryRootsOverride; + archiveWriter?: (sourceDir: string, archivePath: string) => Promise | void; +}; + +function recoveryExportRoot(override?: string): string { + return override ?? RECOVERY_EXPORT_ROOT; +} + +function hasHiddenPathSegment(filePath: string): boolean { + return path.resolve(filePath).split(path.sep).some((segment) => segment.startsWith(".")); +} + +export function resolveRecoveryBackupArchivePath( + archivePath: string, + overrideExportRoot?: string, +): string | null { + const requested = String(archivePath || "").trim(); + if (!requested) return null; + const exportRoot = path.resolve(recoveryExportRoot(overrideExportRoot)); + const resolved = path.resolve(requested); + const relative = path.relative(exportRoot, resolved); + if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) return null; + if (!resolved.endsWith(".zip")) return null; + return resolved; +} + +function sanitizeBackupExportSegment(backupId: string): string { + return String(backupId || "") + .replace(/[\\/]+/g, "__") + .replace(/[^a-zA-Z0-9._-]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 120) || "backup-set"; +} + +async function defaultRecoveryArchiveWriter(sourceDir: string, archivePath: string) { + await writePortableZipArchive(sourceDir, archivePath); +} + +export async function openRecoveryBackupArchiveReadStream( + archivePath: string, + overrideExportRoot?: string, +) { + const resolved = resolveRecoveryBackupArchivePath(archivePath, overrideExportRoot); + if (!resolved) return null; + try { + const archiveStat = await stat(resolved); + if (!archiveStat.isFile()) return null; + } catch { + return null; + } + return { archivePath: resolved, stream: createReadStream(resolved) }; +} + +export async function exportRecoveryBackupsTs( + options: RecoveryBackupExportOptions = {}, +) { + const requestedIds = Array.from( + new Set( + (options.backup_ids ?? []) + .map((item) => String(item || "").trim()) + .filter(Boolean), + ), + ); + const backupRoot = options.roots?.backup_root ?? BACKUP_ROOT; + const exportRoot = recoveryExportRoot(options.roots?.export_root); + if (hasHiddenPathSegment(backupRoot)) { + return { + ok: false, + error: "backup_root_hidden", + selected_backup_ids: requestedIds, + missing_backup_ids: [], + backup_root: backupRoot, + export_root: exportRoot, + }; + } + if (hasHiddenPathSegment(exportRoot)) { + return { + ok: false, + error: "export_root_hidden", + selected_backup_ids: requestedIds, + missing_backup_ids: [], + backup_root: backupRoot, + export_root: exportRoot, + }; + } + const allSets = await scanBackupSets(200, { backupRoot }); + const selectedSets = requestedIds.length + ? allSets.filter((set) => requestedIds.includes(set.backup_id)) + : allSets; + const missingBackupIds = requestedIds.filter( + (backupId) => !selectedSets.some((set) => set.backup_id === backupId), + ); + + if (!selectedSets.length) { + return { + ok: false, + error: requestedIds.length ? "backup-ids-not-found" : "no-backups-found", + selected_backup_ids: requestedIds, + missing_backup_ids: missingBackupIds, + backup_root: backupRoot, + export_root: exportRoot, + }; + } + + const timestamp = nowIsoUtc().replace(/[:.]/g, "-"); + const exportDir = path.join(exportRoot, `backup-export-${timestamp}`); + const payloadRoot = path.join(exportDir, "backup-sets"); + await mkdir(payloadRoot, { recursive: true }); + + const exportedSets: Array<{ + backup_id: string; + source_path: string; + export_path: string; + file_count: number; + total_bytes: number; + latest_mtime: string; + }> = []; + + for (const set of selectedSets) { + const exportPath = path.join( + payloadRoot, + sanitizeBackupExportSegment(set.backup_id), + ); + await cp(set.path, exportPath, { recursive: true }); + exportedSets.push({ + backup_id: set.backup_id, + source_path: set.path, + export_path: exportPath, + file_count: set.file_count, + total_bytes: set.total_bytes, + latest_mtime: set.latest_mtime, + }); + } + + const manifestPath = path.join(exportDir, "manifest.json"); + await writeFile( + manifestPath, + JSON.stringify( + { + generated_at: nowIsoUtc(), + backup_root: backupRoot, + export_root: exportRoot, + selected_backup_ids: selectedSets.map((set) => set.backup_id), + missing_backup_ids: missingBackupIds, + exported_count: exportedSets.length, + exported_sets: exportedSets, + }, + null, + 2, + ), + "utf-8", + ); + + const archivePath = `${exportDir}.zip`; + const archiveWriter = options.archiveWriter ?? defaultRecoveryArchiveWriter; + await archiveWriter(exportDir, archivePath); + + return { + ok: true, + generated_at: nowIsoUtc(), + backup_root: backupRoot, + export_root: exportRoot, + export_dir: exportDir, + archive_path: archivePath, + manifest_path: manifestPath, + selected_backup_ids: selectedSets.map((set) => set.backup_id), + missing_backup_ids: missingBackupIds, + exported_count: exportedSets.length, + exported_sets: exportedSets, + }; +} diff --git a/apps/api-ts/src/lib/recovery-portable-zip.ts b/apps/api-ts/src/lib/recovery-portable-zip.ts new file mode 100644 index 0000000..d963bd4 --- /dev/null +++ b/apps/api-ts/src/lib/recovery-portable-zip.ts @@ -0,0 +1,159 @@ +import { readFile, readdir, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; + +const ZIP_UINT16_MAX = 0xffff; +const ZIP_UINT32_MAX = 0xffffffff; +const DEFAULT_PORTABLE_ZIP_MAX_BYTES = 512 * 1024 * 1024; + +function portableZipMaxBytes(): number { + const raw = Number(process.env.THREADLENS_PORTABLE_ZIP_MAX_BYTES ?? ""); + if (Number.isFinite(raw) && raw > 0) { + return Math.min(Math.floor(raw), ZIP_UINT32_MAX); + } + return DEFAULT_PORTABLE_ZIP_MAX_BYTES; +} + +const CRC32_TABLE = new Uint32Array(256).map((_, index) => { + let value = index; + for (let bit = 0; bit < 8; bit += 1) { + value = value & 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1; + } + return value >>> 0; +}); + +function crc32(data: Buffer): number { + let crc = 0xffffffff; + for (const byte of data) { + crc = CRC32_TABLE[(crc ^ byte) & 0xff] ^ (crc >>> 8); + } + return (crc ^ 0xffffffff) >>> 0; +} + +function dosTimestamp(date: Date): { time: number; date: number } { + const year = Math.max(1980, date.getFullYear()); + return { + time: (date.getHours() << 11) | (date.getMinutes() << 5) | Math.floor(date.getSeconds() / 2), + date: ((year - 1980) << 9) | ((date.getMonth() + 1) << 5) | date.getDate(), + }; +} + +async function collectZipEntries( + sourceDir: string, + rootName = path.basename(sourceDir), +): Promise> { + const entries: Array<{ name: string; data: Buffer; mtime: Date; directory: boolean }> = []; + const maxTotalBytes = portableZipMaxBytes(); + let totalBytes = 0; + const walk = async (absoluteDir: string, relativeDir: string) => { + const dirStat = await stat(absoluteDir); + const directoryName = `${relativeDir.replace(/\\/g, "/").replace(/\/?$/, "/")}`; + if (Buffer.byteLength(directoryName, "utf8") > ZIP_UINT16_MAX) { + throw new Error("portable-zip-entry-name-too-long"); + } + entries.push({ + name: directoryName, + data: Buffer.alloc(0), + mtime: dirStat.mtime, + directory: true, + }); + const children = await readdir(absoluteDir, { withFileTypes: true }); + for (const child of children) { + const childPath = path.join(absoluteDir, child.name); + const childRelative = `${relativeDir}/${child.name}`.replace(/\\/g, "/"); + if (child.isDirectory()) { + await walk(childPath, childRelative); + } else if (child.isFile()) { + const fileStat = await stat(childPath); + if (fileStat.size > ZIP_UINT32_MAX) { + throw new Error("portable-zip-entry-too-large"); + } + totalBytes += fileStat.size; + if (totalBytes > maxTotalBytes) { + throw new Error("portable-zip-total-size-limit-exceeded"); + } + if (Buffer.byteLength(childRelative, "utf8") > ZIP_UINT16_MAX) { + throw new Error("portable-zip-entry-name-too-long"); + } + entries.push({ + name: childRelative, + data: await readFile(childPath), + mtime: fileStat.mtime, + directory: false, + }); + } + } + }; + await walk(sourceDir, rootName || "recovery-export"); + if (entries.length > ZIP_UINT16_MAX) { + throw new Error("portable-zip-entry-count-limit-exceeded"); + } + return entries; +} + +export async function writePortableZipArchive( + sourceDir: string, + archivePath: string, +) { + const entries = await collectZipEntries(sourceDir); + const localParts: Buffer[] = []; + const centralParts: Buffer[] = []; + let offset = 0; + for (const entry of entries) { + const name = Buffer.from(entry.name, "utf8"); + const { time, date } = dosTimestamp(entry.mtime); + const checksum = crc32(entry.data); + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(20, 4); + local.writeUInt16LE(0x0800, 6); + local.writeUInt16LE(0, 8); + local.writeUInt16LE(time, 10); + local.writeUInt16LE(date, 12); + local.writeUInt32LE(checksum, 14); + local.writeUInt32LE(entry.data.length, 18); + local.writeUInt32LE(entry.data.length, 22); + local.writeUInt16LE(name.length, 26); + local.writeUInt16LE(0, 28); + localParts.push(local, name, entry.data); + + const central = Buffer.alloc(46); + central.writeUInt32LE(0x02014b50, 0); + central.writeUInt16LE(20, 4); + central.writeUInt16LE(20, 6); + central.writeUInt16LE(0x0800, 8); + central.writeUInt16LE(0, 10); + central.writeUInt16LE(time, 12); + central.writeUInt16LE(date, 14); + central.writeUInt32LE(checksum, 16); + central.writeUInt32LE(entry.data.length, 20); + central.writeUInt32LE(entry.data.length, 24); + central.writeUInt16LE(name.length, 28); + central.writeUInt16LE(0, 30); + central.writeUInt16LE(0, 32); + central.writeUInt16LE(0, 34); + central.writeUInt16LE(0, 36); + central.writeUInt32LE(entry.directory ? 0x10 : 0, 38); + central.writeUInt32LE(offset, 42); + centralParts.push(central, name); + offset += local.length + name.length + entry.data.length; + if (offset > ZIP_UINT32_MAX) { + throw new Error("portable-zip-archive-too-large"); + } + } + + const centralDirectory = Buffer.concat(centralParts); + if (centralDirectory.length > ZIP_UINT32_MAX) { + throw new Error("portable-zip-central-directory-too-large"); + } + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(0, 4); + end.writeUInt16LE(0, 6); + end.writeUInt16LE(entries.length, 8); + end.writeUInt16LE(entries.length, 10); + end.writeUInt32LE(centralDirectory.length, 12); + end.writeUInt32LE(offset, 16); + end.writeUInt16LE(0, 20); + + await writeFile(archivePath, Buffer.concat([...localParts, centralDirectory, end])); +} diff --git a/apps/api-ts/src/lib/recovery-smoke.ts b/apps/api-ts/src/lib/recovery-smoke.ts new file mode 100644 index 0000000..cfa8c7c --- /dev/null +++ b/apps/api-ts/src/lib/recovery-smoke.ts @@ -0,0 +1,477 @@ +import { readFile, readdir } from "node:fs/promises"; +import path from "node:path"; +import { PROJECT_ROOT } from "./constants.js"; +import { + isRecord, + nowIsoUtc, + parseNumber, + pathExists, + safeJsonParse, +} from "./utils.js"; + +const SMOKE_SUMMARY_DIR = path.join(PROJECT_ROOT, ".run", "smoke"); +const SMOKE_SUMMARY_FILE_RE = /^smoke-summary-(\d{8}T\d{6}Z)\.json$/; +const PERF_SMOKE_DIR = path.join(PROJECT_ROOT, ".run", "perf"); +const PERF_SMOKE_FILE_RE = /^perf-smoke-(\d{8}T\d{6}Z)\.json$/; +const FORENSICS_SMOKE_DIR = path.join(PROJECT_ROOT, ".run", "forensics"); +const FORENSICS_SMOKE_FILE_RE = /^forensics-smoke-(\d{8}T\d{6}Z)\.json$/; + +type SmokeStatusRootOverrides = Partial<{ + summary_dir_abs: string; + summary_dir_rel: string; + perf_dir_abs: string; + perf_dir_rel: string; + forensics_dir_abs: string; + forensics_dir_rel: string; +}>; + +type SmokeStatusRoots = { + summary_dir_abs: string; + summary_dir_rel: string; + perf_dir_abs: string; + perf_dir_rel: string; + forensics_dir_abs: string; + forensics_dir_rel: string; +}; + +function resolveSmokeStatusRoots( + overrides?: SmokeStatusRootOverrides, +): SmokeStatusRoots { + const summaryDirRel = String(overrides?.summary_dir_rel ?? ".run/smoke").trim(); + const perfDirRel = String(overrides?.perf_dir_rel ?? ".run/perf").trim(); + const forensicsDirRel = String(overrides?.forensics_dir_rel ?? ".run/forensics").trim(); + return { + summary_dir_abs: String(overrides?.summary_dir_abs ?? SMOKE_SUMMARY_DIR), + summary_dir_rel: summaryDirRel || ".run/smoke", + perf_dir_abs: String(overrides?.perf_dir_abs ?? PERF_SMOKE_DIR), + perf_dir_rel: perfDirRel || ".run/perf", + forensics_dir_abs: String(overrides?.forensics_dir_abs ?? FORENSICS_SMOKE_DIR), + forensics_dir_rel: forensicsDirRel || ".run/forensics", + }; +} + +function parseNullableNumber(value: unknown): number | null { + const n = Number(value); + return Number.isFinite(n) ? n : null; +} + +function parseTimestampFromFileName(fileName: string, fileNameRe: RegExp): string { + const match = fileNameRe.exec(String(fileName || "").trim()); + return match?.[1] ?? ""; +} + +function parseSmokeTimestampFromName(fileName: string): string { + return parseTimestampFromFileName(fileName, SMOKE_SUMMARY_FILE_RE); +} + +function parseSmokeTimestampUtcMs(timestampUtc: string): number | null { + const match = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/.exec( + String(timestampUtc || "").trim(), + ); + if (!match) return null; + const [, yyyy, mm, dd, hh, mi, ss] = match; + const ms = Date.UTC( + Number(yyyy), + Number(mm) - 1, + Number(dd), + Number(hh), + Number(mi), + Number(ss), + ); + return Number.isFinite(ms) ? ms : null; +} + +function normalizeSmokePath(value: unknown): string { + const text = String(value ?? "").trim(); + return text ? text.replace(/\\/g, "/") : ""; +} + +type LatestSmokeFile = { + file_name: string; + abs_path: string; + rel_path: string; + timestamp_utc: string; +}; + +async function findLatestSmokeFile( + dirPath: string, + fileNameRe: RegExp, + relDirPath: string, +): Promise { + if (!(await pathExists(dirPath))) return null; + const entries = await readdir(dirPath, { withFileTypes: true }).catch(() => []); + const names = entries + .filter((entry) => entry.isFile() && fileNameRe.test(entry.name)) + .map((entry) => entry.name) + .sort(); + if (!names.length) return null; + const fileName = names[names.length - 1]; + return { + file_name: fileName, + abs_path: path.join(dirPath, fileName), + rel_path: path.posix.join(relDirPath, fileName), + timestamp_utc: parseTimestampFromFileName(fileName, fileNameRe), + }; +} + +function readPerfMetricSeconds( + metrics: unknown, + key: string, +): number | null { + if (!Array.isArray(metrics)) return null; + const row = metrics.find((item) => { + if (!isRecord(item)) return false; + return String(item.key ?? "").trim() === key; + }); + if (!isRecord(row)) return null; + return parseNullableNumber(row.time_total); +} + +type SmokeStatusFlag = "pass" | "fail" | "missing" | "invalid"; +type SmokeResult = "PASS" | "FAIL" | "MISSING" | "INVALID"; + +function buildSmokeStatusSkeleton( + status: SmokeStatusFlag, + result: SmokeResult, + overrides?: Partial<{ + timestamp_utc: string; + path: string; + age_sec: number | null; + parse_error: string; + }>, +) { + return { + status, + result, + ok: status === "pass", + timestamp_utc: overrides?.timestamp_utc ?? "", + age_sec: overrides?.age_sec ?? null, + path: overrides?.path ?? "", + sources: { + perf_report: "", + forensics_report: "", + }, + perf: { + ok: false, + agent_runtime_sec: null as number | null, + provider_sessions_30_sec: null as number | null, + threads_60_sec: null as number | null, + threads_160_sec: null as number | null, + }, + forensics: { + result: "", + analyze_status: null as number | null, + cleanup_status: null as number | null, + cleanup_token_valid: null as boolean | null, + }, + parse_error: overrides?.parse_error ?? "", + }; +} + +async function buildSmokeStatusFromRawReports(roots: SmokeStatusRoots) { + const latestPerf = await findLatestSmokeFile( + roots.perf_dir_abs, + PERF_SMOKE_FILE_RE, + roots.perf_dir_rel, + ); + const latestForensics = await findLatestSmokeFile( + roots.forensics_dir_abs, + FORENSICS_SMOKE_FILE_RE, + roots.forensics_dir_rel, + ); + if (!latestPerf && !latestForensics) return null; + + const issues: string[] = []; + const perfRaw = latestPerf + ? safeJsonParse(await readFile(latestPerf.abs_path, "utf-8").catch(() => "")) + : null; + const forensicsRaw = latestForensics + ? safeJsonParse(await readFile(latestForensics.abs_path, "utf-8").catch(() => "")) + : null; + const perfObj = isRecord(perfRaw) ? perfRaw : null; + const forensicsObj = isRecord(forensicsRaw) ? forensicsRaw : null; + + if (latestPerf && !perfObj) issues.push("perf-report-parse-failed"); + if (latestForensics && !forensicsObj) issues.push("forensics-report-parse-failed"); + + const perfOk = perfObj ? Boolean(perfObj.ok) : false; + const forensicsResultRaw = String(forensicsObj?.result ?? "").trim().toUpperCase(); + const forensicsOk = forensicsResultRaw === "PASS"; + + const latestTimestamp = [latestPerf?.timestamp_utc ?? "", latestForensics?.timestamp_utc ?? ""] + .filter(Boolean) + .sort() + .at(-1) ?? ""; + const timestampMs = parseSmokeTimestampUtcMs(latestTimestamp); + const ageSec = + timestampMs === null + ? null + : Math.max(0, Math.round((Date.now() - timestampMs) / 1000)); + + let status: SmokeStatusFlag = "invalid"; + let result: SmokeResult = "INVALID"; + if (perfObj && forensicsObj) { + status = perfOk && forensicsOk ? "pass" : "fail"; + result = perfOk && forensicsOk ? "PASS" : "FAIL"; + } + + const forensicsMetrics = isRecord(forensicsObj?.metrics) + ? (forensicsObj.metrics as Record) + : {}; + const forensicsAnalyze = isRecord(forensicsMetrics.analyze_delete) + ? (forensicsMetrics.analyze_delete as Record) + : {}; + const forensicsCleanup = isRecord(forensicsMetrics.local_cleanup) + ? (forensicsMetrics.local_cleanup as Record) + : {}; + + return { + latest: { + status, + result, + ok: status === "pass", + timestamp_utc: latestTimestamp, + age_sec: ageSec, + path: "", + sources: { + perf_report: latestPerf?.rel_path ?? "", + forensics_report: latestForensics?.rel_path ?? "", + }, + perf: { + ok: perfObj ? perfOk : false, + agent_runtime_sec: readPerfMetricSeconds(perfObj?.metrics, "agent_runtime"), + provider_sessions_30_sec: readPerfMetricSeconds( + perfObj?.metrics, + "provider_sessions_30", + ), + threads_60_sec: readPerfMetricSeconds(perfObj?.metrics, "threads_60"), + threads_160_sec: readPerfMetricSeconds(perfObj?.metrics, "threads_160"), + }, + forensics: { + result: forensicsResultRaw, + analyze_status: parseNullableNumber(forensicsAnalyze.status), + cleanup_status: parseNullableNumber(forensicsCleanup.status), + cleanup_token_valid: + typeof forensicsCleanup.confirm_token_valid === "boolean" + ? forensicsCleanup.confirm_token_valid + : null, + }, + parse_error: issues.join(","), + }, + history: [] as Array<{ timestamp_utc: string; path: string }>, + }; +} + +type SmokeStatusData = { + generated_at: string; + summary_dir: string; + latest: ReturnType; + history: Array<{ timestamp_utc: string; path: string }>; +}; + +const SMOKE_STATUS_CACHE_TTL_MS = 10_000; +const smokeStatusCache = new Map< + number, + { expires_at: number; data: SmokeStatusData } +>(); +const smokeStatusInflight = new Map>(); + +export async function getLatestSmokeStatusTs(options?: { + historyLimit?: number; + roots?: SmokeStatusRootOverrides; + forceRefresh?: boolean; +}) { + const historyLimit = Math.max( + 1, + Math.min(20, parseNumber(options?.historyLimit, 6)), + ); + const roots = resolveSmokeStatusRoots(options?.roots); + const useDefaultRoots = !options?.roots; + const forceRefresh = Boolean(options?.forceRefresh); + const now = Date.now(); + + if (useDefaultRoots && !forceRefresh) { + const cached = smokeStatusCache.get(historyLimit); + if (cached && cached.expires_at > now) return cached.data; + const inflight = smokeStatusInflight.get(historyLimit); + if (inflight) return inflight; + } + + const compute = async (): Promise => { + const summaryDirRelative = roots.summary_dir_rel; + const generatedAt = nowIsoUtc(); + + if (!(await pathExists(roots.summary_dir_abs))) { + const fallback = await buildSmokeStatusFromRawReports(roots); + if (fallback) { + return { + generated_at: generatedAt, + summary_dir: summaryDirRelative, + latest: fallback.latest, + history: fallback.history, + }; + } + return { + generated_at: generatedAt, + summary_dir: summaryDirRelative, + latest: buildSmokeStatusSkeleton("missing", "MISSING"), + history: [], + }; + } + + const entries = await readdir(roots.summary_dir_abs, { withFileTypes: true }).catch( + () => [], + ); + const summaryFiles = entries + .filter((entry) => entry.isFile() && SMOKE_SUMMARY_FILE_RE.test(entry.name)) + .map((entry) => entry.name) + .sort(); + const history = summaryFiles + .slice(-historyLimit) + .reverse() + .map((fileName) => ({ + timestamp_utc: parseSmokeTimestampFromName(fileName), + path: path.posix.join(summaryDirRelative, fileName), + })); + + if (!summaryFiles.length) { + const fallback = await buildSmokeStatusFromRawReports(roots); + if (fallback) { + return { + generated_at: generatedAt, + summary_dir: summaryDirRelative, + latest: fallback.latest, + history: fallback.history, + }; + } + return { + generated_at: generatedAt, + summary_dir: summaryDirRelative, + latest: buildSmokeStatusSkeleton("missing", "MISSING"), + history, + }; + } + + const latestFileName = summaryFiles[summaryFiles.length - 1]; + const latestPath = path.join(roots.summary_dir_abs, latestFileName); + const latestTimestamp = parseSmokeTimestampFromName(latestFileName); + const latestPathRelative = path.posix.join(summaryDirRelative, latestFileName); + + try { + const raw = await readFile(latestPath, "utf-8"); + const parsed = safeJsonParse(raw); + if (!isRecord(parsed)) { + return { + generated_at: generatedAt, + summary_dir: summaryDirRelative, + latest: buildSmokeStatusSkeleton("invalid", "INVALID", { + timestamp_utc: latestTimestamp, + path: latestPathRelative, + parse_error: "invalid-json-structure", + }), + history, + }; + } + + const sourceObj = isRecord(parsed.sources) ? parsed.sources : {}; + const perfObj = isRecord(parsed.perf) ? parsed.perf : {}; + const forensicsObj = isRecord(parsed.forensics) ? parsed.forensics : {}; + + const resultRaw = String(parsed.result ?? "").trim().toUpperCase(); + const result: SmokeResult = + resultRaw === "PASS" + ? "PASS" + : resultRaw === "FAIL" + ? "FAIL" + : "INVALID"; + const ok = Boolean(parsed.ok); + + const status: SmokeStatusFlag = + result === "PASS" && ok + ? "pass" + : result === "FAIL" || (result === "PASS" && !ok) + ? "fail" + : "invalid"; + + const timestampUtcCandidate = String( + parsed.timestamp_utc ?? latestTimestamp, + ) + .trim() + .toUpperCase(); + const timestampUtc = /^(\d{8}T\d{6}Z)$/.test(timestampUtcCandidate) + ? timestampUtcCandidate + : latestTimestamp; + const timestampMs = parseSmokeTimestampUtcMs(timestampUtc); + const ageSec = + timestampMs === null + ? null + : Math.max(0, Math.round((Date.now() - timestampMs) / 1000)); + + return { + generated_at: generatedAt, + summary_dir: summaryDirRelative, + latest: { + status, + result, + ok: status === "pass", + timestamp_utc: timestampUtc, + age_sec: ageSec, + path: latestPathRelative, + sources: { + perf_report: normalizeSmokePath(sourceObj.perf_report), + forensics_report: normalizeSmokePath(sourceObj.forensics_report), + }, + perf: { + ok: Boolean(perfObj.ok), + agent_runtime_sec: parseNullableNumber(perfObj.agent_runtime_sec), + provider_sessions_30_sec: parseNullableNumber( + perfObj.provider_sessions_30_sec, + ), + threads_60_sec: parseNullableNumber(perfObj.threads_60_sec), + threads_160_sec: parseNullableNumber(perfObj.threads_160_sec), + }, + forensics: { + result: String(forensicsObj.result ?? "").trim().toUpperCase(), + analyze_status: parseNullableNumber(forensicsObj.analyze_status), + cleanup_status: parseNullableNumber(forensicsObj.cleanup_status), + cleanup_token_valid: + typeof forensicsObj.cleanup_token_valid === "boolean" + ? forensicsObj.cleanup_token_valid + : null, + }, + parse_error: "", + }, + history, + }; + } catch (error) { + return { + generated_at: generatedAt, + summary_dir: summaryDirRelative, + latest: buildSmokeStatusSkeleton("invalid", "INVALID", { + timestamp_utc: latestTimestamp, + path: latestPathRelative, + parse_error: String(error), + }), + history, + }; + } + }; + + if (!useDefaultRoots) { + return compute(); + } + + const inflight = compute() + .then((data) => { + smokeStatusCache.set(historyLimit, { + expires_at: Date.now() + SMOKE_STATUS_CACHE_TTL_MS, + data, + }); + return data; + }) + .finally(() => { + smokeStatusInflight.delete(historyLimit); + }); + smokeStatusInflight.set(historyLimit, inflight); + return inflight; +} diff --git a/apps/api-ts/src/lib/recovery.ts b/apps/api-ts/src/lib/recovery.ts index 2e2e868..e09ccec 100644 --- a/apps/api-ts/src/lib/recovery.ts +++ b/apps/api-ts/src/lib/recovery.ts @@ -3,12 +3,8 @@ * related-tools status, and roadmap operations. */ -import { createReadStream } from "node:fs"; import { - cp, mkdir, - readFile, - readdir, stat, writeFile, chmod, @@ -20,87 +16,30 @@ import { CODEX_HOME, BACKUP_ROOT, RECOVERY_EXPORT_ROOT, - RECOVERY_CHECKLIST_FILE, RECOVERY_PLAN_DIR, CHAT_DIR, } from "./constants.js"; import { pathExists, - readJsonFile, walkFiles, isRecord, nowIsoUtc, - parseNumber, runCmdText, safeJsonParse, countDirsWithPrefix, quickFileCount, countJsonlFilesRecursive, } from "./utils.js"; +import { isCleanupBackupId, scanBackupSets } from "./recovery-backups.js"; +import { loadRecoveryChecklist } from "./recovery-checklist.js"; -/* ─────────────────────────────────────────────────────────────────── * - * Recovery checklist * - * ─────────────────────────────────────────────────────────────────── */ - -type RecoveryChecklistItem = { - id: string; - label: string; - done: boolean; -}; - -function defaultRecoveryChecklist(): RecoveryChecklistItem[] { - return [ - { id: "backup_exists", label: "Confirm the latest backup set exists", done: false }, - { id: "dry_run_ok", label: "Review the cleanup dry-run result", done: false }, - { id: "token_verified", label: "Verify the execution token", done: false }, - { id: "drill_run", label: "Run and review the recovery drill", done: false }, - { id: "post_verify", label: "Verify state after execution", done: false }, - ]; -} - -async function loadRecoveryChecklist(): Promise { - const data = await readJsonFile(RECOVERY_CHECKLIST_FILE); - if (isRecord(data) && Array.isArray(data.items)) { - return data.items - .filter((item) => isRecord(item)) - .map((item) => ({ - id: String(item.id ?? ""), - label: String(item.label ?? ""), - done: Boolean(item.done), - })) - .filter((item) => item.id && item.label); - } - const defaults = defaultRecoveryChecklist(); - await saveRecoveryChecklist(defaults); - return defaults; -} - -async function saveRecoveryChecklist(items: RecoveryChecklistItem[]) { - await mkdir(path.dirname(RECOVERY_CHECKLIST_FILE), { recursive: true }); - await writeFile( - RECOVERY_CHECKLIST_FILE, - JSON.stringify({ items }, null, 2), - "utf-8", - ); -} - -export async function updateRecoveryChecklistItem( - itemId: string, - done: boolean, -) { - const id = String(itemId ?? "").trim(); - if (!id) return { ok: false, error: "item_id is required" }; - const items = await loadRecoveryChecklist(); - let changed = false; - const next = items.map((item) => { - if (item.id !== id) return item; - changed = true; - return { ...item, done: Boolean(done) }; - }); - if (!changed) return { ok: false, error: "checklist item not found" }; - await saveRecoveryChecklist(next); - return { ok: true, items: next }; -} +export { updateRecoveryChecklistItem } from "./recovery-checklist.js"; +export { + exportRecoveryBackupsTs, + openRecoveryBackupArchiveReadStream, + resolveRecoveryBackupArchivePath, +} from "./recovery-export.js"; +export { getLatestSmokeStatusTs } from "./recovery-smoke.js"; /* ─────────────────────────────────────────────────────────────────── * * Backup scanning & restore plans * @@ -112,29 +51,8 @@ type RecoveryItem = { rel: string; }; -type RecoveryBackupSet = { - backup_id: string; - path: string; - file_count: number; - total_bytes: number; - latest_mtime: string; - sample_files: string[]; -}; - const LEGACY_BACKUP_ROOT = path.join(CODEX_HOME, "local_cleanup_backups"); -type RecoveryBackupCandidate = { - backup_id: string; - path: string; - rank: number; - mtime_ms: number; -}; - -type RecoveryRootsOverride = { - backup_root?: string; - export_root?: string; -}; - type RelatedToolConfig = { id: string; name: string; @@ -241,326 +159,6 @@ async function resolveRelatedToolStatus( }; } -function isCleanupBackupId(name: string): boolean { - return /^\d{8}T\d{6}Z$/.test(String(name || "").trim()); -} - -async function listBackupCandidates( - options?: { backupRoot?: string }, -): Promise { - const backupRoot = options?.backupRoot ?? BACKUP_ROOT; - const entries = await readdir(backupRoot, { withFileTypes: true }).catch( - () => [], - ); - const out: RecoveryBackupCandidate[] = []; - - for (const entry of entries) { - if (!entry.isDirectory()) continue; - const backupId = entry.name; - const fullPath = path.join(backupRoot, backupId); - const dirStat = await stat(fullPath).catch(() => null); - const dirMtimeMs = Number(dirStat?.mtimeMs ?? 0); - - if (isCleanupBackupId(backupId)) { - out.push({ - backup_id: backupId, - path: fullPath, - rank: 0, - mtime_ms: dirMtimeMs, - }); - continue; - } - - if (backupId !== "provider_actions") { - out.push({ - backup_id: backupId, - path: fullPath, - rank: 2, - mtime_ms: dirMtimeMs, - }); - continue; - } - - // Provider action archives are nested: provider_actions//. - const providers = await readdir(fullPath, { withFileTypes: true }).catch( - () => [], - ); - for (const providerEntry of providers) { - if (!providerEntry.isDirectory()) continue; - const providerDir = path.join(fullPath, providerEntry.name); - const runs = await readdir(providerDir, { withFileTypes: true }).catch( - () => [], - ); - for (const runEntry of runs) { - if (!runEntry.isDirectory()) continue; - const runDir = path.join(providerDir, runEntry.name); - const runStat = await stat(runDir).catch(() => null); - out.push({ - backup_id: `provider_actions/${providerEntry.name}/${runEntry.name}`, - path: runDir, - rank: 1, - mtime_ms: Number(runStat?.mtimeMs ?? dirMtimeMs), - }); - } - } - } - - return out; -} - -async function scanBackupSets( - limit = 20, - options?: { backupRoot?: string }, -): Promise { - try { - const candidates = await listBackupCandidates(options); - const dirs = candidates - .sort((a, b) => { - if (a.rank !== b.rank) return a.rank - b.rank; - if (a.mtime_ms !== b.mtime_ms) return b.mtime_ms - a.mtime_ms; - return b.backup_id.localeCompare(a.backup_id); - }) - .slice(0, Math.max(1, limit)); - - const result: RecoveryBackupSet[] = []; - for (const candidate of dirs) { - const root = candidate.path; - const files = await walkFiles(root, 20_000); - let totalBytes = 0; - let latestMtime = 0; - for (const file of files) { - try { - const st = await stat(file); - totalBytes += Number(st.size); - latestMtime = Math.max(latestMtime, Number(st.mtimeMs)); - } catch { - // no-op - } - } - result.push({ - backup_id: candidate.backup_id, - path: root, - file_count: files.length, - total_bytes: totalBytes, - latest_mtime: latestMtime ? new Date(latestMtime).toISOString() : "", - sample_files: files.slice(0, 20), - }); - } - return result; - } catch { - return []; - } -} - -type RecoveryBackupExportOptions = { - backup_ids?: string[]; - roots?: RecoveryRootsOverride; - archiveWriter?: (sourceDir: string, archivePath: string) => Promise | void; -}; - -const ZIP_UINT16_MAX = 0xffff; -const ZIP_UINT32_MAX = 0xffffffff; -const DEFAULT_PORTABLE_ZIP_MAX_BYTES = 512 * 1024 * 1024; - -function portableZipMaxBytes(): number { - const raw = Number(process.env.THREADLENS_PORTABLE_ZIP_MAX_BYTES ?? ""); - if (Number.isFinite(raw) && raw > 0) { - return Math.min(Math.floor(raw), ZIP_UINT32_MAX); - } - return DEFAULT_PORTABLE_ZIP_MAX_BYTES; -} - -function recoveryExportRoot(override?: string): string { - return override ?? RECOVERY_EXPORT_ROOT; -} - -function hasHiddenPathSegment(filePath: string): boolean { - return path.resolve(filePath).split(path.sep).some((segment) => segment.startsWith(".")); -} - -export function resolveRecoveryBackupArchivePath( - archivePath: string, - overrideExportRoot?: string, -): string | null { - const requested = String(archivePath || "").trim(); - if (!requested) return null; - const exportRoot = path.resolve(recoveryExportRoot(overrideExportRoot)); - const resolved = path.resolve(requested); - const relative = path.relative(exportRoot, resolved); - if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) return null; - if (!resolved.endsWith(".zip")) return null; - return resolved; -} - -function sanitizeBackupExportSegment(backupId: string): string { - return String(backupId || "") - .replace(/[\\/]+/g, "__") - .replace(/[^a-zA-Z0-9._-]+/g, "-") - .replace(/^-+|-+$/g, "") - .slice(0, 120) || "backup-set"; -} - -async function defaultRecoveryArchiveWriter(sourceDir: string, archivePath: string) { - await writePortableZipArchive(sourceDir, archivePath); -} - -const CRC32_TABLE = new Uint32Array(256).map((_, index) => { - let value = index; - for (let bit = 0; bit < 8; bit += 1) { - value = value & 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1; - } - return value >>> 0; -}); - -function crc32(data: Buffer): number { - let crc = 0xffffffff; - for (const byte of data) { - crc = CRC32_TABLE[(crc ^ byte) & 0xff] ^ (crc >>> 8); - } - return (crc ^ 0xffffffff) >>> 0; -} - -function dosTimestamp(date: Date): { time: number; date: number } { - const year = Math.max(1980, date.getFullYear()); - return { - time: (date.getHours() << 11) | (date.getMinutes() << 5) | Math.floor(date.getSeconds() / 2), - date: ((year - 1980) << 9) | ((date.getMonth() + 1) << 5) | date.getDate(), - }; -} - -async function collectZipEntries( - sourceDir: string, - rootName = path.basename(sourceDir), -): Promise> { - const entries: Array<{ name: string; data: Buffer; mtime: Date; directory: boolean }> = []; - const maxTotalBytes = portableZipMaxBytes(); - let totalBytes = 0; - const walk = async (absoluteDir: string, relativeDir: string) => { - const dirStat = await stat(absoluteDir); - const directoryName = `${relativeDir.replace(/\\/g, "/").replace(/\/?$/, "/")}`; - if (Buffer.byteLength(directoryName, "utf8") > ZIP_UINT16_MAX) { - throw new Error("portable-zip-entry-name-too-long"); - } - entries.push({ - name: directoryName, - data: Buffer.alloc(0), - mtime: dirStat.mtime, - directory: true, - }); - const children = await readdir(absoluteDir, { withFileTypes: true }); - for (const child of children) { - const childPath = path.join(absoluteDir, child.name); - const childRelative = `${relativeDir}/${child.name}`.replace(/\\/g, "/"); - if (child.isDirectory()) { - await walk(childPath, childRelative); - } else if (child.isFile()) { - const fileStat = await stat(childPath); - if (fileStat.size > ZIP_UINT32_MAX) { - throw new Error("portable-zip-entry-too-large"); - } - totalBytes += fileStat.size; - if (totalBytes > maxTotalBytes) { - throw new Error("portable-zip-total-size-limit-exceeded"); - } - if (Buffer.byteLength(childRelative, "utf8") > ZIP_UINT16_MAX) { - throw new Error("portable-zip-entry-name-too-long"); - } - entries.push({ - name: childRelative, - data: await readFile(childPath), - mtime: fileStat.mtime, - directory: false, - }); - } - } - }; - await walk(sourceDir, rootName || "recovery-export"); - if (entries.length > ZIP_UINT16_MAX) { - throw new Error("portable-zip-entry-count-limit-exceeded"); - } - return entries; -} - -async function writePortableZipArchive(sourceDir: string, archivePath: string) { - const entries = await collectZipEntries(sourceDir); - const localParts: Buffer[] = []; - const centralParts: Buffer[] = []; - let offset = 0; - for (const entry of entries) { - const name = Buffer.from(entry.name, "utf8"); - const { time, date } = dosTimestamp(entry.mtime); - const checksum = crc32(entry.data); - const local = Buffer.alloc(30); - local.writeUInt32LE(0x04034b50, 0); - local.writeUInt16LE(20, 4); - local.writeUInt16LE(0x0800, 6); - local.writeUInt16LE(0, 8); - local.writeUInt16LE(time, 10); - local.writeUInt16LE(date, 12); - local.writeUInt32LE(checksum, 14); - local.writeUInt32LE(entry.data.length, 18); - local.writeUInt32LE(entry.data.length, 22); - local.writeUInt16LE(name.length, 26); - local.writeUInt16LE(0, 28); - localParts.push(local, name, entry.data); - - const central = Buffer.alloc(46); - central.writeUInt32LE(0x02014b50, 0); - central.writeUInt16LE(20, 4); - central.writeUInt16LE(20, 6); - central.writeUInt16LE(0x0800, 8); - central.writeUInt16LE(0, 10); - central.writeUInt16LE(time, 12); - central.writeUInt16LE(date, 14); - central.writeUInt32LE(checksum, 16); - central.writeUInt32LE(entry.data.length, 20); - central.writeUInt32LE(entry.data.length, 24); - central.writeUInt16LE(name.length, 28); - central.writeUInt16LE(0, 30); - central.writeUInt16LE(0, 32); - central.writeUInt16LE(0, 34); - central.writeUInt16LE(0, 36); - central.writeUInt32LE(entry.directory ? 0x10 : 0, 38); - central.writeUInt32LE(offset, 42); - centralParts.push(central, name); - offset += local.length + name.length + entry.data.length; - if (offset > ZIP_UINT32_MAX) { - throw new Error("portable-zip-archive-too-large"); - } - } - - const centralDirectory = Buffer.concat(centralParts); - if (centralDirectory.length > ZIP_UINT32_MAX) { - throw new Error("portable-zip-central-directory-too-large"); - } - const end = Buffer.alloc(22); - end.writeUInt32LE(0x06054b50, 0); - end.writeUInt16LE(0, 4); - end.writeUInt16LE(0, 6); - end.writeUInt16LE(entries.length, 8); - end.writeUInt16LE(entries.length, 10); - end.writeUInt32LE(centralDirectory.length, 12); - end.writeUInt32LE(offset, 16); - end.writeUInt16LE(0, 20); - - await writeFile(archivePath, Buffer.concat([...localParts, centralDirectory, end])); -} - -export async function openRecoveryBackupArchiveReadStream( - archivePath: string, - overrideExportRoot?: string, -) { - const resolved = resolveRecoveryBackupArchivePath(archivePath, overrideExportRoot); - if (!resolved) return null; - try { - const archiveStat = await stat(resolved); - if (!archiveStat.isFile()) return null; - } catch { - return null; - } - return { archivePath: resolved, stream: createReadStream(resolved) }; -} - async function buildRestorePlan( backupDir: string, maxFiles = 400, @@ -697,125 +295,6 @@ export async function runRecoveryDrillTs() { }; } -export async function exportRecoveryBackupsTs( - options: RecoveryBackupExportOptions = {}, -) { - const requestedIds = Array.from( - new Set( - (options.backup_ids ?? []) - .map((item) => String(item || "").trim()) - .filter(Boolean), - ), - ); - const backupRoot = options.roots?.backup_root ?? BACKUP_ROOT; - const exportRoot = recoveryExportRoot(options.roots?.export_root); - if (hasHiddenPathSegment(backupRoot)) { - return { - ok: false, - error: "backup_root_hidden", - selected_backup_ids: requestedIds, - missing_backup_ids: [], - backup_root: backupRoot, - export_root: exportRoot, - }; - } - if (hasHiddenPathSegment(exportRoot)) { - return { - ok: false, - error: "export_root_hidden", - selected_backup_ids: requestedIds, - missing_backup_ids: [], - backup_root: backupRoot, - export_root: exportRoot, - }; - } - const allSets = await scanBackupSets(200, { backupRoot }); - const selectedSets = requestedIds.length - ? allSets.filter((set) => requestedIds.includes(set.backup_id)) - : allSets; - const missingBackupIds = requestedIds.filter( - (backupId) => !selectedSets.some((set) => set.backup_id === backupId), - ); - - if (!selectedSets.length) { - return { - ok: false, - error: requestedIds.length ? "backup-ids-not-found" : "no-backups-found", - selected_backup_ids: requestedIds, - missing_backup_ids: missingBackupIds, - backup_root: backupRoot, - export_root: exportRoot, - }; - } - - const timestamp = nowIsoUtc().replace(/[:.]/g, "-"); - const exportDir = path.join(exportRoot, `backup-export-${timestamp}`); - const payloadRoot = path.join(exportDir, "backup-sets"); - await mkdir(payloadRoot, { recursive: true }); - - const exportedSets: Array<{ - backup_id: string; - source_path: string; - export_path: string; - file_count: number; - total_bytes: number; - latest_mtime: string; - }> = []; - - for (const set of selectedSets) { - const exportPath = path.join( - payloadRoot, - sanitizeBackupExportSegment(set.backup_id), - ); - await cp(set.path, exportPath, { recursive: true }); - exportedSets.push({ - backup_id: set.backup_id, - source_path: set.path, - export_path: exportPath, - file_count: set.file_count, - total_bytes: set.total_bytes, - latest_mtime: set.latest_mtime, - }); - } - - const manifestPath = path.join(exportDir, "manifest.json"); - await writeFile( - manifestPath, - JSON.stringify( - { - generated_at: nowIsoUtc(), - backup_root: backupRoot, - export_root: exportRoot, - selected_backup_ids: selectedSets.map((set) => set.backup_id), - missing_backup_ids: missingBackupIds, - exported_count: exportedSets.length, - exported_sets: exportedSets, - }, - null, - 2, - ), - "utf-8", - ); - - const archivePath = `${exportDir}.zip`; - const archiveWriter = options.archiveWriter ?? defaultRecoveryArchiveWriter; - await archiveWriter(exportDir, archivePath); - - return { - ok: true, - generated_at: nowIsoUtc(), - backup_root: backupRoot, - export_root: exportRoot, - export_dir: exportDir, - archive_path: archivePath, - manifest_path: manifestPath, - selected_backup_ids: selectedSets.map((set) => set.backup_id), - missing_backup_ids: missingBackupIds, - exported_count: exportedSets.length, - exported_sets: exportedSets, - }; -} - /* ─────────────────────────────────────────────────────────────────── * * Related-tools status * * ─────────────────────────────────────────────────────────────────── */ @@ -910,474 +389,3 @@ export async function getRuntimeHealthTs() { quick_counts: quickCounts, }; } - -/* ─────────────────────────────────────────────────────────────────── * - * Smoke status * - * ─────────────────────────────────────────────────────────────────── */ - -const SMOKE_SUMMARY_DIR = path.join(PROJECT_ROOT, ".run", "smoke"); -const SMOKE_SUMMARY_FILE_RE = /^smoke-summary-(\d{8}T\d{6}Z)\.json$/; -const PERF_SMOKE_DIR = path.join(PROJECT_ROOT, ".run", "perf"); -const PERF_SMOKE_FILE_RE = /^perf-smoke-(\d{8}T\d{6}Z)\.json$/; -const FORENSICS_SMOKE_DIR = path.join(PROJECT_ROOT, ".run", "forensics"); -const FORENSICS_SMOKE_FILE_RE = /^forensics-smoke-(\d{8}T\d{6}Z)\.json$/; - -type SmokeStatusRootOverrides = Partial<{ - summary_dir_abs: string; - summary_dir_rel: string; - perf_dir_abs: string; - perf_dir_rel: string; - forensics_dir_abs: string; - forensics_dir_rel: string; -}>; - -type SmokeStatusRoots = { - summary_dir_abs: string; - summary_dir_rel: string; - perf_dir_abs: string; - perf_dir_rel: string; - forensics_dir_abs: string; - forensics_dir_rel: string; -}; - -function resolveSmokeStatusRoots( - overrides?: SmokeStatusRootOverrides, -): SmokeStatusRoots { - const summaryDirRel = String(overrides?.summary_dir_rel ?? ".run/smoke").trim(); - const perfDirRel = String(overrides?.perf_dir_rel ?? ".run/perf").trim(); - const forensicsDirRel = String(overrides?.forensics_dir_rel ?? ".run/forensics").trim(); - return { - summary_dir_abs: String(overrides?.summary_dir_abs ?? SMOKE_SUMMARY_DIR), - summary_dir_rel: summaryDirRel || ".run/smoke", - perf_dir_abs: String(overrides?.perf_dir_abs ?? PERF_SMOKE_DIR), - perf_dir_rel: perfDirRel || ".run/perf", - forensics_dir_abs: String(overrides?.forensics_dir_abs ?? FORENSICS_SMOKE_DIR), - forensics_dir_rel: forensicsDirRel || ".run/forensics", - }; -} - -function parseNullableNumber(value: unknown): number | null { - const n = Number(value); - return Number.isFinite(n) ? n : null; -} - -function parseTimestampFromFileName(fileName: string, fileNameRe: RegExp): string { - const match = fileNameRe.exec(String(fileName || "").trim()); - return match?.[1] ?? ""; -} - -function parseSmokeTimestampFromName(fileName: string): string { - return parseTimestampFromFileName(fileName, SMOKE_SUMMARY_FILE_RE); -} - -function parseSmokeTimestampUtcMs(timestampUtc: string): number | null { - const match = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/.exec( - String(timestampUtc || "").trim(), - ); - if (!match) return null; - const [, yyyy, mm, dd, hh, mi, ss] = match; - const ms = Date.UTC( - Number(yyyy), - Number(mm) - 1, - Number(dd), - Number(hh), - Number(mi), - Number(ss), - ); - return Number.isFinite(ms) ? ms : null; -} - -function normalizeSmokePath(value: unknown): string { - const text = String(value ?? "").trim(); - return text ? text.replace(/\\/g, "/") : ""; -} - -type LatestSmokeFile = { - file_name: string; - abs_path: string; - rel_path: string; - timestamp_utc: string; -}; - -async function findLatestSmokeFile( - dirPath: string, - fileNameRe: RegExp, - relDirPath: string, -): Promise { - if (!(await pathExists(dirPath))) return null; - const entries = await readdir(dirPath, { withFileTypes: true }).catch(() => []); - const names = entries - .filter((entry) => entry.isFile() && fileNameRe.test(entry.name)) - .map((entry) => entry.name) - .sort(); - if (!names.length) return null; - const fileName = names[names.length - 1]; - return { - file_name: fileName, - abs_path: path.join(dirPath, fileName), - rel_path: path.posix.join(relDirPath, fileName), - timestamp_utc: parseTimestampFromFileName(fileName, fileNameRe), - }; -} - -function readPerfMetricSeconds( - metrics: unknown, - key: string, -): number | null { - if (!Array.isArray(metrics)) return null; - const row = metrics.find((item) => { - if (!isRecord(item)) return false; - return String(item.key ?? "").trim() === key; - }); - if (!isRecord(row)) return null; - return parseNullableNumber(row.time_total); -} - -type SmokeStatusFlag = "pass" | "fail" | "missing" | "invalid"; -type SmokeResult = "PASS" | "FAIL" | "MISSING" | "INVALID"; - -function buildSmokeStatusSkeleton( - status: SmokeStatusFlag, - result: SmokeResult, - overrides?: Partial<{ - timestamp_utc: string; - path: string; - age_sec: number | null; - parse_error: string; - }>, -) { - return { - status, - result, - ok: status === "pass", - timestamp_utc: overrides?.timestamp_utc ?? "", - age_sec: overrides?.age_sec ?? null, - path: overrides?.path ?? "", - sources: { - perf_report: "", - forensics_report: "", - }, - perf: { - ok: false, - agent_runtime_sec: null as number | null, - provider_sessions_30_sec: null as number | null, - threads_60_sec: null as number | null, - threads_160_sec: null as number | null, - }, - forensics: { - result: "", - analyze_status: null as number | null, - cleanup_status: null as number | null, - cleanup_token_valid: null as boolean | null, - }, - parse_error: overrides?.parse_error ?? "", - }; -} - -async function buildSmokeStatusFromRawReports(roots: SmokeStatusRoots) { - const latestPerf = await findLatestSmokeFile( - roots.perf_dir_abs, - PERF_SMOKE_FILE_RE, - roots.perf_dir_rel, - ); - const latestForensics = await findLatestSmokeFile( - roots.forensics_dir_abs, - FORENSICS_SMOKE_FILE_RE, - roots.forensics_dir_rel, - ); - if (!latestPerf && !latestForensics) return null; - - const issues: string[] = []; - const perfRaw = latestPerf - ? safeJsonParse(await readFile(latestPerf.abs_path, "utf-8").catch(() => "")) - : null; - const forensicsRaw = latestForensics - ? safeJsonParse(await readFile(latestForensics.abs_path, "utf-8").catch(() => "")) - : null; - const perfObj = isRecord(perfRaw) ? perfRaw : null; - const forensicsObj = isRecord(forensicsRaw) ? forensicsRaw : null; - - if (latestPerf && !perfObj) issues.push("perf-report-parse-failed"); - if (latestForensics && !forensicsObj) issues.push("forensics-report-parse-failed"); - - const perfOk = perfObj ? Boolean(perfObj.ok) : false; - const forensicsResultRaw = String(forensicsObj?.result ?? "").trim().toUpperCase(); - const forensicsOk = forensicsResultRaw === "PASS"; - - const latestTimestamp = [latestPerf?.timestamp_utc ?? "", latestForensics?.timestamp_utc ?? ""] - .filter(Boolean) - .sort() - .at(-1) ?? ""; - const timestampMs = parseSmokeTimestampUtcMs(latestTimestamp); - const ageSec = - timestampMs === null - ? null - : Math.max(0, Math.round((Date.now() - timestampMs) / 1000)); - - let status: SmokeStatusFlag = "invalid"; - let result: SmokeResult = "INVALID"; - if (perfObj && forensicsObj) { - status = perfOk && forensicsOk ? "pass" : "fail"; - result = perfOk && forensicsOk ? "PASS" : "FAIL"; - } - - const forensicsMetrics = isRecord(forensicsObj?.metrics) - ? (forensicsObj.metrics as Record) - : {}; - const forensicsAnalyze = isRecord(forensicsMetrics.analyze_delete) - ? (forensicsMetrics.analyze_delete as Record) - : {}; - const forensicsCleanup = isRecord(forensicsMetrics.local_cleanup) - ? (forensicsMetrics.local_cleanup as Record) - : {}; - - return { - latest: { - status, - result, - ok: status === "pass", - timestamp_utc: latestTimestamp, - age_sec: ageSec, - path: "", - sources: { - perf_report: latestPerf?.rel_path ?? "", - forensics_report: latestForensics?.rel_path ?? "", - }, - perf: { - ok: perfObj ? perfOk : false, - agent_runtime_sec: readPerfMetricSeconds(perfObj?.metrics, "agent_runtime"), - provider_sessions_30_sec: readPerfMetricSeconds( - perfObj?.metrics, - "provider_sessions_30", - ), - threads_60_sec: readPerfMetricSeconds(perfObj?.metrics, "threads_60"), - threads_160_sec: readPerfMetricSeconds(perfObj?.metrics, "threads_160"), - }, - forensics: { - result: forensicsResultRaw, - analyze_status: parseNullableNumber(forensicsAnalyze.status), - cleanup_status: parseNullableNumber(forensicsCleanup.status), - cleanup_token_valid: - typeof forensicsCleanup.confirm_token_valid === "boolean" - ? forensicsCleanup.confirm_token_valid - : null, - }, - parse_error: issues.join(","), - }, - history: [] as Array<{ timestamp_utc: string; path: string }>, - }; -} - -type SmokeStatusData = { - generated_at: string; - summary_dir: string; - latest: ReturnType; - history: Array<{ timestamp_utc: string; path: string }>; -}; - -const SMOKE_STATUS_CACHE_TTL_MS = 10_000; -const smokeStatusCache = new Map< - number, - { expires_at: number; data: SmokeStatusData } ->(); -const smokeStatusInflight = new Map>(); - -export async function getLatestSmokeStatusTs(options?: { - historyLimit?: number; - roots?: SmokeStatusRootOverrides; - forceRefresh?: boolean; -}) { - const historyLimit = Math.max( - 1, - Math.min(20, parseNumber(options?.historyLimit, 6)), - ); - const roots = resolveSmokeStatusRoots(options?.roots); - const useDefaultRoots = !options?.roots; - const forceRefresh = Boolean(options?.forceRefresh); - const now = Date.now(); - - if (useDefaultRoots && !forceRefresh) { - const cached = smokeStatusCache.get(historyLimit); - if (cached && cached.expires_at > now) return cached.data; - const inflight = smokeStatusInflight.get(historyLimit); - if (inflight) return inflight; - } - - const compute = async (): Promise => { - const summaryDirRelative = roots.summary_dir_rel; - const generatedAt = nowIsoUtc(); - - if (!(await pathExists(roots.summary_dir_abs))) { - const fallback = await buildSmokeStatusFromRawReports(roots); - if (fallback) { - return { - generated_at: generatedAt, - summary_dir: summaryDirRelative, - latest: fallback.latest, - history: fallback.history, - }; - } - return { - generated_at: generatedAt, - summary_dir: summaryDirRelative, - latest: buildSmokeStatusSkeleton("missing", "MISSING"), - history: [], - }; - } - - const entries = await readdir(roots.summary_dir_abs, { withFileTypes: true }).catch( - () => [], - ); - const summaryFiles = entries - .filter((entry) => entry.isFile() && SMOKE_SUMMARY_FILE_RE.test(entry.name)) - .map((entry) => entry.name) - .sort(); - const history = summaryFiles - .slice(-historyLimit) - .reverse() - .map((fileName) => ({ - timestamp_utc: parseSmokeTimestampFromName(fileName), - path: path.posix.join(summaryDirRelative, fileName), - })); - - if (!summaryFiles.length) { - const fallback = await buildSmokeStatusFromRawReports(roots); - if (fallback) { - return { - generated_at: generatedAt, - summary_dir: summaryDirRelative, - latest: fallback.latest, - history: fallback.history, - }; - } - return { - generated_at: generatedAt, - summary_dir: summaryDirRelative, - latest: buildSmokeStatusSkeleton("missing", "MISSING"), - history, - }; - } - - const latestFileName = summaryFiles[summaryFiles.length - 1]; - const latestPath = path.join(roots.summary_dir_abs, latestFileName); - const latestTimestamp = parseSmokeTimestampFromName(latestFileName); - const latestPathRelative = path.posix.join(summaryDirRelative, latestFileName); - - try { - const raw = await readFile(latestPath, "utf-8"); - const parsed = safeJsonParse(raw); - if (!isRecord(parsed)) { - return { - generated_at: generatedAt, - summary_dir: summaryDirRelative, - latest: buildSmokeStatusSkeleton("invalid", "INVALID", { - timestamp_utc: latestTimestamp, - path: latestPathRelative, - parse_error: "invalid-json-structure", - }), - history, - }; - } - - const sourceObj = isRecord(parsed.sources) ? parsed.sources : {}; - const perfObj = isRecord(parsed.perf) ? parsed.perf : {}; - const forensicsObj = isRecord(parsed.forensics) ? parsed.forensics : {}; - - const resultRaw = String(parsed.result ?? "").trim().toUpperCase(); - const result: SmokeResult = - resultRaw === "PASS" - ? "PASS" - : resultRaw === "FAIL" - ? "FAIL" - : "INVALID"; - const ok = Boolean(parsed.ok); - - const status: SmokeStatusFlag = - result === "PASS" && ok - ? "pass" - : result === "FAIL" || (result === "PASS" && !ok) - ? "fail" - : "invalid"; - - const timestampUtcCandidate = String( - parsed.timestamp_utc ?? latestTimestamp, - ) - .trim() - .toUpperCase(); - const timestampUtc = /^(\d{8}T\d{6}Z)$/.test(timestampUtcCandidate) - ? timestampUtcCandidate - : latestTimestamp; - const timestampMs = parseSmokeTimestampUtcMs(timestampUtc); - const ageSec = - timestampMs === null - ? null - : Math.max(0, Math.round((Date.now() - timestampMs) / 1000)); - - return { - generated_at: generatedAt, - summary_dir: summaryDirRelative, - latest: { - status, - result, - ok: status === "pass", - timestamp_utc: timestampUtc, - age_sec: ageSec, - path: latestPathRelative, - sources: { - perf_report: normalizeSmokePath(sourceObj.perf_report), - forensics_report: normalizeSmokePath(sourceObj.forensics_report), - }, - perf: { - ok: Boolean(perfObj.ok), - agent_runtime_sec: parseNullableNumber(perfObj.agent_runtime_sec), - provider_sessions_30_sec: parseNullableNumber( - perfObj.provider_sessions_30_sec, - ), - threads_60_sec: parseNullableNumber(perfObj.threads_60_sec), - threads_160_sec: parseNullableNumber(perfObj.threads_160_sec), - }, - forensics: { - result: String(forensicsObj.result ?? "").trim().toUpperCase(), - analyze_status: parseNullableNumber(forensicsObj.analyze_status), - cleanup_status: parseNullableNumber(forensicsObj.cleanup_status), - cleanup_token_valid: - typeof forensicsObj.cleanup_token_valid === "boolean" - ? forensicsObj.cleanup_token_valid - : null, - }, - parse_error: "", - }, - history, - }; - } catch (error) { - return { - generated_at: generatedAt, - summary_dir: summaryDirRelative, - latest: buildSmokeStatusSkeleton("invalid", "INVALID", { - timestamp_utc: latestTimestamp, - path: latestPathRelative, - parse_error: String(error), - }), - history, - }; - } - }; - - if (!useDefaultRoots) { - return compute(); - } - - const inflight = compute() - .then((data) => { - smokeStatusCache.set(historyLimit, { - expires_at: Date.now() + SMOKE_STATUS_CACHE_TTL_MS, - data, - }); - return data; - }) - .finally(() => { - smokeStatusInflight.delete(historyLimit); - }); - smokeStatusInflight.set(historyLimit, inflight); - return inflight; -} diff --git a/packages/shared-contracts/src/api-client.test.ts b/packages/shared-contracts/src/api-client.test.ts index 707fd42..bece6ed 100644 --- a/packages/shared-contracts/src/api-client.test.ts +++ b/packages/shared-contracts/src/api-client.test.ts @@ -17,6 +17,29 @@ describe("shared api client helpers", () => { }); }); + it("unwraps successful null envelope data without treating it as an error", async () => { + const response = new Response( + JSON.stringify({ + ok: true, + data: null, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + + await expect(parseApiPayload(response, "/api/noop", { unwrapEnvelope: true })).resolves.toBeNull(); + }); + + it("rejects successful envelopes that omit the data field", async () => { + const response = new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + + await expect( + parseApiPayload(response, "/api/malformed", { unwrapEnvelope: true }), + ).rejects.toThrow("/api/malformed returned malformed envelope"); + }); + it("keeps envelope payloads intact for web-style consumers", async () => { const response = new Response( JSON.stringify({ diff --git a/packages/shared-contracts/src/api-client.ts b/packages/shared-contracts/src/api-client.ts index f587686..580e375 100644 --- a/packages/shared-contracts/src/api-client.ts +++ b/packages/shared-contracts/src/api-client.ts @@ -36,6 +36,15 @@ function isEnvelopeLike(payload: unknown): payload is ApiEnvelope { ); } +function isEnvelopeStatusLike(payload: unknown): payload is { ok: boolean; error?: unknown } { + return Boolean( + payload && + typeof payload === "object" && + "ok" in payload && + typeof (payload as { ok?: unknown }).ok === "boolean", + ); +} + async function parseJsonPayload(response: Response, path: string): Promise { try { return await response.json(); @@ -68,11 +77,14 @@ export async function parseApiPayload( const payload = await parseJsonPayload(response, path); - if (options.unwrapEnvelope && isEnvelopeLike(payload)) { - if (!payload.ok || payload.data == null) { - throw new Error(payload.error || `${path} failed`); + if (options.unwrapEnvelope && isEnvelopeStatusLike(payload)) { + if (!payload.ok) { + throw new Error(String(payload.error || `${path} failed`)); + } + if (!("data" in payload)) { + throw new Error(`${path} returned malformed envelope`); } - return payload.data; + return (payload as ApiEnvelope).data as T; } return payload as T; From ebb3356f941a53720d91fcde819e17d16147cebb Mon Sep 17 00:00:00 2001 From: hanityx Date: Mon, 18 May 2026 02:23:44 +0900 Subject: [PATCH 4/4] refactor(api): organize provider boundaries and dependency rules --- .dependency-cruiser.cjs | 47 ++++ apps/api-ts/src/app/create-server.test.ts | 15 +- apps/api-ts/src/app/create-server.ts | 10 +- .../routes/{platform.ts => platform/index.ts} | 26 +- .../recovery.ts} | 4 +- .../{providers.ts => providers/index.ts} | 45 ++-- .../index.test.ts} | 16 +- .../routes/{threads.ts => threads/index.ts} | 18 +- apps/api-ts/src/domains/ops/alert-hooks.ts | 2 +- apps/api-ts/src/domains/providers/actions.ts | 247 +----------------- .../src/domains/providers/adapters.test.ts | 16 +- apps/api-ts/src/domains/providers/adapters.ts | 58 +--- .../chatgpt.ts => adapters/chatgpt/roots.ts} | 12 +- .../claude/title.ts} | 8 +- .../codex.ts => adapters/codex/roots.ts} | 4 +- .../codex/title-map.ts} | 6 +- .../gemini/health.ts} | 6 +- .../src/domains/providers/capabilities.ts | 100 +++++++ .../providers/index.test.ts} | 4 +- apps/api-ts/src/domains/providers/index.ts | 96 +++++++ .../src/domains/providers/matrix.test.ts | 7 +- apps/api-ts/src/domains/providers/matrix.ts | 73 +----- .../src/domains/providers/path-safety.test.ts | 45 +++- .../src/domains/providers/path-safety.ts | 139 +--------- .../src/domains/providers/provider-roots.ts | 113 +------- apps/api-ts/src/domains/providers/registry.ts | 63 +++++ .../src/domains/providers/search.test.ts | 6 +- apps/api-ts/src/domains/providers/search.ts | 38 +-- .../{ => services}/actions/archives.ts | 4 +- .../{ => services}/actions/backups.ts | 8 +- .../{ => services}/actions/confirm-tokens.ts | 2 +- .../{ => services}/actions/execution.ts | 2 +- .../providers/services/actions/index.ts | 246 +++++++++++++++++ .../{ => services}/actions/responses.ts | 2 +- .../{ => services}/actions/selection.ts | 2 +- .../{ => services}/actions/targets.ts | 2 +- .../{ => services}/matrix/adapter-access.ts | 2 +- .../{ => services}/matrix/copilot.ts | 6 +- .../providers/services/matrix/index.ts | 72 +++++ .../{ => services}/matrix/provider-entries.ts | 4 +- .../{ => services}/matrix/signals.ts | 8 +- .../{ => services}/matrix/summary.ts | 2 +- .../providers/{ => services}/search/abort.ts | 0 .../providers/{ => services}/search/cache.ts | 2 +- .../search/codex-session-path.ts | 4 +- .../{ => services}/search/concurrency.ts | 0 .../{ => services}/search/constants.ts | 9 +- .../providers/services/search/index.ts | 37 +++ .../providers/{ => services}/search/limits.ts | 2 +- .../{ => services}/search/local-search.ts | 6 +- .../{ => services}/search/manifest-store.ts | 4 +- .../{ => services}/search/openable-threads.ts | 2 +- .../search/provider-session-reports.ts | 17 +- .../search/provider-session-scan.ts | 4 +- .../{ => services}/search/raw-files.ts | 4 +- .../{ => services}/search/row-search.ts | 6 +- .../search/session-accumulator.ts | 2 +- .../{ => services}/search/session-dedupe.ts | 6 +- .../search/session-hit-resolver.ts | 4 +- .../search/session-hit-search.ts | 6 +- .../search/session-manifest-files.ts | 8 +- .../search/session-manifest-rows.ts | 10 +- .../{ => services}/search/session-manifest.ts | 6 +- .../search/session-metadata-results.ts | 10 +- .../search/session-raw-search.ts | 2 +- .../{ => services}/search/session-results.ts | 2 +- .../search/session-row-order.ts | 2 +- .../{ => services}/search/session-search.ts | 2 +- .../search/transcript-loader-cache.ts | 0 .../search/transcript-matches.ts | 6 +- .../search/transcript-message-matches.ts | 4 +- .../search/transcript-policy.ts | 0 .../search/transcript-raw-matches.ts | 4 +- .../providers/{ => services}/search/types.ts | 2 +- .../providers/services/transcripts/index.ts | 91 +++++++ .../{ => services}/transcripts/json.ts | 4 +- .../{ => services}/transcripts/jsonl.ts | 4 +- .../{ => services}/transcripts/normalizers.ts | 2 +- .../backups.ts => shared/backup-roots.ts} | 0 .../domains/providers/shared/file-roots.ts | 134 ++++++++++ .../domains/providers/shared/path-safety.ts | 130 +++++++++ .../src/domains/providers/title-detection.ts | 4 +- .../src/domains/providers/transcript.test.ts | 2 +- .../src/domains/providers/transcript.ts | 92 +------ .../recovery/backups.ts} | 4 +- .../recovery/checklist.ts} | 4 +- .../recovery/export.ts} | 8 +- apps/api-ts/src/domains/recovery/index.ts | 1 + .../recovery/portable-zip.ts} | 0 apps/api-ts/src/domains/recovery/roadmap.ts | 2 +- .../recovery/runtime.test.ts} | 2 +- .../recovery/runtime.ts} | 14 +- .../recovery/smoke.ts} | 4 +- apps/api-ts/src/lib/providers.ts | 114 -------- apps/api-ts/src/server.test.ts | 2 +- package.json | 4 +- pnpm-lock.yaml | 180 +++++++++++++ 97 files changed, 1482 insertions(+), 1079 deletions(-) create mode 100644 .dependency-cruiser.cjs rename apps/api-ts/src/app/routes/{platform.ts => platform/index.ts} (92%) rename apps/api-ts/src/app/routes/{platform-recovery.ts => platform/recovery.ts} (99%) rename apps/api-ts/src/app/routes/{providers.ts => providers/index.ts} (90%) rename apps/api-ts/src/app/routes/{threads.test.ts => threads/index.test.ts} (94%) rename apps/api-ts/src/app/routes/{threads.ts => threads/index.ts} (96%) rename apps/api-ts/src/domains/providers/{roots/chatgpt.ts => adapters/chatgpt/roots.ts} (83%) rename apps/api-ts/src/domains/providers/{claude-title-detection.ts => adapters/claude/title.ts} (80%) rename apps/api-ts/src/domains/providers/{roots/codex.ts => adapters/codex/roots.ts} (97%) rename apps/api-ts/src/domains/providers/{codex-title-map.ts => adapters/codex/title-map.ts} (91%) rename apps/api-ts/src/domains/providers/{adapter-health.ts => adapters/gemini/health.ts} (81%) create mode 100644 apps/api-ts/src/domains/providers/capabilities.ts rename apps/api-ts/src/{lib/providers.test.ts => domains/providers/index.test.ts} (98%) create mode 100644 apps/api-ts/src/domains/providers/index.ts create mode 100644 apps/api-ts/src/domains/providers/registry.ts rename apps/api-ts/src/domains/providers/{ => services}/actions/archives.ts (96%) rename apps/api-ts/src/domains/providers/{ => services}/actions/backups.ts (97%) rename apps/api-ts/src/domains/providers/{ => services}/actions/confirm-tokens.ts (99%) rename apps/api-ts/src/domains/providers/{ => services}/actions/execution.ts (99%) create mode 100644 apps/api-ts/src/domains/providers/services/actions/index.ts rename apps/api-ts/src/domains/providers/{ => services}/actions/responses.ts (98%) rename apps/api-ts/src/domains/providers/{ => services}/actions/selection.ts (98%) rename apps/api-ts/src/domains/providers/{ => services}/actions/targets.ts (98%) rename apps/api-ts/src/domains/providers/{ => services}/matrix/adapter-access.ts (97%) rename apps/api-ts/src/domains/providers/{ => services}/matrix/copilot.ts (93%) create mode 100644 apps/api-ts/src/domains/providers/services/matrix/index.ts rename apps/api-ts/src/domains/providers/{ => services}/matrix/provider-entries.ts (98%) rename apps/api-ts/src/domains/providers/{ => services}/matrix/signals.ts (95%) rename apps/api-ts/src/domains/providers/{ => services}/matrix/summary.ts (97%) rename apps/api-ts/src/domains/providers/{ => services}/search/abort.ts (100%) rename apps/api-ts/src/domains/providers/{ => services}/search/cache.ts (98%) rename apps/api-ts/src/domains/providers/{ => services}/search/codex-session-path.ts (92%) rename apps/api-ts/src/domains/providers/{ => services}/search/concurrency.ts (100%) rename apps/api-ts/src/domains/providers/{ => services}/search/constants.ts (88%) create mode 100644 apps/api-ts/src/domains/providers/services/search/index.ts rename apps/api-ts/src/domains/providers/{ => services}/search/limits.ts (97%) rename apps/api-ts/src/domains/providers/{ => services}/search/local-search.ts (97%) rename apps/api-ts/src/domains/providers/{ => services}/search/manifest-store.ts (96%) rename apps/api-ts/src/domains/providers/{ => services}/search/openable-threads.ts (94%) rename apps/api-ts/src/domains/providers/{ => services}/search/provider-session-reports.ts (90%) rename apps/api-ts/src/domains/providers/{ => services}/search/provider-session-scan.ts (98%) rename apps/api-ts/src/domains/providers/{ => services}/search/raw-files.ts (98%) rename apps/api-ts/src/domains/providers/{ => services}/search/row-search.ts (97%) rename apps/api-ts/src/domains/providers/{ => services}/search/session-accumulator.ts (99%) rename apps/api-ts/src/domains/providers/{ => services}/search/session-dedupe.ts (95%) rename apps/api-ts/src/domains/providers/{ => services}/search/session-hit-resolver.ts (98%) rename apps/api-ts/src/domains/providers/{ => services}/search/session-hit-search.ts (97%) rename apps/api-ts/src/domains/providers/{ => services}/search/session-manifest-files.ts (96%) rename apps/api-ts/src/domains/providers/{ => services}/search/session-manifest-rows.ts (96%) rename apps/api-ts/src/domains/providers/{ => services}/search/session-manifest.ts (97%) rename apps/api-ts/src/domains/providers/{ => services}/search/session-metadata-results.ts (94%) rename apps/api-ts/src/domains/providers/{ => services}/search/session-raw-search.ts (96%) rename apps/api-ts/src/domains/providers/{ => services}/search/session-results.ts (97%) rename apps/api-ts/src/domains/providers/{ => services}/search/session-row-order.ts (97%) rename apps/api-ts/src/domains/providers/{ => services}/search/session-search.ts (99%) rename apps/api-ts/src/domains/providers/{ => services}/search/transcript-loader-cache.ts (100%) rename apps/api-ts/src/domains/providers/{ => services}/search/transcript-matches.ts (97%) rename apps/api-ts/src/domains/providers/{ => services}/search/transcript-message-matches.ts (96%) rename apps/api-ts/src/domains/providers/{ => services}/search/transcript-policy.ts (100%) rename apps/api-ts/src/domains/providers/{ => services}/search/transcript-raw-matches.ts (95%) rename apps/api-ts/src/domains/providers/{ => services}/search/types.ts (97%) create mode 100644 apps/api-ts/src/domains/providers/services/transcripts/index.ts rename apps/api-ts/src/domains/providers/{ => services}/transcripts/json.ts (98%) rename apps/api-ts/src/domains/providers/{ => services}/transcripts/jsonl.ts (98%) rename apps/api-ts/src/domains/providers/{ => services}/transcripts/normalizers.ts (99%) rename apps/api-ts/src/domains/providers/{roots/backups.ts => shared/backup-roots.ts} (100%) create mode 100644 apps/api-ts/src/domains/providers/shared/file-roots.ts create mode 100644 apps/api-ts/src/domains/providers/shared/path-safety.ts rename apps/api-ts/src/{lib/recovery-backups.ts => domains/recovery/backups.ts} (97%) rename apps/api-ts/src/{lib/recovery-checklist.ts => domains/recovery/checklist.ts} (94%) rename apps/api-ts/src/{lib/recovery-export.ts => domains/recovery/export.ts} (95%) create mode 100644 apps/api-ts/src/domains/recovery/index.ts rename apps/api-ts/src/{lib/recovery-portable-zip.ts => domains/recovery/portable-zip.ts} (100%) rename apps/api-ts/src/{lib/recovery.test.ts => domains/recovery/runtime.test.ts} (99%) rename apps/api-ts/src/{lib/recovery.ts => domains/recovery/runtime.ts} (97%) rename apps/api-ts/src/{lib/recovery-smoke.ts => domains/recovery/smoke.ts} (99%) delete mode 100644 apps/api-ts/src/lib/providers.ts diff --git a/.dependency-cruiser.cjs b/.dependency-cruiser.cjs new file mode 100644 index 0000000..84955ed --- /dev/null +++ b/.dependency-cruiser.cjs @@ -0,0 +1,47 @@ +/** @type {import('dependency-cruiser').IConfiguration} */ +module.exports = { + forbidden: [ + { + name: "domains-must-not-import-app", + severity: "error", + from: { path: "^apps/api-ts/src/domains/" }, + to: { path: "^apps/api-ts/src/app/" }, + }, + { + name: "lib-must-not-import-domains", + severity: "error", + from: { path: "^apps/api-ts/src/lib/" }, + to: { path: "^apps/api-ts/src/domains/" }, + }, + { + name: "lib-must-not-import-app", + severity: "error", + from: { path: "^apps/api-ts/src/lib/" }, + to: { path: "^apps/api-ts/src/app/" }, + }, + { + name: "provider-services-must-not-import-app", + severity: "error", + from: { path: "^apps/api-ts/src/domains/providers/services/" }, + to: { path: "^apps/api-ts/src/app/" }, + }, + ], + options: { + doNotFollow: { + path: "node_modules", + }, + exclude: { + path: "(^|/)(dist|dist-electron|coverage|node_modules)/", + }, + tsPreCompilationDeps: true, + enhancedResolveOptions: { + extensions: [".ts", ".tsx", ".js", ".jsx", ".json"], + }, + reporterOptions: { + dot: { + collapsePattern: + "node_modules/[^/]+|packages/[^/]+|apps/api-ts/src/(app|domains|lib|platform)", + }, + }, + }, +}; diff --git a/apps/api-ts/src/app/create-server.test.ts b/apps/api-ts/src/app/create-server.test.ts index 65337d9..502fd82 100644 --- a/apps/api-ts/src/app/create-server.test.ts +++ b/apps/api-ts/src/app/create-server.test.ts @@ -12,21 +12,28 @@ afterEach(() => { describe("parseConversationSearchProviders", () => { it("accepts comma-separated provider ids", () => { expect( - parseConversationSearchProviders("codex,chatgpt,copilot"), + parseConversationSearchProviders("codex,claude,copilot"), ).toEqual({ - providers: ["codex", "chatgpt", "copilot"], + providers: ["codex", "claude", "copilot"], invalid: [], }); }); it("dedupes repeated providers and reports invalid tokens", () => { expect( - parseConversationSearchProviders(["codex,chatgpt", "codex,unknown"]), + parseConversationSearchProviders(["codex,claude", "codex,unknown"]), ).toEqual({ - providers: ["codex", "chatgpt"], + providers: ["codex", "claude"], invalid: ["unknown"], }); }); + + it("keeps internal ChatGPT legacy data out of public search scope", () => { + expect(parseConversationSearchProviders("codex,chatgpt")).toEqual({ + providers: ["codex"], + invalid: ["chatgpt"], + }); + }); }); describe("local API auth guard", () => { diff --git a/apps/api-ts/src/app/create-server.ts b/apps/api-ts/src/app/create-server.ts index 31cc17d..5587176 100644 --- a/apps/api-ts/src/app/create-server.ts +++ b/apps/api-ts/src/app/create-server.ts @@ -29,7 +29,7 @@ import { } from "../lib/utils.js"; import type { ProviderId } from "../domains/providers/types.js"; -import { parseProviderId } from "../domains/providers/path-safety.js"; +import { parseSearchableProviderId } from "../domains/providers/capabilities.js"; import { getDataSourceInventoryTs } from "../domains/recovery/inventory.js"; import { invalidateOverviewTsCache } from "../domains/threads/overview.js"; import { @@ -38,9 +38,9 @@ import { import { registerPlatformRoutes, type ProxyRequest, -} from "./routes/platform.js"; -import { registerThreadRoutes } from "./routes/threads.js"; -import { registerProviderRoutes } from "./routes/providers.js"; +} from "./routes/platform/index.js"; +import { registerThreadRoutes } from "./routes/threads/index.js"; +import { registerProviderRoutes } from "./routes/providers/index.js"; type RuntimeCacheEntry = { expires_at: number; @@ -120,7 +120,7 @@ export function parseConversationSearchProviders( const providers: ProviderId[] = []; const invalid: string[] = []; for (const token of tokens) { - const parsed = parseProviderId(token); + const parsed = parseSearchableProviderId(token); if (!parsed) { invalid.push(token); continue; diff --git a/apps/api-ts/src/app/routes/platform.ts b/apps/api-ts/src/app/routes/platform/index.ts similarity index 92% rename from apps/api-ts/src/app/routes/platform.ts rename to apps/api-ts/src/app/routes/platform/index.ts index a19a552..3adef24 100644 --- a/apps/api-ts/src/app/routes/platform.ts +++ b/apps/api-ts/src/app/routes/platform/index.ts @@ -4,45 +4,45 @@ import { SCHEMA_VERSION, type AgentRuntimeState, } from "@threadlens/shared-contracts"; -import { getExecutionGraphData } from "../../execution-graph.js"; +import { getExecutionGraphData } from "../../../execution-graph.js"; import { APP_VERSION, CODEX_HOME, START_TS, -} from "../../lib/constants.js"; -import { checkForUpdates } from "../../lib/update-check.js"; +} from "../../../lib/constants.js"; +import { checkForUpdates } from "../../../lib/update-check.js"; import { cleanTitleText, envelope, parseQueryString, type QueryMap, withSchemaVersion, -} from "../../lib/utils.js"; +} from "../../../lib/utils.js"; import { getCompareAppsStatusTs, getLatestSmokeStatusTs, getRelatedToolsStatusTs, getRuntimeHealthTs, -} from "../../lib/recovery.js"; -import { getDataSourceInventoryTs } from "../../domains/recovery/inventory.js"; +} from "../../../domains/recovery/index.js"; +import { getDataSourceInventoryTs } from "../../../domains/recovery/inventory.js"; import { appendRoadmapCheckinTs, getRoadmapStatusTs, -} from "../../domains/recovery/roadmap.js"; -import { getOverviewTs } from "../../domains/threads/overview.js"; -import { getCodexObservatoryTs } from "../../domains/ops/observatory.js"; +} from "../../../domains/recovery/roadmap.js"; +import { getOverviewTs } from "../../../domains/threads/overview.js"; +import { getCodexObservatoryTs } from "../../../domains/ops/observatory.js"; import { evaluateAlertHooksTs, updateAlertHooksConfigTs, updateAlertRuleTs, -} from "../../domains/ops/alert-hooks.js"; +} from "../../../domains/ops/alert-hooks.js"; import { getAgentLoopsStatusTs, runAgentLoopActionTs, -} from "../../domains/ops/agent-loops.js"; -import { registerRecoveryRoutes } from "./platform-recovery.js"; +} from "../../../domains/ops/agent-loops.js"; +import { registerRecoveryRoutes } from "./recovery.js"; -export { issueRecoveryBackupDownloadTokenForTests } from "./platform-recovery.js"; +export { issueRecoveryBackupDownloadTokenForTests } from "./recovery.js"; export type ProxyRequest = FastifyRequest<{ Params: { "*": string }; diff --git a/apps/api-ts/src/app/routes/platform-recovery.ts b/apps/api-ts/src/app/routes/platform/recovery.ts similarity index 99% rename from apps/api-ts/src/app/routes/platform-recovery.ts rename to apps/api-ts/src/app/routes/platform/recovery.ts index 641a309..d266f8c 100644 --- a/apps/api-ts/src/app/routes/platform-recovery.ts +++ b/apps/api-ts/src/app/routes/platform/recovery.ts @@ -9,13 +9,13 @@ import { openRecoveryBackupArchiveReadStream, runRecoveryDrillTs, updateRecoveryChecklistItem, -} from "../../lib/recovery.js"; +} from "../../../domains/recovery/index.js"; import { envelope, pathExists, type QueryMap, withSchemaVersion, -} from "../../lib/utils.js"; +} from "../../../lib/utils.js"; const RECOVERY_BACKUP_DOWNLOAD_TOKEN_TTL_MS = 10 * 60 * 1000; diff --git a/apps/api-ts/src/app/routes/providers.ts b/apps/api-ts/src/app/routes/providers/index.ts similarity index 90% rename from apps/api-ts/src/app/routes/providers.ts rename to apps/api-ts/src/app/routes/providers/index.ts index 84e83fa..83ac3f5 100644 --- a/apps/api-ts/src/app/routes/providers.ts +++ b/apps/api-ts/src/app/routes/providers/index.ts @@ -2,28 +2,33 @@ import { execFile } from "node:child_process"; import path from "node:path"; import type { FastifyInstance } from "fastify"; import { z } from "zod"; -import type { ProviderId } from "../../domains/providers/types.js"; +import type { ProviderId } from "../../../domains/providers/types.js"; +import { + listProviderActionProviderIds, + listSessionReadableProviderIds, + parseSearchableProviderId, + parseSessionReadableProviderId, + parseTranscriptReadableProviderId, +} from "../../../domains/providers/capabilities.js"; import { - listProviderIds, - parseProviderId, resolveAllowedProviderFilePath, -} from "../../domains/providers/path-safety.js"; -import { getProviderMatrixTs } from "../../domains/providers/matrix.js"; -import { getProviderSessionTranscript } from "../../domains/providers/session-transcript-service.js"; -import { runProviderSessionAction } from "../../lib/providers.js"; +} from "../../../domains/providers/path-safety.js"; +import { getProviderMatrixTs } from "../../../domains/providers/matrix.js"; +import { getProviderSessionTranscript } from "../../../domains/providers/session-transcript-service.js"; +import { runProviderSessionAction } from "../../../domains/providers/index.js"; import { getProviderParserHealthTs, getProviderSessionsTs, searchConversationSessionHitsTs, searchLocalConversationsTs, -} from "../../domains/providers/search.js"; +} from "../../../domains/providers/search.js"; import { envelope, parseQueryNumber, pathExists, type QueryMap, withSchemaVersion, -} from "../../lib/utils.js"; +} from "../../../lib/utils.js"; export async function registerProviderRoutes( app: FastifyInstance, @@ -33,11 +38,17 @@ export async function registerProviderRoutes( ) => { providers?: ProviderId[]; invalid: string[] }; }, ): Promise { - const providerIds = listProviderIds(); - const providerIdTuple = providerIds as [ProviderId, ...ProviderId[]]; + const providerActionIdTuple = listProviderActionProviderIds() as [ + ProviderId, + ...ProviderId[], + ]; + const sessionReadableProviderIdTuple = listSessionReadableProviderIds() as [ + ProviderId, + ...ProviderId[], + ]; const providerSessionActionSchema = z.object({ - provider: z.enum(providerIdTuple), + provider: z.enum(providerActionIdTuple), action: z.enum(["backup_local", "archive_local", "unarchive_local", "delete_local"]), file_paths: z.array(z.string().min(1)).min(1).max(500), dry_run: z.boolean().optional().default(true), @@ -46,7 +57,7 @@ export async function registerProviderRoutes( backup_root: z.string().optional().default(""), }); const providerOpenFolderSchema = z.object({ - provider: z.enum(providerIdTuple), + provider: z.enum(sessionReadableProviderIdTuple), file_path: z.string().min(1), }); @@ -170,7 +181,7 @@ export async function registerProviderRoutes( const limitRaw = Array.isArray(req.query.limit) ? req.query.limit[0] : req.query.limit; - const provider = parseProviderId(providerRaw); + const provider = parseSessionReadableProviderId(providerRaw); if (providerRaw && !provider) { return reply.code(400).send(envelope(null, "invalid provider")); } @@ -201,7 +212,7 @@ export async function registerProviderRoutes( const limitRaw = Array.isArray(req.query.limit) ? req.query.limit[0] : req.query.limit; - const provider = parseProviderId(providerRaw); + const provider = parseSessionReadableProviderId(providerRaw); if (providerRaw && !provider) { return reply.code(400).send(envelope(null, "invalid provider")); } @@ -297,7 +308,7 @@ export async function registerProviderRoutes( const providerRaw = Array.isArray(req.query.provider) ? req.query.provider[0] : req.query.provider; - const provider = parseProviderId(providerRaw); + const provider = parseSearchableProviderId(providerRaw); if (!provider) { return reply.code(400).send(envelope(null, "invalid provider")); } @@ -364,7 +375,7 @@ export async function registerProviderRoutes( const limitRaw = Array.isArray(req.query.limit) ? req.query.limit[0] : req.query.limit; - const provider = parseProviderId(providerRaw); + const provider = parseTranscriptReadableProviderId(providerRaw); if (!provider) { return reply.code(400).send(envelope(null, "invalid provider")); } diff --git a/apps/api-ts/src/app/routes/threads.test.ts b/apps/api-ts/src/app/routes/threads/index.test.ts similarity index 94% rename from apps/api-ts/src/app/routes/threads.test.ts rename to apps/api-ts/src/app/routes/threads/index.test.ts index 76fe5b6..ecc02a7 100644 --- a/apps/api-ts/src/app/routes/threads.test.ts +++ b/apps/api-ts/src/app/routes/threads/index.test.ts @@ -23,7 +23,7 @@ vi.mock("node:child_process", () => ({ execFile: mockExecFile, })); -vi.mock("../../domains/threads/state.js", () => ({ +vi.mock("../../../domains/threads/state.js", () => ({ archiveThreadsLocalTs: mockArchiveThreadsLocalTs, unarchiveThreadsLocalTs: mockUnarchiveThreadsLocalTs, getThreadResumeCommandsTs: mockGetThreadResumeCommandsTs, @@ -31,33 +31,33 @@ vi.mock("../../domains/threads/state.js", () => ({ setThreadPinnedTs: mockSetThreadPinnedTs, })); -vi.mock("../../domains/threads/cleanup.js", () => ({ +vi.mock("../../../domains/threads/cleanup.js", () => ({ analyzeDeleteTs: mockAnalyzeDeleteTs, executeBackupCleanupTs: mockExecuteBackupCleanupTs, executeLocalCleanupTs: mockExecuteLocalCleanupTs, })); -vi.mock("../../domains/threads/forensics.js", () => ({ +vi.mock("../../../domains/threads/forensics.js", () => ({ getThreadForensicsTs: mockGetThreadForensicsTs, })); -vi.mock("../../domains/threads/query.js", () => ({ +vi.mock("../../../domains/threads/query.js", () => ({ getThreadsTs: mockGetThreadsTs, })); -vi.mock("../../domains/providers/transcript.js", () => ({ +vi.mock("../../../domains/providers/transcript.js", () => ({ buildSessionTranscript: mockBuildSessionTranscript, })); -vi.mock("../../domains/providers/title-detection.js", () => ({ +vi.mock("../../../domains/providers/title-detection.js", () => ({ invalidateCodexThreadTitleMapCache: mockInvalidateCodexThreadTitleMapCache, })); -vi.mock("../../domains/providers/search.js", () => ({ +vi.mock("../../../domains/providers/search.js", () => ({ resolveCodexSessionPathByThreadId: mockResolveCodexSessionPathByThreadId, })); -import { registerThreadRoutes } from "./threads.js"; +import { registerThreadRoutes } from "./index.js"; describe("registerThreadRoutes cleanup invalidation", () => { let app: FastifyInstance; diff --git a/apps/api-ts/src/app/routes/threads.ts b/apps/api-ts/src/app/routes/threads/index.ts similarity index 96% rename from apps/api-ts/src/app/routes/threads.ts rename to apps/api-ts/src/app/routes/threads/index.ts index 2d93f21..e36bb1d 100644 --- a/apps/api-ts/src/app/routes/threads.ts +++ b/apps/api-ts/src/app/routes/threads/index.ts @@ -12,27 +12,27 @@ import { renameThreadTitleTs, setThreadPinnedTs, unarchiveThreadsLocalTs, -} from "../../domains/threads/state.js"; +} from "../../../domains/threads/state.js"; import { analyzeDeleteTs, executeBackupCleanupTs, executeLocalCleanupTs, -} from "../../domains/threads/cleanup.js"; -import { getThreadForensicsTs } from "../../domains/threads/forensics.js"; -import { parseSafeThreadId } from "../../domains/threads/thread-id.js"; -import { getThreadsTs } from "../../domains/threads/query.js"; +} from "../../../domains/threads/cleanup.js"; +import { getThreadForensicsTs } from "../../../domains/threads/forensics.js"; +import { parseSafeThreadId } from "../../../domains/threads/thread-id.js"; +import { getThreadsTs } from "../../../domains/threads/query.js"; import { buildSessionTranscript, -} from "../../domains/providers/transcript.js"; -import { invalidateCodexThreadTitleMapCache } from "../../domains/providers/title-detection.js"; -import { resolveCodexSessionPathByThreadId } from "../../domains/providers/search.js"; +} from "../../../domains/providers/transcript.js"; +import { invalidateCodexThreadTitleMapCache } from "../../../domains/providers/title-detection.js"; +import { resolveCodexSessionPathByThreadId } from "../../../domains/providers/search.js"; import { envelope, isRecord, pathExists, type QueryMap, withSchemaVersion, -} from "../../lib/utils.js"; +} from "../../../lib/utils.js"; async function openDirectoryInOs(directoryPath: string): Promise { const [command, args] = diff --git a/apps/api-ts/src/domains/ops/alert-hooks.ts b/apps/api-ts/src/domains/ops/alert-hooks.ts index aeacc77..4f8a472 100644 --- a/apps/api-ts/src/domains/ops/alert-hooks.ts +++ b/apps/api-ts/src/domains/ops/alert-hooks.ts @@ -6,7 +6,7 @@ import { ALERT_RULES_FILE, ALERT_STATE_FILE, } from "../../lib/constants.js"; -import { getRuntimeHealthTs } from "../../lib/recovery.js"; +import { getRuntimeHealthTs } from "../recovery/index.js"; import { isRecord, nowIsoUtc, diff --git a/apps/api-ts/src/domains/providers/actions.ts b/apps/api-ts/src/domains/providers/actions.ts index 5ee37cf..0f85028 100644 --- a/apps/api-ts/src/domains/providers/actions.ts +++ b/apps/api-ts/src/domains/providers/actions.ts @@ -1,246 +1 @@ -import { BACKUP_ROOT } from "../../lib/constants.js"; -import { - deriveProviderBackupId, - resolveProviderActionBackupRoot, - stageProviderActionBackup, -} from "./actions/backups.js"; -import { - consumeProviderActionConfirmToken, - issueProviderActionConfirmToken, - pruneProviderActionTokens, -} from "./actions/confirm-tokens.js"; -import { - buildProviderActionFingerprint, - normalizeProviderActionOptions, -} from "./actions/selection.js"; -import { - buildProviderActionAppliedResponse, - buildProviderActionFailureResponse, - buildProviderActionPreviewResponse, -} from "./actions/responses.js"; -import { - type ProviderActionExecutionFailure, - runArchiveProviderAction, - runDeleteProviderAction, - runUnarchiveProviderAction, -} from "./actions/execution.js"; -import { - normalizeProviderActionTargetPaths, - resolveProviderActionTargets, -} from "./actions/targets.js"; -import type { - ProviderId, - ProviderSessionAction, - ProviderSessionActionOptions, -} from "./types.js"; - -type ProviderActionDeps = { - resolveAllowedProviderFilePath: ( - provider: ProviderId, - filePath: string, - ) => Promise; - supportsProviderAction: ( - provider: ProviderId, - action: ProviderSessionAction, - ) => boolean; - invalidateProviderCaches: (provider: ProviderId) => void; -}; - -export { buildProviderActionFingerprint, buildProviderActionToken } from "./actions/selection.js"; -export { deriveProviderBackupRelativePath } from "./actions/backups.js"; - -export async function runProviderSessionAction( - deps: ProviderActionDeps, - provider: ProviderId, - action: ProviderSessionAction, - filePaths: string[], - dryRun: boolean, - confirmToken: string, - options?: ProviderSessionActionOptions, -) { - pruneProviderActionTokens(); - const normalizedOptions = normalizeProviderActionOptions(options); - const uniquePaths = normalizeProviderActionTargetPaths(filePaths); - if (!deps.supportsProviderAction(provider, action)) { - return buildProviderActionFailureResponse({ - provider, - action, - dryRun, - targetCount: uniquePaths.length, - validCount: 0, - selectionFingerprint: "", - options: normalizedOptions, - skipped: [], - error: "cleanup-disabled-provider", - }); - } - const { skipped, valid } = await resolveProviderActionTargets( - provider, - uniquePaths, - deps.resolveAllowedProviderFilePath, - ); - if (!valid.length && !dryRun) { - return buildProviderActionFailureResponse({ - provider, - action, - dryRun: false, - targetCount: uniquePaths.length, - validCount: valid.length, - selectionFingerprint: "", - options: normalizedOptions, - skipped, - error: "no-valid-targets", - }); - } - - const selectionFingerprint = valid.length - ? buildProviderActionFingerprint(provider, action, valid, normalizedOptions) - : ""; - const shouldBackup = - action === "backup_local" || - normalizedOptions.backup_before_delete; - const backupRootResult = await resolveProviderActionBackupRoot( - normalizedOptions.backup_root, - ); - if (shouldBackup && !backupRootResult.ok) { - return buildProviderActionFailureResponse({ - provider, - action, - dryRun, - targetCount: uniquePaths.length, - validCount: valid.length, - selectionFingerprint, - options: normalizedOptions, - skipped, - error: backupRootResult.error, - }); - } - - if (dryRun) { - const expectedToken = valid.length - ? issueProviderActionConfirmToken(provider, action, valid, normalizedOptions) - : ""; - return buildProviderActionPreviewResponse({ - provider, - action, - dryRun: true, - targetCount: uniquePaths.length, - validCount: valid.length, - selectionFingerprint, - options: normalizedOptions, - skipped, - confirmTokenExpected: expectedToken, - }); - } - - if (action !== "backup_local") { - const consume = consumeProviderActionConfirmToken( - confirmToken, - provider, - action, - valid, - normalizedOptions, - ); - if (!consume.ok) { - const expectedToken = valid.length - ? issueProviderActionConfirmToken(provider, action, valid, normalizedOptions) - : ""; - return buildProviderActionFailureResponse({ - provider, - action, - dryRun: false, - targetCount: uniquePaths.length, - validCount: valid.length, - selectionFingerprint, - options: normalizedOptions, - skipped, - confirmTokenExpected: expectedToken, - error: consume.reason, - }); - } - } - - let applied = 0; - let archivedTo: string | null = null; - let backupTo: string | null = null; - let backupManifestPath: string | null = null; - let backedUpCount = 0; - const failed: ProviderActionExecutionFailure[] = []; - const backupRoot = shouldBackup - ? backupRootResult.ok - ? backupRootResult.backupRoot - : (() => { - throw new Error(`unreachable invalid backup root: ${backupRootResult.error}`); - })() - : BACKUP_ROOT; - const backupStage = shouldBackup - ? await stageProviderActionBackup(provider, action, valid, backupRoot) - : null; - if (backupStage) { - failed.push(...backupStage.failed); - backupTo = backupStage.destination; - backupManifestPath = backupStage.manifest_path; - backedUpCount = backupStage.items.length; - } - const backupManifestWriteFailed = - backupStage?.failed.some((failure) => failure.step === "manifest_write") ?? false; - if (backupManifestWriteFailed && action !== "backup_local") { - return buildProviderActionAppliedResponse({ - provider, - action, - targetCount: uniquePaths.length, - validCount: valid.length, - selectionFingerprint, - options: normalizedOptions, - skipped, - failed, - applied: 0, - backedUpCount, - backupId: deriveProviderBackupId(backupRoot, backupTo), - backupTo, - backupManifestPath, - shouldBackup, - backupStageFailedCount: backupStage?.failed.length ?? 0, - archivedTo, - }); - } - - if (action === "backup_local") { - applied = backupStage?.items.length ?? 0; - } else if (action === "archive_local") { - const result = await runArchiveProviderAction(provider, valid); - applied = result.applied; - archivedTo = result.archivedTo; - failed.push(...result.failed); - } else if (action === "unarchive_local") { - const result = await runUnarchiveProviderAction(provider, valid); - applied = result.applied; - failed.push(...result.failed); - } else { - const deleteTargets = backupStage?.items.map((item) => item.source_path) ?? valid; - const result = await runDeleteProviderAction(deleteTargets); - applied = result.applied; - failed.push(...result.failed); - } - - deps.invalidateProviderCaches(provider); - - return buildProviderActionAppliedResponse({ - provider, - action, - targetCount: uniquePaths.length, - validCount: valid.length, - selectionFingerprint, - options: normalizedOptions, - skipped, - failed, - applied, - backedUpCount, - backupId: deriveProviderBackupId(backupRoot, backupTo), - backupTo, - backupManifestPath, - shouldBackup, - backupStageFailedCount: backupStage?.failed.length ?? 0, - archivedTo, - }); -} +export * from "./services/actions/index.js"; diff --git a/apps/api-ts/src/domains/providers/adapters.test.ts b/apps/api-ts/src/domains/providers/adapters.test.ts index 19e2517..f826e7e 100644 --- a/apps/api-ts/src/domains/providers/adapters.test.ts +++ b/apps/api-ts/src/domains/providers/adapters.test.ts @@ -1,25 +1,29 @@ import { getProviderCapability, - PROVIDER_IDS, } from "@threadlens/shared-contracts"; import { describe, expect, it } from "vitest"; +import { + IMPLEMENTED_PROVIDER_IDS, +} from "./capabilities.js"; import { type ProviderAdapter, type ProviderSessionLocator, getProviderAdapter, listProviderAdapters, PROVIDER_ADAPTERS, -} from "./adapters.js"; +} from "./registry.js"; import { providerRootSpecs } from "./path-safety.js"; describe("provider adapters", () => { it("registers one internal adapter for each provider contract id", () => { - expect(listProviderAdapters().map((adapter) => adapter.id)).toEqual(PROVIDER_IDS); - expect(Object.keys(PROVIDER_ADAPTERS)).toEqual([...PROVIDER_IDS]); + expect(listProviderAdapters().map((adapter) => adapter.id)).toEqual( + IMPLEMENTED_PROVIDER_IDS, + ); + expect(Object.keys(PROVIDER_ADAPTERS)).toEqual([...IMPLEMENTED_PROVIDER_IDS]); }); it("keeps adapter roots equivalent to current provider root specs", () => { - for (const provider of PROVIDER_IDS) { + for (const provider of IMPLEMENTED_PROVIDER_IDS) { const adapter = getProviderAdapter(provider); expect(adapter).toBeDefined(); expect(adapter?.roots()).toEqual(providerRootSpecs(provider)); @@ -31,7 +35,7 @@ describe("provider adapters", () => { }); it("does not duplicate provider capabilities inside adapters", () => { - for (const provider of PROVIDER_IDS) { + for (const provider of IMPLEMENTED_PROVIDER_IDS) { const adapter = getProviderAdapter(provider); expect(adapter).toBeDefined(); if (!adapter) throw new Error(`missing adapter for ${provider}`); diff --git a/apps/api-ts/src/domains/providers/adapters.ts b/apps/api-ts/src/domains/providers/adapters.ts index c09ff1d..0e8ed1b 100644 --- a/apps/api-ts/src/domains/providers/adapters.ts +++ b/apps/api-ts/src/domains/providers/adapters.ts @@ -1,57 +1 @@ -import { - PROVIDER_IDS, - PROVIDER_LABELS, - type ProviderId, -} from "@threadlens/shared-contracts"; -import { getGeminiProviderHealth } from "./adapter-health.js"; -import { providerRootSpecs, providerScanRootSpecs } from "./path-safety.js"; -import type { ProviderHealthEvidence, ProviderRootSpec } from "./types.js"; - -export type ProviderSessionLocator = - | { kind: "file"; file_path: string } - | { kind: "sqlite"; db_path: string; session_id: string }; - -export type ProviderAdapter = { - id: ProviderId; - label: string; - roots(): ProviderRootSpec[]; - scanRoots?(): Promise; - health?(): Promise; -}; - -function providerHealth(provider: ProviderId): ProviderAdapter["health"] { - if (provider === "gemini") return getGeminiProviderHealth; - return undefined; -} - -export const PROVIDER_ADAPTERS = Object.freeze( - Object.fromEntries( - PROVIDER_IDS.map((id) => { - const health = providerHealth(id); - return [ - id, - Object.freeze({ - id, - label: PROVIDER_LABELS[id], - roots: () => providerRootSpecs(id), - scanRoots: () => providerScanRootSpecs(id), - ...(health ? { health } : {}), - } satisfies ProviderAdapter), - ]; - }), - ), -) as Readonly>; - -const PROVIDER_ADAPTERS_BY_ID: Readonly< - Partial> -> = PROVIDER_ADAPTERS; - -export function getProviderAdapter( - provider: ProviderId, -): ProviderAdapter | undefined { - return PROVIDER_ADAPTERS_BY_ID[provider]; -} - -export function listProviderAdapters(): ProviderAdapter[] { - return PROVIDER_IDS.map((id) => PROVIDER_ADAPTERS[id]); -} +export * from "./registry.js"; diff --git a/apps/api-ts/src/domains/providers/roots/chatgpt.ts b/apps/api-ts/src/domains/providers/adapters/chatgpt/roots.ts similarity index 83% rename from apps/api-ts/src/domains/providers/roots/chatgpt.ts rename to apps/api-ts/src/domains/providers/adapters/chatgpt/roots.ts index d5eeb0f..ac766ba 100644 --- a/apps/api-ts/src/domains/providers/roots/chatgpt.ts +++ b/apps/api-ts/src/domains/providers/adapters/chatgpt/roots.ts @@ -1,10 +1,10 @@ import { readdir } from "node:fs/promises"; import path from "node:path"; -import { CHAT_DIR } from "../../../lib/constants.js"; -import { pathExists } from "../../../lib/utils.js"; +import { CHAT_DIR } from "../../../../lib/constants.js"; +import { pathExists } from "../../../../lib/utils.js"; import type { ProviderRootSpec, -} from "../types.js"; +} from "../../types.js"; type ChatGptRootsCacheEntry = { expires_at: number; @@ -14,6 +14,10 @@ type ChatGptRootsCacheEntry = { const CHATGPT_ROOT_DISCOVERY_TTL_MS = 45_000; let chatGptRootsCache: ChatGptRootsCacheEntry | null = null; +export function invalidateChatGptConversationRootsCache() { + chatGptRootsCache = null; +} + export async function discoverChatGptConversationRoots(): Promise { const now = Date.now(); if (chatGptRootsCache && chatGptRootsCache.expires_at > now) { @@ -42,7 +46,7 @@ export async function discoverChatGptConversationRoots(): Promise { const rootExists = await pathExists(GEMINI_HOME); diff --git a/apps/api-ts/src/domains/providers/capabilities.ts b/apps/api-ts/src/domains/providers/capabilities.ts new file mode 100644 index 0000000..892fca8 --- /dev/null +++ b/apps/api-ts/src/domains/providers/capabilities.ts @@ -0,0 +1,100 @@ +import { + findProviderCapability, + getProviderCapability, + type ProviderCapability, + type ProviderId, +} from "@threadlens/shared-contracts"; + +import type { ProviderSessionAction } from "./types.js"; + +export const IMPLEMENTED_PROVIDER_IDS = [ + "codex", + "chatgpt", + "claude", + "gemini", + "copilot", +] as const satisfies readonly ProviderId[]; + +function isImplementedProviderId(value: string): value is ProviderId { + return (IMPLEMENTED_PROVIDER_IDS as readonly string[]).includes(value); +} + +function implementedCapabilities(): ProviderCapability[] { + return IMPLEMENTED_PROVIDER_IDS.map((provider) => getProviderCapability(provider)); +} + +export function listProviderIds(): ProviderId[] { + return [...IMPLEMENTED_PROVIDER_IDS]; +} + +export function parseProviderId(raw: unknown): ProviderId | undefined { + const value = String(raw ?? "") + .trim() + .toLowerCase(); + return isImplementedProviderId(value) ? value : undefined; +} + +export function listSearchableProviderIds(): ProviderId[] { + return implementedCapabilities() + .filter( + (capability) => + capability.search_scope_visibility === "public" && + capability.read_sessions && + capability.analyze_context, + ) + .map((capability) => capability.id); +} + +export function parseSearchableProviderId(raw: unknown): ProviderId | undefined { + const provider = parseProviderId(raw); + if (!provider) return undefined; + return listSearchableProviderIds().includes(provider) ? provider : undefined; +} + +export function listSessionReadableProviderIds(): ProviderId[] { + return implementedCapabilities() + .filter((capability) => capability.read_sessions) + .map((capability) => capability.id); +} + +export function parseSessionReadableProviderId(raw: unknown): ProviderId | undefined { + const provider = parseProviderId(raw); + if (!provider) return undefined; + return listSessionReadableProviderIds().includes(provider) ? provider : undefined; +} + +export function listTranscriptReadableProviderIds(): ProviderId[] { + return implementedCapabilities() + .filter((capability) => capability.read_transcript) + .map((capability) => capability.id); +} + +export function parseTranscriptReadableProviderId(raw: unknown): ProviderId | undefined { + const provider = parseProviderId(raw); + if (!provider) return undefined; + return listTranscriptReadableProviderIds().includes(provider) ? provider : undefined; +} + +export function listProviderActionProviderIds(): ProviderId[] { + return implementedCapabilities() + .filter( + (capability) => + capability.read_sessions || capability.safe_cleanup || capability.hard_delete, + ) + .map((capability) => capability.id); +} + +export function supportsProviderAction( + provider: ProviderId, + action: ProviderSessionAction, +): boolean { + const capability = findProviderCapability(provider); + if (!capability) return false; + if (action === "backup_local") { + return capability.read_sessions === true; + } + if (action === "delete_local") { + return capability.safe_cleanup === true && capability.hard_delete === true; + } + return capability.safe_cleanup === true; +} diff --git a/apps/api-ts/src/lib/providers.test.ts b/apps/api-ts/src/domains/providers/index.test.ts similarity index 98% rename from apps/api-ts/src/lib/providers.test.ts rename to apps/api-ts/src/domains/providers/index.test.ts index 8b17503..2253691 100644 --- a/apps/api-ts/src/lib/providers.test.ts +++ b/apps/api-ts/src/domains/providers/index.test.ts @@ -3,7 +3,7 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; import type { ProviderId } from "@threadlens/shared-contracts"; -import { APP_DATA_DIR, CHAT_DIR } from "./constants.js"; +import { APP_DATA_DIR, CHAT_DIR } from "../../lib/constants.js"; import { buildProviderActionToken, codexTranscriptSearchRoots, @@ -13,7 +13,7 @@ import { providerRootSpecs, resolveSafePathWithinRoots, runProviderSessionAction, -} from "./providers"; +} from "./index.js"; describe("buildProviderActionToken", () => { it("is stable regardless of file path order", () => { diff --git a/apps/api-ts/src/domains/providers/index.ts b/apps/api-ts/src/domains/providers/index.ts new file mode 100644 index 0000000..2da2e85 --- /dev/null +++ b/apps/api-ts/src/domains/providers/index.ts @@ -0,0 +1,96 @@ +import { + buildProviderActionToken, + runProviderSessionAction as runProviderSessionActionInternal, +} from "./actions.js"; +import { invalidateProviderMatrixCache } from "./matrix.js"; +import { resolveAllowedProviderFilePath } from "./path-safety.js"; +import { invalidateProviderSearchCaches } from "./search.js"; +import { buildSessionTranscript } from "./transcript.js"; +import { supportsProviderAction } from "./capabilities.js"; +import type { ProviderId, ProviderSessionActionOptions } from "./types.js"; + +export type * from "./types.js"; +export type { ProviderAdapter } from "./registry.js"; + +export { + getProviderAdapter, + listProviderAdapters, +} from "./registry.js"; +export { + listProviderIds, + listProviderActionProviderIds, + parseProviderId, + listSearchableProviderIds, + parseSearchableProviderId, + listSessionReadableProviderIds, + parseSessionReadableProviderId, + listTranscriptReadableProviderIds, + parseTranscriptReadableProviderId, + supportsProviderAction, +} from "./capabilities.js"; +export { + providerName, + providerRootSpecs, + providerScanRootSpecs, + codexTranscriptSearchRoots, + isAllowedProviderFilePath, + isPathInsideRoot, + resolveSafePathWithinRoots, + resolveAllowedProviderFilePath, +} from "./path-safety.js"; +export { + inferSessionId, + isWorkspaceChatSessionPath, + isCopilotGlobalSessionLikeFile, + probeSessionFile, +} from "./probe.js"; +export { + invalidateCodexThreadTitleMapCache, + extractCodexThreadIdFromSessionName, + normalizeDetectedTitle, + detectSessionTitleFromHead, + detectClaudeRenamedTitle, + getCodexThreadTitleMap, + fallbackDisplayTitle as buildDisplayTitleFallback, +} from "./title-detection.js"; +export { + providerStatus, + capabilityLevel, + invalidateProviderMatrixCache, + getProviderMatrixTs, +} from "./matrix.js"; +export { + normalizeSearchText, + normalizeSearchQuery, + buildSearchTokens, + matchesConversationSearch, + buildSearchSnippet, + fallbackDisplayTitle, +} from "./search-helpers.js"; +export { buildProviderActionToken, buildSessionTranscript }; + +export async function runProviderSessionAction( + provider: ProviderId, + action: import("./types.js").ProviderSessionAction, + filePaths: string[], + dryRun: boolean, + confirmToken: string, + options?: ProviderSessionActionOptions, +) { + return runProviderSessionActionInternal( + { + resolveAllowedProviderFilePath, + supportsProviderAction, + invalidateProviderCaches: (targetProvider) => { + invalidateProviderSearchCaches(targetProvider); + invalidateProviderMatrixCache(); + }, + }, + provider, + action, + filePaths, + dryRun, + confirmToken, + options, + ); +} diff --git a/apps/api-ts/src/domains/providers/matrix.test.ts b/apps/api-ts/src/domains/providers/matrix.test.ts index 8e18b77..89284b9 100644 --- a/apps/api-ts/src/domains/providers/matrix.test.ts +++ b/apps/api-ts/src/domains/providers/matrix.test.ts @@ -10,6 +10,7 @@ vi.mock("@threadlens/shared-contracts", async (importOriginal) => { }); vi.mock("../../lib/constants.js", () => ({ + BACKUP_ROOT: "/mock/backups", CHAT_DIR: "/mock/chat", CODEX_HOME: "/mock/codex", CLAUDE_HOME: "/mock/claude", @@ -53,7 +54,7 @@ vi.mock("./probe.js", () => ({ isWorkspaceChatSessionPath: () => true, })); -import { listProviderAdapters } from "./adapters.js"; +import { listProviderAdapters } from "./registry.js"; import { getProviderMatrixTs, invalidateProviderMatrixCache } from "./matrix.js"; describe("provider matrix notes", () => { @@ -115,8 +116,8 @@ describe("provider matrix notes", () => { const data = await getProviderMatrixTs({ forceRefresh: true }); const copilot = data.providers.find((provider) => provider.provider === "copilot"); - expect(copilot?.evidence.session_log_count).toBe(3); - expect(copilot?.evidence.roots).toContain("/mock/backups/copilot"); + expect(copilot?.evidence.session_log_count).toBeGreaterThanOrEqual(3); + expect(copilot?.evidence.roots).toContain("/mock/backups/provider_actions/copilot"); }); it("uses adapter health evidence for Gemini without changing the matrix shape", async () => { diff --git a/apps/api-ts/src/domains/providers/matrix.ts b/apps/api-ts/src/domains/providers/matrix.ts index bf712f4..d3f9f7a 100644 --- a/apps/api-ts/src/domains/providers/matrix.ts +++ b/apps/api-ts/src/domains/providers/matrix.ts @@ -1,72 +1 @@ -import { - nowIsoUtc, -} from "../../lib/utils.js"; -import { - buildProviderMatrixProviders, -} from "./matrix/provider-entries.js"; -import { readProviderMatrixSignals } from "./matrix/signals.js"; -import { - buildProviderMatrixSummary, -} from "./matrix/summary.js"; -import type { ProviderMatrixData } from "./types.js"; - -type ProviderMatrixCacheEntry = { - expires_at: number; - data: ProviderMatrixData; -}; - -const PROVIDER_MATRIX_CACHE_TTL_MS = 30_000; -let providerMatrixCache: ProviderMatrixCacheEntry | null = null; -let providerMatrixInflight: Promise | null = null; - -export { capabilityLevel, providerStatus } from "./matrix/summary.js"; - -export function invalidateProviderMatrixCache() { - providerMatrixCache = null; -} - -async function buildProviderMatrixData(): Promise { - const signals = await readProviderMatrixSignals(); - const providers = buildProviderMatrixProviders(signals); - - const summary = buildProviderMatrixSummary(providers); - - return { - generated_at: nowIsoUtc(), - mode: "multi-provider-phase-1", - summary, - providers, - policy: { - cleanup_gate: "provider capability matrix controls destructive actions", - default_non_codex: "all detected providers are visible for local analysis", - }, - }; -} - -export async function getProviderMatrixTs(options?: { forceRefresh?: boolean }) { - const forceRefresh = Boolean(options?.forceRefresh); - const now = Date.now(); - if (!forceRefresh && providerMatrixCache && providerMatrixCache.expires_at > now) { - return providerMatrixCache.data; - } - if (forceRefresh) { - providerMatrixCache = null; - } - if (providerMatrixInflight) { - return providerMatrixInflight; - } - - providerMatrixInflight = buildProviderMatrixData() - .then((data) => { - providerMatrixCache = { - expires_at: Date.now() + PROVIDER_MATRIX_CACHE_TTL_MS, - data, - }; - return data; - }) - .finally(() => { - providerMatrixInflight = null; - }); - - return providerMatrixInflight; -} +export * from "./services/matrix/index.js"; diff --git a/apps/api-ts/src/domains/providers/path-safety.test.ts b/apps/api-ts/src/domains/providers/path-safety.test.ts index 86e6901..d6b3562 100644 --- a/apps/api-ts/src/domains/providers/path-safety.test.ts +++ b/apps/api-ts/src/domains/providers/path-safety.test.ts @@ -4,10 +4,19 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { getProviderCapability, - PROVIDER_IDS, type ProviderId, } from "@threadlens/shared-contracts"; import { APP_DATA_DIR, CHAT_DIR, CODEX_HOME } from "../../lib/constants.js"; +import { + IMPLEMENTED_PROVIDER_IDS, +} from "./capabilities.js"; +import { + isValidProviderRootSource, + providerScanRootSpecs, +} from "./provider-roots.js"; +import { + invalidateChatGptConversationRootsCache, +} from "./adapters/chatgpt/roots.js"; import { codexTranscriptSearchRoots, isAllowedProviderFilePath, @@ -64,7 +73,7 @@ describe("provider path safety", () => { it("keeps cleanup capability source of truth explicit for current providers", () => { expect( Object.fromEntries( - PROVIDER_IDS.map((provider) => [ + IMPLEMENTED_PROVIDER_IDS.map((provider) => [ provider, getProviderCapability(provider).safe_cleanup, ]), @@ -128,6 +137,7 @@ describe("provider path safety", () => { }); it("rejects loose ChatGPT data files outside conversations-v3 roots", async () => { + invalidateChatGptConversationRootsCache(); const testDir = path.join(CHAT_DIR, "__threadlens-vitest__"); const looseFile = path.join(testDir, "loose.data"); try { @@ -137,8 +147,39 @@ describe("provider path safety", () => { await expect(resolveAllowedProviderFilePath("chatgpt", looseFile)).resolves.toBeNull(); } finally { await rm(testDir, { recursive: true, force: true }); + invalidateChatGptConversationRootsCache(); + } + }); + + it("allows ChatGPT data files only inside discovered conversations-v3 roots", async () => { + invalidateChatGptConversationRootsCache(); + const root = path.join(CHAT_DIR, "__threadlens-vitest__", "conversations-v3-test"); + const filePath = path.join(root, "conversation.data"); + try { + await mkdir(root, { recursive: true }); + await writeFile(filePath, "{}", "utf-8"); + + await expect(resolveAllowedProviderFilePath("chatgpt", filePath)).resolves.toBe( + await realpath(filePath), + ); + const scanRoots = await providerScanRootSpecs("chatgpt"); + expect(scanRoots.map((spec) => spec.root)).toContain(root); + expect(scanRoots.every((spec) => spec.source === "conversations")).toBe(true); + } finally { + await rm(path.join(CHAT_DIR, "__threadlens-vitest__"), { + recursive: true, + force: true, + }); + invalidateChatGptConversationRootsCache(); } }); + + it("validates provider root source names before they can be used for backup paths", () => { + expect(isValidProviderRootSource("cleanup_backups")).toBe(true); + expect(isValidProviderRootSource("nested/source")).toBe(false); + expect(isValidProviderRootSource("../escape")).toBe(false); + expect(isValidProviderRootSource("bad-source")).toBe(false); + }); }); describe("resolveSafePathWithinRoots", () => { diff --git a/apps/api-ts/src/domains/providers/path-safety.ts b/apps/api-ts/src/domains/providers/path-safety.ts index 7fdf403..df3df4d 100644 --- a/apps/api-ts/src/domains/providers/path-safety.ts +++ b/apps/api-ts/src/domains/providers/path-safety.ts @@ -1,138 +1 @@ -import { lstat, realpath } from "node:fs/promises"; -import path from "node:path"; -import { - PROVIDER_IDS, - PROVIDER_LABELS, - type ProviderId, -} from "@threadlens/shared-contracts"; -import { - CHAT_DIR, - CODEX_HOME, -} from "../../lib/constants.js"; -import { - providerRootSpecs, - providerScanRootSpecs, -} from "./provider-roots.js"; - -export { - codexTranscriptSearchRoots, - providerRootSpecs, - providerScanRootSpecs, -} from "./provider-roots.js"; - -export function providerName(provider: ProviderId): string { - return PROVIDER_LABELS[provider] ?? provider; -} - -export function listProviderIds(): ProviderId[] { - return [...PROVIDER_IDS]; -} - -export function parseProviderId(raw: unknown): ProviderId | undefined { - const value = String(raw ?? "") - .trim() - .toLowerCase(); - if ((PROVIDER_IDS as readonly string[]).includes(value)) { - return value as ProviderId; - } - return undefined; -} - -export function isPathInsideRoot( - targetPath: string, - rootPath: string, -): boolean { - const fullTarget = path.resolve(targetPath); - const fullRoot = path.resolve(rootPath); - return ( - fullTarget === fullRoot || fullTarget.startsWith(`${fullRoot}${path.sep}`) - ); -} - -function isCodexCwdBackupPath(filePath: string): boolean { - const ext = path.extname(filePath).toLowerCase(); - if (ext !== ".jsonl") return false; - if (!isPathInsideRoot(filePath, CODEX_HOME)) return false; - const relativePath = path.relative(CODEX_HOME, path.resolve(filePath)); - const [firstSegment = ""] = relativePath.split(path.sep); - return ( - firstSegment.startsWith("jsonl-cwd-backups-") && - !relativePath.startsWith(`..${path.sep}`) && - relativePath !== ".." - ); -} - -export function isAllowedProviderFilePath( - provider: ProviderId, - filePath: string, -): boolean { - if (provider === "codex" && isCodexCwdBackupPath(filePath)) { - return true; - } - if (provider === "chatgpt") { - const ext = path.extname(filePath).toLowerCase(); - if (ext !== ".data") return false; - if (!isPathInsideRoot(filePath, CHAT_DIR)) return false; - const normalized = path.resolve(filePath); - return /(^|[\\/])conversations-v3-[^\\/]+[\\/]/.test(normalized); - } - const specs = providerRootSpecs(provider); - const ext = path.extname(filePath).toLowerCase(); - return specs.some( - (spec) => spec.exts.includes(ext) && isPathInsideRoot(filePath, spec.root), - ); -} - -export async function resolveSafePathWithinRoots( - filePath: string, - rootPaths: string[], -): Promise { - const normalizedTarget = path.resolve(filePath); - let targetLstat; - try { - targetLstat = await lstat(normalizedTarget); - } catch { - return null; - } - if (targetLstat.isSymbolicLink()) return null; - - let realTarget = ""; - try { - realTarget = await realpath(normalizedTarget); - } catch { - return null; - } - - for (const rootPath of rootPaths) { - try { - const realRoot = await realpath(rootPath); - if ( - realTarget === realRoot || - realTarget.startsWith(`${realRoot}${path.sep}`) - ) { - return realTarget; - } - } catch { - continue; - } - } - - return null; -} - -export async function resolveAllowedProviderFilePath( - provider: ProviderId, - filePath: string, -): Promise { - if (!isAllowedProviderFilePath(provider, filePath)) return null; - const specs = await providerScanRootSpecs(provider); - const ext = path.extname(filePath).toLowerCase(); - const matchingRoots = specs - .filter( - (spec) => spec.exts.includes(ext) && isPathInsideRoot(filePath, spec.root), - ) - .map((spec) => spec.root); - - if (!matchingRoots.length) return null; - return resolveSafePathWithinRoots(filePath, matchingRoots); -} +export * from "./shared/path-safety.js"; diff --git a/apps/api-ts/src/domains/providers/provider-roots.ts b/apps/api-ts/src/domains/providers/provider-roots.ts index 02c0d0e..b1ba382 100644 --- a/apps/api-ts/src/domains/providers/provider-roots.ts +++ b/apps/api-ts/src/domains/providers/provider-roots.ts @@ -1,112 +1 @@ -import path from "node:path"; -import type { - ProviderId, -} from "@threadlens/shared-contracts"; - -import { - CHAT_DIR, - CODEX_HOME, - CLAUDE_PROJECTS_DIR, - CLAUDE_TRANSCRIPTS_DIR, - COPILOT_CURSOR_GLOBAL, - COPILOT_CURSOR_WORKSPACE_STORAGE, - COPILOT_VSCODE_GLOBAL, - COPILOT_VSCODE_WORKSPACE_STORAGE, - GEMINI_ANTIGRAVITY_CONVERSATIONS_DIR, - GEMINI_HISTORY_DIR, - GEMINI_TMP_DIR, -} from "../../lib/constants.js"; -import { discoverChatGptConversationRoots } from "./roots/chatgpt.js"; -import { discoverCodexCwdBackupRoots } from "./roots/codex.js"; -import { - providerArchivedRootSpec, - providerBackupRootSpecs, -} from "./roots/backups.js"; -import type { - ProviderRootSpec, -} from "./types.js"; - -export { codexTranscriptSearchRoots } from "./roots/codex.js"; - -export function providerRootSpecs(provider: ProviderId): ProviderRootSpec[] { - const archivedSpec = providerArchivedRootSpec(provider); - if (provider === "codex") { - return [ - { - source: "sessions", - root: path.join(CODEX_HOME, "sessions"), - exts: [".jsonl"], - }, - ...(archivedSpec ? [archivedSpec] : []), - ...providerBackupRootSpecs(provider), - ]; - } - if (provider === "chatgpt") { - return [{ source: "chat_cache", root: CHAT_DIR, exts: [".data"] }]; - } - if (provider === "claude") { - return [ - { source: "projects", root: CLAUDE_PROJECTS_DIR, exts: [".jsonl"] }, - { - source: "transcripts", - root: CLAUDE_TRANSCRIPTS_DIR, - exts: [".jsonl", ".json"], - }, - ...(archivedSpec ? [archivedSpec] : []), - ...providerBackupRootSpecs(provider), - ]; - } - if (provider === "gemini") { - return [ - { source: "tmp", root: GEMINI_TMP_DIR, exts: [".jsonl", ".json"] }, - { source: "history", root: GEMINI_HISTORY_DIR, exts: [".jsonl", ".json"] }, - { - source: "antigravity_conversations", - root: GEMINI_ANTIGRAVITY_CONVERSATIONS_DIR, - exts: [".pb"], - }, - ...(archivedSpec ? [archivedSpec] : []), - ...providerBackupRootSpecs(provider), - ]; - } - if (provider === "copilot") { - return [ - { - source: "vscode_global", - root: COPILOT_VSCODE_GLOBAL, - exts: [".jsonl", ".json"], - }, - { - source: "cursor_global", - root: COPILOT_CURSOR_GLOBAL, - exts: [".jsonl", ".json"], - }, - { - source: "vscode_workspace_chats", - root: COPILOT_VSCODE_WORKSPACE_STORAGE, - exts: [".json"], - }, - { - source: "cursor_workspace_chats", - root: COPILOT_CURSOR_WORKSPACE_STORAGE, - exts: [".json"], - }, - ...(archivedSpec ? [archivedSpec] : []), - ...providerBackupRootSpecs(provider), - ]; - } - return []; -} - -export async function providerScanRootSpecs( - provider: ProviderId, -): Promise { - if (provider === "chatgpt") { - const discovered = await discoverChatGptConversationRoots(); - if (discovered.length > 0) return discovered; - return providerRootSpecs(provider); - } - if (provider !== "codex") return providerRootSpecs(provider); - const extraRoots = await discoverCodexCwdBackupRoots(); - return [...providerRootSpecs(provider), ...extraRoots]; -} +export * from "./shared/file-roots.js"; diff --git a/apps/api-ts/src/domains/providers/registry.ts b/apps/api-ts/src/domains/providers/registry.ts new file mode 100644 index 0000000..844c965 --- /dev/null +++ b/apps/api-ts/src/domains/providers/registry.ts @@ -0,0 +1,63 @@ +import { + PROVIDER_LABELS, + type ProviderId, +} from "@threadlens/shared-contracts"; +import { + IMPLEMENTED_PROVIDER_IDS, +} from "./capabilities.js"; +import { getGeminiProviderHealth } from "./adapters/gemini/health.js"; +import { providerRootSpecs, providerScanRootSpecs } from "./provider-roots.js"; +import type { ProviderHealthEvidence, ProviderRootSpec } from "./types.js"; + +export type ProviderSessionLocator = + | { kind: "file"; file_path: string } + | { kind: "sqlite"; db_path: string; session_id: string }; + +export type ProviderAdapter = { + id: ProviderId; + label: string; + roots(): ProviderRootSpec[]; + scanRoots?(): Promise; + health?(): Promise; +}; + +function buildProviderAdapter(provider: ProviderId): ProviderAdapter { + const health = providerHealth(provider); + return Object.freeze({ + id: provider, + label: PROVIDER_LABELS[provider], + roots: () => providerRootSpecs(provider), + scanRoots: () => providerScanRootSpecs(provider), + ...(health ? { health } : {}), + } satisfies ProviderAdapter); +} + +function providerHealth(provider: ProviderId): ProviderAdapter["health"] { + if (provider === "gemini") return getGeminiProviderHealth; + return undefined; +} + +export const PROVIDER_ADAPTERS = Object.freeze({ + codex: buildProviderAdapter("codex"), + chatgpt: buildProviderAdapter("chatgpt"), + claude: buildProviderAdapter("claude"), + gemini: buildProviderAdapter("gemini"), + copilot: buildProviderAdapter("copilot"), +} satisfies Partial>); + +const PROVIDER_ADAPTERS_BY_ID: Readonly< + Partial> +> = PROVIDER_ADAPTERS; + +export function getProviderAdapter( + provider: ProviderId, +): ProviderAdapter | undefined { + return PROVIDER_ADAPTERS_BY_ID[provider]; +} + +export function listProviderAdapters(): ProviderAdapter[] { + return IMPLEMENTED_PROVIDER_IDS.flatMap((id) => { + const adapter = PROVIDER_ADAPTERS[id]; + return adapter ? [adapter] : []; + }); +} diff --git a/apps/api-ts/src/domains/providers/search.test.ts b/apps/api-ts/src/domains/providers/search.test.ts index d02f696..e87212c 100644 --- a/apps/api-ts/src/domains/providers/search.test.ts +++ b/apps/api-ts/src/domains/providers/search.test.ts @@ -7,7 +7,7 @@ import { describe, expect, it, vi } from "vitest"; import type { ProviderSessionRow, TranscriptPayload, -} from "../../lib/providers.js"; +} from "./index.js"; import { buildConversationSearchProviderBudgets, createCachedConversationTranscriptLoader, @@ -1345,7 +1345,9 @@ describe("provider manifest cache behavior", () => { })); try { - const { readPersistedProviderManifest } = await import("./search/manifest-store.js"); + const { readPersistedProviderManifest } = await import( + "./services/search/manifest-store.js" + ); const persisted = await readPersistedProviderManifest("claude"); expect(persisted?.manifest.candidates).toHaveLength(1); diff --git a/apps/api-ts/src/domains/providers/search.ts b/apps/api-ts/src/domains/providers/search.ts index 75a4358..7d24e8e 100644 --- a/apps/api-ts/src/domains/providers/search.ts +++ b/apps/api-ts/src/domains/providers/search.ts @@ -1,37 +1 @@ -export { - isMetadataOnlyConversationQuery, -} from "./search/session-results.js"; -export { - buildConversationSearchProviderBudgets, - resolveConversationSearchLimits, -} from "./search/limits.js"; -export { - searchConversationRows, -} from "./search/row-search.js"; -export { - createCachedConversationTranscriptLoader, -} from "./search/transcript-matches.js"; -export { - searchConversationSessionHits, - searchConversationSessions, -} from "./search/session-search.js"; -export { - getProviderParserHealthTs, - getProviderSessionsTs, -} from "./search/provider-session-reports.js"; -export { - searchLocalConversationsTs, -} from "./search/local-search.js"; -export { - searchConversationSessionHitsTs, - selectConversationSessionHitsRow, -} from "./search/session-hit-resolver.js"; -export { - resolveCodexSessionPathByThreadId, -} from "./search/codex-session-path.js"; -export { - defaultConversationSearchProviders, - getProviderSessionScan, - invalidateProviderSearchCaches, - primeConversationSearchCaches, -} from "./search/provider-session-scan.js"; +export * from "./services/search/index.js"; diff --git a/apps/api-ts/src/domains/providers/actions/archives.ts b/apps/api-ts/src/domains/providers/services/actions/archives.ts similarity index 96% rename from apps/api-ts/src/domains/providers/actions/archives.ts rename to apps/api-ts/src/domains/providers/services/actions/archives.ts index 7bccc69..75ff416 100644 --- a/apps/api-ts/src/domains/providers/actions/archives.ts +++ b/apps/api-ts/src/domains/providers/services/actions/archives.ts @@ -1,7 +1,7 @@ import path from "node:path"; import { realpath } from "node:fs/promises"; -import { isPathInsideRoot, providerRootSpecs } from "../path-safety.js"; -import type { ProviderId, ProviderRootSpec } from "../types.js"; +import { isPathInsideRoot, providerRootSpecs } from "../../path-safety.js"; +import type { ProviderId, ProviderRootSpec } from "../../types.js"; function isSafeProviderRelativePath(relativePath: string): boolean { return Boolean(relativePath) && diff --git a/apps/api-ts/src/domains/providers/actions/backups.ts b/apps/api-ts/src/domains/providers/services/actions/backups.ts similarity index 97% rename from apps/api-ts/src/domains/providers/actions/backups.ts rename to apps/api-ts/src/domains/providers/services/actions/backups.ts index 4af58ab..cfce5b9 100644 --- a/apps/api-ts/src/domains/providers/actions/backups.ts +++ b/apps/api-ts/src/domains/providers/services/actions/backups.ts @@ -1,16 +1,16 @@ import path from "node:path"; import { copyFile, mkdir, realpath, writeFile } from "node:fs/promises"; -import { BACKUP_ROOT, HOME_DIR } from "../../../lib/constants.js"; -import { nowIsoUtc } from "../../../lib/utils.js"; +import { BACKUP_ROOT, HOME_DIR } from "../../../../lib/constants.js"; +import { nowIsoUtc } from "../../../../lib/utils.js"; import { isPathInsideRoot, providerRootSpecs, providerScanRootSpecs, -} from "../path-safety.js"; +} from "../../path-safety.js"; import type { ProviderId, ProviderSessionAction, -} from "../types.js"; +} from "../../types.js"; type ProviderBackupManifestItem = { source_path: string; diff --git a/apps/api-ts/src/domains/providers/actions/confirm-tokens.ts b/apps/api-ts/src/domains/providers/services/actions/confirm-tokens.ts similarity index 99% rename from apps/api-ts/src/domains/providers/actions/confirm-tokens.ts rename to apps/api-ts/src/domains/providers/services/actions/confirm-tokens.ts index c08ea2f..0ea0b79 100644 --- a/apps/api-ts/src/domains/providers/actions/confirm-tokens.ts +++ b/apps/api-ts/src/domains/providers/services/actions/confirm-tokens.ts @@ -7,7 +7,7 @@ import type { ProviderId, ProviderSessionAction, ProviderSessionActionOptions, -} from "../types.js"; +} from "../../types.js"; type ProviderActionTokenEntry = { provider: ProviderId; diff --git a/apps/api-ts/src/domains/providers/actions/execution.ts b/apps/api-ts/src/domains/providers/services/actions/execution.ts similarity index 99% rename from apps/api-ts/src/domains/providers/actions/execution.ts rename to apps/api-ts/src/domains/providers/services/actions/execution.ts index b5d875b..5f475c3 100644 --- a/apps/api-ts/src/domains/providers/actions/execution.ts +++ b/apps/api-ts/src/domains/providers/services/actions/execution.ts @@ -8,7 +8,7 @@ import { } from "./archives.js"; import type { ProviderId, -} from "../types.js"; +} from "../../types.js"; export type ProviderActionExecutionFailure = { file_path: string; diff --git a/apps/api-ts/src/domains/providers/services/actions/index.ts b/apps/api-ts/src/domains/providers/services/actions/index.ts new file mode 100644 index 0000000..c4a8ba5 --- /dev/null +++ b/apps/api-ts/src/domains/providers/services/actions/index.ts @@ -0,0 +1,246 @@ +import { BACKUP_ROOT } from "../../../../lib/constants.js"; +import { + deriveProviderBackupId, + resolveProviderActionBackupRoot, + stageProviderActionBackup, +} from "./backups.js"; +import { + consumeProviderActionConfirmToken, + issueProviderActionConfirmToken, + pruneProviderActionTokens, +} from "./confirm-tokens.js"; +import { + buildProviderActionFingerprint, + normalizeProviderActionOptions, +} from "./selection.js"; +import { + buildProviderActionAppliedResponse, + buildProviderActionFailureResponse, + buildProviderActionPreviewResponse, +} from "./responses.js"; +import { + type ProviderActionExecutionFailure, + runArchiveProviderAction, + runDeleteProviderAction, + runUnarchiveProviderAction, +} from "./execution.js"; +import { + normalizeProviderActionTargetPaths, + resolveProviderActionTargets, +} from "./targets.js"; +import type { + ProviderId, + ProviderSessionAction, + ProviderSessionActionOptions, +} from "../../types.js"; + +type ProviderActionDeps = { + resolveAllowedProviderFilePath: ( + provider: ProviderId, + filePath: string, + ) => Promise; + supportsProviderAction: ( + provider: ProviderId, + action: ProviderSessionAction, + ) => boolean; + invalidateProviderCaches: (provider: ProviderId) => void; +}; + +export { buildProviderActionFingerprint, buildProviderActionToken } from "./selection.js"; +export { deriveProviderBackupRelativePath } from "./backups.js"; + +export async function runProviderSessionAction( + deps: ProviderActionDeps, + provider: ProviderId, + action: ProviderSessionAction, + filePaths: string[], + dryRun: boolean, + confirmToken: string, + options?: ProviderSessionActionOptions, +) { + pruneProviderActionTokens(); + const normalizedOptions = normalizeProviderActionOptions(options); + const uniquePaths = normalizeProviderActionTargetPaths(filePaths); + if (!deps.supportsProviderAction(provider, action)) { + return buildProviderActionFailureResponse({ + provider, + action, + dryRun, + targetCount: uniquePaths.length, + validCount: 0, + selectionFingerprint: "", + options: normalizedOptions, + skipped: [], + error: "cleanup-disabled-provider", + }); + } + const { skipped, valid } = await resolveProviderActionTargets( + provider, + uniquePaths, + deps.resolveAllowedProviderFilePath, + ); + if (!valid.length && !dryRun) { + return buildProviderActionFailureResponse({ + provider, + action, + dryRun: false, + targetCount: uniquePaths.length, + validCount: valid.length, + selectionFingerprint: "", + options: normalizedOptions, + skipped, + error: "no-valid-targets", + }); + } + + const selectionFingerprint = valid.length + ? buildProviderActionFingerprint(provider, action, valid, normalizedOptions) + : ""; + const shouldBackup = + action === "backup_local" || + normalizedOptions.backup_before_delete; + const backupRootResult = await resolveProviderActionBackupRoot( + normalizedOptions.backup_root, + ); + if (shouldBackup && !backupRootResult.ok) { + return buildProviderActionFailureResponse({ + provider, + action, + dryRun, + targetCount: uniquePaths.length, + validCount: valid.length, + selectionFingerprint, + options: normalizedOptions, + skipped, + error: backupRootResult.error, + }); + } + + if (dryRun) { + const expectedToken = valid.length + ? issueProviderActionConfirmToken(provider, action, valid, normalizedOptions) + : ""; + return buildProviderActionPreviewResponse({ + provider, + action, + dryRun: true, + targetCount: uniquePaths.length, + validCount: valid.length, + selectionFingerprint, + options: normalizedOptions, + skipped, + confirmTokenExpected: expectedToken, + }); + } + + if (action !== "backup_local") { + const consume = consumeProviderActionConfirmToken( + confirmToken, + provider, + action, + valid, + normalizedOptions, + ); + if (!consume.ok) { + const expectedToken = valid.length + ? issueProviderActionConfirmToken(provider, action, valid, normalizedOptions) + : ""; + return buildProviderActionFailureResponse({ + provider, + action, + dryRun: false, + targetCount: uniquePaths.length, + validCount: valid.length, + selectionFingerprint, + options: normalizedOptions, + skipped, + confirmTokenExpected: expectedToken, + error: consume.reason, + }); + } + } + + let applied = 0; + let archivedTo: string | null = null; + let backupTo: string | null = null; + let backupManifestPath: string | null = null; + let backedUpCount = 0; + const failed: ProviderActionExecutionFailure[] = []; + const backupRoot = shouldBackup + ? backupRootResult.ok + ? backupRootResult.backupRoot + : (() => { + throw new Error(`unreachable invalid backup root: ${backupRootResult.error}`); + })() + : BACKUP_ROOT; + const backupStage = shouldBackup + ? await stageProviderActionBackup(provider, action, valid, backupRoot) + : null; + if (backupStage) { + failed.push(...backupStage.failed); + backupTo = backupStage.destination; + backupManifestPath = backupStage.manifest_path; + backedUpCount = backupStage.items.length; + } + const backupManifestWriteFailed = + backupStage?.failed.some((failure) => failure.step === "manifest_write") ?? false; + if (backupManifestWriteFailed && action !== "backup_local") { + return buildProviderActionAppliedResponse({ + provider, + action, + targetCount: uniquePaths.length, + validCount: valid.length, + selectionFingerprint, + options: normalizedOptions, + skipped, + failed, + applied: 0, + backedUpCount, + backupId: deriveProviderBackupId(backupRoot, backupTo), + backupTo, + backupManifestPath, + shouldBackup, + backupStageFailedCount: backupStage?.failed.length ?? 0, + archivedTo, + }); + } + + if (action === "backup_local") { + applied = backupStage?.items.length ?? 0; + } else if (action === "archive_local") { + const result = await runArchiveProviderAction(provider, valid); + applied = result.applied; + archivedTo = result.archivedTo; + failed.push(...result.failed); + } else if (action === "unarchive_local") { + const result = await runUnarchiveProviderAction(provider, valid); + applied = result.applied; + failed.push(...result.failed); + } else { + const deleteTargets = backupStage?.items.map((item) => item.source_path) ?? valid; + const result = await runDeleteProviderAction(deleteTargets); + applied = result.applied; + failed.push(...result.failed); + } + + deps.invalidateProviderCaches(provider); + + return buildProviderActionAppliedResponse({ + provider, + action, + targetCount: uniquePaths.length, + validCount: valid.length, + selectionFingerprint, + options: normalizedOptions, + skipped, + failed, + applied, + backedUpCount, + backupId: deriveProviderBackupId(backupRoot, backupTo), + backupTo, + backupManifestPath, + shouldBackup, + backupStageFailedCount: backupStage?.failed.length ?? 0, + archivedTo, + }); +} diff --git a/apps/api-ts/src/domains/providers/actions/responses.ts b/apps/api-ts/src/domains/providers/services/actions/responses.ts similarity index 98% rename from apps/api-ts/src/domains/providers/actions/responses.ts rename to apps/api-ts/src/domains/providers/services/actions/responses.ts index 0121b31..2eff2fd 100644 --- a/apps/api-ts/src/domains/providers/actions/responses.ts +++ b/apps/api-ts/src/domains/providers/services/actions/responses.ts @@ -1,4 +1,4 @@ -import type { ProviderId, ProviderSessionAction } from "../types.js"; +import type { ProviderId, ProviderSessionAction } from "../../types.js"; import type { NormalizedProviderActionOptions } from "./selection.js"; import type { ProviderActionSkippedTarget } from "./targets.js"; import type { ProviderActionExecutionFailure } from "./execution.js"; diff --git a/apps/api-ts/src/domains/providers/actions/selection.ts b/apps/api-ts/src/domains/providers/services/actions/selection.ts similarity index 98% rename from apps/api-ts/src/domains/providers/actions/selection.ts rename to apps/api-ts/src/domains/providers/services/actions/selection.ts index 60cda11..f37c642 100644 --- a/apps/api-ts/src/domains/providers/actions/selection.ts +++ b/apps/api-ts/src/domains/providers/services/actions/selection.ts @@ -4,7 +4,7 @@ import type { ProviderId, ProviderSessionAction, ProviderSessionActionOptions, -} from "../types.js"; +} from "../../types.js"; export type NormalizedProviderActionOptions = { backup_before_delete: boolean; diff --git a/apps/api-ts/src/domains/providers/actions/targets.ts b/apps/api-ts/src/domains/providers/services/actions/targets.ts similarity index 98% rename from apps/api-ts/src/domains/providers/actions/targets.ts rename to apps/api-ts/src/domains/providers/services/actions/targets.ts index 24e1b4d..336db50 100644 --- a/apps/api-ts/src/domains/providers/actions/targets.ts +++ b/apps/api-ts/src/domains/providers/services/actions/targets.ts @@ -1,7 +1,7 @@ import { stat } from "node:fs/promises"; import type { ProviderId, -} from "../types.js"; +} from "../../types.js"; export type ProviderActionSkippedTarget = { file_path: string; diff --git a/apps/api-ts/src/domains/providers/matrix/adapter-access.ts b/apps/api-ts/src/domains/providers/services/matrix/adapter-access.ts similarity index 97% rename from apps/api-ts/src/domains/providers/matrix/adapter-access.ts rename to apps/api-ts/src/domains/providers/services/matrix/adapter-access.ts index 7e73401..281ed6e 100644 --- a/apps/api-ts/src/domains/providers/matrix/adapter-access.ts +++ b/apps/api-ts/src/domains/providers/services/matrix/adapter-access.ts @@ -5,7 +5,7 @@ import { import { getProviderAdapter, -} from "../adapters.js"; +} from "../../registry.js"; export function supportsProviderCleanup(provider: ProviderId): boolean { return getProviderCapability(provider).safe_cleanup; diff --git a/apps/api-ts/src/domains/providers/matrix/copilot.ts b/apps/api-ts/src/domains/providers/services/matrix/copilot.ts similarity index 93% rename from apps/api-ts/src/domains/providers/matrix/copilot.ts rename to apps/api-ts/src/domains/providers/services/matrix/copilot.ts index 5926757..7fb57a6 100644 --- a/apps/api-ts/src/domains/providers/matrix/copilot.ts +++ b/apps/api-ts/src/domains/providers/services/matrix/copilot.ts @@ -2,14 +2,14 @@ import path from "node:path"; import { walkFilesByExt, -} from "../../../lib/utils.js"; +} from "../../../../lib/utils.js"; import { isCopilotGlobalSessionLikeFile, isWorkspaceChatSessionPath, -} from "../probe.js"; +} from "../../probe.js"; import type { ProviderRootSpec, -} from "../types.js"; +} from "../../types.js"; const COPILOT_MATRIX_SESSION_SCAN_LIMIT = 5_000; diff --git a/apps/api-ts/src/domains/providers/services/matrix/index.ts b/apps/api-ts/src/domains/providers/services/matrix/index.ts new file mode 100644 index 0000000..a62c6cc --- /dev/null +++ b/apps/api-ts/src/domains/providers/services/matrix/index.ts @@ -0,0 +1,72 @@ +import { + nowIsoUtc, +} from "../../../../lib/utils.js"; +import { + buildProviderMatrixProviders, +} from "./provider-entries.js"; +import { readProviderMatrixSignals } from "./signals.js"; +import { + buildProviderMatrixSummary, +} from "./summary.js"; +import type { ProviderMatrixData } from "../../types.js"; + +type ProviderMatrixCacheEntry = { + expires_at: number; + data: ProviderMatrixData; +}; + +const PROVIDER_MATRIX_CACHE_TTL_MS = 30_000; +let providerMatrixCache: ProviderMatrixCacheEntry | null = null; +let providerMatrixInflight: Promise | null = null; + +export { capabilityLevel, providerStatus } from "./summary.js"; + +export function invalidateProviderMatrixCache() { + providerMatrixCache = null; +} + +async function buildProviderMatrixData(): Promise { + const signals = await readProviderMatrixSignals(); + const providers = buildProviderMatrixProviders(signals); + + const summary = buildProviderMatrixSummary(providers); + + return { + generated_at: nowIsoUtc(), + mode: "multi-provider-phase-1", + summary, + providers, + policy: { + cleanup_gate: "provider capability matrix controls destructive actions", + default_non_codex: "all detected providers are visible for local analysis", + }, + }; +} + +export async function getProviderMatrixTs(options?: { forceRefresh?: boolean }) { + const forceRefresh = Boolean(options?.forceRefresh); + const now = Date.now(); + if (!forceRefresh && providerMatrixCache && providerMatrixCache.expires_at > now) { + return providerMatrixCache.data; + } + if (forceRefresh) { + providerMatrixCache = null; + } + if (providerMatrixInflight) { + return providerMatrixInflight; + } + + providerMatrixInflight = buildProviderMatrixData() + .then((data) => { + providerMatrixCache = { + expires_at: Date.now() + PROVIDER_MATRIX_CACHE_TTL_MS, + data, + }; + return data; + }) + .finally(() => { + providerMatrixInflight = null; + }); + + return providerMatrixInflight; +} diff --git a/apps/api-ts/src/domains/providers/matrix/provider-entries.ts b/apps/api-ts/src/domains/providers/services/matrix/provider-entries.ts similarity index 98% rename from apps/api-ts/src/domains/providers/matrix/provider-entries.ts rename to apps/api-ts/src/domains/providers/services/matrix/provider-entries.ts index 3c16b1c..8701106 100644 --- a/apps/api-ts/src/domains/providers/matrix/provider-entries.ts +++ b/apps/api-ts/src/domains/providers/services/matrix/provider-entries.ts @@ -8,7 +8,7 @@ import { COPILOT_CURSOR_WORKSPACE_STORAGE, COPILOT_VSCODE_GLOBAL, COPILOT_VSCODE_WORKSPACE_STORAGE, -} from "../../../lib/constants.js"; +} from "../../../../lib/constants.js"; import { providerLabel, supportsProviderCleanup, @@ -23,7 +23,7 @@ import type { } from "./signals.js"; import type { ProviderMatrixData, -} from "../types.js"; +} from "../../types.js"; export function buildProviderMatrixProviders( signals: ProviderMatrixSignals, diff --git a/apps/api-ts/src/domains/providers/matrix/signals.ts b/apps/api-ts/src/domains/providers/services/matrix/signals.ts similarity index 95% rename from apps/api-ts/src/domains/providers/matrix/signals.ts rename to apps/api-ts/src/domains/providers/services/matrix/signals.ts index 03f1ae9..c2a6ce8 100644 --- a/apps/api-ts/src/domains/providers/matrix/signals.ts +++ b/apps/api-ts/src/domains/providers/services/matrix/signals.ts @@ -9,13 +9,13 @@ import { COPILOT_CURSOR_WORKSPACE_STORAGE, COPILOT_VSCODE_GLOBAL, COPILOT_VSCODE_WORKSPACE_STORAGE, -} from "../../../lib/constants.js"; +} from "../../../../lib/constants.js"; import { countFilesRecursiveByExt, countJsonlFilesRecursive, pathExists, quickFileCount, -} from "../../../lib/utils.js"; +} from "../../../../lib/utils.js"; import { providerHealth, providerRoots, @@ -24,10 +24,10 @@ import { import { countCopilotMatrixSessionFiles, } from "./copilot.js"; -import { codexTranscriptSearchRoots } from "../path-safety.js"; +import { codexTranscriptSearchRoots } from "../../path-safety.js"; import type { ProviderRootSpec, -} from "../types.js"; +} from "../../types.js"; export type ProviderMatrixSignals = { codexHomes: string[]; diff --git a/apps/api-ts/src/domains/providers/matrix/summary.ts b/apps/api-ts/src/domains/providers/services/matrix/summary.ts similarity index 97% rename from apps/api-ts/src/domains/providers/matrix/summary.ts rename to apps/api-ts/src/domains/providers/services/matrix/summary.ts index ee72e70..beaf5f1 100644 --- a/apps/api-ts/src/domains/providers/matrix/summary.ts +++ b/apps/api-ts/src/domains/providers/services/matrix/summary.ts @@ -1,7 +1,7 @@ import type { ProviderMatrixData, ProviderStatus, -} from "../types.js"; +} from "../../types.js"; type ProviderMatrixProvider = ProviderMatrixData["providers"][number]; diff --git a/apps/api-ts/src/domains/providers/search/abort.ts b/apps/api-ts/src/domains/providers/services/search/abort.ts similarity index 100% rename from apps/api-ts/src/domains/providers/search/abort.ts rename to apps/api-ts/src/domains/providers/services/search/abort.ts diff --git a/apps/api-ts/src/domains/providers/search/cache.ts b/apps/api-ts/src/domains/providers/services/search/cache.ts similarity index 98% rename from apps/api-ts/src/domains/providers/search/cache.ts rename to apps/api-ts/src/domains/providers/services/search/cache.ts index c8d0786..9f8ae2f 100644 --- a/apps/api-ts/src/domains/providers/search/cache.ts +++ b/apps/api-ts/src/domains/providers/services/search/cache.ts @@ -1,6 +1,6 @@ import path from "node:path"; -import type { ProviderId, ProviderSessionRow, ProviderSessionScan } from "../types.js"; +import type { ProviderId, ProviderSessionRow, ProviderSessionScan } from "../../types.js"; import { SEARCH_TRANSCRIPT_CACHE_MAX_ENTRIES } from "./constants.js"; import type { ConversationSearchResponseCacheEntry, diff --git a/apps/api-ts/src/domains/providers/search/codex-session-path.ts b/apps/api-ts/src/domains/providers/services/search/codex-session-path.ts similarity index 92% rename from apps/api-ts/src/domains/providers/search/codex-session-path.ts rename to apps/api-ts/src/domains/providers/services/search/codex-session-path.ts index a7173ca..53938d3 100644 --- a/apps/api-ts/src/domains/providers/search/codex-session-path.ts +++ b/apps/api-ts/src/domains/providers/services/search/codex-session-path.ts @@ -2,10 +2,10 @@ import path from "node:path"; import { codexTranscriptSearchRoots, -} from "../path-safety.js"; +} from "../../path-safety.js"; import { walkFilesByExt, -} from "../../../lib/utils.js"; +} from "../../../../lib/utils.js"; import { getProviderSessionScan, } from "./provider-session-scan.js"; diff --git a/apps/api-ts/src/domains/providers/search/concurrency.ts b/apps/api-ts/src/domains/providers/services/search/concurrency.ts similarity index 100% rename from apps/api-ts/src/domains/providers/search/concurrency.ts rename to apps/api-ts/src/domains/providers/services/search/concurrency.ts diff --git a/apps/api-ts/src/domains/providers/search/constants.ts b/apps/api-ts/src/domains/providers/services/search/constants.ts similarity index 88% rename from apps/api-ts/src/domains/providers/search/constants.ts rename to apps/api-ts/src/domains/providers/services/search/constants.ts index 6110c80..869d1eb 100644 --- a/apps/api-ts/src/domains/providers/search/constants.ts +++ b/apps/api-ts/src/domains/providers/services/search/constants.ts @@ -1,13 +1,14 @@ -import { SEARCHABLE_PROVIDER_IDS } from "@threadlens/shared-contracts"; - -import type { ProviderId } from "../types.js"; +import type { ProviderId } from "../../types.js"; +import { + listSearchableProviderIds, +} from "../../capabilities.js"; export const PROVIDER_SCAN_CACHE_TTL_MS = 60_000; export const PROVIDER_MANIFEST_CACHE_TTL_MS = 5 * 60_000; export const OPENABLE_THREAD_IDS_CACHE_TTL_MS = 15_000; export const CONVERSATION_SEARCH_RESPONSE_CACHE_TTL_MS = 15_000; export const DEFAULT_CONVERSATION_SEARCH_PROVIDERS: ProviderId[] = [ - ...SEARCHABLE_PROVIDER_IDS, + ...listSearchableProviderIds(), ]; export const DEFAULT_CONVERSATION_SEARCH_LIMIT = 40; export const MAX_CONVERSATION_SEARCH_LIMIT = 200; diff --git a/apps/api-ts/src/domains/providers/services/search/index.ts b/apps/api-ts/src/domains/providers/services/search/index.ts new file mode 100644 index 0000000..db5e9b7 --- /dev/null +++ b/apps/api-ts/src/domains/providers/services/search/index.ts @@ -0,0 +1,37 @@ +export { + isMetadataOnlyConversationQuery, +} from "./session-results.js"; +export { + buildConversationSearchProviderBudgets, + resolveConversationSearchLimits, +} from "./limits.js"; +export { + searchConversationRows, +} from "./row-search.js"; +export { + createCachedConversationTranscriptLoader, +} from "./transcript-matches.js"; +export { + searchConversationSessionHits, + searchConversationSessions, +} from "./session-search.js"; +export { + getProviderParserHealthTs, + getProviderSessionsTs, +} from "./provider-session-reports.js"; +export { + searchLocalConversationsTs, +} from "./local-search.js"; +export { + searchConversationSessionHitsTs, + selectConversationSessionHitsRow, +} from "./session-hit-resolver.js"; +export { + resolveCodexSessionPathByThreadId, +} from "./codex-session-path.js"; +export { + defaultConversationSearchProviders, + getProviderSessionScan, + invalidateProviderSearchCaches, + primeConversationSearchCaches, +} from "./provider-session-scan.js"; diff --git a/apps/api-ts/src/domains/providers/search/limits.ts b/apps/api-ts/src/domains/providers/services/search/limits.ts similarity index 97% rename from apps/api-ts/src/domains/providers/search/limits.ts rename to apps/api-ts/src/domains/providers/services/search/limits.ts index 86f82bb..aef0ed5 100644 --- a/apps/api-ts/src/domains/providers/search/limits.ts +++ b/apps/api-ts/src/domains/providers/services/search/limits.ts @@ -1,4 +1,4 @@ -import type { ProviderId } from "../types.js"; +import type { ProviderId } from "../../types.js"; import { DEFAULT_CONVERSATION_SEARCH_LIMIT, DEFAULT_CONVERSATION_SEARCH_SCAN_FLOOR, diff --git a/apps/api-ts/src/domains/providers/search/local-search.ts b/apps/api-ts/src/domains/providers/services/search/local-search.ts similarity index 97% rename from apps/api-ts/src/domains/providers/search/local-search.ts rename to apps/api-ts/src/domains/providers/services/search/local-search.ts index 74a73c1..c7401de 100644 --- a/apps/api-ts/src/domains/providers/search/local-search.ts +++ b/apps/api-ts/src/domains/providers/services/search/local-search.ts @@ -1,11 +1,11 @@ import type { ConversationSearchPayload, ProviderId, -} from "../types.js"; +} from "../../types.js"; import { normalizeSearchText, -} from "../search-helpers.js"; -import { nowIsoUtc } from "../../../lib/utils.js"; +} from "../../search-helpers.js"; +import { nowIsoUtc } from "../../../../lib/utils.js"; import { throwIfAborted, } from "./abort.js"; diff --git a/apps/api-ts/src/domains/providers/search/manifest-store.ts b/apps/api-ts/src/domains/providers/services/search/manifest-store.ts similarity index 96% rename from apps/api-ts/src/domains/providers/search/manifest-store.ts rename to apps/api-ts/src/domains/providers/services/search/manifest-store.ts index 04c79fd..5e3a81c 100644 --- a/apps/api-ts/src/domains/providers/search/manifest-store.ts +++ b/apps/api-ts/src/domains/providers/services/search/manifest-store.ts @@ -3,8 +3,8 @@ import { mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import type { ProviderId } from "../types.js"; -import { providerScanRootSpecs } from "../path-safety.js"; +import type { ProviderId } from "../../types.js"; +import { providerScanRootSpecs } from "../../path-safety.js"; import type { PersistedProviderManifestCacheEntry, ProviderSessionCandidate, diff --git a/apps/api-ts/src/domains/providers/search/openable-threads.ts b/apps/api-ts/src/domains/providers/services/search/openable-threads.ts similarity index 94% rename from apps/api-ts/src/domains/providers/search/openable-threads.ts rename to apps/api-ts/src/domains/providers/services/search/openable-threads.ts index 75a1edf..c2dc295 100644 --- a/apps/api-ts/src/domains/providers/search/openable-threads.ts +++ b/apps/api-ts/src/domains/providers/services/search/openable-threads.ts @@ -20,7 +20,7 @@ export async function resolveOpenableThreadIds( } const loadIds = (async () => { - const { getThreadsTs } = await import("../../threads/query.js"); + const { getThreadsTs } = await import("../../../threads/query.js"); const threads = await getThreadsTs({ offset: "0", limit: "240", diff --git a/apps/api-ts/src/domains/providers/search/provider-session-reports.ts b/apps/api-ts/src/domains/providers/services/search/provider-session-reports.ts similarity index 90% rename from apps/api-ts/src/domains/providers/search/provider-session-reports.ts rename to apps/api-ts/src/domains/providers/services/search/provider-session-reports.ts index e6502f9..c55f7f5 100644 --- a/apps/api-ts/src/domains/providers/search/provider-session-reports.ts +++ b/apps/api-ts/src/domains/providers/services/search/provider-session-reports.ts @@ -1,15 +1,8 @@ -import type { ProviderId } from "../types.js"; -import { nowIsoUtc } from "../../../lib/utils.js"; +import type { ProviderId } from "../../types.js"; +import { nowIsoUtc } from "../../../../lib/utils.js"; +import { listSessionReadableProviderIds } from "../../capabilities.js"; import { getProviderSessionScan } from "./provider-session-scan.js"; -const PROVIDER_SESSION_REPORT_TARGETS: ProviderId[] = [ - "codex", - "chatgpt", - "claude", - "gemini", - "copilot", -]; - export async function getProviderSessionsTs( provider?: ProviderId, limit = 80, @@ -17,7 +10,7 @@ export async function getProviderSessionsTs( ) { const targets: ProviderId[] = provider ? [provider] - : PROVIDER_SESSION_REPORT_TARGETS; + : listSessionReadableProviderIds(); const scans = await Promise.all( targets.map((p) => getProviderSessionScan(p, limit, options)), ); @@ -51,7 +44,7 @@ export async function getProviderParserHealthTs( ) { const targets: ProviderId[] = provider ? [provider] - : PROVIDER_SESSION_REPORT_TARGETS; + : listSessionReadableProviderIds(); const scans = await Promise.all( targets.map((item) => getProviderSessionScan(item, limitPerProvider, options)), ); diff --git a/apps/api-ts/src/domains/providers/search/provider-session-scan.ts b/apps/api-ts/src/domains/providers/services/search/provider-session-scan.ts similarity index 98% rename from apps/api-ts/src/domains/providers/search/provider-session-scan.ts rename to apps/api-ts/src/domains/providers/services/search/provider-session-scan.ts index a29e6d2..c52b46e 100644 --- a/apps/api-ts/src/domains/providers/search/provider-session-scan.ts +++ b/apps/api-ts/src/domains/providers/services/search/provider-session-scan.ts @@ -1,10 +1,10 @@ import type { ProviderId, ProviderSessionScan, -} from "../types.js"; +} from "../../types.js"; import { invalidateCodexThreadTitleMapCache, -} from "../title-detection.js"; +} from "../../title-detection.js"; import { awaitAbortable, throwIfAborted, diff --git a/apps/api-ts/src/domains/providers/search/raw-files.ts b/apps/api-ts/src/domains/providers/services/search/raw-files.ts similarity index 98% rename from apps/api-ts/src/domains/providers/search/raw-files.ts rename to apps/api-ts/src/domains/providers/services/search/raw-files.ts index 30ef6ea..3a8a6fd 100644 --- a/apps/api-ts/src/domains/providers/search/raw-files.ts +++ b/apps/api-ts/src/domains/providers/services/search/raw-files.ts @@ -1,6 +1,6 @@ import { spawn } from "node:child_process"; -import type { ProviderSessionRow } from "../types.js"; +import type { ProviderSessionRow } from "../../types.js"; import { buildSearchSnippet, buildSearchTokens, @@ -8,7 +8,7 @@ import { matchesConversationSearch, normalizeSearchQuery, normalizeSearchText, -} from "../search-helpers.js"; +} from "../../search-helpers.js"; import { createAbortError, throwIfAborted, diff --git a/apps/api-ts/src/domains/providers/search/row-search.ts b/apps/api-ts/src/domains/providers/services/search/row-search.ts similarity index 97% rename from apps/api-ts/src/domains/providers/search/row-search.ts rename to apps/api-ts/src/domains/providers/services/search/row-search.ts index 68dad0e..ea399d5 100644 --- a/apps/api-ts/src/domains/providers/search/row-search.ts +++ b/apps/api-ts/src/domains/providers/services/search/row-search.ts @@ -1,17 +1,17 @@ import path from "node:path"; -import { buildSessionTranscript } from "../transcript.js"; +import { buildSessionTranscript } from "../../transcript.js"; import type { ConversationSearchResult, ProviderId, ProviderSessionRow, -} from "../types.js"; +} from "../../types.js"; import { buildSearchSnippet, buildSearchTokens, matchesConversationSearch, normalizeSearchQuery, normalizeSearchText, -} from "../search-helpers.js"; +} from "../../search-helpers.js"; import { DEFAULT_CONVERSATION_SEARCH_TRANSCRIPT_LIMIT, SEARCH_TRANSCRIPT_CONCURRENCY, diff --git a/apps/api-ts/src/domains/providers/search/session-accumulator.ts b/apps/api-ts/src/domains/providers/services/search/session-accumulator.ts similarity index 99% rename from apps/api-ts/src/domains/providers/search/session-accumulator.ts rename to apps/api-ts/src/domains/providers/services/search/session-accumulator.ts index e471713..18bb7ff 100644 --- a/apps/api-ts/src/domains/providers/search/session-accumulator.ts +++ b/apps/api-ts/src/domains/providers/services/search/session-accumulator.ts @@ -2,7 +2,7 @@ import type { ConversationSearchResult, ConversationSearchSessionResult, ProviderSessionRow, -} from "../types.js"; +} from "../../types.js"; import { conversationSearchResultDedupKey, } from "./session-dedupe.js"; diff --git a/apps/api-ts/src/domains/providers/search/session-dedupe.ts b/apps/api-ts/src/domains/providers/services/search/session-dedupe.ts similarity index 95% rename from apps/api-ts/src/domains/providers/search/session-dedupe.ts rename to apps/api-ts/src/domains/providers/services/search/session-dedupe.ts index c32d659..ebd2ed4 100644 --- a/apps/api-ts/src/domains/providers/search/session-dedupe.ts +++ b/apps/api-ts/src/domains/providers/services/search/session-dedupe.ts @@ -3,13 +3,13 @@ import path from "node:path"; import type { ConversationSearchResult, ProviderSessionRow, -} from "../types.js"; +} from "../../types.js"; import { normalizeSearchText, -} from "../search-helpers.js"; +} from "../../search-helpers.js"; import { inferSessionId, -} from "../probe.js"; +} from "../../probe.js"; import { providerSessionSourcePriority, } from "./session-manifest.js"; diff --git a/apps/api-ts/src/domains/providers/search/session-hit-resolver.ts b/apps/api-ts/src/domains/providers/services/search/session-hit-resolver.ts similarity index 98% rename from apps/api-ts/src/domains/providers/search/session-hit-resolver.ts rename to apps/api-ts/src/domains/providers/services/search/session-hit-resolver.ts index 9b699be..13ca395 100644 --- a/apps/api-ts/src/domains/providers/search/session-hit-resolver.ts +++ b/apps/api-ts/src/domains/providers/services/search/session-hit-resolver.ts @@ -2,12 +2,12 @@ import path from "node:path"; import { inferSessionId, -} from "../probe.js"; +} from "../../probe.js"; import type { ConversationSearchSessionHitsPayload, ProviderId, ProviderSessionRow, -} from "../types.js"; +} from "../../types.js"; import { materializeProviderSessionRow, selectProviderSessionManifestCandidate, diff --git a/apps/api-ts/src/domains/providers/search/session-hit-search.ts b/apps/api-ts/src/domains/providers/services/search/session-hit-search.ts similarity index 97% rename from apps/api-ts/src/domains/providers/search/session-hit-search.ts rename to apps/api-ts/src/domains/providers/services/search/session-hit-search.ts index e38fac5..0a63aa3 100644 --- a/apps/api-ts/src/domains/providers/search/session-hit-search.ts +++ b/apps/api-ts/src/domains/providers/services/search/session-hit-search.ts @@ -1,11 +1,11 @@ import type { ConversationSearchSessionHitsPayload, ProviderSessionRow, -} from "../types.js"; +} from "../../types.js"; import { normalizeSearchText, -} from "../search-helpers.js"; -import { nowIsoUtc } from "../../../lib/utils.js"; +} from "../../search-helpers.js"; +import { nowIsoUtc } from "../../../../lib/utils.js"; import { throwIfAborted, } from "./abort.js"; diff --git a/apps/api-ts/src/domains/providers/search/session-manifest-files.ts b/apps/api-ts/src/domains/providers/services/search/session-manifest-files.ts similarity index 96% rename from apps/api-ts/src/domains/providers/search/session-manifest-files.ts rename to apps/api-ts/src/domains/providers/services/search/session-manifest-files.ts index be4a169..d795e50 100644 --- a/apps/api-ts/src/domains/providers/search/session-manifest-files.ts +++ b/apps/api-ts/src/domains/providers/services/search/session-manifest-files.ts @@ -3,18 +3,18 @@ import path from "node:path"; import { normalizeSearchText, -} from "../search-helpers.js"; +} from "../../search-helpers.js"; import { isCopilotGlobalSessionLikeFile, isWorkspaceChatSessionPath, -} from "../probe.js"; +} from "../../probe.js"; import type { ProviderId, ProviderRootSpec, -} from "../types.js"; +} from "../../types.js"; import { walkFilesByExt, -} from "../../../lib/utils.js"; +} from "../../../../lib/utils.js"; import { throwIfAborted, } from "./abort.js"; diff --git a/apps/api-ts/src/domains/providers/search/session-manifest-rows.ts b/apps/api-ts/src/domains/providers/services/search/session-manifest-rows.ts similarity index 96% rename from apps/api-ts/src/domains/providers/search/session-manifest-rows.ts rename to apps/api-ts/src/domains/providers/services/search/session-manifest-rows.ts index 5e39901..7537fda 100644 --- a/apps/api-ts/src/domains/providers/search/session-manifest-rows.ts +++ b/apps/api-ts/src/domains/providers/services/search/session-manifest-rows.ts @@ -1,21 +1,21 @@ import { fallbackDisplayTitle, -} from "../search-helpers.js"; +} from "../../search-helpers.js"; import { extractCodexThreadIdFromSessionName, getCodexThreadTitleMap, -} from "../title-detection.js"; +} from "../../title-detection.js"; import { inferSessionId, probeSessionFile, -} from "../probe.js"; +} from "../../probe.js"; import { providerStatus, -} from "../matrix.js"; +} from "../../matrix.js"; import type { ProviderSessionRow, ProviderSessionScan, -} from "../types.js"; +} from "../../types.js"; import { throwIfAborted, } from "./abort.js"; diff --git a/apps/api-ts/src/domains/providers/search/session-manifest.ts b/apps/api-ts/src/domains/providers/services/search/session-manifest.ts similarity index 97% rename from apps/api-ts/src/domains/providers/search/session-manifest.ts rename to apps/api-ts/src/domains/providers/services/search/session-manifest.ts index 2b41507..0beddd6 100644 --- a/apps/api-ts/src/domains/providers/search/session-manifest.ts +++ b/apps/api-ts/src/domains/providers/services/search/session-manifest.ts @@ -3,13 +3,13 @@ import path from "node:path"; import { providerName, providerScanRootSpecs, -} from "../path-safety.js"; +} from "../../path-safety.js"; import { inferSessionId, -} from "../probe.js"; +} from "../../probe.js"; import type { ProviderId, -} from "../types.js"; +} from "../../types.js"; import { throwIfAborted, } from "./abort.js"; diff --git a/apps/api-ts/src/domains/providers/search/session-metadata-results.ts b/apps/api-ts/src/domains/providers/services/search/session-metadata-results.ts similarity index 94% rename from apps/api-ts/src/domains/providers/search/session-metadata-results.ts rename to apps/api-ts/src/domains/providers/services/search/session-metadata-results.ts index 8266454..9870fe3 100644 --- a/apps/api-ts/src/domains/providers/search/session-metadata-results.ts +++ b/apps/api-ts/src/domains/providers/services/search/session-metadata-results.ts @@ -3,23 +3,23 @@ import path from "node:path"; import type { ConversationSearchResult, ProviderSessionRow, -} from "../types.js"; +} from "../../types.js"; import { buildSearchSnippet, fallbackDisplayTitle, isExactPhraseSearchMatch, matchesConversationSearch, normalizeSearchText, -} from "../search-helpers.js"; +} from "../../search-helpers.js"; import { providerName, -} from "../path-safety.js"; +} from "../../path-safety.js"; import { extractCodexThreadIdFromSessionName, -} from "../title-detection.js"; +} from "../../title-detection.js"; import { inferSessionId, -} from "../probe.js"; +} from "../../probe.js"; export type SearchIdentity = { sessionId: string; diff --git a/apps/api-ts/src/domains/providers/search/session-raw-search.ts b/apps/api-ts/src/domains/providers/services/search/session-raw-search.ts similarity index 96% rename from apps/api-ts/src/domains/providers/search/session-raw-search.ts rename to apps/api-ts/src/domains/providers/services/search/session-raw-search.ts index 480dd2e..b542840 100644 --- a/apps/api-ts/src/domains/providers/search/session-raw-search.ts +++ b/apps/api-ts/src/domains/providers/services/search/session-raw-search.ts @@ -1,4 +1,4 @@ -import type { ProviderSessionRow } from "../types.js"; +import type { ProviderSessionRow } from "../../types.js"; import { type RawConversationFileMatch, type RawConversationFileSearchLoader, diff --git a/apps/api-ts/src/domains/providers/search/session-results.ts b/apps/api-ts/src/domains/providers/services/search/session-results.ts similarity index 97% rename from apps/api-ts/src/domains/providers/search/session-results.ts rename to apps/api-ts/src/domains/providers/services/search/session-results.ts index b0b85d7..12cc861 100644 --- a/apps/api-ts/src/domains/providers/search/session-results.ts +++ b/apps/api-ts/src/domains/providers/services/search/session-results.ts @@ -1,7 +1,7 @@ import { normalizeSearchQuery, normalizeSearchText, -} from "../search-helpers.js"; +} from "../../search-helpers.js"; export type { SearchIdentity, diff --git a/apps/api-ts/src/domains/providers/search/session-row-order.ts b/apps/api-ts/src/domains/providers/services/search/session-row-order.ts similarity index 97% rename from apps/api-ts/src/domains/providers/search/session-row-order.ts rename to apps/api-ts/src/domains/providers/services/search/session-row-order.ts index 779ee4a..ea9b079 100644 --- a/apps/api-ts/src/domains/providers/search/session-row-order.ts +++ b/apps/api-ts/src/domains/providers/services/search/session-row-order.ts @@ -1,7 +1,7 @@ import path from "node:path"; import type { ProviderSessionRow, -} from "../types.js"; +} from "../../types.js"; import { dedupeConversationSearchRows, } from "./session-results.js"; diff --git a/apps/api-ts/src/domains/providers/search/session-search.ts b/apps/api-ts/src/domains/providers/services/search/session-search.ts similarity index 99% rename from apps/api-ts/src/domains/providers/search/session-search.ts rename to apps/api-ts/src/domains/providers/services/search/session-search.ts index 2067f70..58a0536 100644 --- a/apps/api-ts/src/domains/providers/search/session-search.ts +++ b/apps/api-ts/src/domains/providers/services/search/session-search.ts @@ -2,7 +2,7 @@ import type { ConversationSearchResult, ConversationSearchSessionResult, ProviderSessionRow, -} from "../types.js"; +} from "../../types.js"; import { throwIfAborted, } from "./abort.js"; diff --git a/apps/api-ts/src/domains/providers/search/transcript-loader-cache.ts b/apps/api-ts/src/domains/providers/services/search/transcript-loader-cache.ts similarity index 100% rename from apps/api-ts/src/domains/providers/search/transcript-loader-cache.ts rename to apps/api-ts/src/domains/providers/services/search/transcript-loader-cache.ts diff --git a/apps/api-ts/src/domains/providers/search/transcript-matches.ts b/apps/api-ts/src/domains/providers/services/search/transcript-matches.ts similarity index 97% rename from apps/api-ts/src/domains/providers/search/transcript-matches.ts rename to apps/api-ts/src/domains/providers/services/search/transcript-matches.ts index 671ddf9..16b0d32 100644 --- a/apps/api-ts/src/domains/providers/search/transcript-matches.ts +++ b/apps/api-ts/src/domains/providers/services/search/transcript-matches.ts @@ -1,13 +1,13 @@ -import { buildSessionTranscript } from "../transcript.js"; +import { buildSessionTranscript } from "../../transcript.js"; import type { ProviderId, ProviderSessionRow, -} from "../types.js"; +} from "../../types.js"; import { buildSearchTokens, normalizeSearchQuery, normalizeSearchText, -} from "../search-helpers.js"; +} from "../../search-helpers.js"; import { throwIfAborted, } from "./abort.js"; diff --git a/apps/api-ts/src/domains/providers/search/transcript-message-matches.ts b/apps/api-ts/src/domains/providers/services/search/transcript-message-matches.ts similarity index 96% rename from apps/api-ts/src/domains/providers/search/transcript-message-matches.ts rename to apps/api-ts/src/domains/providers/services/search/transcript-message-matches.ts index 61814c7..fef72b6 100644 --- a/apps/api-ts/src/domains/providers/search/transcript-message-matches.ts +++ b/apps/api-ts/src/domains/providers/services/search/transcript-message-matches.ts @@ -1,12 +1,12 @@ import type { ProviderSessionRow, TranscriptMessage, -} from "../types.js"; +} from "../../types.js"; import { buildSearchSnippet, isExactPhraseSearchMatch, matchesConversationSearch, -} from "../search-helpers.js"; +} from "../../search-helpers.js"; import { throwIfAborted, } from "./abort.js"; diff --git a/apps/api-ts/src/domains/providers/search/transcript-policy.ts b/apps/api-ts/src/domains/providers/services/search/transcript-policy.ts similarity index 100% rename from apps/api-ts/src/domains/providers/search/transcript-policy.ts rename to apps/api-ts/src/domains/providers/services/search/transcript-policy.ts diff --git a/apps/api-ts/src/domains/providers/search/transcript-raw-matches.ts b/apps/api-ts/src/domains/providers/services/search/transcript-raw-matches.ts similarity index 95% rename from apps/api-ts/src/domains/providers/search/transcript-raw-matches.ts rename to apps/api-ts/src/domains/providers/services/search/transcript-raw-matches.ts index 45552d5..25180e4 100644 --- a/apps/api-ts/src/domains/providers/search/transcript-raw-matches.ts +++ b/apps/api-ts/src/domains/providers/services/search/transcript-raw-matches.ts @@ -1,9 +1,9 @@ import type { ProviderSessionRow, -} from "../types.js"; +} from "../../types.js"; import { isExactPhraseSearchMatch, -} from "../search-helpers.js"; +} from "../../search-helpers.js"; import type { RawConversationFileMatch, } from "./raw-files.js"; diff --git a/apps/api-ts/src/domains/providers/search/types.ts b/apps/api-ts/src/domains/providers/services/search/types.ts similarity index 97% rename from apps/api-ts/src/domains/providers/search/types.ts rename to apps/api-ts/src/domains/providers/services/search/types.ts index 161749f..0bf6ee3 100644 --- a/apps/api-ts/src/domains/providers/search/types.ts +++ b/apps/api-ts/src/domains/providers/services/search/types.ts @@ -4,7 +4,7 @@ import type { ProviderSessionRow, ProviderSessionScan, TranscriptPayload, -} from "../types.js"; +} from "../../types.js"; export type ProviderScanCacheEntry = { expires_at: number; diff --git a/apps/api-ts/src/domains/providers/services/transcripts/index.ts b/apps/api-ts/src/domains/providers/services/transcripts/index.ts new file mode 100644 index 0000000..bd376a9 --- /dev/null +++ b/apps/api-ts/src/domains/providers/services/transcripts/index.ts @@ -0,0 +1,91 @@ +import { readFile } from "node:fs/promises"; +import { readFileTail } from "../../../../lib/utils.js"; +import type { + ProviderId, + TranscriptPayload, +} from "../../types.js"; +import { + parseJsonTranscriptPayload, +} from "./json.js"; +import { + parseJsonlTranscriptLines, +} from "./jsonl.js"; +import { + extractCodexThreadIdFromSessionName, + inferFormat, + inferSessionId, +} from "./normalizers.js"; + +export async function buildSessionTranscript( + provider: ProviderId, + filePath: string, + limit: number, +): Promise { + const safeLimit = Math.max(20, Math.min(10_000, Number(limit) || 200)); + const format = inferFormat(filePath); + const threadId = + provider === "codex" + ? extractCodexThreadIdFromSessionName(inferSessionId(filePath)) + : ""; + + if (format === "json") { + const raw = await readFile(filePath, "utf-8").catch(() => ""); + const messages = parseJsonTranscriptPayload(raw, provider); + const sliced = messages + .slice(Math.max(0, messages.length - safeLimit)) + .map((msg, index) => ({ ...msg, idx: index + 1 })); + return { + provider, + thread_id: threadId || null, + file_path: filePath, + scanned_lines: raw ? raw.split("\n").length : 0, + message_count: sliced.length, + truncated: messages.length > safeLimit, + messages: sliced, + }; + } + + if (format !== "jsonl") { + return { + provider, + thread_id: threadId || null, + file_path: filePath, + scanned_lines: 0, + message_count: 0, + truncated: false, + messages: [], + }; + } + + const tail = await readFileTail(filePath, 2_621_440); + let lines = tail.text + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + let messages = parseJsonlTranscriptLines(lines); + + if (tail.truncated && safeLimit > messages.length) { + const fullRaw = await readFile(filePath, "utf-8").catch(() => ""); + if (fullRaw) { + lines = fullRaw + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + messages = parseJsonlTranscriptLines(lines); + } + } + + const sliced = messages + .slice(Math.max(0, messages.length - safeLimit)) + .map((msg, index) => ({ ...msg, idx: index + 1 })); + + return { + provider, + thread_id: threadId || null, + file_path: filePath, + scanned_lines: lines.length, + message_count: sliced.length, + truncated: messages.length > safeLimit, + messages: sliced, + }; +} diff --git a/apps/api-ts/src/domains/providers/transcripts/json.ts b/apps/api-ts/src/domains/providers/services/transcripts/json.ts similarity index 98% rename from apps/api-ts/src/domains/providers/transcripts/json.ts rename to apps/api-ts/src/domains/providers/services/transcripts/json.ts index d74d6e5..5757480 100644 --- a/apps/api-ts/src/domains/providers/transcripts/json.ts +++ b/apps/api-ts/src/domains/providers/services/transcripts/json.ts @@ -1,10 +1,10 @@ import { isRecord, -} from "../../../lib/utils.js"; +} from "../../../../lib/utils.js"; import type { ProviderId, TranscriptMessage, -} from "../types.js"; +} from "../../types.js"; import { extractTranscriptText, isBoilerplateTitle, diff --git a/apps/api-ts/src/domains/providers/transcripts/jsonl.ts b/apps/api-ts/src/domains/providers/services/transcripts/jsonl.ts similarity index 98% rename from apps/api-ts/src/domains/providers/transcripts/jsonl.ts rename to apps/api-ts/src/domains/providers/services/transcripts/jsonl.ts index f49ffb9..84087af 100644 --- a/apps/api-ts/src/domains/providers/transcripts/jsonl.ts +++ b/apps/api-ts/src/domains/providers/services/transcripts/jsonl.ts @@ -1,9 +1,9 @@ import { isRecord, -} from "../../../lib/utils.js"; +} from "../../../../lib/utils.js"; import type { TranscriptMessage, -} from "../types.js"; +} from "../../types.js"; import { extractTranscriptText, isBoilerplateTitle, diff --git a/apps/api-ts/src/domains/providers/transcripts/normalizers.ts b/apps/api-ts/src/domains/providers/services/transcripts/normalizers.ts similarity index 99% rename from apps/api-ts/src/domains/providers/transcripts/normalizers.ts rename to apps/api-ts/src/domains/providers/services/transcripts/normalizers.ts index d6a0cf3..068551e 100644 --- a/apps/api-ts/src/domains/providers/transcripts/normalizers.ts +++ b/apps/api-ts/src/domains/providers/services/transcripts/normalizers.ts @@ -1,7 +1,7 @@ import type { ProviderId, TranscriptMessage, -} from "../types.js"; +} from "../../types.js"; export function inferFormat(filePath: string): "jsonl" | "json" | "unknown" { const lower = filePath.toLowerCase(); diff --git a/apps/api-ts/src/domains/providers/roots/backups.ts b/apps/api-ts/src/domains/providers/shared/backup-roots.ts similarity index 100% rename from apps/api-ts/src/domains/providers/roots/backups.ts rename to apps/api-ts/src/domains/providers/shared/backup-roots.ts diff --git a/apps/api-ts/src/domains/providers/shared/file-roots.ts b/apps/api-ts/src/domains/providers/shared/file-roots.ts new file mode 100644 index 0000000..2ec37db --- /dev/null +++ b/apps/api-ts/src/domains/providers/shared/file-roots.ts @@ -0,0 +1,134 @@ +import path from "node:path"; +import type { + ProviderId, +} from "@threadlens/shared-contracts"; + +import { + CHAT_DIR, + CODEX_HOME, + CLAUDE_PROJECTS_DIR, + CLAUDE_TRANSCRIPTS_DIR, + COPILOT_CURSOR_GLOBAL, + COPILOT_CURSOR_WORKSPACE_STORAGE, + COPILOT_VSCODE_GLOBAL, + COPILOT_VSCODE_WORKSPACE_STORAGE, + GEMINI_ANTIGRAVITY_CONVERSATIONS_DIR, + GEMINI_HISTORY_DIR, + GEMINI_TMP_DIR, +} from "../../../lib/constants.js"; +import { discoverChatGptConversationRoots } from "../adapters/chatgpt/roots.js"; +import { discoverCodexCwdBackupRoots } from "../adapters/codex/roots.js"; +import { + providerArchivedRootSpec, + providerBackupRootSpecs, +} from "./backup-roots.js"; +import type { + ProviderRootSpec, +} from "../types.js"; + +export { codexTranscriptSearchRoots } from "../adapters/codex/roots.js"; + +const PROVIDER_ROOT_SOURCE_PATTERN = /^[a-z0-9][a-z0-9_]*$/; + +export function isValidProviderRootSource(source: string): boolean { + return PROVIDER_ROOT_SOURCE_PATTERN.test(source); +} + +export function isValidProviderRootSpec(spec: ProviderRootSpec): boolean { + return ( + isValidProviderRootSource(spec.source) && + path.isAbsolute(spec.root) && + spec.exts.length > 0 && + spec.exts.every((ext) => ext.startsWith(".") && ext === ext.toLowerCase()) + ); +} + +export function validateProviderRootSpecs( + specs: ProviderRootSpec[], +): ProviderRootSpec[] { + return specs.filter(isValidProviderRootSpec); +} + +export function providerRootSpecs(provider: ProviderId): ProviderRootSpec[] { + const archivedSpec = providerArchivedRootSpec(provider); + if (provider === "codex") { + return validateProviderRootSpecs([ + { + source: "sessions", + root: path.join(CODEX_HOME, "sessions"), + exts: [".jsonl"], + }, + ...(archivedSpec ? [archivedSpec] : []), + ...providerBackupRootSpecs(provider), + ]); + } + if (provider === "chatgpt") { + return validateProviderRootSpecs([ + { source: "chat_cache", root: CHAT_DIR, exts: [".data"] }, + ]); + } + if (provider === "claude") { + return validateProviderRootSpecs([ + { source: "projects", root: CLAUDE_PROJECTS_DIR, exts: [".jsonl"] }, + { + source: "transcripts", + root: CLAUDE_TRANSCRIPTS_DIR, + exts: [".jsonl", ".json"], + }, + ...(archivedSpec ? [archivedSpec] : []), + ...providerBackupRootSpecs(provider), + ]); + } + if (provider === "gemini") { + return validateProviderRootSpecs([ + { source: "tmp", root: GEMINI_TMP_DIR, exts: [".jsonl", ".json"] }, + { source: "history", root: GEMINI_HISTORY_DIR, exts: [".jsonl", ".json"] }, + { + source: "antigravity_conversations", + root: GEMINI_ANTIGRAVITY_CONVERSATIONS_DIR, + exts: [".pb"], + }, + ...(archivedSpec ? [archivedSpec] : []), + ...providerBackupRootSpecs(provider), + ]); + } + if (provider === "copilot") { + return validateProviderRootSpecs([ + { + source: "vscode_global", + root: COPILOT_VSCODE_GLOBAL, + exts: [".jsonl", ".json"], + }, + { + source: "cursor_global", + root: COPILOT_CURSOR_GLOBAL, + exts: [".jsonl", ".json"], + }, + { + source: "vscode_workspace_chats", + root: COPILOT_VSCODE_WORKSPACE_STORAGE, + exts: [".json"], + }, + { + source: "cursor_workspace_chats", + root: COPILOT_CURSOR_WORKSPACE_STORAGE, + exts: [".json"], + }, + ...(archivedSpec ? [archivedSpec] : []), + ...providerBackupRootSpecs(provider), + ]); + } + return []; +} + +export async function providerScanRootSpecs( + provider: ProviderId, +): Promise { + if (provider === "chatgpt") { + const discovered = await discoverChatGptConversationRoots(); + return validateProviderRootSpecs(discovered); + } + if (provider !== "codex") return providerRootSpecs(provider); + const extraRoots = await discoverCodexCwdBackupRoots(); + return validateProviderRootSpecs([...providerRootSpecs(provider), ...extraRoots]); +} diff --git a/apps/api-ts/src/domains/providers/shared/path-safety.ts b/apps/api-ts/src/domains/providers/shared/path-safety.ts new file mode 100644 index 0000000..a9a0339 --- /dev/null +++ b/apps/api-ts/src/domains/providers/shared/path-safety.ts @@ -0,0 +1,130 @@ +import { lstat, realpath } from "node:fs/promises"; +import path from "node:path"; +import { + PROVIDER_LABELS, + type ProviderId, +} from "@threadlens/shared-contracts"; +import { + CHAT_DIR, + CODEX_HOME, +} from "../../../lib/constants.js"; +import { + providerRootSpecs, + providerScanRootSpecs, +} from "./file-roots.js"; +import { + listProviderIds, + parseProviderId, +} from "../capabilities.js"; + +export { + codexTranscriptSearchRoots, + providerRootSpecs, + providerScanRootSpecs, +} from "./file-roots.js"; +export { + listProviderIds, + parseProviderId, +} from "../capabilities.js"; + +export function providerName(provider: ProviderId): string { + return PROVIDER_LABELS[provider] ?? provider; +} + +export function isPathInsideRoot( + targetPath: string, + rootPath: string, +): boolean { + const fullTarget = path.resolve(targetPath); + const fullRoot = path.resolve(rootPath); + return ( + fullTarget === fullRoot || fullTarget.startsWith(`${fullRoot}${path.sep}`) + ); +} + +function isCodexCwdBackupPath(filePath: string): boolean { + const ext = path.extname(filePath).toLowerCase(); + if (ext !== ".jsonl") return false; + if (!isPathInsideRoot(filePath, CODEX_HOME)) return false; + const relativePath = path.relative(CODEX_HOME, path.resolve(filePath)); + const [firstSegment = ""] = relativePath.split(path.sep); + return ( + firstSegment.startsWith("jsonl-cwd-backups-") && + !relativePath.startsWith(`..${path.sep}`) && + relativePath !== ".." + ); +} + +export function isAllowedProviderFilePath( + provider: ProviderId, + filePath: string, +): boolean { + if (provider === "codex" && isCodexCwdBackupPath(filePath)) { + return true; + } + if (provider === "chatgpt") { + const ext = path.extname(filePath).toLowerCase(); + if (ext !== ".data") return false; + if (!isPathInsideRoot(filePath, CHAT_DIR)) return false; + const normalized = path.resolve(filePath); + return /(^|[\\/])conversations-v3-[^\\/]+[\\/]/.test(normalized); + } + const specs = providerRootSpecs(provider); + const ext = path.extname(filePath).toLowerCase(); + return specs.some( + (spec) => spec.exts.includes(ext) && isPathInsideRoot(filePath, spec.root), + ); +} + +export async function resolveSafePathWithinRoots( + filePath: string, + rootPaths: string[], +): Promise { + const normalizedTarget = path.resolve(filePath); + let targetLstat; + try { + targetLstat = await lstat(normalizedTarget); + } catch { + return null; + } + if (targetLstat.isSymbolicLink()) return null; + + let realTarget = ""; + try { + realTarget = await realpath(normalizedTarget); + } catch { + return null; + } + + for (const rootPath of rootPaths) { + try { + const realRoot = await realpath(rootPath); + if ( + realTarget === realRoot || + realTarget.startsWith(`${realRoot}${path.sep}`) + ) { + return realTarget; + } + } catch { + continue; + } + } + + return null; +} + +export async function resolveAllowedProviderFilePath( + provider: ProviderId, + filePath: string, +): Promise { + const specs = await providerScanRootSpecs(provider); + const ext = path.extname(filePath).toLowerCase(); + const matchingRoots = specs + .filter( + (spec) => spec.exts.includes(ext) && isPathInsideRoot(filePath, spec.root), + ) + .map((spec) => spec.root); + + if (!matchingRoots.length) return null; + return resolveSafePathWithinRoots(filePath, matchingRoots); +} diff --git a/apps/api-ts/src/domains/providers/title-detection.ts b/apps/api-ts/src/domains/providers/title-detection.ts index 0e70e26..09bac62 100644 --- a/apps/api-ts/src/domains/providers/title-detection.ts +++ b/apps/api-ts/src/domains/providers/title-detection.ts @@ -6,8 +6,8 @@ export { extractCodexThreadIdFromSessionName, getCodexThreadTitleMap, invalidateCodexThreadTitleMapCache, -} from "./codex-title-map.js"; -export { detectClaudeRenamedTitle } from "./claude-title-detection.js"; +} from "./adapters/codex/title-map.js"; +export { detectClaudeRenamedTitle } from "./adapters/claude/title.js"; export { normalizeDetectedTitle } from "./title-normalization.js"; export function fallbackDisplayTitle( diff --git a/apps/api-ts/src/domains/providers/transcript.test.ts b/apps/api-ts/src/domains/providers/transcript.test.ts index 8d9de9e..bc84ec3 100644 --- a/apps/api-ts/src/domains/providers/transcript.test.ts +++ b/apps/api-ts/src/domains/providers/transcript.test.ts @@ -3,7 +3,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { buildSessionTranscript } from "./transcript"; -import { inferSessionId } from "./transcripts/normalizers.js"; +import { inferSessionId } from "./services/transcripts/normalizers.js"; const tmpDirs: string[] = []; diff --git a/apps/api-ts/src/domains/providers/transcript.ts b/apps/api-ts/src/domains/providers/transcript.ts index e6a3826..6f9d429 100644 --- a/apps/api-ts/src/domains/providers/transcript.ts +++ b/apps/api-ts/src/domains/providers/transcript.ts @@ -1,91 +1 @@ -import { readFile } from "node:fs/promises"; -import { readFileTail } from "../../lib/utils.js"; -import type { - ProviderId, - TranscriptPayload, -} from "./types.js"; -import { - parseJsonTranscriptPayload, -} from "./transcripts/json.js"; -import { - parseJsonlTranscriptLines, -} from "./transcripts/jsonl.js"; -import { - extractCodexThreadIdFromSessionName, - inferFormat, - inferSessionId, -} from "./transcripts/normalizers.js"; - -export async function buildSessionTranscript( - provider: ProviderId, - filePath: string, - limit: number, -): Promise { - const safeLimit = Math.max(20, Math.min(10_000, Number(limit) || 200)); - const format = inferFormat(filePath); - const threadId = - provider === "codex" - ? extractCodexThreadIdFromSessionName(inferSessionId(filePath)) - : ""; - - if (format === "json") { - const raw = await readFile(filePath, "utf-8").catch(() => ""); - const messages = parseJsonTranscriptPayload(raw, provider); - const sliced = messages - .slice(Math.max(0, messages.length - safeLimit)) - .map((msg, index) => ({ ...msg, idx: index + 1 })); - return { - provider, - thread_id: threadId || null, - file_path: filePath, - scanned_lines: raw ? raw.split("\n").length : 0, - message_count: sliced.length, - truncated: messages.length > safeLimit, - messages: sliced, - }; - } - - if (format !== "jsonl") { - return { - provider, - thread_id: threadId || null, - file_path: filePath, - scanned_lines: 0, - message_count: 0, - truncated: false, - messages: [], - }; - } - - const tail = await readFileTail(filePath, 2_621_440); - let lines = tail.text - .split("\n") - .map((line) => line.trim()) - .filter(Boolean); - let messages = parseJsonlTranscriptLines(lines); - - if (tail.truncated && safeLimit > messages.length) { - const fullRaw = await readFile(filePath, "utf-8").catch(() => ""); - if (fullRaw) { - lines = fullRaw - .split("\n") - .map((line) => line.trim()) - .filter(Boolean); - messages = parseJsonlTranscriptLines(lines); - } - } - - const sliced = messages - .slice(Math.max(0, messages.length - safeLimit)) - .map((msg, index) => ({ ...msg, idx: index + 1 })); - - return { - provider, - thread_id: threadId || null, - file_path: filePath, - scanned_lines: lines.length, - message_count: sliced.length, - truncated: messages.length > safeLimit, - messages: sliced, - }; -} +export * from "./services/transcripts/index.js"; diff --git a/apps/api-ts/src/lib/recovery-backups.ts b/apps/api-ts/src/domains/recovery/backups.ts similarity index 97% rename from apps/api-ts/src/lib/recovery-backups.ts rename to apps/api-ts/src/domains/recovery/backups.ts index fab7716..e9e0383 100644 --- a/apps/api-ts/src/lib/recovery-backups.ts +++ b/apps/api-ts/src/domains/recovery/backups.ts @@ -1,7 +1,7 @@ import { readdir, stat } from "node:fs/promises"; import path from "node:path"; -import { BACKUP_ROOT } from "./constants.js"; -import { walkFiles } from "./utils.js"; +import { BACKUP_ROOT } from "../../lib/constants.js"; +import { walkFiles } from "../../lib/utils.js"; export type RecoveryBackupSet = { backup_id: string; diff --git a/apps/api-ts/src/lib/recovery-checklist.ts b/apps/api-ts/src/domains/recovery/checklist.ts similarity index 94% rename from apps/api-ts/src/lib/recovery-checklist.ts rename to apps/api-ts/src/domains/recovery/checklist.ts index bb426c3..1aef8d9 100644 --- a/apps/api-ts/src/lib/recovery-checklist.ts +++ b/apps/api-ts/src/domains/recovery/checklist.ts @@ -1,7 +1,7 @@ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; -import { RECOVERY_CHECKLIST_FILE } from "./constants.js"; -import { isRecord, readJsonFile } from "./utils.js"; +import { RECOVERY_CHECKLIST_FILE } from "../../lib/constants.js"; +import { isRecord, readJsonFile } from "../../lib/utils.js"; export type RecoveryChecklistItem = { id: string; diff --git a/apps/api-ts/src/lib/recovery-export.ts b/apps/api-ts/src/domains/recovery/export.ts similarity index 95% rename from apps/api-ts/src/lib/recovery-export.ts rename to apps/api-ts/src/domains/recovery/export.ts index 11ee53c..01a1a8a 100644 --- a/apps/api-ts/src/lib/recovery-export.ts +++ b/apps/api-ts/src/domains/recovery/export.ts @@ -1,10 +1,10 @@ import { createReadStream } from "node:fs"; import { cp, mkdir, stat, writeFile } from "node:fs/promises"; import path from "node:path"; -import { BACKUP_ROOT, RECOVERY_EXPORT_ROOT } from "./constants.js"; -import { scanBackupSets } from "./recovery-backups.js"; -import { writePortableZipArchive } from "./recovery-portable-zip.js"; -import { nowIsoUtc } from "./utils.js"; +import { BACKUP_ROOT, RECOVERY_EXPORT_ROOT } from "../../lib/constants.js"; +import { scanBackupSets } from "./backups.js"; +import { writePortableZipArchive } from "./portable-zip.js"; +import { nowIsoUtc } from "../../lib/utils.js"; type RecoveryRootsOverride = { backup_root?: string; diff --git a/apps/api-ts/src/domains/recovery/index.ts b/apps/api-ts/src/domains/recovery/index.ts new file mode 100644 index 0000000..01ddd67 --- /dev/null +++ b/apps/api-ts/src/domains/recovery/index.ts @@ -0,0 +1 @@ +export * from "./runtime.js"; diff --git a/apps/api-ts/src/lib/recovery-portable-zip.ts b/apps/api-ts/src/domains/recovery/portable-zip.ts similarity index 100% rename from apps/api-ts/src/lib/recovery-portable-zip.ts rename to apps/api-ts/src/domains/recovery/portable-zip.ts diff --git a/apps/api-ts/src/domains/recovery/roadmap.ts b/apps/api-ts/src/domains/recovery/roadmap.ts index 58eb1ea..56d7f75 100644 --- a/apps/api-ts/src/domains/recovery/roadmap.ts +++ b/apps/api-ts/src/domains/recovery/roadmap.ts @@ -17,7 +17,7 @@ import { getOverviewTs } from "../threads/overview.js"; import { getRelatedToolsStatusTs, getRuntimeHealthTs, -} from "../../lib/recovery.js"; +} from "./index.js"; async function readRoadmapState(): Promise[]> { const data = await readJsonFile(ROADMAP_STATE_FILE); diff --git a/apps/api-ts/src/lib/recovery.test.ts b/apps/api-ts/src/domains/recovery/runtime.test.ts similarity index 99% rename from apps/api-ts/src/lib/recovery.test.ts rename to apps/api-ts/src/domains/recovery/runtime.test.ts index ae8de7a..cc52ffd 100644 --- a/apps/api-ts/src/lib/recovery.test.ts +++ b/apps/api-ts/src/domains/recovery/runtime.test.ts @@ -6,7 +6,7 @@ import { exportRecoveryBackupsTs, getLatestSmokeStatusTs, getRecoveryCenterDataTs, -} from "./recovery"; +} from "./index.js"; async function withTempDir(fn: (dir: string) => Promise): Promise { const dir = await mkdtemp(path.join(os.tmpdir(), "threadlens-smoke-")); diff --git a/apps/api-ts/src/lib/recovery.ts b/apps/api-ts/src/domains/recovery/runtime.ts similarity index 97% rename from apps/api-ts/src/lib/recovery.ts rename to apps/api-ts/src/domains/recovery/runtime.ts index e09ccec..38564a3 100644 --- a/apps/api-ts/src/lib/recovery.ts +++ b/apps/api-ts/src/domains/recovery/runtime.ts @@ -18,7 +18,7 @@ import { RECOVERY_EXPORT_ROOT, RECOVERY_PLAN_DIR, CHAT_DIR, -} from "./constants.js"; +} from "../../lib/constants.js"; import { pathExists, walkFiles, @@ -29,17 +29,17 @@ import { countDirsWithPrefix, quickFileCount, countJsonlFilesRecursive, -} from "./utils.js"; -import { isCleanupBackupId, scanBackupSets } from "./recovery-backups.js"; -import { loadRecoveryChecklist } from "./recovery-checklist.js"; +} from "../../lib/utils.js"; +import { isCleanupBackupId, scanBackupSets } from "./backups.js"; +import { loadRecoveryChecklist } from "./checklist.js"; -export { updateRecoveryChecklistItem } from "./recovery-checklist.js"; +export { updateRecoveryChecklistItem } from "./checklist.js"; export { exportRecoveryBackupsTs, openRecoveryBackupArchiveReadStream, resolveRecoveryBackupArchivePath, -} from "./recovery-export.js"; -export { getLatestSmokeStatusTs } from "./recovery-smoke.js"; +} from "./export.js"; +export { getLatestSmokeStatusTs } from "./smoke.js"; /* ─────────────────────────────────────────────────────────────────── * * Backup scanning & restore plans * diff --git a/apps/api-ts/src/lib/recovery-smoke.ts b/apps/api-ts/src/domains/recovery/smoke.ts similarity index 99% rename from apps/api-ts/src/lib/recovery-smoke.ts rename to apps/api-ts/src/domains/recovery/smoke.ts index cfa8c7c..6fc6b46 100644 --- a/apps/api-ts/src/lib/recovery-smoke.ts +++ b/apps/api-ts/src/domains/recovery/smoke.ts @@ -1,13 +1,13 @@ import { readFile, readdir } from "node:fs/promises"; import path from "node:path"; -import { PROJECT_ROOT } from "./constants.js"; +import { PROJECT_ROOT } from "../../lib/constants.js"; import { isRecord, nowIsoUtc, parseNumber, pathExists, safeJsonParse, -} from "./utils.js"; +} from "../../lib/utils.js"; const SMOKE_SUMMARY_DIR = path.join(PROJECT_ROOT, ".run", "smoke"); const SMOKE_SUMMARY_FILE_RE = /^smoke-summary-(\d{8}T\d{6}Z)\.json$/; diff --git a/apps/api-ts/src/lib/providers.ts b/apps/api-ts/src/lib/providers.ts deleted file mode 100644 index a7f26a8..0000000 --- a/apps/api-ts/src/lib/providers.ts +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Provider-domain compatibility facade. - * - * New code should prefer `src/domains/providers/*`. - */ - -import { buildSessionTranscript } from "../domains/providers/transcript.js"; -import { - buildProviderActionToken, - runProviderSessionAction as runProviderSessionActionInternal, -} from "../domains/providers/actions.js"; -import { invalidateProviderSearchCaches } from "../domains/providers/search.js"; -import { invalidateProviderMatrixCache } from "../domains/providers/matrix.js"; -import { resolveAllowedProviderFilePath } from "../domains/providers/path-safety.js"; -import type { ProviderId } from "../domains/providers/types.js"; -import type { ProviderSessionAction } from "../domains/providers/types.js"; -import { findProviderCapability } from "@threadlens/shared-contracts"; - -export type * from "../domains/providers/types.js"; -export type { ProviderAdapter } from "../domains/providers/adapters.js"; - -export { - getProviderAdapter, - listProviderAdapters, -} from "../domains/providers/adapters.js"; - -export { - listProviderIds, - parseProviderId, - providerName, - providerRootSpecs, - providerScanRootSpecs, - codexTranscriptSearchRoots, - isAllowedProviderFilePath, - isPathInsideRoot, - resolveSafePathWithinRoots, - resolveAllowedProviderFilePath, -} from "../domains/providers/path-safety.js"; -export { - inferSessionId, - isWorkspaceChatSessionPath, - isCopilotGlobalSessionLikeFile, - probeSessionFile, -} from "../domains/providers/probe.js"; -export { - invalidateCodexThreadTitleMapCache, - extractCodexThreadIdFromSessionName, - normalizeDetectedTitle, - detectSessionTitleFromHead, - detectClaudeRenamedTitle, - getCodexThreadTitleMap, - fallbackDisplayTitle as buildDisplayTitleFallback, -} from "../domains/providers/title-detection.js"; -export { - providerStatus, - capabilityLevel, - invalidateProviderMatrixCache, - getProviderMatrixTs, -} from "../domains/providers/matrix.js"; -export { - normalizeSearchText, - normalizeSearchQuery, - buildSearchTokens, - matchesConversationSearch, - buildSearchSnippet, - fallbackDisplayTitle, -} from "../domains/providers/search-helpers.js"; -export { buildProviderActionToken, buildSessionTranscript }; - -function supportsProviderAction( - provider: ProviderId, - action: ProviderSessionAction, -): boolean { - const capability = findProviderCapability(provider); - if (!capability) { - return false; - } - if (action === "backup_local") { - return capability.read_sessions === true; - } - if (action === "delete_local") { - return ( - capability.safe_cleanup === true && - capability.hard_delete === true - ); - } - return capability.safe_cleanup === true; -} - -export async function runProviderSessionAction( - provider: ProviderId, - action: import("../domains/providers/types.js").ProviderSessionAction, - filePaths: string[], - dryRun: boolean, - confirmToken: string, - options?: import("../domains/providers/types.js").ProviderSessionActionOptions, -) { - return runProviderSessionActionInternal( - { - resolveAllowedProviderFilePath, - supportsProviderAction, - invalidateProviderCaches: (targetProvider) => { - invalidateProviderSearchCaches(targetProvider); - invalidateProviderMatrixCache(); - }, - }, - provider, - action, - filePaths, - dryRun, - confirmToken, - options, - ); -} diff --git a/apps/api-ts/src/server.test.ts b/apps/api-ts/src/server.test.ts index c72e857..673580d 100644 --- a/apps/api-ts/src/server.test.ts +++ b/apps/api-ts/src/server.test.ts @@ -4,7 +4,7 @@ import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { FastifyInstance } from "fastify"; import { RECOVERY_EXPORT_ROOT, UPDATE_CHECK_CACHE_FILE } from "./lib/constants"; import { resetUpdateCheckCacheForTests } from "./lib/update-check"; -import { issueRecoveryBackupDownloadTokenForTests } from "./app/routes/platform"; +import { issueRecoveryBackupDownloadTokenForTests } from "./app/routes/platform/index"; import { createServer } from "./server"; vi.mock("./domains/threads/state.js", async (importOriginal) => { diff --git a/package.json b/package.json index 1a000ce..a9c8aea 100644 --- a/package.json +++ b/package.json @@ -53,9 +53,11 @@ "package:desktop:win": "pnpm build:desktop && pnpm --filter @threadlens/desktop-electron dist:win", "package:desktop:linux": "pnpm build:desktop && pnpm --filter @threadlens/desktop-electron dist:linux", "test": "pnpm -r test", - "lint": "pnpm -r lint" + "lint": "pnpm -r lint", + "lint:deps": "depcruise --config .dependency-cruiser.cjs apps/api-ts/src" }, "devDependencies": { + "dependency-cruiser": "^17.4.0", "typescript": "^6.0.3" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4433712..e2a2675 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,6 +13,9 @@ importers: .: devDependencies: + dependency-cruiser: + specifier: ^17.4.0 + version: 17.4.0 typescript: specifier: ^6.0.3 version: 6.0.3 @@ -1034,6 +1037,22 @@ packages: abstract-logging@2.0.1: resolution: {integrity: sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==} + acorn-jsx-walk@2.0.0: + resolution: {integrity: sha512-uuo6iJj4D4ygkdzd6jPtcxs8vZgDX9YFIkqczGImoypX2fQ4dVImmu3UzA4ynixCIMTrEOWW+95M2HuBaCEOVA==} + + acorn-jsx@5.3.2: + resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} + peerDependencies: + acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 + + acorn-loose@8.5.2: + resolution: {integrity: sha512-PPvV6g8UGMGgjrMu+n/f9E/tCSkNQ2Y97eFvuVdJfG11+xdIeDcLyNdC8SHcrHbRqkfwLASdplyR6B6sKM1U4A==} + engines: {node: '>=0.4.0'} + + acorn-walk@8.3.5: + resolution: {integrity: sha512-HEHNfbars9v4pgpW6SO1KSPkfoS0xVOM/9UzkJltjlsHZmJasxg8aXkuZa7SMf8vKGIBhpUsPluQSqhJFCqebw==} + engines: {node: '>=0.4.0'} + acorn@8.16.0: resolution: {integrity: sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==} engines: {node: '>=0.4.0'} @@ -1322,6 +1341,10 @@ packages: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} + commander@14.0.3: + resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==} + engines: {node: '>=20'} + commander@5.1.0: resolution: {integrity: sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==} engines: {node: '>= 6'} @@ -1415,6 +1438,11 @@ packages: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} + dependency-cruiser@17.4.0: + resolution: {integrity: sha512-+WdFoOb+fT1XNC0iPqOyLpfhLd8xVh7eLXJxPAtiXCS+YmXzGrjqVTte7+L8SZIsnJj0aFhb8LxECIBJY5TTIA==} + engines: {node: ^20.12||^22||>=24} + hasBin: true + dequal@2.0.3: resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} engines: {node: '>=6'} @@ -1507,6 +1535,10 @@ packages: end-of-stream@1.4.5: resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} + enhanced-resolve@5.21.0: + resolution: {integrity: sha512-otxSQPw4lkOZWkHpB3zaEQs6gWYEsmX4xQF68ElXC/TWvGxGMSGOvoNbaLXm6/cS/fSfHtsEdw90y20PCd+sCA==} + engines: {node: '>=10.13.0'} + env-paths@2.2.1: resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==} engines: {node: '>=6'} @@ -1726,6 +1758,10 @@ packages: resolution: {integrity: sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==} engines: {node: '>=10.0'} + global-directory@4.0.1: + resolution: {integrity: sha512-wHTUcDUoZ1H5/0iVqEudYW4/kAlN5cZ3j/bXn0Dpbizl9iaUVeWSHqiOjsgk6OW2bkLclbBjzewBz6weQ1zA2Q==} + engines: {node: '>=18'} + globalthis@1.0.4: resolution: {integrity: sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==} engines: {node: '>= 0.4'} @@ -1794,6 +1830,10 @@ packages: ieee754@1.2.1: resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + ignore@7.0.5: + resolution: {integrity: sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==} + engines: {node: '>= 4'} + imurmurhash@0.1.4: resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} engines: {node: '>=0.8.19'} @@ -1813,6 +1853,10 @@ packages: inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + ini@4.1.1: + resolution: {integrity: sha512-QQnnxNyfvmHFIsj7gkPcYymR8Jdw/o7mp5ZFihxn6h8Ci6fh3Dx4E1gPjpQEpIuPo9XVNY/ZUwh4BPMjGyL01g==} + engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + ink@7.0.1: resolution: {integrity: sha512-o6LAC268PLawlGVYrXTyaTfke4VtJftEheuwbgkQf7yvSXyWp1nRwBbAyKEkWXFZZsW/la5wrMuNbuBvZK2C1w==} engines: {node: '>=22'} @@ -1826,6 +1870,10 @@ packages: react-devtools-core: optional: true + interpret@3.1.1: + resolution: {integrity: sha512-6xwYfHbajpoF0xLW+iwLkhwgvLoZDfjYfoFNu8ftMoXINzwuymNLd9u/KmwtdT2GbR+/Cz66otEGEVVUHX9QLQ==} + engines: {node: '>=10.13.0'} + ip-address@10.1.0: resolution: {integrity: sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==} engines: {node: '>= 12'} @@ -1861,10 +1909,18 @@ packages: engines: {node: '>=14.16'} hasBin: true + is-installed-globally@1.0.0: + resolution: {integrity: sha512-K55T22lfpQ63N4KEN57jZUAaAYqYHEe8veb/TycJRk9DdSCLLcovXz/mL6mOnhQaZsQGwPhuFopdQIlqGSEjiQ==} + engines: {node: '>=18'} + is-interactive@1.0.0: resolution: {integrity: sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==} engines: {node: '>=8'} + is-path-inside@4.0.0: + resolution: {integrity: sha512-lJJV/5dYS+RcL8uQdBDW9c9uWFLLBNRyFhnAKXw5tVqLlKZ4RMGZKv+YQ/IA3OhD+RpbJa1LLFM1FQPGyIXvOA==} + engines: {node: '>=12'} + is-unicode-supported@0.1.0: resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} engines: {node: '>=10'} @@ -1956,6 +2012,10 @@ packages: keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + kleur@3.0.3: + resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==} + engines: {node: '>=6'} + lazy-val@1.0.5: resolution: {integrity: sha512-0/BnGCCfyUMkBpeDgWihanIAF9JmZhHBgUhEqzvf+adhNGLoP6TaiI5oF8oyb3I45P+PcnrqihSf01M0l0G5+Q==} @@ -2365,6 +2425,10 @@ packages: resolution: {integrity: sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==} engines: {node: '>=10'} + prompts@2.4.2: + resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==} + engines: {node: '>= 6'} + proper-lockfile@4.1.2: resolution: {integrity: sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==} @@ -2425,10 +2489,18 @@ packages: resolution: {integrity: sha512-YTUo+Flmw4ZXiWfQKGcwwc11KnoRAYgzAE2E7mXKCjSviTKShtxBsN6YUUBB2gtaBzKzeKunxhUwNHQuRryhWA==} engines: {node: '>= 4'} + rechoir@0.8.0: + resolution: {integrity: sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==} + engines: {node: '>= 10.13.0'} + redent@3.0.0: resolution: {integrity: sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==} engines: {node: '>=8'} + regexp-tree@0.1.27: + resolution: {integrity: sha512-iETxpjK6YoRWJG5o6hXLwvjYAoW+FEZn9os0PD/b6AP6xQwsa/Y7lCVgIixBbUPMfhu+i2LtdeAqVTgGlQarfA==} + hasBin: true + require-directory@2.1.1: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} @@ -2507,6 +2579,9 @@ packages: safe-regex2@5.0.0: resolution: {integrity: sha512-YwJwe5a51WlK7KbOJREPdjNrpViQBI3p4T50lfwPuDhZnE3XGVTlGvi+aolc5+RvxDD6bnUmjVsU9n1eboLUYw==} + safe-regex@2.1.1: + resolution: {integrity: sha512-rx+x8AMzKb5Q5lQ95Zoi6ZbJqwCLkqi3XuJXp5P3rT8OEc6sZCJG5AE5dU3lsgRr/F4Bs31jSlVN+j5KrsGu9A==} + safe-stable-stringify@2.5.0: resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==} engines: {node: '>=10'} @@ -2572,6 +2647,9 @@ packages: resolution: {integrity: sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==} engines: {node: '>=10'} + sisteransi@1.0.5: + resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} + slice-ansi@3.0.0: resolution: {integrity: sha512-pSyv7bSTC7ig9Dcgbw9AuRNUb5k5V6oDudjZoMBSr13qpLBG7tB+zgCkARjq7xIUgdz5P1Qe8u+rSGdouOOIyQ==} engines: {node: '>=8'} @@ -2691,6 +2769,10 @@ packages: resolution: {integrity: sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==} engines: {node: '>=20'} + tapable@2.3.3: + resolution: {integrity: sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==} + engines: {node: '>=6'} + tar@7.5.11: resolution: {integrity: sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==} engines: {node: '>=18'} @@ -2761,6 +2843,10 @@ packages: resolution: {integrity: sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ==} engines: {node: '>=6.10'} + tsconfig-paths-webpack-plugin@4.2.0: + resolution: {integrity: sha512-zbem3rfRS8BgeNK50Zz5SIQgXzLafiHjOwUAvk/38/o1jHn/V5QAgVUcz884or7WYcPaH3N2CIfUc2u0ul7UcA==} + engines: {node: '>=10.13.0'} + tsconfig-paths@4.2.0: resolution: {integrity: sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg==} engines: {node: '>=6'} @@ -2920,6 +3006,11 @@ packages: jsdom: optional: true + watskeburt@5.0.3: + resolution: {integrity: sha512-g9CXukMjazlJJVQ3OHzXsnG25KFYgSgKMIyoJrD8ggr0DbS9UNF7OzIqWmmKKBMedkxj3T01uqEaGnn+y7QhMA==} + engines: {node: ^20.12||^22.13||>=24.0} + hasBin: true + wcwidth@1.0.1: resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} @@ -3875,6 +3966,20 @@ snapshots: abstract-logging@2.0.1: {} + acorn-jsx-walk@2.0.0: {} + + acorn-jsx@5.3.2(acorn@8.16.0): + dependencies: + acorn: 8.16.0 + + acorn-loose@8.5.2: + dependencies: + acorn: 8.16.0 + + acorn-walk@8.3.5: + dependencies: + acorn: 8.16.0 + acorn@8.16.0: {} agent-base@7.1.4: {} @@ -4199,6 +4304,8 @@ snapshots: dependencies: delayed-stream: 1.0.0 + commander@14.0.3: {} + commander@5.1.0: {} commander@9.5.0: @@ -4276,6 +4383,27 @@ snapshots: delayed-stream@1.0.0: {} + dependency-cruiser@17.4.0: + dependencies: + acorn: 8.16.0 + acorn-jsx: 5.3.2(acorn@8.16.0) + acorn-jsx-walk: 2.0.0 + acorn-loose: 8.5.2 + acorn-walk: 8.3.5 + commander: 14.0.3 + enhanced-resolve: 5.21.0 + ignore: 7.0.5 + interpret: 3.1.1 + is-installed-globally: 1.0.0 + json5: 2.2.3 + picomatch: 4.0.4 + prompts: 2.4.2 + rechoir: 0.8.0 + safe-regex: 2.1.1 + semver: 7.7.4 + tsconfig-paths-webpack-plugin: 4.2.0 + watskeburt: 5.0.3 + dequal@2.0.3: {} detect-libc@2.1.2: {} @@ -4414,6 +4542,11 @@ snapshots: dependencies: once: 1.4.0 + enhanced-resolve@5.21.0: + dependencies: + graceful-fs: 4.2.11 + tapable: 2.3.3 + env-paths@2.2.1: {} environment@1.1.0: {} @@ -4692,6 +4825,10 @@ snapshots: serialize-error: 7.0.1 optional: true + global-directory@4.0.1: + dependencies: + ini: 4.1.1 + globalthis@1.0.4: dependencies: define-properties: 1.2.1 @@ -4772,6 +4909,8 @@ snapshots: ieee754@1.2.1: {} + ignore@7.0.5: {} + imurmurhash@0.1.4: {} indent-string@4.0.0: {} @@ -4785,6 +4924,8 @@ snapshots: inherits@2.0.4: {} + ini@4.1.1: {} + ink@7.0.1(@types/react@19.2.14)(react@19.2.5): dependencies: '@alcalzone/ansi-tokenize': 0.3.0 @@ -4819,6 +4960,8 @@ snapshots: - bufferutil - utf-8-validate + interpret@3.1.1: {} + ip-address@10.1.0: {} ipaddr.js@2.3.0: {} @@ -4841,8 +4984,15 @@ snapshots: dependencies: is-docker: 3.0.0 + is-installed-globally@1.0.0: + dependencies: + global-directory: 4.0.1 + is-path-inside: 4.0.0 + is-interactive@1.0.0: {} + is-path-inside@4.0.0: {} + is-unicode-supported@0.1.0: {} is-wsl@3.1.1: @@ -4923,6 +5073,8 @@ snapshots: dependencies: json-buffer: 3.0.1 + kleur@3.0.3: {} + lazy-val@1.0.5: {} light-my-request@6.6.0: @@ -5302,6 +5454,11 @@ snapshots: err-code: 2.0.3 retry: 0.12.0 + prompts@2.4.2: + dependencies: + kleur: 3.0.3 + sisteransi: 1.0.5 + proper-lockfile@4.1.2: dependencies: graceful-fs: 4.2.11 @@ -5374,11 +5531,17 @@ snapshots: tiny-invariant: 1.3.3 tslib: 2.8.1 + rechoir@0.8.0: + dependencies: + resolve: 1.22.12 + redent@3.0.0: dependencies: indent-string: 4.0.0 strip-indent: 3.0.0 + regexp-tree@0.1.27: {} + require-directory@2.1.1: {} require-from-string@2.0.2: {} @@ -5495,6 +5658,10 @@ snapshots: dependencies: ret: 0.5.0 + safe-regex@2.1.1: + dependencies: + regexp-tree: 0.1.27 + safe-stable-stringify@2.5.0: {} safer-buffer@2.1.2: {} @@ -5541,6 +5708,8 @@ snapshots: dependencies: semver: 7.7.4 + sisteransi@1.0.5: {} + slice-ansi@3.0.0: dependencies: ansi-styles: 4.3.0 @@ -5673,6 +5842,8 @@ snapshots: tagged-tag@1.0.0: {} + tapable@2.3.3: {} + tar@7.5.11: dependencies: '@isaacs/fs-minipass': 4.0.1 @@ -5737,6 +5908,13 @@ snapshots: ts-dedent@2.2.0: {} + tsconfig-paths-webpack-plugin@4.2.0: + dependencies: + chalk: 4.1.2 + enhanced-resolve: 5.21.0 + tapable: 2.3.3 + tsconfig-paths: 4.2.0 + tsconfig-paths@4.2.0: dependencies: json5: 2.2.3 @@ -5851,6 +6029,8 @@ snapshots: transitivePeerDependencies: - msw + watskeburt@5.0.3: {} + wcwidth@1.0.1: dependencies: defaults: 1.0.4