From 8e7db89fe3940ed8af7c1c830463deaaf47a5272 Mon Sep 17 00:00:00 2001 From: Andrew Mikofalvy <5668128+amikofalvy@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:28:45 -0700 Subject: [PATCH] perf(server): incremental workspace search corpus maintenance (#2815) GitOrigin-RevId: 9426c0bb6e59a99b138a6f05e49fef091aa34c00 --- .changeset/incremental-search-corpus.md | 5 + packages/core/src/index.ts | 3 + .../workspace-search.incremental.test.ts | 246 ++++++++++++++++ packages/core/src/search/workspace-search.ts | 150 +++++++++- packages/server/src/api-extension.ts | 62 +++- .../server/src/api-search-incremental.test.ts | 267 ++++++++++++++++++ 6 files changed, 728 insertions(+), 5 deletions(-) create mode 100644 .changeset/incremental-search-corpus.md create mode 100644 packages/core/src/search/workspace-search.incremental.test.ts create mode 100644 packages/server/src/api-search-incremental.test.ts diff --git a/.changeset/incremental-search-corpus.md b/.changeset/incremental-search-corpus.md new file mode 100644 index 000000000..9af5d4a11 --- /dev/null +++ b/.changeset/incremental-search-corpus.md @@ -0,0 +1,5 @@ +--- +"@inkeep/open-knowledge": patch +--- + +Workspace search now maintains its full-text index incrementally. Previously any file change invalidated the whole search corpus and the next search rebuilt the entire index from scratch on the server's event loop — on a large workspace, a write burst with interleaved searches meant repeated full re-indexing. The server now diffs the document set against the live index and applies per-document insert/update/remove, so one write re-indexes one document. A from-scratch rebuild remains as the cold-start path and as an automatic fallback whenever the incremental patch cannot be proven consistent. diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 5d72bbe86..3b9fee46c 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1260,12 +1260,15 @@ export { MAX_WORKSPACE_SEARCH_LIMIT, searchWorkspaceCorpus, searchWorkspaceDocuments, + updateWorkspaceSearchCorpus, type WorkspaceSearchCorpus, + type WorkspaceSearchCorpusUpdate, type WorkspaceSearchDocument, type WorkspaceSearchIntent, type WorkspaceSearchKind, type WorkspaceSearchOptions, type WorkspaceSearchRanking, + type WorkspaceSearchRebuildReason, type WorkspaceSearchResult, type WorkspaceSearchScope, type WorkspaceSemanticInput, diff --git a/packages/core/src/search/workspace-search.incremental.test.ts b/packages/core/src/search/workspace-search.incremental.test.ts new file mode 100644 index 000000000..f1b25056a --- /dev/null +++ b/packages/core/src/search/workspace-search.incremental.test.ts @@ -0,0 +1,246 @@ +import { describe, expect, test } from 'vitest'; +import { + createWorkspaceSearchCorpus, + createWorkspaceSearchDocument, + searchWorkspaceCorpus, + updateWorkspaceSearchCorpus, + type WorkspaceSearchDocument, + type WorkspaceSearchOptions, +} from './workspace-search.ts'; + +function page( + path: string, + content: string, + modifiedTs = 10, + title?: string, +): WorkspaceSearchDocument { + return createWorkspaceSearchDocument({ kind: 'page', path, title, content, modifiedTs }); +} + +function folder(path: string, modifiedTs = 0): WorkspaceSearchDocument { + return createWorkspaceSearchDocument({ kind: 'folder', path, modifiedTs }); +} + +function file(path: string, modifiedTs = 5): WorkspaceSearchDocument { + return createWorkspaceSearchDocument({ kind: 'file', path, modifiedTs }); +} + +const baseDocuments: WorkspaceSearchDocument[] = [ + page('docs/api', 'HTTP endpoint contracts', 10, 'API Reference'), + page('architecture/overview', 'Observer bridge and CRDT topology', 30, 'Architecture Overview'), + page('notes/graphing', 'Visual explorer notes', 20, 'Graphing Notes'), + folder('docs'), + folder('architecture'), + folder('notes'), + file('assets/diagram.png'), +]; + +/** + * The correctness bar for the incremental path: for any query/options pair, an + * incrementally-maintained corpus must be indistinguishable from a corpus + * built from scratch over the same documents — paths, scores, and signals. + */ +function expectSearchEquivalence( + incremental: ReturnType, + documents: readonly WorkspaceSearchDocument[], + queries: readonly string[], + options: WorkspaceSearchOptions = { intent: 'full_text' }, +) { + const fresh = createWorkspaceSearchCorpus(documents); + for (const query of queries) { + const incrementalResults = searchWorkspaceCorpus(incremental, query, options); + const freshResults = searchWorkspaceCorpus(fresh, query, options); + expect( + incrementalResults.map((r) => ({ + path: r.document.path, + kind: r.document.kind, + score: r.score, + signals: r.signals, + })), + ).toEqual( + freshResults.map((r) => ({ + path: r.document.path, + kind: r.document.kind, + score: r.score, + signals: r.signals, + })), + ); + } +} + +describe('updateWorkspaceSearchCorpus', () => { + test('inserting a document makes it searchable without a rebuild', () => { + const corpus = createWorkspaceSearchCorpus(baseDocuments); + const next = [...baseDocuments, page('guides/tutorial', 'Zebra migration walkthrough', 40)]; + + const update = updateWorkspaceSearchCorpus(corpus, next); + + expect(update.rebuilt).toBe(false); + expect(update.inserted).toBe(1); + expect(update.updated).toBe(0); + expect(update.removed).toBe(0); + const results = searchWorkspaceCorpus(update.corpus, 'zebra', { intent: 'full_text' }); + expect(results.map((r) => r.document.path)).toContain('guides/tutorial'); + expectSearchEquivalence(update.corpus, next, ['zebra', 'tutorial', 'topology', 'arch']); + }); + + test('updating a document swaps its indexed content', () => { + const corpus = createWorkspaceSearchCorpus(baseDocuments); + const next = baseDocuments.map((doc) => + doc.path === 'notes/graphing' + ? page('notes/graphing', 'Quokka habitat research', 21, 'Graphing Notes') + : doc, + ); + + const update = updateWorkspaceSearchCorpus(corpus, next); + + expect(update.rebuilt).toBe(false); + expect(update.inserted).toBe(0); + expect(update.updated).toBe(1); + expect(update.removed).toBe(0); + const newHits = searchWorkspaceCorpus(update.corpus, 'quokka', { intent: 'full_text' }); + expect(newHits.map((r) => r.document.path)).toContain('notes/graphing'); + const oldHits = searchWorkspaceCorpus(update.corpus, 'explorer', { intent: 'full_text' }); + expect(oldHits.map((r) => r.document.path)).not.toContain('notes/graphing'); + expectSearchEquivalence(update.corpus, next, ['quokka', 'explorer', 'notes']); + }); + + test('removing a document drops it from every search tier', () => { + const corpus = createWorkspaceSearchCorpus(baseDocuments); + const next = baseDocuments.filter((doc) => doc.path !== 'docs/api'); + + const update = updateWorkspaceSearchCorpus(corpus, next); + + expect(update.rebuilt).toBe(false); + expect(update.inserted).toBe(0); + expect(update.updated).toBe(0); + expect(update.removed).toBe(1); + for (const query of ['endpoint', 'api']) { + const hits = searchWorkspaceCorpus(update.corpus, query, { intent: 'full_text' }); + expect(hits.map((r) => r.document.path)).not.toContain('docs/api'); + } + expectSearchEquivalence(update.corpus, next, ['endpoint', 'api', 'contracts']); + }); + + test('unchanged document set patches nothing and stays searchable', () => { + const corpus = createWorkspaceSearchCorpus(baseDocuments); + // New array, mixed identity: some reused objects, some re-created equal ones + // (the server reuses page objects via its cache but re-creates metadata-only + // folder/file documents every build). + const next = baseDocuments.map((doc) => + doc.kind === 'page' ? doc : createWorkspaceSearchDocument({ ...doc }), + ); + + const update = updateWorkspaceSearchCorpus(corpus, next); + + expect(update.rebuilt).toBe(false); + expect(update.inserted).toBe(0); + expect(update.updated).toBe(0); + expect(update.removed).toBe(0); + expectSearchEquivalence(update.corpus, next, ['topology', 'arch', 'diagram']); + }); + + test('a removed id can be re-added across successive updates', () => { + const corpus = createWorkspaceSearchCorpus(baseDocuments); + const without = baseDocuments.filter((doc) => doc.path !== 'docs/api'); + const first = updateWorkspaceSearchCorpus(corpus, without); + expect(first.rebuilt).toBe(false); + + const readded = [...without, page('docs/api', 'Fresh endpoint catalog', 50, 'API Reference')]; + const second = updateWorkspaceSearchCorpus(first.corpus, readded); + + expect(second.rebuilt).toBe(false); + const hits = searchWorkspaceCorpus(second.corpus, 'catalog', { intent: 'full_text' }); + expect(hits.map((r) => r.document.path)).toContain('docs/api'); + expectSearchEquivalence(second.corpus, readded, ['catalog', 'endpoint']); + }); + + test('emptying the corpus rebuilds electively and leaves a searchable empty index', () => { + const corpus = createWorkspaceSearchCorpus(baseDocuments); + const update = updateWorkspaceSearchCorpus(corpus, []); + // Removing everything mutates more entries than an (empty) rebuild inserts, + // so the bulk-change gate takes the cheap from-scratch path. + expect(update.rebuilt).toBe(true); + expect(update.rebuildReason).toBe('bulk-change'); + expect(searchWorkspaceCorpus(update.corpus, 'topology', { intent: 'full_text' })).toEqual([]); + }); + + test('duplicate ids fail loudly, matching the from-scratch build', () => { + const corpus = createWorkspaceSearchCorpus(baseDocuments); + const duplicated = [...baseDocuments, page('docs/api', 'Shadow copy', 60)]; + expect(() => createWorkspaceSearchCorpus(duplicated)).toThrow(); + expect(() => updateWorkspaceSearchCorpus(corpus, duplicated)).toThrow(); + }); + + test('reusing a consumed base falls back to a from-scratch rebuild', () => { + const corpus = createWorkspaceSearchCorpus(baseDocuments); + const first = updateWorkspaceSearchCorpus(corpus, [ + ...baseDocuments, + page('guides/alpha', 'Alpha content', 40), + ]); + expect(first.rebuilt).toBe(false); + + // Diffing against `corpus` again would patch an index that has already + // moved past it; the consumed-base gate must force a rebuild instead. + const next = [...baseDocuments, page('guides/beta', 'Beta content', 41)]; + const second = updateWorkspaceSearchCorpus(corpus, next); + + expect(second.rebuilt).toBe(true); + expect(second.rebuildReason).toBe('stale-base'); + expectSearchEquivalence(second.corpus, next, ['beta', 'alpha', 'topology']); + }); + + test('a diff touching more documents than a rebuild would insert rebuilds electively', () => { + const corpus = createWorkspaceSearchCorpus(baseDocuments); + // Replace everything: every old id removed, every new id inserted. + const next = [page('brand/new', 'Entirely new workspace', 70)]; + + const update = updateWorkspaceSearchCorpus(corpus, next); + + expect(update.rebuilt).toBe(true); + expect(update.rebuildReason).toBe('bulk-change'); + expectSearchEquivalence(update.corpus, next, ['workspace', 'topology']); + }); + + test('mixed burst of inserts, updates, and removes matches a from-scratch build', () => { + // Deterministic pseudo-random burst: enough documents and churn to exercise + // the diff paths together, seeded so a failure replays exactly. + let seed = 0xc0ffee; + const rand = () => { + // LCG (Numerical Recipes constants) — deterministic across runs/platforms. + seed = (seed * 1664525 + 1013904223) >>> 0; + return seed / 0x100000000; + }; + const words = ['alpha', 'bravo', 'crdt', 'delta', 'editor', 'fjord', 'graph', 'hocus']; + const makeContent = () => + Array.from({ length: 5 }, () => words[Math.trunc(rand() * words.length)]).join(' '); + + let documents: WorkspaceSearchDocument[] = Array.from({ length: 40 }, (_, i) => + page(`corpus/doc-${i}`, makeContent(), i), + ); + let corpus = createWorkspaceSearchCorpus(documents); + + for (let step = 0; step < 8; step++) { + const next = [...documents]; + // Remove one, update two, insert one per step — small deltas, like a + // write burst with interleaved searches. + next.splice(Math.trunc(rand() * next.length), 1); + for (let u = 0; u < 2; u++) { + const i = Math.trunc(rand() * next.length); + const target = next[i]; + if (target) next[i] = page(target.path, makeContent(), target.modifiedTs + 100); + } + next.push(page(`corpus/new-${step}`, makeContent(), 1000 + step)); + + const update = updateWorkspaceSearchCorpus(corpus, next); + expect(update.rebuilt).toBe(false); + corpus = update.corpus; + documents = next; + + expectSearchEquivalence(corpus, documents, words, { intent: 'full_text' }); + expectSearchEquivalence(corpus, documents, ['corpus', 'doc-3', 'new-'], { + intent: 'omnibar', + }); + } + }); +}); diff --git a/packages/core/src/search/workspace-search.ts b/packages/core/src/search/workspace-search.ts index 02a6f90d3..f03433a16 100644 --- a/packages/core/src/search/workspace-search.ts +++ b/packages/core/src/search/workspace-search.ts @@ -1,4 +1,4 @@ -import { type AnyOrama, create, insertMultiple, search } from '@orama/orama'; +import { type AnyOrama, count, create, insertMultiple, removeMultiple, search } from '@orama/orama'; import { isHiddenDocName } from '../util/doc-name.ts'; export type WorkspaceSearchKind = 'page' | 'folder' | 'file'; @@ -281,6 +281,154 @@ function getWorkspaceSearchIndex(corpus: WorkspaceSearchCorpus): WorkspaceSearch return index; } +/** + * Corpora already consumed as the base of an incremental update. A diff is only + * valid against the exact document set the shared index currently reflects, and + * that is the corpus most recently RETURNED by `createWorkspaceSearchCorpus` / + * `updateWorkspaceSearchCorpus` — reusing an older base would apply a diff the + * index has already moved past, silently desynchronizing BM25 from + * `corpus.documents`. Membership here makes that misuse deterministic: a + * consumed base forces the from-scratch rebuild path instead. + */ +const consumedUpdateBases = new WeakSet(); + +/** Why an incremental corpus update fell back to a from-scratch index build. */ +export type WorkspaceSearchRebuildReason = + | 'no-index' + | 'stale-base' + | 'duplicate-id' + | 'bulk-change' + | 'mutation-failed'; + +export interface WorkspaceSearchCorpusUpdate { + corpus: WorkspaceSearchCorpus; + /** Documents newly added to the index (includes the insert half of updates). */ + inserted: number; + /** Documents whose content/metadata changed (counted once, not as insert+remove). */ + updated: number; + /** Documents removed from the index (excludes the remove half of updates). */ + removed: number; + /** True when the index was rebuilt from scratch instead of patched in place. */ + rebuilt: boolean; + rebuildReason?: WorkspaceSearchRebuildReason; +} + +/** + * Field-level equality for diffing. Object identity short-circuits the common + * case (callers reuse unchanged document objects across builds); the field + * compare covers metadata-only documents that are re-created each build with + * identical values. `id` is the diff key, so it is compared by the caller. + */ +function sameSearchDocument(a: WorkspaceSearchDocument, b: WorkspaceSearchDocument): boolean { + return ( + a === b || + (a.kind === b.kind && + a.path === b.path && + a.title === b.title && + a.name === b.name && + a.pathSegments === b.pathSegments && + a.modifiedTs === b.modifiedTs && + a.content === b.content) + ); +} + +/** + * Advance a corpus to a new document set by patching the existing Orama index + * in place — per-document remove/insert for only the changed entries — instead + * of re-tokenizing the entire workspace. On a large corpus the from-scratch + * build is the dominant cost of search-cache invalidation, and it runs on the + * event loop; this keeps steady-state maintenance proportional to the change. + * + * Falls back to a from-scratch build (`rebuilt: true`) whenever the + * incremental path cannot be proven safe: `previous` has no live index or was + * already consumed as an update base, the new set carries a duplicate id (the + * rebuild then throws exactly like `createWorkspaceSearchCorpus`), the diff + * touches more documents than a rebuild would insert, or an index mutation + * fails / leaves the index count inconsistent. + * + * The returned corpus shares the (now-patched) index with `previous`. A search + * still holding `previous` keeps a coherent `documents` snapshot for the + * lexical/recency tiers but reads the patched index for BM25 — a bounded, + * one-step freshness skew, accepted over the alternatives (an index copy per + * update, or re-building the old snapshot's index on demand). + */ +export function updateWorkspaceSearchCorpus( + previous: WorkspaceSearchCorpus, + documents: readonly WorkspaceSearchDocument[], +): WorkspaceSearchCorpusUpdate { + const rebuild = (rebuildReason: WorkspaceSearchRebuildReason): WorkspaceSearchCorpusUpdate => ({ + corpus: createWorkspaceSearchCorpus(documents), + inserted: 0, + updated: 0, + removed: 0, + rebuilt: true, + rebuildReason, + }); + + const index = workspaceSearchIndexes.get(previous); + if (!index) return rebuild('no-index'); + if (consumedUpdateBases.has(previous)) return rebuild('stale-base'); + + const nextById = new Map(documents.map((document) => [document.id, document] as const)); + if (nextById.size !== documents.length) return rebuild('duplicate-id'); + + const removedIds: string[] = []; + const insertedDocuments: WorkspaceSearchDocument[] = []; + let updated = 0; + for (const previousDocument of previous.documents) { + const nextDocument = nextById.get(previousDocument.id); + if (!nextDocument) { + removedIds.push(previousDocument.id); + } else if (!sameSearchDocument(previousDocument, nextDocument)) { + // Changed document: remove the stale tokens, insert the new ones. Orama's + // `update` is the same remove+insert internally; batching both halves + // keeps the mutation count observable for the bulk-change gate below. + removedIds.push(previousDocument.id); + insertedDocuments.push(nextDocument); + updated += 1; + } + } + const previousIds = new Set(previous.documents.map((document) => document.id)); + for (const document of documents) { + if (!previousIds.has(document.id)) insertedDocuments.push(document); + } + + // When the diff would mutate more index entries than a rebuild would insert + // (e.g. a branch switch replacing most of the workspace), the from-scratch + // build is the cheaper operation — take it electively. + if (removedIds.length + insertedDocuments.length > documents.length) { + return rebuild('bulk-change'); + } + + consumedUpdateBases.add(previous); + try { + if (removedIds.length > 0) removeMultiple(index, removedIds); + if (insertedDocuments.length > 0) { + insertMultiple(index, insertedDocuments as WorkspaceSearchDocument[]); + } + // A count mismatch means a removal targeted an id the index did not hold + // (or an insert landed twice) — the index no longer mirrors `documents`, + // so discard it rather than serve drifted BM25 results. + if (count(index) !== documents.length) { + throw new Error( + `Search index count ${count(index)} diverged from document count ${documents.length} after incremental update`, + ); + } + } catch { + return rebuild('mutation-failed'); + } + + const corpus: WorkspaceSearchCorpus = { documents }; + workspaceSearchIndexes.set(corpus, index); + return { + corpus, + inserted: insertedDocuments.length - updated, + updated, + removed: removedIds.length - updated, + rebuilt: false, + }; +} + function defaultScopes(intent: WorkspaceSearchIntent): readonly WorkspaceSearchScope[] { // Autocomplete stays page-only: it backs the `[[wikilink]]` target picker, // whose resolution model is markdown-scoped (the all-files autocomplete target diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.ts index ee3fe2ed4..acb33394f 100644 --- a/packages/server/src/api-extension.ts +++ b/packages/server/src/api-extension.ts @@ -241,6 +241,7 @@ import { UploadAssetSuccessSchema, UploadRequestSchema, unwrapFrontmatterFences, + updateWorkspaceSearchCorpus, type WorkspaceSearchCorpus, type WorkspaceSearchDocument, type WorkspaceSearchIntent, @@ -742,6 +743,21 @@ function searchCorpusTruncatedCounter(): ReturnType[ return _searchCorpusTruncatedCounter; } +// Counter for search-corpus builds by mode: `cold` (no prior corpus), +// `incremental` (in-place index patch of only the changed documents), or +// `rebuild` (incremental path fell back to a from-scratch build; the bounded +// `reason` attribute says why). A steady stream of `rebuild` where +// `incremental` is expected is the drift signal worth alerting on. +let _searchCorpusUpdateCounter: ReturnType['createCounter']> | null = + null; +function searchCorpusUpdateCounter(): ReturnType['createCounter']> { + _searchCorpusUpdateCounter ||= getMeter().createCounter('ok.search.corpus_update_total', { + description: + 'Count of workspace search corpus builds, by mode (cold | incremental | rebuild) and, for rebuilds, the bounded fallback reason.', + }); + return _searchCorpusUpdateCounter; +} + /** Bounded handler label for the content-divergence counters. */ type DivergenceHandler = 'agent-write-md' | 'agent-patch' | 'rollback'; @@ -17235,10 +17251,48 @@ export function createApiExtension(options: ApiExtensionOptions): Extension { return workspaceSearchCache.pending; } - const pending = buildWorkspaceSearchDocumentsFromIndex().then(({ documents, truncated }) => ({ - corpus: createWorkspaceSearchCorpus(documents), - truncated, - })); + // Stale-but-live corpus (or the build about to produce one): the base for + // an incremental index patch, so one write re-indexes one document instead + // of re-tokenizing the whole workspace on the event loop. + const priorCorpus = workspaceSearchCache?.corpus; + const priorPending = workspaceSearchCache?.pending; + const pending = (async () => { + // Serialize behind any in-flight build: an incremental diff is only valid + // against the corpus it was computed from, and the in-flight build owns + // the shared index right now. Chaining keeps updates linear (each build + // bases on its predecessor's output) without coalescing away freshness — + // the document snapshot below is read AFTER this fingerprint was seen. + const base = priorPending + ? await priorPending.then( + (result) => result.corpus, + () => undefined, + ) + : priorCorpus; + const { documents, truncated } = await buildWorkspaceSearchDocumentsFromIndex(); + if (!base) { + searchCorpusUpdateCounter().add(1, { mode: 'cold' }); + return { corpus: createWorkspaceSearchCorpus(documents), truncated }; + } + const update = updateWorkspaceSearchCorpus(base, documents); + if (update.rebuilt) { + searchCorpusUpdateCounter().add(1, { mode: 'rebuild', reason: update.rebuildReason }); + // `mutation-failed` means the patched index diverged from the document + // set (or a mutation threw) — recovered by the rebuild, but worth an + // operator-visible signal; the elective reasons are routine. + const logLevel = update.rebuildReason === 'mutation-failed' ? 'warn' : 'debug'; + getLogger('search')[logLevel]( + { reason: update.rebuildReason, documents: documents.length }, + '[search] corpus update fell back to a full index rebuild', + ); + } else { + searchCorpusUpdateCounter().add(1, { mode: 'incremental' }); + getLogger('search').debug( + { inserted: update.inserted, updated: update.updated, removed: update.removed }, + '[search] corpus updated incrementally', + ); + } + return { corpus: update.corpus, truncated }; + })(); workspaceSearchCaches.set(cacheKey, { fingerprint, pending }); try { const result = await pending; diff --git a/packages/server/src/api-search-incremental.test.ts b/packages/server/src/api-search-incremental.test.ts new file mode 100644 index 000000000..c37404db3 --- /dev/null +++ b/packages/server/src/api-search-incremental.test.ts @@ -0,0 +1,267 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { Readable } from 'node:stream'; +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; +import { createApiExtension } from './api-extension.ts'; +import type { FileIndexEntry } from './file-watcher.ts'; + +// Count how the server maintains the search corpus: `create` = from-scratch +// builds requested directly by the server (cold start), `update` = incremental +// maintenance. `updateWorkspaceSearchCorpus`'s internal fallback rebuild calls +// the module-local `createWorkspaceSearchCorpus`, which this mock does NOT +// intercept — so `create` counts exactly the server's own cold builds. +const corpusCalls = vi.hoisted(() => ({ create: 0, update: 0 })); +vi.mock('@inkeep/open-knowledge-core', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createWorkspaceSearchCorpus: ( + ...args: Parameters + ) => { + corpusCalls.create += 1; + return actual.createWorkspaceSearchCorpus(...args); + }, + updateWorkspaceSearchCorpus: ( + ...args: Parameters + ) => { + corpusCalls.update += 1; + return actual.updateWorkspaceSearchCorpus(...args); + }, + }; +}); + +interface CapturedResponse { + status: number; + headers: Record; + body: string; +} + +interface SearchResultEntry { + kind: string; + path: string; + title: string; + score: number; + signals: Record; + snippet?: string; +} + +function makeReq(method: string, url: string): IncomingMessage { + const readable = Readable.from(Buffer.from('')) as unknown as IncomingMessage; + readable.method = method; + readable.url = url; + readable.headers = { host: 'localhost' }; + return readable; +} + +function makeRes(): { res: ServerResponse; captured: CapturedResponse } { + const captured: CapturedResponse = { status: 0, headers: {}, body: '' }; + const res = { + writeHead(status: number, headers?: Record) { + captured.status = status; + if (headers) Object.assign(captured.headers, headers); + }, + end(body?: string) { + captured.body = body ?? ''; + }, + } as unknown as ServerResponse; + return { res, captured }; +} + +/** + * A persistent extension over a mutable in-memory file index — the shape the + * production server has (one long-lived extension whose index the file watcher + * mutates), unlike the per-call throwaway extension in `api-search.test.ts`. + * `modified` stamps are assigned from a deterministic sequence so every content + * change also changes the entry fingerprint, mirroring the watcher's behavior. + */ +function createHarness(contentDir: string, options: { withGeneration: boolean }) { + const index = new Map(); + let generation = 0; + let stampSeq = 0; + const nextStamp = () => new Date(1700000000000 + ++stampSeq * 1000).toISOString(); + + const ext = createApiExtension({ + hocuspocus: {} as unknown as Parameters[0]['hocuspocus'], + sessionManager: {} as unknown as Parameters[0]['sessionManager'], + contentDir, + serverInstanceId: 'test-server', + getFileIndex: () => index, + getAllFilesIndex: () => index, + ...(options.withGeneration ? { getFileIndexGeneration: () => generation } : {}), + }); + + const setDoc = (docName: string, content: string, modified = nextStamp()) => { + const abs = join(contentDir, `${docName}.md`); + mkdirSync(dirname(abs), { recursive: true }); + writeFileSync(abs, content, 'utf-8'); + index.set(docName, { + size: Buffer.byteLength(content), + modified, + canonicalPath: abs, + inode: ++stampSeq, + aliases: [], + kind: 'markdown', + }); + generation += 1; + return modified; + }; + + const setRawEntry = (docName: string, entry: Partial) => { + index.set(docName, { + size: 0, + modified: nextStamp(), + canonicalPath: join(contentDir, docName), + inode: ++stampSeq, + aliases: [], + kind: 'markdown', + ...entry, + }); + generation += 1; + }; + + const removeDoc = (docName: string) => { + index.delete(docName); + generation += 1; + }; + + const search = async (query: string, intent = 'full_text') => { + const req = makeReq('GET', `/api/search?query=${encodeURIComponent(query)}&intent=${intent}`); + const { res, captured } = makeRes(); + await ( + ext as { + onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise; + } + ).onRequest({ request: req, response: res }); + expect(captured.status).toBe(200); + return JSON.parse(captured.body) as { results: SearchResultEntry[]; ready: boolean }; + }; + + return { setDoc, setRawEntry, removeDoc, search }; +} + +describe('incremental workspace search corpus maintenance', () => { + let contentDir: string; + + beforeEach(() => { + contentDir = mkdtempSync(join(tmpdir(), 'ok-search-incr-')); + corpusCalls.create = 0; + corpusCalls.update = 0; + }); + + afterEach(() => { + rmSync(contentDir, { recursive: true, force: true }); + }); + + test('one cold build, then per-write incremental updates (generation fingerprint)', async () => { + const harness = createHarness(contentDir, { withGeneration: true }); + harness.setDoc('docs/api', '# API Reference\n\nHTTP endpoint contracts\n'); + harness.setDoc('notes/graphing', '# Graphing Notes\n\nVisual explorer notes\n'); + + const cold = await harness.search('endpoint'); + expect(cold.results.map((r) => r.path)).toContain('docs/api'); + expect(corpusCalls).toEqual({ create: 1, update: 0 }); + + // Insert: a new doc is searchable without another from-scratch build. + harness.setDoc('guides/tutorial', '# Tutorial\n\nZebra migration walkthrough\n'); + const afterInsert = await harness.search('zebra'); + expect(afterInsert.results.map((r) => r.path)).toContain('guides/tutorial'); + expect(corpusCalls).toEqual({ create: 1, update: 1 }); + + // Update: changed body content is re-indexed; the old tokens are gone. + harness.setDoc('notes/graphing', '# Graphing Notes\n\nQuokka habitat research\n'); + const afterUpdate = await harness.search('quokka'); + expect(afterUpdate.results.map((r) => r.path)).toContain('notes/graphing'); + // The second search after the same write hits the fingerprint cache — no + // further corpus work of either kind. + const oldTokens = await harness.search('explorer'); + expect(oldTokens.results.map((r) => r.path)).not.toContain('notes/graphing'); + expect(corpusCalls).toEqual({ create: 1, update: 2 }); + + // Remove: the doc disappears from name and body tiers. + harness.removeDoc('docs/api'); + const afterRemove = await harness.search('endpoint'); + expect(afterRemove.results.map((r) => r.path)).not.toContain('docs/api'); + expect(corpusCalls).toEqual({ create: 1, update: 3 }); + }); + + test('incremental maintenance also runs on the fallback (no-generation) fingerprint', async () => { + const harness = createHarness(contentDir, { withGeneration: false }); + harness.setDoc('docs/api', '# API\n\nendpoint contracts\n'); + + await harness.search('endpoint'); + expect(corpusCalls).toEqual({ create: 1, update: 0 }); + + harness.setDoc('guides/new-page', '# New Page\n\nxylophone content\n'); + const results = await harness.search('xylophone'); + expect(results.results.map((r) => r.path)).toContain('guides/new-page'); + expect(corpusCalls).toEqual({ create: 1, update: 1 }); + }); + + test('system and config docs added after the cold build never enter the corpus', async () => { + const harness = createHarness(contentDir, { withGeneration: true }); + harness.setDoc('docs/real-page', '# Real\n\nsystem architecture content\n'); + await harness.search('system'); + + // The build-time predicate skips these before any disk read, so the raw + // entries need no backing file. + harness.setRawEntry('__system__', {}); + harness.setRawEntry('__config__/project', {}); + // full_text searches name AND body, so this asserts the synthetic names are + // absent from every tier while the real page still matches on content. + const results = await harness.search('system', 'full_text'); + const paths = results.results.map((r) => r.path); + expect(paths).not.toContain('__system__'); + expect(paths).not.toContain('__config__/project'); + expect(paths).toContain('docs/real-page'); + }); + + test('after a mixed write burst, results match a from-scratch build of the same workspace', async () => { + const harness = createHarness(contentDir, { withGeneration: true }); + const stamps = new Map(); + stamps.set('docs/api', harness.setDoc('docs/api', '# API\n\nendpoint contracts\n')); + stamps.set('notes/one', harness.setDoc('notes/one', '# One\n\nalpha bravo crdt\n')); + stamps.set('notes/two', harness.setDoc('notes/two', '# Two\n\ndelta editor fjord\n')); + await harness.search('alpha'); + + // Burst: update, insert, remove, insert — each interleaved with a search so + // every step goes through the incremental path rather than one big diff. + stamps.set('notes/one', harness.setDoc('notes/one', '# One\n\nalpha graph hocus\n')); + await harness.search('graph'); + stamps.set('guides/four', harness.setDoc('guides/four', '# Four\n\nbravo delta alpha\n')); + await harness.search('bravo'); + harness.removeDoc('notes/two'); + stamps.delete('notes/two'); + await harness.search('delta'); + stamps.set('guides/five', harness.setDoc('guides/five', '# Five\n\nhocus fjord\n')); + expect(corpusCalls.create).toBe(1); + + // Reference: an identical workspace (same docNames, contents, and modified + // stamps) in a fresh contentDir, indexed from scratch by its own harness. + const referenceDir = mkdtempSync(join(tmpdir(), 'ok-search-ref-')); + try { + const reference = createHarness(referenceDir, { withGeneration: true }); + const finalDocs: Array<[string, string]> = [ + ['docs/api', '# API\n\nendpoint contracts\n'], + ['notes/one', '# One\n\nalpha graph hocus\n'], + ['guides/four', '# Four\n\nbravo delta alpha\n'], + ['guides/five', '# Five\n\nhocus fjord\n'], + ]; + for (const [docName, content] of finalDocs) { + const stamp = stamps.get(docName); + if (!stamp) throw new Error(`missing stamp for ${docName}`); + reference.setDoc(docName, content, stamp); + } + for (const query of ['alpha', 'bravo', 'delta', 'fjord', 'graph', 'hocus', 'guides']) { + for (const intent of ['full_text', 'omnibar'] as const) { + const incremental = await harness.search(query, intent); + const fresh = await reference.search(query, intent); + expect(incremental.results).toEqual(fresh.results); + } + } + } finally { + rmSync(referenceDir, { recursive: true, force: true }); + } + }); +});