diff --git a/.changeset/unify-doc-existence-oracle.md b/.changeset/unify-doc-existence-oracle.md new file mode 100644 index 000000000..c2b24f51f --- /dev/null +++ b/.changeset/unify-doc-existence-oracle.md @@ -0,0 +1,6 @@ +--- +"@inkeep/open-knowledge-server": patch +"@inkeep/open-knowledge": patch +--- + +Stop the file-watcher's indexing lag from making freshly-written docs look missing across link, title, and dead-link results. The file index the watcher maintains can lag an in-session write — or permanently drop the create event for a file written into a just-created subfolder — which made a brand-new doc render as a raw red-link name (instead of its title) and could flag a valid link to it as dead, until a server restart. Two complementary fixes close the gap: doc-existence now also trusts the link graph (updated in-process on every write), and the agent write/edit/frontmatter-patch handlers register the doc they just wrote into the file index immediately, mirroring page creation. Both paths honor content-scope exclusions, so a gitignore/okignore'd doc never leaks its title through these endpoints. diff --git a/packages/server/src/api-agent-write-file-index.test.ts b/packages/server/src/api-agent-write-file-index.test.ts new file mode 100644 index 000000000..60077348d --- /dev/null +++ b/packages/server/src/api-agent-write-file-index.test.ts @@ -0,0 +1,257 @@ +import { describe, expect, test } from 'bun:test'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Readable } from 'node:stream'; +import { Hocuspocus } from '@hocuspocus/server'; +import { + AGENT_WRITE_ORIGIN, + AgentSessionManager, + applyAgentMarkdownWrite, +} from './agent-sessions.ts'; +import { createApiExtension } from './api-extension.ts'; +import { createContentFilter } from './content-filter.ts'; +import type { DiskEvent, FileIndexEntry } from './file-watcher.ts'; + +function makeJsonPostReq(url: string, body: unknown): IncomingMessage { + const readable = Readable.from(Buffer.from(JSON.stringify(body))) as unknown as IncomingMessage; + readable.method = 'POST'; + readable.url = url; + readable.headers = { host: 'localhost', 'content-type': 'application/json' }; + return readable; +} + +function makeRes(): { res: ServerResponse; captured: { status: number; body: string } } { + const captured = { status: 0, body: '' }; + const res = { + writeHead(status: number) { + captured.status = status; + }, + end(body?: string) { + captured.body = body ?? ''; + }, + } as unknown as ServerResponse; + return { res, captured }; +} + +async function post( + ext: { + onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise; + }, + url: string, + body: unknown, +): Promise<{ status: number; body: string }> { + const req = makeJsonPostReq(url, body); + const { res, captured } = makeRes(); + await ext.onRequest({ request: req, response: res }); + return captured; +} + +describe('agent write self-registers the file index (PRD-7201)', () => { + test('agent-write-md registers the just-written doc into the file index', async () => { + const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-')); + const contentDir = join(projectDir, 'content'); + mkdirSync(contentDir, { recursive: true }); + const hocuspocus = new Hocuspocus({ quiet: true }); + const sessionManager = new AgentSessionManager(hocuspocus); + const events: DiskEvent[] = []; + + try { + const ext = createApiExtension({ + hocuspocus, + sessionManager, + contentDir, + getFileIndex: () => new Map(), + mutateFileIndex: (event) => events.push(event), + }) as { + onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise; + }; + + const captured = await post(ext, '/api/agent-write-md', { + docName: 'evidence/new-target', + markdown: '# New Target\n', + position: 'replace', + }); + + expect(captured.status).toBe(200); + const docEvents = events.filter( + (e) => 'docName' in e && (e as { docName?: string }).docName === 'evidence/new-target', + ); + expect(docEvents).toHaveLength(1); + expect(docEvents[0].kind).toBe('create'); + } finally { + await sessionManager.closeAll(); + rmSync(projectDir, { recursive: true, force: true }); + } + }); + + test('a write to a doc already in the file index registers as kind:update', async () => { + const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-')); + const contentDir = join(projectDir, 'content'); + mkdirSync(contentDir, { recursive: true }); + const hocuspocus = new Hocuspocus({ quiet: true }); + const sessionManager = new AgentSessionManager(hocuspocus); + const events: DiskEvent[] = []; + const fileIndex = new Map([ + [ + 'notes', + { + size: 1, + modified: new Date(0).toISOString(), + canonicalPath: '', + inode: 0, + aliases: [], + kind: 'markdown', + }, + ], + ]); + + try { + const ext = createApiExtension({ + hocuspocus, + sessionManager, + contentDir, + getFileIndex: () => fileIndex, + mutateFileIndex: (event) => events.push(event), + }) as { + onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise; + }; + + const captured = await post(ext, '/api/agent-write-md', { + docName: 'notes', + markdown: '# Notes\n', + position: 'replace', + }); + + expect(captured.status).toBe(200); + const docEvents = events.filter( + (e) => 'docName' in e && (e as { docName?: string }).docName === 'notes', + ); + expect(docEvents).toHaveLength(1); + expect(docEvents[0].kind).toBe('update'); + } finally { + await sessionManager.closeAll(); + rmSync(projectDir, { recursive: true, force: true }); + } + }); + + test('agent-patch registers the just-edited doc into the file index', async () => { + const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-')); + const contentDir = join(projectDir, 'content'); + mkdirSync(contentDir, { recursive: true }); + const hocuspocus = new Hocuspocus({ quiet: true }); + const sessionManager = new AgentSessionManager(hocuspocus); + const events: DiskEvent[] = []; + + try { + const session = await sessionManager.getSession('notes'); + session.dc.document.transact(() => { + applyAgentMarkdownWrite(session.dc.document, '# Notes\n\nalpha\n', 'replace'); + }, AGENT_WRITE_ORIGIN); + + const ext = createApiExtension({ + hocuspocus, + sessionManager, + contentDir, + getFileIndex: () => new Map(), + mutateFileIndex: (event) => events.push(event), + }) as { + onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise; + }; + + const captured = await post(ext, '/api/agent-patch', { + docName: 'notes', + find: 'alpha', + replace: 'beta', + }); + + expect(captured.status).toBe(200); + const docEvents = events.filter( + (e) => 'docName' in e && (e as { docName?: string }).docName === 'notes', + ); + expect(docEvents).toHaveLength(1); + } finally { + await sessionManager.closeAll(); + rmSync(projectDir, { recursive: true, force: true }); + } + }); + + test('frontmatter-patch registers the patched doc into the file index', async () => { + const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-')); + const contentDir = join(projectDir, 'content'); + mkdirSync(contentDir, { recursive: true }); + const hocuspocus = new Hocuspocus({ quiet: true }); + const sessionManager = new AgentSessionManager(hocuspocus); + const events: DiskEvent[] = []; + + try { + const session = await sessionManager.getSession('notes'); + session.dc.document.transact(() => { + applyAgentMarkdownWrite(session.dc.document, '# Notes\n\nbody text\n', 'replace'); + }, AGENT_WRITE_ORIGIN); + + const ext = createApiExtension({ + hocuspocus, + sessionManager, + contentDir, + getFileIndex: () => new Map(), + mutateFileIndex: (event) => events.push(event), + }) as { + onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise; + }; + + const captured = await post(ext, '/api/frontmatter-patch', { + docName: 'notes', + patch: { addedkey: 'v1' }, + }); + + expect(captured.status).toBe(200); + const docEvents = events.filter( + (e) => 'docName' in e && (e as { docName?: string }).docName === 'notes', + ); + expect(docEvents).toHaveLength(1); + } finally { + await sessionManager.closeAll(); + rmSync(projectDir, { recursive: true, force: true }); + } + }); + + test('does NOT register a content-scope-excluded doc (mirrors the watcher gate)', async () => { + const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-excluded-')); + const contentDir = join(projectDir, 'content'); + mkdirSync(contentDir, { recursive: true }); + writeFileSync(join(contentDir, '.okignore'), 'secret.md\n', 'utf-8'); + const hocuspocus = new Hocuspocus({ quiet: true }); + const sessionManager = new AgentSessionManager(hocuspocus); + const events: DiskEvent[] = []; + + try { + const ext = createApiExtension({ + hocuspocus, + sessionManager, + contentDir, + getFileIndex: () => new Map(), + contentFilter: createContentFilter({ projectDir, contentDir }), + mutateFileIndex: (event) => events.push(event), + }) as { + onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise; + }; + + const captured = await post(ext, '/api/agent-write-md', { + docName: 'secret', + markdown: '# Top Secret\n', + position: 'replace', + }); + + expect(captured.status).toBe(200); + const docEvents = events.filter( + (e) => 'docName' in e && (e as { docName?: string }).docName === 'secret', + ); + expect(docEvents).toHaveLength(0); + } finally { + await sessionManager.closeAll(); + rmSync(projectDir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/server/src/api-backlinks.test.ts b/packages/server/src/api-backlinks.test.ts index 04fcc9fae..576f6b161 100644 --- a/packages/server/src/api-backlinks.test.ts +++ b/packages/server/src/api-backlinks.test.ts @@ -6,6 +6,7 @@ import { join } from 'node:path'; import { Readable } from 'node:stream'; import { createApiExtension } from './api-extension.ts'; import { BacklinkIndex } from './backlink-index.ts'; +import { type ContentFilter, createContentFilter } from './content-filter.ts'; import type { FileIndexEntry } from './file-watcher.ts'; interface CapturedResponse { @@ -41,7 +42,7 @@ async function callRoute( url: string, fileIndex: ReadonlyMap, backlinkIndex?: BacklinkIndex, - options?: { method?: string; enableTestRoutes?: boolean }, + options?: { method?: string; enableTestRoutes?: boolean; contentFilter?: ContentFilter }, ): Promise { const ext = createApiExtension({ hocuspocus: {} as never, @@ -50,6 +51,7 @@ async function callRoute( getFileIndex: () => fileIndex, backlinkIndex, enableTestRoutes: options?.enableTestRoutes, + ...(options?.contentFilter ? { contentFilter: options.contentFilter } : {}), }); const req = makeReq(url, options?.method ?? 'GET'); const { res, captured } = makeRes(); @@ -386,6 +388,51 @@ describe('graph endpoints', () => { } }); + test('link/title consumers resolve a graph-indexed doc the file index is missing (PRD-7201)', async () => { + const projectDir = mkdtempSync(join(tmpdir(), 'ok-graph-api-prd7201-')); + const contentDir = join(projectDir, 'content'); + mkdirSync(join(contentDir, 'evidence'), { recursive: true }); + try { + writeFileSync( + join(contentDir, 'alpha.md'), + '# Alpha\n\nLinks to [[evidence/beta]].\n', + 'utf-8', + ); + writeFileSync(join(contentDir, 'evidence', 'beta.md'), '# Beta\n\nBody.\n', 'utf-8'); + + const backlinkIndex = new BacklinkIndex({ projectDir, contentDir }); + await backlinkIndex.rebuildFromDisk(); + + const fileIndex = new Map([ + [ + 'alpha', + { + size: 10, + modified: new Date(0).toISOString(), + canonicalPath: '', + inode: 0, + aliases: [], + }, + ], + ]); + + const forward = JSON.parse( + (await callRoute(contentDir, '/api/forward-links?docName=alpha', fileIndex, backlinkIndex)) + .body, + ) as { forwardLinks: Array<{ kind: string; docName: string; title: string }> }; + expect(forward.forwardLinks).toEqual([ + expect.objectContaining({ kind: 'doc', docName: 'evidence/beta', title: 'Beta' }), + ]); + + const dead = JSON.parse( + (await callRoute(contentDir, '/api/dead-links', fileIndex, backlinkIndex)).body, + ) as { deadLinks: Array<{ target: string }> }; + expect(dead.deadLinks).toEqual([]); + } finally { + rmSync(projectDir, { recursive: true, force: true }); + } + }); + test('returns 503 when the backlink index is unavailable', async () => { const projectDir = mkdtempSync(join(tmpdir(), 'ok-dead-links-unavailable-')); const contentDir = join(projectDir, 'content'); @@ -407,6 +454,7 @@ describe('graph endpoints', () => { const contentDir = join(projectDir, 'content'); mkdirSync(contentDir, { recursive: true }); try { + writeFileSync(join(contentDir, '.okignore'), 'secret.md\n', 'utf-8'); writeFileSync(join(contentDir, 'public.md'), '# Public\n\nLinks to [[secret]].\n', 'utf-8'); writeFileSync( join(contentDir, 'secret.md'), @@ -429,9 +477,20 @@ describe('graph endpoints', () => { const backlinkIndex = new BacklinkIndex({ projectDir, contentDir }); await backlinkIndex.rebuildFromDisk(); + const contentFilter = createContentFilter({ projectDir, contentDir }); + const forward = JSON.parse( - (await callRoute(contentDir, '/api/forward-links?docName=public', fileIndex, backlinkIndex)) - .body, + ( + await callRoute( + contentDir, + '/api/forward-links?docName=public', + fileIndex, + backlinkIndex, + { + contentFilter, + }, + ) + ).body, ) as { forwardLinks: Array<{ kind: 'doc'; docName: string; title: string }>; }; @@ -440,12 +499,17 @@ describe('graph endpoints', () => { expect(forward.forwardLinks[0].title).toBe('secret'); const hubs = JSON.parse( - (await callRoute(contentDir, '/api/hubs', fileIndex, backlinkIndex)).body, + (await callRoute(contentDir, '/api/hubs', fileIndex, backlinkIndex, { contentFilter })) + .body, ) as { hubs: Array<{ docName: string; title: string; count: number }> }; expect(hubs.hubs).toContainEqual({ docName: 'secret', title: 'secret', count: 1 }); const linkGraph = JSON.parse( - (await callRoute(contentDir, '/api/link-graph', fileIndex, backlinkIndex)).body, + ( + await callRoute(contentDir, '/api/link-graph', fileIndex, backlinkIndex, { + contentFilter, + }) + ).body, ) as { nodes: Array<{ id: string; diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.ts index 2ac46d7f0..48cc5a373 100644 --- a/packages/server/src/api-extension.ts +++ b/packages/server/src/api-extension.ts @@ -2172,6 +2172,12 @@ export function createApiExtension(options: ApiExtensionOptions): Extension { }; } + function isDocNameContentExcluded(docName: string): boolean { + if (!contentFilter) return false; + const relPath = isSupportedDocFile(docName) ? docName : `${docName}${getDocExtension(docName)}`; + return contentFilter.isExcluded(relPath); + } + function collectAdmittedDocNames(): Set { const admitted = new Set(); for (const [docName, entry] of getFileIndex()) { @@ -2196,12 +2202,26 @@ export function createApiExtension(options: ApiExtensionOptions): Extension { } catch (err) { log.warn({ err }, '[collectAdmittedDocNames] managed-artifact enumeration failed'); } + for (const docName of backlinkIndex?.getIndexedDocNames() ?? []) { + if (admitted.has(docName)) continue; + if (!isDocNameContentExcluded(docName)) admitted.add(docName); + } return admitted; } const linkedFileExists = (contentRootRelativePath: string): boolean => existsSync(resolve(contentDir, contentRootRelativePath)); + function registerWrittenDocInFileIndex(docName: string, content: string): void { + if (isDocNameContentExcluded(docName)) return; + mutateFileIndex?.({ + kind: getFileIndex().has(docName) ? 'update' : 'create', + path: resolveContentEntryPath(contentDir, 'file', docName), + docName, + content, + }); + } + function createSerializedRunner() { let pending = Promise.resolve(); return async function runSerialized(task: () => Promise): Promise { @@ -3724,6 +3744,8 @@ export function createApiExtension(options: ApiExtensionOptions): Extension { const writtenSource = session.dc.document.getText('source').toString(); + registerWrittenDocInFileIndex(resolvedDocName, writtenSource); + const renderWarnings = await validateMermaidFences(writtenSource, resolvedDocName); const admittedForLinks = collectAdmittedDocNames(); @@ -3999,6 +4021,11 @@ export function createApiExtension(options: ApiExtensionOptions): Extension { const fmWarning = buildReconcileWarning(fmReconcile); + registerWrittenDocInFileIndex( + resolvedDocName, + session.dc.document.getText('source').toString(), + ); + const admittedForLinks = collectAdmittedDocNames(); admittedForLinks.add(resolvedDocName); const brokenLinks = computeBrokenOutboundLinks( @@ -5194,6 +5221,8 @@ export function createApiExtension(options: ApiExtensionOptions): Extension { const patchedSource = session.dc.document.getText('source').toString(); + registerWrittenDocInFileIndex(docName, patchedSource); + const renderWarnings = await validateMermaidFences(patchedSource, docName); const admittedForLinks = collectAdmittedDocNames(); diff --git a/packages/server/src/backlink-index.test.ts b/packages/server/src/backlink-index.test.ts index 0d3bb4ce8..92367ef93 100644 --- a/packages/server/src/backlink-index.test.ts +++ b/packages/server/src/backlink-index.test.ts @@ -373,6 +373,30 @@ describe('BacklinkIndex', () => { } }); + test('getIndexedDocNames returns one entry per indexed doc and never a referenced-but-missing target', () => { + const projectDir = mkdtempSync(join(tmpdir(), 'ok-backlinks-indexed-names-')); + const contentDir = join(projectDir, 'content'); + mkdirSync(contentDir, { recursive: true }); + try { + const index = new BacklinkIndex({ projectDir, contentDir }); + index.updateDocument('report', [ + { target: 'evidence/new-target', anchor: null, snippet: 'see new target' }, + { target: 'evidence/ghost', anchor: null, snippet: 'see ghost' }, + ]); + index.updateDocument('evidence/new-target', []); + + expect(new Set(index.getIndexedDocNames())).toEqual( + new Set(['report', 'evidence/new-target']), + ); + expect(index.getIndexedDocNames()).not.toContain('evidence/ghost'); + + index.deleteDocument('evidence/new-target'); + expect(index.getIndexedDocNames()).toEqual(['report']); + } finally { + rmSync(projectDir, { recursive: true, force: true }); + } + }); + test('getDeadLinks reports a target as dead again after deleteDocument removes its forward node', () => { const projectDir = mkdtempSync(join(tmpdir(), 'ok-backlinks-dead-after-delete-')); const contentDir = join(projectDir, 'content'); diff --git a/packages/server/src/backlink-index.ts b/packages/server/src/backlink-index.ts index 0ee06a95b..2d805dd81 100644 --- a/packages/server/src/backlink-index.ts +++ b/packages/server/src/backlink-index.ts @@ -1163,6 +1163,10 @@ export class BacklinkIndex { .slice(0, limit); } + getIndexedDocNames(branch = this.activeBranch): string[] { + return [...this.getState(branch).forward.keys()]; + } + getDeadLinks( admittedDocs: Iterable, sourceDocNames?: readonly string[],