Stop the file-watcher's indexing lag from making freshly-written docs…

.changeset/unify-doc-existence-oracle.md

@@ -0,0 +1,6 @@
+---
+"@inkeep/open-knowledge-server": patch
+"@inkeep/open-knowledge": patch
+---
+
+Stop the file-watcher's indexing lag from making freshly-written docs look missing across link, title, and dead-link results. The file index the watcher maintains can lag an in-session write — or permanently drop the create event for a file written into a just-created subfolder — which made a brand-new doc render as a raw red-link name (instead of its title) and could flag a valid link to it as dead, until a server restart. Two complementary fixes close the gap: doc-existence now also trusts the link graph (updated in-process on every write), and the agent write/edit/frontmatter-patch handlers register the doc they just wrote into the file index immediately, mirroring page creation. Both paths honor content-scope exclusions, so a gitignore/okignore'd doc never leaks its title through these endpoints.

packages/server/src/api-agent-write-file-index.test.ts

@@ -0,0 +1,257 @@
+import { describe, expect, test } from 'bun:test';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Readable } from 'node:stream';
+import { Hocuspocus } from '@hocuspocus/server';
+import {
+  AGENT_WRITE_ORIGIN,
+  AgentSessionManager,
+  applyAgentMarkdownWrite,
+} from './agent-sessions.ts';
+import { createApiExtension } from './api-extension.ts';
+import { createContentFilter } from './content-filter.ts';
+import type { DiskEvent, FileIndexEntry } from './file-watcher.ts';
+
+function makeJsonPostReq(url: string, body: unknown): IncomingMessage {
+  const readable = Readable.from(Buffer.from(JSON.stringify(body))) as unknown as IncomingMessage;
+  readable.method = 'POST';
+  readable.url = url;
+  readable.headers = { host: 'localhost', 'content-type': 'application/json' };
+  return readable;
+}
+
+function makeRes(): { res: ServerResponse; captured: { status: number; body: string } } {
+  const captured = { status: 0, body: '' };
+  const res = {
+    writeHead(status: number) {
+      captured.status = status;
+    },
+    end(body?: string) {
+      captured.body = body ?? '';
+    },
+  } as unknown as ServerResponse;
+  return { res, captured };
+}
+
+async function post(
+  ext: {
+    onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;
+  },
+  url: string,
+  body: unknown,
+): Promise<{ status: number; body: string }> {
+  const req = makeJsonPostReq(url, body);
+  const { res, captured } = makeRes();
+  await ext.onRequest({ request: req, response: res });
+  return captured;
+}
+
+describe('agent write self-registers the file index (PRD-7201)', () => {
+  test('agent-write-md registers the just-written doc into the file index', async () => {
+    const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-'));
+    const contentDir = join(projectDir, 'content');
+    mkdirSync(contentDir, { recursive: true });
+    const hocuspocus = new Hocuspocus({ quiet: true });
+    const sessionManager = new AgentSessionManager(hocuspocus);
+    const events: DiskEvent[] = [];
+
+    try {
+      const ext = createApiExtension({
+        hocuspocus,
+        sessionManager,
+        contentDir,
+        getFileIndex: () => new Map(),
+        mutateFileIndex: (event) => events.push(event),
+      }) as {
+        onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;
+      };
+
+      const captured = await post(ext, '/api/agent-write-md', {
+        docName: 'evidence/new-target',
+        markdown: '# New Target\n',
+        position: 'replace',
+      });
+
+      expect(captured.status).toBe(200);
+      const docEvents = events.filter(
+        (e) => 'docName' in e && (e as { docName?: string }).docName === 'evidence/new-target',
+      );
+      expect(docEvents).toHaveLength(1);
+      expect(docEvents[0].kind).toBe('create');
+    } finally {
+      await sessionManager.closeAll();
+      rmSync(projectDir, { recursive: true, force: true });
+    }
+  });
+
+  test('a write to a doc already in the file index registers as kind:update', async () => {
+    const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-'));
+    const contentDir = join(projectDir, 'content');
+    mkdirSync(contentDir, { recursive: true });
+    const hocuspocus = new Hocuspocus({ quiet: true });
+    const sessionManager = new AgentSessionManager(hocuspocus);
+    const events: DiskEvent[] = [];
+    const fileIndex = new Map<string, FileIndexEntry>([
+      [
+        'notes',
+        {
+          size: 1,
+          modified: new Date(0).toISOString(),
+          canonicalPath: '',
+          inode: 0,
+          aliases: [],
+          kind: 'markdown',
+        },
+      ],
+    ]);
+
+    try {
+      const ext = createApiExtension({
+        hocuspocus,
+        sessionManager,
+        contentDir,
+        getFileIndex: () => fileIndex,
+        mutateFileIndex: (event) => events.push(event),
+      }) as {
+        onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;
+      };
+
+      const captured = await post(ext, '/api/agent-write-md', {
+        docName: 'notes',
+        markdown: '# Notes\n',
+        position: 'replace',
+      });
+
+      expect(captured.status).toBe(200);
+      const docEvents = events.filter(
+        (e) => 'docName' in e && (e as { docName?: string }).docName === 'notes',
+      );
+      expect(docEvents).toHaveLength(1);
+      expect(docEvents[0].kind).toBe('update');
+    } finally {
+      await sessionManager.closeAll();
+      rmSync(projectDir, { recursive: true, force: true });
+    }
+  });
+
+  test('agent-patch registers the just-edited doc into the file index', async () => {
+    const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-'));
+    const contentDir = join(projectDir, 'content');
+    mkdirSync(contentDir, { recursive: true });
+    const hocuspocus = new Hocuspocus({ quiet: true });
+    const sessionManager = new AgentSessionManager(hocuspocus);
+    const events: DiskEvent[] = [];
+
+    try {
+      const session = await sessionManager.getSession('notes');
+      session.dc.document.transact(() => {
+        applyAgentMarkdownWrite(session.dc.document, '# Notes\n\nalpha\n', 'replace');
+      }, AGENT_WRITE_ORIGIN);
+
+      const ext = createApiExtension({
+        hocuspocus,
+        sessionManager,
+        contentDir,
+        getFileIndex: () => new Map(),
+        mutateFileIndex: (event) => events.push(event),
+      }) as {
+        onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;
+      };
+
+      const captured = await post(ext, '/api/agent-patch', {
+        docName: 'notes',
+        find: 'alpha',
+        replace: 'beta',
+      });
+
+      expect(captured.status).toBe(200);
+      const docEvents = events.filter(
+        (e) => 'docName' in e && (e as { docName?: string }).docName === 'notes',
+      );
+      expect(docEvents).toHaveLength(1);
+    } finally {
+      await sessionManager.closeAll();
+      rmSync(projectDir, { recursive: true, force: true });
+    }
+  });
+
+  test('frontmatter-patch registers the patched doc into the file index', async () => {
+    const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-'));
+    const contentDir = join(projectDir, 'content');
+    mkdirSync(contentDir, { recursive: true });
+    const hocuspocus = new Hocuspocus({ quiet: true });
+    const sessionManager = new AgentSessionManager(hocuspocus);
+    const events: DiskEvent[] = [];
+
+    try {
+      const session = await sessionManager.getSession('notes');
+      session.dc.document.transact(() => {
+        applyAgentMarkdownWrite(session.dc.document, '# Notes\n\nbody text\n', 'replace');
+      }, AGENT_WRITE_ORIGIN);
+
+      const ext = createApiExtension({
+        hocuspocus,
+        sessionManager,
+        contentDir,
+        getFileIndex: () => new Map(),
+        mutateFileIndex: (event) => events.push(event),
+      }) as {
+        onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;
+      };
+
+      const captured = await post(ext, '/api/frontmatter-patch', {
+        docName: 'notes',
+        patch: { addedkey: 'v1' },
+      });
+
+      expect(captured.status).toBe(200);
+      const docEvents = events.filter(
+        (e) => 'docName' in e && (e as { docName?: string }).docName === 'notes',
+      );
+      expect(docEvents).toHaveLength(1);
+    } finally {
+      await sessionManager.closeAll();
+      rmSync(projectDir, { recursive: true, force: true });
+    }
+  });
+
+  test('does NOT register a content-scope-excluded doc (mirrors the watcher gate)', async () => {
+    const projectDir = mkdtempSync(join(tmpdir(), 'ok-write-file-index-excluded-'));
+    const contentDir = join(projectDir, 'content');
+    mkdirSync(contentDir, { recursive: true });
+    writeFileSync(join(contentDir, '.okignore'), 'secret.md\n', 'utf-8');
+    const hocuspocus = new Hocuspocus({ quiet: true });
+    const sessionManager = new AgentSessionManager(hocuspocus);
+    const events: DiskEvent[] = [];
+
+    try {
+      const ext = createApiExtension({
+        hocuspocus,
+        sessionManager,
+        contentDir,
+        getFileIndex: () => new Map(),
+        contentFilter: createContentFilter({ projectDir, contentDir }),
+        mutateFileIndex: (event) => events.push(event),
+      }) as {
+        onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;
+      };
+
+      const captured = await post(ext, '/api/agent-write-md', {
+        docName: 'secret',
+        markdown: '# Top Secret\n',
+        position: 'replace',
+      });
+
+      expect(captured.status).toBe(200);
+      const docEvents = events.filter(
+        (e) => 'docName' in e && (e as { docName?: string }).docName === 'secret',
+      );
+      expect(docEvents).toHaveLength(0);
+    } finally {
+      await sessionManager.closeAll();
+      rmSync(projectDir, { recursive: true, force: true });
+    }
+  });
+});

packages/server/src/api-backlinks.test.ts

@@ -6,6 +6,7 @@ import { join } from 'node:path';
 import { Readable } from 'node:stream';
 import { createApiExtension } from './api-extension.ts';
 import { BacklinkIndex } from './backlink-index.ts';
+import { type ContentFilter, createContentFilter } from './content-filter.ts';
 import type { FileIndexEntry } from './file-watcher.ts';
 
 interface CapturedResponse {
@@ -41,7 +42,7 @@ async function callRoute(
   url: string,
   fileIndex: ReadonlyMap<string, FileIndexEntry>,
   backlinkIndex?: BacklinkIndex,
-  options?: { method?: string; enableTestRoutes?: boolean },
+  options?: { method?: string; enableTestRoutes?: boolean; contentFilter?: ContentFilter },
 ): Promise<CapturedResponse> {
   const ext = createApiExtension({
     hocuspocus: {} as never,
@@ -50,6 +51,7 @@ async function callRoute(
     getFileIndex: () => fileIndex,
     backlinkIndex,
     enableTestRoutes: options?.enableTestRoutes,
+    ...(options?.contentFilter ? { contentFilter: options.contentFilter } : {}),
   });
   const req = makeReq(url, options?.method ?? 'GET');
   const { res, captured } = makeRes();
@@ -386,6 +388,51 @@ describe('graph endpoints', () => {
     }
   });
 
+  test('link/title consumers resolve a graph-indexed doc the file index is missing (PRD-7201)', async () => {
+    const projectDir = mkdtempSync(join(tmpdir(), 'ok-graph-api-prd7201-'));
+    const contentDir = join(projectDir, 'content');
+    mkdirSync(join(contentDir, 'evidence'), { recursive: true });
+    try {
+      writeFileSync(
+        join(contentDir, 'alpha.md'),
+        '# Alpha\n\nLinks to [[evidence/beta]].\n',
+        'utf-8',
+      );
+      writeFileSync(join(contentDir, 'evidence', 'beta.md'), '# Beta\n\nBody.\n', 'utf-8');
+
+      const backlinkIndex = new BacklinkIndex({ projectDir, contentDir });
+      await backlinkIndex.rebuildFromDisk();
+
+      const fileIndex = new Map<string, FileIndexEntry>([
+        [
+          'alpha',
+          {
+            size: 10,
+            modified: new Date(0).toISOString(),
+            canonicalPath: '',
+            inode: 0,
+            aliases: [],
+          },
+        ],
+      ]);
+
+      const forward = JSON.parse(
+        (await callRoute(contentDir, '/api/forward-links?docName=alpha', fileIndex, backlinkIndex))
+          .body,
+      ) as { forwardLinks: Array<{ kind: string; docName: string; title: string }> };
+      expect(forward.forwardLinks).toEqual([
+        expect.objectContaining({ kind: 'doc', docName: 'evidence/beta', title: 'Beta' }),
+      ]);
+
+      const dead = JSON.parse(
+        (await callRoute(contentDir, '/api/dead-links', fileIndex, backlinkIndex)).body,
+      ) as { deadLinks: Array<{ target: string }> };
+      expect(dead.deadLinks).toEqual([]);
+    } finally {
+      rmSync(projectDir, { recursive: true, force: true });
+    }
+  });
+
   test('returns 503 when the backlink index is unavailable', async () => {
     const projectDir = mkdtempSync(join(tmpdir(), 'ok-dead-links-unavailable-'));
     const contentDir = join(projectDir, 'content');
@@ -407,6 +454,7 @@ describe('graph endpoints', () => {
     const contentDir = join(projectDir, 'content');
     mkdirSync(contentDir, { recursive: true });
     try {
+      writeFileSync(join(contentDir, '.okignore'), 'secret.md\n', 'utf-8');
       writeFileSync(join(contentDir, 'public.md'), '# Public\n\nLinks to [[secret]].\n', 'utf-8');
       writeFileSync(
         join(contentDir, 'secret.md'),
@@ -429,9 +477,20 @@ describe('graph endpoints', () => {
       const backlinkIndex = new BacklinkIndex({ projectDir, contentDir });
       await backlinkIndex.rebuildFromDisk();
 
+      const contentFilter = createContentFilter({ projectDir, contentDir });
+
       const forward = JSON.parse(
-        (await callRoute(contentDir, '/api/forward-links?docName=public', fileIndex, backlinkIndex))
-          .body,
+        (
+          await callRoute(
+            contentDir,
+            '/api/forward-links?docName=public',
+            fileIndex,
+            backlinkIndex,
+            {
+              contentFilter,
+            },
+          )
+        ).body,
       ) as {
         forwardLinks: Array<{ kind: 'doc'; docName: string; title: string }>;
       };
@@ -440,12 +499,17 @@ describe('graph endpoints', () => {
       expect(forward.forwardLinks[0].title).toBe('secret');
 
       const hubs = JSON.parse(
-        (await callRoute(contentDir, '/api/hubs', fileIndex, backlinkIndex)).body,
+        (await callRoute(contentDir, '/api/hubs', fileIndex, backlinkIndex, { contentFilter }))
+          .body,
       ) as { hubs: Array<{ docName: string; title: string; count: number }> };
       expect(hubs.hubs).toContainEqual({ docName: 'secret', title: 'secret', count: 1 });
 
       const linkGraph = JSON.parse(
-        (await callRoute(contentDir, '/api/link-graph', fileIndex, backlinkIndex)).body,
+        (
+          await callRoute(contentDir, '/api/link-graph', fileIndex, backlinkIndex, {
+            contentFilter,
+          })
+        ).body,
       ) as {
         nodes: Array<{
           id: string;