Speed up project load for knowledge bases with many files. `GET…

.changeset/pages-list-from-index-cache.md

@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Speed up project load for knowledge bases with many files. `GET /api/pages` previously re-read and re-parsed every markdown file from disk on every request — including the redundant full-directory pass that ran concurrently with the watcher's seed walk on cold load, and a fresh full re-read on every window focus / file-change refresh. Page titles and icons are now cached on the in-memory file index (derived from the content the watcher already reads for its change-detection hash, so no extra disk reads), and `/api/pages` serves them straight from memory. Titles/icons stay current through create, edit, and rename events. Behavior is unchanged; only the cost of listing pages drops from O(files) disk reads per request to an in-memory scan.

packages/server/src/api-extension.ts

@@ -8060,16 +8060,22 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
           icon?: string;
         }[] = [];
         for (const [docName, entry] of index) {
-          let title = docName;
-          let icon: string | undefined;
           const docExt = getDocExtension(docName);
-          try {
-            const filePath = resolve(contentDir, `${docName}${docExt}`);
-            const content = readFileSync(filePath, 'utf-8');
-            title = extractPageTitle(content, docName);
-            icon = extractPageIcon(content);
-          } catch (err) {
-            console.warn(`[pages] Failed to read title for ${docName}:`, err);
+          let title: string;
+          let icon: string | undefined;
+          if (entry.title !== undefined) {
+            title = entry.title;
+            icon = entry.icon;
+          } else {
+            title = docName;
+            try {
+              const filePath = resolve(contentDir, `${docName}${docExt}`);
+              const content = readFileSync(filePath, 'utf-8');
+              title = extractPageTitle(content, docName);
+              icon = extractPageIcon(content);
+            } catch (err) {
+              console.warn(`[pages] Failed to read title for ${docName}:`, err);
+            }
           }
           pages.push({ docName, title, docExt, size: entry.size, modified: entry.modified, icon });
         }

packages/server/src/api-pages.test.ts

@@ -230,8 +230,11 @@ function buildFileIndex(dir: string, base = ''): ReadonlyMap<string, FileIndexEn
   return index;
 }
 
-async function callPages(contentDir: string, method = 'GET'): Promise<CapturedResponse> {
-  const fileIndex = buildFileIndex(contentDir);
+async function callPagesWithIndex(
+  contentDir: string,
+  fileIndex: ReadonlyMap<string, FileIndexEntry>,
+  method = 'GET',
+): Promise<CapturedResponse> {
   const ext = createApiExtension({
     hocuspocus: {} as unknown as Parameters<typeof createApiExtension>[0]['hocuspocus'],
     sessionManager: {} as unknown as Parameters<typeof createApiExtension>[0]['sessionManager'],
@@ -248,6 +251,10 @@ async function callPages(contentDir: string, method = 'GET'): Promise<CapturedRe
   return captured;
 }
 
+async function callPages(contentDir: string, method = 'GET'): Promise<CapturedResponse> {
+  return callPagesWithIndex(contentDir, buildFileIndex(contentDir), method);
+}
+
 describe('GET /api/pages', () => {
   test('returns flat { pages } success body and lists markdown files recursively', async () => {
     const dir = mkdtempSync(join(tmpdir(), 'ok-pages-'));
@@ -300,6 +307,68 @@ describe('GET /api/pages', () => {
     }
   });
 
+  test('serves cached title/icon from the index without reading disk', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'ok-pages-cache-'));
+    try {
+      const index = new Map<string, FileIndexEntry>([
+        [
+          'ghost',
+          {
+            size: 123,
+            modified: new Date(0).toISOString(),
+            canonicalPath: join(dir, 'ghost.md'),
+            inode: 1,
+            aliases: [],
+            kind: 'markdown',
+            title: 'Cached Title',
+            icon: '🎯',
+          },
+        ],
+      ]);
+
+      const result = await callPagesWithIndex(dir, index);
+      expect(result.status).toBe(200);
+      const body = JSON.parse(result.body) as {
+        pages?: Array<{ docName: string; title: string; icon?: string }>;
+      };
+      const entry = (body.pages ?? []).find((p) => p.docName === 'ghost');
+      expect(entry?.title).toBe('Cached Title');
+      expect(entry?.icon).toBe('🎯');
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  test('bare entry whose file is missing falls back to the docName title (ENOENT path)', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'ok-pages-bare-'));
+    try {
+      const index = new Map<string, FileIndexEntry>([
+        [
+          'orphan',
+          {
+            size: 0,
+            modified: new Date(0).toISOString(),
+            canonicalPath: join(dir, 'orphan.md'),
+            inode: 1,
+            aliases: [],
+            kind: 'markdown',
+          },
+        ],
+      ]);
+
+      const result = await callPagesWithIndex(dir, index);
+      expect(result.status).toBe(200);
+      const body = JSON.parse(result.body) as {
+        pages?: Array<{ docName: string; title: string; icon?: string }>;
+      };
+      const entry = (body.pages ?? []).find((p) => p.docName === 'orphan');
+      expect(entry?.title).toBe('orphan');
+      expect(entry?.icon ?? undefined).toBeUndefined();
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
   test('returns RFC 9457 problem+json 405 envelope for unsupported methods', async () => {
     const dir = mkdtempSync(join(tmpdir(), 'ok-pages-'));
     try {

packages/server/src/file-watcher.test.ts

@@ -391,6 +391,27 @@ describe('startWatcher file index', () => {
     }
   });
 
+  test('initial scan caches page title + icon on each markdown entry', async () => {
+    writeFileSync(
+      resolve(contentDir, 'with-meta.md'),
+      '---\ntitle: Meta Title\nicon: 📝\n---\n\n# Heading Ignored\n',
+    );
+    writeFileSync(resolve(contentDir, 'heading-only.md'), '# Heading Title\n');
+    writeFileSync(resolve(contentDir, 'plain.md'), 'just body text, no heading\n');
+
+    const handle = await startWatcher(contentDir, async () => {});
+    try {
+      const index = handle.getFileIndex();
+      expect(index.get('with-meta')?.title).toBe('Meta Title');
+      expect(index.get('with-meta')?.icon).toBe('📝');
+      expect(index.get('heading-only')?.title).toBe('Heading Title');
+      expect(index.get('heading-only')?.icon).toBeUndefined();
+      expect(index.get('plain')?.title).toBe('plain');
+    } finally {
+      await handle.unsubscribe();
+    }
+  });
+
   test('initial scan preserves uppercase .MD/.MDX extension casing', async () => {
     writeFileSync(resolve(contentDir, 'Upper.MD'), '# Upper\n');
     writeFileSync(resolve(contentDir, 'Mixed.MdX'), '# Mixed\n');
@@ -519,6 +540,61 @@ describe('startWatcher file index', () => {
     expect(index.get('new-name')?.size).toBe(Buffer.byteLength('# Renamed\n', 'utf-8'));
   });
 
+  test('file index caches title + icon on create/update/rename/conflict events', () => {
+    const { updateFileIndex } = require('./file-watcher.ts');
+    const index = new Map();
+
+    updateFileIndex(
+      {
+        kind: 'create' as const,
+        path: resolve(contentDir, 'doc.md'),
+        docName: 'doc',
+        content: '---\ntitle: Created\nicon: 🚀\n---\n\nBody\n',
+      },
+      index,
+    );
+    expect(index.get('doc')?.title).toBe('Created');
+    expect(index.get('doc')?.icon).toBe('🚀');
+
+    updateFileIndex(
+      {
+        kind: 'update' as const,
+        path: resolve(contentDir, 'doc.md'),
+        docName: 'doc',
+        content: '# Edited Title\n',
+      },
+      index,
+    );
+    expect(index.get('doc')?.title).toBe('Edited Title');
+    expect(index.get('doc')?.icon).toBeUndefined();
+
+    updateFileIndex(
+      {
+        kind: 'rename' as const,
+        oldPath: resolve(contentDir, 'doc.md'),
+        newPath: resolve(contentDir, 'renamed.md'),
+        oldDocName: 'doc',
+        newDocName: 'renamed',
+        content: '---\ntitle: Renamed Title\nicon: 🔖\n---\n\nBody\n',
+      },
+      index,
+    );
+    expect(index.get('renamed')?.title).toBe('Renamed Title');
+    expect(index.get('renamed')?.icon).toBe('🔖');
+
+    updateFileIndex(
+      {
+        kind: 'conflict' as const,
+        path: resolve(contentDir, 'renamed.md'),
+        docName: 'renamed',
+        content: '---\ntitle: Conflicted\nicon: ⚠️\n---\n<<<<<<< HEAD\na\n=======\nb\n>>>>>>> x\n',
+      },
+      index,
+    );
+    expect(index.get('renamed')?.title).toBe('Conflicted');
+    expect(index.get('renamed')?.icon).toBe('⚠️');
+  });
+
   test('getFileIndex returns empty map when no .md files exist', async () => {
     const handle = await startWatcher(contentDir, async () => {});
     try {

packages/server/src/file-watcher.ts

@@ -14,6 +14,7 @@ import {
 } from './doc-extensions.ts';
 import { classifyFsPath, normalizeFsPath } from './fs-traced.ts';
 import { getLogger } from './logger.ts';
+import { extractPageIcon, extractPageTitle } from './page-identity.ts';
 import { toPosix } from './path-utils.ts';
 import { isWithinContentDir } from './persistence.ts';
 import { containsConflictMarkers } from './reconciliation.ts';
@@ -79,6 +80,8 @@ export interface FileIndexEntry {
   inode: number;
   aliases: string[];
   kind: 'markdown' | 'file';
+  title?: string;
+  icon?: string;
 }
 
 export interface FolderIndexEntry {
@@ -88,6 +91,13 @@ export interface FolderIndexEntry {
   inode: number;
 }
 
+function derivePageMeta(
+  content: string,
+  docName: string,
+): { title: string; icon: string | undefined } {
+  return { title: extractPageTitle(content, docName), icon: extractPageIcon(content) };
+}
+
 function markdownIndexView(
   inner: ReadonlyMap<string, FileIndexEntry>,
 ): ReadonlyMap<string, FileIndexEntry> {
@@ -554,6 +564,7 @@ async function seedLastKnownHashes(
                 inode: canonStat.ino,
                 aliases: [aliasDocName],
                 kind: 'markdown',
+                ...derivePageMeta(content, canonicalDocName),
               });
             } catch (err) {
               const code = (err as NodeJS.ErrnoException).code;
@@ -626,6 +637,7 @@ async function seedLastKnownHashes(
             inode: lst.ino,
             aliases: [],
             kind: 'markdown',
+            ...derivePageMeta(content, docName),
           });
         } catch (err) {
           const code = (err as NodeJS.ErrnoException).code;
@@ -717,6 +729,7 @@ export function updateFileIndex(event: DiskEvent, fileIndex: Map<string, FileInd
         inode: existing?.inode ?? 0,
         aliases: existing?.aliases ?? [],
         kind: 'markdown',
+        ...derivePageMeta(event.content, docName),
       });
       break;
     }
@@ -748,6 +761,7 @@ export function updateFileIndex(event: DiskEvent, fileIndex: Map<string, FileInd
         inode: existing?.inode ?? 0,
         aliases: existing?.aliases ?? [],
         kind: 'markdown',
+        ...derivePageMeta(event.content, event.newDocName),
       });
       break;
     }