diff --git a/.changeset/pages-list-from-index-cache.md b/.changeset/pages-list-from-index-cache.md new file mode 100644 index 000000000..ab7f2a8da --- /dev/null +++ b/.changeset/pages-list-from-index-cache.md @@ -0,0 +1,5 @@ +--- +"@inkeep/open-knowledge": patch +--- + +Speed up project load for knowledge bases with many files. `GET /api/pages` previously re-read and re-parsed every markdown file from disk on every request — including the redundant full-directory pass that ran concurrently with the watcher's seed walk on cold load, and a fresh full re-read on every window focus / file-change refresh. Page titles and icons are now cached on the in-memory file index (derived from the content the watcher already reads for its change-detection hash, so no extra disk reads), and `/api/pages` serves them straight from memory. Titles/icons stay current through create, edit, and rename events. Behavior is unchanged; only the cost of listing pages drops from O(files) disk reads per request to an in-memory scan. diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.ts index ff2708efe..ede6e5e22 100644 --- a/packages/server/src/api-extension.ts +++ b/packages/server/src/api-extension.ts @@ -8060,16 +8060,22 @@ export function createApiExtension(options: ApiExtensionOptions): Extension { icon?: string; }[] = []; for (const [docName, entry] of index) { - let title = docName; - let icon: string | undefined; const docExt = getDocExtension(docName); - try { - const filePath = resolve(contentDir, `${docName}${docExt}`); - const content = readFileSync(filePath, 'utf-8'); - title = extractPageTitle(content, docName); - icon = extractPageIcon(content); - } catch (err) { - console.warn(`[pages] Failed to read title for ${docName}:`, err); + let title: string; + let icon: string | undefined; + if (entry.title !== undefined) { + title = entry.title; + icon = entry.icon; + } else { + title = docName; + try { + const filePath = resolve(contentDir, `${docName}${docExt}`); + const content = readFileSync(filePath, 'utf-8'); + title = extractPageTitle(content, docName); + icon = extractPageIcon(content); + } catch (err) { + console.warn(`[pages] Failed to read title for ${docName}:`, err); + } } pages.push({ docName, title, docExt, size: entry.size, modified: entry.modified, icon }); } diff --git a/packages/server/src/api-pages.test.ts b/packages/server/src/api-pages.test.ts index f7beb4944..d3226448a 100644 --- a/packages/server/src/api-pages.test.ts +++ b/packages/server/src/api-pages.test.ts @@ -230,8 +230,11 @@ function buildFileIndex(dir: string, base = ''): ReadonlyMap { - const fileIndex = buildFileIndex(contentDir); +async function callPagesWithIndex( + contentDir: string, + fileIndex: ReadonlyMap, + method = 'GET', +): Promise { const ext = createApiExtension({ hocuspocus: {} as unknown as Parameters[0]['hocuspocus'], sessionManager: {} as unknown as Parameters[0]['sessionManager'], @@ -248,6 +251,10 @@ async function callPages(contentDir: string, method = 'GET'): Promise { + return callPagesWithIndex(contentDir, buildFileIndex(contentDir), method); +} + describe('GET /api/pages', () => { test('returns flat { pages } success body and lists markdown files recursively', async () => { const dir = mkdtempSync(join(tmpdir(), 'ok-pages-')); @@ -300,6 +307,68 @@ describe('GET /api/pages', () => { } }); + test('serves cached title/icon from the index without reading disk', async () => { + const dir = mkdtempSync(join(tmpdir(), 'ok-pages-cache-')); + try { + const index = new Map([ + [ + 'ghost', + { + size: 123, + modified: new Date(0).toISOString(), + canonicalPath: join(dir, 'ghost.md'), + inode: 1, + aliases: [], + kind: 'markdown', + title: 'Cached Title', + icon: '🎯', + }, + ], + ]); + + const result = await callPagesWithIndex(dir, index); + expect(result.status).toBe(200); + const body = JSON.parse(result.body) as { + pages?: Array<{ docName: string; title: string; icon?: string }>; + }; + const entry = (body.pages ?? []).find((p) => p.docName === 'ghost'); + expect(entry?.title).toBe('Cached Title'); + expect(entry?.icon).toBe('🎯'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test('bare entry whose file is missing falls back to the docName title (ENOENT path)', async () => { + const dir = mkdtempSync(join(tmpdir(), 'ok-pages-bare-')); + try { + const index = new Map([ + [ + 'orphan', + { + size: 0, + modified: new Date(0).toISOString(), + canonicalPath: join(dir, 'orphan.md'), + inode: 1, + aliases: [], + kind: 'markdown', + }, + ], + ]); + + const result = await callPagesWithIndex(dir, index); + expect(result.status).toBe(200); + const body = JSON.parse(result.body) as { + pages?: Array<{ docName: string; title: string; icon?: string }>; + }; + const entry = (body.pages ?? []).find((p) => p.docName === 'orphan'); + expect(entry?.title).toBe('orphan'); + expect(entry?.icon ?? undefined).toBeUndefined(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + test('returns RFC 9457 problem+json 405 envelope for unsupported methods', async () => { const dir = mkdtempSync(join(tmpdir(), 'ok-pages-')); try { diff --git a/packages/server/src/file-watcher.test.ts b/packages/server/src/file-watcher.test.ts index 868bd6123..4658dcad6 100644 --- a/packages/server/src/file-watcher.test.ts +++ b/packages/server/src/file-watcher.test.ts @@ -391,6 +391,27 @@ describe('startWatcher file index', () => { } }); + test('initial scan caches page title + icon on each markdown entry', async () => { + writeFileSync( + resolve(contentDir, 'with-meta.md'), + '---\ntitle: Meta Title\nicon: 📝\n---\n\n# Heading Ignored\n', + ); + writeFileSync(resolve(contentDir, 'heading-only.md'), '# Heading Title\n'); + writeFileSync(resolve(contentDir, 'plain.md'), 'just body text, no heading\n'); + + const handle = await startWatcher(contentDir, async () => {}); + try { + const index = handle.getFileIndex(); + expect(index.get('with-meta')?.title).toBe('Meta Title'); + expect(index.get('with-meta')?.icon).toBe('📝'); + expect(index.get('heading-only')?.title).toBe('Heading Title'); + expect(index.get('heading-only')?.icon).toBeUndefined(); + expect(index.get('plain')?.title).toBe('plain'); + } finally { + await handle.unsubscribe(); + } + }); + test('initial scan preserves uppercase .MD/.MDX extension casing', async () => { writeFileSync(resolve(contentDir, 'Upper.MD'), '# Upper\n'); writeFileSync(resolve(contentDir, 'Mixed.MdX'), '# Mixed\n'); @@ -519,6 +540,61 @@ describe('startWatcher file index', () => { expect(index.get('new-name')?.size).toBe(Buffer.byteLength('# Renamed\n', 'utf-8')); }); + test('file index caches title + icon on create/update/rename/conflict events', () => { + const { updateFileIndex } = require('./file-watcher.ts'); + const index = new Map(); + + updateFileIndex( + { + kind: 'create' as const, + path: resolve(contentDir, 'doc.md'), + docName: 'doc', + content: '---\ntitle: Created\nicon: 🚀\n---\n\nBody\n', + }, + index, + ); + expect(index.get('doc')?.title).toBe('Created'); + expect(index.get('doc')?.icon).toBe('🚀'); + + updateFileIndex( + { + kind: 'update' as const, + path: resolve(contentDir, 'doc.md'), + docName: 'doc', + content: '# Edited Title\n', + }, + index, + ); + expect(index.get('doc')?.title).toBe('Edited Title'); + expect(index.get('doc')?.icon).toBeUndefined(); + + updateFileIndex( + { + kind: 'rename' as const, + oldPath: resolve(contentDir, 'doc.md'), + newPath: resolve(contentDir, 'renamed.md'), + oldDocName: 'doc', + newDocName: 'renamed', + content: '---\ntitle: Renamed Title\nicon: 🔖\n---\n\nBody\n', + }, + index, + ); + expect(index.get('renamed')?.title).toBe('Renamed Title'); + expect(index.get('renamed')?.icon).toBe('🔖'); + + updateFileIndex( + { + kind: 'conflict' as const, + path: resolve(contentDir, 'renamed.md'), + docName: 'renamed', + content: '---\ntitle: Conflicted\nicon: ⚠️\n---\n<<<<<<< HEAD\na\n=======\nb\n>>>>>>> x\n', + }, + index, + ); + expect(index.get('renamed')?.title).toBe('Conflicted'); + expect(index.get('renamed')?.icon).toBe('⚠️'); + }); + test('getFileIndex returns empty map when no .md files exist', async () => { const handle = await startWatcher(contentDir, async () => {}); try { diff --git a/packages/server/src/file-watcher.ts b/packages/server/src/file-watcher.ts index 6b85dbafa..cee6539a9 100644 --- a/packages/server/src/file-watcher.ts +++ b/packages/server/src/file-watcher.ts @@ -14,6 +14,7 @@ import { } from './doc-extensions.ts'; import { classifyFsPath, normalizeFsPath } from './fs-traced.ts'; import { getLogger } from './logger.ts'; +import { extractPageIcon, extractPageTitle } from './page-identity.ts'; import { toPosix } from './path-utils.ts'; import { isWithinContentDir } from './persistence.ts'; import { containsConflictMarkers } from './reconciliation.ts'; @@ -79,6 +80,8 @@ export interface FileIndexEntry { inode: number; aliases: string[]; kind: 'markdown' | 'file'; + title?: string; + icon?: string; } export interface FolderIndexEntry { @@ -88,6 +91,13 @@ export interface FolderIndexEntry { inode: number; } +function derivePageMeta( + content: string, + docName: string, +): { title: string; icon: string | undefined } { + return { title: extractPageTitle(content, docName), icon: extractPageIcon(content) }; +} + function markdownIndexView( inner: ReadonlyMap, ): ReadonlyMap { @@ -554,6 +564,7 @@ async function seedLastKnownHashes( inode: canonStat.ino, aliases: [aliasDocName], kind: 'markdown', + ...derivePageMeta(content, canonicalDocName), }); } catch (err) { const code = (err as NodeJS.ErrnoException).code; @@ -626,6 +637,7 @@ async function seedLastKnownHashes( inode: lst.ino, aliases: [], kind: 'markdown', + ...derivePageMeta(content, docName), }); } catch (err) { const code = (err as NodeJS.ErrnoException).code; @@ -717,6 +729,7 @@ export function updateFileIndex(event: DiskEvent, fileIndex: Map