diff --git a/.changeset/sqlite-vec-graceful.md b/.changeset/sqlite-vec-graceful.md new file mode 100644 index 0000000..46d31c9 --- /dev/null +++ b/.changeset/sqlite-vec-graceful.md @@ -0,0 +1,5 @@ +--- +"@enbox/memoryd": patch +--- + +Gracefully handle sqlite-vec extension loading failure — falls back to FTS5-only search when the SQLite build lacks dynamic extension loading support (e.g. some Bun versions on macOS). diff --git a/src/cli/agent.ts b/src/cli/agent.ts index 066c927..d59aca1 100644 --- a/src/cli/agent.ts +++ b/src/cli/agent.ts @@ -221,10 +221,21 @@ export async function bootstrapSidecar( mkdirSync(dirname(cfg.sidecarPath), { recursive: true }); const sidecarDb = new SDB(cfg.sidecarPath, provider.dimensions); - const searchIndex = new SI(sidecarDb.db, provider); + + if (!sidecarDb.hasVectorSearch) { + console.error( + 'Warning: sqlite-vec could not be loaded — vector search is disabled.\n' + + ' Search will use keyword matching (FTS5) only.\n' + + ' This typically happens when the SQLite build lacks dynamic extension loading.\n' + + ' Upgrading Bun (bun upgrade) or installing from source may resolve this.', + ); + } + + const searchIndex = new SI(sidecarDb.db, provider, sidecarDb.hasVectorSearch); const compaction = new CE(ctx.memoryStore, ctx.taskStore, { - sidecarDb: sidecarDb.db, + sidecarDb : sidecarDb.db, searchIndex, + hasVectorSearch : sidecarDb.hasVectorSearch, }); ctx.sidecarDb = sidecarDb; diff --git a/src/core/compaction.ts b/src/core/compaction.ts index 4e24dbd..d201d06 100644 --- a/src/core/compaction.ts +++ b/src/core/compaction.ts @@ -67,6 +67,7 @@ export class CompactionEngine { private readonly sidecarDb? : Database; private readonly searchIndex? : SearchIndex; private readonly summarize? : SummarizeFn; + private readonly hasVectorSearch : boolean; constructor( memoryStore : MemoryStore, @@ -75,6 +76,7 @@ export class CompactionEngine { sidecarDb? : Database; searchIndex? : SearchIndex; summarize? : SummarizeFn; + hasVectorSearch? : boolean; }, ) { this.memoryStore = memoryStore; @@ -82,6 +84,7 @@ export class CompactionEngine { this.sidecarDb = opts?.sidecarDb; this.searchIndex = opts?.searchIndex; this.summarize = opts?.summarize; + this.hasVectorSearch = opts?.hasVectorSearch ?? true; } // ------------------------------------------------------------------------- @@ -120,7 +123,7 @@ export class CompactionEngine { ): Promise { const effectiveSummarize = summarize ?? this.summarize; - if (!this.sidecarDb) { + if (!this.sidecarDb || !this.hasVectorSearch) { return 0; } if (!effectiveSummarize) { diff --git a/src/sidecar/database.ts b/src/sidecar/database.ts index 61e4d49..e637e37 100644 --- a/src/sidecar/database.ts +++ b/src/sidecar/database.ts @@ -2,38 +2,62 @@ // SidecarDatabase — SQLite lifecycle management for the vector sidecar. // --------------------------------------------------------------------------- -import * as sqliteVec from 'sqlite-vec'; import { Database } from 'bun:sqlite'; export class SidecarDatabase { readonly db: Database; + /** + * Whether the sqlite-vec extension loaded successfully. + * When `false`, the sidecar operates in FTS-only mode — + * keyword search still works, but vector KNN is unavailable. + */ + readonly hasVectorSearch: boolean; + constructor(dbPath: string, dimensions: number = 768) { this.db = new Database(dbPath); - sqliteVec.load(this.db); + this.hasVectorSearch = this.tryLoadVec(); this.db.exec('PRAGMA journal_mode = WAL;'); this.createTables(dimensions); } - private createTables(dimensions: number): void { - // Vector search (sqlite-vec) - // vec0 does not support IF NOT EXISTS; use try/catch for idempotency. + /** + * Attempt to load the sqlite-vec extension. + * Returns `true` on success, `false` when the extension cannot load + * (e.g. the SQLite build lacks dynamic extension loading). + */ + private tryLoadVec(): boolean { try { - this.db.exec(` - CREATE VIRTUAL TABLE memory_embeddings USING vec0( - embedding float[${dimensions}], - +record_id TEXT NOT NULL, - +protocol_path TEXT NOT NULL, - +content_preview TEXT, - +category TEXT, - +collection TEXT, - +updated_at TEXT NOT NULL - ); - `); - } catch (e: unknown) { - // Table already exists — safe to ignore. - if (!(e instanceof Error) || !e.message.includes('already exists')) { - throw e; + + const sqliteVec = require('sqlite-vec') as { load: (db: Database) => void }; + sqliteVec.load(this.db); + return true; + } catch { + return false; + } + } + + private createTables(dimensions: number): void { + // Vector search (sqlite-vec) — only when the extension loaded. + if (this.hasVectorSearch) { + // vec0 does not support IF NOT EXISTS; use try/catch for idempotency. + try { + this.db.exec(` + CREATE VIRTUAL TABLE memory_embeddings USING vec0( + embedding float[${dimensions}], + +record_id TEXT NOT NULL, + +protocol_path TEXT NOT NULL, + +content_preview TEXT, + +category TEXT, + +collection TEXT, + +updated_at TEXT NOT NULL + ); + `); + } catch (e: unknown) { + // Table already exists — safe to ignore. + if (!(e instanceof Error) || !e.message.includes('already exists')) { + throw e; + } } } diff --git a/src/sidecar/search.ts b/src/sidecar/search.ts index a7f6025..e1799ef 100644 --- a/src/sidecar/search.ts +++ b/src/sidecar/search.ts @@ -39,33 +39,45 @@ export type IndexRecord = { // --------------------------------------------------------------------------- export class SearchIndex { + /** + * When `false`, the sidecar has no vec0 table — vector upserts and + * KNN queries are skipped, and search falls back to FTS5-only. + */ + readonly hasVectorSearch: boolean; + constructor( private readonly db: Database, private readonly embeddings: EmbeddingProvider, - ) {} + hasVectorSearch: boolean = true, + ) { + this.hasVectorSearch = hasVectorSearch; + } /** Upsert a record into both vector and FTS indexes. */ async upsert(record: IndexRecord): Promise { - const embedding = await this.embeddings.embed(record.content); - const now = new Date().toISOString(); - // Delete existing entries for this record_id (if updating) - this.db.prepare('DELETE FROM memory_embeddings WHERE record_id = ?').run(record.recordId); this.db.prepare('DELETE FROM memory_fts WHERE record_id = ?').run(record.recordId); - // Insert into vec0 - this.db.prepare(` - INSERT INTO memory_embeddings(embedding, record_id, protocol_path, content_preview, category, collection, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?) - `).run( - new Float32Array(embedding), - record.recordId, - record.protocolPath, - record.content.substring(0, 200), - record.category ?? null, - record.collection ?? null, - now, - ); + if (this.hasVectorSearch) { + const embedding = await this.embeddings.embed(record.content); + const now = new Date().toISOString(); + + this.db.prepare('DELETE FROM memory_embeddings WHERE record_id = ?').run(record.recordId); + + // Insert into vec0 + this.db.prepare(` + INSERT INTO memory_embeddings(embedding, record_id, protocol_path, content_preview, category, collection, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + `).run( + new Float32Array(embedding), + record.recordId, + record.protocolPath, + record.content.substring(0, 200), + record.category ?? null, + record.collection ?? null, + now, + ); + } // Insert into FTS5 this.db.prepare(` @@ -82,7 +94,9 @@ export class SearchIndex { /** Remove a record from both indexes. */ remove(recordId: string): void { - this.db.prepare('DELETE FROM memory_embeddings WHERE record_id = ?').run(recordId); + if (this.hasVectorSearch) { + this.db.prepare('DELETE FROM memory_embeddings WHERE record_id = ?').run(recordId); + } this.db.prepare('DELETE FROM memory_fts WHERE record_id = ?').run(recordId); } @@ -94,14 +108,17 @@ export class SearchIndex { const limit = opts?.limit ?? 10; const k = 60; // RRF constant - // 1. Vector KNN search - const queryEmbedding = await this.embeddings.embed(query); - const vectorResults = this.vectorSearch(new Float32Array(queryEmbedding), limit * 2, opts?.filters); + // 1. Vector KNN search (when available) + let vectorResults: RankedResult[] = []; + if (this.hasVectorSearch) { + const queryEmbedding = await this.embeddings.embed(query); + vectorResults = this.vectorSearch(new Float32Array(queryEmbedding), limit * 2, opts?.filters); + } // 2. FTS5 search const ftsResults = this.ftsSearch(query, limit * 2, opts?.filters); - // 3. Reciprocal Rank Fusion + // 3. Reciprocal Rank Fusion (degrades to FTS-only ranking when no vector results) const merged = this.reciprocalRankFusion(vectorResults, ftsResults, k); return merged.slice(0, limit); diff --git a/tests/sidecar/database.spec.ts b/tests/sidecar/database.spec.ts index 2e70028..6c49f0c 100644 --- a/tests/sidecar/database.spec.ts +++ b/tests/sidecar/database.spec.ts @@ -27,6 +27,11 @@ describe('SidecarDatabase', () => { expect(sidecar.db).toBeDefined(); }); + it('exposes hasVectorSearch property', () => { + sidecar = new SidecarDatabase(':memory:'); + expect(typeof sidecar.hasVectorSearch).toBe('boolean'); + }); + it('creates all expected tables', () => { sidecar = new SidecarDatabase(':memory:'); @@ -36,7 +41,10 @@ describe('SidecarDatabase', () => { const tableNames = tables.map(t => t.name); - expect(tableNames).toContain('memory_embeddings'); + // memory_embeddings only exists when sqlite-vec loaded + if (sidecar.hasVectorSearch) { + expect(tableNames).toContain('memory_embeddings'); + } expect(tableNames).toContain('memory_fts'); expect(tableNames).toContain('task_graph'); expect(tableNames).toContain('task_status');