From cdebfea4501d57ba9fa5e86b1d07a4747f13e496 Mon Sep 17 00:00:00 2001 From: Bence Laky Date: Wed, 29 Sep 2021 15:22:08 +0200 Subject: [PATCH 01/14] added missing null check to dot sub scan --- spec/generic/ops.spec.js | 11 +++++++++++ src/lokijs.js | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/spec/generic/ops.spec.js b/spec/generic/ops.spec.js index e3952cf3..337a6f23 100644 --- a/spec/generic/ops.spec.js +++ b/spec/generic/ops.spec.js @@ -359,6 +359,17 @@ describe("Individual operator tests", function() { expect(coll.find({ "c": { $eq: undefined } }).length).toEqual(4); }); + it('query nested documents with nullable object', function() { + var db = new loki('db'); + var coll = db.addCollection('coll'); + + coll.insert({ a: null, b: 5, c: { a: 1 }}); + coll.insert({ a: "11", b: 5, c: { a: 1 }}); + coll.insert({ a: "11", b: 5, c: null}); + + expect(coll.find({ "c.a": { $eq: 1 } }).length).toEqual(2); + }); + it('$exists ops work as expected', function() { var db = new loki('db'); var coll = db.addCollection('coll'); diff --git a/src/lokijs.js b/src/lokijs.js index fb35f141..ef37400e 100644 --- a/src/lokijs.js +++ b/src/lokijs.js @@ -415,7 +415,7 @@ var valueFound = false; var element; - if (typeof root === 'object' && path in root) { + if (root !== null && typeof root === 'object' && path in root) { element = root[path]; } if (pathOffset + 1 >= paths.length) { From f1ecdc4e22fd092ade703062a7d3083f3ebe92b7 Mon Sep 17 00:00:00 2001 From: radex Date: Thu, 30 Sep 2021 14:16:45 +0200 Subject: [PATCH 02/14] [IncrementalIDB] PoC IDB preloading --- src/incremental-indexeddb-adapter.js | 64 ++++++++++++++++++---------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/src/incremental-indexeddb-adapter.js b/src/incremental-indexeddb-adapter.js index 19d06e12..35d66126 100644 --- a/src/incremental-indexeddb-adapter.js +++ b/src/incremental-indexeddb-adapter.js @@ -529,24 +529,8 @@ } this.idbInitInProgress = true; - var openRequest = indexedDB.open(dbname, 1); - - openRequest.onupgradeneeded = function(e) { - var db = e.target.result; - DEBUG && console.log('onupgradeneeded, old version: ' + e.oldVersion); - - if (e.oldVersion < 1) { - // Version 1 - Initial - Create database - db.createObjectStore('LokiIncrementalData', { keyPath: "key" }); - } else { - // Unknown version - throw new Error("Invalid old version " + e.oldVersion + " for IndexedDB upgrade"); - } - }; - - openRequest.onsuccess = function(e) { + function onDidOpen(db) { that.idbInitInProgress = false; - var db = e.target.result; that.idb = db; if (!db.objectStoreNames.contains('LokiIncrementalData')) { @@ -579,6 +563,33 @@ }; onSuccess(); + } + + if (window.__idb && window.__idb.idb) { + console.warn('using preloaded idb') + onDidOpen(window.__idb.idb); + return; + } + + var openRequest = indexedDB.open(dbname, 1); + + openRequest.onupgradeneeded = function(e) { + var db = e.target.result; + DEBUG && console.log('onupgradeneeded, old version: ' + e.oldVersion); + + if (e.oldVersion < 1) { + // Version 1 - Initial - Create database + db.createObjectStore('LokiIncrementalData', { keyPath: "key" }); + } else { + // Unknown version + throw new Error("Invalid old version " + e.oldVersion + " for IndexedDB upgrade"); + } + }; + + openRequest.onsuccess = function(e) { + that.idbInitInProgress = false; + var db = e.target.result; + onDidOpen(db); }; openRequest.onblocked = function(e) { @@ -667,16 +678,25 @@ } function getAllKeys() { - idbReq(store.getAllKeys(), function(e) { - var keys = e.target.result.sort(); + function onDidGetKeys(keys) { + keys.sort(); if (keys.length > 100) { getMegachunks(keys); } else { getAllChunks(); } - }, function(e) { - callback(e); - }); + } + + if (window.__idb && window.__idb.keys) { + console.warn('using preloaded keys') + onDidGetKeys(window.__idb.keys); + } else { + idbReq(store.getAllKeys(), function(e) { + onDidGetKeys(e.target.result); + }, function(e) { + callback(e); + }); + } if (that.options.onFetchStart) { that.options.onFetchStart(); From 321fff5ad91622312f2572f7d2689978a25af4c3 Mon Sep 17 00:00:00 2001 From: radex Date: Thu, 30 Sep 2021 14:17:03 +0200 Subject: [PATCH 03/14] [IncrementalIDB] Tunable megachunk waving --- src/incremental-indexeddb-adapter.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/incremental-indexeddb-adapter.js b/src/incremental-indexeddb-adapter.js index 35d66126..a625f2b1 100644 --- a/src/incremental-indexeddb-adapter.js +++ b/src/incremental-indexeddb-adapter.js @@ -53,7 +53,7 @@ this.mode = "incremental"; this.options = options || {}; this.chunkSize = 100; - this.megachunkCount = this.options.megachunkCount || 20; + this.megachunkCount = this.options.megachunkCount || 24; this.idb = null; // will be lazily loaded on first operation that needs it this._prevLokiVersionId = null; this._prevCollectionVersionIds = {}; @@ -647,11 +647,14 @@ // Stagger megachunk requests - first one half, then request the second when first one comes // back. This further improves concurrency. - function requestMegachunk(index) { + const megachunkWaves = 2 + const megachunksPerWave = megachunkCount / megachunkWaves + console.log(megachunkWaves, megachunksPerWave) + function requestMegachunk(index, wave) { var keyRange = keyRanges[index]; idbReq(store.getAll(keyRange), function(e) { - if (index < megachunkCount / 2) { - requestMegachunk(index + megachunkCount / 2); + if (wave < megachunkWaves) { + requestMegachunk(index + megachunksPerWave, wave + 1); } processMegachunk(e, index, keyRange); @@ -660,8 +663,8 @@ }); } - for (var i = 0; i < megachunkCount / 2; i += 1) { - requestMegachunk(i); + for (var i = 0; i < megachunksPerWave; i += 1) { + requestMegachunk(i, 1); } } From 0083d2d4fd1efcdb4f03d6d3b162f55db8a12816 Mon Sep 17 00:00:00 2001 From: radex Date: Fri, 1 Oct 2021 10:13:40 +0200 Subject: [PATCH 04/14] [IncrementalIDB] Process megachunks in random order maybe? --- src/incremental-indexeddb-adapter.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/incremental-indexeddb-adapter.js b/src/incremental-indexeddb-adapter.js index a625f2b1..a6a014c2 100644 --- a/src/incremental-indexeddb-adapter.js +++ b/src/incremental-indexeddb-adapter.js @@ -565,11 +565,11 @@ onSuccess(); } - if (window.__idb && window.__idb.idb) { - console.warn('using preloaded idb') - onDidOpen(window.__idb.idb); - return; - } + // if (window.__idb && window.__idb.idb) { + // console.warn('using preloaded idb') + // onDidOpen(window.__idb.idb); + // return; + // } var openRequest = indexedDB.open(dbname, 1); @@ -623,21 +623,21 @@ // while IDB process is still fetching data. Details: https://github.com/techfort/LokiJS/pull/874 function getMegachunks(keys) { var megachunkCount = that.megachunkCount; - var keyRanges = createKeyRanges(keys, megachunkCount); + var keyRanges = createKeyRanges(keys, megachunkCount).sort(() => Math.random() - 0.5); var allChunks = []; var megachunksReceived = 0; function processMegachunk(e, megachunkIndex, keyRange) { - // var debugMsg = 'processing chunk ' + megachunkIndex + ' (' + keyRange.lower + ' -- ' + keyRange.upper + ')' - // DEBUG && console.time(debugMsg); + var debugMsg = 'processing chunk ' + megachunkIndex + ' (' + keyRange.lower + ' -- ' + keyRange.upper + ')' + DEBUG && console.time(debugMsg); var megachunk = e.target.result; megachunk.forEach(function (chunk, i) { parseChunk(chunk, deserializeChunk); allChunks.push(chunk); megachunk[i] = null; // gc }); - // DEBUG && console.timeEnd(debugMsg); + DEBUG && console.timeEnd(debugMsg); megachunksReceived += 1; if (megachunksReceived === megachunkCount) { From ebbc1ff2eac82292cca95b45a868887e61a459f2 Mon Sep 17 00:00:00 2001 From: radex Date: Fri, 1 Oct 2021 10:26:07 +0200 Subject: [PATCH 05/14] [IncrementalIDB] Lazy JSON parse&deserialize of chunks --- src/incremental-indexeddb-adapter.js | 48 ++++++++++++++++---------- src/lokijs.js | 50 ++++++++++++++++++---------- 2 files changed, 62 insertions(+), 36 deletions(-) diff --git a/src/incremental-indexeddb-adapter.js b/src/incremental-indexeddb-adapter.js index a6a014c2..398b63a8 100644 --- a/src/incremental-indexeddb-adapter.js +++ b/src/incremental-indexeddb-adapter.js @@ -427,11 +427,11 @@ // repack chunks into a map chunks = chunksToMap(chunks); - var loki = chunks.loki; + var loki = JSON.parse(chunks.loki); chunks.loki = null; // gc // populate collections with data - populateLoki(loki, chunks.chunkMap); + populateLoki(loki, chunks.chunkMap, that.options.deserializeChunk); chunks = null; // gc // remember previous version IDs @@ -497,25 +497,37 @@ return { loki: loki, chunkMap: chunkMap }; } - function populateLoki(loki, chunkMap) { + function populateLoki(loki, chunkMap, deserializeChunk) { + // console.warn(chunkMap) + // console.warn(loki.collections.length) loki.collections.forEach(function populateCollection(collectionStub, i) { var chunkCollection = chunkMap[collectionStub.name]; if (chunkCollection) { if (!chunkCollection.metadata) { throw new Error("Corrupted database - missing metadata chunk for " + collectionStub.name); } - var collection = chunkCollection.metadata; + var collection = JSON.parse(chunkCollection.metadata); chunkCollection.metadata = null; loki.collections[i] = collection; - var dataChunks = chunkCollection.dataChunks; - dataChunks.forEach(function populateChunk(chunk, i) { - chunk.forEach(function(doc) { - collection.data.push(doc); + + const getData = () => { + var data = [] + var dataChunks = chunkCollection.dataChunks; + dataChunks.forEach(function populateChunk(chunk, i) { + chunk = JSON.parse(chunk); + if (deserializeChunk) { + chunk = deserializeChunk(collection.name, chunk); + } + chunk.forEach(function(doc) { + data.push(doc); + }); + dataChunks[i] = null; }); - dataChunks[i] = null; - }); + return data + } + collection.getData = getData; } }); } @@ -710,14 +722,14 @@ }; function parseChunk(chunk, deserializeChunk) { - chunk.value = JSON.parse(chunk.value); - if (deserializeChunk) { - var segments = chunk.key.split('.'); - if (segments.length === 3 && segments[1] === 'chunk') { - var collectionName = segments[0]; - chunk.value = deserializeChunk(collectionName, chunk.value); - } - } + // chunk.value = JSON.parse(chunk.value); + // if (deserializeChunk) { + // var segments = chunk.key.split('.'); + // if (segments.length === 3 && segments[1] === 'chunk') { + // var collectionName = segments[0]; + // chunk.value = deserializeChunk(collectionName, chunk.value); + // } + // } } /** diff --git a/src/lokijs.js b/src/lokijs.js index fb35f141..5639b075 100644 --- a/src/lokijs.js +++ b/src/lokijs.js @@ -1774,27 +1774,41 @@ copyColl.dirty = false; } - // load each element individually - clen = coll.data.length; - j = 0; - if (options && options.hasOwnProperty(coll.name)) { - loader = makeLoader(coll); - - for (j; j < clen; j++) { - collObj = loader(coll.data[j]); - copyColl.data[j] = collObj; - copyColl.addAutoUpdateObserver(collObj); - if (!copyColl.disableFreeze) { - deepFreeze(copyColl.data[j]); + + if (coll.getData) { + const {name, getData}=coll; + Object.defineProperty(copyColl, 'data', { + get() { + console.log(`hello ${name}`); + console.log(this) + const data = getData() + Object.defineProperty(this, 'data', { value: data }) + return data; } - } + }); } else { + // load each element individually + clen = coll.data.length; + j = 0; + if (options && options.hasOwnProperty(coll.name)) { + loader = makeLoader(coll); + + for (j; j < clen; j++) { + collObj = loader(coll.data[j]); + copyColl.data[j] = collObj; + copyColl.addAutoUpdateObserver(collObj); + if (!copyColl.disableFreeze) { + deepFreeze(copyColl.data[j]); + } + } + } else { - for (j; j < clen; j++) { - copyColl.data[j] = coll.data[j]; - copyColl.addAutoUpdateObserver(copyColl.data[j]); - if (!copyColl.disableFreeze) { - deepFreeze(copyColl.data[j]); + for (j; j < clen; j++) { + copyColl.data[j] = coll.data[j]; + copyColl.addAutoUpdateObserver(copyColl.data[j]); + if (!copyColl.disableFreeze) { + deepFreeze(copyColl.data[j]); + } } } } From 6676736f55b7eedcd151e1b77b83162e8833928d Mon Sep 17 00:00:00 2001 From: radex Date: Fri, 1 Oct 2021 12:03:15 +0200 Subject: [PATCH 06/14] [IncrementalIDB] Classify chunks earlier --- src/incremental-indexeddb-adapter.js | 65 ++++++++++++++++++---------- 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/src/incremental-indexeddb-adapter.js b/src/incremental-indexeddb-adapter.js index 398b63a8..f24200e0 100644 --- a/src/incremental-indexeddb-adapter.js +++ b/src/incremental-indexeddb-adapter.js @@ -427,7 +427,7 @@ // repack chunks into a map chunks = chunksToMap(chunks); - var loki = JSON.parse(chunks.loki); + var loki = chunks.loki; chunks.loki = null; // gc // populate collections with data @@ -456,38 +456,30 @@ sortChunksInPlace(chunks); - chunks.forEach(function(object) { - var key = object.key; - var value = object.value; - if (key === "loki") { + chunks.forEach(function(chunk) { + var type = chunk.type; + var value = chunk.value; + var name = chunk.collectionName; + if (type === "loki") { loki = value; - return; - } else if (key.includes(".")) { - var keySegments = key.split("."); - if (keySegments.length === 3 && keySegments[1] === "chunk") { - var colName = keySegments[0]; - if (chunkMap[colName]) { - chunkMap[colName].dataChunks.push(value); + } else if (type === "data") { + if (chunkMap[name]) { + chunkMap[name].dataChunks.push(value); } else { - chunkMap[colName] = { + chunkMap[name] = { metadata: null, dataChunks: [value], }; } - return; - } else if (keySegments.length === 2 && keySegments[1] === "metadata") { - var name = keySegments[0]; + } else if (type === "metadata") { if (chunkMap[name]) { chunkMap[name].metadata = value; } else { chunkMap[name] = { metadata: value, dataChunks: [] }; } - return; - } + } else { + throw new Error("unreachable"); } - - console.error("Unknown chunk " + key); - throw new Error("Corrupted database - unknown chunk found"); }); if (!loki) { @@ -506,7 +498,7 @@ if (!chunkCollection.metadata) { throw new Error("Corrupted database - missing metadata chunk for " + collectionStub.name); } - var collection = JSON.parse(chunkCollection.metadata); + var collection = chunkCollection.metadata; chunkCollection.metadata = null; loki.collections[i] = collection; @@ -721,7 +713,36 @@ getAllKeys(); }; + function classifyChunk(chunk) { + var key = chunk.key; + + if (key === 'loki') { + chunk.type = 'loki'; + return; + } else if (key.includes('.')) { + var keySegments = key.split("."); + if (keySegments.length === 3 && keySegments[1] === "chunk") { + chunk.type = 'data'; + chunk.collectionName = keySegments[0]; + return; + } else if (keySegments.length === 2 && keySegments[1] === "metadata") { + chunk.type = 'metadata'; + chunk.collectionName = keySegments[0]; + return; + } + } + + console.error("Unknown chunk " + key); + throw new Error("Corrupted database - unknown chunk found"); + } + function parseChunk(chunk, deserializeChunk) { + classifyChunk(chunk); + + if (chunk.type !== 'data') { + chunk.value = JSON.parse(chunk.value); + } + // chunk.value = JSON.parse(chunk.value); // if (deserializeChunk) { // var segments = chunk.key.split('.'); From 79a1435fb9566cdeff164dd5c46f2adb949aa2f0 Mon Sep 17 00:00:00 2001 From: radex Date: Fri, 1 Oct 2021 12:15:49 +0200 Subject: [PATCH 07/14] [IncrementalIDB] Specified lazy collections --- src/incremental-indexeddb-adapter.js | 42 ++++++++++++++-------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/incremental-indexeddb-adapter.js b/src/incremental-indexeddb-adapter.js index f24200e0..d38e209a 100644 --- a/src/incremental-indexeddb-adapter.js +++ b/src/incremental-indexeddb-adapter.js @@ -54,6 +54,7 @@ this.options = options || {}; this.chunkSize = 100; this.megachunkCount = this.options.megachunkCount || 24; + this.lazyCollections = this.options.lazyCollections || []; this.idb = null; // will be lazily loaded on first operation that needs it this._prevLokiVersionId = null; this._prevCollectionVersionIds = {}; @@ -431,7 +432,7 @@ chunks.loki = null; // gc // populate collections with data - populateLoki(loki, chunks.chunkMap, that.options.deserializeChunk); + populateLoki(loki, chunks.chunkMap, that.options.deserializeChunk, that.lazyCollections); chunks = null; // gc // remember previous version IDs @@ -489,28 +490,29 @@ return { loki: loki, chunkMap: chunkMap }; } - function populateLoki(loki, chunkMap, deserializeChunk) { - // console.warn(chunkMap) - // console.warn(loki.collections.length) + function populateLoki(loki, chunkMap, deserializeChunk, lazyCollections) { loki.collections.forEach(function populateCollection(collectionStub, i) { - var chunkCollection = chunkMap[collectionStub.name]; + var name = collectionStub.name; + var chunkCollection = chunkMap[name]; if (chunkCollection) { if (!chunkCollection.metadata) { - throw new Error("Corrupted database - missing metadata chunk for " + collectionStub.name); + throw new Error("Corrupted database - missing metadata chunk for " + name); } var collection = chunkCollection.metadata; chunkCollection.metadata = null; - loki.collections[i] = collection; + var isLazy = lazyCollections.includes(name); const getData = () => { var data = [] var dataChunks = chunkCollection.dataChunks; dataChunks.forEach(function populateChunk(chunk, i) { + if (isLazy) { chunk = JSON.parse(chunk); if (deserializeChunk) { - chunk = deserializeChunk(collection.name, chunk); + chunk = deserializeChunk(name, chunk); + } } chunk.forEach(function(doc) { data.push(doc); @@ -621,6 +623,7 @@ var store = tx.objectStore('LokiIncrementalData'); var deserializeChunk = this.options.deserializeChunk; + var lazyCollections = this.lazyCollections; // If there are a lot of chunks (>100), don't request them all in one go, but in multiple // "megachunks" (chunks of chunks). This improves concurrency, as main thread is already busy @@ -637,7 +640,7 @@ DEBUG && console.time(debugMsg); var megachunk = e.target.result; megachunk.forEach(function (chunk, i) { - parseChunk(chunk, deserializeChunk); + parseChunk(chunk, deserializeChunk, lazyCollections); allChunks.push(chunk); megachunk[i] = null; // gc }); @@ -676,7 +679,7 @@ idbReq(store.getAll(), function(e) { var allChunks = e.target.result; allChunks.forEach(function (chunk) { - parseChunk(chunk, deserializeChunk); + parseChunk(chunk, deserializeChunk, lazyCollections); }); callback(allChunks); }, function(e) { @@ -736,21 +739,18 @@ throw new Error("Corrupted database - unknown chunk found"); } - function parseChunk(chunk, deserializeChunk) { + function parseChunk(chunk, deserializeChunk, lazyCollections) { classifyChunk(chunk); - if (chunk.type !== 'data') { + var isData = chunk.type === 'data' + var isLazy = lazyCollections.includes(chunk.collectionName); + + if (!(isData && isLazy)) { chunk.value = JSON.parse(chunk.value); } - - // chunk.value = JSON.parse(chunk.value); - // if (deserializeChunk) { - // var segments = chunk.key.split('.'); - // if (segments.length === 3 && segments[1] === 'chunk') { - // var collectionName = segments[0]; - // chunk.value = deserializeChunk(collectionName, chunk.value); - // } - // } + if (deserializeChunk && isData && !isLazy) { + chunk.value = deserializeChunk(chunk.collectionName, chunk.value); + } } /** From 49f08441b93012ab7843f065addd0da62ed8486c Mon Sep 17 00:00:00 2001 From: radex Date: Fri, 1 Oct 2021 12:28:25 +0200 Subject: [PATCH 08/14] [IncrementalIDB] split keys only N times --- src/incremental-indexeddb-adapter.js | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/src/incremental-indexeddb-adapter.js b/src/incremental-indexeddb-adapter.js index d38e209a..98af06e4 100644 --- a/src/incremental-indexeddb-adapter.js +++ b/src/incremental-indexeddb-adapter.js @@ -727,6 +727,7 @@ if (keySegments.length === 3 && keySegments[1] === "chunk") { chunk.type = 'data'; chunk.collectionName = keySegments[0]; + chunk.index = parseInt(keySegments[2], 10); return; } else if (keySegments.length === 2 && keySegments[1] === "metadata") { chunk.type = 'metadata'; @@ -814,27 +815,11 @@ return Math.random().toString(36).substring(2); } - function _getSortKey(object) { - var key = object.key; - if (key.includes(".")) { - var segments = key.split("."); - if (segments.length === 3 && segments[1] === "chunk") { - return parseInt(segments[2], 10); - } - } - - return -1; // consistent type must be returned - } - function sortChunksInPlace(chunks) { // sort chunks in place to load data in the right order (ascending loki ids) // on both Safari and Chrome, we'll get chunks in order like this: 0, 1, 10, 100... chunks.sort(function(a, b) { - var aKey = _getSortKey(a), - bKey = _getSortKey(b); - if (aKey < bKey) return -1; - if (aKey > bKey) return 1; - return 0; + return a.index - b.index }); } From f1606241999cd185942fb9070d9eb19cb976f8c5 Mon Sep 17 00:00:00 2001 From: radex Date: Wed, 13 Oct 2021 12:03:01 +0200 Subject: [PATCH 09/14] [IncrementalIDB] Clean up preloads --- src/incremental-indexeddb-adapter.js | 53 +++++++++++++++------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/src/incremental-indexeddb-adapter.js b/src/incremental-indexeddb-adapter.js index 98af06e4..7f36ab0c 100644 --- a/src/incremental-indexeddb-adapter.js +++ b/src/incremental-indexeddb-adapter.js @@ -149,6 +149,7 @@ * @memberof IncrementalIndexedDBAdapter */ IncrementalIndexedDBAdapter.prototype.saveDatabase = function(dbname, getLokiCopy, callback) { + // throw new Error('save NOPE') var that = this; if (!this.idb) { @@ -466,18 +467,18 @@ } else if (type === "data") { if (chunkMap[name]) { chunkMap[name].dataChunks.push(value); - } else { + } else { chunkMap[name] = { - metadata: null, - dataChunks: [value], - }; - } + metadata: null, + dataChunks: [value], + }; + } } else if (type === "metadata") { - if (chunkMap[name]) { - chunkMap[name].metadata = value; - } else { - chunkMap[name] = { metadata: value, dataChunks: [] }; - } + if (chunkMap[name]) { + chunkMap[name].metadata = value; + } else { + chunkMap[name] = { metadata: value, dataChunks: [] }; + } } else { throw new Error("unreachable"); } @@ -509,8 +510,8 @@ var dataChunks = chunkCollection.dataChunks; dataChunks.forEach(function populateChunk(chunk, i) { if (isLazy) { - chunk = JSON.parse(chunk); - if (deserializeChunk) { + chunk = JSON.parse(chunk); + if (deserializeChunk) { chunk = deserializeChunk(name, chunk); } } @@ -571,11 +572,12 @@ onSuccess(); } - // if (window.__idb && window.__idb.idb) { - // console.warn('using preloaded idb') - // onDidOpen(window.__idb.idb); - // return; - // } + if (this.options.preloads && this.options.preloads.idb) { + DEBUG && console.log('using preloaded idb'); + onDidOpen(this.options.preloads.idb); + this.options.preloads.idb = null; + return; + } var openRequest = indexedDB.open(dbname, 1); @@ -630,21 +632,21 @@ // while IDB process is still fetching data. Details: https://github.com/techfort/LokiJS/pull/874 function getMegachunks(keys) { var megachunkCount = that.megachunkCount; - var keyRanges = createKeyRanges(keys, megachunkCount).sort(() => Math.random() - 0.5); + var keyRanges = createKeyRanges(keys, megachunkCount) var allChunks = []; var megachunksReceived = 0; function processMegachunk(e, megachunkIndex, keyRange) { - var debugMsg = 'processing chunk ' + megachunkIndex + ' (' + keyRange.lower + ' -- ' + keyRange.upper + ')' - DEBUG && console.time(debugMsg); + // var debugMsg = 'processing chunk ' + megachunkIndex + ' (' + keyRange.lower + ' -- ' + keyRange.upper + ')' + // DEBUG && console.time(debugMsg); var megachunk = e.target.result; megachunk.forEach(function (chunk, i) { parseChunk(chunk, deserializeChunk, lazyCollections); allChunks.push(chunk); megachunk[i] = null; // gc }); - DEBUG && console.timeEnd(debugMsg); + // DEBUG && console.timeEnd(debugMsg); megachunksReceived += 1; if (megachunksReceived === megachunkCount) { @@ -697,9 +699,10 @@ } } - if (window.__idb && window.__idb.keys) { - console.warn('using preloaded keys') - onDidGetKeys(window.__idb.keys); + if (that.options.preloads && that.options.preloads.keys) { + DEBUG && console.log('using preloaded keys'); + onDidGetKeys(that.options.preloads.keys); + that.options.preloads.keys = null; } else { idbReq(store.getAllKeys(), function(e) { onDidGetKeys(e.target.result); @@ -769,6 +772,8 @@ * @memberof IncrementalIndexedDBAdapter */ IncrementalIndexedDBAdapter.prototype.deleteDatabase = function(dbname, callback) { + // debugger + // throw new Error('delete NOPE') if (this.operationInProgress) { throw new Error("Error while deleting database - another operation is already in progress. Please use throttledSaves=true option on Loki object"); } From ad494478fc27942c21d07c8f2b99550b399751db Mon Sep 17 00:00:00 2001 From: radex Date: Wed, 13 Oct 2021 12:32:53 +0200 Subject: [PATCH 10/14] [IncrementalIDB] Clean up --- src/incremental-indexeddb-adapter.js | 25 +++++++++++++------------ src/lokijs.js | 18 ++++++++++-------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/incremental-indexeddb-adapter.js b/src/incremental-indexeddb-adapter.js index 7f36ab0c..35cbff9e 100644 --- a/src/incremental-indexeddb-adapter.js +++ b/src/incremental-indexeddb-adapter.js @@ -504,9 +504,11 @@ loki.collections[i] = collection; var isLazy = lazyCollections.includes(name); - - const getData = () => { - var data = [] + var lokiDeserializeCollectionChunks = function () { + if (isLazy) { + DEBUG && console.log("lazy loading " + name); + } + var data = []; var dataChunks = chunkCollection.dataChunks; dataChunks.forEach(function populateChunk(chunk, i) { if (isLazy) { @@ -520,9 +522,9 @@ }); dataChunks[i] = null; }); - return data - } - collection.getData = getData; + return data; + }; + collection.getData = lokiDeserializeCollectionChunks; } }); } @@ -632,7 +634,7 @@ // while IDB process is still fetching data. Details: https://github.com/techfort/LokiJS/pull/874 function getMegachunks(keys) { var megachunkCount = that.megachunkCount; - var keyRanges = createKeyRanges(keys, megachunkCount) + var keyRanges = createKeyRanges(keys, megachunkCount); var allChunks = []; var megachunksReceived = 0; @@ -656,9 +658,8 @@ // Stagger megachunk requests - first one half, then request the second when first one comes // back. This further improves concurrency. - const megachunkWaves = 2 - const megachunksPerWave = megachunkCount / megachunkWaves - console.log(megachunkWaves, megachunksPerWave) + var megachunkWaves = 2; + var megachunksPerWave = megachunkCount / megachunkWaves; function requestMegachunk(index, wave) { var keyRange = keyRanges[index]; idbReq(store.getAll(keyRange), function(e) { @@ -746,7 +747,7 @@ function parseChunk(chunk, deserializeChunk, lazyCollections) { classifyChunk(chunk); - var isData = chunk.type === 'data' + var isData = chunk.type === 'data'; var isLazy = lazyCollections.includes(chunk.collectionName); if (!(isData && isLazy)) { @@ -824,7 +825,7 @@ // sort chunks in place to load data in the right order (ascending loki ids) // on both Safari and Chrome, we'll get chunks in order like this: 0, 1, 10, 100... chunks.sort(function(a, b) { - return a.index - b.index + return a.index - b.index; }); } diff --git a/src/lokijs.js b/src/lokijs.js index 5639b075..1f4413b2 100644 --- a/src/lokijs.js +++ b/src/lokijs.js @@ -1774,17 +1774,20 @@ copyColl.dirty = false; } - if (coll.getData) { - const {name, getData}=coll; + if ((options && options.hasOwnProperty(coll.name)) || !copyColl.disableFreeze || copyColl.autoupdate) { + throw new Error("this collection cannot be loaded lazily: " + coll.name); + } + copyColl.getData = coll.getData; Object.defineProperty(copyColl, 'data', { - get() { - console.log(`hello ${name}`); - console.log(this) - const data = getData() - Object.defineProperty(this, 'data', { value: data }) + /* jshint loopfunc:true */ + get: function() { + var data = this.getData(); + this.getData = null; + Object.defineProperty(this, 'data', { value: data }); return data; } + /* jshint loopfunc:false */ }); } else { // load each element individually @@ -1802,7 +1805,6 @@ } } } else { - for (j; j < clen; j++) { copyColl.data[j] = coll.data[j]; copyColl.addAutoUpdateObserver(copyColl.data[j]); From a2d64874c5dc4b959ea00a4c79a5dc6bc80cc79e Mon Sep 17 00:00:00 2001 From: radex Date: Fri, 15 Oct 2021 13:19:50 +0200 Subject: [PATCH 11/14] [IncrementalIDB] clean up --- src/incremental-indexeddb-adapter.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/incremental-indexeddb-adapter.js b/src/incremental-indexeddb-adapter.js index 35cbff9e..ef77276e 100644 --- a/src/incremental-indexeddb-adapter.js +++ b/src/incremental-indexeddb-adapter.js @@ -149,7 +149,6 @@ * @memberof IncrementalIndexedDBAdapter */ IncrementalIndexedDBAdapter.prototype.saveDatabase = function(dbname, getLokiCopy, callback) { - // throw new Error('save NOPE') var that = this; if (!this.idb) { @@ -505,9 +504,7 @@ var isLazy = lazyCollections.includes(name); var lokiDeserializeCollectionChunks = function () { - if (isLazy) { - DEBUG && console.log("lazy loading " + name); - } + DEBUG && isLazy && console.log("lazy loading " + name); var data = []; var dataChunks = chunkCollection.dataChunks; dataChunks.forEach(function populateChunk(chunk, i) { @@ -773,8 +770,6 @@ * @memberof IncrementalIndexedDBAdapter */ IncrementalIndexedDBAdapter.prototype.deleteDatabase = function(dbname, callback) { - // debugger - // throw new Error('delete NOPE') if (this.operationInProgress) { throw new Error("Error while deleting database - another operation is already in progress. Please use throttledSaves=true option on Loki object"); } From 35d6c76a699912b042bc1a276cf952dc102ff59c Mon Sep 17 00:00:00 2001 From: radex Date: Fri, 15 Oct 2021 13:57:50 +0200 Subject: [PATCH 12/14] [IncrementalIDB] Fix critical regression with chunk ordering at load time --- src/incremental-indexeddb-adapter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/incremental-indexeddb-adapter.js b/src/incremental-indexeddb-adapter.js index ef77276e..6799250a 100644 --- a/src/incremental-indexeddb-adapter.js +++ b/src/incremental-indexeddb-adapter.js @@ -820,7 +820,7 @@ // sort chunks in place to load data in the right order (ascending loki ids) // on both Safari and Chrome, we'll get chunks in order like this: 0, 1, 10, 100... chunks.sort(function(a, b) { - return a.index - b.index; + return (a.index || 0) - (b.index || 0); }); } From b01a3399b6040b34bf265d19f766a3683ba65705 Mon Sep 17 00:00:00 2001 From: radex Date: Fri, 15 Oct 2021 14:01:58 +0200 Subject: [PATCH 13/14] [IncrementalIDB] Add test --- spec/incrementalidb.html | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/spec/incrementalidb.html b/spec/incrementalidb.html index fa446422..ed60b836 100644 --- a/spec/incrementalidb.html +++ b/spec/incrementalidb.html @@ -269,6 +269,17 @@

IncrementalIDB tests and benchmark

trace('ok') } + trace('==> lazy collection deserialization') + + { + let db2 = new loki('incremental_idb_tester', { adapter: new IncrementalIndexedDBAdapter({ + lazyCollections: ['test_collection'] + }) }); + await saveAndCheckDatabaseCopyIntegrity(db2); + + trace('ok') + } + trace('==> long running fuzz tests') function fuzz(dbToFuzz) { From 534882e9112cdb218ef00d0f517d61d3d4f708f9 Mon Sep 17 00:00:00 2001 From: radex Date: Fri, 29 Oct 2021 15:47:52 +0200 Subject: [PATCH 14/14] [IncrementalIDB] Remove preloads --- src/incremental-indexeddb-adapter.js | 64 +++++++++++----------------- 1 file changed, 24 insertions(+), 40 deletions(-) diff --git a/src/incremental-indexeddb-adapter.js b/src/incremental-indexeddb-adapter.js index 6799250a..eddd73e1 100644 --- a/src/incremental-indexeddb-adapter.js +++ b/src/incremental-indexeddb-adapter.js @@ -48,6 +48,8 @@ * Expects an array of Loki documents as the return value * @param {number} options.megachunkCount Number of parallel requests for data when loading database. * Can be tuned for a specific application + * @param {array} options.lazyCollections Names of collections that should be deserialized lazily + * Only use this for collections that aren't used at launch */ function IncrementalIndexedDBAdapter(options) { this.mode = "incremental"; @@ -535,8 +537,24 @@ } this.idbInitInProgress = true; - function onDidOpen(db) { + var openRequest = indexedDB.open(dbname, 1); + + openRequest.onupgradeneeded = function(e) { + var db = e.target.result; + DEBUG && console.log('onupgradeneeded, old version: ' + e.oldVersion); + + if (e.oldVersion < 1) { + // Version 1 - Initial - Create database + db.createObjectStore('LokiIncrementalData', { keyPath: "key" }); + } else { + // Unknown version + throw new Error("Invalid old version " + e.oldVersion + " for IndexedDB upgrade"); + } + }; + + openRequest.onsuccess = function(e) { that.idbInitInProgress = false; + var db = e.target.result; that.idb = db; if (!db.objectStoreNames.contains('LokiIncrementalData')) { @@ -569,34 +587,6 @@ }; onSuccess(); - } - - if (this.options.preloads && this.options.preloads.idb) { - DEBUG && console.log('using preloaded idb'); - onDidOpen(this.options.preloads.idb); - this.options.preloads.idb = null; - return; - } - - var openRequest = indexedDB.open(dbname, 1); - - openRequest.onupgradeneeded = function(e) { - var db = e.target.result; - DEBUG && console.log('onupgradeneeded, old version: ' + e.oldVersion); - - if (e.oldVersion < 1) { - // Version 1 - Initial - Create database - db.createObjectStore('LokiIncrementalData', { keyPath: "key" }); - } else { - // Unknown version - throw new Error("Invalid old version " + e.oldVersion + " for IndexedDB upgrade"); - } - }; - - openRequest.onsuccess = function(e) { - that.idbInitInProgress = false; - var db = e.target.result; - onDidOpen(db); }; openRequest.onblocked = function(e) { @@ -697,17 +687,11 @@ } } - if (that.options.preloads && that.options.preloads.keys) { - DEBUG && console.log('using preloaded keys'); - onDidGetKeys(that.options.preloads.keys); - that.options.preloads.keys = null; - } else { - idbReq(store.getAllKeys(), function(e) { - onDidGetKeys(e.target.result); - }, function(e) { - callback(e); - }); - } + idbReq(store.getAllKeys(), function(e) { + onDidGetKeys(e.target.result); + }, function(e) { + callback(e); + }); if (that.options.onFetchStart) { that.options.onFetchStart();