diff --git a/spec/generic/ops.spec.js b/spec/generic/ops.spec.js index e3952cf3..337a6f23 100644 --- a/spec/generic/ops.spec.js +++ b/spec/generic/ops.spec.js @@ -359,6 +359,17 @@ describe("Individual operator tests", function() { expect(coll.find({ "c": { $eq: undefined } }).length).toEqual(4); }); + it('query nested documents with nullable object', function() { + var db = new loki('db'); + var coll = db.addCollection('coll'); + + coll.insert({ a: null, b: 5, c: { a: 1 }}); + coll.insert({ a: "11", b: 5, c: { a: 1 }}); + coll.insert({ a: "11", b: 5, c: null}); + + expect(coll.find({ "c.a": { $eq: 1 } }).length).toEqual(2); + }); + it('$exists ops work as expected', function() { var db = new loki('db'); var coll = db.addCollection('coll'); diff --git a/spec/incrementalidb.html b/spec/incrementalidb.html index fa446422..ed60b836 100644 --- a/spec/incrementalidb.html +++ b/spec/incrementalidb.html @@ -269,6 +269,17 @@

IncrementalIDB tests and benchmark

trace('ok') } + trace('==> lazy collection deserialization') + + { + let db2 = new loki('incremental_idb_tester', { adapter: new IncrementalIndexedDBAdapter({ + lazyCollections: ['test_collection'] + }) }); + await saveAndCheckDatabaseCopyIntegrity(db2); + + trace('ok') + } + trace('==> long running fuzz tests') function fuzz(dbToFuzz) { diff --git a/src/incremental-indexeddb-adapter.js b/src/incremental-indexeddb-adapter.js index 19d06e12..eddd73e1 100644 --- a/src/incremental-indexeddb-adapter.js +++ b/src/incremental-indexeddb-adapter.js @@ -48,12 +48,15 @@ * Expects an array of Loki documents as the return value * @param {number} options.megachunkCount Number of parallel requests for data when loading database. * Can be tuned for a specific application + * @param {array} options.lazyCollections Names of collections that should be deserialized lazily + * Only use this for collections that aren't used at launch */ function IncrementalIndexedDBAdapter(options) { this.mode = "incremental"; this.options = options || {}; this.chunkSize = 100; - this.megachunkCount = this.options.megachunkCount || 20; + this.megachunkCount = this.options.megachunkCount || 24; + this.lazyCollections = this.options.lazyCollections || []; this.idb = null; // will be lazily loaded on first operation that needs it this._prevLokiVersionId = null; this._prevCollectionVersionIds = {}; @@ -431,7 +434,7 @@ chunks.loki = null; // gc // populate collections with data - populateLoki(loki, chunks.chunkMap); + populateLoki(loki, chunks.chunkMap, that.options.deserializeChunk, that.lazyCollections); chunks = null; // gc // remember previous version IDs @@ -456,38 +459,30 @@ sortChunksInPlace(chunks); - chunks.forEach(function(object) { - var key = object.key; - var value = object.value; - if (key === "loki") { + chunks.forEach(function(chunk) { + var type = chunk.type; + var value = chunk.value; + var name = chunk.collectionName; + if (type === "loki") { loki = value; - return; - } else if (key.includes(".")) { - var keySegments = key.split("."); - if (keySegments.length === 3 && keySegments[1] === "chunk") { - var colName = keySegments[0]; - if (chunkMap[colName]) { - chunkMap[colName].dataChunks.push(value); - } else { - chunkMap[colName] = { - metadata: null, - dataChunks: [value], - }; - } - return; - } else if (keySegments.length === 2 && keySegments[1] === "metadata") { - var name = keySegments[0]; - if (chunkMap[name]) { - chunkMap[name].metadata = value; - } else { - chunkMap[name] = { metadata: value, dataChunks: [] }; - } - return; + } else if (type === "data") { + if (chunkMap[name]) { + chunkMap[name].dataChunks.push(value); + } else { + chunkMap[name] = { + metadata: null, + dataChunks: [value], + }; } + } else if (type === "metadata") { + if (chunkMap[name]) { + chunkMap[name].metadata = value; + } else { + chunkMap[name] = { metadata: value, dataChunks: [] }; + } + } else { + throw new Error("unreachable"); } - - console.error("Unknown chunk " + key); - throw new Error("Corrupted database - unknown chunk found"); }); if (!loki) { @@ -497,25 +492,38 @@ return { loki: loki, chunkMap: chunkMap }; } - function populateLoki(loki, chunkMap) { + function populateLoki(loki, chunkMap, deserializeChunk, lazyCollections) { loki.collections.forEach(function populateCollection(collectionStub, i) { - var chunkCollection = chunkMap[collectionStub.name]; + var name = collectionStub.name; + var chunkCollection = chunkMap[name]; if (chunkCollection) { if (!chunkCollection.metadata) { - throw new Error("Corrupted database - missing metadata chunk for " + collectionStub.name); + throw new Error("Corrupted database - missing metadata chunk for " + name); } var collection = chunkCollection.metadata; chunkCollection.metadata = null; - loki.collections[i] = collection; - var dataChunks = chunkCollection.dataChunks; - dataChunks.forEach(function populateChunk(chunk, i) { - chunk.forEach(function(doc) { - collection.data.push(doc); + var isLazy = lazyCollections.includes(name); + var lokiDeserializeCollectionChunks = function () { + DEBUG && isLazy && console.log("lazy loading " + name); + var data = []; + var dataChunks = chunkCollection.dataChunks; + dataChunks.forEach(function populateChunk(chunk, i) { + if (isLazy) { + chunk = JSON.parse(chunk); + if (deserializeChunk) { + chunk = deserializeChunk(name, chunk); + } + } + chunk.forEach(function(doc) { + data.push(doc); + }); + dataChunks[i] = null; }); - dataChunks[i] = null; - }); + return data; + }; + collection.getData = lokiDeserializeCollectionChunks; } }); } @@ -606,6 +614,7 @@ var store = tx.objectStore('LokiIncrementalData'); var deserializeChunk = this.options.deserializeChunk; + var lazyCollections = this.lazyCollections; // If there are a lot of chunks (>100), don't request them all in one go, but in multiple // "megachunks" (chunks of chunks). This improves concurrency, as main thread is already busy @@ -622,7 +631,7 @@ // DEBUG && console.time(debugMsg); var megachunk = e.target.result; megachunk.forEach(function (chunk, i) { - parseChunk(chunk, deserializeChunk); + parseChunk(chunk, deserializeChunk, lazyCollections); allChunks.push(chunk); megachunk[i] = null; // gc }); @@ -636,11 +645,13 @@ // Stagger megachunk requests - first one half, then request the second when first one comes // back. This further improves concurrency. - function requestMegachunk(index) { + var megachunkWaves = 2; + var megachunksPerWave = megachunkCount / megachunkWaves; + function requestMegachunk(index, wave) { var keyRange = keyRanges[index]; idbReq(store.getAll(keyRange), function(e) { - if (index < megachunkCount / 2) { - requestMegachunk(index + megachunkCount / 2); + if (wave < megachunkWaves) { + requestMegachunk(index + megachunksPerWave, wave + 1); } processMegachunk(e, index, keyRange); @@ -649,8 +660,8 @@ }); } - for (var i = 0; i < megachunkCount / 2; i += 1) { - requestMegachunk(i); + for (var i = 0; i < megachunksPerWave; i += 1) { + requestMegachunk(i, 1); } } @@ -658,7 +669,7 @@ idbReq(store.getAll(), function(e) { var allChunks = e.target.result; allChunks.forEach(function (chunk) { - parseChunk(chunk, deserializeChunk); + parseChunk(chunk, deserializeChunk, lazyCollections); }); callback(allChunks); }, function(e) { @@ -667,13 +678,17 @@ } function getAllKeys() { - idbReq(store.getAllKeys(), function(e) { - var keys = e.target.result.sort(); + function onDidGetKeys(keys) { + keys.sort(); if (keys.length > 100) { getMegachunks(keys); } else { getAllChunks(); } + } + + idbReq(store.getAllKeys(), function(e) { + onDidGetKeys(e.target.result); }, function(e) { callback(e); }); @@ -686,15 +701,42 @@ getAllKeys(); }; - function parseChunk(chunk, deserializeChunk) { - chunk.value = JSON.parse(chunk.value); - if (deserializeChunk) { - var segments = chunk.key.split('.'); - if (segments.length === 3 && segments[1] === 'chunk') { - var collectionName = segments[0]; - chunk.value = deserializeChunk(collectionName, chunk.value); + function classifyChunk(chunk) { + var key = chunk.key; + + if (key === 'loki') { + chunk.type = 'loki'; + return; + } else if (key.includes('.')) { + var keySegments = key.split("."); + if (keySegments.length === 3 && keySegments[1] === "chunk") { + chunk.type = 'data'; + chunk.collectionName = keySegments[0]; + chunk.index = parseInt(keySegments[2], 10); + return; + } else if (keySegments.length === 2 && keySegments[1] === "metadata") { + chunk.type = 'metadata'; + chunk.collectionName = keySegments[0]; + return; } } + + console.error("Unknown chunk " + key); + throw new Error("Corrupted database - unknown chunk found"); + } + + function parseChunk(chunk, deserializeChunk, lazyCollections) { + classifyChunk(chunk); + + var isData = chunk.type === 'data'; + var isLazy = lazyCollections.includes(chunk.collectionName); + + if (!(isData && isLazy)) { + chunk.value = JSON.parse(chunk.value); + } + if (deserializeChunk && isData && !isLazy) { + chunk.value = deserializeChunk(chunk.collectionName, chunk.value); + } } /** @@ -758,27 +800,11 @@ return Math.random().toString(36).substring(2); } - function _getSortKey(object) { - var key = object.key; - if (key.includes(".")) { - var segments = key.split("."); - if (segments.length === 3 && segments[1] === "chunk") { - return parseInt(segments[2], 10); - } - } - - return -1; // consistent type must be returned - } - function sortChunksInPlace(chunks) { // sort chunks in place to load data in the right order (ascending loki ids) // on both Safari and Chrome, we'll get chunks in order like this: 0, 1, 10, 100... chunks.sort(function(a, b) { - var aKey = _getSortKey(a), - bKey = _getSortKey(b); - if (aKey < bKey) return -1; - if (aKey > bKey) return 1; - return 0; + return (a.index || 0) - (b.index || 0); }); } diff --git a/src/lokijs.js b/src/lokijs.js index fb35f141..0588569c 100644 --- a/src/lokijs.js +++ b/src/lokijs.js @@ -415,7 +415,7 @@ var valueFound = false; var element; - if (typeof root === 'object' && path in root) { + if (root !== null && typeof root === 'object' && path in root) { element = root[path]; } if (pathOffset + 1 >= paths.length) { @@ -1774,27 +1774,43 @@ copyColl.dirty = false; } - // load each element individually - clen = coll.data.length; - j = 0; - if (options && options.hasOwnProperty(coll.name)) { - loader = makeLoader(coll); - - for (j; j < clen; j++) { - collObj = loader(coll.data[j]); - copyColl.data[j] = collObj; - copyColl.addAutoUpdateObserver(collObj); - if (!copyColl.disableFreeze) { - deepFreeze(copyColl.data[j]); - } + if (coll.getData) { + if ((options && options.hasOwnProperty(coll.name)) || !copyColl.disableFreeze || copyColl.autoupdate) { + throw new Error("this collection cannot be loaded lazily: " + coll.name); } + copyColl.getData = coll.getData; + Object.defineProperty(copyColl, 'data', { + /* jshint loopfunc:true */ + get: function() { + var data = this.getData(); + this.getData = null; + Object.defineProperty(this, 'data', { value: data }); + return data; + } + /* jshint loopfunc:false */ + }); } else { - - for (j; j < clen; j++) { - copyColl.data[j] = coll.data[j]; - copyColl.addAutoUpdateObserver(copyColl.data[j]); - if (!copyColl.disableFreeze) { - deepFreeze(copyColl.data[j]); + // load each element individually + clen = coll.data.length; + j = 0; + if (options && options.hasOwnProperty(coll.name)) { + loader = makeLoader(coll); + + for (j; j < clen; j++) { + collObj = loader(coll.data[j]); + copyColl.data[j] = collObj; + copyColl.addAutoUpdateObserver(collObj); + if (!copyColl.disableFreeze) { + deepFreeze(copyColl.data[j]); + } + } + } else { + for (j; j < clen; j++) { + copyColl.data[j] = coll.data[j]; + copyColl.addAutoUpdateObserver(copyColl.data[j]); + if (!copyColl.disableFreeze) { + deepFreeze(copyColl.data[j]); + } } } }