diff --git a/Makefile b/Makefile index e436e54..aa150b0 100644 --- a/Makefile +++ b/Makefile @@ -153,7 +153,8 @@ ${JIOVERSION}: ${EXTERNALDIR}/URI.js \ ${SRCDIR}/jio.storage/fbstorage.js \ ${SRCDIR}/jio.storage/cloudooostorage.js \ ${SRCDIR}/jio.storage/nocapacitystorage.js \ - ${SRCDIR}/jio.storage/liststorage.js + ${SRCDIR}/jio.storage/liststorage.js \ + ${SRCDIR}/jio.storage/indexstorage2.js @mkdir -p $(@D) cat $^ > $@ diff --git a/examples/index2_test.html b/examples/index2_test.html new file mode 100644 index 0000000..2aa5227 --- /dev/null +++ b/examples/index2_test.html @@ -0,0 +1,42 @@ + + + + + + + jIO Query Performance test + + + + + + + + + + + +

Testing index2 query

+ Test +

+
+ + diff --git a/examples/index2_test.js b/examples/index2_test.js new file mode 100644 index 0000000..d8fbdcc --- /dev/null +++ b/examples/index2_test.js @@ -0,0 +1,163 @@ +/*global performance, String*/ +(function (window, jIO, rJS) { + "use strict"; + + var test_count = 10; + /*function randomi(limit) { + return Math.floor(Math.random() * Math.floor(limit)); + } + + function randomSentence(length) { + var alphabet = ['a', 'b', 'c', 'd', 'e', ' ', 'f', 'g', 'h', 'i', 'j', 'k', + ' ', 'l', 'm', 'n', 'o', ' ', 'p', 'q', 'r', 's', 't', ' ', + 'u', 'v', 'w', ' ', 'x', 'y', 'z', ' '], sentence = '', z; + for (z = 0; z < length; z += 1) { + sentence += alphabet[randomi(alphabet.length - 1)]; + } + return sentence; + } + + function randomSentenceArray(sentence_length, array_length) { + var y, sentence_array = []; + for (y = 0; y < array_length; y += 1) { + sentence_array.push(randomSentence(sentence_length)); + } + return sentence_array; + }*/ + + function get_fake_data_values2(i) { + if (i === 0 || i === 1 || i === 2) { + return {'url': 'renderjs.com', 'name': 'erp5', 'user': 'preet'}; + } + if (i === 3 || i === 4) { + return {'url': 'erp5.com', 'name': 'erp5', 'user': 'test'}; + } + if (i === 5 || i === 6 || i === 7) { + return {'url': 'nexedi.com', 'name': 'nexedi', 'user': 'preet'}; + } + return {'url': 'jio.nexedi.com', 'name': 'jio', 'user': 'obscure'}; + } + + /*function get_fake_data_values(i) { + var data_value = { + 'id': i, + 'url': 'https://streetsite.com/profiles/' + i, + 'pic_url': 'https://cdn.streetsite.com/pictures/saoteuhcu/' + i, + 'short_description': randomSentence(10 + randomi(40)), + 'description': randomSentence(randomi(250)), + 'comments': randomSentenceArray(randomi(500), randomi(20)) + }; + if (i === 9900) { + data_value.short_description = 'test'; + } + if (i === 7500) { + data_value.short_description = 'preet'; + } + if (i === 5400) { + data_value.short_description = 'obscure'; + } + if (i === 3200) { + data_value.short_description = 'precise'; + } + if (i === 1200) { + data_value.short_description = 'environ'; + } + return data_value; + }*/ + + /* function sequential_test(i, storage) { + if (i < test_count) { + var data_value = { + 'id': i, + 'url': 'https://streetsite.com/profiles/' + i, + 'pic_url': 'https://cdn.streetsite.com/pictures/saoteuhcu/' + i, + 'short_description': randomSentence(10 + randomi(40)), + 'description': randomSentence(randomi(250)), + 'comments': randomSentenceArray(randomi(500), randomi(20)) + }; + if (i === 99000) { + data_value.short_description = 'test'; + } + if (i % 100 === 0) { + data_value.short_description = 'preet'; + } + if (i % 1000 === 0) { + data_value.short_description = 'obscure'; + } + if (i === 32000) { + data_value.short_description = 'precise precise precise'; + } + if (i === 120000) { + data_value.short_description = 'environ'; + } + return storage.put(String(i), data_value) + .then(function () { + if (i % 1000 === 0) { + console.log(i); + } + data_value = null; + return sequential_test(i + 1, storage); + }); + } + return; + }*/ + + + rJS(window) + + .declareService(function () { + var storage = jIO.createJIO({ + type: "index2", + database: "index2test2", + index_keys: ["name", "user", "url"], + sub_storage: { + type: "indexeddb", + database: "index2testdata2", + } + }), promise_list = [], i, time1; + console.log('Staring to write ' + test_count + ' documents'); + //sequential_test(0, storage); + for (i = 0; i < test_count; i += 1) { + promise_list.push(storage.put(String(i), get_fake_data_values2(i))); + } + time1 = performance.now(); + return RSVP.all(promise_list) + .then(function () { + console.log('Time to write - ', (performance.now() - time1)); + console.log('Starting queries'); + console.log('Query 1'); + var time2 = performance.now(); + return storage.allDocs({query: "name:erp5 AND user:preet AND " + + "url:renderjs.com"}) + .then(function (result) { + console.log('Time to query 1 - ', (performance.now() - time2)); + console.log(result); + console.log('Query 2'); + var time3 = performance.now(); + return storage.allDocs({query: 'user:preet'}) + .then(function (result) { + console.log('Time to query 2 - ', + (performance.now() - time3)); + console.log(result); + console.log('Query 3'); + var time4 = performance.now(); + return storage.allDocs({query: "(name:jio OR url:nexedi.com" + + ") AND user:obscure"}) + .then(function (result) { + console.log('Time to query 3 - ', + performance.now() - time4); + console.log(result); + console.log('Query 4'); + var time5 = performance.now(); + return storage.allDocs({query: 'name:not'}) + .then(function (result) { + console.log('Time to query 4 - ', + (performance.now() - time5)); + console.log(result); + }); + }); + }); + }); + }); + }); +}(window, jIO, rJS)); \ No newline at end of file diff --git a/src/jio.storage/indexstorage2.js b/src/jio.storage/indexstorage2.js new file mode 100644 index 0000000..65e61ae --- /dev/null +++ b/src/jio.storage/indexstorage2.js @@ -0,0 +1,313 @@ +/* + * Copyright 2019, Nexedi SA + * + * This program is free software: you can Use, Study, Modify and Redistribute + * it under the terms of the GNU General Public License version 3, or (at your + * option) any later version, as published by the Free Software Foundation. + * + * You can also Link and Combine this program with other software covered by + * the terms of any of the Free Software licenses or any of the Open Source + * Initiative approved licenses and Convey the resulting work. Corresponding + * source of such a combination shall include the source code for all other + * software used. + * + * This program is distributed WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * + * See COPYING file for full licensing terms. + * See https://www.nexedi.com/licensing for rationale and options. + */ +/*jslint nomen: true */ +/*global indexedDB, jIO, RSVP, IDBOpenDBRequest, + DOMError, Event, parseStringToObject, Set*/ + +(function (indexedDB, jIO, RSVP, IDBOpenDBRequest, + DOMError, parseStringToObject) { + "use strict"; + + function IndexStorage2(description) { + if (typeof description.database !== "string" || + description.database === "") { + throw new TypeError("IndexStorage2 'database' description property " + + "must be a non-empty string"); + } + this._sub_storage = jIO.createJIO(description.sub_storage); + this._database_name = "jio:" + description.database; + this._index_keys = description.index_keys; + } + + IndexStorage2.prototype.hasCapacity = function (name) { + return ((name === "list") || (name === "include") || (name === "query")); + }; + + function handleUpgradeNeeded(evt, index_keys) { + var db = evt.target.result, store, i; + + store = db.createObjectStore("index-store", { + keyPath: "id", + autoIncrement: false + }); + for (i = 0; i < index_keys.length; i += 1) { + store.createIndex("Index-" + index_keys[i], "doc." + index_keys[i], + {unique: false}); + } + } + + function waitForOpenIndexedDB(db_name, index_keys, callback) { + function resolver(resolve, reject) { + // Open DB // + var request = indexedDB.open(db_name); + request.onerror = function (error) { + if (request.result) { + request.result.close(); + } + if ((error !== undefined) && + (error.target instanceof IDBOpenDBRequest) && + (error.target.error instanceof DOMError)) { + reject("Connection to: " + db_name + " failed: " + + error.target.error.message); + } else { + reject(error); + } + }; + + request.onabort = function () { + request.result.close(); + reject("Aborting connection to: " + db_name); + }; + + request.ontimeout = function () { + request.result.close(); + reject("Connection to: " + db_name + " timeout"); + }; + + request.onblocked = function () { + request.result.close(); + reject("Connection to: " + db_name + " was blocked"); + }; + + // Create DB if necessary // + request.onupgradeneeded = function (evt) { + handleUpgradeNeeded(evt, index_keys); + }; + + request.onversionchange = function () { + request.result.close(); + reject(db_name + " was upgraded"); + }; + + request.onsuccess = function () { + return new RSVP.Queue() + .push(function () { + return callback(request.result); + }) + .push(function (result) { + request.result.close(); + resolve(result); + }, function (error) { + request.result.close(); + reject(error); + }); + }; + } + + return new RSVP.Promise(resolver); + } + + function waitForTransaction(db, stores, flag, callback) { + var tx = db.transaction(stores, flag); + function canceller() { + try { + tx.abort(); + } catch (unused) { + // Transaction already finished + return; + } + } + function resolver(resolve, reject) { + var result; + try { + result = callback(tx); + } catch (error) { + reject(error); + } + tx.oncomplete = function () { + return new RSVP.Queue() + .push(function () { + return result; + }) + .push(resolve, function (error) { + canceller(); + reject(error); + }); + }; + tx.onerror = function (error) { + canceller(); + reject(error); + }; + tx.onabort = function (evt) { + reject(evt.target); + }; + return tx; + } + return new RSVP.Promise(resolver, canceller); + } + + function waitForIDBRequest(request) { + return new RSVP.Promise(function (resolve, reject) { + request.onerror = reject; + request.onsuccess = resolve; + }); + } + + IndexStorage2.prototype._runQuery = function (index, value) { + var context = this; + return new RSVP.Queue() + .push(function () { + return waitForOpenIndexedDB(context._database_name, + context._index_keys, function (db) { + return waitForTransaction(db, ["index-store"], "readonly", + function (tx) { + return waitForIDBRequest(tx.objectStore("index-store") + .index("Index-" + index).getAll(value)); + }); + }); + }) + .push(function (evt) { + return evt.target.result; + }); + }; + + IndexStorage2.prototype._processQueryObject = function (object) { + var promise_list = [], context = this, i, j, query_result = new Set(); + return RSVP.Queue() + .push(function () { + if (object.type === "simple") { + return context._runQuery(object.key, object.value); + } + if (object.type === "complex") { + for (i = 0; i < object.query_list.length; i += 1) { + promise_list.push(context + ._processQueryObject(object.query_list[i])); + } + return RSVP.all(promise_list) + .then(function (result) { + if (object.operator === "OR") { + for (i = 0; i < result.length; i += 1) { + for (j = 0; j < result[i].length; j += 1) { + query_result.add(result[i][j].id); + } + } + return Array.from(query_result); + } + if (object.operator === "AND") { + var temp_set = new Set(); + for (i = 0; i < result[0].length; i += 1) { + query_result.add(result[0][i].id); + } + for (i = 1; i < result.length; i += 1) { + for (j = 0; j < result[i].length; j += 1) { + if (query_result.has(result[i][j].id)) { + temp_set.add(result[i][j].id); + } + } + query_result = temp_set; + temp_set = new Set(); + } + return Array.from(query_result); + } + }); + } + }); + }; + + IndexStorage2.prototype.buildQuery = function (options) { + return this._processQueryObject(parseStringToObject(options.query)); + }; + + IndexStorage2.prototype.get = function () { + return this._sub_storage.get.apply(this._sub_storage, arguments); + }; + + IndexStorage2.prototype._filter_doc_values = function (doc, keys) { + var filtered_doc = {}, i; + for (i = 0; i < keys.length; i += 1) { + filtered_doc[keys[i]] = doc[keys[i]]; + } + return filtered_doc; + }; + + IndexStorage2.prototype.put = function (id, value) { + var context = this; + return context._sub_storage.put(id, value) + .push(function (result) { + return waitForOpenIndexedDB(context._database_name, + context._index_keys, function (db) { + return waitForTransaction(db, ["index-store"], "readwrite", + function (tx) { + return waitForIDBRequest(tx.objectStore("index-store").put({ + "id": id, + "doc": context._filter_doc_values(value, context._index_keys) + })) + .then(function () { + return result; + }); + }); + }); + }); + }; + + IndexStorage2.prototype.post = function (value) { + var context = this; + return context._sub_storage.post(value) + .push(function (id) { + return waitForOpenIndexedDB(context._database_name, + context._index_keys, function (db) { + return waitForTransaction(db, ["index-store"], "readwrite", + function (tx) { + return waitForIDBRequest(tx.objectStore("index-store").put({ + "id": id, + "doc": context._filter_doc_values(value, context._index_keys) + })) + .then(function () { + return id; + }); + }); + }); + }); + }; + + IndexStorage2.prototype.remove = function (id) { + var context = this; + return context._sub_storage.remove(id) + .push(function (result) { + return waitForOpenIndexedDB(context._database_name, context._index_keys, + function (db) { + return waitForTransaction(db, ["index-store"], "readwrite", + function (tx) { + return waitForIDBRequest(tx.objectStore("index-store") + .delete(id)) + .then(function () { + return result; + }); + }); + }); + }); + }; + + IndexStorage2.prototype.getAttachment = function () { + return this._sub_storage.getAttachment.apply(this._sub_storage, arguments); + }; + + IndexStorage2.prototype.putAttachment = function () { + return this._sub_storage.putAttachment.apply(this._sub_storage, arguments); + }; + + IndexStorage2.prototype.removeAttachment = function () { + return this._sub_storage.removeAttachment.apply(this._sub_storage, + arguments); + }; + + jIO.addStorage("index2", IndexStorage2); +}(indexedDB, jIO, RSVP, IDBOpenDBRequest, DOMError, + parseStringToObject)); \ No newline at end of file