diff --git a/Makefile b/Makefile index e436e54..aa150b0 100644 --- a/Makefile +++ b/Makefile @@ -153,7 +153,8 @@ ${JIOVERSION}: ${EXTERNALDIR}/URI.js \ ${SRCDIR}/jio.storage/fbstorage.js \ ${SRCDIR}/jio.storage/cloudooostorage.js \ ${SRCDIR}/jio.storage/nocapacitystorage.js \ - ${SRCDIR}/jio.storage/liststorage.js + ${SRCDIR}/jio.storage/liststorage.js \ + ${SRCDIR}/jio.storage/indexstorage2.js @mkdir -p $(@D) cat $^ > $@ diff --git a/examples/index2_benchmark.html b/examples/index2_benchmark.html new file mode 100644 index 0000000..3100f65 --- /dev/null +++ b/examples/index2_benchmark.html @@ -0,0 +1,42 @@ + + + + + + + jIO Query Performance test + + + + + + + + + + + +

Testing index2 query

+ Test +

+
+ + diff --git a/examples/index2_benchmark.js b/examples/index2_benchmark.js new file mode 100644 index 0000000..b9c83be --- /dev/null +++ b/examples/index2_benchmark.js @@ -0,0 +1,172 @@ +/*global performance, String*/ +(function (window, jIO, rJS) { + "use strict"; + + var test_count = 15; + /*function randomi(limit) { + return Math.floor(Math.random() * Math.floor(limit)); + } + + function randomSentence(length) { + var alphabet = ['a', 'b', 'c', 'd', 'e', ' ', 'f', 'g', 'h', 'i', 'j', 'k', + ' ', 'l', 'm', 'n', 'o', ' ', 'p', 'q', 'r', 's', 't', ' ', + 'u', 'v', 'w', ' ', 'x', 'y', 'z', ' '], sentence = '', z; + for (z = 0; z < length; z += 1) { + sentence += alphabet[randomi(alphabet.length - 1)]; + } + return sentence; + } + + function randomSentenceArray(sentence_length, array_length) { + var y, sentence_array = []; + for (y = 0; y < array_length; y += 1) { + sentence_array.push(randomSentence(sentence_length)); + } + return sentence_array; + }*/ + + function get_fake_data_values2(i) { + if (i === 0 || i === 1 || i === 2) { + return {'url': 'renderjs.com', 'name': 'erp5', 'user': 'preet'}; + } + if (i === 3 || i === 4) { + return {'url': 'erp5.com', 'name': 'erp5', 'user': 'test'}; + } + if (i === 5 || i === 6 || i === 7) { + return {'url': 'nexedi.com', 'name': 'nexedi', 'user': 'prabetcder'}; + } + if (i === 10 || i === 11) { + return {'url': 'vifib.com', 'name': 'renderjs', 'user': 'preetwinder'}; + } + if (i === 12 || i === 13) { + return {'url': 'renderjs.com', 'name': 'jio', 'user': 'obscure'}; + } + return {'url': 'jio.nexedi.com', 'name': 'jio', 'user': 'praounsteter'}; + } + + /*function get_fake_data_values(i) { + var data_value = { + 'id': i, + 'url': 'https://streetsite.com/profiles/' + i, + 'pic_url': 'https://cdn.streetsite.com/pictures/saoteuhcu/' + i, + 'short_description': randomSentence(10 + randomi(40)), + 'description': randomSentence(randomi(250)), + 'comments': randomSentenceArray(randomi(500), randomi(20)) + }; + if (i === 9900) { + data_value.short_description = 'test'; + } + if (i === 7500) { + data_value.short_description = 'preet'; + } + if (i === 5400) { + data_value.short_description = 'obscure'; + } + if (i === 3200) { + data_value.short_description = 'precise'; + } + if (i === 1200) { + data_value.short_description = 'environ'; + } + return data_value; + }*/ + + /* function sequential_test(i, storage) { + if (i < test_count) { + var data_value = { + 'id': i, + 'url': 'https://streetsite.com/profiles/' + i, + 'pic_url': 'https://cdn.streetsite.com/pictures/saoteuhcu/' + i, + 'short_description': randomSentence(10 + randomi(40)), + 'description': randomSentence(randomi(250)), + 'comments': randomSentenceArray(randomi(500), randomi(20)) + }; + if (i === 99000) { + data_value.short_description = 'test'; + } + if (i % 100 === 0) { + data_value.short_description = 'preet'; + } + if (i % 1000 === 0) { + data_value.short_description = 'obscure'; + } + if (i === 32000) { + data_value.short_description = 'precise precise precise'; + } + if (i === 120000) { + data_value.short_description = 'environ'; + } + return storage.put(String(i), data_value) + .then(function () { + if (i % 1000 === 0) { + console.log(i); + } + data_value = null; + return sequential_test(i + 1, storage); + }); + } + return; + }*/ + + + rJS(window) + + .declareService(function () { + var storage = jIO.createJIO({ + type: "index2", + database: "index2test2", + index_keys: {"user": ["normal", "reverse"], + "name": ["normal", "reverse"], + "url": ["normal"]}, + //index_keys: ["name", "user", "url"], + sub_storage: { + type: "indexeddb", + database: "index2testdata2", + } + }), promise_list = [], i, time1; + console.log('Staring to write ' + test_count + ' documents'); + //sequential_test(0, storage); + for (i = 0; i < test_count; i += 1) { + promise_list.push(storage.put(String(i), get_fake_data_values2(i))); + } + promise_list.push(storage.put('325', get_fake_data_values2(325))); + time1 = performance.now(); + return RSVP.all(promise_list) + .then(function () { + console.log('Time to write - ', (performance.now() - time1)); + console.log('Starting queries'); + console.log('Query 1'); + var time2 = performance.now(); + return storage.allDocs({query: "user:pr%et%er"}) + .then(function (result) { + console.log('Time to query 1 - ', (performance.now() - time2)); + console.log(result); + console.log('Query 2'); + var time3 = performance.now(); + return storage.allDocs({query: 'user:preet%'}) + .then(function (result) { + console.log('Time to query 2 - ', + (performance.now() - time3)); + console.log(result); + console.log('Query 3'); + var time4 = performance.now(); + return storage.allDocs({query: "(name:jio OR url:nexedi.com" + + ") AND user:obscure"}) + .then(function (result) { + console.log('Time to query 3 - ', + performance.now() - time4); + console.log(result); + console.log('Query 4'); + var time5 = performance.now(); + return storage.allDocs({query: 'name:not'}) + .then(function (result) { + console.log('Time to query 4 - ', + (performance.now() - time5)); + console.log(result); + }); + }); + }); + }); + }); + }); +}(window, jIO, rJS)); \ No newline at end of file diff --git a/src/jio.storage/indexstorage2.js b/src/jio.storage/indexstorage2.js new file mode 100644 index 0000000..1166627 --- /dev/null +++ b/src/jio.storage/indexstorage2.js @@ -0,0 +1,342 @@ +/* + * Copyright 2019, Nexedi SA + * + * This program is free software: you can Use, Study, Modify and Redistribute + * it under the terms of the GNU General Public License version 3, or (at your + * option) any later version, as published by the Free Software Foundation. + * + * You can also Link and Combine this program with other software covered by + * the terms of any of the Free Software licenses or any of the Open Source + * Initiative approved licenses and Convey the resulting work. Corresponding + * source of such a combination shall include the source code for all other + * software used. + * + * This program is distributed WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * + * See COPYING file for full licensing terms. + * See https://www.nexedi.com/licensing for rationale and options. + */ +/*jslint nomen: true */ +/*global indexedDB, jIO, RSVP, IDBOpenDBRequest, DOMError, Event, + parseStringToObject, Set*/ + +(function (indexedDB, jIO, RSVP, IDBOpenDBRequest, DOMError, + parseStringToObject) { + "use strict"; + + function IndexStorage2(description) { + if (typeof description.database !== "string" || + description.database === "") { + throw new TypeError("IndexStorage2 'database' description property " + + "must be a non-empty string"); + } + this._sub_storage = jIO.createJIO(description.sub_storage); + this._database_name = "jio:" + description.database; + this._index_keys = description.index_keys; + } + + IndexStorage2.prototype.hasCapacity = function (name) { + return ((name === "list") || (name === "query")); + }; + + function handleUpgradeNeeded(evt, index_keys) { + var db = evt.target.result, store, i; + + store = db.createObjectStore("index-store", { + keyPath: "id", + autoIncrement: false + }); + for (i = 0; i < index_keys.length; i += 1) { + store.createIndex("Index-" + index_keys[i], "doc." + index_keys[i], + {unique: false}); + } + } + + function waitForOpenIndexedDB(db_name, index_keys, callback) { + function resolver(resolve, reject) { + // Open DB // + var request = indexedDB.open(db_name); + request.onerror = function (error) { + if (request.result) { + request.result.close(); + } + if ((error !== undefined) && + (error.target instanceof IDBOpenDBRequest) && + (error.target.error instanceof DOMError)) { + reject("Connection to: " + db_name + " failed: " + + error.target.error.message); + } else { + reject(error); + } + }; + + request.onabort = function () { + request.result.close(); + reject("Aborting connection to: " + db_name); + }; + + request.ontimeout = function () { + request.result.close(); + reject("Connection to: " + db_name + " timeout"); + }; + + request.onblocked = function () { + request.result.close(); + reject("Connection to: " + db_name + " was blocked"); + }; + + // Create DB if necessary // + request.onupgradeneeded = function (evt) { + handleUpgradeNeeded(evt, index_keys); + }; + + request.onversionchange = function () { + request.result.close(); + reject(db_name + " was upgraded"); + }; + + request.onsuccess = function () { + return new RSVP.Queue() + .push(function () { + return callback(request.result); + }) + .push(function (result) { + request.result.close(); + resolve(result); + }, function (error) { + request.result.close(); + reject(error); + }); + }; + } + + return new RSVP.Promise(resolver); + } + + function waitForTransaction(db, stores, flag, callback) { + var tx = db.transaction(stores, flag); + function canceller() { + try { + tx.abort(); + } catch (unused) { + // Transaction already finished + return; + } + } + function resolver(resolve, reject) { + var result; + try { + result = callback(tx); + } catch (error) { + reject(error); + } + tx.oncomplete = function () { + return new RSVP.Queue() + .push(function () { + return result; + }) + .push(resolve, function (error) { + canceller(); + reject(error); + }); + }; + tx.onerror = function (error) { + canceller(); + reject(error); + }; + tx.onabort = function (evt) { + reject(evt.target); + }; + return tx; + } + return new RSVP.Promise(resolver, canceller); + } + + function waitForIDBRequest(request) { + return new RSVP.Promise(function (resolve, reject) { + request.onerror = reject; + request.onsuccess = resolve; + }); + } + + IndexStorage2.prototype._runQuery = function (index, value) { + var context = this; + return new RSVP.Queue() + .push(function () { + if ((context._index_keys.indexOf(index) === -1)) { + if (context._sub_storage.hasCapacity("query")) { + return context._sub_storage.buildQuery( + {"query": index + ":" + value} + ) + .then(function (result) { + return result; + }); + } + } + return waitForOpenIndexedDB(context._database_name, + context._index_keys, function (db) { + return waitForTransaction(db, ["index-store"], "readonly", + function (tx) { + return waitForIDBRequest(tx.objectStore("index-store") + .index("Index-" + index).getAll(value)) + .then(function (evt) { + return evt.target.result; + }); + }); + }); + }); + }; + + IndexStorage2.prototype._processQueryObject = function (object) { + var promise_list = [], context = this, i, j, query_result = new Set(); + return RSVP.Queue() + .push(function () { + if (object.type === "simple") { + return context._runQuery(object.key, object.value); + } + if (object.type === "complex") { + for (i = 0; i < object.query_list.length; i += 1) { + promise_list.push(context + ._processQueryObject(object.query_list[i])); + } + return RSVP.all(promise_list) + .then(function (result) { + if (object.operator === "OR") { + for (i = 0; i < result.length; i += 1) { + for (j = 0; j < result[i].length; j += 1) { + query_result.add(result[i][j]); + } + } + return Array.from(query_result); + } + if (object.operator === "AND") { + var temp_set = new Set(); + for (i = 0; i < result[0].length; i += 1) { + query_result.add(result[0][i].id); + } + for (i = 1; i < result.length; i += 1) { + for (j = 0; j < result[i].length; j += 1) { + if (query_result.has(result[i][j].id)) { + temp_set.add(result[i][j]); + } + } + query_result = temp_set; + temp_set = new Set(); + } + return Array.from(query_result); + } + }); + } + }); + }; + + IndexStorage2.prototype.buildQuery = function (options) { + var context = this; + if (options.query) { + return this._processQueryObject(parseStringToObject(options.query)) + .then(function (result) { + return result.map(function (value) { + return {"id": value.id, "value": {} }; + }); + }); + } + return waitForOpenIndexedDB(context._database_name, + context._index_keys, function (db) { + return waitForTransaction(db, ["index-store"], "readonly", + function (tx) { + return waitForIDBRequest(tx.objectStore("index-store").getAll()) + .then(function (evt) { + return evt.target.result.map(function (value) { + return {"id": value.id, "value": {} }; + }); + }); + }); + }); + }; + + IndexStorage2.prototype.get = function () { + return this._sub_storage.get.apply(this._sub_storage, arguments); + }; + + IndexStorage2.prototype._filter_doc_values = function (doc, keys) { + var filtered_doc = {}, i; + for (i = 0; i < keys.length; i += 1) { + filtered_doc[keys[i]] = doc[keys[i]]; + } + return filtered_doc; + }; + + IndexStorage2.prototype.put = function (id, value) { + var context = this; + return context._sub_storage.put(id, value) + .push(function (result) { + return waitForOpenIndexedDB(context._database_name, + context._index_keys, function (db) { + return waitForTransaction(db, ["index-store"], "readwrite", + function (tx) { + return waitForIDBRequest(tx.objectStore("index-store").put({ + "id": id, + "doc": context._filter_doc_values(value, context._index_keys) + })) + .then(function () { + return result; + }); + }); + }); + }); + }; + + IndexStorage2.prototype.post = function (value) { + var context = this; + return context._sub_storage.post(value) + .push(function (id) { + return waitForOpenIndexedDB(context._database_name, + context._index_keys, function (db) { + return waitForTransaction(db, ["index-store"], "readwrite", + function (tx) { + return waitForIDBRequest(tx.objectStore("index-store").put({ + "id": id, + "doc": context._filter_doc_values(value, context._index_keys) + })) + .then(function () { + return id; + }); + }); + }); + }); + }; + + IndexStorage2.prototype.remove = function (id) { + var context = this; + return context._sub_storage.remove(id) + .push(function (result) { + return waitForOpenIndexedDB(context._database_name, context._index_keys, + function (db) { + return waitForTransaction(db, ["index-store"], "readwrite", + function (tx) { + return waitForIDBRequest(tx.objectStore("index-store") + .delete(id)) + .then(function () { + return result; + }); + }); + }); + }); + }; + + IndexStorage2.prototype.getAttachment = function () { + return this._sub_storage.getAttachment.apply(this._sub_storage, arguments); + }; + + IndexStorage2.prototype.putAttachment = function () { + return this._sub_storage.putAttachment.apply(this._sub_storage, arguments); + }; + + IndexStorage2.prototype.removeAttachment = function () { + return this._sub_storage.removeAttachment.apply(this._sub_storage, + arguments); + }; + + jIO.addStorage("index2", IndexStorage2); +}(indexedDB, jIO, RSVP, IDBOpenDBRequest, DOMError, parseStringToObject)); \ No newline at end of file diff --git a/test/jio.storage/indexstorage2.tests.js b/test/jio.storage/indexstorage2.tests.js new file mode 100644 index 0000000..3c8fe62 --- /dev/null +++ b/test/jio.storage/indexstorage2.tests.js @@ -0,0 +1,259 @@ +/* + * Copyright 2019, Nexedi SA + * + * This program is free software: you can Use, Study, Modify and Redistribute + * it under the terms of the GNU General Public License version 3, or (at your + * option) any later version, as published by the Free Software Foundation. + * + * You can also Link and Combine this program with other software covered by + * the terms of any of the Free Software licenses or any of the Open Source + * Initiative approved licenses and Convey the resulting work. Corresponding + * source of such a combination shall include the source code for all other + * software used. + * + * This program is distributed WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * + * See COPYING file for full licensing terms. + * See https://www.nexedi.com/licensing for rationale and options. + */ +/*jslint nomen: true */ +/*global indexedDB*/ +(function (jIO, QUnit) { + "use strict"; + var test = QUnit.test, + stop = QUnit.stop, + start = QUnit.start, + ok = QUnit.ok, + expect = QUnit.expect, + deepEqual = QUnit.deepEqual, + equal = QUnit.equal, + module = QUnit.module; + + function deleteIndexedDB(storage) { + return new RSVP.Promise(function resolver(resolve, reject) { + var request = indexedDB.deleteDatabase( + storage.__storage._database_name + ); + request.onerror = reject; + request.onblocked = reject; + request.onsuccess = resolve; + }); + } + + ///////////////////////////////////////////////////////////////// + // Custom test substorage definition + ///////////////////////////////////////////////////////////////// + function DummyStorage3() { + return this; + } + jIO.addStorage('dummystorage3', DummyStorage3); + + ///////////////////////////////////////////////////////////////// + // indexStorage2.constructor + ///////////////////////////////////////////////////////////////// + module("indexStorage2.constructor"); + test("Constructor with empty index_keys", function () { + var jio = jIO.createJIO({ + type: "index2", + database: "index2_test", + index_keys: [], + sub_storage: { + type: "dummystorage3" + } + }); + + equal(jio.__type, "index2"); + equal(jio.__storage._sub_storage.__type, "dummystorage3"); + equal(jio.__storage._database_name, "jio:index2_test"); + }); + + ///////////////////////////////////////////////////////////////// + // indexStorage2.hasCapacity + ///////////////////////////////////////////////////////////////// + module("indexStorage2.hasCapacity"); + test("can list documents", function () { + var jio = jIO.createJIO({ + type: "index2", + database: "index2_test", + index_keys: [], + sub_storage: { + type: "dummystorage3" + } + }); + + ok(jio.hasCapacity("list")); + ok(jio.hasCapacity("query")); + }); + + ///////////////////////////////////////////////////////////////// + // indexStorage2.get + ///////////////////////////////////////////////////////////////// + module("indexStorage2.get", { + setup: function () { + this.jio = jIO.createJIO({ + type: "index2", + database: "index2_test", + index_keys: [], + sub_storage: { + type: "dummystorage3" + } + }); + } + }); + test("Simple put get", function () { + var context = this; + stop(); + expect(4); + + DummyStorage3.prototype.put = function (id, value) { + equal(id, "32"); + deepEqual(value, {"a": 3, "b": 2, "c": 8}); + return id; + }; + + DummyStorage3.prototype.get = function (id) { + equal(id, "32"); + return {"a": 3, "b": 2, "c": 8}; + }; + + context.jio.put("32", {"a": 3, "b": 2, "c": 8}) + .then(function () { + return context.jio.get("32"); + }) + .then(function (result) { + deepEqual(result, {"a": 3, "b": 2, "c": 8}); + }) + .fail(function (error) { + console.log(error); + }) + .then(function () { + return deleteIndexedDB(context.jio); + }) + .always(function () { + start(); + }); + }); + + ///////////////////////////////////////////////////////////////// + // indexStorage2.buildQuery + ///////////////////////////////////////////////////////////////// + module("indexStorage2.buildQuery"); + + test("Simple query", function () { + var jio = jIO.createJIO({ + type: "index2", + database: "index2_test", + index_keys: ["a", "b"], + sub_storage: { + type: "dummystorage3" + } + }); + stop(); + expect(3); + + DummyStorage3.prototype.put = function (id, value) { + equal(id, "32"); + deepEqual(value, {a: "3", b: "2"}); + return id; + }; + + jio.put("32", {"a": "3", "b": "2"}) + .then(function () { + return jio.allDocs({query: 'a: "3"'}); + }) + .then(function (result) { + deepEqual(result.data.rows[0], {"id": "32", "value": {}}); + }) + .fail(function (error) { + console.log(error); + }) + .then(function () { + return deleteIndexedDB(jio); + }) + .always(function () { + start(); + }); + }); + + test("No index keys provided", function () { + var jio = jIO.createJIO({ + type: "index2", + database: "index2_test", + index_keys: [], + sub_storage: { + type: "dummystorage3" + } + }); + stop(); + expect(4); + + DummyStorage3.prototype.put = function (id, value) { + equal(id, "32"); + deepEqual(value, {"a": "3", "b": "2"}); + return id; + }; + DummyStorage3.prototype.buildQuery = function (options) { + equal(options.query, 'a:3'); + }; + DummyStorage3.prototype.hasCapacity = function (capacity) { + equal(capacity, "query"); + return false; + }; + + jio.put("32", {"a": "3", "b": "2"}) + .then(function () { + return jio.allDocs({query: 'a:"3"'}); + }) + .fail(function (error) { + equal(error.message, + "Capacity 'query' is not implemented on 'dummystorage3'"); + }) + .then(function () { + return deleteIndexedDB(jio); + }) + .always(function () { + start(); + }); + }); + + test("No Query", function () { + var jio = jIO.createJIO({ + type: "index2", + database: "index2_test", + index_keys: ["a"], + sub_storage: { + type: "dummystorage3" + } + }); + stop(); + expect(1); + + DummyStorage3.prototype.put = function (id) { + return id; + }; + + RSVP.all([ + jio.put("32", {"a": "3", "b": "2"}), + jio.put("21", {"a": "6", "b": "9"}), + jio.put("3", {"a": "8", "b": "5"}) + ]) + .then(function () { + return jio.allDocs(); + }) + .then(function (result) { + equal(result.data.total_rows, 3); + }) + .fail(function (error) { + console.log(error); + }) + .then(function () { + return deleteIndexedDB(jio); + }) + .always(function () { + start(); + }); + }); + + +}(jIO, QUnit)); \ No newline at end of file diff --git a/test/node.js b/test/node.js index f409e6e..6b86823 100644 --- a/test/node.js +++ b/test/node.js @@ -63,6 +63,7 @@ 'test/jio.storage/erp5storage.tests.js', 'test/jio.storage/fbstorage.tests.js', 'test/jio.storage/gdrivestorage.tests.js', + 'test/jio.storage/indexstorage2.tests.js', 'test/jio.storage/liststorage.tests.js', 'test/jio.storage/memorystorage.tests.js', 'test/jio.storage/nocapacitystorage.tests.js', diff --git a/test/tests.html b/test/tests.html index 1e510a9..bcb0190 100644 --- a/test/tests.html +++ b/test/tests.html @@ -79,6 +79,7 @@ See https://www.nexedi.com/licensing for rationale and options. +