From 4ca45156e4eaf2dde1899de069afe8d4ca42c4e1 Mon Sep 17 00:00:00 2001 From: Romain Courteaud Date: Mon, 5 Aug 2019 11:00:03 +0200 Subject: [PATCH 1/7] WIP [jio] Add index capacity --- src/jio.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/jio.js b/src/jio.js index 789eec6..90b5e3e 100644 --- a/src/jio.js +++ b/src/jio.js @@ -467,6 +467,7 @@ return ensurePushableQueue(function () { if (context.hasCapacity("list") && ((options.query === undefined) || context.hasCapacity("query")) && + ((options.index === undefined) || context.hasCapacity("index")) && ((options.sort_on === undefined) || context.hasCapacity("sort")) && ((options.group_by === undefined) || context.hasCapacity("group")) && ((options.select_list === undefined) || -- 2.30.9 From 1753ee5f155a45eb4c8353107da9fecb580b386e Mon Sep 17 00:00:00 2001 From: Romain Courteaud Date: Mon, 5 Aug 2019 10:56:56 +0200 Subject: [PATCH 2/7] WIP [indexedDB] Add pseudo index support in allDocs --- src/jio.storage/indexeddbstorage.js | 22 +++++-- test/jio.storage/indexeddbstorage.tests.js | 72 ++++++++++++++++++++++ 2 files changed, 89 insertions(+), 5 deletions(-) diff --git a/src/jio.storage/indexeddbstorage.js b/src/jio.storage/indexeddbstorage.js index 1bfa078..51b3c72 100644 --- a/src/jio.storage/indexeddbstorage.js +++ b/src/jio.storage/indexeddbstorage.js @@ -65,7 +65,7 @@ } IndexedDBStorage.prototype.hasCapacity = function (name) { - return ((name === "list") || (name === "include")); + return ((name === "list") || (name === "include") || (name === "index")); }; function buildKeyPath(key_list) { @@ -268,7 +268,9 @@ IndexedDBStorage.prototype.buildQuery = function (options) { var result_list = [], - context = this; + context = this, + key = "_id", + value; function pushIncludedMetadata(cursor) { result_list.push({ @@ -285,20 +287,30 @@ }); } + if (options.index) { + if (context._index_key_list.indexOf(options.index.key) === -1) { + throw new jIO.util.jIOError( + "IndexedDB: unsupported index '" + options.index.key + "'", + 400 + ); + } + key = INDEX_PREFIX + options.index.key; + value = options.index.value; + } + return new RSVP.Queue() .push(function () { return waitForOpenIndexedDB(context, function (db) { return waitForTransaction(db, ["metadata"], "readonly", function (tx) { - var key = "_id"; if (options.include_docs === true) { return waitForAllSynchronousCursor( - tx.objectStore("metadata").index(key).openCursor(), + tx.objectStore("metadata").index(key).openCursor(value), pushIncludedMetadata ); } return waitForAllSynchronousCursor( - tx.objectStore("metadata").index(key).openKeyCursor(), + tx.objectStore("metadata").index(key).openKeyCursor(value), pushMetadata ); }); diff --git a/test/jio.storage/indexeddbstorage.tests.js b/test/jio.storage/indexeddbstorage.tests.js index ccee743..4cd36ef 100644 --- a/test/jio.storage/indexeddbstorage.tests.js +++ b/test/jio.storage/indexeddbstorage.tests.js @@ -700,6 +700,78 @@ }); }); + test("Unhandled index", function () { + var context = this; + stop(); + expect(3); + + deleteIndexedDB(context.jio) + .then(function () { + return context.jio.allDocs({index: {key: 'a', value: '3'}}); + }) + .fail(function (error) { + ok(error instanceof jIO.util.jIOError); + equal( + error.message, + "IndexedDB: unsupported index 'a'" + ); + equal(error.status_code, 400); + }) + .fail(function (error) { + ok(false, error); + }) + .always(function () { + start(); + }); + }); + + test("Handled index", function () { + var context = this; + this.jio = jIO.createJIO({ + type: "indexeddb", + database: "qunit", + index_key_list: ['foo'] + }); + stop(); + expect(1); + + deleteIndexedDB(context.jio) + .then(function () { + return RSVP.all([ + context.jio.put("1", {"foo": "bar"}), + context.jio.put("2", {"foo": "bar2"}), + context.jio.put("3", {"foo2": "bar"}), + context.jio.put("4", {"foo": "bar"}) + ]); + }) + .then(function () { + return context.jio.allDocs({index: {key: 'foo', value: 'bar'}}); + }) + .then(function (result) { + deepEqual(result, { + "data": { + "rows": [ + { + "id": "1", + "value": {} + }, + { + "id": "4", + "value": {} + } + ], + "total_rows": 2 + } + }); + }) + .fail(function (error) { + ok(false, error); + }) + .always(function () { + start(); + }); + }); + ///////////////////////////////////////////////////////////////// // indexeddbStorage.get ///////////////////////////////////////////////////////////////// -- 2.30.9 From 9bb8ec08166320095132a74627e68494d9dd4076 Mon Sep 17 00:00:00 2001 From: Romain Courteaud Date: Thu, 1 Aug 2019 16:14:05 +0200 Subject: [PATCH 3/7] [HARDCODED] Check idb perf --- examples/perfidb.html | 46 +++++++++++ examples/perfidb.js | 177 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+) create mode 100644 examples/perfidb.html create mode 100644 examples/perfidb.js diff --git a/examples/perfidb.html b/examples/perfidb.html new file mode 100644 index 0000000..59fc932 --- /dev/null +++ b/examples/perfidb.html @@ -0,0 +1,46 @@ + + + + + + + + + jIO Performance Scenario + + + + + + + + + + + +

jIO Performance Scenario

+

+
+

+
    +
    test markup, will be hidden
    + + diff --git a/examples/perfidb.js b/examples/perfidb.js new file mode 100644 index 0000000..30d94d9 --- /dev/null +++ b/examples/perfidb.js @@ -0,0 +1,177 @@ +/* + * Copyright 2014, Nexedi SA + * + * This program is free software: you can Use, Study, Modify and Redistribute + * it under the terms of the GNU General Public License version 3, or (at your + * option) any later version, as published by the Free Software Foundation. + * + * You can also Link and Combine this program with other software covered by + * the terms of any of the Free Software licenses or any of the Open Source + * Initiative approved licenses and Convey the resulting work. Corresponding + * source of such a combination shall include the source code for all other + * software used. + * + * This program is distributed WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * + * See COPYING file for full licensing terms. + * See https://www.nexedi.com/licensing for rationale and options. + */ + +/*global console, btoa, Blob, indexedDB*/ +/*jslint nomen: true, maxlen: 200*/ +(function (window, QUnit, jIO, rJS) { + "use strict"; + var test = QUnit.test, + equal = QUnit.equal, + expect = QUnit.expect, + ok = QUnit.ok, + stop = QUnit.stop, + start = QUnit.start, + // deepEqual = QUnit.deepEqual; + document_idb_name = "index_document_test_performance", + index_idb_name = "index_test_performance"; + + function dispatchQueue(context, function_used, argument_list, + number_queue) { + var result_promise_list = [], + i, + defer; + function pushAndExecute(global_defer) { + if ((global_defer.promise.isFulfilled) || + (global_defer.promise.isRejected)) { + return; + } + if (argument_list.length > 0) { + function_used.apply(context, argument_list.shift()) + .then(function () { + pushAndExecute(global_defer); + }) + .fail(global_defer.reject); + return; + } + global_defer.resolve(); + } + for (i = 0; i < number_queue; i += 1) { + defer = RSVP.defer(); + result_promise_list.push(defer.promise); + pushAndExecute(defer); + } + if (number_queue > 1) { + return RSVP.all(result_promise_list); + } + return result_promise_list[0]; + } + + rJS(window) + + .declareService(function () { + return this.run(); + }) + + .declareMethod('run', function () { + console.info(index_idb_name); + var jio_options = { + type: "uuid", + sub_storage: { + type: "indexeddb", + database: document_idb_name, + index_key_list: ["title", "portal_type"], + version: 3 + } + }, + document_count = 10000, + parallel_operation = 10; + + test('Test index storage speed', function () { + var jio, + i, + document_list = []; + stop(); + expect(6); + + // Create the storage + try { + jio = jIO.createJIO(jio_options); + } catch (error) { + console.error(error.stack); + console.error(error); + throw error; + } + + function postNewDocument(index) { + return jio.post({title: index, modification_date: index, + parent_relative_url: 'jio_perf_module', + portal_type: 'Jio Perf'}); + } + + function generateCheckQuerySpeed(query) { + return function () { + var now = Date.now(); + return jio.allDocs(query) + .push(function (result) { + equal(result.data.total_rows, query.limit || document_count, query.index); + console.log(query.index, result.data.total_rows, Date.now() - now, 'ms'); + }); + }; + } + function checkQueryListSpeed() { + var queue = new RSVP.Queue(), + query_list = arguments, + j; + for (j = 0; j < query_list.length; j += 1) { + queue.push(generateCheckQuerySpeed(query_list[j])); + } + return queue; + } + + // Put in the jio storage + return new RSVP.Queue() + .then(function () { + ok(true, 'Index repair... ' + new Date()); + // Ensure the index is correct + // return jio.repair(); + }) + .then(function () { + ok(true, 'Index repaired ' + new Date()); + // Check if all documents have been created + return jio.allDocs(); + }) + .then(function (result) { + // Initialize all documents to create + for (i = result.data.total_rows; i < document_count; i += 1) { + document_list.push([i]); + } + ok(true, 'Documents created ' + new Date()); + console.info('creating ' + document_list.length + ' documents'); + + return dispatchQueue(this, postNewDocument, document_list, + parallel_operation); + }) + .then(function () { + return checkQueryListSpeed( + {}, + // {limit: [0, 1]}, + // Monovalued index + {index: {key: "portal_type", value: "Jio Perf"}}, + // {query: 'portal_type:"Jio Perf"', limit: [0, 1]}, + {index: {key: "portal_type", value: "NOTMATCHING"}}, + // Multi valued index + {index: {key: "title", value: 1234}}, + // {query: 'title:"12345"', limit: [0, 1]}, + {index: {key: "title", value: "NOTMATCHING"}} + ); + }) + .fail(function (error) { + console.error("---"); + console.error(error.stack); + console.error(error); + ok(false, error); + }) + .always(function () { + start(); + }); + }); + }); + +}(window, QUnit, jIO, rJS)); \ No newline at end of file -- 2.30.9 From 12c51b53f3932b173312192c9465c4b957bc3a74 Mon Sep 17 00:00:00 2001 From: Romain Courteaud Date: Thu, 11 Jan 2018 09:53:04 +0100 Subject: [PATCH 4/7] XXX wip: check full text parser --- test/queries/tests.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test/queries/tests.js b/test/queries/tests.js index 98637d5..290f25e 100644 --- a/test/queries/tests.js +++ b/test/queries/tests.js @@ -17,8 +17,8 @@ * See COPYING file for full licensing terms. * See https://www.nexedi.com/licensing for rationale and options. */ -/*global jiodate*/ -(function (jIO, jiodate) { +/*global jiodate, SimpleQuery*/ +(function (jIO, jiodate, SimpleQuery) { "use strict"; var test = QUnit.test, stop = QUnit.stop, @@ -479,9 +479,13 @@ {"identifier": "a", "value": "test post", "time": "2016"}, {"identifier": "b", "value": "test post 1", "time": "2017"}, {"identifier": "c", "value": "test post 2016", "time": "2017"} - ]; + ], lala; stop(); - expect(2); + expect(3); + lala = jIO.QueryFactory.create('test post'); + ok(lala instanceof SimpleQuery, lala); + /*global console */ + console.log(lala); jIO.QueryFactory.create('test post').exec(doc_list). then(function (doc_list) { deepEqual(doc_list, [ @@ -734,4 +738,4 @@ }).always(start); }); -}(jIO, jiodate)); +}(jIO, jiodate, SimpleQuery)); -- 2.30.9 From 11d27ff8f603aee0f3419605f5f93e1bb259e176 Mon Sep 17 00:00:00 2001 From: Romain Courteaud Date: Fri, 19 Apr 2019 12:08:31 +0200 Subject: [PATCH 5/7] WIP compare some storage --- examples/perf.html | 44 ++++++++ examples/perf.js | 256 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 300 insertions(+) create mode 100644 examples/perf.html create mode 100644 examples/perf.js diff --git a/examples/perf.html b/examples/perf.html new file mode 100644 index 0000000..bff532b --- /dev/null +++ b/examples/perf.html @@ -0,0 +1,44 @@ + + + + + + + jIO Performance Scenario + + + + + + + + + + + +

    jIO Performance Scenario

    +

    +
    +

    +
      +
      test markup, will be hidden
      + + diff --git a/examples/perf.js b/examples/perf.js new file mode 100644 index 0000000..907d755 --- /dev/null +++ b/examples/perf.js @@ -0,0 +1,256 @@ +/* + * Copyright 2014, Nexedi SA + * + * This program is free software: you can Use, Study, Modify and Redistribute + * it under the terms of the GNU General Public License version 3, or (at your + * option) any later version, as published by the Free Software Foundation. + * + * You can also Link and Combine this program with other software covered by + * the terms of any of the Free Software licenses or any of the Open Source + * Initiative approved licenses and Convey the resulting work. Corresponding + * source of such a combination shall include the source code for all other + * software used. + * + * This program is distributed WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * + * See COPYING file for full licensing terms. + * See https://www.nexedi.com/licensing for rationale and options. + */ + +/*global console, btoa, Blob, indexedDB*/ +/*jslint nomen: true, maxlen: 200*/ +(function (window, QUnit, jIO, rJS) { + "use strict"; + var test = QUnit.test, + equal = QUnit.equal, + expect = QUnit.expect, + ok = QUnit.ok, + stop = QUnit.stop, + start = QUnit.start, + // deepEqual = QUnit.deepEqual; + local_idb_name = "local_test_performance", + remote_idb_name = "remote_test_performance", + signature_idb_name = "signature_test_performance"; + + function dispatchQueue(context, function_used, argument_list, + number_queue) { + var result_promise_list = [], + i, + defer; + function pushAndExecute(global_defer) { + if ((global_defer.promise.isFulfilled) || + (global_defer.promise.isRejected)) { + return; + } + if (argument_list.length > 0) { + function_used.apply(context, argument_list.shift()) + .then(function () { + pushAndExecute(global_defer); + }) + .fail(global_defer.reject); + return; + } + global_defer.resolve(); + } + for (i = 0; i < number_queue; i += 1) { + defer = RSVP.defer(); + result_promise_list.push(defer.promise); + pushAndExecute(defer); + } + if (number_queue > 1) { + return RSVP.all(result_promise_list); + } + return result_promise_list[0]; + } + + function deleteIndexedDB(idb_name) { + return new RSVP.Promise(function resolver(resolve, reject) { + var request = indexedDB.deleteDatabase( + idb_name + ); + request.onerror = reject; + request.onblocked = reject; + request.onsuccess = resolve; + }); + } + + rJS(window) + + .declareService(function () { + return this.run(); + }) + + .declareMethod('run', function () { + var jio_options = { + type: "replicate", + query: { + query: 'portal_type:"Jio Perf"' + }, + parallel_operation_amount: 10, + signature_hash_key: 'modification_date', + check_local_deletion: false, + check_local_modification: false, + check_remote_creation: false, + check_remote_deletion: false, + check_remote_modification: false, + local_sub_storage: { + /* + type: "erp5", + url: 'https://softinst114089.host.vifib.net/erp5/web_site_module/renderjs_runner/hateoas/', + default_view_reference: "jio_view" + */ + type: "uuid", + sub_storage: { + type: "query", + sub_storage: { + // type: "memory", + type: "indexeddb", + database: local_idb_name + } + /* + type: "drivetojiomapping", + sub_storage: { + type: "dropbox", + access_token: "" + } + */ + } + }, + use_remote_post: true, + remote_sub_storage: { + /* + type: "uuid", + sub_storage: { + type: "query", + sub_storage: { + // type: "memory", + type: "indexeddb", + database: remote_idb_name + } + } + */ + type: "erp5", + url: '', + default_view_reference: "jio_view" + }, + signature_sub_storage: { + type: "query", + sub_storage: { + type: "memory", + // type: "indexeddb", + database: signature_idb_name + } + } + }, + document_count = 1000, + parallel_operation = 10; + + test('Test "' + jio_options.type + '"scenario', function () { + var jio, + i, + document_list = []; + stop(); + expect(6); + + // Create the storage + try { + jio = jIO.createJIO(jio_options); + } catch (error) { + console.error(error.stack); + console.error(error); + throw error; + } + + function postNewDocument(index) { + // return RSVP.Queue(); + /* + return RSVP.resolve(JSON.stringify({ + title: index, modification_date: index, + parent_relative_url: 'jio_perf_module', + portal_type: 'Jio Perf' + })); + */ + return jio.post({title: index, modification_date: index, + parent_relative_url: 'jio_perf_module', + portal_type: 'Jio Perf'}); + } + + // Put in the jio storage + return new RSVP.Queue() + .then(function () { + return RSVP.all([ + deleteIndexedDB('jio:' + local_idb_name), + deleteIndexedDB('jio:' + remote_idb_name), + deleteIndexedDB('jio:' + signature_idb_name) + ]); + }) + .then(function () { + ok(true, 'IDB deleted ' + new Date()); + }) + /* + .then(function () { + // Initialize all documents to create + for (i = 0; i < document_count; i += 1) { + document_list.push([{title: i, modification_date: i, + parent_relative_url: 'jio_perf_module', + portal_type: 'Jio Perf'}]); + } + equal(document_list.length, document_count, + 'Documents created ' + new Date()); + + return dispatchQueue(jio, jio.post, document_list, + parallel_operation); + }) + */ + .then(function () { + // Initialize all documents to create + for (i = 0; i < document_count; i += 1) { + document_list.push([i]); + } + equal(document_list.length, document_count, + 'Documents created ' + new Date()); + + return dispatchQueue(this, postNewDocument, document_list, + parallel_operation); + }) + .then(function () { + return jio.allDocs(); + }) + .then(function (result) { + equal(result.data.total_rows, document_count, + 'Local Storage filled ' + new Date()); + }) + /* + .then(function () { + return jio.repair(); + }) + .then(function () { + ok(true, 'First sync finished ' + new Date()); + }) + .then(function () { + return jio.repair(); + }) + .then(function () { + ok(true, 'Second sync finished ' + new Date()); + }) + .then(function () { + return jio.repair(); + }) + .then(function () { + ok(true, 'Third sync finished ' + new Date()); + }) + */ + .fail(function (error) { + console.error("---"); + console.error(error.stack); + console.error(error); + ok(false, error); + }) + .always(function () { + start(); + }); + }); + }); + +}(window, QUnit, jIO, rJS)); -- 2.30.9 From 9e16cbd42edc61940fe252adad43c2f20a56f5d2 Mon Sep 17 00:00:00 2001 From: Stefane Fermigier Date: Wed, 24 Feb 2021 08:37:09 +0100 Subject: [PATCH 6/7] Fix package.json. 1) Use proper SPDX code for the licence. 2) Add dev dependencies. --- package.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index b312d8b..7965a4c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "jio", "version": "v3.45.0", - "license": "GPLv3+", + "license": "GPL-3.0-or-later", "author": "Nexedi SA", "contributors": [ "Tristan Cavelier ", @@ -22,5 +22,11 @@ ], "engines": { "npm": ">=1.3" + }, + "devDependencies": { + "jslint": "^0.9.2", + "jison": "^0.4.16", + "qunit": "github:qunitjs/node-qunit#v0.9.3", + "sinon": "^1.7.3" } } -- 2.30.9 From 397bd917d5c8366d75e2a5b0fd8ba81a1b23347e Mon Sep 17 00:00:00 2001 From: Stefane Fermigier Date: Wed, 24 Feb 2021 08:37:52 +0100 Subject: [PATCH 7/7] Fix README. --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e11f3fc..f623768 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,10 @@ jIO is a promise-based JavaScript library that offers connectors to many storage The documentation can be found on [https://jio.nexedi.com/](https://jio.nexedi.com/) ### jIO Quickstart + git clone https://lab.nexedi.com/nexedi/jio.git - cd jio.git - npm install jslint@0.9.2 jison@0.4.16 https://github.com/qunitjs/node-qunit.git#v0.9.3 sinon@1.7.3 + cd jio + npm install # Or yarn install make -- 2.30.9