diff --git a/README.md b/README.md
index e11f3fc70bbe4296cd39e4185eb723d39d90fead..f623768e97fb9644208618fd06f59a70beb33819 100644
--- a/README.md
+++ b/README.md
@@ -7,9 +7,10 @@ jIO is a promise-based JavaScript library that offers connectors to many storage
The documentation can be found on [https://jio.nexedi.com/](https://jio.nexedi.com/)
### jIO Quickstart
+
git clone https://lab.nexedi.com/nexedi/jio.git
- cd jio.git
- npm install jslint@0.9.2 jison@0.4.16 https://github.com/qunitjs/node-qunit.git#v0.9.3 sinon@1.7.3
+ cd jio
+ npm install # Or yarn install
make
diff --git a/examples/perf.html b/examples/perf.html
new file mode 100644
index 0000000000000000000000000000000000000000..bff532b0216b49aba57d22e7ff8916c84a62cb4f
--- /dev/null
+++ b/examples/perf.html
@@ -0,0 +1,44 @@
+
+
+
+
+
+
+ jIO Performance Scenario
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ test markup, will be hidden
+
+
diff --git a/examples/perf.js b/examples/perf.js
new file mode 100644
index 0000000000000000000000000000000000000000..907d755ba11658fbfe37c17768d4fe118145505f
--- /dev/null
+++ b/examples/perf.js
@@ -0,0 +1,256 @@
+/*
+ * Copyright 2014, Nexedi SA
+ *
+ * This program is free software: you can Use, Study, Modify and Redistribute
+ * it under the terms of the GNU General Public License version 3, or (at your
+ * option) any later version, as published by the Free Software Foundation.
+ *
+ * You can also Link and Combine this program with other software covered by
+ * the terms of any of the Free Software licenses or any of the Open Source
+ * Initiative approved licenses and Convey the resulting work. Corresponding
+ * source of such a combination shall include the source code for all other
+ * software used.
+ *
+ * This program is distributed WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See COPYING file for full licensing terms.
+ * See https://www.nexedi.com/licensing for rationale and options.
+ */
+
+/*global console, btoa, Blob, indexedDB*/
+/*jslint nomen: true, maxlen: 200*/
+(function (window, QUnit, jIO, rJS) {
+ "use strict";
+ var test = QUnit.test,
+ equal = QUnit.equal,
+ expect = QUnit.expect,
+ ok = QUnit.ok,
+ stop = QUnit.stop,
+ start = QUnit.start,
+ // deepEqual = QUnit.deepEqual;
+ local_idb_name = "local_test_performance",
+ remote_idb_name = "remote_test_performance",
+ signature_idb_name = "signature_test_performance";
+
+ function dispatchQueue(context, function_used, argument_list,
+ number_queue) {
+ var result_promise_list = [],
+ i,
+ defer;
+ function pushAndExecute(global_defer) {
+ if ((global_defer.promise.isFulfilled) ||
+ (global_defer.promise.isRejected)) {
+ return;
+ }
+ if (argument_list.length > 0) {
+ function_used.apply(context, argument_list.shift())
+ .then(function () {
+ pushAndExecute(global_defer);
+ })
+ .fail(global_defer.reject);
+ return;
+ }
+ global_defer.resolve();
+ }
+ for (i = 0; i < number_queue; i += 1) {
+ defer = RSVP.defer();
+ result_promise_list.push(defer.promise);
+ pushAndExecute(defer);
+ }
+ if (number_queue > 1) {
+ return RSVP.all(result_promise_list);
+ }
+ return result_promise_list[0];
+ }
+
+ function deleteIndexedDB(idb_name) {
+ return new RSVP.Promise(function resolver(resolve, reject) {
+ var request = indexedDB.deleteDatabase(
+ idb_name
+ );
+ request.onerror = reject;
+ request.onblocked = reject;
+ request.onsuccess = resolve;
+ });
+ }
+
+ rJS(window)
+
+ .declareService(function () {
+ return this.run();
+ })
+
+ .declareMethod('run', function () {
+ var jio_options = {
+ type: "replicate",
+ query: {
+ query: 'portal_type:"Jio Perf"'
+ },
+ parallel_operation_amount: 10,
+ signature_hash_key: 'modification_date',
+ check_local_deletion: false,
+ check_local_modification: false,
+ check_remote_creation: false,
+ check_remote_deletion: false,
+ check_remote_modification: false,
+ local_sub_storage: {
+ /*
+ type: "erp5",
+ url: 'https://softinst114089.host.vifib.net/erp5/web_site_module/renderjs_runner/hateoas/',
+ default_view_reference: "jio_view"
+ */
+ type: "uuid",
+ sub_storage: {
+ type: "query",
+ sub_storage: {
+ // type: "memory",
+ type: "indexeddb",
+ database: local_idb_name
+ }
+ /*
+ type: "drivetojiomapping",
+ sub_storage: {
+ type: "dropbox",
+ access_token: ""
+ }
+ */
+ }
+ },
+ use_remote_post: true,
+ remote_sub_storage: {
+ /*
+ type: "uuid",
+ sub_storage: {
+ type: "query",
+ sub_storage: {
+ // type: "memory",
+ type: "indexeddb",
+ database: remote_idb_name
+ }
+ }
+ */
+ type: "erp5",
+ url: '',
+ default_view_reference: "jio_view"
+ },
+ signature_sub_storage: {
+ type: "query",
+ sub_storage: {
+ type: "memory",
+ // type: "indexeddb",
+ database: signature_idb_name
+ }
+ }
+ },
+ document_count = 1000,
+ parallel_operation = 10;
+
+ test('Test "' + jio_options.type + '"scenario', function () {
+ var jio,
+ i,
+ document_list = [];
+ stop();
+ expect(6);
+
+ // Create the storage
+ try {
+ jio = jIO.createJIO(jio_options);
+ } catch (error) {
+ console.error(error.stack);
+ console.error(error);
+ throw error;
+ }
+
+ function postNewDocument(index) {
+ // return RSVP.Queue();
+ /*
+ return RSVP.resolve(JSON.stringify({
+ title: index, modification_date: index,
+ parent_relative_url: 'jio_perf_module',
+ portal_type: 'Jio Perf'
+ }));
+ */
+ return jio.post({title: index, modification_date: index,
+ parent_relative_url: 'jio_perf_module',
+ portal_type: 'Jio Perf'});
+ }
+
+ // Put in the jio storage
+ return new RSVP.Queue()
+ .then(function () {
+ return RSVP.all([
+ deleteIndexedDB('jio:' + local_idb_name),
+ deleteIndexedDB('jio:' + remote_idb_name),
+ deleteIndexedDB('jio:' + signature_idb_name)
+ ]);
+ })
+ .then(function () {
+ ok(true, 'IDB deleted ' + new Date());
+ })
+ /*
+ .then(function () {
+ // Initialize all documents to create
+ for (i = 0; i < document_count; i += 1) {
+ document_list.push([{title: i, modification_date: i,
+ parent_relative_url: 'jio_perf_module',
+ portal_type: 'Jio Perf'}]);
+ }
+ equal(document_list.length, document_count,
+ 'Documents created ' + new Date());
+
+ return dispatchQueue(jio, jio.post, document_list,
+ parallel_operation);
+ })
+ */
+ .then(function () {
+ // Initialize all documents to create
+ for (i = 0; i < document_count; i += 1) {
+ document_list.push([i]);
+ }
+ equal(document_list.length, document_count,
+ 'Documents created ' + new Date());
+
+ return dispatchQueue(this, postNewDocument, document_list,
+ parallel_operation);
+ })
+ .then(function () {
+ return jio.allDocs();
+ })
+ .then(function (result) {
+ equal(result.data.total_rows, document_count,
+ 'Local Storage filled ' + new Date());
+ })
+ /*
+ .then(function () {
+ return jio.repair();
+ })
+ .then(function () {
+ ok(true, 'First sync finished ' + new Date());
+ })
+ .then(function () {
+ return jio.repair();
+ })
+ .then(function () {
+ ok(true, 'Second sync finished ' + new Date());
+ })
+ .then(function () {
+ return jio.repair();
+ })
+ .then(function () {
+ ok(true, 'Third sync finished ' + new Date());
+ })
+ */
+ .fail(function (error) {
+ console.error("---");
+ console.error(error.stack);
+ console.error(error);
+ ok(false, error);
+ })
+ .always(function () {
+ start();
+ });
+ });
+ });
+
+}(window, QUnit, jIO, rJS));
diff --git a/examples/perfidb.html b/examples/perfidb.html
new file mode 100644
index 0000000000000000000000000000000000000000..59fc93224feef332b8c6439f381ce00831689388
--- /dev/null
+++ b/examples/perfidb.html
@@ -0,0 +1,46 @@
+
+
+
+
+
+
+
+
+ jIO Performance Scenario
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ test markup, will be hidden
+
+
diff --git a/examples/perfidb.js b/examples/perfidb.js
new file mode 100644
index 0000000000000000000000000000000000000000..30d94d9c0e61948ad73fcba767c69dc21c89b1cc
--- /dev/null
+++ b/examples/perfidb.js
@@ -0,0 +1,177 @@
+/*
+ * Copyright 2014, Nexedi SA
+ *
+ * This program is free software: you can Use, Study, Modify and Redistribute
+ * it under the terms of the GNU General Public License version 3, or (at your
+ * option) any later version, as published by the Free Software Foundation.
+ *
+ * You can also Link and Combine this program with other software covered by
+ * the terms of any of the Free Software licenses or any of the Open Source
+ * Initiative approved licenses and Convey the resulting work. Corresponding
+ * source of such a combination shall include the source code for all other
+ * software used.
+ *
+ * This program is distributed WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See COPYING file for full licensing terms.
+ * See https://www.nexedi.com/licensing for rationale and options.
+ */
+
+/*global console, btoa, Blob, indexedDB*/
+/*jslint nomen: true, maxlen: 200*/
+(function (window, QUnit, jIO, rJS) {
+ "use strict";
+ var test = QUnit.test,
+ equal = QUnit.equal,
+ expect = QUnit.expect,
+ ok = QUnit.ok,
+ stop = QUnit.stop,
+ start = QUnit.start,
+ // deepEqual = QUnit.deepEqual;
+ document_idb_name = "index_document_test_performance",
+ index_idb_name = "index_test_performance";
+
+ function dispatchQueue(context, function_used, argument_list,
+ number_queue) {
+ var result_promise_list = [],
+ i,
+ defer;
+ function pushAndExecute(global_defer) {
+ if ((global_defer.promise.isFulfilled) ||
+ (global_defer.promise.isRejected)) {
+ return;
+ }
+ if (argument_list.length > 0) {
+ function_used.apply(context, argument_list.shift())
+ .then(function () {
+ pushAndExecute(global_defer);
+ })
+ .fail(global_defer.reject);
+ return;
+ }
+ global_defer.resolve();
+ }
+ for (i = 0; i < number_queue; i += 1) {
+ defer = RSVP.defer();
+ result_promise_list.push(defer.promise);
+ pushAndExecute(defer);
+ }
+ if (number_queue > 1) {
+ return RSVP.all(result_promise_list);
+ }
+ return result_promise_list[0];
+ }
+
+ rJS(window)
+
+ .declareService(function () {
+ return this.run();
+ })
+
+ .declareMethod('run', function () {
+ console.info(index_idb_name);
+ var jio_options = {
+ type: "uuid",
+ sub_storage: {
+ type: "indexeddb",
+ database: document_idb_name,
+ index_key_list: ["title", "portal_type"],
+ version: 3
+ }
+ },
+ document_count = 10000,
+ parallel_operation = 10;
+
+ test('Test index storage speed', function () {
+ var jio,
+ i,
+ document_list = [];
+ stop();
+ expect(6);
+
+ // Create the storage
+ try {
+ jio = jIO.createJIO(jio_options);
+ } catch (error) {
+ console.error(error.stack);
+ console.error(error);
+ throw error;
+ }
+
+ function postNewDocument(index) {
+ return jio.post({title: index, modification_date: index,
+ parent_relative_url: 'jio_perf_module',
+ portal_type: 'Jio Perf'});
+ }
+
+ function generateCheckQuerySpeed(query) {
+ return function () {
+ var now = Date.now();
+ return jio.allDocs(query)
+ .push(function (result) {
+ equal(result.data.total_rows, query.limit || document_count, query.index);
+ console.log(query.index, result.data.total_rows, Date.now() - now, 'ms');
+ });
+ };
+ }
+ function checkQueryListSpeed() {
+ var queue = new RSVP.Queue(),
+ query_list = arguments,
+ j;
+ for (j = 0; j < query_list.length; j += 1) {
+ queue.push(generateCheckQuerySpeed(query_list[j]));
+ }
+ return queue;
+ }
+
+ // Put in the jio storage
+ return new RSVP.Queue()
+ .then(function () {
+ ok(true, 'Index repair... ' + new Date());
+ // Ensure the index is correct
+ // return jio.repair();
+ })
+ .then(function () {
+ ok(true, 'Index repaired ' + new Date());
+ // Check if all documents have been created
+ return jio.allDocs();
+ })
+ .then(function (result) {
+ // Initialize all documents to create
+ for (i = result.data.total_rows; i < document_count; i += 1) {
+ document_list.push([i]);
+ }
+ ok(true, 'Documents created ' + new Date());
+ console.info('creating ' + document_list.length + ' documents');
+
+ return dispatchQueue(this, postNewDocument, document_list,
+ parallel_operation);
+ })
+ .then(function () {
+ return checkQueryListSpeed(
+ {},
+ // {limit: [0, 1]},
+ // Monovalued index
+ {index: {key: "portal_type", value: "Jio Perf"}},
+ // {query: 'portal_type:"Jio Perf"', limit: [0, 1]},
+ {index: {key: "portal_type", value: "NOTMATCHING"}},
+ // Multi valued index
+ {index: {key: "title", value: 1234}},
+ // {query: 'title:"12345"', limit: [0, 1]},
+ {index: {key: "title", value: "NOTMATCHING"}}
+ );
+ })
+ .fail(function (error) {
+ console.error("---");
+ console.error(error.stack);
+ console.error(error);
+ ok(false, error);
+ })
+ .always(function () {
+ start();
+ });
+ });
+ });
+
+}(window, QUnit, jIO, rJS));
\ No newline at end of file
diff --git a/package.json b/package.json
index b312d8bd8e2743eefadfee2099fbd988527bbb6c..7965a4c1f2583070c02224957b62c357a35add9f 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
{
"name": "jio",
"version": "v3.45.0",
- "license": "GPLv3+",
+ "license": "GPL-3.0-or-later",
"author": "Nexedi SA",
"contributors": [
"Tristan Cavelier ",
@@ -22,5 +22,11 @@
],
"engines": {
"npm": ">=1.3"
+ },
+ "devDependencies": {
+ "jslint": "^0.9.2",
+ "jison": "^0.4.16",
+ "qunit": "github:qunitjs/node-qunit#v0.9.3",
+ "sinon": "^1.7.3"
}
}
diff --git a/src/jio.js b/src/jio.js
index 789eec61d4c1399cf93da6e3ac3572628921290e..90b5e3e8f157b11b6bfbc65835529d425697a26c 100644
--- a/src/jio.js
+++ b/src/jio.js
@@ -467,6 +467,7 @@
return ensurePushableQueue(function () {
if (context.hasCapacity("list") &&
((options.query === undefined) || context.hasCapacity("query")) &&
+ ((options.index === undefined) || context.hasCapacity("index")) &&
((options.sort_on === undefined) || context.hasCapacity("sort")) &&
((options.group_by === undefined) || context.hasCapacity("group")) &&
((options.select_list === undefined) ||
diff --git a/src/jio.storage/indexeddbstorage.js b/src/jio.storage/indexeddbstorage.js
index 1bfa078ef46a1989831674cfab9d8f637f1c2880..51b3c723848e9dbca4286de6cddcd341758d90a2 100644
--- a/src/jio.storage/indexeddbstorage.js
+++ b/src/jio.storage/indexeddbstorage.js
@@ -65,7 +65,7 @@
}
IndexedDBStorage.prototype.hasCapacity = function (name) {
- return ((name === "list") || (name === "include"));
+ return ((name === "list") || (name === "include") || (name === "index"));
};
function buildKeyPath(key_list) {
@@ -268,7 +268,9 @@
IndexedDBStorage.prototype.buildQuery = function (options) {
var result_list = [],
- context = this;
+ context = this,
+ key = "_id",
+ value;
function pushIncludedMetadata(cursor) {
result_list.push({
@@ -285,20 +287,30 @@
});
}
+ if (options.index) {
+ if (context._index_key_list.indexOf(options.index.key) === -1) {
+ throw new jIO.util.jIOError(
+ "IndexedDB: unsupported index '" + options.index.key + "'",
+ 400
+ );
+ }
+ key = INDEX_PREFIX + options.index.key;
+ value = options.index.value;
+ }
+
return new RSVP.Queue()
.push(function () {
return waitForOpenIndexedDB(context, function (db) {
return waitForTransaction(db, ["metadata"], "readonly",
function (tx) {
- var key = "_id";
if (options.include_docs === true) {
return waitForAllSynchronousCursor(
- tx.objectStore("metadata").index(key).openCursor(),
+ tx.objectStore("metadata").index(key).openCursor(value),
pushIncludedMetadata
);
}
return waitForAllSynchronousCursor(
- tx.objectStore("metadata").index(key).openKeyCursor(),
+ tx.objectStore("metadata").index(key).openKeyCursor(value),
pushMetadata
);
});
diff --git a/test/jio.storage/indexeddbstorage.tests.js b/test/jio.storage/indexeddbstorage.tests.js
index ccee7435ba7db654b7efac32644304cc437331d5..4cd36ef8e2df03164f8a705f37727d90560735d6 100644
--- a/test/jio.storage/indexeddbstorage.tests.js
+++ b/test/jio.storage/indexeddbstorage.tests.js
@@ -700,6 +700,78 @@
});
});
+ test("Unhandled index", function () {
+ var context = this;
+ stop();
+ expect(3);
+
+ deleteIndexedDB(context.jio)
+ .then(function () {
+ return context.jio.allDocs({index: {key: 'a', value: '3'}});
+ })
+ .fail(function (error) {
+ ok(error instanceof jIO.util.jIOError);
+ equal(
+ error.message,
+ "IndexedDB: unsupported index 'a'"
+ );
+ equal(error.status_code, 400);
+ })
+ .fail(function (error) {
+ ok(false, error);
+ })
+ .always(function () {
+ start();
+ });
+ });
+
+ test("Handled index", function () {
+ var context = this;
+ this.jio = jIO.createJIO({
+ type: "indexeddb",
+ database: "qunit",
+ index_key_list: ['foo']
+ });
+ stop();
+ expect(1);
+
+ deleteIndexedDB(context.jio)
+ .then(function () {
+ return RSVP.all([
+ context.jio.put("1", {"foo": "bar"}),
+ context.jio.put("2", {"foo": "bar2"}),
+ context.jio.put("3", {"foo2": "bar"}),
+ context.jio.put("4", {"foo": "bar"})
+ ]);
+ })
+ .then(function () {
+ return context.jio.allDocs({index: {key: 'foo', value: 'bar'}});
+ })
+ .then(function (result) {
+ deepEqual(result, {
+ "data": {
+ "rows": [
+ {
+ "id": "1",
+ "value": {}
+ },
+ {
+ "id": "4",
+ "value": {}
+ }
+ ],
+ "total_rows": 2
+ }
+ });
+ })
+ .fail(function (error) {
+ ok(false, error);
+ })
+ .always(function () {
+ start();
+ });
+ });
+
/////////////////////////////////////////////////////////////////
// indexeddbStorage.get
/////////////////////////////////////////////////////////////////
diff --git a/test/queries/tests.js b/test/queries/tests.js
index 98637d5261b5f85574ab3880454406cfbb3d3289..290f25ed59786044a41262912b63f71816565943 100644
--- a/test/queries/tests.js
+++ b/test/queries/tests.js
@@ -17,8 +17,8 @@
* See COPYING file for full licensing terms.
* See https://www.nexedi.com/licensing for rationale and options.
*/
-/*global jiodate*/
-(function (jIO, jiodate) {
+/*global jiodate, SimpleQuery*/
+(function (jIO, jiodate, SimpleQuery) {
"use strict";
var test = QUnit.test,
stop = QUnit.stop,
@@ -479,9 +479,13 @@
{"identifier": "a", "value": "test post", "time": "2016"},
{"identifier": "b", "value": "test post 1", "time": "2017"},
{"identifier": "c", "value": "test post 2016", "time": "2017"}
- ];
+ ], lala;
stop();
- expect(2);
+ expect(3);
+ lala = jIO.QueryFactory.create('test post');
+ ok(lala instanceof SimpleQuery, lala);
+ /*global console */
+ console.log(lala);
jIO.QueryFactory.create('test post').exec(doc_list).
then(function (doc_list) {
deepEqual(doc_list, [
@@ -734,4 +738,4 @@
}).always(start);
});
-}(jIO, jiodate));
+}(jIO, jiodate, SimpleQuery));