diff --git a/src/jio.storage/indexstorage2.js b/src/jio.storage/indexstorage2.js index 0b2ab1c..2c63ec4 100644 --- a/src/jio.storage/indexstorage2.js +++ b/src/jio.storage/indexstorage2.js @@ -19,10 +19,10 @@ */ /*jslint nomen: true */ /*global indexedDB, jIO, RSVP, IDBOpenDBRequest, DOMError, Event, - parseStringToObject, Set*/ + parseStringToObject, Set, DOMException*/ (function (indexedDB, jIO, RSVP, IDBOpenDBRequest, DOMError, - parseStringToObject) { + parseStringToObject, DOMException) { "use strict"; function IndexStorage2(description) { @@ -35,11 +35,16 @@ throw new TypeError("IndexStorage2 'index_keys' description property " + "must be an Array"); } + if (description.version && (typeof description.version !== "number")) { + throw new TypeError("IndexStorage2 'version' description property " + + "must be a number"); + } this._sub_storage_description = description.sub_storage; this._sub_storage = jIO.createJIO(description.sub_storage); this._database_name = "jio:" + description.database; this._index_keys = description.index_keys || []; - this._version = description.version || undefined; + this._version = description.version; + this._signature_storage_name = description.database + "_signatures"; } IndexStorage2.prototype.hasCapacity = function (name) { @@ -76,55 +81,194 @@ }); } + function iterateCursor(on, query, limit) { + return new RSVP.Promise(function (resolve, reject) { + var result = [], count = 0, cursor; + cursor = on.openKeyCursor(query); + cursor.onsuccess = function (cursor) { + if (cursor.target.result && count !== limit) { + count += 1; + result.push({id: cursor.target.result.primaryKey, value: {}}); + cursor.target.result.continue(); + } else { + resolve(result); + } + }; + cursor.onerror = function (error) { + reject(error.message); + }; + }); + } + function VirtualIDB(description) { - this._write_operations = description.write_operations; + this._operations = description.operations; } - VirtualIDB.prototype.put = function () { - this._write_operations.put.push(arguments); + VirtualIDB.prototype.hasCapacity = function (name) { + return (name === "list"); }; - VirtualIDB.prototype.hasCapacity = function (name) { - return (name === 'list') || (name === 'select'); + VirtualIDB.prototype.put = function (id, value) { + var context = this; + return new RSVP.Promise(function (resolve, reject) { + context._operations.push({type: "put", arguments: [id, value], + onsuccess: resolve, onerror: reject}); + }); + }; + + VirtualIDB.prototype.remove = function (id) { + var context = this; + return new RSVP.Promise(function (resolve, reject) { + context._operations.push({type: "remove", arguments: [id], + onsuccess: resolve, onerror: reject}); + }); }; VirtualIDB.prototype.get = function (id) { - throw new jIO.util.jIOError("Cannot find document: " + id, 404); + var context = this; + return new RSVP.Promise(function (resolve, reject) { + context._operations.push({type: "get", arguments: [id], + onsuccess: resolve, onerror: reject}); + }); + }; + + VirtualIDB.prototype.buildQuery = function (options) { + var context = this; + return new RSVP.Promise(function (resolve, reject) { + context._operations.push({type: "buildQuery", arguments: [options], + onsuccess: resolve, onerror: reject}); + }); }; - VirtualIDB.prototype.buildQuery = function () { - return []; + VirtualIDB.prototype.allAttachments = function () { + return {}; }; jIO.addStorage("virtualidb", VirtualIDB); - function getRepairStorage(write_operations, sub_storage_description) { + function getRepairStorage(operations, sub_storage_description, + signature_storage_name) { return jIO.createJIO({ type: "replicate", local_sub_storage: sub_storage_description, - check_local_modification: false, - check_local_deletion: false, - check_local_creation: true, - check_remote_modification: false, - check_remote_creation: false, - check_remote_deletion: false, remote_sub_storage: { type: "virtualidb", - write_operations: write_operations, + operations: operations }, signature_sub_storage: { type: "query", sub_storage: { - type: "memory" + type: "indexeddb", + database: signature_storage_name } - } + }, + check_remote_modification: false, + check_remote_creation: false, + check_remote_deletion: false, + conflict_handling: 1, + parallel_operation_amount: 16 }); } - function handleUpgradeNeeded(evt, index_keys, sub_storage_description) { - var db = evt.target.result, store, i, current_indices, required_indices, - put_promise_list = [], repair_promise, repeatUntilPromiseFulfilled, - write_operations; + var transaction_failure_reason; + + function handleVirtualGetSuccess(id, onsuccess, onerror) { + return function (result) { + if (result.target.result === undefined) { + return onerror(new jIO.util.jIOError("Cannot find document: " + + id, 404)); + } + return onsuccess(result.target.result.doc); + }; + } + + function processVirtualOperation(operation, store, index_keys, disable_get) { + var request, get_success_handler; + if (operation.type === "put") { + request = store.put({ + id: operation.arguments[0], + doc: filterDocValues(operation.arguments[1], index_keys), + }); + request.onerror = operation.onerror; + return {request: request, onsuccess: operation.onsuccess}; + } + if (operation.type === "get") { + // if storage was cleared, get can return without checking the database + if (disable_get) { + operation.onerror(new jIO.util.jIOError("Cannot find document: " + + operation.arguments[0], 404)); + } else { + get_success_handler = handleVirtualGetSuccess(operation.arguments[0], + operation.onsuccess, operation.onerror); + request = store.get(operation.arguments[0]); + request.onerror = operation.onerror; + return {request: request, onsuccess: get_success_handler}; + } + } + if (operation.type === "buildQuery") { + request = iterateCursor(store); + request.then(operation.onsuccess).fail(operation.onerror); + return; + } + if (operation.type === "remove") { + request = store.delete(operation.arguments[0]); + request.onerror = operation.onerror; + return {request: request, onsuccess: operation.onsuccess}; + } + } + + function repairInTransaction(sub_storage_description, transaction, + index_keys, signature_storage_name, clear_storage) { + var repair_promise, repeatUntilPromiseFulfilled, store, + operations = []; + if (clear_storage) { + indexedDB.deleteDatabase("jio:" + signature_storage_name); + } + store = transaction.objectStore("index-store"); + repair_promise = getRepairStorage(operations, + sub_storage_description, signature_storage_name).repair(); + repeatUntilPromiseFulfilled = function repeatUntilPromiseFulfilled( + continuation_request, + continuation_resolve + ) { + var operation_result, next_continuation_request, + next_continuation_resolve; + continuation_request.onsuccess = function () { + if (continuation_resolve) { + continuation_resolve.apply(null, arguments); + } + while (true) { + if (operations.length === 0) { + break; + } + operation_result = processVirtualOperation(operations.shift(), store, + index_keys, clear_storage); + // use the current request to continue the repeat loop if possible + if (next_continuation_request && operation_result) { + operation_result.request.onsuccess = operation_result.onsuccess; + } else if (operation_result) { + next_continuation_request = operation_result.request; + next_continuation_resolve = operation_result.onsuccess; + } + } + if (repair_promise.isRejected) { + transaction.abort(); + transaction_failure_reason = repair_promise.rejectedReason; + return; + } + if (repair_promise.isFulfilled) { + return; + } + return repeatUntilPromiseFulfilled(next_continuation_request || + store.get("inexistent"), next_continuation_resolve); + }; + }; + repeatUntilPromiseFulfilled(store.get("inexistent")); + } + + function handleUpgradeNeeded(evt, index_keys, sub_storage_description, + signature_storage_name) { + var db = evt.target.result, store, i, current_indices, required_indices; required_indices = new Set(index_keys.map(function (name) { return 'Index-' + name; })); @@ -134,12 +278,11 @@ current_indices = new Set(store ? store.indexNames : []); if (isSubset(current_indices, required_indices)) { - if (!store) { - return; - } - for (i = 0; i < store.indexNames.length; i += 1) { - if (!required_indices.has(store.indexNames[i])) { - store.deleteIndex(store.indexNames[i]); + if (store) { + for (i = 0; i < store.indexNames.length; i += 1) { + if (!required_indices.has(store.indexNames[i])) { + store.deleteIndex(store.indexNames[i]); + } } } } else { @@ -155,49 +298,33 @@ store.createIndex('Index-' + index_keys[i], 'doc.' + index_keys[i], { unique: false }); } - - write_operations = {put: []}; - repair_promise = getRepairStorage(write_operations, - sub_storage_description).repair(); - repeatUntilPromiseFulfilled = function repeatUntilPromiseFulfilled(req) { - req.onsuccess = function () { - if (repair_promise.isRejected) { - evt.target.transaction.abort(); - return; - } - if (repair_promise.isFulfilled) { - for (i = 0; i < write_operations.put.length; i += 1) { - put_promise_list.push(waitForIDBRequest(store.put({ - id: write_operations.put[i][0], - doc: filterDocValues(write_operations.put[i][1], index_keys) - }))); - } - write_operations.put = []; - return RSVP.all(put_promise_list); - } - return repeatUntilPromiseFulfilled(store.getAll()); - }; - }; - repeatUntilPromiseFulfilled(store.getAll()); + return repairInTransaction(sub_storage_description, + evt.target.transaction, index_keys, signature_storage_name, true); } } function waitForOpenIndexedDB(db_name, version, index_keys, - sub_storage_description, callback) { + sub_storage_description, signature_storage_name, callback) { function resolver(resolve, reject) { // Open DB // var request = indexedDB.open(db_name, version); request.onerror = function (error) { + var error_sub_message; if (request.result) { request.result.close(); } if ((error !== undefined) && (error.target instanceof IDBOpenDBRequest) && - (error.target.error instanceof DOMError)) { - reject("Connection to: " + db_name + " failed: " + - error.target.error.message); + ((error.target.error instanceof DOMError) || + (error.target.error instanceof DOMException))) { + error_sub_message = error.target.error.message; + if (transaction_failure_reason) { + error_sub_message += " " + transaction_failure_reason; + transaction_failure_reason = undefined; + } + reject("Connection to: " + db_name + " failed: " + error_sub_message); } else { - reject(error.target.error); + reject(error); } }; @@ -218,7 +345,8 @@ // Create DB if necessary // request.onupgradeneeded = function (evt) { - handleUpgradeNeeded(evt, index_keys, sub_storage_description); + handleUpgradeNeeded(evt, index_keys, sub_storage_description, + signature_storage_name); }; request.onversionchange = function () { @@ -283,32 +411,14 @@ return new RSVP.Promise(resolver, canceller); } - IndexStorage2.prototype._iterateCursor = function (on, query, limit) { - return new RSVP.Promise(function (resolve, reject) { - var result_list = [], count = 0, cursor; - cursor = on.openKeyCursor(query); - cursor.onsuccess = function (cursor) { - if (cursor.target.result && count !== limit) { - count += 1; - result_list.push({id: cursor.target.result.primaryKey, value: {}}); - cursor.target.result.continue(); - } else { - resolve(result_list); - } - }; - cursor.onerror = function (error) { - reject(error.message); - }; - }); - }; - IndexStorage2.prototype._runQuery = function (key, value, limit) { var context = this; return waitForOpenIndexedDB(context._database_name, context._version, - context._index_keys, context._sub_storage_description, function (db) { + context._index_keys, context._sub_storage_description, + context._signature_storage_name, function (db) { return waitForTransaction(db, ["index-store"], "readonly", function (tx) { - return context._iterateCursor(tx.objectStore("index-store") + return iterateCursor(tx.objectStore("index-store") .index("Index-" + key), value, limit); }); }); @@ -349,7 +459,8 @@ return; } return waitForOpenIndexedDB(context._database_name, context._version, - context._index_keys, context._sub_storage_description, function (db) { + context._index_keys, context._sub_storage_description, + context._signature_storage_name, function (db) { return waitForTransaction(db, ["index-store"], "readwrite", function (tx) { return waitForIDBRequest(tx.objectStore("index-store").put({ @@ -381,7 +492,8 @@ return context._sub_storage.remove(id) .push(function () { return waitForOpenIndexedDB(context._database_name, context._version, - context._index_keys, context._sub_storage_description, function (db) { + context._index_keys, context._sub_storage_description, + context._signature_storage_name, function (db) { return waitForTransaction(db, ["index-store"], "readwrite", function (tx) { return waitForIDBRequest(tx.objectStore("index-store") @@ -391,6 +503,19 @@ }); }; + IndexStorage2.prototype.repair = function () { + var context = this; + return waitForOpenIndexedDB(context._database_name, context._version, + context._index_keys, context._sub_storage_description, + context._signature_storage_name, function (db) { + return waitForTransaction(db, ["index-store"], "readwrite", + function (tx) { + return repairInTransaction(context._sub_storage_description, tx, + context._index_keys, context._signature_storage_name); + }); + }); + }; + IndexStorage2.prototype.getAttachment = function () { return this._sub_storage.getAttachment.apply(this._sub_storage, arguments); }; @@ -405,4 +530,5 @@ }; jIO.addStorage("index2", IndexStorage2); -}(indexedDB, jIO, RSVP, IDBOpenDBRequest, DOMError, parseStringToObject)); \ No newline at end of file +}(indexedDB, jIO, RSVP, IDBOpenDBRequest, DOMError, parseStringToObject, + DOMException)); \ No newline at end of file diff --git a/test/jio.storage/indexstorage2.tests.js b/test/jio.storage/indexstorage2.tests.js index 07a719e..4594ecc 100644 --- a/test/jio.storage/indexstorage2.tests.js +++ b/test/jio.storage/indexstorage2.tests.js @@ -31,14 +31,19 @@ module = QUnit.module, throws = QUnit.throws; - function deleteIndexedDB(storage) { + function deleteIndexStorage2(storage) { return new RSVP.Promise(function resolver(resolve, reject) { - var request = indexedDB.deleteDatabase( + var storage_deletion_request = indexedDB.deleteDatabase( storage.__storage._database_name + ), signature_deletion_request = indexedDB.deleteDatabase( + "jio:" + storage.__storage._signature_storage_name ); - request.onerror = reject; - request.onblocked = reject; - request.onsuccess = resolve; + storage_deletion_request.onerror = reject; + storage_deletion_request.onblocked = reject; + storage_deletion_request.onsuccess = resolve; + signature_deletion_request.onerror = reject; + signature_deletion_request.onblocked = reject; + signature_deletion_request.onsuccess = resolve; }); } @@ -65,7 +70,7 @@ ///////////////////////////////////////////////////////////////// module("indexStorage2.constructor", { teardown: function () { - deleteIndexedDB(this.jio); + deleteIndexStorage2(this.jio); } }); test("Constructor without index_keys", function () { @@ -80,6 +85,7 @@ equal(this.jio.__type, "index2"); equal(this.jio.__storage._sub_storage.__type, "dummystorage3"); equal(this.jio.__storage._database_name, "jio:index2_test"); + equal(this.jio.__storage._signature_storage_name, "index2_test_signatures"); deepEqual(this.jio.__storage._index_keys, []); }); @@ -141,12 +147,32 @@ return true; } ); + throws( + function () { + this.jio = jIO.createJIO({ + type: "index2", + database: "index2_test", + index_keys: ["a", "b"], + version: "1", + sub_storage: { + type: "dummystorage3" + } + }); + }, + function (error) { + ok(error instanceof TypeError); + equal(error.message, "IndexStorage2 'version' description property" + + " must be a number"); + return true; + } + ); }); - test("Constructor with index_keys", function () { + test("Constructor with index_keys and version", function () { this.jio = jIO.createJIO({ type: "index2", database: "index2_test", + version: 4, index_keys: ["a", "b"], sub_storage: { type: "dummystorage3" @@ -156,6 +182,10 @@ equal(this.jio.__type, "index2"); equal(this.jio.__storage._sub_storage.__type, "dummystorage3"); equal(this.jio.__storage._database_name, "jio:index2_test"); + equal(this.jio.__storage._version, 4); + equal(this.jio.__storage._signature_storage_name, "index2_test_signatures"); + deepEqual(this.jio.__storage._sub_storage_description, + {type: "dummystorage3"}); deepEqual(this.jio.__storage._index_keys, ["a", "b"]); }); @@ -164,10 +194,10 @@ ///////////////////////////////////////////////////////////////// module("indexStorage2.hasCapacity", { teardown: function () { - deleteIndexedDB(this.jio); + deleteIndexStorage2(this.jio); } }); - test("can list documents", function () { + test("Test various capacities", function () { this.jio = jIO.createJIO({ type: "index2", database: "index2_test", @@ -213,7 +243,7 @@ }); }, teardown: function () { - deleteIndexedDB(this.jio); + deleteIndexStorage2(this.jio); } }); test("Get calls substorage", function () { @@ -251,7 +281,7 @@ }; }, teardown: function () { - deleteIndexedDB(this.jio); + deleteIndexStorage2(this.jio); } }); @@ -650,26 +680,21 @@ stop(); expect(8); - dummy_data = { - "32": {id: "32", doc: {"a": "3", "b": "2", "c": "inverse"}, - value: {"a": "3", "b": "2", "c": "inverse"}}, - "5": {id: "5", doc: {"a": "6", "b": "2", "c": "strong"}, - value: {"a": "6", "b": "2", "c": "strong"}}, - "14": {id: "14", doc: {"a": "67", "b": "3", "c": "disolve"}, - value: {"a": "67", "b": "3", "c": "disolve"}} - }; + dummy_data = {}; DummyStorage3.prototype.put = function (id, value) { dummy_data[id] = {id: id, doc: value, value: value}; return id; }; DummyStorage3.prototype.get = function (id) { - return dummy_data[id].doc; + if (dummy_data[id]) { + return dummy_data[id].doc; + } + throw new jIO.util.jIOError("Cannot find document: " + id, 404); }; DummyStorage3.prototype.hasCapacity = function (name) { return (name === 'list') || (name === 'include') || (name === 'select'); }; - DummyStorage3.prototype.buildQuery = function () { return Object.values(dummy_data); }; @@ -781,8 +806,164 @@ context.jio.allDocs({query: 'c: "control"'}) .fail(function (error) { - equal(error.message, "Version change transaction was aborted in" + - " upgradeneeded event handler."); + equal(error, "Connection to: jio:index2_test failed: Version change " + + "transaction was aborted in upgradeneeded event handler. " + + "Error: Capacity 'buildQuery' is not implemented on 'dummystorage3'"); + }) + .always(function () { + start(); + }); + }); + + test("Manual repair", function () { + var context = this, fake_data; + context.jio = jIO.createJIO({ + type: "index2", + database: "index2_test", + index_keys: ["a", "c"], + sub_storage: { + type: "dummystorage3" + } + }); + stop(); + expect(15); + + fake_data = { + "1": {a: "id54", b: "9", c: "night"}, + "4": {a: "vn92", b: "7", c: "matter"}, + "9": {a: "ru23", b: "3", c: "control"}, + "42": {a: "k422", b: "100", c: "grape"} + }; + + DummyStorage3.prototype.hasCapacity = function (name) { + return (name === 'list') || (name === 'select'); + }; + DummyStorage3.prototype.put = function (id, value) { + fake_data[id] = value; + return id; + }; + DummyStorage3.prototype.get = function (id) { + if (fake_data[id]) { + return fake_data[id]; + } + throw new jIO.util.jIOError("Cannot find document: " + id, 404); + }; + DummyStorage3.prototype.remove = function (id) { + delete fake_data[id]; + return id; + }; + DummyStorage3.prototype.buildQuery = function () { + var keys = Object.keys(fake_data); + return keys.map(function (v) { return {id: v, value: {}}; }); + }; + + context.jio.allDocs({query: 'c: "control"'}) + .then(function (result) { + equal(result.data.total_rows, 1); + deepEqual(result.data.rows, [{id: "9", value: {}}]); + }) + .then(function () { + fake_data["2"] = {a: "zu64", b: "1", c: "matter"}; + fake_data["13"] = {a: "tk32", b: "9", c: "matter"}; + return context.jio.repair(); + }) + .then(function () { + return context.jio.allDocs({query: 'c: "matter"'}); + }) + .then(function (result) { + equal(result.data.total_rows, 3); + deepEqual(result.data.rows.sort(idCompare), [{id: "13", value: {}}, + {id: "2", value: {}}, {id: "4", value: {}}]); + }) + .then(function () { + fake_data["2"] = {a: "zu64", b: "1", c: "observe"}; + return context.jio.repair(); + }) + .then(function () { + return context.jio.allDocs({query: 'c: "observe"'}); + }) + .then(function (result) { + equal(result.data.total_rows, 1); + deepEqual(result.data.rows, [{id: "2", value: {}}]); + }) + .then(function () { + delete fake_data["2"]; + return context.jio.repair(); + }) + .then(function () { + return context.jio.allDocs({query: 'c: "observe"'}); + }) + .then(function (result) { + equal(result.data.total_rows, 0); + deepEqual(result.data.rows, []); + }) + .then(function () { + return RSVP.all([ + context.jio.put("43", {a: "t345", b: "101", c: "pear"}), + context.jio.put("44", {a: "j939", b: "121", c: "grape"}), + context.jio.put("45", {a: "q423", b: "131", c: "grape"}), + context.jio.remove("42") + ]); + }) + .then(function () { + return context.jio.repair(); + }) + .then(function () { + return context.jio.allDocs({query: "c:grape"}); + }) + .then(function (result) { + equal(result.data.total_rows, 2); + deepEqual(result.data.rows.sort(idCompare), [{id: "44", value: {}}, + {id: "45", value: {}}]); + }) + .then(function () { + context.jio = jIO.createJIO({ + type: "index2", + database: "index2_test", + index_keys: ["a", "c"], + sub_storage: { + type: "dummystorage3" + } + }); + delete fake_data["13"]; + fake_data["3"] = {a: "gg38", b: "4", c: "matter"}; + fake_data["9"] = {a: "tk32", b: "9", c: "matter"}; + return context.jio.repair(); + }) + .then(function () { + return context.jio.allDocs({query: "c:matter"}); + }) + .then(function (result) { + equal(result.data.total_rows, 3); + deepEqual(result.data.rows.sort(idCompare), [{id: "3", value: {}}, + {id: "4", value: {}}, {id: "9", value: {}}]); + }) + .then(function () { + delete fake_data["9"]; + fake_data["3"] = {a: "xu76", b: "9", c: "night"}; + fake_data["7"] = {a: "bn02", b: "9", c: "matter"}; + context.jio = jIO.createJIO({ + type: "index2", + database: "index2_test", + index_keys: ["b"], + version: 2, + sub_storage: { + type: "dummystorage3" + } + }); + return context.jio.allDocs({query: "b:9"}); + }) + .then(function (result) { + equal(result.data.total_rows, 3); + deepEqual(result.data.rows.sort(idCompare), [{id: "1", value: {}}, + {id: "3", value: {}}, {id: "7", value: {}}]); + }) + .then(function () { + return context.jio.allDocs({query: "c:matter"}); + }) + .fail(function (error) { + equal(error.message, + "Capacity 'query' is not implemented on 'dummystorage3'"); }) .always(function () { start(); @@ -794,7 +975,7 @@ ///////////////////////////////////////////////////////////////// module("IndexStorage2.getAttachment", { teardown: function () { - deleteIndexedDB(this.jio); + deleteIndexStorage2(this.jio); } }); test("getAttachment called substorage getAttachment", function () { @@ -834,7 +1015,7 @@ ///////////////////////////////////////////////////////////////// module("IndexStorage2.putAttachment", { teardown: function () { - deleteIndexedDB(this.jio); + deleteIndexStorage2(this.jio); } }); test("putAttachment called substorage putAttachment", function () { @@ -875,7 +1056,7 @@ ///////////////////////////////////////////////////////////////// module("IndexStorage2.removeAttachment", { teardown: function () { - deleteIndexedDB(this.jio); + deleteIndexStorage2(this.jio); } }); test("removeAttachment called substorage removeAttachment", function () { @@ -914,7 +1095,7 @@ ///////////////////////////////////////////////////////////////// module("indexStorage2.put", { teardown: function () { - deleteIndexedDB(this.jio); + deleteIndexStorage2(this.jio); } }); test("Put creates index", function () { @@ -998,7 +1179,7 @@ ///////////////////////////////////////////////////////////////// module("indexStorage2.post", { teardown: function () { - deleteIndexedDB(this.jio); + deleteIndexStorage2(this.jio); } }); test("Post creates index", function () { @@ -1090,7 +1271,7 @@ ///////////////////////////////////////////////////////////////// module("indexStorage2.remove", { teardown: function () { - deleteIndexedDB(this.jio); + deleteIndexStorage2(this.jio); } }); test("Remove values", function () {