Skip to content

jio
jio
Commit 78cca6d2 authored by preetwinder's avatar preetwinder
Browse files
Options

Add support for manual repair

parent 04daba61
Hide whitespace changes
Inline Side-by-side
Showing with 353 additions and 78 deletions
src/jio.storage/indexstorage2.js
View file @ 78cca6d2
......@@ -39,7 +39,8 @@
this._sub_storage = jIO.createJIO(description.sub_storage);
this._database_name = "jio:" + description.database;
this._index_keys = description.index_keys || [];
this._version = description.version || undefined;
this._version = description.version;
this._signature_storage_name = description.database + "_signatures";
}
IndexStorage2.prototype.hasCapacity = function (name) {
......@@ -76,55 +77,189 @@
});
}
function VirtualIDB(description) {
this._write_operations = description.write_operations;
function iterateCursor(on, query, limit) {
return new RSVP.Promise(function (resolve, reject) {
var result = [], count = 0, cursor;
cursor = on.openKeyCursor(query);
cursor.onsuccess = function (cursor) {
if (cursor.target.result && count !== limit) {
count += 1;
result.push({id: cursor.target.result.primaryKey, value: {}});
cursor.target.result.continue();
} else {
resolve(result);
}
};
cursor.onerror = function (error) {
reject(error.message);
};
});
}
VirtualIDB.prototype.put = function () {
this._write_operations.put.push(arguments);
};
function VirtualIDB(description) {
this._operations = description.operations;
}
VirtualIDB.prototype.hasCapacity = function (name) {
return (name === 'list') || (name === 'select');
};
VirtualIDB.prototype.put = function (id, value) {
var context = this;
return new RSVP.Promise(function (resolve, reject) {
context._operations.push({type: 'put', arguments: [id, value],
onsuccess: resolve, onerror: reject});
});
};
VirtualIDB.prototype.remove = function (id) {
var context = this;
return new RSVP.Promise(function (resolve, reject) {
context._operations.push({type: 'remove', arguments: [id],
onsuccess: resolve, onerror: reject});
});
};
VirtualIDB.prototype.get = function (id) {
throw new jIO.util.jIOError("Cannot find document: " + id, 404);
var context = this;
return new RSVP.Promise(function (resolve, reject) {
context._operations.push({type: 'get', arguments: [id],
onsuccess: resolve, onerror: reject});
});
};
VirtualIDB.prototype.buildQuery = function () {
return [];
VirtualIDB.prototype.buildQuery = function (options) {
var context = this;
return new RSVP.Promise(function (resolve, reject) {
context._operations.push({type: 'buildQuery', arguments: [options],
onsuccess: resolve, onerror: reject});
});
};
VirtualIDB.prototype.allAttachments = function () {
return {};
};
jIO.addStorage("virtualidb", VirtualIDB);
function getRepairStorage(write_operations, sub_storage_description) {
function getRepairStorage(operations, sub_storage_description,
signature_storage_name) {
return jIO.createJIO({
type: "replicate",
local_sub_storage: sub_storage_description,
check_local_modification: false,
check_local_deletion: false,
check_local_modification: true,
check_local_deletion: true,
check_local_creation: true,
check_remote_modification: false,
check_remote_creation: false,
check_remote_deletion: false,
conflict_handling: 1,
parallel_operation_amount: 16,
remote_sub_storage: {
type: "virtualidb",
write_operations: write_operations,
operations: operations
},
signature_sub_storage: {
type: "query",
sub_storage: {
type: "memory"
type: "indexeddb",
database: signature_storage_name
}
}
});
}
function handleUpgradeNeeded(evt, index_keys, sub_storage_description) {
var db = evt.target.result, store, i, current_indices, required_indices,
put_promise_list = [], repair_promise, repeatUntilPromiseFulfilled,
write_operations;
function repairInTransaction(sub_storage_description, transaction,
index_keys, signature_storage_name, clear_signature) {
var repair_promise, repeatUntilPromiseFulfilled, store,
operations = [], handle_get;
if (clear_signature) {
indexedDB.deleteDatabase("jio:" + signature_storage_name);
}
handle_get = function handle_get(id, onsuccess, onerror) {
return function (result) {
if (result.target.result === undefined) {
return onerror(new jIO.util.jIOError("Cannot find document: " +
id, 404));
}
return onsuccess(result.target.result.doc);
};
};
store = transaction.objectStore('index-store');
repair_promise = getRepairStorage(operations,
sub_storage_description, signature_storage_name).repair();
repeatUntilPromiseFulfilled = function repeatUntilPromiseFulfilled(
continuation_request,
continuation_resolve
) {
var operation, request, next_continuation_request,
next_continuation_resolve;
continuation_request.onsuccess = function () {
if (continuation_resolve) {
continuation_resolve.apply(null, arguments);
}
while (true) {
if (operations.length === 0) {
break;
}
operation = operations.shift();
if (operation.type === 'put') {
request = store.put({
id: operation.arguments[0],
doc: filterDocValues(operation.arguments[1], index_keys),
});
if (!next_continuation_request) {
next_continuation_request = request;
next_continuation_resolve = operation.onsuccess;
} else {
request.onsuccess = operation.onsuccess;
}
request.onerror = operation.onerror;
} else if (operation.type === 'get') {
request = store.get(operation.arguments[0]);
if (!next_continuation_request) {
next_continuation_request = request;
next_continuation_resolve = handle_get(operation.arguments[0],
operation.onsuccess, operation.onerror);
} else {
request.onsuccess = handle_get(operation.arguments[0],
operation.onsuccess, operation.onerror);
}
request.onerror = operation.onerror;
} else if (operation.type === 'buildQuery') {
request = iterateCursor(store);
request.then(operation.onsuccess).fail(operation.onerror);
} else if (operation.type === 'remove') {
request = store.delete(operation.arguments[0]);
if (!next_continuation_request) {
next_continuation_request = request;
next_continuation_resolve = operation.onsuccess;
} else {
request.onsuccess = operation.onsuccess;
}
request.onerror = operation.onerror;
}
}
if (repair_promise.isRejected) {
transaction.abort();
return;
}
if (repair_promise.isFulfilled) {
return;
}
if (next_continuation_request) {
return repeatUntilPromiseFulfilled(next_continuation_request,
next_continuation_resolve);
}
return repeatUntilPromiseFulfilled(store.count());
};
};
repeatUntilPromiseFulfilled(store.count());
}
function handleUpgradeNeeded(evt, index_keys, sub_storage_description,
signature_storage_name) {
var db = evt.target.result, store, i, current_indices, required_indices;
required_indices = new Set(index_keys.map(function (name) {
return 'Index-' + name;
}));
......@@ -155,35 +290,13 @@
store.createIndex('Index-' + index_keys[i],
'doc.' + index_keys[i], { unique: false });
}
write_operations = {put: []};
repair_promise = getRepairStorage(write_operations,
sub_storage_description).repair();
repeatUntilPromiseFulfilled = function repeatUntilPromiseFulfilled(req) {
req.onsuccess = function () {
if (repair_promise.isRejected) {
evt.target.transaction.abort();
return;
}
if (repair_promise.isFulfilled) {
for (i = 0; i < write_operations.put.length; i += 1) {
put_promise_list.push(waitForIDBRequest(store.put({
id: write_operations.put[i][0],
doc: filterDocValues(write_operations.put[i][1], index_keys)
})));
}
write_operations.put = [];
return RSVP.all(put_promise_list);
}
return repeatUntilPromiseFulfilled(store.getAll());
};
};
repeatUntilPromiseFulfilled(store.getAll());
return repairInTransaction(sub_storage_description,
evt.target.transaction, index_keys, signature_storage_name, true);
}
}
function waitForOpenIndexedDB(db_name, version, index_keys,
sub_storage_description, callback) {
sub_storage_description, signature_storage_name, callback) {
function resolver(resolve, reject) {
// Open DB //
var request = indexedDB.open(db_name, version);
......@@ -218,7 +331,8 @@
// Create DB if necessary //
request.onupgradeneeded = function (evt) {
handleUpgradeNeeded(evt, index_keys, sub_storage_description);
handleUpgradeNeeded(evt, index_keys, sub_storage_description,
signature_storage_name);
};
request.onversionchange = function () {
......@@ -283,32 +397,14 @@
return new RSVP.Promise(resolver, canceller);
}
IndexStorage2.prototype._iterateCursor = function (on, query, limit) {
return new RSVP.Promise(function (resolve, reject) {
var result_list = [], count = 0, cursor;
cursor = on.openKeyCursor(query);
cursor.onsuccess = function (cursor) {
if (cursor.target.result && count !== limit) {
count += 1;
result_list.push({id: cursor.target.result.primaryKey, value: {}});
cursor.target.result.continue();
} else {
resolve(result_list);
}
};
cursor.onerror = function (error) {
reject(error.message);
};
});
};
IndexStorage2.prototype._runQuery = function (key, value, limit) {
var context = this;
return waitForOpenIndexedDB(context._database_name, context._version,
context._index_keys, context._sub_storage_description, function (db) {
context._index_keys, context._sub_storage_description,
context._signature_storage_name, function (db) {
return waitForTransaction(db, ["index-store"], "readonly",
function (tx) {
return context._iterateCursor(tx.objectStore("index-store")
return iterateCursor(tx.objectStore("index-store")
.index("Index-" + key), value, limit);
});
});
......@@ -349,7 +445,8 @@
return;
}
return waitForOpenIndexedDB(context._database_name, context._version,
context._index_keys, context._sub_storage_description, function (db) {
context._index_keys, context._sub_storage_description,
context._signature_storage_name, function (db) {
return waitForTransaction(db, ["index-store"], "readwrite",
function (tx) {
return waitForIDBRequest(tx.objectStore("index-store").put({
......@@ -381,7 +478,8 @@
return context._sub_storage.remove(id)
.push(function () {
return waitForOpenIndexedDB(context._database_name, context._version,
context._index_keys, context._sub_storage_description, function (db) {
context._index_keys, context._sub_storage_description,
context._signature_storage_name, function (db) {
return waitForTransaction(db, ["index-store"], "readwrite",
function (tx) {
return waitForIDBRequest(tx.objectStore("index-store")
......@@ -391,6 +489,19 @@
});
};
IndexStorage2.prototype.repair = function () {
var context = this;
return waitForOpenIndexedDB(context._database_name, context._version,
context._index_keys, context._sub_storage_description,
context._signature_storage_name, function (db) {
return waitForTransaction(db, ["index-store"], "readwrite",
function (tx) {
return repairInTransaction(context._sub_storage_description, tx,
context._index_keys, context._signature_storage_name);
});
});
};
IndexStorage2.prototype.getAttachment = function () {
return this._sub_storage.getAttachment.apply(this._sub_storage, arguments);
};
......
test/jio.storage/indexstorage2.tests.js
View file @ 78cca6d2
......@@ -66,6 +66,7 @@
module("indexStorage2.constructor", {
teardown: function () {
deleteIndexedDB(this.jio);
indexedDB.deleteDatabase("jio:index2_test_signatures");
}
});
test("Constructor without index_keys", function () {
......@@ -143,10 +144,11 @@
);
});
test("Constructor with index_keys", function () {
test("Constructor with index_keys and version", function () {
this.jio = jIO.createJIO({
type: "index2",
database: "index2_test",
version: 4,
index_keys: ["a", "b"],
sub_storage: {
type: "dummystorage3"
......@@ -156,6 +158,9 @@
equal(this.jio.__type, "index2");
equal(this.jio.__storage._sub_storage.__type, "dummystorage3");
equal(this.jio.__storage._database_name, "jio:index2_test");
equal(this.jio.__storage._version, 4);
deepEqual(this.jio.__storage._sub_storage_description,
{type: "dummystorage3"});
deepEqual(this.jio.__storage._index_keys, ["a", "b"]);
});
......@@ -165,6 +170,7 @@
module("indexStorage2.hasCapacity", {
teardown: function () {
deleteIndexedDB(this.jio);
indexedDB.deleteDatabase("jio:index2_test_signatures");
}
});
test("can list documents", function () {
......@@ -214,6 +220,7 @@
},
teardown: function () {
deleteIndexedDB(this.jio);
indexedDB.deleteDatabase("jio:index2_test_signatures");
}
});
test("Get calls substorage", function () {
......@@ -252,6 +259,7 @@
},
teardown: function () {
deleteIndexedDB(this.jio);
indexedDB.deleteDatabase("jio:index2_test_signatures");
}
});
......@@ -650,26 +658,21 @@
stop();
expect(8);
dummy_data = {
"32": {id: "32", doc: {"a": "3", "b": "2", "c": "inverse"},
value: {"a": "3", "b": "2", "c": "inverse"}},
"5": {id: "5", doc: {"a": "6", "b": "2", "c": "strong"},
value: {"a": "6", "b": "2", "c": "strong"}},
"14": {id: "14", doc: {"a": "67", "b": "3", "c": "disolve"},
value: {"a": "67", "b": "3", "c": "disolve"}}
};
dummy_data = {};
DummyStorage3.prototype.put = function (id, value) {
dummy_data[id] = {id: id, doc: value, value: value};
return id;
};
DummyStorage3.prototype.get = function (id) {
return dummy_data[id].doc;
if (dummy_data[id]) {
return dummy_data[id].doc;
}
throw new jIO.util.jIOError("Cannot find document: " + id, 404);
};
DummyStorage3.prototype.hasCapacity = function (name) {
return (name === 'list') || (name === 'include') || (name === 'select');
};
DummyStorage3.prototype.buildQuery = function () {
return Object.values(dummy_data);
};
......@@ -789,12 +792,168 @@
});
});
test("Manual repair", function () {
var context = this, fake_data;
context.jio = jIO.createJIO({
type: "index2",
database: "index2_test",
index_keys: ["a", "c"],
sub_storage: {
type: "dummystorage3"
}
});
stop();
expect(15);
fake_data = {
"1": {a: "id54", b: "9", c: "night"},
"4": {a: "vn92", b: "7", c: "matter"},
"9": {a: "ru23", b: "3", c: "control"},
"42": {a: "k422", b: "100", c: "grape"}
};
DummyStorage3.prototype.hasCapacity = function (name) {
return (name === 'list') || (name === 'select');
};
DummyStorage3.prototype.put = function (id, value) {
fake_data[id] = value;
return id;
};
DummyStorage3.prototype.get = function (id) {
if (fake_data[id]) {
return fake_data[id];
}
throw new jIO.util.jIOError("Cannot find document: " + id, 404);
};
DummyStorage3.prototype.remove = function (id) {
delete fake_data[id];
return id;
};
DummyStorage3.prototype.buildQuery = function () {
var keys = Object.keys(fake_data);
return keys.map(function (v) { return {id: v, value: {}}; });
};
context.jio.allDocs({query: 'c: "control"'})
.then(function (result) {
equal(result.data.total_rows, 1);
deepEqual(result.data.rows, [{id: "9", value: {}}]);
})
.then(function () {
fake_data["2"] = {a: "zu64", b: "1", c: "matter"};
fake_data["13"] = {a: "tk32", b: "9", c: "matter"};
return context.jio.repair();
})
.then(function () {
return context.jio.allDocs({query: 'c: "matter"'});
})
.then(function (result) {
equal(result.data.total_rows, 3);
deepEqual(result.data.rows.sort(idCompare), [{id: "13", value: {}},
{id: "2", value: {}}, {id: "4", value: {}}]);
})
.then(function () {
fake_data["2"] = {a: "zu64", b: "1", c: "observe"};
return context.jio.repair();
})
.then(function () {
return context.jio.allDocs({query: 'c: "observe"'});
})
.then(function (result) {
equal(result.data.total_rows, 1);
deepEqual(result.data.rows, [{id: "2", value: {}}]);
})
.then(function () {
delete fake_data["2"];
return context.jio.repair();
})
.then(function () {
return context.jio.allDocs({query: 'c: "observe"'});
})
.then(function (result) {
equal(result.data.total_rows, 0);
deepEqual(result.data.rows, []);
})
.then(function () {
return RSVP.all([
context.jio.put("43", {a: "t345", b: "101", c: "pear"}),
context.jio.put("44", {a: "j939", b: "121", c: "grape"}),
context.jio.put("45", {a: "q423", b: "131", c: "grape"}),
context.jio.remove("42")
]);
})
.then(function () {
return context.jio.repair();
})
.then(function () {
return context.jio.allDocs({query: "c:grape"});
})
.then(function (result) {
equal(result.data.total_rows, 2);
deepEqual(result.data.rows.sort(idCompare), [{id: "44", value: {}},
{id: "45", value: {}}]);
})
.then(function () {
context.jio = jIO.createJIO({
type: "index2",
database: "index2_test",
index_keys: ["a", "c"],
sub_storage: {
type: "dummystorage3"
}
});
delete fake_data["13"];
fake_data["3"] = {a: "gg38", b: "4", c: "matter"};
fake_data["9"] = {a: "tk32", b: "9", c: "matter"};
return context.jio.repair();
})
.then(function () {
return context.jio.allDocs({query: "c:matter"});
})
.then(function (result) {
equal(result.data.total_rows, 3);
deepEqual(result.data.rows.sort(idCompare), [{id: "3", value: {}},
{id: "4", value: {}}, {id: "9", value: {}}]);
})
.then(function () {
delete fake_data["9"];
fake_data["3"] = {a: "xu76", b: "9", c: "night"};
fake_data["7"] = {a: "bn02", b: "9", c: "matter"};
context.jio = jIO.createJIO({
type: "index2",
database: "index2_test",
index_keys: ["b"],
version: 2,
sub_storage: {
type: "dummystorage3"
}
});
return context.jio.allDocs({query: "b:9"});
})
.then(function (result) {
equal(result.data.total_rows, 3);
deepEqual(result.data.rows.sort(idCompare), [{id: "1", value: {}},
{id: "3", value: {}}, {id: "7", value: {}}]);
})
.then(function () {
return context.jio.allDocs({query: "c:matter"});
})
.fail(function (error) {
equal(error.message,
"Capacity 'query' is not implemented on 'dummystorage3'");
})
.always(function () {
start();
});
});
/////////////////////////////////////////////////////////////////
// IndexStorage2.getAttachment
/////////////////////////////////////////////////////////////////
module("IndexStorage2.getAttachment", {
teardown: function () {
deleteIndexedDB(this.jio);
indexedDB.deleteDatabase("jio:index2_test_signatures");
}
});
test("getAttachment called substorage getAttachment", function () {
......@@ -835,6 +994,7 @@
module("IndexStorage2.putAttachment", {
teardown: function () {
deleteIndexedDB(this.jio);
indexedDB.deleteDatabase("jio:index2_test_signatures");
}
});
test("putAttachment called substorage putAttachment", function () {
......@@ -876,6 +1036,7 @@
module("IndexStorage2.removeAttachment", {
teardown: function () {
deleteIndexedDB(this.jio);
indexedDB.deleteDatabase("jio:index2_test_signatures");
}
});
test("removeAttachment called substorage removeAttachment", function () {
......@@ -915,6 +1076,7 @@
module("indexStorage2.put", {
teardown: function () {
deleteIndexedDB(this.jio);
indexedDB.deleteDatabase("jio:index2_test_signatures");
}
});
test("Put creates index", function () {
......@@ -999,6 +1161,7 @@
module("indexStorage2.post", {
teardown: function () {
deleteIndexedDB(this.jio);
indexedDB.deleteDatabase("jio:index2_test_signatures");
}
});
test("Post creates index", function () {
......@@ -1091,6 +1254,7 @@
module("indexStorage2.remove", {
teardown: function () {
deleteIndexedDB(this.jio);
indexedDB.deleteDatabase("jio:index2_test_signatures");
}
});
test("Remove values", function () {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7