Skip to content

J
jio
Commit 0e2d44a4 authored by preetwinder's avatar preetwinder
Browse files
Options

reimplement repair

parent 9f2e7bd5
Hide whitespace changes
Inline Side-by-side
Showing with 208 additions and 240 deletions
src/jio.storage/indexstorage2.js
View file @ 0e2d44a4
...@@ -43,61 +43,132 @@ ...@@ -43,61 +43,132 @@
IndexStorage2.prototype.hasCapacity = function (name) { IndexStorage2.prototype.hasCapacity = function (name) {
return (name === 'query') || (name === 'limit') || (name === 'list') || return (name === 'query') || (name === 'limit') || (name === 'list') ||
(name === 'select') || this._sub_storage.hasCapacity(name); this._sub_storage.hasCapacity(name);
}; };
function isSubset(array1, array2) { function isSubset(set1, set2) {
var i; var i, values;
array1 = new Set(array1); values = Array.from(set2);
for (i = 0; i < array2.length; i += 1) { for (i = 0; i < values.length; i += 1) {
if (!(array1.has(array2[i]))) { if (!set1.has(values[i])) {
return false; return false;
} }
} }
return true; return true;
} }
function handleUpgradeNeeded(evt, index_keys) { function filterDocValues(doc, keys) {
var filtered_doc = {}, i;
if (keys) {
for (i = 0; i < keys.length; i += 1) {
filtered_doc[keys[i]] = doc[keys[i]];
}
return filtered_doc;
}
return doc;
}
function getDocs(storage) {
var promise_hash = {}, i;
try {
storage.hasCapacity('include');
return storage.allDocs({include_docs: true});
} catch (error) {
if ((error instanceof jIO.util.jIOError) &&
(error.status_code === 501)) {
storage.hasCapacity('list');
return storage.allDocs()
.push(function (result) {
for (i = 0; i < result.data.total_rows; i += 1) {
promise_hash[result.data.rows[i].id] =
storage.get(result.data.rows[i].id);
}
return RSVP.hash(promise_hash);
})
.push(function (temp_result) {
var final_result = {data: {rows: []}}, keys;
keys = Object.keys(temp_result);
for (i = 0; i < keys.length; i += 1) {
final_result.data.rows.push({id: keys[i],
doc: temp_result[keys[i]]});
}
final_result.data.total_rows = final_result.data.rows.length;
return final_result;
});
}
throw error;
}
}
function waitForIDBRequest(request) {
return new RSVP.Promise(function (resolve, reject) {
request.onerror = reject;
request.onsuccess = resolve;
});
}
function handleUpgradeNeeded(evt, index_keys, repair_storage) {
var db = evt.target.result, store, i, current_indices, required_indices, var db = evt.target.result, store, i, current_indices, required_indices,
needs_repair = false; put_promise_list = [], docs_promise,
required_indices = index_keys.map(function (name) { repeatUntilPromiseFulfilled;
required_indices = new Set(index_keys.map(function (name) {
return 'Index-' + name; return 'Index-' + name;
}); }));
if (!(db.objectStoreNames[0])) { if (db.objectStoreNames[0] === 'index-store') {
store = db.createObjectStore("index-store", {
keyPath: "id",
autoIncrement: false
});
current_indices = new Set();
} else {
store = evt.target.transaction.objectStore('index-store'); store = evt.target.transaction.objectStore('index-store');
current_indices = new Set(store.indexNames);
if (!isSubset(store.indexNames, required_indices)) {
db.deleteObjectStore("index-store");
store = db.createObjectStore("index-store", {
keyPath: "id",
autoIncrement: false
});
current_indices = new Set();
needs_repair = true;
}
} }
for (i = 0; i < index_keys.length; i += 1) { current_indices = new Set(store ? store.indexNames : []);
if (!(current_indices.has(required_indices[i]))) { if (isSubset(current_indices, required_indices)) {
store.createIndex(required_indices[i], if (!store) {
return;
}
for (i = 0; i < store.indexNames.length; i += 1) {
if (!required_indices.has(store.indexNames[i])) {
store.deleteIndex(store.indexNames[i]);
}
}
} else {
if (store) {
db.deleteObjectStore('index-store');
current_indices.clear();
}
store = db.createObjectStore('index-store', {
keyPath: 'id',
autoIncrement: false
});
for (i = 0; i < index_keys.length; i += 1) {
store.createIndex('Index-' + index_keys[i],
'doc.' + index_keys[i], { unique: false }); 'doc.' + index_keys[i], { unique: false });
} }
current_indices.delete(required_indices[i]); docs_promise = getDocs(repair_storage);
} repeatUntilPromiseFulfilled = function repeatUntilPromiseFulfilled(req) {
current_indices = Array.from(current_indices); req.onsuccess = function () {
for (i = 0; i < current_indices.length; i += 1) { if (docs_promise.isRejected) {
store.deleteIndex(current_indices[i]); throw new jIO.util.jIOError(docs_promise.rejectedReason.message,
docs_promise.rejectedReason.status_code);
}
if (docs_promise.isFulfilled) {
for (i = 0; i < docs_promise.fulfillmentValue.data.total_rows;
i += 1) {
put_promise_list.push(waitForIDBRequest(store.put({
id: docs_promise.fulfillmentValue.data.rows[i].id,
doc: filterDocValues(
docs_promise.fulfillmentValue.data.rows[i].doc,
index_keys
)
})));
}
return RSVP.all(put_promise_list);
}
repeatUntilPromiseFulfilled(store.getAll());
};
};
repeatUntilPromiseFulfilled(store.getAll());
} }
return needs_repair;
} }
function waitForOpenIndexedDB(db_name, version, index_keys, callback) { function waitForOpenIndexedDB(db_name, version, index_keys, repair_storage,
callback) {
function resolver(resolve, reject) { function resolver(resolve, reject) {
// Open DB // // Open DB //
var request = indexedDB.open(db_name, version); var request = indexedDB.open(db_name, version);
...@@ -111,7 +182,7 @@ ...@@ -111,7 +182,7 @@
reject("Connection to: " + db_name + " failed: " + reject("Connection to: " + db_name + " failed: " +
error.target.error.message); error.target.error.message);
} else { } else {
reject(error); reject(error.target.error);
} }
}; };
...@@ -132,7 +203,7 @@ ...@@ -132,7 +203,7 @@
// Create DB if necessary // // Create DB if necessary //
request.onupgradeneeded = function (evt) { request.onupgradeneeded = function (evt) {
this.needs_repair = handleUpgradeNeeded(evt, index_keys); handleUpgradeNeeded(evt, index_keys, repair_storage);
}; };
request.onversionchange = function () { request.onversionchange = function () {
...@@ -141,10 +212,9 @@ ...@@ -141,10 +212,9 @@
}; };
request.onsuccess = function () { request.onsuccess = function () {
var context = this;
return new RSVP.Queue() return new RSVP.Queue()
.push(function () { .push(function () {
return callback(request.result, context.needs_repair); return callback(request.result);
}) })
.push(function (result) { .push(function (result) {
request.result.close(); request.result.close();
...@@ -198,69 +268,50 @@ ...@@ -198,69 +268,50 @@
return new RSVP.Promise(resolver, canceller); return new RSVP.Promise(resolver, canceller);
} }
function waitForIDBRequest(request) { IndexStorage2.prototype._iterateCursor = function (on, query, limit) {
return new RSVP.Promise(function (resolve, reject) { return new RSVP.Promise(function (resolve, reject) {
request.onerror = reject; var result_list = [], count = 0, cursor;
request.onsuccess = resolve; cursor = on.openKeyCursor(query);
}); cursor.onsuccess = function (cursor) {
} if (cursor.target.result && count !== limit) {
count += 1;
function filterDocValues(doc, keys) { result_list.push({id: cursor.target.result.primaryKey, value: {}});
var filtered_doc = {}, i; cursor.target.result.continue();
if (keys) { } else {
for (i = 0; i < keys.length; i += 1) { resolve(result_list);
filtered_doc[keys[i]] = doc[keys[i]];
}
return filtered_doc;
}
return doc;
}
IndexStorage2.prototype._repairIfNeeded = function (needs_repair) {
var context = this;
return RSVP.Queue()
.push(function () {
if (needs_repair) {
return context.repair();
} }
}); };
cursor.onerror = function (error) {
reject(error.message);
};
});
}; };
IndexStorage2.prototype._runQuery = function (key, value, limit) { IndexStorage2.prototype._runQuery = function (key, value, limit) {
var context = this; var context = this;
return new RSVP.Queue() return waitForOpenIndexedDB(context._database_name, context._version,
.push(function () { context._index_keys, context._sub_storage, function (db) {
return waitForOpenIndexedDB(context._database_name, context._version, return waitForTransaction(db, ["index-store"], "readonly",
context._index_keys, function (db, needs_repair) { function (tx) {
return context._repairIfNeeded(needs_repair) return context._iterateCursor(tx.objectStore("index-store")
.push(function () { .index("Index-" + key), value, limit);
return waitForTransaction(db, ["index-store"], "readonly",
function (tx) {
return waitForIDBRequest(tx.objectStore("index-store")
.index("Index-" + key).getAll(value, limit))
.then(function (evt) {
return evt.target.result;
});
});
});
}); });
}); });
}; };
IndexStorage2.prototype.buildQuery = function (options) { IndexStorage2.prototype.buildQuery = function (options) {
var context = this, query, select; var context = this, query;
select = options.select_list; if (options.query && !options.include_docs && !options.sort_on &&
if (options.query && !options.sort_on && !options.include) { !options.select_list) {
query = parseStringToObject(options.query); query = parseStringToObject(options.query);
if (query.type === 'simple') { if (query.type === 'simple') {
if (context._index_keys.indexOf(query.key) !== -1 && if (context._index_keys.indexOf(query.key) !== -1) {
(!select || isSubset(context._index_keys, select))) {
return context._runQuery(query.key, query.value, options.limit) return context._runQuery(query.key, query.value, options.limit)
.push(function (result) { .then(function (result) {
return result.map(function (value) { return result.map(function (value) {
return { return {
id: value.id, id: value.id,
value: select ? filterDocValues(value.doc, select) : {} value: {}
}; };
}); });
}); });
...@@ -279,17 +330,17 @@ ...@@ -279,17 +330,17 @@
IndexStorage2.prototype._put = function (id, value) { IndexStorage2.prototype._put = function (id, value) {
var context = this; var context = this;
if (context._index_keys.length === 0) {
return;
}
return waitForOpenIndexedDB(context._database_name, context._version, return waitForOpenIndexedDB(context._database_name, context._version,
context._index_keys, function (db, needs_repair) { context._index_keys, context._sub_storage, function (db) {
return context._repairIfNeeded(needs_repair) return waitForTransaction(db, ["index-store"], "readwrite",
.push(function () { function (tx) {
return waitForTransaction(db, ["index-store"], "readwrite", return waitForIDBRequest(tx.objectStore("index-store").put({
function (tx) { "id": id,
return waitForIDBRequest(tx.objectStore("index-store").put({ "doc": filterDocValues(value, context._index_keys)
"id": id, }));
"doc": filterDocValues(value, context._index_keys)
}));
});
}); });
}); });
}; };
...@@ -315,14 +366,11 @@ ...@@ -315,14 +366,11 @@
return context._sub_storage.remove(id) return context._sub_storage.remove(id)
.push(function () { .push(function () {
return waitForOpenIndexedDB(context._database_name, context._version, return waitForOpenIndexedDB(context._database_name, context._version,
context._index_keys, function (db, needs_repair) { context._index_keys, context._sub_storage, function (db) {
return context._repairIfNeeded(needs_repair) return waitForTransaction(db, ["index-store"], "readwrite",
.push(function () { function (tx) {
return waitForTransaction(db, ["index-store"], "readwrite", return waitForIDBRequest(tx.objectStore("index-store")
function (tx) { .delete(id));
return waitForIDBRequest(tx.objectStore("index-store")
.delete(id));
});
}); });
}); });
}); });
...@@ -341,17 +389,5 @@ ...@@ -341,17 +389,5 @@
arguments); arguments);
}; };
IndexStorage2.prototype.repair = function () {
var context = this, promise_list = [], i;
return context._sub_storage.allDocs()
.push(function (result) {
for (i = 0; i < result.data.total_rows; i += 1) {
promise_list.push(context.put(result.data.rows[i].id,
filterDocValues(result.data.rows[i].value, context._index_keys)));
}
return RSVP.all(promise_list);
});
};
jIO.addStorage("index2", IndexStorage2); jIO.addStorage("index2", IndexStorage2);
}(indexedDB, jIO, RSVP, IDBOpenDBRequest, DOMError, parseStringToObject)); }(indexedDB, jIO, RSVP, IDBOpenDBRequest, DOMError, parseStringToObject));
\ No newline at end of file
test/jio.storage/indexstorage2.tests.js
View file @ 0e2d44a4
...@@ -241,6 +241,14 @@ ...@@ -241,6 +241,14 @@
// indexStorage2.buildQuery // indexStorage2.buildQuery
///////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////
module("indexStorage2.buildQuery", { module("indexStorage2.buildQuery", {
setup: function () {
DummyStorage3.prototype.hasCapacity = function (name) {
return (name === 'list') || (name === 'include');
};
DummyStorage3.prototype.buildQuery = function () {
return [];
};
},
teardown: function () { teardown: function () {
deleteIndexedDB(this.jio); deleteIndexedDB(this.jio);
} }
...@@ -257,24 +265,22 @@ ...@@ -257,24 +265,22 @@
} }
}); });
stop(); stop();
expect(3); expect(2);
DummyStorage3.prototype.put = function (id) { DummyStorage3.prototype.put = function (id) {
return id; return id;
}; };
DummyStorage3.prototype.hasCapacity = function (name) {
return name === "list";
};
DummyStorage3.prototype.buildQuery = function (options) { DummyStorage3.prototype.buildQuery = function (options) {
deepEqual(options, {}); if (options.include_docs !== true) {
return [ return [
{id: "2", value: {}}, {id: "2", value: {}},
{id: "32", value: {}}, {id: "32", value: {}},
{id: "16", value: {}}, {id: "16", value: {}},
{id: "21", value: {}} {id: "21", value: {}}
]; ];
}
return [];
}; };
RSVP.all([ RSVP.all([
...@@ -322,6 +328,9 @@ ...@@ -322,6 +328,9 @@
}; };
DummyStorage3.prototype.buildQuery = function (options) { DummyStorage3.prototype.buildQuery = function (options) {
if (options.include_docs === true) {
return [];
}
deepEqual(options, {include_docs: false, select_list: ["a", "c"], deepEqual(options, {include_docs: false, select_list: ["a", "c"],
limit: 3, sort_on: [["a", "descending"], ["b", "ascending"]]}); limit: 3, sort_on: [["a", "descending"], ["b", "ascending"]]});
return [ return [
...@@ -463,10 +472,20 @@ ...@@ -463,10 +472,20 @@
deepEqual(value, {"a": "3", "b": "2"}); deepEqual(value, {"a": "3", "b": "2"});
return id; return id;
}; };
DummyStorage3.prototype.hasCapacity = function (capacity) { DummyStorage3.prototype.hasCapacity = function (name) {
return capacity === 'list'; return (name === 'list');
};
DummyStorage3.prototype.buildQuery = function () {
return [{id: "32", doc: {"a": "3", "b": "2"}}];
};
DummyStorage3.prototype.buildQuery = function () {
return [{id: "32", value: {}}];
};
DummyStorage3.prototype.get = function (id) {
if (id === "32") {
return {"a": "3", "b": "2"};
}
}; };
DummyStorage3.prototype.buildQuery = undefined;
context.jio.put("32", {"a": "3", "b": "2"}) context.jio.put("32", {"a": "3", "b": "2"})
.then(function () { .then(function () {
...@@ -542,9 +561,6 @@ ...@@ -542,9 +561,6 @@
DummyStorage3.prototype.put = function (id) { DummyStorage3.prototype.put = function (id) {
return id; return id;
}; };
DummyStorage3.prototype.hasCapacity = function (name) {
return (name === 'list');
};
RSVP.all([ RSVP.all([
context.jio.put("1", {"a": "55", "b": "2"}), context.jio.put("1", {"a": "55", "b": "2"}),
...@@ -569,106 +585,6 @@ ...@@ -569,106 +585,6 @@
}); });
}); });
test("select_list option is preset", function () {
var context = this;
context.jio = jIO.createJIO({
type: "index2",
database: "index2_test",
index_keys: ["a", "b", "c"],
sub_storage: {
type: "dummystorage3"
}
});
stop();
expect(2);
DummyStorage3.prototype.put = function (id) {
return id;
};
DummyStorage3.prototype.hasCapacity = function (name) {
return (name === 'list');
};
RSVP.all([
context.jio.put("1", {"a": "55", "b": "2", "c": "try"}),
context.jio.put("2", {"a": "98", "b": "2", "c": "adverse"}),
context.jio.put("3", {"a": "75", "b": "2", "c": "invite"}),
context.jio.put("8", {"a": "43", "b": "3", "c": "absolve"}),
context.jio.put("6", {"a": "21", "b": "2", "c": "defy"}),
context.jio.put("4", {"a": "65", "b": "3", "c": "odd"})
])
.then(function () {
return context.jio.allDocs({select_list: ["a", "c"],
query: "b:2"});
})
.then(function (result) {
equal(result.data.total_rows, 4);
deepEqual(result.data.rows.sort(idCompare), [
{id: "1", value: {"a": "55", "c": "try"}},
{id: "2", value: {"a": "98", "c": "adverse"}},
{id: "3", value: {"a": "75", "c": "invite"}},
{id: "6", value: {"a": "21", "c": "defy"}}
]);
})
.fail(function (error) {
console.log(error);
})
.always(function () {
start();
});
});
test("Select_list key is not present in the index", function () {
var context = this;
context.jio = jIO.createJIO({
type: "index2",
database: "index2_test",
index_keys: ["a", "c"],
sub_storage: {
type: "dummystorage3"
}
});
stop();
expect(3);
DummyStorage3.prototype.put = function (id) {
return id;
};
DummyStorage3.prototype.hasCapacity = function (name) {
return (name === 'list') || (name === 'query') || (name === 'select');
};
DummyStorage3.prototype.buildQuery = function (options) {
deepEqual(options, {select_list: ["a", "b"], query: "a:55"});
return [
{id: "1", value: {"a": "55", "b": "2"}},
{id: "2", value: {"a": "55", "b": "5"}},
{id: "3", value: {"a": "55", "b": "1"}}
];
};
RSVP.all([
context.jio.put("1", {"a": "55", "b": "2", "c": "try"}),
context.jio.put("2", {"a": "55", "b": "5", "c": "adverse"}),
context.jio.put("3", {"a": "55", "b": "1", "c": "invite"})
])
.then(function () {
return context.jio.allDocs({select_list: ["a", "b"],
query: "a:55"});
})
.then(function (result) {
equal(result.data.total_rows, 3);
deepEqual(result.data.rows.sort(idCompare), [
{id: "1", value: {"a": "55", "b": "2"}},
{id: "2", value: {"a": "55", "b": "5"}},
{id: "3", value: {"a": "55", "b": "1"}}
]);
})
.fail(function (error) {
console.log(error);
})
.always(function () {
start();
});
});
test("Index keys are modified", function () { test("Index keys are modified", function () {
var context = this, dummy_data; var context = this, dummy_data;
context.jio = jIO.createJIO({ context.jio = jIO.createJIO({
...@@ -684,18 +600,18 @@ ...@@ -684,18 +600,18 @@
expect(8); expect(8);
dummy_data = { dummy_data = {
"32": {id: "32", value: {"a": "3", "b": "2", "c": "inverse"}}, "32": {id: "32", doc: {"a": "3", "b": "2", "c": "inverse"}},
"5": {id: "5", value: {"a": "6", "b": "2", "c": "strong"}}, "5": {id: "5", doc: {"a": "6", "b": "2", "c": "strong"}},
"14": {id: "14", value: {"a": "67", "b": "3", "c": "disolve"}} "14": {id: "14", doc: {"a": "67", "b": "3", "c": "disolve"}}
}; };
DummyStorage3.prototype.put = function (id, value) { DummyStorage3.prototype.put = function (id, value) {
dummy_data[id] = {id: id, value: value}; dummy_data[id] = {id: id, doc: value};
return id; return id;
}; };
DummyStorage3.prototype.hasCapacity = function (name) { DummyStorage3.prototype.hasCapacity = function (name) {
return (name === 'list'); return (name === 'list') || (name === 'include');
}; };
DummyStorage3.prototype.buildQuery = function () { DummyStorage3.prototype.buildQuery = function () {
...@@ -937,6 +853,14 @@ ...@@ -937,6 +853,14 @@
return id; return id;
}; };
DummyStorage3.prototype.hasCapacity = function (name) {
return (name === 'list') || (name === 'include');
};
DummyStorage3.prototype.buildQuery = function () {
return [];
};
RSVP.all([ RSVP.all([
context.jio.put("32", {"a": "894", "b": "inversion", "c": 2}), context.jio.put("32", {"a": "894", "b": "inversion", "c": 2}),
context.jio.put("33", {"a": "65", "b": "division", "c": 4}), context.jio.put("33", {"a": "65", "b": "division", "c": 4}),
...@@ -1021,6 +945,14 @@ ...@@ -1021,6 +945,14 @@
} }
}; };
DummyStorage3.prototype.hasCapacity = function (name) {
return (name === 'list') || (name === 'include');
};
DummyStorage3.prototype.buildQuery = function () {
return [];
};
RSVP.all([ RSVP.all([
context.jio.post({"a": "5", "b": "inversion", "c": 2}), context.jio.post({"a": "5", "b": "inversion", "c": 2}),
context.jio.post({"a": "62", "b": "division", "c": 4}), context.jio.post({"a": "62", "b": "division", "c": 4}),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7