Skip to content

J
jio
Commit 9084abd9 authored by preetwinder's avatar preetwinder Committed by Romain Courteaud
Browse files
Options

reimplement repair

parent 29bef154
Hide whitespace changes
Inline Side-by-side
Showing with 277 additions and 213 deletions
src/jio.storage/indexstorage2.js
View file @ 9084abd9
......@@ -43,61 +43,134 @@
IndexStorage2.prototype.hasCapacity = function (name) {
return (name === 'query') || (name === 'limit') || (name === 'list') ||
(name === 'select') || this._sub_storage.hasCapacity(name);
this._sub_storage.hasCapacity(name);
};
function isSubset(array1, array2) {
var i;
array1 = new Set(array1);
for (i = 0; i < array2.length; i += 1) {
if (!(array1.has(array2[i]))) {
function isSubset(set1, set2) {
var i, values;
values = Array.from(set2);
for (i = 0; i < values.length; i += 1) {
if (!set1.has(values[i])) {
return false;
}
}
return true;
}
function handleUpgradeNeeded(evt, index_keys) {
function filterDocValues(doc, keys) {
var filtered_doc = {}, i;
if (keys) {
for (i = 0; i < keys.length; i += 1) {
filtered_doc[keys[i]] = doc[keys[i]];
}
return filtered_doc;
}
return doc;
}
function getDocs(storage) {
var promise_hash = {}, i;
try {
storage.hasCapacity('include');
return storage.allDocs({include_docs: true});
} catch (error) {
if ((error instanceof jIO.util.jIOError) &&
(error.status_code === 501)) {
storage.hasCapacity('list');
return storage.allDocs()
.push(function (result) {
var id;
for (i = 0; i < result.data.total_rows; i += 1) {
id = result.data.rows[i].id;
promise_hash[id] = storage.get(id);
}
return RSVP.hash(promise_hash);
})
.push(function (temp_result) {
var final_result = {data: {rows: []}}, keys;
keys = Object.keys(temp_result);
for (i = 0; i < keys.length; i += 1) {
final_result.data.rows.push({id: keys[i],
doc: temp_result[keys[i]]});
}
final_result.data.total_rows = final_result.data.rows.length;
return final_result;
});
}
throw error;
}
}
function waitForIDBRequest(request) {
return new RSVP.Promise(function (resolve, reject) {
request.onerror = reject;
request.onsuccess = resolve;
});
}
function handleUpgradeNeeded(evt, index_keys, repair_storage) {
var db = evt.target.result, store, i, current_indices, required_indices,
needs_repair = false;
required_indices = index_keys.map(function (name) {
put_promise_list = [], docs_promise,
repeatUntilPromiseFulfilled;
required_indices = new Set(index_keys.map(function (name) {
return 'Index-' + name;
});
}));
if (db.objectStoreNames[0] === 'index-store') {
store = evt.target.transaction.objectStore('index-store');
}
if (!(db.objectStoreNames[0])) {
store = db.createObjectStore("index-store", {
keyPath: "id",
autoIncrement: false
});
current_indices = new Set();
current_indices = new Set(store ? store.indexNames : []);
if (isSubset(current_indices, required_indices)) {
if (!store) {
return;
}
for (i = 0; i < store.indexNames.length; i += 1) {
if (!required_indices.has(store.indexNames[i])) {
store.deleteIndex(store.indexNames[i]);
}
}
} else {
store = evt.target.transaction.objectStore('index-store');
current_indices = new Set(store.indexNames);
if (!isSubset(store.indexNames, required_indices)) {
db.deleteObjectStore("index-store");
store = db.createObjectStore("index-store", {
keyPath: "id",
autoIncrement: false
});
current_indices = new Set();
needs_repair = true;
if (store) {
db.deleteObjectStore('index-store');
current_indices.clear();
}
}
for (i = 0; i < index_keys.length; i += 1) {
if (!(current_indices.has(required_indices[i]))) {
store.createIndex(required_indices[i],
store = db.createObjectStore('index-store', {
keyPath: 'id',
autoIncrement: false
});
for (i = 0; i < index_keys.length; i += 1) {
store.createIndex('Index-' + index_keys[i],
'doc.' + index_keys[i], { unique: false });
}
current_indices.delete(required_indices[i]);
}
current_indices = Array.from(current_indices);
for (i = 0; i < current_indices.length; i += 1) {
store.deleteIndex(current_indices[i]);
docs_promise = getDocs(repair_storage);
repeatUntilPromiseFulfilled = function repeatUntilPromiseFulfilled(req) {
req.onsuccess = function () {
if (docs_promise.isRejected) {
throw new jIO.util.jIOError(docs_promise.rejectedReason.message,
docs_promise.rejectedReason.status_code);
}
if (docs_promise.isFulfilled) {
for (i = 0; i < docs_promise.fulfillmentValue.data.total_rows;
i += 1) {
put_promise_list.push(waitForIDBRequest(store.put({
id: docs_promise.fulfillmentValue.data.rows[i].id,
doc: filterDocValues(
docs_promise.fulfillmentValue.data.rows[i].doc,
index_keys
)
})));
}
return RSVP.all(put_promise_list);
}
repeatUntilPromiseFulfilled(store.getAll());
};
};
repeatUntilPromiseFulfilled(store.getAll());
}
return needs_repair;
}
function waitForOpenIndexedDB(db_name, version, index_keys, callback) {
function waitForOpenIndexedDB(db_name, version, index_keys, repair_storage,
callback) {
function resolver(resolve, reject) {
// Open DB //
var request = indexedDB.open(db_name, version);
......@@ -111,7 +184,7 @@
reject("Connection to: " + db_name + " failed: " +
error.target.error.message);
} else {
reject(error);
reject(error.target.error);
}
};
......@@ -132,7 +205,7 @@
// Create DB if necessary //
request.onupgradeneeded = function (evt) {
this.needs_repair = handleUpgradeNeeded(evt, index_keys);
handleUpgradeNeeded(evt, index_keys, repair_storage);
};
request.onversionchange = function () {
......@@ -141,10 +214,9 @@
};
request.onsuccess = function () {
var context = this;
return new RSVP.Queue()
.push(function () {
return callback(request.result, context.needs_repair);
return callback(request.result);
})
.push(function (result) {
request.result.close();
......@@ -198,69 +270,50 @@
return new RSVP.Promise(resolver, canceller);
}
function waitForIDBRequest(request) {
IndexStorage2.prototype._iterateCursor = function (on, query, limit) {
return new RSVP.Promise(function (resolve, reject) {
request.onerror = reject;
request.onsuccess = resolve;
});
}
function filterDocValues(doc, keys) {
var filtered_doc = {}, i;
if (keys) {
for (i = 0; i < keys.length; i += 1) {
filtered_doc[keys[i]] = doc[keys[i]];
}
return filtered_doc;
}
return doc;
}
IndexStorage2.prototype._repairIfNeeded = function (needs_repair) {
var context = this;
return RSVP.Queue()
.push(function () {
if (needs_repair) {
return context.repair();
var result_list = [], count = 0, cursor;
cursor = on.openKeyCursor(query);
cursor.onsuccess = function (cursor) {
if (cursor.target.result && count !== limit) {
count += 1;
result_list.push({id: cursor.target.result.primaryKey, value: {}});
cursor.target.result.continue();
} else {
resolve(result_list);
}
});
};
cursor.onerror = function (error) {
reject(error.message);
};
});
};
IndexStorage2.prototype._runQuery = function (key, value, limit) {
var context = this;
return new RSVP.Queue()
.push(function () {
return waitForOpenIndexedDB(context._database_name, context._version,
context._index_keys, function (db, needs_repair) {
return context._repairIfNeeded(needs_repair)
.push(function () {
return waitForTransaction(db, ["index-store"], "readonly",
function (tx) {
return waitForIDBRequest(tx.objectStore("index-store")
.index("Index-" + key).getAll(value, limit))
.then(function (evt) {
return evt.target.result;
});
});
});
return waitForOpenIndexedDB(context._database_name, context._version,
context._index_keys, context._sub_storage, function (db) {
return waitForTransaction(db, ["index-store"], "readonly",
function (tx) {
return context._iterateCursor(tx.objectStore("index-store")
.index("Index-" + key), value, limit);
});
});
};
IndexStorage2.prototype.buildQuery = function (options) {
var context = this, query, select;
select = options.select_list;
if (options.query && !options.sort_on && !options.include) {
var context = this, query;
if (options.query && !options.include_docs && !options.sort_on &&
!options.select_list) {
query = parseStringToObject(options.query);
if (query.type === 'simple') {
if (context._index_keys.indexOf(query.key) !== -1 &&
(!select || isSubset(context._index_keys, select))) {
if (context._index_keys.indexOf(query.key) !== -1) {
return context._runQuery(query.key, query.value, options.limit)
.push(function (result) {
.then(function (result) {
return result.map(function (value) {
return {
id: value.id,
value: select ? filterDocValues(value.doc, select) : {}
value: {}
};
});
});
......@@ -279,17 +332,17 @@
IndexStorage2.prototype._put = function (id, value) {
var context = this;
if (context._index_keys.length === 0) {
return;
}
return waitForOpenIndexedDB(context._database_name, context._version,
context._index_keys, function (db, needs_repair) {
return context._repairIfNeeded(needs_repair)
.push(function () {
return waitForTransaction(db, ["index-store"], "readwrite",
function (tx) {
return waitForIDBRequest(tx.objectStore("index-store").put({
"id": id,
"doc": filterDocValues(value, context._index_keys)
}));
});
context._index_keys, context._sub_storage, function (db) {
return waitForTransaction(db, ["index-store"], "readwrite",
function (tx) {
return waitForIDBRequest(tx.objectStore("index-store").put({
"id": id,
"doc": filterDocValues(value, context._index_keys)
}));
});
});
};
......@@ -315,14 +368,11 @@
return context._sub_storage.remove(id)
.push(function () {
return waitForOpenIndexedDB(context._database_name, context._version,
context._index_keys, function (db, needs_repair) {
return context._repairIfNeeded(needs_repair)
.push(function () {
return waitForTransaction(db, ["index-store"], "readwrite",
function (tx) {
return waitForIDBRequest(tx.objectStore("index-store")
.delete(id));
});
context._index_keys, context._sub_storage, function (db) {
return waitForTransaction(db, ["index-store"], "readwrite",
function (tx) {
return waitForIDBRequest(tx.objectStore("index-store")
.delete(id));
});
});
});
......@@ -341,17 +391,5 @@
arguments);
};
IndexStorage2.prototype.repair = function () {
var context = this, promise_list = [], i;
return context._sub_storage.allDocs()
.push(function (result) {
for (i = 0; i < result.data.total_rows; i += 1) {
promise_list.push(context.put(result.data.rows[i].id,
filterDocValues(result.data.rows[i].value, context._index_keys)));
}
return RSVP.all(promise_list);
});
};
jIO.addStorage("index2", IndexStorage2);
}(indexedDB, jIO, RSVP, IDBOpenDBRequest, DOMError, parseStringToObject));
\ No newline at end of file
test/jio.storage/indexstorage2.tests.js
View file @ 9084abd9
......@@ -194,6 +194,7 @@
);
ok(this.jio.hasCapacity("query"));
ok(this.jio.hasCapacity("limit"));
ok(this.jio.hasCapacity("sort"));
});
......@@ -241,6 +242,14 @@
// indexStorage2.buildQuery
/////////////////////////////////////////////////////////////////
module("indexStorage2.buildQuery", {
setup: function () {
DummyStorage3.prototype.hasCapacity = function (name) {
return (name === 'list') || (name === 'include');
};
DummyStorage3.prototype.buildQuery = function () {
return [];
};
},
teardown: function () {
deleteIndexedDB(this.jio);
}
......@@ -257,24 +266,22 @@
}
});
stop();
expect(3);
expect(2);
DummyStorage3.prototype.put = function (id) {
return id;
};
DummyStorage3.prototype.hasCapacity = function (name) {
return name === "list";
};
DummyStorage3.prototype.buildQuery = function (options) {
deepEqual(options, {});
return [
{id: "2", value: {}},
{id: "32", value: {}},
{id: "16", value: {}},
{id: "21", value: {}}
];
if (options.include_docs !== true) {
return [
{id: "2", value: {}},
{id: "32", value: {}},
{id: "16", value: {}},
{id: "21", value: {}}
];
}
return [];
};
RSVP.all([
......@@ -322,6 +329,9 @@
};
DummyStorage3.prototype.buildQuery = function (options) {
if (options.include_docs === true) {
return [];
}
deepEqual(options, {include_docs: false, select_list: ["a", "c"],
limit: 3, sort_on: [["a", "descending"], ["b", "ascending"]]});
return [
......@@ -463,10 +473,6 @@
deepEqual(value, {"a": "3", "b": "2"});
return id;
};
DummyStorage3.prototype.hasCapacity = function (capacity) {
return capacity === 'list';
};
DummyStorage3.prototype.buildQuery = undefined;
context.jio.put("32", {"a": "3", "b": "2"})
.then(function () {
......@@ -482,12 +488,12 @@
});
});
test("No index keys provided but substorage supports querying", function () {
var context = this;
test("Repair on storage without include_docs support", function () {
var context = this, fake_data;
context.jio = jIO.createJIO({
type: "index2",
database: "index2_test",
index_keys: [],
index_keys: ["a", "c"],
sub_storage: {
type: "dummystorage3"
}
......@@ -495,28 +501,41 @@
stop();
expect(3);
DummyStorage3.prototype.put = function (id) {
fake_data = {
"1": {a: "id54", b: 2, c: "night"},
"4": {a: "vn92", b: 7, c: "matter"},
"9": {a: "ru23", b: 3, c: "control"}
};
DummyStorage3.prototype.hasCapacity = function (name) {
return name === 'list';
};
DummyStorage3.prototype.put = function (id, value) {
fake_data[id] = value;
return id;
};
DummyStorage3.prototype.hasCapacity = function (capacity) {
return (capacity === 'list') || (capacity === 'query');
DummyStorage3.prototype.get = function (id) {
return fake_data[id];
};
DummyStorage3.prototype.buildQuery = function (options) {
equal(options.query, 'a: "5"');
return [{id: "3", value: {}}];
deepEqual(options, {});
var keys = Object.keys(fake_data);
return keys.map(function (v) { return {id: v, value: {}}; });
};
RSVP.all([
context.jio.put("32", {a: "3", b: "1"}),
context.jio.put("21", {a: "8", b: "1"}),
context.jio.put("3", {a: "5", b: "1"})
context.jio.put("3", {a: "ab43", b: 9, c: "absorb"}),
context.jio.put("5", {a: "gu31", b: 5, c: "control"}),
context.jio.put("7", {a: "zf76", b: 3, c: "rules"}),
context.jio.put("6", {a: "cc92", b: 6, c: "afraid"})
])
.then(function () {
return context.jio.allDocs({query: 'a: "5"'});
return context.jio.allDocs({query: 'c:"control"'});
})
.then(function (result) {
equal(result.data.total_rows, 1);
deepEqual(result.data.rows, [{"id": "3", "value": {}}]);
equal(result.data.total_rows, 2);
deepEqual(result.data.rows.sort(idCompare), [{id: "5", value: {}},
{id: "9", value: {}}]);
})
.fail(function (error) {
console.log(error);
......@@ -526,40 +545,47 @@
});
});
test("Limit with query", function () {
var context = this;
test("Repair on storage with include_docs support", function () {
var context = this, fake_data;
context.jio = jIO.createJIO({
type: "index2",
database: "index2_test",
index_keys: ["a", "b"],
index_keys: ["a", "c"],
sub_storage: {
type: "dummystorage3"
}
});
stop();
expect(1);
expect(3);
DummyStorage3.prototype.put = function (id) {
fake_data = [
{id: "1", doc: {a: "id54", b: 2, c: "night"}},
{id: "4", doc: {a: "vn92", b: 7, c: "matter"}},
{id: "9", doc: {a: "ru23", b: 3, c: "control"}}
];
DummyStorage3.prototype.put = function (id, value) {
fake_data.push({id: id, doc: value});
return id;
};
DummyStorage3.prototype.hasCapacity = function (name) {
return (name === 'list');
DummyStorage3.prototype.buildQuery = function (options) {
deepEqual(options, {include_docs: true});
return fake_data;
};
RSVP.all([
context.jio.put("1", {"a": "55", "b": "2"}),
context.jio.put("2", {"a": "98", "b": "2"}),
context.jio.put("3", {"a": "75", "b": "2"}),
context.jio.put("8", {"a": "43", "b": "3"}),
context.jio.put("6", {"a": "21", "b": "3"}),
context.jio.put("16", {"a": "39", "b": "3"}),
context.jio.put("11", {"a": "16", "b": "3"})
context.jio.put("3", {a: "ab43", b: 9, c: "absorb"}),
context.jio.put("5", {a: "gu31", b: 5, c: "control"}),
context.jio.put("7", {a: "zf76", b: 3, c: "rules"}),
context.jio.put("6", {a: "cc92", b: 6, c: "afraid"})
])
.then(function () {
return context.jio.allDocs({limit: 4, query: "b:2"});
return context.jio.allDocs({query: 'c:"control"'});
})
.then(function (result) {
equal(result.data.total_rows, 3);
equal(result.data.total_rows, 2);
deepEqual(result.data.rows.sort(idCompare), [{id: "5", value: {}},
{id: "9", value: {}}]);
})
.fail(function (error) {
console.log(error);
......@@ -569,44 +595,41 @@
});
});
test("select_list option is preset", function () {
test("No index keys provided but substorage supports querying", function () {
var context = this;
context.jio = jIO.createJIO({
type: "index2",
database: "index2_test",
index_keys: ["a", "b", "c"],
index_keys: [],
sub_storage: {
type: "dummystorage3"
}
});
stop();
expect(2);
expect(3);
DummyStorage3.prototype.put = function (id) {
return id;
};
DummyStorage3.prototype.hasCapacity = function (name) {
return (name === 'list');
DummyStorage3.prototype.hasCapacity = function (capacity) {
return (capacity === 'list') || (capacity === 'query');
};
DummyStorage3.prototype.buildQuery = function (options) {
equal(options.query, 'a: "5"');
return [{id: "3", value: {}}];
};
RSVP.all([
context.jio.put("1", {"a": "55", "b": "2", "c": "try"}),
context.jio.put("2", {"a": "98", "b": "2", "c": "adverse"}),
context.jio.put("3", {"a": "75", "b": "2", "c": "invite"}),
context.jio.put("8", {"a": "43", "b": "3", "c": "absolve"}),
context.jio.put("6", {"a": "21", "b": "2", "c": "defy"}),
context.jio.put("4", {"a": "65", "b": "3", "c": "odd"})
context.jio.put("32", {a: "3", b: "1"}),
context.jio.put("21", {a: "8", b: "1"}),
context.jio.put("3", {a: "5", b: "1"})
])
.then(function () {
return context.jio.allDocs({select_list: ["a", "c"],
query: "b:2"});
return context.jio.allDocs({query: 'a: "5"'});
})
.then(function (result) {
equal(result.data.total_rows, 4);
deepEqual(result.data.rows.sort(idCompare), [
{id: "1", value: {"a": "55", "c": "try"}},
{id: "2", value: {"a": "98", "c": "adverse"}},
{id: "3", value: {"a": "75", "c": "invite"}},
{id: "6", value: {"a": "21", "c": "defy"}}
]);
equal(result.data.total_rows, 1);
deepEqual(result.data.rows, [{"id": "3", "value": {}}]);
})
.fail(function (error) {
console.log(error);
......@@ -616,50 +639,37 @@
});
});
test("Select_list key is not present in the index", function () {
test("Limit with query", function () {
var context = this;
context.jio = jIO.createJIO({
type: "index2",
database: "index2_test",
index_keys: ["a", "c"],
index_keys: ["a", "b"],
sub_storage: {
type: "dummystorage3"
}
});
stop();
expect(3);
expect(1);
DummyStorage3.prototype.put = function (id) {
return id;
};
DummyStorage3.prototype.hasCapacity = function (name) {
return (name === 'list') || (name === 'query') || (name === 'select');
};
DummyStorage3.prototype.buildQuery = function (options) {
deepEqual(options, {select_list: ["a", "b"], query: "a:55"});
return [
{id: "1", value: {"a": "55", "b": "2"}},
{id: "2", value: {"a": "55", "b": "5"}},
{id: "3", value: {"a": "55", "b": "1"}}
];
};
RSVP.all([
context.jio.put("1", {"a": "55", "b": "2", "c": "try"}),
context.jio.put("2", {"a": "55", "b": "5", "c": "adverse"}),
context.jio.put("3", {"a": "55", "b": "1", "c": "invite"})
context.jio.put("1", {"a": "55", "b": "2"}),
context.jio.put("2", {"a": "98", "b": "2"}),
context.jio.put("3", {"a": "75", "b": "2"}),
context.jio.put("8", {"a": "43", "b": "3"}),
context.jio.put("6", {"a": "21", "b": "2"}),
context.jio.put("16", {"a": "39", "b": "2"}),
context.jio.put("11", {"a": "16", "b": "2"})
])
.then(function () {
return context.jio.allDocs({select_list: ["a", "b"],
query: "a:55"});
return context.jio.allDocs({limit: 4, query: "b:2"});
})
.then(function (result) {
equal(result.data.total_rows, 3);
deepEqual(result.data.rows.sort(idCompare), [
{id: "1", value: {"a": "55", "b": "2"}},
{id: "2", value: {"a": "55", "b": "5"}},
{id: "3", value: {"a": "55", "b": "1"}}
]);
equal(result.data.total_rows, 4);
})
.fail(function (error) {
console.log(error);
......@@ -684,18 +694,18 @@
expect(8);
dummy_data = {
"32": {id: "32", value: {"a": "3", "b": "2", "c": "inverse"}},
"5": {id: "5", value: {"a": "6", "b": "2", "c": "strong"}},
"14": {id: "14", value: {"a": "67", "b": "3", "c": "disolve"}}
"32": {id: "32", doc: {"a": "3", "b": "2", "c": "inverse"}},
"5": {id: "5", doc: {"a": "6", "b": "2", "c": "strong"}},
"14": {id: "14", doc: {"a": "67", "b": "3", "c": "disolve"}}
};
DummyStorage3.prototype.put = function (id, value) {
dummy_data[id] = {id: id, value: value};
dummy_data[id] = {id: id, doc: value};
return id;
};
DummyStorage3.prototype.hasCapacity = function (name) {
return (name === 'list');
return (name === 'list') || (name === 'include');
};
DummyStorage3.prototype.buildQuery = function () {
......@@ -937,6 +947,14 @@
return id;
};
DummyStorage3.prototype.hasCapacity = function (name) {
return (name === 'list') || (name === 'include');
};
DummyStorage3.prototype.buildQuery = function () {
return [];
};
RSVP.all([
context.jio.put("32", {"a": "894", "b": "inversion", "c": 2}),
context.jio.put("33", {"a": "65", "b": "division", "c": 4}),
......@@ -1021,6 +1039,14 @@
}
};
DummyStorage3.prototype.hasCapacity = function (name) {
return (name === 'list') || (name === 'include');
};
DummyStorage3.prototype.buildQuery = function () {
return [];
};
RSVP.all([
context.jio.post({"a": "5", "b": "inversion", "c": 2}),
context.jio.post({"a": "62", "b": "division", "c": 4}),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7