Commit 31ab09f1 authored by Romain Courteaud's avatar Romain Courteaud

ReplicateStorage: prototype of bulk usage to speed up first replication

parent cbb8be6e
...@@ -258,6 +258,7 @@ ...@@ -258,6 +258,7 @@
return argument_list[0]; return argument_list[0];
}); });
declareMethod(JioProxyStorage, "get", checkId); declareMethod(JioProxyStorage, "get", checkId);
declareMethod(JioProxyStorage, "bulk");
declareMethod(JioProxyStorage, "remove", checkId, function (argument_list) { declareMethod(JioProxyStorage, "remove", checkId, function (argument_list) {
return argument_list[0]; return argument_list[0];
}); });
...@@ -403,7 +404,11 @@ ...@@ -403,7 +404,11 @@
}; };
JioProxyStorage.prototype.hasCapacity = function (name) { JioProxyStorage.prototype.hasCapacity = function (name) {
var storage_method = this.__storage.hasCapacity; var storage_method = this.__storage.hasCapacity,
capacity_method = this.__storage[name];
if (capacity_method !== undefined) {
return true;
}
if ((storage_method === undefined) || if ((storage_method === undefined) ||
!storage_method.apply(this.__storage, arguments)) { !storage_method.apply(this.__storage, arguments)) {
throw new jIO.util.jIOError( throw new jIO.util.jIOError(
......
...@@ -167,7 +167,8 @@ ...@@ -167,7 +167,8 @@
}); });
} }
function checkLocalCreation(queue, source, destination, id, options) { function checkLocalCreation(queue, source, destination, id, options,
getMethod) {
var remote_doc; var remote_doc;
queue queue
.push(function () { .push(function () {
...@@ -187,7 +188,7 @@ ...@@ -187,7 +188,7 @@
.push(function () { .push(function () {
// This document was never synced. // This document was never synced.
// Push it to the remote storage and store sync information // Push it to the remote storage and store sync information
return source.get(id); return getMethod(id);
}) })
.push(function (doc) { .push(function (doc) {
var local_hash = generateHash(JSON.stringify(doc)), var local_hash = generateHash(JSON.stringify(doc)),
...@@ -213,6 +214,34 @@ ...@@ -213,6 +214,34 @@
}); });
} }
function checkBulkLocalCreation(queue, source, destination, id_list,
options) {
queue
.push(function () {
return source.bulk(id_list);
})
.push(function (result_list) {
var i,
sub_queue = new RSVP.Queue();
function getResult(j) {
return function (id) {
if (id !== id_list[j].parameter_list[0]) {
throw new Error("Does not access expected ID " + id);
}
return result_list[j];
};
}
for (i = 0; i < result_list.length; i += 1) {
checkLocalCreation(sub_queue, source, destination,
id_list[i].parameter_list[0],
options, getResult(i));
}
return sub_queue;
});
}
function checkLocalDeletion(queue, destination, id, source) { function checkLocalDeletion(queue, destination, id, source) {
var status_hash; var status_hash;
queue queue
...@@ -312,6 +341,7 @@ ...@@ -312,6 +341,7 @@
.push(function (result_list) { .push(function (result_list) {
var i, var i,
local_dict = {}, local_dict = {},
new_list = [],
signature_dict = {}, signature_dict = {},
key; key;
for (i = 0; i < result_list[0].data.total_rows; i += 1) { for (i = 0; i < result_list[0].data.total_rows; i += 1) {
...@@ -328,15 +358,28 @@ ...@@ -328,15 +358,28 @@
signature_dict[result_list[1].data.rows[i].id] = i; signature_dict[result_list[1].data.rows[i].id] = i;
} }
} }
if (options.check_creation === true) { if (options.check_creation === true) {
for (key in local_dict) { for (key in local_dict) {
if (local_dict.hasOwnProperty(key)) { if (local_dict.hasOwnProperty(key)) {
if (!signature_dict.hasOwnProperty(key)) { if (!signature_dict.hasOwnProperty(key)) {
checkLocalCreation(queue, source, destination, key, options); if (options.use_bulk_get === true) {
new_list.push({
method: "get",
parameter_list: [key]
});
} else {
checkLocalCreation(queue, source, destination, key,
options, source.get.bind(source));
} }
} }
} }
} }
if ((options.use_bulk_get === true) && (new_list.length !== 0)) {
checkBulkLocalCreation(queue, source, destination, new_list,
options);
}
}
for (key in signature_dict) { for (key in signature_dict) {
if (signature_dict.hasOwnProperty(key)) { if (signature_dict.hasOwnProperty(key)) {
if (local_dict.hasOwnProperty(key)) { if (local_dict.hasOwnProperty(key)) {
...@@ -404,11 +447,23 @@ ...@@ -404,11 +447,23 @@
} }
}) })
.push(function () { .push(function () {
// Autoactivate bulk if substorage implements it
// Keep it like this until the bulk API is stabilized
var use_bulk_get = false;
try {
use_bulk_get = context._remote_sub_storage.hasCapacity("bulk");
} catch (error) {
if (!((error instanceof jIO.util.jIOError) &&
(error.status_code === 501))) {
throw error;
}
}
if (context._check_remote_modification || if (context._check_remote_modification ||
context._check_remote_creation || context._check_remote_creation ||
context._check_remote_deletion) { context._check_remote_deletion) {
return pushStorage(context._remote_sub_storage, return pushStorage(context._remote_sub_storage,
context._local_sub_storage, { context._local_sub_storage, {
use_bulk_get: use_bulk_get,
check_modification: context._check_remote_modification, check_modification: context._check_remote_modification,
check_creation: context._check_remote_creation, check_creation: context._check_remote_creation,
check_deletion: context._check_remote_deletion check_deletion: context._check_remote_deletion
......
...@@ -1825,4 +1825,84 @@ ...@@ -1825,4 +1825,84 @@
}); });
}); });
test("bulk remote document creation", function () {
stop();
expect(3);
var id,
post_id = "123456789",
context = this;
function Storage200Bulk(spec) {
this._sub_storage = jIO.createJIO(spec.sub_storage);
}
Storage200Bulk.prototype.bulk = function (args) {
deepEqual(args, [{
method: "get",
parameter_list: [post_id]
}]);
return this._sub_storage.get(post_id)
.push(function (doc) {
return [doc];
});
};
Storage200Bulk.prototype.post = function (param) {
return this.put(post_id, param);
};
Storage200Bulk.prototype.put = function () {
return this._sub_storage.put.apply(this._sub_storage, arguments);
};
Storage200Bulk.prototype.hasCapacity = function () {
return this._sub_storage.hasCapacity.apply(this._sub_storage, arguments);
};
Storage200Bulk.prototype.buildQuery = function () {
return this._sub_storage.buildQuery.apply(this._sub_storage, arguments);
};
jIO.addStorage(
'replicatestorage200bulk',
Storage200Bulk
);
this.jio = jIO.createJIO({
type: "replicate",
local_sub_storage: {
type: "memory"
},
remote_sub_storage: {
type: "replicatestorage200bulk",
sub_storage: {
type: "memory"
}
}
});
context.jio.__storage._remote_sub_storage.post({"title": "bar"})
.then(function (result) {
id = result;
return context.jio.repair();
})
.then(function () {
return context.jio.get(id);
})
.then(function (result) {
deepEqual(result, {
title: "bar"
});
})
.then(function () {
return context.jio.__storage._signature_sub_storage.get(id);
})
.then(function (result) {
deepEqual(result, {
hash: "6799f3ea80e325b89f19589282a343c376c1f1af"
});
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
}(jIO, QUnit)); }(jIO, QUnit));
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment