Skip to content

J
jio
Commit 3e6379c8 authored by Romain Courteaud's avatar Romain Courteaud
Browse files
Options

Drop bulk prototype. 

Bulk was implemented to see if it improves ERP5 replication.
Using parallel replication functionality is much more efficient from server and client point of view.

It does not generate huge transaction on server side.
Client does not wait for all documents before writing them locally.
parent db73ae15
Hide whitespace changes
Inline Side-by-side
Showing with 10 additions and 628 deletions
src/jio.storage/erp5storage.js
View file @ 3e6379c8
...@@ -172,56 +172,6 @@ ...@@ -172,56 +172,6 @@
}); });
}; };
ERP5Storage.prototype.bulk = function (request_list) {
var i,
storage = this,
bulk_list = [];
for (i = 0; i < request_list.length; i += 1) {
if (request_list[i].method !== "get") {
throw new Error("ERP5Storage: not supported " +
request_list[i].method + " in bulk");
}
bulk_list.push({
relative_url: request_list[i].parameter_list[0],
view: storage._default_view_reference
});
}
return getSiteDocument(storage)
.push(function (site_hal) {
var form_data = new FormData();
form_data.append("bulk_list", JSON.stringify(bulk_list));
return jIO.util.ajax({
"type": "POST",
"url": site_hal._actions.bulk.href,
"data": form_data,
// "headers": {
// "Content-Type": "application/json"
// },
"xhrFields": {
withCredentials: true
}
});
})
.push(function (response) {
var result_list = [],
hateoas = JSON.parse(response.target.responseText);
function pushResult(json) {
return extractPropertyFromFormJSON(json)
.push(function (json2) {
return convertJSONToGet(json2);
});
}
for (i = 0; i < hateoas.result_list.length; i += 1) {
result_list.push(pushResult(hateoas.result_list[i]));
}
return RSVP.all(result_list);
});
};
ERP5Storage.prototype.post = function (data) { ERP5Storage.prototype.post = function (data) {
var context = this, var context = this,
new_id; new_id;
...@@ -448,7 +398,7 @@ ...@@ -448,7 +398,7 @@
ERP5Storage.prototype.hasCapacity = function (name) { ERP5Storage.prototype.hasCapacity = function (name) {
return ((name === "list") || (name === "query") || return ((name === "list") || (name === "query") ||
(name === "select") || (name === "limit") || (name === "select") || (name === "limit") ||
(name === "sort")) || (name === "bulk_get"); (name === "sort"));
}; };
function isSingleLocalRoles(parsed_query) { function isSingleLocalRoles(parsed_query) {
......
src/jio.storage/replicatestorage.js
View file @ 3e6379c8
...@@ -822,44 +822,6 @@ ...@@ -822,44 +822,6 @@
}); });
} }
function checkBulkSignatureDifference(queue, source, destination, id_list,
document_status_list, options,
conflict_force, conflict_revert,
conflict_ignore) {
queue
.push(function () {
return source.bulk(id_list);
})
.push(function (result_list) {
var i,
argument_list = [];
function getResult(j) {
return function (id) {
if (id !== id_list[j].parameter_list[0]) {
throw new Error("Does not access expected ID " + id);
}
return result_list[j];
};
}
for (i = 0; i < result_list.length; i += 1) {
argument_list[i] = [undefined, source, destination,
id_list[i].parameter_list[0],
conflict_force, conflict_revert,
conflict_ignore,
document_status_list[i].is_creation,
document_status_list[i].is_modification,
getResult(i), options];
}
return dispatchQueue(
checkSignatureDifference,
argument_list,
options.operation_amount
);
});
}
function pushStorage(source, destination, signature_allDocs, options) { function pushStorage(source, destination, signature_allDocs, options) {
var argument_list = [], var argument_list = [],
argument_list_deletion = []; argument_list_deletion = [];
...@@ -873,8 +835,6 @@ ...@@ -873,8 +835,6 @@
.push(function (source_allDocs) { .push(function (source_allDocs) {
var i, var i,
local_dict = {}, local_dict = {},
document_list = [],
document_status_list = [],
signature_dict = {}, signature_dict = {},
is_modification, is_modification,
is_creation, is_creation,
...@@ -902,26 +862,15 @@ ...@@ -902,26 +862,15 @@
is_creation = !signature_dict.hasOwnProperty(key) is_creation = !signature_dict.hasOwnProperty(key)
&& options.check_creation; && options.check_creation;
if (is_modification === true || is_creation === true) { if (is_modification === true || is_creation === true) {
if (options.use_bulk_get === true) { argument_list[i] = [undefined, source, destination,
document_list.push({ key,
method: "get", options.conflict_force,
parameter_list: [key] options.conflict_revert,
}); options.conflict_ignore,
document_status_list.push({ is_creation, is_modification,
is_creation: is_creation, source.get.bind(source),
is_modification: is_modification options];
}); i += 1;
} else {
argument_list[i] = [undefined, source, destination,
key,
options.conflict_force,
options.conflict_revert,
options.conflict_ignore,
is_creation, is_modification,
source.get.bind(source),
options];
i += 1;
}
} }
} }
} }
...@@ -957,14 +906,6 @@ ...@@ -957,14 +906,6 @@
); );
}); });
} }
if ((options.use_bulk_get === true) && (document_list.length !== 0)) {
checkBulkSignatureDifference(queue, source, destination,
document_list, document_status_list,
options,
options.conflict_force,
options.conflict_revert,
options.conflict_ignore);
}
return queue; return queue;
}); });
} }
...@@ -1049,24 +990,12 @@ ...@@ -1049,24 +990,12 @@
return signature_allDocs; return signature_allDocs;
}) })
.push(function (signature_allDocs) { .push(function (signature_allDocs) {
// Autoactivate bulk if substorage implements it
// Keep it like this until the bulk API is stabilized
var use_bulk_get = false;
try {
use_bulk_get = context._remote_sub_storage.hasCapacity("bulk_get");
} catch (error) {
if (!((error instanceof jIO.util.jIOError) &&
(error.status_code === 501))) {
throw error;
}
}
if (context._check_remote_modification || if (context._check_remote_modification ||
context._check_remote_creation || context._check_remote_creation ||
context._check_remote_deletion) { context._check_remote_deletion) {
return pushStorage(context._remote_sub_storage, return pushStorage(context._remote_sub_storage,
context._local_sub_storage, context._local_sub_storage,
signature_allDocs, { signature_allDocs, {
use_bulk_get: use_bulk_get,
use_revert_post: context._use_remote_post, use_revert_post: context._use_remote_post,
conflict_force: (context._conflict_handling === conflict_force: (context._conflict_handling ===
CONFLICT_KEEP_REMOTE), CONFLICT_KEEP_REMOTE),
......
test/jio.storage/erp5storage.tests.js
View file @ 3e6379c8
...@@ -15,7 +15,6 @@ ...@@ -15,7 +15,6 @@
search_template = domain + "?mode=search{&query,select_list*,limit*," + search_template = domain + "?mode=search{&query,select_list*,limit*," +
"sort_on*,local_roles*}", "sort_on*,local_roles*}",
add_url = domain + "lets?add=somedocument", add_url = domain + "lets?add=somedocument",
bulk_url = domain + "lets?run=bulk",
root_hateoas = JSON.stringify({ root_hateoas = JSON.stringify({
"_links": { "_links": {
traverse: { traverse: {
...@@ -30,9 +29,6 @@ ...@@ -30,9 +29,6 @@
"_actions": { "_actions": {
add: { add: {
href: add_url href: add_url
},
bulk: {
href: bulk_url
} }
} }
}); });
...@@ -1964,202 +1960,4 @@ ...@@ -1964,202 +1960,4 @@
}); });
}); });
/////////////////////////////////////////////////////////////////
// erp5Storage.bulk
/////////////////////////////////////////////////////////////////
module("erp5Storage.bulk", {
setup: function () {
this.server = sinon.fakeServer.create();
this.server.autoRespond = true;
this.server.autoRespondAfter = 5;
this.spy = sinon.spy(FormData.prototype, "append");
this.jio = jIO.createJIO({
type: "erp5",
url: domain,
default_view_reference: "bar_view"
});
},
teardown: function () {
this.server.restore();
delete this.server;
this.spy.restore();
delete this.spy;
}
});
test("bulk get ERP5 document list", function () {
var id = "person_module/20150119_azerty",
id2 = "person_module/20150219_azerty",
context = this,
document_hateoas = JSON.stringify({
// Kept property
"title": "foo",
// Remove all _ properties
"_bar": "john doo",
"_links": {
type: {
name: "Person"
},
parent: {
href: "urn:jio:get:bar_module"
}
},
"_embedded": {
"_view": {
form_id: {
key: "form_id",
"default": "Base_view"
},
my_title: {
key: "field_my_title",
"default": "foo",
editable: true,
type: "StringField"
},
my_id: {
key: "field_my_id",
"default": "",
editable: true,
type: "StringField"
},
my_title_non_editable: {
key: "field_my_title_non_editable",
"default": "foo",
editable: false,
type: "StringField"
},
my_start_date: {
key: "field_my_start_date",
"default": "foo",
editable: true,
type: "DateTimeField"
},
your_reference: {
key: "field_your_reference",
"default": "bar",
editable: true,
type: "StringField"
},
your_reference_non_editable: {
key: "field_your_reference_non_editable",
"default": "bar",
editable: false,
type: "StringField"
},
sort_index: {
key: "field_sort_index",
"default": "foobar",
editable: true,
type: "StringField"
},
"_actions": {
put: {
href: "one erp5 url"
}
}
}
}
}),
document_hateoas2 = JSON.stringify({
// Kept property
"title": "foo2",
// Remove all _ properties
"_bar": "john doo2",
"_links": {
type: {
name: "Person"
}
},
"_embedded": {
"_view": {
form_id: {
key: "form_id",
"default": "Base_view"
},
"_actions": {
put: {
href: "one erp5 url"
}
}
}
}
}),
bulk_hateoas = JSON.parse(root_hateoas),
server = this.server;
bulk_hateoas.result_list = [
JSON.parse(document_hateoas),
JSON.parse(document_hateoas2)
];
bulk_hateoas = JSON.stringify(bulk_hateoas);
this.server.respondWith("GET", domain, [200, {
"Content-Type": "application/hal+json"
}, root_hateoas]);
this.server.respondWith("POST", bulk_url, [200, {
"Content-Type": "application/hal+json"
}, bulk_hateoas]);
stop();
expect(15);
this.jio.bulk([{
method: "get",
parameter_list: [id]
}, {
method: "get",
parameter_list: [id2]
}])
.then(function (result_list) {
equal(server.requests.length, 2);
equal(server.requests[0].method, "GET");
equal(server.requests[0].url, domain);
equal(server.requests[0].requestBody, undefined);
equal(server.requests[0].withCredentials, true);
equal(server.requests[1].method, "POST");
equal(server.requests[1].url, bulk_url);
// XXX Check form data
ok(server.requests[1].requestBody instanceof FormData);
ok(context.spy.calledOnce, "FormData.append count " +
context.spy.callCount);
equal(context.spy.firstCall.args[0], "bulk_list", "First append call");
equal(context.spy.firstCall.args[1],
JSON.stringify([{
relative_url: "person_module/20150119_azerty",
view: "bar_view"
}, {
relative_url: "person_module/20150219_azerty",
view: "bar_view"
}]),
"First append call");
equal(server.requests[1].withCredentials, true);
var result = result_list[0],
result2 = result_list[1];
equal(result_list.length, 2);
deepEqual(result, {
portal_type: "Person",
parent_relative_url: "bar_module",
reference: "bar",
reference_non_editable: "bar",
title: "foo"
}, "Check document");
deepEqual(result2, {
portal_type: "Person"
}, "Check document2");
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
}(jIO, QUnit, Blob, sinon, encodeURIComponent, FormData)); }(jIO, QUnit, Blob, sinon, encodeURIComponent, FormData));
test/jio.storage/replicatestorage.tests.js
View file @ 3e6379c8
...@@ -4039,301 +4039,6 @@ ...@@ -4039,301 +4039,6 @@
}); });
}); });
test("bulk remote document creation", function () {
stop();
expect(3);
var id,
post_id = "123456789",
context = this;
function Storage200Bulk(spec) {
this._sub_storage = jIO.createJIO(spec.sub_storage);
}
Storage200Bulk.prototype.bulk = function (args) {
deepEqual(args, [{
method: "get",
parameter_list: [post_id]
}]);
return this._sub_storage.get(post_id)
.push(function (doc) {
return [doc];
});
};
Storage200Bulk.prototype.post = function (param) {
return this.put(post_id, param);
};
Storage200Bulk.prototype.put = function () {
return this._sub_storage.put.apply(this._sub_storage, arguments);
};
Storage200Bulk.prototype.hasCapacity = function (name) {
if (name === "bulk_get") {
return true;
}
return this._sub_storage.hasCapacity.apply(this._sub_storage, arguments);
};
Storage200Bulk.prototype.buildQuery = function () {
return this._sub_storage.buildQuery.apply(this._sub_storage, arguments);
};
jIO.addStorage(
'replicatestorage200bulk',
Storage200Bulk
);
this.jio = jIO.createJIO({
type: "replicate",
local_sub_storage: {
type: "memory"
},
remote_sub_storage: {
type: "replicatestorage200bulk",
sub_storage: {
type: "memory"
}
}
});
context.jio.__storage._remote_sub_storage.post({"title": "bar"})
.then(function (result) {
id = result;
return context.jio.repair();
})
.then(function () {
return context.jio.get(id);
})
.then(function (result) {
deepEqual(result, {
title: "bar"
});
})
.then(function () {
return context.jio.__storage._signature_sub_storage.get(id);
})
.then(function (result) {
deepEqual(result, {
hash: "6799f3ea80e325b89f19589282a343c376c1f1af"
});
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("bulk remote document modification", function () {
stop();
expect(5);
var id,
post_id = "123456789",
context = this;
function Storage200Bulk(spec) {
this._sub_storage = jIO.createJIO(spec.sub_storage);
}
Storage200Bulk.prototype.bulk = function (args) {
deepEqual(args, [{
method: "get",
parameter_list: [post_id]
}]);
return this._sub_storage.get(post_id)
.push(function (doc) {
return [doc];
});
};
Storage200Bulk.prototype.post = function (param) {
return this.put(post_id, param);
};
Storage200Bulk.prototype.put = function () {
return this._sub_storage.put.apply(this._sub_storage, arguments);
};
Storage200Bulk.prototype.hasCapacity = function (name) {
if (name === "bulk_get") {
return true;
}
return this._sub_storage.hasCapacity.apply(this._sub_storage, arguments);
};
Storage200Bulk.prototype.buildQuery = function () {
return this._sub_storage.buildQuery.apply(this._sub_storage, arguments);
};
jIO.addStorage(
'replicatestorage200bulkremotemodification',
Storage200Bulk
);
this.jio = jIO.createJIO({
type: "replicate",
local_sub_storage: {
type: "memory"
},
remote_sub_storage: {
type: "replicatestorage200bulkremotemodification",
sub_storage: {
type: "memory"
}
}
});
context.jio.__storage._remote_sub_storage.post({"title": "bar"})
.then(function (result) {
id = result;
return context.jio.repair();
})
.then(function () {
return context.jio.get(id);
})
.then(function (result) {
deepEqual(result, {
title: "bar"
});
})
.then(function () {
return context.jio.__storage._signature_sub_storage.get(id);
})
.then(function (result) {
deepEqual(result, {
hash: "6799f3ea80e325b89f19589282a343c376c1f1af"
});
})
.then(function () {
return context.jio.__storage._remote_sub_storage.put(
id,
{"title": "foo"}
);
})
.then(function (result) {
id = result;
return context.jio.repair();
})
.then(function () {
return context.jio.get(id);
})
.then(function (result) {
deepEqual(result, {
title: "foo"
});
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("Donot bulk if substorage don't implement it", function () {
stop();
expect(2);
var id,
post_id = "123456789",
context = this;
function SubStorage200Bulk() {
return;
}
function Storage200Bulk(spec) {
this._sub_storage = jIO.createJIO(spec.sub_storage);
}
Storage200Bulk.prototype.bulk = function (args) {
deepEqual(args, [{
method: "get",
parameter_list: [post_id]
}]);
return this._sub_storage.bulk(args);
};
Storage200Bulk.prototype.get = function () {
return this._sub_storage.get.apply(this._sub_storage, arguments);
};
Storage200Bulk.prototype.post = function (param) {
return this.put(post_id, param);
};
Storage200Bulk.prototype.put = function () {
return this._sub_storage.put.apply(this._sub_storage, arguments);
};
Storage200Bulk.prototype.hasCapacity = function () {
return this._sub_storage.hasCapacity.apply(this._sub_storage, arguments);
};
Storage200Bulk.prototype.buildQuery = function () {
return this._sub_storage.buildQuery.apply(this._sub_storage, arguments);
};
jIO.addStorage(
'replicatestorage200nobulk',
Storage200Bulk
);
SubStorage200Bulk.prototype.get = function () {
return {title: "bar"};
};
SubStorage200Bulk.prototype.post = function (param) {
return this.put(post_id, param);
};
SubStorage200Bulk.prototype.put = function () {
return post_id;
};
SubStorage200Bulk.prototype.hasCapacity = function (name) {
if (name === "bulk_get") {
return false;
}
return true;
};
SubStorage200Bulk.prototype.buildQuery = function () {
return [{id: "123456789", value: {"title": "bar"}}];
};
jIO.addStorage(
'replicatesubstorage200nobulk',
SubStorage200Bulk
);
this.jio = jIO.createJIO({
type: "replicate",
local_sub_storage: {
type: "memory"
},
remote_sub_storage: {
type: "replicatestorage200nobulk",
sub_storage: {
type: "replicatesubstorage200nobulk"
}
}
});
context.jio.__storage._remote_sub_storage.post({"title": "bar"})
.then(function (result) {
id = result;
return context.jio.repair();
})
.then(function () {
return context.jio.get(id);
})
.then(function (result) {
deepEqual(result, {
title: "bar"
});
})
.then(function () {
return context.jio.__storage._signature_sub_storage.get(id);
})
.then(function (result) {
deepEqual(result, {
hash: "6799f3ea80e325b89f19589282a343c376c1f1af"
});
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("use 1 parallel operation", function () { test("use 1 parallel operation", function () {
stop(); stop();
expect(16); expect(16);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7