Commit 3e6379c8 authored by Romain Courteaud's avatar Romain Courteaud

Drop bulk prototype.

Bulk was implemented to see if it improves ERP5 replication.
Using parallel replication functionality is much more efficient from server and client point of view.

It does not generate huge transaction on server side.
Client does not wait for all documents before writing them locally.
parent db73ae15
...@@ -172,56 +172,6 @@ ...@@ -172,56 +172,6 @@
}); });
}; };
ERP5Storage.prototype.bulk = function (request_list) {
var i,
storage = this,
bulk_list = [];
for (i = 0; i < request_list.length; i += 1) {
if (request_list[i].method !== "get") {
throw new Error("ERP5Storage: not supported " +
request_list[i].method + " in bulk");
}
bulk_list.push({
relative_url: request_list[i].parameter_list[0],
view: storage._default_view_reference
});
}
return getSiteDocument(storage)
.push(function (site_hal) {
var form_data = new FormData();
form_data.append("bulk_list", JSON.stringify(bulk_list));
return jIO.util.ajax({
"type": "POST",
"url": site_hal._actions.bulk.href,
"data": form_data,
// "headers": {
// "Content-Type": "application/json"
// },
"xhrFields": {
withCredentials: true
}
});
})
.push(function (response) {
var result_list = [],
hateoas = JSON.parse(response.target.responseText);
function pushResult(json) {
return extractPropertyFromFormJSON(json)
.push(function (json2) {
return convertJSONToGet(json2);
});
}
for (i = 0; i < hateoas.result_list.length; i += 1) {
result_list.push(pushResult(hateoas.result_list[i]));
}
return RSVP.all(result_list);
});
};
ERP5Storage.prototype.post = function (data) { ERP5Storage.prototype.post = function (data) {
var context = this, var context = this,
new_id; new_id;
...@@ -448,7 +398,7 @@ ...@@ -448,7 +398,7 @@
ERP5Storage.prototype.hasCapacity = function (name) { ERP5Storage.prototype.hasCapacity = function (name) {
return ((name === "list") || (name === "query") || return ((name === "list") || (name === "query") ||
(name === "select") || (name === "limit") || (name === "select") || (name === "limit") ||
(name === "sort")) || (name === "bulk_get"); (name === "sort"));
}; };
function isSingleLocalRoles(parsed_query) { function isSingleLocalRoles(parsed_query) {
......
...@@ -822,44 +822,6 @@ ...@@ -822,44 +822,6 @@
}); });
} }
function checkBulkSignatureDifference(queue, source, destination, id_list,
document_status_list, options,
conflict_force, conflict_revert,
conflict_ignore) {
queue
.push(function () {
return source.bulk(id_list);
})
.push(function (result_list) {
var i,
argument_list = [];
function getResult(j) {
return function (id) {
if (id !== id_list[j].parameter_list[0]) {
throw new Error("Does not access expected ID " + id);
}
return result_list[j];
};
}
for (i = 0; i < result_list.length; i += 1) {
argument_list[i] = [undefined, source, destination,
id_list[i].parameter_list[0],
conflict_force, conflict_revert,
conflict_ignore,
document_status_list[i].is_creation,
document_status_list[i].is_modification,
getResult(i), options];
}
return dispatchQueue(
checkSignatureDifference,
argument_list,
options.operation_amount
);
});
}
function pushStorage(source, destination, signature_allDocs, options) { function pushStorage(source, destination, signature_allDocs, options) {
var argument_list = [], var argument_list = [],
argument_list_deletion = []; argument_list_deletion = [];
...@@ -873,8 +835,6 @@ ...@@ -873,8 +835,6 @@
.push(function (source_allDocs) { .push(function (source_allDocs) {
var i, var i,
local_dict = {}, local_dict = {},
document_list = [],
document_status_list = [],
signature_dict = {}, signature_dict = {},
is_modification, is_modification,
is_creation, is_creation,
...@@ -902,26 +862,15 @@ ...@@ -902,26 +862,15 @@
is_creation = !signature_dict.hasOwnProperty(key) is_creation = !signature_dict.hasOwnProperty(key)
&& options.check_creation; && options.check_creation;
if (is_modification === true || is_creation === true) { if (is_modification === true || is_creation === true) {
if (options.use_bulk_get === true) { argument_list[i] = [undefined, source, destination,
document_list.push({ key,
method: "get", options.conflict_force,
parameter_list: [key] options.conflict_revert,
}); options.conflict_ignore,
document_status_list.push({ is_creation, is_modification,
is_creation: is_creation, source.get.bind(source),
is_modification: is_modification options];
}); i += 1;
} else {
argument_list[i] = [undefined, source, destination,
key,
options.conflict_force,
options.conflict_revert,
options.conflict_ignore,
is_creation, is_modification,
source.get.bind(source),
options];
i += 1;
}
} }
} }
} }
...@@ -957,14 +906,6 @@ ...@@ -957,14 +906,6 @@
); );
}); });
} }
if ((options.use_bulk_get === true) && (document_list.length !== 0)) {
checkBulkSignatureDifference(queue, source, destination,
document_list, document_status_list,
options,
options.conflict_force,
options.conflict_revert,
options.conflict_ignore);
}
return queue; return queue;
}); });
} }
...@@ -1049,24 +990,12 @@ ...@@ -1049,24 +990,12 @@
return signature_allDocs; return signature_allDocs;
}) })
.push(function (signature_allDocs) { .push(function (signature_allDocs) {
// Autoactivate bulk if substorage implements it
// Keep it like this until the bulk API is stabilized
var use_bulk_get = false;
try {
use_bulk_get = context._remote_sub_storage.hasCapacity("bulk_get");
} catch (error) {
if (!((error instanceof jIO.util.jIOError) &&
(error.status_code === 501))) {
throw error;
}
}
if (context._check_remote_modification || if (context._check_remote_modification ||
context._check_remote_creation || context._check_remote_creation ||
context._check_remote_deletion) { context._check_remote_deletion) {
return pushStorage(context._remote_sub_storage, return pushStorage(context._remote_sub_storage,
context._local_sub_storage, context._local_sub_storage,
signature_allDocs, { signature_allDocs, {
use_bulk_get: use_bulk_get,
use_revert_post: context._use_remote_post, use_revert_post: context._use_remote_post,
conflict_force: (context._conflict_handling === conflict_force: (context._conflict_handling ===
CONFLICT_KEEP_REMOTE), CONFLICT_KEEP_REMOTE),
......
...@@ -15,7 +15,6 @@ ...@@ -15,7 +15,6 @@
search_template = domain + "?mode=search{&query,select_list*,limit*," + search_template = domain + "?mode=search{&query,select_list*,limit*," +
"sort_on*,local_roles*}", "sort_on*,local_roles*}",
add_url = domain + "lets?add=somedocument", add_url = domain + "lets?add=somedocument",
bulk_url = domain + "lets?run=bulk",
root_hateoas = JSON.stringify({ root_hateoas = JSON.stringify({
"_links": { "_links": {
traverse: { traverse: {
...@@ -30,9 +29,6 @@ ...@@ -30,9 +29,6 @@
"_actions": { "_actions": {
add: { add: {
href: add_url href: add_url
},
bulk: {
href: bulk_url
} }
} }
}); });
...@@ -1964,202 +1960,4 @@ ...@@ -1964,202 +1960,4 @@
}); });
}); });
/////////////////////////////////////////////////////////////////
// erp5Storage.bulk
/////////////////////////////////////////////////////////////////
module("erp5Storage.bulk", {
setup: function () {
this.server = sinon.fakeServer.create();
this.server.autoRespond = true;
this.server.autoRespondAfter = 5;
this.spy = sinon.spy(FormData.prototype, "append");
this.jio = jIO.createJIO({
type: "erp5",
url: domain,
default_view_reference: "bar_view"
});
},
teardown: function () {
this.server.restore();
delete this.server;
this.spy.restore();
delete this.spy;
}
});
test("bulk get ERP5 document list", function () {
var id = "person_module/20150119_azerty",
id2 = "person_module/20150219_azerty",
context = this,
document_hateoas = JSON.stringify({
// Kept property
"title": "foo",
// Remove all _ properties
"_bar": "john doo",
"_links": {
type: {
name: "Person"
},
parent: {
href: "urn:jio:get:bar_module"
}
},
"_embedded": {
"_view": {
form_id: {
key: "form_id",
"default": "Base_view"
},
my_title: {
key: "field_my_title",
"default": "foo",
editable: true,
type: "StringField"
},
my_id: {
key: "field_my_id",
"default": "",
editable: true,
type: "StringField"
},
my_title_non_editable: {
key: "field_my_title_non_editable",
"default": "foo",
editable: false,
type: "StringField"
},
my_start_date: {
key: "field_my_start_date",
"default": "foo",
editable: true,
type: "DateTimeField"
},
your_reference: {
key: "field_your_reference",
"default": "bar",
editable: true,
type: "StringField"
},
your_reference_non_editable: {
key: "field_your_reference_non_editable",
"default": "bar",
editable: false,
type: "StringField"
},
sort_index: {
key: "field_sort_index",
"default": "foobar",
editable: true,
type: "StringField"
},
"_actions": {
put: {
href: "one erp5 url"
}
}
}
}
}),
document_hateoas2 = JSON.stringify({
// Kept property
"title": "foo2",
// Remove all _ properties
"_bar": "john doo2",
"_links": {
type: {
name: "Person"
}
},
"_embedded": {
"_view": {
form_id: {
key: "form_id",
"default": "Base_view"
},
"_actions": {
put: {
href: "one erp5 url"
}
}
}
}
}),
bulk_hateoas = JSON.parse(root_hateoas),
server = this.server;
bulk_hateoas.result_list = [
JSON.parse(document_hateoas),
JSON.parse(document_hateoas2)
];
bulk_hateoas = JSON.stringify(bulk_hateoas);
this.server.respondWith("GET", domain, [200, {
"Content-Type": "application/hal+json"
}, root_hateoas]);
this.server.respondWith("POST", bulk_url, [200, {
"Content-Type": "application/hal+json"
}, bulk_hateoas]);
stop();
expect(15);
this.jio.bulk([{
method: "get",
parameter_list: [id]
}, {
method: "get",
parameter_list: [id2]
}])
.then(function (result_list) {
equal(server.requests.length, 2);
equal(server.requests[0].method, "GET");
equal(server.requests[0].url, domain);
equal(server.requests[0].requestBody, undefined);
equal(server.requests[0].withCredentials, true);
equal(server.requests[1].method, "POST");
equal(server.requests[1].url, bulk_url);
// XXX Check form data
ok(server.requests[1].requestBody instanceof FormData);
ok(context.spy.calledOnce, "FormData.append count " +
context.spy.callCount);
equal(context.spy.firstCall.args[0], "bulk_list", "First append call");
equal(context.spy.firstCall.args[1],
JSON.stringify([{
relative_url: "person_module/20150119_azerty",
view: "bar_view"
}, {
relative_url: "person_module/20150219_azerty",
view: "bar_view"
}]),
"First append call");
equal(server.requests[1].withCredentials, true);
var result = result_list[0],
result2 = result_list[1];
equal(result_list.length, 2);
deepEqual(result, {
portal_type: "Person",
parent_relative_url: "bar_module",
reference: "bar",
reference_non_editable: "bar",
title: "foo"
}, "Check document");
deepEqual(result2, {
portal_type: "Person"
}, "Check document2");
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
}(jIO, QUnit, Blob, sinon, encodeURIComponent, FormData)); }(jIO, QUnit, Blob, sinon, encodeURIComponent, FormData));
...@@ -4039,301 +4039,6 @@ ...@@ -4039,301 +4039,6 @@
}); });
}); });
test("bulk remote document creation", function () {
stop();
expect(3);
var id,
post_id = "123456789",
context = this;
function Storage200Bulk(spec) {
this._sub_storage = jIO.createJIO(spec.sub_storage);
}
Storage200Bulk.prototype.bulk = function (args) {
deepEqual(args, [{
method: "get",
parameter_list: [post_id]
}]);
return this._sub_storage.get(post_id)
.push(function (doc) {
return [doc];
});
};
Storage200Bulk.prototype.post = function (param) {
return this.put(post_id, param);
};
Storage200Bulk.prototype.put = function () {
return this._sub_storage.put.apply(this._sub_storage, arguments);
};
Storage200Bulk.prototype.hasCapacity = function (name) {
if (name === "bulk_get") {
return true;
}
return this._sub_storage.hasCapacity.apply(this._sub_storage, arguments);
};
Storage200Bulk.prototype.buildQuery = function () {
return this._sub_storage.buildQuery.apply(this._sub_storage, arguments);
};
jIO.addStorage(
'replicatestorage200bulk',
Storage200Bulk
);
this.jio = jIO.createJIO({
type: "replicate",
local_sub_storage: {
type: "memory"
},
remote_sub_storage: {
type: "replicatestorage200bulk",
sub_storage: {
type: "memory"
}
}
});
context.jio.__storage._remote_sub_storage.post({"title": "bar"})
.then(function (result) {
id = result;
return context.jio.repair();
})
.then(function () {
return context.jio.get(id);
})
.then(function (result) {
deepEqual(result, {
title: "bar"
});
})
.then(function () {
return context.jio.__storage._signature_sub_storage.get(id);
})
.then(function (result) {
deepEqual(result, {
hash: "6799f3ea80e325b89f19589282a343c376c1f1af"
});
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("bulk remote document modification", function () {
stop();
expect(5);
var id,
post_id = "123456789",
context = this;
function Storage200Bulk(spec) {
this._sub_storage = jIO.createJIO(spec.sub_storage);
}
Storage200Bulk.prototype.bulk = function (args) {
deepEqual(args, [{
method: "get",
parameter_list: [post_id]
}]);
return this._sub_storage.get(post_id)
.push(function (doc) {
return [doc];
});
};
Storage200Bulk.prototype.post = function (param) {
return this.put(post_id, param);
};
Storage200Bulk.prototype.put = function () {
return this._sub_storage.put.apply(this._sub_storage, arguments);
};
Storage200Bulk.prototype.hasCapacity = function (name) {
if (name === "bulk_get") {
return true;
}
return this._sub_storage.hasCapacity.apply(this._sub_storage, arguments);
};
Storage200Bulk.prototype.buildQuery = function () {
return this._sub_storage.buildQuery.apply(this._sub_storage, arguments);
};
jIO.addStorage(
'replicatestorage200bulkremotemodification',
Storage200Bulk
);
this.jio = jIO.createJIO({
type: "replicate",
local_sub_storage: {
type: "memory"
},
remote_sub_storage: {
type: "replicatestorage200bulkremotemodification",
sub_storage: {
type: "memory"
}
}
});
context.jio.__storage._remote_sub_storage.post({"title": "bar"})
.then(function (result) {
id = result;
return context.jio.repair();
})
.then(function () {
return context.jio.get(id);
})
.then(function (result) {
deepEqual(result, {
title: "bar"
});
})
.then(function () {
return context.jio.__storage._signature_sub_storage.get(id);
})
.then(function (result) {
deepEqual(result, {
hash: "6799f3ea80e325b89f19589282a343c376c1f1af"
});
})
.then(function () {
return context.jio.__storage._remote_sub_storage.put(
id,
{"title": "foo"}
);
})
.then(function (result) {
id = result;
return context.jio.repair();
})
.then(function () {
return context.jio.get(id);
})
.then(function (result) {
deepEqual(result, {
title: "foo"
});
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("Donot bulk if substorage don't implement it", function () {
stop();
expect(2);
var id,
post_id = "123456789",
context = this;
function SubStorage200Bulk() {
return;
}
function Storage200Bulk(spec) {
this._sub_storage = jIO.createJIO(spec.sub_storage);
}
Storage200Bulk.prototype.bulk = function (args) {
deepEqual(args, [{
method: "get",
parameter_list: [post_id]
}]);
return this._sub_storage.bulk(args);
};
Storage200Bulk.prototype.get = function () {
return this._sub_storage.get.apply(this._sub_storage, arguments);
};
Storage200Bulk.prototype.post = function (param) {
return this.put(post_id, param);
};
Storage200Bulk.prototype.put = function () {
return this._sub_storage.put.apply(this._sub_storage, arguments);
};
Storage200Bulk.prototype.hasCapacity = function () {
return this._sub_storage.hasCapacity.apply(this._sub_storage, arguments);
};
Storage200Bulk.prototype.buildQuery = function () {
return this._sub_storage.buildQuery.apply(this._sub_storage, arguments);
};
jIO.addStorage(
'replicatestorage200nobulk',
Storage200Bulk
);
SubStorage200Bulk.prototype.get = function () {
return {title: "bar"};
};
SubStorage200Bulk.prototype.post = function (param) {
return this.put(post_id, param);
};
SubStorage200Bulk.prototype.put = function () {
return post_id;
};
SubStorage200Bulk.prototype.hasCapacity = function (name) {
if (name === "bulk_get") {
return false;
}
return true;
};
SubStorage200Bulk.prototype.buildQuery = function () {
return [{id: "123456789", value: {"title": "bar"}}];
};
jIO.addStorage(
'replicatesubstorage200nobulk',
SubStorage200Bulk
);
this.jio = jIO.createJIO({
type: "replicate",
local_sub_storage: {
type: "memory"
},
remote_sub_storage: {
type: "replicatestorage200nobulk",
sub_storage: {
type: "replicatesubstorage200nobulk"
}
}
});
context.jio.__storage._remote_sub_storage.post({"title": "bar"})
.then(function (result) {
id = result;
return context.jio.repair();
})
.then(function () {
return context.jio.get(id);
})
.then(function (result) {
deepEqual(result, {
title: "bar"
});
})
.then(function () {
return context.jio.__storage._signature_sub_storage.get(id);
})
.then(function (result) {
deepEqual(result, {
hash: "6799f3ea80e325b89f19589282a343c376c1f1af"
});
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("use 1 parallel operation", function () { test("use 1 parallel operation", function () {
stop(); stop();
expect(16); expect(16);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment