Skip to content

jio
jio
Commit f71fb6cb authored by Bryan Kaperick's avatar Bryan Kaperick
Browse files
Options

Implemented attachment methods so that scenario_officejs.js runs with... 

Implemented attachment methods so that scenario_officejs.js runs with historystorage as the local sub storage.
parent fede0168
Hide whitespace changes
Inline Side-by-side
Showing with 440 additions and 46 deletions
examples/scenario_officejs.js
View file @ f71fb6cb
......@@ -135,6 +135,7 @@
},
count: {}
};
this.jio = jIO.createJIO({
type: "replicate",
query: {
......@@ -153,7 +154,7 @@
parallel_operation_amount: 10,
parallel_operation_attachment_amount: 10,
local_sub_storage: {
type: "query",
type: "history",
sub_storage: {
type: "uuid",
sub_storage: {
......@@ -213,7 +214,6 @@
doc_id = 'foo_module/1',
doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'},
blob = new Blob(['a']);
putFullDoc(this.jio.__storage._remote_sub_storage, doc_id, doc, blob)
.then(function () {
return test.jio.repair();
......@@ -279,7 +279,6 @@
test("remote document modification: copy", function () {
expect(2);
stop();
var test = this,
doc_id = 'foo_module/1',
doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'},
......@@ -367,7 +366,6 @@
doc_id = 'abc',
doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'},
blob = new Blob(['a']);
putFullDoc(this.jio, doc_id, doc, blob)
.then(function () {
resetCount(test.remote_mock_options.count);
......
src/jio.storage/historystorage.js
View file @ f71fb6cb
......@@ -3,6 +3,9 @@
(function (jIO) {
"use strict";
//local mod/frozen remote
//local del/remote mod
// Used to distinguish between operations done within the same millisecond
var unique_timestamp = function () {
......@@ -30,11 +33,13 @@
HistoryStorage.prototype.get = function (id_in) {
// Query to get the last edit made to this document
var substorage = this._sub_storage,
metadata_query = function (id) {
return "doc_id: " + id + " AND ((op: put) OR (op: remove))";
},
options = {
query: "doc_id: " + id_in,
query: metadata_query(id_in),
sort_on: [["timestamp", "descending"]],
limit: [0, 1]
};
......@@ -45,7 +50,7 @@
return substorage.get(results.data.rows[0].id);
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id_in + "'",
"HistoryStorage: cannot find object '" + id_in + "' (0)",
404
);
})
......@@ -67,7 +72,7 @@
// is not found
if (steps_loc === -1) {
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id_in + "'",
"HistoryStorage: cannot find object '" + id_in + "' (1)",
404
);
}
......@@ -76,7 +81,7 @@
steps = Number(id_in.slice(steps_loc + 2));
id_in = id_in.slice(0, steps_loc);
options = {
query: "doc_id: " + id_in,
query: metadata_query(id_in),
sort_on: [["timestamp", "descending"]],
limit: [steps, 1]
};
......@@ -86,7 +91,7 @@
return substorage.get(results.data.rows[0].id);
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id_in + "'",
"HistoryStorage: cannot find object '" + id_in + "' (2)",
404
);
})
......@@ -95,7 +100,8 @@
return result.doc;
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id_in + "' (removed)",
"HistoryStorage: cannot find object '" + id_in +
"' (removed) (1)",
404
);
});
......@@ -129,18 +135,143 @@
return this._sub_storage.put(timestamp, metadata);
};
HistoryStorage.prototype.allAttachments = function () {
return this._sub_storage.allAttachments.apply(this._sub_storage, arguments);
HistoryStorage.prototype.allAttachments = function (id) {
var substorage = this._sub_storage,
options = {
query: "(doc_id: " + id + ") AND " +
"((op: putAttachment) OR (op: removeAttachment))",
sort_on: [["timestamp", "descending"]]
};
return this._sub_storage.allDocs(options)
.push(function (results) {
var promises = results.data.rows.map(function (data) {
return substorage.get(data.id);
});
return RSVP.all(promises);
})
.push(function (results) {
var seen = {},
attachments = {},
ind,
doc;
for (ind = 0; ind < results.length; ind += 1) {
doc = results[ind];
if (!seen.hasOwnProperty(doc.name)) {
if (doc.op === "putAttachment") {
attachments[doc.name] = {};
}
seen[doc.name] = {};
}
}
return attachments;
});
};
HistoryStorage.prototype.getAttachment = function () {
return this._sub_storage.getAttachment.apply(this._sub_storage, arguments);
HistoryStorage.prototype.putAttachment = function (id, name, blob) {
var timestamp = unique_timestamp(),
metadata = {
// XXX: remove this attribute once query can sort_on id
timestamp: timestamp,
doc_id: id,
name: name,
op: "putAttachment"
},
substorage = this._sub_storage;
return this._sub_storage.put(timestamp, metadata)
.push(function () {
return substorage.putAttachment(timestamp, name, blob);
});
};
HistoryStorage.prototype.putAttachment = function () {
return this._sub_storage.putAttachment.apply(this._sub_storage, arguments);
HistoryStorage.prototype.getAttachment = function (id, name) {
// Query to get the last edit made to this document
var substorage = this._sub_storage,
metadata_query = function (id) {
return "(doc_id: " + id +
") AND (name: " + name +
") AND ((op: putAttachment) OR (op: removeAttachment))";
},
options = {
query: metadata_query(id),
sort_on: [["timestamp", "descending"]],
limit: [0, 1]
};
return substorage.allDocs(options)
.push(function (results) {
if (results.data.rows.length > 0) {
return substorage.get(results.data.rows[0].id);
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id + "' (0)",
404
);
})
.push(function (result) {
if (result.op === "putAttachment") {
return substorage.getAttachment(result.timestamp, result.name);
//return result.blob;
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id + "' (removed)",
404
);
// If no documents returned in first query, check if the id is encoding
// revision information
}, function () {
var steps,
steps_loc = id.lastIndexOf("_-");
// If revision signature is not in id_in, than return 404, since id
// is not found
if (steps_loc === -1) {
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id + "' (1)",
404
);
}
// If revision signature is found, query storage based on this
steps = Number(id.slice(steps_loc + 2));
id = id.slice(0, steps_loc);
options = {
query: metadata_query(id),
sort_on: [["timestamp", "descending"]],
limit: [steps, 1]
};
return substorage.allDocs(options)
.push(function (results) {
if (results.data.rows.length > 0) {
return substorage.get(results.data.rows[0].id);
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id + "' (2)",
404
);
})
.push(function (result) {
if (result.op === "putAttachment") {
return substorage.getAttachment(result.timestamp, result.name);
//return result.blob;
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id + "' (removed) (1)",
404
);
});
});
};
HistoryStorage.prototype.removeAttachment = function () {
return this._sub_storage.removeAttachment
.apply(this._sub_storage, arguments);
HistoryStorage.prototype.removeAttachment = function (id, name) {
var timestamp = unique_timestamp(),
metadata = {
// XXX: remove this attribute once query can sort_on id
timestamp: timestamp,
doc_id: id,
name: name,
op: "removeAttachment"
};
return this._sub_storage.put(timestamp, metadata);
};
HistoryStorage.prototype.repair = function () {
return this._sub_storage.repair.apply(this._sub_storage, arguments);
......@@ -150,7 +281,9 @@
};
HistoryStorage.prototype.buildQuery = function (options) {
if (options.message === "give all docs") {
return this._sub_storage.allDocs(options.opts);
}
if (options === undefined) {
options = {};
}
......@@ -160,16 +293,20 @@
if (options.sort_on === undefined) {
options.sort_on = [];
}
if (options.select_list === undefined) {
options.select_list = [];
}
options.sort_on.push(["timestamp", "descending"]);
options.query = jIO.QueryFactory.create(options.query);
var meta_options = {
// XXX: I don't believe it's currently possible to query on
// sub-attributes so for now, we just use the inputted query, which
// obviously will fail
query: "",
//query: "",//(op: put) OR (op: remove)",
// XXX: same here, we cannot sort correctly because we cannot access
// attributes of doc
query: "(op: remove) OR (op: put)",
sort_on: options.sort_on
},
substorage = this._sub_storage,
......@@ -201,49 +338,61 @@
// Get all documents found in query
.push(function (results) {
var promises = results.data.rows.map(function (data) {
return substorage.get(data.id);
});
return RSVP.all(promises);
})
.push(function (results) {
// Label all documents with their current revision status
var doc,
var docum,
revision_tracker = {},
latest_rev_query;
latest_rev_query,
results_reduced;
for (ind = 0; ind < results.length; ind += 1) {
doc = results[ind];
if (revision_tracker.hasOwnProperty(doc.doc_id)) {
revision_tracker[doc.doc_id] += 1;
docum = results[ind];
if (revision_tracker.hasOwnProperty(docum.doc_id)) {
revision_tracker[docum.doc_id] += 1;
} else {
revision_tracker[doc.doc_id] = 0;
revision_tracker[docum.doc_id] = 0;
}
doc._REVISION = revision_tracker[doc.doc_id];
if (docum.op === "remove") {
docum.doc = {};
}
results[ind].doc._REVISION = revision_tracker[docum.doc_id];
results[ind].doc.op = docum.op;
}
// Create a new query to only get non-removed revisions and abide by
// whatever the inputted query says
latest_rev_query = jIO.QueryFactory.create(
"(_REVISION: >= 0) AND (NOT op: remove)"
"(_REVISION: >= 0) AND (op: put)"
);
if (rev_query) {
latest_rev_query.query_list[0] = options.query;
} else {
latest_rev_query.query_list[0] = jIO.QueryFactory.create(
"(_REVISION: =0)"
"(_REVISION: = 0)"
);
if (options.query.type === "simple" ||
options.query.type === "complex") {
latest_rev_query.query_list.push(options.query);
}
}
return results
//return results
results_reduced = results
// Only return results which match latest_rev_query
.filter(function (doc) {
return latest_rev_query.match(doc);
})
.filter(function (docum) {
var filtered_res = latest_rev_query.match(docum.doc);
// Remove extra metadata used in revision query
delete docum.doc.op;
delete docum.doc._REVISION;
return filtered_res;
});
return results_reduced
// Only return the correct range of valid results specified by
// options.limit
......@@ -257,9 +406,18 @@
// Format results to be expected output of allDocs
.map(function (current_doc) {
var val = {},
ind,
key;
for (ind = 0; ind < options.select_list.length; ind += 1) {
key = options.select_list[ind];
if (current_doc.doc.hasOwnProperty(key)) {
val[key] = current_doc.doc[key];
}
}
return {
doc: current_doc.doc,
value: {},
value: val,
id: current_doc.doc_id
};
});
......
test/jio.storage/historystorage.tests.js
View file @ f71fb6cb
......@@ -11,6 +11,16 @@
equal = QUnit.equal,
module = QUnit.module;
function putFullDoc(storage, id, doc, attachment_name, attachment) {
return storage.put(id, doc)
.push(function () {
return storage.putAttachment(
id,
attachment_name,
attachment
);
});
}
/////////////////////////////////////////////////////////////////
// _revision parameter updating with RSVP all
......@@ -279,6 +289,148 @@
.always(function () {start(); });
});
**/
/////////////////////////////////////////////////////////////////
// Attachments
/////////////////////////////////////////////////////////////////
module("HistoryStorage.attachments");
test("Testing proper adding/removing attachments",
function () {
stop();
expect(9);
// create storage of type "history" with memory as substorage
var jio = jIO.createJIO({
type: "history",
sub_storage: {
type: "uuid",
sub_storage: {
type: "memory"
//type: "indexeddb",
//database: dbname
}
}
}),
not_history = jIO.createJIO({
type: "query",
sub_storage: {
type: "uuid",
sub_storage: {
type: "memory"
//type: "indexeddb",
//database: dbname
}
}
}),
blob1 = new Blob(['a']),
blob2 = new Blob(['b']),
other_blob = new Blob(['1']);
jio.put("doc", {title: "foo0"})
.push(function () {
return jio.putAttachment("doc", "attached", blob1);
})
.push(function () {
return jio.putAttachment("doc", "attached", blob2);
})
.push(function () {
return jio.putAttachment("doc", "other_attached", other_blob);
})
.push(function () {
return jio.get("doc");
})
.push(function (result) {
deepEqual(result, {
title: "foo0"
}, "Get does not return any attachment information");
return jio.getAttachment("doc", "attached");
})
.push(function (result) {
deepEqual(result,
blob2,
"Return the attachment information with getAttachment"
);
return jio.getAttachment("doc", "attached_-0");
})
.push(function (result) {
deepEqual(result,
blob2,
"Return the attachment information with getAttachment"
);
return jio.getAttachment("doc", "attached_-1");
})
.push(function (result) {
deepEqual(result,
blob1,
"Return the attachment information with getAttachment"
);
return jio.getAttachment("doc", "attached_-2");
})
.push(function () {
ok(false, "This query should have thrown a 404 error");
},
function (error) {
deepEqual(error.status_code,
404,
"Error if you try to go back more revisions than what exists");
return jio.allAttachments("doc");
})
.push(function (results) {
deepEqual(results, {
"attached": {},
"other_attached": {}
}, "allAttachments works as expected.");
return jio.removeAttachment("doc", "attached");
})
.push(function () {
return jio.get("doc");
})
.push(function (result) {
deepEqual(result, {
title: "foo0"
}, "Get does not return any attachment information");
return jio.getAttachment("doc", "attached");
})
.push(function () {
ok(false, "This query should have thrown a 404 error");
},
function (error) {
deepEqual(error.status_code,
404,
"Removed attachments cannot be queried");
return jio.allAttachments("doc");
})
.push(function (results) {
deepEqual(results, {
"other_attached": {}
}, "allAttachments works as expected with a removed attachment");
})
.push(function () {
return jio.allDocs();
})
.push(function (results) {
equal(results.data.rows.length, 1, "Only one document in storage");
return jio.get(results.data.rows[0].id);
})
.push(function (result) {
deepEqual(result, {
"title": "foo0"
});
return not_history.allDocs();
})
.push(function (results) {
return RSVP.all(results.data.rows.map(function (d) {
return not_history.get(d.id);
}));
})
.fail(function (error) {
//console.log(error);
ok(false, error);
})
.always(function () {start(); });
});
/////////////////////////////////////////////////////////////////
// Accessing older revisions
......@@ -483,7 +635,7 @@
test("Testing retrieval of older revisions via allDocs calls",
function () {
stop();
expect(37);
expect(42);
// create storage of type "history" with memory as substorage
var jio = jIO.createJIO({
......@@ -527,7 +679,7 @@
.push(function (result) {
deepEqual(result, {
"k": "v3"
});
}, "One correct result.");
})
.push(function () {
return jio.allDocs({
......@@ -540,7 +692,7 @@
"Only one query returned with options.revision_limit == [1,1]");
deepEqual(results.data.rows[0].doc, {
"k": "v2"
});
}, "One correct result.");
return jio.allDocs({
query: "_REVISION : =2"
});
......@@ -576,13 +728,14 @@
});
})
.push(function (results) {
equal(results.data.rows.length, 2);
equal(results.data.rows.length, 2,
"Only retrieve two most recent revions");
deepEqual(results.data.rows[0].doc, {
"k": "v3"
}, "Only retrieve two most recent revions");
}, "First retrieved revision is correct");
deepEqual(results.data.rows[1].doc, {
"k": "v2"
});
}, "Second retrieved revision is correct");
})
.push(function () {
return jio.remove("doc");
......@@ -660,10 +813,10 @@
})
.push(function (results) {
equal(results.data.rows.length, 1,
"There is only one non-removed doc");
"There is only one non-removed doc.");
deepEqual(results.data.rows[0].doc, {
"k2": "w1"
});
}, "Returned the one correct document.");
})
.push(function () {
return jio.remove("doc2");
......@@ -714,6 +867,91 @@
deepEqual(results.data.rows[3].doc, {
"k": "v1"
}, "Correct results with options.limit set");
return jio.allDocs({
query: "_REVISION: = 1",
select_list: ["k"]
});
})
.push(function (results) {
equal(results.data.rows.length, 2);
deepEqual(results.data.rows[0].doc, {
"k2": "w1"
});
deepEqual(results.data.rows[0].value, {});
deepEqual(results.data.rows[1].doc, {
"k": "v3"
});
deepEqual(results.data.rows[1].value, {
"k": "v3"
});
})
.fail(function (error) {
//console.log(error);
ok(false, error);
})
.always(function () {start(); });
});
/////////////////////////////////////////////////////////////////
// Complex Queries
/////////////////////////////////////////////////////////////////
module("HistoryStorage.complex_queries");
test("Testing retrieval of older revisions via allDocs calls",
function () {
stop();
expect(3);
// create storage of type "history" with memory as substorage
var jio = jIO.createJIO({
type: "history",
sub_storage: {
type: "uuid",
sub_storage: {
type: "memory"
}
}
}),
doc = {
"modification_date": "a",
"portal_type": "Foo",
"title": "foo_module/1"
},
blob = new Blob(['a']);
putFullDoc(jio, "foo_module/1", doc, "data", blob)
.push(function () {
return jio.get("foo_module/1");
})
.push(function (result) {
deepEqual(result, {
"modification_date": "a",
"portal_type": "Foo",
"title": "foo_module/1"
}, "Can retrieve a document after attachment placed."
);
})
.push(function () {
return jio.allDocs({
query: "portal_type: Foo",
select_list: ["modification_date", "__id", "__id"],
sort_on: [["modification_date", "descending"],
["timestamp", "descending"],
["timestamp", "descending"]
]
});
})
.push(function (results) {
equal(results.data.rows.length, 1);
deepEqual(results.data.rows[0], {
doc: {
"modification_date": "a",
"portal_type": "Foo",
"title": "foo_module/1"
},
id: "foo_module/1",
value: {modification_date: "a"}
});
})
.fail(function (error) {
//console.log(error);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7