Skip to content

jio
jio
Commit c2386704 authored by Bryan Kaperick's avatar Bryan Kaperick
Browse files
Options

Added additional testing, fixed attachment methods, incorporated Romain's... 

Added additional testing, fixed attachment methods, incorporated Romain's comments, and added more comments and simplified code structure.
parent f71fb6cb
Hide whitespace changes
Inline Side-by-side
Showing with 359 additions and 202 deletions
src/jio.storage/historystorage.js
View file @ c2386704
/*jslint nomen: true*/
/*global RSVP, jiodate*/
(function (jIO) {
/*global RSVP*/
(function (jIO, RSVP) {
"use strict";
//local mod/frozen remote
//local del/remote mod
// Used to distinguish between operations done within the same millisecond
var unique_timestamp = function () {
......@@ -15,7 +12,89 @@
.toString(16)).slice(-4),
timestamp = Date.now().toString();
return timestamp + "-" + uuid;
};
},
// Helper function for getAttachment
findAttachment = function (substorage, name, metadata_query, steps) {
var options = {
query: metadata_query,
sort_on: [["timestamp", "descending"], ["op", "ascending"]],
select_list: ["op", "name"]
};
return substorage.allDocs(options)
.push(function (results) {
var ind,
id = metadata_query.value,
count = 0;
// At the least, a document needs to have been put and an attachment
// needs to have been put
if (results.data.rows.length > 1) {
for (ind = 0; ind < results.data.rows.length; ind += 1) {
// Cannot get the attachment of a removed document
if (results.data.rows[ind].value.op === "remove") {
throw new jIO.util.jIOError(
"HistoryStorage: cannot find attachment '" + name +
"' of object '" + id + "' (removed)",
404
);
}
// Make sure to get the correct revision of the attachment
// and throw 404 error if it was removed
if (results.data.rows[ind].value.name === name) {
if (count === steps) {
if (results.data.rows[ind].value.op === "removeAttachment") {
throw new jIO.util.jIOError(
"HistoryStorage: cannot find attachment '" + name +
"' of object '" + id + "' (removed)",
404
);
}
return substorage.getAttachment(
results.data.rows[ind].id,
name
);
}
count += 1;
}
}
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find attachment '" + name +
"' of object '" + id + "'",
404
);
});
},
findDoc = function (substorage, metadata_query, steps) {
var options = {
query: metadata_query,
sort_on: [["timestamp", "descending"]],
select_list: ["op"],
limit: [steps, 1]
};
return substorage.allDocs(options)
.push(function (results) {
var id_in = metadata_query.query_list[0].value;
if (results.data.rows.length > 0) {
if (results.data.rows[0].value.op === "put") {
return substorage.get(results.data.rows[0].id)
.push(function (result) {
return result.doc;
});
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id_in + "' (removed)",
404
);
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id_in + "'",
404
);
});
};
/**
* The jIO HistoryStorage extension
......@@ -24,92 +103,53 @@
* @constructor
*/
function HistoryStorage(spec) {
this._sub_storage = jIO.createJIO({
type: "query",
sub_storage: spec.sub_storage
});
this._sub_storage = jIO.createJIO(spec.sub_storage);
}
HistoryStorage.prototype.get = function (id_in) {
// Query to get the last edit made to this document
var substorage = this._sub_storage,
metadata_query = function (id) {
return "doc_id: " + id + " AND ((op: put) OR (op: remove))";
},
options = {
query: metadata_query(id_in),
sort_on: [["timestamp", "descending"]],
limit: [0, 1]
};
return substorage.allDocs(options)
.push(function (results) {
if (results.data.rows.length > 0) {
return substorage.get(results.data.rows[0].id);
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id_in + "' (0)",
404
);
})
.push(function (result) {
if (result.op === "put") {
return result.doc;
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id_in + "' (removed)",
404
);
metadata_query;
// Include id_in as value in query object for safety
metadata_query = jIO.QueryFactory.create(
"(doc_id: undefined) AND ((op: put) OR (op: remove))"
);
metadata_query.query_list[0].value = id_in;
return findDoc(substorage, metadata_query, 0)
.push(undefined,
// If no documents returned in first query, check if the id is encoding
// revision information
function (error) {
if (!(error instanceof jIO.util.jIOError) ||
(error.status_code !== 404) ||
(error.message !== "HistoryStorage: cannot find object '" +
id_in + "'")) {
throw error;
}
// If no documents returned in first query, check if the id is encoding
// revision information
}, function () {
var steps,
steps_loc = id_in.lastIndexOf("_-");
// If revision signature is not in id_in, than return 404, since id
// is not found
if (steps_loc === -1) {
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id_in + "' (1)",
404
);
}
// "_-" is the revision signature used to indicate a previous revision
var steps,
steps_loc = id_in.lastIndexOf("_-");
// If revision signature is found, query storage based on this
steps = Number(id_in.slice(steps_loc + 2));
id_in = id_in.slice(0, steps_loc);
options = {
query: metadata_query(id_in),
sort_on: [["timestamp", "descending"]],
limit: [steps, 1]
};
return substorage.allDocs(options)
.push(function (results) {
if (results.data.rows.length > 0) {
return substorage.get(results.data.rows[0].id);
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id_in + "' (2)",
404
);
})
.push(function (result) {
if (result.op === "put") {
return result.doc;
}
// If the revision signature '_-' is not contained in the id, then
// the first findDoc call should have found the id if it exists
if (steps_loc === -1) {
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id_in +
"' (removed) (1)",
"HistoryStorage: cannot find object '" + id_in + "'",
404
);
});
});
};
}
HistoryStorage.prototype.post = function (metadata) {
return this._sub_storage.post(metadata);
// If revision signature is found, query storage based on this
steps = Number(id_in.slice(steps_loc + 2));
id_in = id_in.slice(0, steps_loc);
metadata_query.query_list[0].value = id_in;
return findDoc(substorage, metadata_query, steps);
});
};
HistoryStorage.prototype.put = function (id, data) {
......@@ -136,12 +176,25 @@
};
HistoryStorage.prototype.allAttachments = function (id) {
// XXX: Do we need to be able to retrieve older revisions with
// allAttachments?
var substorage = this._sub_storage,
options = {
query: "(doc_id: " + id + ") AND " +
"((op: putAttachment) OR (op: removeAttachment))",
sort_on: [["timestamp", "descending"]]
};
query_obj,
options;
// Include id as value in query object for safety (as opposed to string
// concatenation)
query_obj = jIO.QueryFactory.create(
"(doc_id: undefined) AND ((op: putAttachment) OR (op: removeAttachment))"
);
query_obj.query_list[0].value = id;
// Only query for attachment edits
options = {
query: query_obj,
sort_on: [["timestamp", "descending"]]
};
return this._sub_storage.allDocs(options)
.push(function (results) {
var promises = results.data.rows.map(function (data) {
......@@ -154,6 +207,8 @@
attachments = {},
ind,
doc;
// Only include attachments whose most recent edit is a putAttachment
// (and not a removeAttachment)
for (ind = 0; ind < results.length; ind += 1) {
doc = results[ind];
if (!seen.hasOwnProperty(doc.name)) {
......@@ -187,79 +242,45 @@
// Query to get the last edit made to this document
var substorage = this._sub_storage,
metadata_query = function (id) {
return "(doc_id: " + id +
") AND (name: " + name +
") AND ((op: putAttachment) OR (op: removeAttachment))";
},
options = {
query: metadata_query(id),
sort_on: [["timestamp", "descending"]],
limit: [0, 1]
};
return substorage.allDocs(options)
.push(function (results) {
if (results.data.rows.length > 0) {
return substorage.get(results.data.rows[0].id);
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id + "' (0)",
404
);
})
.push(function (result) {
if (result.op === "putAttachment") {
return substorage.getAttachment(result.timestamp, result.name);
//return result.blob;
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id + "' (removed)",
404
);
metadata_query;
// Include id_in as value in query object for safety
metadata_query = jIO.QueryFactory.create(
"(doc_id: undefined)"
);
metadata_query.value = id;
return findAttachment(substorage, name, metadata_query, 0)
.push(undefined,
// If no documents returned in first query, check if the id is encoding
// revision information
function (error) {
if (!(error instanceof jIO.util.jIOError) ||
(error.status_code !== 404) ||
(error.message !== "HistoryStorage: cannot find attachment '" +
name + "' of object '" + id + "'")) {
throw error;
}
// If no documents returned in first query, check if the id is encoding
// revision information
}, function () {
var steps,
steps_loc = id.lastIndexOf("_-");
// If revision signature is not in id_in, than return 404, since id
// is not found
if (steps_loc === -1) {
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id + "' (1)",
404
);
}
var steps,
steps_loc = name.lastIndexOf("_-");
// If revision signature is found, query storage based on this
steps = Number(id.slice(steps_loc + 2));
id = id.slice(0, steps_loc);
options = {
query: metadata_query(id),
sort_on: [["timestamp", "descending"]],
limit: [steps, 1]
};
return substorage.allDocs(options)
.push(function (results) {
if (results.data.rows.length > 0) {
return substorage.get(results.data.rows[0].id);
}
// If revision signature is not in id_in, than return 404, since id
// is not found
if (steps_loc === -1) {
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id + "' (2)",
"HistoryStorage: cannot find attachment '" + name +
"' of object '" + id + "'",
404
);
})
.push(function (result) {
if (result.op === "putAttachment") {
return substorage.getAttachment(result.timestamp, result.name);
//return result.blob;
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id + "' (removed) (1)",
404
);
});
});
}
// If revision signature is found, query storage based on this
steps = Number(name.slice(steps_loc + 2));
name = name.slice(0, steps_loc);
return findAttachment(substorage, name, metadata_query, steps);
});
};
HistoryStorage.prototype.removeAttachment = function (id, name) {
......@@ -281,9 +302,7 @@
};
HistoryStorage.prototype.buildQuery = function (options) {
if (options.message === "give all docs") {
return this._sub_storage.allDocs(options.opts);
}
// Set default values
if (options === undefined) {
options = {};
}
......@@ -298,17 +317,7 @@
}
options.sort_on.push(["timestamp", "descending"]);
options.query = jIO.QueryFactory.create(options.query);
var meta_options = {
// XXX: I don't believe it's currently possible to query on
// sub-attributes so for now, we just use the inputted query, which
// obviously will fail
//query: "",//(op: put) OR (op: remove)",
// XXX: same here, we cannot sort correctly because we cannot access
// attributes of doc
query: "(op: remove) OR (op: put)",
sort_on: options.sort_on
},
var meta_options,
substorage = this._sub_storage,
// Check if query involved _REVISION. If not, we will later place a
......@@ -322,6 +331,8 @@
} else {
rev_query = (query_obj.key === "_REVISION");
}
// Traverse through query tree to find mentions of _REVISION
// and stop as soon as it is found once
while (query_stack.length > 0 && (!rev_query)) {
query_obj = query_stack.pop();
for (ind = 0; ind < query_obj.query_list.length; ind += 1) {
......@@ -334,18 +345,24 @@
}
}
// Query for all edits putting or removing documents (and nothing about
// attachments)
meta_options = {
query: "(op: remove) OR (op: put)",
sort_on: options.sort_on
};
return this._sub_storage.allDocs(meta_options)
// Get all documents found in query
// XXX: Once include_docs is implemented, this step can be simplified
.push(function (results) {
var promises = results.data.rows.map(function (data) {
return substorage.get(data.id);
});
return RSVP.all(promises);
})
.push(function (results) {
.push(function (results) {
// Label all documents with their current revision status
var docum,
revision_tracker = {},
......@@ -361,6 +378,9 @@
if (docum.op === "remove") {
docum.doc = {};
}
// Add op and _REVISION to the docum.doc (temporarily) so the
// document can be matched manually with the inputted query
results[ind].doc._REVISION = revision_tracker[docum.doc_id];
results[ind].doc.op = docum.op;
}
......@@ -370,12 +390,16 @@
latest_rev_query = jIO.QueryFactory.create(
"(_REVISION: >= 0) AND (op: put)"
);
// If query does not use _REVISION, then by default set _REVISION := 0
if (rev_query) {
latest_rev_query.query_list[0] = options.query;
} else {
latest_rev_query.query_list[0] = jIO.QueryFactory.create(
"(_REVISION: = 0)"
);
// Check if options.query is nonempty
if (options.query.type === "simple" ||
options.query.type === "complex") {
latest_rev_query.query_list.push(options.query);
......@@ -404,7 +428,8 @@
return true;
})
// Format results to be expected output of allDocs
// Return certain attributes in .val as specified by
// options.select_list
.map(function (current_doc) {
var val = {},
ind,
......@@ -415,6 +440,7 @@
val[key] = current_doc.doc[key];
}
}
// Format results to be expected output of allDocs
return {
doc: current_doc.doc,
value: val,
......@@ -426,4 +452,4 @@
jIO.addStorage('history', HistoryStorage);
}(jIO));
\ No newline at end of file
}(jIO, RSVP));
\ No newline at end of file
test/jio.storage/historystorage.tests.js
View file @ c2386704
/*jslint nomen: true*/
/*global Blob, jiodate*/
(function (jIO, QUnit) {
/*global Blob*/
(function (jIO, RSVP, Blob, QUnit) {
"use strict";
var test = QUnit.test,
stop = QUnit.stop,
......@@ -37,11 +37,14 @@
jio = jIO.createJIO({
type: "history",
sub_storage: {
type: "uuid",
type: "query",
sub_storage: {
//type: "memory"
type: "indexeddb",
database: dbname
type: "uuid",
sub_storage: {
//type: "memory"
type: "indexeddb",
database: dbname
}
}
}
}),
......@@ -297,17 +300,20 @@
test("Testing proper adding/removing attachments",
function () {
stop();
expect(9);
expect(26);
// create storage of type "history" with memory as substorage
var jio = jIO.createJIO({
var dbname = "db_" + Date.now(),
jio = jIO.createJIO({
type: "history",
sub_storage: {
type: "uuid",
type: "query",
sub_storage: {
type: "memory"
//type: "indexeddb",
//database: dbname
type: "uuid",
sub_storage: {
type: "indexeddb",
database: dbname
}
}
}
}),
......@@ -316,17 +322,20 @@
sub_storage: {
type: "uuid",
sub_storage: {
type: "memory"
//type: "indexeddb",
//database: dbname
type: "indexeddb",
database: dbname
}
}
}),
blob1 = new Blob(['a']),
blob2 = new Blob(['b']),
blob3 = new Blob(['ccc']),
other_blob = new Blob(['1']);
jio.put("doc", {title: "foo0"})
.push(function () {
return jio.put("doc2", {key: "val"});
})
.push(function () {
return jio.putAttachment("doc", "attached", blob1);
})
......@@ -342,7 +351,7 @@
.push(function (result) {
deepEqual(result, {
title: "foo0"
}, "Get does not return any attachment information");
}, "Get does not return any attachment/revision information");
return jio.getAttachment("doc", "attached");
})
.push(function (result) {
......@@ -355,26 +364,40 @@
.push(function (result) {
deepEqual(result,
blob2,
"Return the attachment information with getAttachment"
"Return the attachment information with getAttachment for " +
"current revision"
);
return jio.getAttachment("doc", "attached_-1");
}, function (error) {
ok(false, error);
})
.push(function (result) {
deepEqual(result,
blob1,
"Return the attachment information with getAttachment"
"Return the attachment information with getAttachment for " +
"previous revision"
);
return jio.getAttachment("doc", "attached_-2");
}, function (error) {
ok(false, error);
})
.push(function () {
ok(false, "This query should have thrown a 404 error");
},
function (error) {
ok(error instanceof jIO.util.jIOError, "Correct type of error");
deepEqual(error.status_code,
404,
"Error if you try to go back more revisions than what exists");
return jio.allAttachments("doc");
return jio.getAttachment("doc", "other_attached");
})
.push(function (result) {
deepEqual(result,
other_blob,
"Other document successfully queried"
);
return jio.allAttachments("doc");
})
.push(function (results) {
deepEqual(results, {
"attached": {},
......@@ -388,13 +411,14 @@
.push(function (result) {
deepEqual(result, {
title: "foo0"
}, "Get does not return any attachment information");
}, "Get does not return any attachment information (9)");
return jio.getAttachment("doc", "attached");
})
.push(function () {
ok(false, "This query should have thrown a 404 error");
},
function (error) {
ok(error instanceof jIO.util.jIOError, "Correct type of error");
deepEqual(error.status_code,
404,
"Removed attachments cannot be queried");
......@@ -404,18 +428,99 @@
deepEqual(results, {
"other_attached": {}
}, "allAttachments works as expected with a removed attachment");
return jio.putAttachment("doc", "attached", blob3);
})
.push(function () {
return not_history.allDocs();
})
.push(function (results) {
var promises = results.data.rows.map(function (data) {
return not_history.get(data.id);
});
return RSVP.all(promises);
})
.push(function (results) {
deepEqual(results, [
{timestamp: results[0].timestamp,
doc_id: "doc", doc: results[0].doc, op: "put"},
{timestamp: results[1].timestamp,
doc_id: "doc2", doc: results[1].doc, op: "put"},
{timestamp: results[2].timestamp,
doc_id: "doc", name: "attached", op: "putAttachment"},
{timestamp: results[3].timestamp,
doc_id: "doc", name: "attached", op: "putAttachment"},
{timestamp: results[4].timestamp,
doc_id: "doc", name: "other_attached", op: "putAttachment"},
{timestamp: results[5].timestamp,
doc_id: "doc", name: "attached", op: "removeAttachment"},
{timestamp: results[6].timestamp,
doc_id: "doc", name: "attached", op: "putAttachment"}
], "Other storage can access all document revisions."
);
})
.push(function () {
return jio.getAttachment("doc", "attached");
})
.push(function (result) {
deepEqual(result,
blob3,
"Return the attachment information with getAttachment"
);
return jio.getAttachment("doc", "attached_-0");
})
.push(function (result) {
deepEqual(result,
blob3,
"Return the attachment information with getAttachment"
);
return jio.getAttachment("doc", "attached_-1");
})
.push(function () {
ok(false, "This query should have thrown a 404 error");
},
function (error) {
ok(error instanceof jIO.util.jIOError, "Correct type of error");
deepEqual(error.status_code,
404,
"Error if you try to go back to a removed attachment state");
return jio.getAttachment("doc", "attached_-2");
})
.push(function (result) {
deepEqual(result,
blob2,
"Return the attachment information with getAttachment (17)"
);
return jio.getAttachment("doc", "attached_-3");
})
.push(function (result) {
deepEqual(result,
blob1,
"Return the attachment information with getAttachment"
);
return jio.getAttachment("doc", "attached_-4");
})
.push(function () {
ok(false, "This query should have thrown a 404 error");
},
function (error) {
ok(error instanceof jIO.util.jIOError, "Correct type of error");
deepEqual(error.status_code,
404,
"Error if you try to go back more revisions than what exists");
})
.push(function () {
return jio.allDocs();
})
.push(function (results) {
equal(results.data.rows.length, 1, "Only one document in storage");
return jio.get(results.data.rows[0].id);
equal(results.data.rows.length,
2,
"Two documents in accessible storage");
return jio.get(results.data.rows[1].id);
})
.push(function (result) {
deepEqual(result, {
"title": "foo0"
});
}, "Get second document accessible from jio storage");
return not_history.allDocs();
})
......@@ -424,6 +529,22 @@
return not_history.get(d.id);
}));
})
.push(function (results) {
equal(results.length, 7, "Seven document revisions in storage (24)");
return jio.remove("doc");
})
.push(function () {
return jio.getAttachment("doc", "attached");
})
.push(function () {
ok(false, "This query should have thrown a 404 error");
},
function (error) {
ok(error instanceof jIO.util.jIOError, "Correct type of error");
deepEqual(error.status_code,
404,
"Cannot get the attachment of a removed document");
})
.fail(function (error) {
//console.log(error);
ok(false, error);
......@@ -446,9 +567,12 @@
var jio = jIO.createJIO({
type: "history",
sub_storage: {
type: "uuid",
type: "query",
sub_storage: {
type: "memory"
type: "uuid",
sub_storage: {
type: "memory"
}
}
}
});
......@@ -627,6 +751,7 @@
.always(function () {start(); });
});
/////////////////////////////////////////////////////////////////
// Querying older revisions
/////////////////////////////////////////////////////////////////
......@@ -641,9 +766,12 @@
var jio = jIO.createJIO({
type: "history",
sub_storage: {
type: "uuid",
type: "query",
sub_storage: {
type: "memory"
type: "uuid",
sub_storage: {
type: "memory"
}
}
}
});
......@@ -854,7 +982,7 @@
})
.push(function (results) {
equal(results.data.rows.length, 4,
"Correct number of results with optins.limit set");
"Correct number of results with options.limit set");
deepEqual(results.data.rows[0].doc, {
"k2": "w0"
}, "Correct results with options.limit set");
......@@ -907,9 +1035,12 @@
var jio = jIO.createJIO({
type: "history",
sub_storage: {
type: "uuid",
type: "query",
sub_storage: {
type: "memory"
type: "uuid",
sub_storage: {
type: "memory"
}
}
}
}),
......@@ -960,4 +1091,4 @@
.always(function () {start(); });
});
}(jIO, QUnit));
\ No newline at end of file
}(jIO, RSVP, Blob, QUnit));
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7