Skip to content

jio
jio
Commit a0aaf798 authored by Sven Franck's avatar Sven Franck
Browse files
Options

indexStorage: ALLDOCS new API and qunit

parent ada4b47d
Hide whitespace changes
Inline Side-by-side
Showing with 246 additions and 81 deletions
src/jio.storage/davstorage.js
View file @ a0aaf798
......@@ -790,6 +790,8 @@ jIO.addStorageType('dav', function (spec, my) {
/**
* Gets a document list from a distant dav storage
* Options:
* - {boolean} include_docs Also retrieve the actual document content.
* @method allDocs
* @param {object} command The JIO command
*/
......
src/jio.storage/indexstorage.js
View file @ a0aaf798
......@@ -72,16 +72,6 @@ jIO.addStorageType('indexed', function (spec, my) {
S4() + S4() + S4();
};
/**
* Escape string before storing
* @method sanitizeValue
* @param {string} s The string to be sanitized
* @return {string} The sanitized string
*/
priv.sanitizeValue = function (s) {
return s.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&')
};
/**
* Get number of elements in object
* @method getObjectSize
......@@ -249,6 +239,91 @@ jIO.addStorageType('indexed', function (spec, my) {
return indices;
};
/**
* Build the alldocs response from the index file (overriding substorage)
* @method allDocsResponseFromIndex
* @param {object} command The JIO command
* @param {boolean} include_docs Whether to also supply the document
* @param {object} option The options set for this method
* @returns {object} response The allDocs response
*/
priv.allDocsResponseFromIndex = function (indices, include_docs, option) {
var i, j, k, m, n = 0, l = priv.indices.length,
index, key, obj, prop, found, file,
unique_count = 0, unique_docids = [], all_doc_response = {};
// loop indices
for (i = 0; i < l; i += 1) {
index = {};
index.reference = priv.indices[i];
index.name = index.reference["name"];
index.current = indices[index.name];
index.current_size = priv.getObjectSize(index.current);
// a lot of loops, not sure this is the fastest way
for (j = 0; j < index.current_size; j += 1) {
for (key in index.current) {
obj = index.current[key];
for (prop in obj) {
for ( k = 0; k < unique_docids.length; k++ ) {
if ( obj[prop] === unique_docids[k] ) {
found = true;
break;
}
}
if (!found) {
unique_docids.push( obj[prop] );
unique_count += 1;
}
}
}
}
}
// construct allDocs response
all_doc_response.total_rows = unique_count;
all_doc_response.rows = [];
for (m = 0; m < unique_count; m += 1) {
// include_docs
if (include_docs) {
that.addJob(
"get",
priv.substorage,
unique_docids[m],
option,
function (content) {
file = { value: {} };
file.id = unique_docids[n];
file.key = unique_docids[n];
file.doc = content;
all_doc_response.rows.push(file);
// async counter, must be in callback
n += 1;
if (n === (unique_count)) {
that.success(all_doc_response);
}
},
function (error) {
that.error({
"status": 404,
"statusText": "Not Found",
"error": "not_found",
"message": "Cannot find the document",
"reason": "Cannot get a document from substorage"
});
}
);
} else {
file = { value: {} };
file.id = unique_docids[m];
file.key = unique_docids[m];
all_doc_response.rows.push(file);
if (m === (unique_count-1)) {
return all_doc_response;
}
}
}
};
/**
* Post document to substorage and create/update index file(s)
* @method post
......@@ -507,7 +582,6 @@ jIO.addStorageType('indexed', function (spec, my) {
f.removeDocument('doc');
},
function (err) {
// xxx do we try to delete the posted document ?
err.message = "Cannot save index file";
that.error(err);
}
......@@ -529,33 +603,60 @@ jIO.addStorageType('indexed', function (spec, my) {
f.getIndices();
};
/**
* Gets a document list.
/**
* Gets a document list from the substorage
* Options:
* - {boolean} include_docs Also retrieve the actual document content.
* @method allDocs
* @param {object} command The JIO command
*/
/*
//{
// "total_rows": 4,
// "rows": [
// {
// "id": "otherdoc",
// "key": "otherdoc",
// "value": {
// "rev": "1-3753476B70A49EA4D8C9039E7B04254C"
// }
// },{...}
// ]
//}
that.allDocs = function (command) {
var obj = localStorage.getItem(storage_file_object_name),
success,
error;
if (obj) {
priv.update();
setTimeout(function () {
that.success(priv.allDocs(obj));
});
} else {
success = function (val) {
priv.setFileArray(val.rows);
that.success(val);
};
error = function (err) {
that.error(err);
};
that.addJob('allDocs', priv.sub_storage_spec, null,
command.cloneOption(), success, error);
var f = {}, indices, option, include_docs, all_docs_response;
option = command.cloneOption();
if (option.max_retry === 0) {
option.max_retry = 3;
}
}; // end allDocs
*/
f.getIndices = function () {
that.addJob(
"get",
priv.substorage,
priv.index_suffix,
option,
function (response) {
if (command.getOption('include_docs')) {
priv.allDocsResponseFromIndex(response, true, option);
} else {
all_docs_response =
priv.allDocsResponseFromIndex(response, false, option);
that.success(all_docs_response);
}
},
function (err) {
that.error({
"status": 404,
"statusText": "Not Found",
"error": "not_found",
"message": "Document index not found",
"reason": "There are no documents in the storage"
});
return;
}
);
};
f.getIndices();
};
return that;
});
test/jiotests.js
View file @ a0aaf798
......@@ -2745,8 +2745,7 @@ test ("Put", function(){
// modify document - modify keyword on index!
o.doc = {"_id": "put1", "title": "myPuttter1", "author":"Jane Doe"};
o.spy (o, "value", {"ok": true, "id": "put1"},
"Modify document, update keyword on index");
o.spy (o, "value", {"ok": true, "id": "put1"}, "Modify existing document");
o.jio.put(o.doc, o.f);
o.tick(o);
......@@ -2996,39 +2995,6 @@ test ("Get", function(){
o.jio.stop();
});
/*
test ('Get document list', function () {
var o = {}; o.clock = this.sandbox.useFakeTimers();
o.clock.tick(base_tick);
o.jio = JIO.newJio({type:'indexed',
storage:{type:'dummyall3tries',
username:'indexgetlist'}});
o.doc1 = {id:'file',key:'file',value:{
_last_modified:15000,_creation_date:10000}};
o.doc2 = {id:'memo',key:'memo',value:{
_last_modified:25000,_creation_date:20000}};
// getting list must take long time with dummyall3tries
o.f = this.spy();
o.jio.allDocs({max_retry:3},o.f);
o.clock.tick(1000);
ok(!o.f.called,'Callback must not be called');
// wail long time too retreive list
o.clock.tick(1000);
// now we can test if the document list is loaded faster
o.f2 = function (err,val) {
deepEqual (err || objectifyDocumentArray(val.rows),
objectifyDocumentArray([o.doc1,o.doc2]),'get document list');
};
this.spy(o,'f2');
o.jio.allDocs({max_retry:3},o.f2);
o.clock.tick(1000)
if (!o.f2.calledOnce) {
ok (false, 'no response / too much results');
}
});
*/
test ("Remove", function(){
// not sure these need to be run, because the index does not change
......@@ -3037,17 +3003,17 @@ test ("Remove", function(){
var o = generateTools(this);
o.jio = JIO.newJio({
"type": "indexed",
"indices": [
{"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]}
],
"sub_storage": {
"type": "local",
"username": "irem",
"application_name": "irem"
}
});
"type": "indexed",
"indices": [
{"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]}
],
"sub_storage": {
"type": "local",
"username": "irem",
"application_name": "irem"
}
});
// remove inexistent document
o.spy(o, "status", 404, "Remove inexistent document");
......@@ -3173,6 +3139,102 @@ test ("Remove", function(){
o.jio.stop();
});
test ("AllDocs", function () {
var o = generateTools(this);
o.jio = JIO.newJio({
"type": "indexed",
"indices": [
{"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]}
],
"sub_storage": {
"type": "local",
"username": "iall",
"application_name": "iall"
}
});
// adding documents
o.all1 = { "_id": "dragon.doc",
"title": "some title", "author": "Dr. No", "year": "1968"
};
o.spy (o, "value", {"ok": true, "id": "dragon.doc"}, "Put 1");
o.jio.put(o.all1, o.f);
o.tick(o);
o.all2 = {"_id": "timemachine",
"title": "hello world", "author": "Dr. Who", "year": "1968"
}
o.spy (o, "value", {"ok": true, "id": "timemachine"}, "Put 2");
o.jio.put(o.all2, o.f);
o.tick(o);
o.all3 = {"_id": "rocket.ppt",
"title": "sunshine.", "author": "Dr. Snuggles", "year": "1985"
}
o.spy (o, "value", {"ok": true, "id": "rocket.ppt"}, "Put 3");
o.jio.put(o.all3, o.f);
o.tick(o);
o.all4 = {"_id": "stick.jpg",
"title": "clouds", "author": "Dr. House", "year": "2005"
}
o.spy (o, "value", {"ok": true, "id": "stick.jpg"}, "Put 4");
o.jio.put(o.all4, o.f);
o.tick(o);
// check index
o.fakeIndex = {
"_id": "iall_indices.json",
"indexA": {
"Dr. No": ["dragon.doc"],
"Dr. Who": ["timemachine"],
"Dr. Snuggles": ["rocket.ppt"],
"Dr. House":["stick.jpg"]
},
"indexAB": {
"Dr. No": ["dragon.doc"],
"Dr. Who": ["timemachine"],
"Dr. Snuggles": ["rocket.ppt"],
"Dr. House":["stick.jpg"],
"1968": ["dragon.doc", "timemachine"],
"1985": ["rocket.ppt"],
"2005":["stick.jpg"]
}
};
o.jio.get("iall_indices.json",function(err, response){
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.tick(o);
o.thisShouldBeTheAnswer = {
"rows": [
{"id": "dragon.doc", "key": "dragon.doc", "value": {} },
{"id": "timemachine", "key": "timemachine", "value": {} },
{"id": "rocket.ppt", "key": "rocket.ppt", "value": {} },
{"id": "stick.jpg", "key": "stick.jpg", "value": {} }
],
"total_rows": 4
}
o.spy(o, "value", o.thisShouldBeTheAnswer, "allDocs (served by index)");
o.jio.allDocs(o.f);
o.tick(o);
o.thisShouldBeTheAnswer2 = {
"rows": [
{"id": "dragon.doc", "key": "dragon.doc", "value": {}, "doc": o.all1 },
{"id": "timemachine", "key": "timemachine", "value": {}, "doc": o.all2 },
{"id": "rocket.ppt", "key": "rocket.ppt", "value": {}, "doc": o.all3 },
{"id": "stick.jpg", "key": "stick.jpg", "value": {}, "doc": o.all4 }
],
"total_rows": 4
}
o.spy(o, "value", o.thisShouldBeTheAnswer2, "allDocs (include_docs)");
o.jio.allDocs({"include_docs":true}, o.f);
o.tick(o);
o.jio.stop();
});
/*
module ('Jio CryptedStorage');
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7