Skip to content

jio
jio
Commit 39feb6e5 authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

replicated opml now check signature and use new parser storage

parent badea138
Hide whitespace changes
Inline Side-by-side
Showing with 996 additions and 1023 deletions
examples/scenario_monitor.html
View file @ 39feb6e5
......@@ -7,6 +7,8 @@
<script src="../node_modules/rsvp/dist/rsvp-2.0.4.js"></script>
<script src="../dist/jio-latest.js"></script>
<script src="../src/jio.storage/replicatedopmltreestorage.js"></script>
<script src="../src/jio.storage/webhttpstorage.js"></script>
<link rel="stylesheet" href="../node_modules/grunt-contrib-qunit/test/libs/qunit.css" type="text/css" media="screen"/>
<script src="../node_modules/grunt-contrib-qunit/test/libs/qunit.js" type="text/javascript"></script>
......
examples/scenario_monitor.js
View file @ 39feb6e5
......@@ -12,6 +12,10 @@
module = QUnit.module,
rusha = new Rusha(),
i,
opml_mock_options,
rss_mock_options,
opml_mock_options2 = {},
rss_mock_options2 = {},
name_list = ['get', 'put', 'remove', 'buildQuery',
'putAttachment', 'getAttachment', 'allAttachments'];
......@@ -24,66 +28,41 @@
}
}
function RSSMockStorage(spec) {
this._rss_storage = jIO.createJIO({
type: "rss",
url: "http://example.com/rss.xml"
function ParserMockStorage(spec) {
this._sub_storage = jIO.createJIO({
type: "parser",
document_id: spec.document_id,
attachment_id: spec.attachment_id,
parser: spec.parser,
sub_storage: spec.sub_storage
});
this._options = spec.options;
resetCount(spec.options.count);
}
RSSMockStorage.prototype.hasCapacity = function (name) {
return this._rss_storage.hasCapacity(name);
};
function WEBMockStorage(spec) {
this._web_storage = jIO.createJIO({
type: "webhttp",
url: "http://example.com/"
});
this._options = spec.options;
resetCount(spec.options.count);
}
WEBMockStorage.prototype.hasCapacity = function (name) {
return this._web_storage.hasCapacity(name);
};
function OPMLMockStorage(spec) {
this._opml_storage = jIO.createJIO({
type: "opml",
url: "http://example.com/opml.xml"
});
this._options = spec.options;
resetCount(spec.options.count);
if (spec.parser === "opml") {
if (spec.document_id === "http://example2.com/opml.xml") {
this._options = opml_mock_options2;
} else {
this._options = opml_mock_options;
}
} else if (spec.parser === "rss") {
if (spec.document_id === "http://example2.com/rss.xml") {
this._options = rss_mock_options2;
} else {
this._options = rss_mock_options;
}
}
resetCount(this._options.count);
}
OPMLMockStorage.prototype.hasCapacity = function (name) {
return this._opml_storage.hasCapacity(name);
ParserMockStorage.prototype.hasCapacity = function (name) {
return this._sub_storage.hasCapacity(name);
};
function mockFunction(name) {
WEBMockStorage.prototype[name] = function () {
ParserMockStorage.prototype[name] = function () {
this._options.count[name] += 1;
if (this._options.mock.hasOwnProperty(name)) {
return this._options.mock[name].apply(this, arguments);
}
return this._web_storage[name].apply(this._web_storage, arguments);
};
RSSMockStorage.prototype[name] = function () {
this._options.count[name] += 1;
if (this._options.mock.hasOwnProperty(name)) {
return this._options.mock[name].apply(this, arguments);
}
return this._rss_storage[name].apply(this._rss_storage, arguments);
};
OPMLMockStorage.prototype[name] = function () {
this._options.count[name] += 1;
if (this._options.mock.hasOwnProperty(name)) {
return this._options.mock[name].apply(this, arguments);
}
return this._opml_storage[name].apply(this._opml_storage, arguments);
return this._sub_storage[name].apply(this._sub_storage, arguments);
};
}
......@@ -91,9 +70,7 @@
mockFunction(name_list[i]);
}
jIO.addStorage('opmlmock', OPMLMockStorage);
jIO.addStorage('rssmock', RSSMockStorage);
jIO.addStorage('webmock', WEBMockStorage);
jIO.addStorage('parsermock', ParserMockStorage);
///////////////////////////////////////////////////////
// Helpers
......@@ -110,9 +87,7 @@
for (i = 0; i < result.data.rows.length; i += 1) {
promise_list.push(RSVP.all([
result.data.rows[i].id,
storage.get(result.data.rows[i].id),
storage.getAttachment(result.data.rows[i].id,
result.data.rows[i].doc.name)
storage.get(result.data.rows[i].id)
]));
}
return RSVP.all(promise_list);
......@@ -146,82 +121,9 @@
this.server.autoRespond = true;
this.server.autoRespondAfter = 5;
this.rss_mock_options = {
mock: {
remove: function () {
throw new Error('remove not supported');
},
removeAttachment: function () {
throw new Error('removeAttachment not supported');
},
allAttachments: function () {
return {data: null};
},
putAttachment: function () {
throw new Error('putAttachment not supported');
}
},
count: {}
};
this.opml_mock_options = {
mock: {
remove: function () {
throw new Error('remove not supported');
},
removeAttachment: function () {
throw new Error('removeAttachment not supported');
},
allAttachments: function () {
return {data: null};
},
putAttachment: function () {
throw new Error('putAttachment not supported');
}
},
count: {}
};
this.web_mock_options = {
mock: {
remove: function () {
throw new Error('remove not supported');
},
removeAttachment: function () {
throw new Error('removeAttachment not supported');
},
allAttachments: function () {
return {data: null};
},
putAttachment: function () {
throw new Error('putAttachment not supported');
}
},
count: {}
};
this.sub_opml_storage = {
type: "opmlmock",
options: this.opml_mock_options,
url: "http://example.com/opml.xml",
sub_storage_list: [
{
type: "rssmock",
url: "http://example.com/rss.xml",
has_include_docs: true,
options: this.rss_mock_options
},
{
type: "webmock",
url: "http://example.com/",
has_include_docs: true,
options: this.web_mock_options
}
],
basic_login: "YWRtaW46endfEzrJUZGw="
};
this.jio = jIO.createJIO({
type: "replicatedopml",
opml_storage_list: [
this.sub_opml_storage
],
remote_parser_storage_type: "parsermock",
local_sub_storage: {
type: "query",
sub_storage: {
......@@ -243,9 +145,28 @@
'</head>' +
'<body>' +
'<outline text="OPML Item List">' +
'<outline text="instance foo" type="link" url="http://example.com/' +
'<outline text="instance foo" xmlUrl="http://example.com/' +
'rss.xml" dateCreated="Thu, 12 Sep 2003 23:35:52 GMT" ' +
'htmlUrl="http://example.com/" title="opml item foo" type="link"/>' +
'</outline>' +
'</body>' +
'</opml>'
]);
this.server.respondWith("GET", "http://example2.com/opml.xml", [200,
{ "Content-Type": "text/xml" },
'<?xml version="1.0" encoding="ISO-8859-1"?>' +
'<opml version="1.0">' +
'<head>' +
'<title>opml foo</title>' +
'<dateCreated>Thu, 12 Sep 2003 23:35:52 GMT</dateCreated>' +
'<dateModified>Fri, 12 Sep 2003 23:45:37 GMT</dateModified>' +
'</head>' +
'<body>' +
'<outline text="OPML Item List">' +
'<outline text="instance foo" xmlUrl="http://example2.com/' +
'rss.xml" dateCreated="Thu, 12 Sep 2003 23:35:52 GMT" ' +
'htmlUrl="http://example.com/" title="opml item foo" />' +
'htmlUrl="http://example2.com/" title="opml item foo" type="link"/>' +
'</outline>' +
'</body>' +
'</opml>'
......@@ -273,21 +194,64 @@
'</rss>'
]);
this.server.respondWith("GET", "http://example.com/_document_list", [200,
{ "Content-Type": "text/plain" },
'monitor.status'
this.server.respondWith("GET", "http://example2.com/rss.xml", [200,
{ "Content-Type": "text/xml" },
'<?xml version="1.0" encoding="UTF-8" ?>' +
'<rss version="2.0">' +
'<channel>' +
'<title>instance foo</title>' +
'<description>This is an example of an RSS feed</description>' +
'<link>http://www.domain.com/link.htm</link>' +
'<lastBuildDate>Mon, 28 Aug 2006 11:12:55 -0400 </lastBuildDate>' +
'<pubDate>Tue, 29 Aug 2006 09:00:00 -0400</pubDate>' +
'<item>' +
'<title>Item Example</title>' +
'<category>ERROR</category>' +
'<description>This is an example of an Item</description>' +
'<link>http://www.domain.com/link.htm</link>' +
'<guid isPermaLink="false">11026875</guid>' +
'<pubDate>Tue, 29 Aug 2006 09:00:00 -0400</pubDate>' +
'</item>' +
'</channel>' +
'</rss>'
]);
this.server.respondWith(
"GET",
"http://example.com/monitor.status.json",
[200,
{ "Content-Type": "application/json" },
'{"title": "document fooo", "status": "ERROR",' +
'"date": "Tue, 29 Aug 2006 09:00:00 -0400",' +
'"type": "global", "foo_p": "fooo parameter",' +
'"bar_p": "bar parameter", "total_error": 12345}']
);
opml_mock_options = {
mock: {
remove: function () {
throw new Error('remove not supported');
},
removeAttachment: function () {
throw new Error('removeAttachment not supported');
},
allAttachments: function () {
return {data: null};
},
putAttachment: function () {
throw new Error('putAttachment not supported');
}
},
count: {}
};
rss_mock_options = {
mock: {
remove: function () {
throw new Error('remove not supported');
},
removeAttachment: function () {
throw new Error('removeAttachment not supported');
},
allAttachments: function () {
return {data: null};
},
putAttachment: function () {
throw new Error('putAttachment not supported');
}
},
count: {}
};
console.log(new Blob([JSON.stringify({toto: ""})]));
},
teardown: function () {
this.server.restore();
......@@ -303,24 +267,19 @@
stop();
var test = this;
test.opml_mock_options.mock.buildQuery = function () {
return [];
};
this.jio.repair()
.then(function () {
return RSVP.all([
isEmptyStorage(test.jio),
deepEqual(test.jio.__storage._remote_storage_dict,
{},
'SubStorage empty'),
equalsubStorageCallCount(
test.opml_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.rss_mock_options.count,
opml_mock_options.count,
{}
),
equalsubStorageCallCount(
test.web_mock_options.count,
rss_mock_options.count,
{}
)
]);
......@@ -333,113 +292,90 @@
});
});
///////////////////////////////////////////////////////
// complete sync - one opml, 2 sub storages
// complete sync - one opml, one sub storages
///////////////////////////////////////////////////////
test("complete storage sync", function () {
expect(4);
expect(3);
stop();
var test = this,
doc_id = "http://example.com/rss.xml",
doc = {
opml_doc = {
title: "opml item foo",
htmlurl: "http://example.com/",
url: "http://example.com/rss.xml",
url: "http://example.com/opml.xml",
portal_type: "opml",
basic_login: "cred foo",
active: true
},
opml_id = generateHash(opml_doc.url),
opml_outline_id = "/1/0/0",
opml_outline = {
opml_title: "opml foo",
dateCreated: "Thu, 12 Sep 2003 23:35:52 GMT",
dateModified: "Fri, 12 Sep 2003 23:45:37 GMT",
text: "instance foo",
type: "link",
opml_title: "opml foo",
created_date: "Thu, 12 Sep 2003 23:35:52 GMT",
modified_date: "Fri, 12 Sep 2003 23:45:37 GMT"
},
parent_id = generateHash(test.sub_opml_storage.url),
opml_item_id = generateHash(parent_id + doc_id),
opml_item = {
name: doc_id,
opml_title: doc.opml_title,
parent_id: parent_id,
reference: generateHash(parent_id + doc_id),
creation_date: doc.created_date,
title: doc.title,
type: "opml-item",
url: test.sub_opml_storage.url,
signature: generateHash(JSON.stringify(doc))
htmlUrl: "http://example.com/",
xmlUrl: "http://example.com/rss.xml",
title: "opml item foo",
portal_type: "opml-outline",
parent_id: opml_id,
parent_url: opml_doc.url,
reference: generateHash(opml_id + opml_outline_id),
active: true
},
full_opml = new Blob([JSON.stringify(doc)]),
rss_id = "1102345",
rss_doc = {
"link": "http://www.domain.com/link.htm",
"date": "Tue, 29 Aug 2006 09:00:00 -0400",
"title": "Item Example",
"category": "ERROR",
"description": "This is an example of an Item",
"guid": "1102345",
"siteTitle": "instance foo",
"reference": "This is an example of an RSS feed",
"siteLink": "http://www.domain.com/link.htm",
"lastBuildDate": "Mon, 28 Aug 2006 11:12:55 -0400 "
promise_id = generateHash(opml_outline.reference + opml_outline.xmlUrl +
"/0/0"),
promise_item = {
link: "http://www.domain.com/link.htm",
title: "Item Example",
category: "ERROR",
description: "This is an example of an Item",
guid: "1102345",
guid_isPermaLink: "false",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
channel: "This is an example of an RSS feed",
channel_item: "instance foo",
parent_id: opml_outline.reference,
reference: promise_id,
status: "ERROR",
portal_type: "promise",
active: true
},
// Sub OPML document (rss)
rss_feed_url = "http://example.com/rss.xml",
rss_item_id = generateHash(opml_item.reference + rss_feed_url + rss_id),
rss_item_id = generateHash(opml_outline.reference + opml_outline.xmlUrl +
"/0"),
rss_item = {
name: rss_id,
opml_title: opml_item.opml_title,
parent_title: opml_item.title,
parent_id: opml_item.reference,
title: "instance foo",
description: "This is an example of an RSS feed",
link: "http://www.domain.com/link.htm",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
parent_id: opml_outline.reference,
reference: rss_item_id,
title: rss_doc.title,
type: rss_doc.type || "rssmock-item",
url: rss_feed_url,
status: rss_doc.category,
creation_date: rss_doc.date,
signature: generateHash(JSON.stringify(rss_doc))
},
full_rss = new Blob([JSON.stringify(rss_doc)]),
json_id = "monitor.status",
json_doc = {
title: "document fooo",
status: "ERROR",
date: "Tue, 29 Aug 2006 09:00:00 -0400",
type: "global",
foo_p: "fooo parameter",
bar_p: "bar parameter",
total_error: 12345
},
// Sub OPML document (webhttp)
http_url = "http://example.com/",
json_item_id = generateHash(opml_item.reference + http_url + json_id),
json_item = {
name: json_id,
opml_title: opml_item.opml_title,
parent_title: opml_item.title,
parent_id: opml_item.reference,
reference: json_item_id,
title: json_doc.title,
type: json_doc.type,
url: http_url,
status: json_doc.status,
creation_date: json_doc.date,
signature: generateHash(JSON.stringify(json_doc))
},
full_json = new Blob([JSON.stringify(json_doc)]);
portal_type: "rss",
active: true
};
test.jio.repair()
test.jio.put(opml_doc.url, opml_doc)
.then(function () {
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[opml_item_id, opml_item, full_opml],
[rss_item_id, rss_item, full_rss],
[json_item_id, json_item, full_json]]),
equalStorage(test.jio, [
[opml_doc.url, opml_doc],
[opml_outline.reference, opml_outline],
[rss_item_id, rss_item],
[promise_id, promise_item]
]),
equalsubStorageCallCount(
test.opml_mock_options.count,
opml_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.rss_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.web_mock_options.count,
rss_mock_options.count,
{buildQuery: 1}
)
]);
......@@ -453,130 +389,125 @@
});
///////////////////////////////////////////////////////
// document update
// remote document update
///////////////////////////////////////////////////////
test("remote document modified", function () {
expect(4);
expect(3);
stop();
var test = this,
doc_id = "http://example.com/rss.xml",
doc = {
opml_doc = {
title: "opml item foo",
htmlurl: "http://example.com/",
url: "http://example.com/rss.xml",
url: "http://example.com/opml.xml",
portal_type: "opml",
basic_login: "cred foo",
active: true
},
opml_id = generateHash(opml_doc.url),
opml_outline_id = "/1/0/0",
opml_outline = {
opml_title: "opml foo",
dateCreated: "Thu, 12 Sep 2003 23:35:52 GMT",
dateModified: "Fri, 12 Sep 2003 23:45:37 GMT",
text: "instance foo",
type: "link",
opml_title: "opml foo",
created_date: "Thu, 12 Sep 2003 23:35:52 GMT",
modified_date: "Fri, 12 Sep 2003 23:45:37 GMT"
},
parent_id = generateHash(test.sub_opml_storage.url),
opml_item_id = generateHash(parent_id + doc_id),
opml_item = {
name: doc_id,
opml_title: doc.opml_title,
parent_id: parent_id,
reference: generateHash(parent_id + doc_id),
creation_date: doc.created_date,
title: doc.title,
type: "opml-item",
url: test.sub_opml_storage.url,
signature: generateHash(JSON.stringify(doc))
},
full_opml = new Blob([JSON.stringify(doc)]),
rss_id = "1102345",
rss_doc = {
"link": "http://www.domain.com/link.htm",
"date": "Tue, 29 Aug 2006 09:00:00 -0400",
"title": "Item Example",
"category": "ERROR",
"description": "This is an example of an Item",
"guid": "1102345",
"siteTitle": "instance foo",
"reference": "This is an example of an RSS feed",
"siteLink": "http://www.domain.com/link.htm",
"lastBuildDate": "Mon, 28 Aug 2006 11:12:55 -0400 "
},
// Sub OPML document (rss)
rss_feed_url = test.sub_opml_storage.sub_storage_list[0].url,
rss_item_id = generateHash(opml_item.reference + rss_feed_url + rss_id),
rss_item2 = {
name: rss_id,
opml_title: opml_item.opml_title,
parent_title: opml_item.title,
parent_id: opml_item.reference,
reference: rss_item_id,
title: rss_doc.title,
type: rss_doc.type || "rssmock-item",
url: rss_feed_url,
status: rss_doc.category,
creation_date: rss_doc.date
htmlUrl: "http://example.com/",
xmlUrl: "http://example.com/rss.xml",
title: "opml item foo",
portal_type: "opml-outline",
parent_id: opml_id,
parent_url: opml_doc.url,
reference: generateHash(opml_id + opml_outline_id),
active: true
},
rss_doc2 = JSON.parse(JSON.stringify(rss_doc)),
full_rss2,
json_id = "monitor.status",
json_doc = {
title: "document fooo",
promise_id = generateHash(opml_outline.reference + opml_outline.xmlUrl +
"/0/0"),
promise_item = {
link: "http://www.domain.com/link.htm",
title: "Item Example",
category: "ERROR",
description: "This is an example of an Item",
guid: "1102345",
guid_isPermaLink: "false",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
channel: "This is an example of an RSS feed",
channel_item: "instance foo",
parent_id: opml_outline.reference,
reference: promise_id,
status: "ERROR",
date: "Tue, 29 Aug 2006 09:00:00 -0400",
type: "global",
foo_p: "fooo parameter",
bar_p: "bar parameter",
total_error: 12345
portal_type: "promise",
active: true
},
// Sub OPML document (webhttp)
http_url = "http://example.com/",
json_item_id = generateHash(opml_item.reference + http_url + json_id),
json_item = {
name: json_id,
opml_title: opml_item.opml_title,
parent_title: opml_item.title,
parent_id: opml_item.reference,
reference: json_item_id,
title: json_doc.title,
type: json_doc.type,
url: http_url,
status: json_doc.status,
creation_date: json_doc.date,
signature: generateHash(JSON.stringify(json_doc))
rss_item_id = generateHash(opml_outline.reference + opml_outline.xmlUrl +
"/0"),
rss_item = {
title: "instance foo",
description: "This is an example of an RSS feed",
link: "http://www.domain.com/link.htm",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
parent_id: opml_outline.reference,
reference: rss_item_id,
portal_type: "rss",
active: true
},
full_json = new Blob([JSON.stringify(json_doc)]);
updated_promise = JSON.parse(JSON.stringify(promise_item));
/* Update rss document */
rss_doc2.date = "new rss date";
// new signature
rss_item2.signature = generateHash(JSON.stringify(rss_doc2));
// modified date
rss_item2.creation_date = rss_doc2.date;
// get the full rss item
full_rss2 = new Blob([JSON.stringify(rss_doc2)]);
test.jio.repair()
test.jio.put(opml_doc.url, opml_doc)
.then(function () {
test.rss_mock_options.mock.buildQuery = function () {
return [{id: rss_id, doc: rss_doc2, value: {}}];
return test.jio.repair();
})
.then(function () {
rss_mock_options.mock.buildQuery = function () {
return [
{
"id": "/0",
"value": {},
"doc": {
"title": "instance foo",
"description": "This is an example of an RSS feed",
"link": "http://www.domain.com/link.htm",
"lastBuildDate": "Mon, 28 Aug 2006 11:12:55 -0400 ",
"pubDate": "Tue, 29 Aug 2006 09:00:00 -0400"
}
},
{
"id": "/0/0",
"value": {},
"doc": {
"link": "http://www.domain.com/link.htm",
"pubDate": "Tue, 29 Aug 2006 10:00:00 -0400", //Changed
"title": "Item Example",
"category": "OK", // changed to OK
"description": "This is an example of an Item",
"guid": "1102345",
"guid_isPermaLink": "false"
}
}
];
};
resetCount(test.opml_mock_options.count);
resetCount(test.rss_mock_options.count);
resetCount(test.web_mock_options.count);
updated_promise.pubDate = "Tue, 29 Aug 2006 10:00:00 -0400";
updated_promise.category = "OK";
updated_promise.status = "OK";
resetCount(opml_mock_options.count);
resetCount(rss_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[opml_item_id, opml_item, full_opml],
[rss_item_id, rss_item2, full_rss2],
[json_item_id, json_item, full_json]]),
equalStorage(test.jio, [
[opml_doc.url, opml_doc],
[opml_outline.reference, opml_outline],
[rss_item_id, rss_item],
[promise_id, updated_promise]
]),
equalsubStorageCallCount(
test.opml_mock_options.count,
opml_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.rss_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.web_mock_options.count,
rss_mock_options.count,
{buildQuery: 1}
)
]);
......@@ -594,127 +525,112 @@
// remote document deleted - non exist in result
///////////////////////////////////////////////////////
test("remote document deleted: empty result", function () {
expect(5);
expect(4);
stop();
var test = this,
doc_id = "http://example.com/rss.xml",
doc = {
opml_doc = {
title: "opml item foo",
htmlurl: "http://example.com/",
url: "http://example.com/rss.xml",
url: "http://example.com/opml.xml",
portal_type: "opml",
basic_login: "cred foo",
active: true
},
opml_id = generateHash(opml_doc.url),
opml_outline_id = "/1/0/0",
opml_outline = {
opml_title: "opml foo",
dateCreated: "Thu, 12 Sep 2003 23:35:52 GMT",
dateModified: "Fri, 12 Sep 2003 23:45:37 GMT",
text: "instance foo",
type: "link",
opml_title: "opml foo",
created_date: "Thu, 12 Sep 2003 23:35:52 GMT",
modified_date: "Fri, 12 Sep 2003 23:45:37 GMT"
},
parent_id = generateHash(test.sub_opml_storage.url),
opml_item_id = generateHash(parent_id + doc_id),
opml_item = {
name: doc_id,
opml_title: doc.opml_title,
parent_id: parent_id,
reference: generateHash(parent_id + doc_id),
creation_date: doc.created_date,
title: doc.title,
type: "opml-item",
url: test.sub_opml_storage.url,
signature: generateHash(JSON.stringify(doc))
htmlUrl: "http://example.com/",
xmlUrl: "http://example.com/rss.xml",
title: "opml item foo",
portal_type: "opml-outline",
parent_id: opml_id,
parent_url: opml_doc.url,
reference: generateHash(opml_id + opml_outline_id),
active: true
},
full_opml = new Blob([JSON.stringify(doc)]),
rss_id = "1102345",
rss_doc = {
"link": "http://www.domain.com/link.htm",
"date": "Tue, 29 Aug 2006 09:00:00 -0400",
"title": "Item Example",
"category": "ERROR",
"description": "This is an example of an Item",
"guid": "1102345",
"siteTitle": "instance foo",
"reference": "This is an example of an RSS feed",
"siteLink": "http://www.domain.com/link.htm",
"lastBuildDate": "Mon, 28 Aug 2006 11:12:55 -0400 "
promise_id = generateHash(opml_outline.reference + opml_outline.xmlUrl +
"/0/0"),
promise_item = {
link: "http://www.domain.com/link.htm",
title: "Item Example",
category: "ERROR",
description: "This is an example of an Item",
guid: "1102345",
guid_isPermaLink: "false",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
channel: "This is an example of an RSS feed",
channel_item: "instance foo",
parent_id: opml_outline.reference,
reference: promise_id,
status: "ERROR",
portal_type: "promise",
active: true
},
// Sub OPML document (rss)
rss_feed_url = "http://example.com/rss.xml",
rss_item_id = generateHash(opml_item.reference + rss_feed_url + rss_id),
rss_item_id = generateHash(opml_outline.reference + opml_outline.xmlUrl +
"/0"),
rss_item = {
name: rss_id,
opml_title: opml_item.opml_title,
parent_title: opml_item.title,
parent_id: opml_item.reference,
title: "instance foo",
description: "This is an example of an RSS feed",
link: "http://www.domain.com/link.htm",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
parent_id: opml_outline.reference,
reference: rss_item_id,
title: rss_doc.title,
type: rss_doc.type || "rssmock-item",
url: rss_feed_url,
status: rss_doc.category,
creation_date: rss_doc.date,
signature: generateHash(JSON.stringify(rss_doc))
},
full_rss = new Blob([JSON.stringify(rss_doc)]),
json_id = "monitor.status",
json_doc = {
title: "document fooo",
status: "ERROR",
date: "Tue, 29 Aug 2006 09:00:00 -0400",
type: "global",
foo_p: "fooo parameter",
bar_p: "bar parameter",
total_error: 12345
},
// Sub OPML document (webhttp)
http_url = "http://example.com/",
json_item_id = generateHash(opml_item.reference + http_url + json_id),
json_item = {
name: json_id,
opml_title: opml_item.opml_title,
parent_title: opml_item.title,
parent_id: opml_item.reference,
reference: json_item_id,
title: json_doc.title,
type: json_doc.type,
url: http_url,
status: json_doc.status,
creation_date: json_doc.date,
signature: generateHash(JSON.stringify(json_doc))
},
full_json = new Blob([JSON.stringify(json_doc)]);
portal_type: "rss",
active: true
};
new RSVP.Queue()
test.jio.put(opml_doc.url, opml_doc)
.then(function () {
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[opml_item_id, opml_item, full_opml],
[rss_item_id, rss_item, full_rss],
[json_item_id, json_item, full_json]])
equalStorage(test.jio, [
[opml_doc.url, opml_doc],
[opml_outline.reference, opml_outline],
[rss_item_id, rss_item],
[promise_id, promise_item]
])
]);
})
.then(function () {
test.rss_mock_options.mock.buildQuery = function () {
return [];
rss_mock_options.mock.buildQuery = function () {
return [{
"id": "/0",
"value": {},
"doc": {
"title": "instance foo",
"description": "This is an example of an RSS feed",
"link": "http://www.domain.com/link.htm",
"lastBuildDate": "Mon, 28 Aug 2006 11:12:55 -0400 ",
"pubDate": "Tue, 29 Aug 2006 09:00:00 -0400"
}
}];
};
resetCount(test.opml_mock_options.count);
resetCount(test.rss_mock_options.count);
resetCount(test.web_mock_options.count);
resetCount(opml_mock_options.count);
resetCount(rss_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[opml_item_id, opml_item, full_opml],
[json_item_id, json_item, full_json]]),
equalStorage(test.jio, [
[opml_doc.url, opml_doc],
[opml_outline.reference, opml_outline],
[rss_item_id, rss_item]
]),
equalsubStorageCallCount(
test.opml_mock_options.count,
opml_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.rss_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.web_mock_options.count,
rss_mock_options.count,
{buildQuery: 1}
)
]);
......@@ -727,336 +643,194 @@
});
});
///////////////////////////////////////////////////////
// some document remove - id has changed
///////////////////////////////////////////////////////
test("remote document removed", function () {
//////////////////////////////////////////////////////////
// complete sync - more than one opml with sub storages
//////////////////////////////////////////////////////////
test("multi opml storage sync", function () {
expect(5);
stop();
var test = this,
doc_id = "http://example.com/rss.xml",
doc = {
opml_doc = {
title: "opml item foo",
htmlurl: "http://example.com/",
url: "http://example.com/rss.xml",
url: "http://example.com/opml.xml",
portal_type: "opml",
basic_login: "cred foo",
active: true
},
opml_id = generateHash(opml_doc.url),
opml_outline_id = "/1/0/0",
opml_outline = {
opml_title: "opml foo",
dateCreated: "Thu, 12 Sep 2003 23:35:52 GMT",
dateModified: "Fri, 12 Sep 2003 23:45:37 GMT",
text: "instance foo",
type: "link",
opml_title: "opml foo",
created_date: "Thu, 12 Sep 2003 23:35:52 GMT",
modified_date: "Fri, 12 Sep 2003 23:45:37 GMT"
},
parent_id = generateHash(test.sub_opml_storage.url),
opml_item_id = generateHash(parent_id + doc_id),
opml_item = {
name: doc_id,
opml_title: doc.opml_title,
parent_id: parent_id,
reference: generateHash(parent_id + doc_id),
creation_date: doc.created_date,
title: doc.title,
type: "opml-item",
url: test.sub_opml_storage.url,
signature: generateHash(JSON.stringify(doc))
htmlUrl: "http://example.com/",
xmlUrl: "http://example.com/rss.xml",
title: "opml item foo",
portal_type: "opml-outline",
parent_id: opml_id,
parent_url: opml_doc.url,
reference: generateHash(opml_id + opml_outline_id),
active: true
},
full_opml = new Blob([JSON.stringify(doc)]),
rss_id = "1102345",
rss_doc = {
"link": "http://www.domain.com/link.htm",
"date": "Tue, 29 Aug 2006 09:00:00 -0400",
"title": "Item Example",
"category": "ERROR",
"description": "This is an example of an Item",
"guid": "1102345",
"siteTitle": "instance foo",
"reference": "This is an example of an RSS feed",
"siteLink": "http://www.domain.com/link.htm",
"lastBuildDate": "Mon, 28 Aug 2006 11:12:55 -0400 "
promise_id = generateHash(opml_outline.reference + opml_outline.xmlUrl +
"/0/0"),
promise_item = {
link: "http://www.domain.com/link.htm",
title: "Item Example",
category: "ERROR",
description: "This is an example of an Item",
guid: "1102345",
guid_isPermaLink: "false",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
channel: "This is an example of an RSS feed",
channel_item: "instance foo",
parent_id: opml_outline.reference,
reference: promise_id,
status: "ERROR",
portal_type: "promise",
active: true
},
// Sub OPML document (rss)
rss_feed_url = "http://example.com/rss.xml",
rss_item_id = generateHash(opml_item.reference + rss_feed_url + rss_id),
rss_item_id = generateHash(opml_outline.reference + opml_outline.xmlUrl +
"/0"),
rss_item = {
name: rss_id,
opml_title: opml_item.opml_title,
parent_title: opml_item.title,
parent_id: opml_item.reference,
title: "instance foo",
description: "This is an example of an RSS feed",
link: "http://www.domain.com/link.htm",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
parent_id: opml_outline.reference,
reference: rss_item_id,
title: rss_doc.title,
type: rss_doc.type || "rssmock-item",
url: rss_feed_url,
status: rss_doc.category,
creation_date: rss_doc.date,
signature: generateHash(JSON.stringify(rss_doc))
portal_type: "rss",
active: true
},
full_rss = new Blob([JSON.stringify(rss_doc)]),
json_id = "monitor.status",
json_doc = {
title: "document fooo",
status: "ERROR",
date: "Tue, 29 Aug 2006 09:00:00 -0400",
type: "global",
foo_p: "fooo parameter",
bar_p: "bar parameter",
total_error: 12345
opml_doc2 = {
title: "opml item bar",
url: "http://example2.com/opml.xml",
portal_type: "opml",
basic_login: "cred bar",
active: true
},
// Sub OPML document (webhttp)
http_url = "http://example.com/",
json_item_id = generateHash(opml_item.reference + http_url + json_id),
json_item = {
name: json_id,
opml_title: opml_item.opml_title,
parent_title: opml_item.title,
parent_id: opml_item.reference,
reference: json_item_id,
title: json_doc.title,
type: json_doc.type,
url: http_url,
status: json_doc.status,
creation_date: json_doc.date,
signature: generateHash(JSON.stringify(json_doc))
},
full_json = new Blob([JSON.stringify(json_doc)]),
/* rss doc 2 with different id */
rss_id2 = "1102345-new",
rss_item2_id = generateHash(opml_item.reference + rss_feed_url + rss_id2),
rss_item2 = JSON.parse(JSON.stringify(rss_item));
rss_item2.name = rss_id2;
rss_item2.reference = rss_item2_id;
test.jio.repair()
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[opml_item_id, opml_item, full_opml],
[rss_item_id, rss_item, full_rss],
[json_item_id, json_item, full_json]])
]);
})
.then(function () {
test.rss_mock_options.mock.buildQuery = function () {
// return a different document
return [{id: rss_id2, doc: rss_doc, value: {}}];
};
resetCount(test.opml_mock_options.count);
resetCount(test.rss_mock_options.count);
resetCount(test.web_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[opml_item_id, opml_item, full_opml],
[json_item_id, json_item, full_json],
[rss_item2_id, rss_item2, full_rss]]),
equalsubStorageCallCount(
test.opml_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.rss_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.web_mock_options.count,
{buildQuery: 1}
)
]);
})
.fail(function (error) {
console.log(error);
ok(false, error);
})
.always(function () {
start();
});
});
///////////////////////////////////////////////////////
// complete sync - 2 opmls, 3 sub storages
///////////////////////////////////////////////////////
test("multi opml storage sync", function () {
expect(6);
stop();
var test = this,
doc_id = "http://example.com/rss.xml",
doc = {
title: "opml item foo",
htmlurl: "http://example.com/",
url: "http://example.com/rss.xml",
opml_id2 = generateHash(opml_doc2.url),
opml_outline2 = {
opml_title: "opml foo",
dateCreated: "Thu, 12 Sep 2003 23:35:52 GMT",
dateModified: "Fri, 12 Sep 2003 23:45:37 GMT",
text: "instance foo",
type: "link",
opml_title: "opml foo",
created_date: "Thu, 12 Sep 2003 23:35:52 GMT",
modified_date: "Fri, 12 Sep 2003 23:45:37 GMT"
},
full_opml = new Blob([JSON.stringify(doc)]),
rss_id = "1102345",
rss_doc = {
"link": "http://www.domain.com/link.htm",
"date": "Tue, 29 Aug 2006 09:00:00 -0400",
"title": "Item Example",
"category": "ERROR",
"description": "This is an example of an Item",
"guid": "1102345",
"siteTitle": "instance foo",
"reference": "This is an example of an RSS feed",
"siteLink": "http://www.domain.com/link.htm",
"lastBuildDate": "Mon, 28 Aug 2006 11:12:55 -0400 "
htmlUrl: "http://example2.com/",
xmlUrl: "http://example2.com/rss.xml",
title: "opml item foo",
portal_type: "opml-outline",
parent_id: opml_id2,
parent_url: opml_doc2.url,
reference: generateHash(opml_id2 + opml_outline_id),
active: true
},
full_rss = new Blob([JSON.stringify(rss_doc)]),
json_id = "monitor.status",
json_doc = {
title: "document fooo",
promise_id2 = generateHash(opml_outline2.reference +
opml_outline2.xmlUrl + "/0/0"),
promise_item2 = {
link: "http://www.domain.com/link.htm",
title: "Item Example",
category: "ERROR",
description: "This is an example of an Item",
guid: "11026875",
guid_isPermaLink: "false",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
channel: "This is an example of an RSS feed",
channel_item: "instance foo",
parent_id: opml_outline2.reference,
reference: promise_id2,
status: "ERROR",
date: "Tue, 29 Aug 2006 09:00:00 -0400",
type: "global",
foo_p: "fooo parameter",
bar_p: "bar parameter",
total_error: 12345
portal_type: "promise",
active: true
},
full_json = new Blob([JSON.stringify(json_doc)]),
item_dict = {},
rss_url = "http://example.com/rss.xml",
rss2_url = "http://example2.com/rss.xml",
http_url = "http://example.com/";
rss_item_id2 = generateHash(opml_outline2.reference +
opml_outline2.xmlUrl + "/0"),
rss_item2 = {
title: "instance foo",
description: "This is an example of an RSS feed",
link: "http://www.domain.com/link.htm",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
parent_id: opml_outline2.reference,
reference: rss_item_id2,
portal_type: "rss",
active: true
};
// update storage with 2 opmls
// opml2 has only rss feed substorage
this.sub_opml_storage2 = {
type: "opmlmock",
options: {
mock: {
opml_mock_options2 = {
mock: {
remove: function () {
throw new Error('remove not supported');
},
count: {}
},
url: "http://example2.com/opml.xml",
sub_storage_list: [
{
type: "rssmock",
url: "http://example2.com/rss.xml",
has_include_docs: true,
options: {
mock: {
},
count: {}
}
removeAttachment: function () {
throw new Error('removeAttachment not supported');
},
allAttachments: function () {
return {data: null};
},
putAttachment: function () {
throw new Error('putAttachment not supported');
}
]
},
count: {}
};
this.jio = jIO.createJIO({
type: "replicatedopml",
opml_storage_list: [
this.sub_opml_storage,
this.sub_opml_storage2
],
local_sub_storage: {
type: "query",
sub_storage: {
type: "uuid",
sub_storage: {
type: "memory"
}
rss_mock_options2 = {
mock: {
remove: function () {
throw new Error('remove not supported');
},
removeAttachment: function () {
throw new Error('removeAttachment not supported');
},
allAttachments: function () {
return {data: null};
},
putAttachment: function () {
throw new Error('putAttachment not supported');
}
}
});
/* Expected item in indexeddb*/
item_dict.opml = {
parent_id: generateHash(test.sub_opml_storage.url),
reference: generateHash(generateHash(test.sub_opml_storage.url) + doc_id),
name: doc_id,
opml_title: doc.opml_title,
creation_date: doc.created_date,
title: doc.title,
type: "opml-item",
url: test.sub_opml_storage.url,
signature: generateHash(JSON.stringify(doc))
};
item_dict.opml2 = {
parent_id: generateHash(test.sub_opml_storage2.url),
reference: generateHash(
generateHash(test.sub_opml_storage2.url) + doc_id
),
name: doc_id,
opml_title: doc.opml_title,
creation_date: doc.created_date,
title: doc.title,
type: "opml-item",
url: test.sub_opml_storage2.url,
signature: generateHash(JSON.stringify(doc))
};
item_dict.rss = {
name: rss_id,
opml_title: item_dict.opml.opml_title,
parent_title: item_dict.opml.title,
parent_id: item_dict.opml.reference,
reference: generateHash(item_dict.opml.reference + rss_url + rss_id),
title: rss_doc.title,
type: rss_doc.type || "rssmock-item",
url: rss_url,
status: rss_doc.category,
creation_date: rss_doc.date,
signature: generateHash(JSON.stringify(rss_doc))
};
item_dict.rss2 = {
name: rss_id,
opml_title: item_dict.opml2.opml_title,
parent_title: item_dict.opml2.title,
parent_id: item_dict.opml2.reference,
reference: generateHash(item_dict.opml2.reference + rss2_url + rss_id),
title: rss_doc.title,
type: "rssmock-item",
url: rss2_url,
status: rss_doc.category,
creation_date: rss_doc.date,
signature: generateHash(JSON.stringify(rss_doc))
};
item_dict.json = {
name: json_id,
opml_title: item_dict.opml.opml_title,
parent_title: item_dict.opml.title,
parent_id: item_dict.opml.reference,
reference: generateHash(item_dict.opml.reference + http_url + json_id),
title: json_doc.title,
type: json_doc.type,
url: http_url,
status: json_doc.status,
creation_date: json_doc.date,
signature: generateHash(JSON.stringify(json_doc))
},
count: {}
};
new RSVP.Queue()
test.jio.put(opml_doc.url, opml_doc)
.then(function () {
return test.jio.put(opml_doc2.url, opml_doc2);
})
.then(function () {
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio,
[[item_dict.opml.reference, item_dict.opml, full_opml],
[item_dict.rss.reference, item_dict.rss, full_rss],
[item_dict.json.reference, item_dict.json, full_json],
[item_dict.opml2.reference, item_dict.opml2, full_opml],
[item_dict.rss2.reference, item_dict.rss2, full_rss]]
),
equalsubStorageCallCount(
test.opml_mock_options.count,
{buildQuery: 1}
),
equalStorage(test.jio, [
[opml_doc.url, opml_doc],
[opml_doc2.url, opml_doc2],
[opml_outline.reference, opml_outline],
[rss_item_id, rss_item],
[promise_id, promise_item],
[opml_outline2.reference, opml_outline2],
[rss_item_id2, rss_item2],
[promise_id2, promise_item2]
]),
equalsubStorageCallCount(
test.rss_mock_options.count,
opml_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.web_mock_options.count,
rss_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.sub_opml_storage2.options.count,
opml_mock_options2.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.sub_opml_storage2.sub_storage_list[0].options.count,
rss_mock_options2.count,
{buildQuery: 1}
)
]);
......
src/jio.storage/replicatedopmltreestorage.js
View file @ 39feb6e5
......@@ -5,7 +5,7 @@
*/
/*jslint nomen: true */
/*global jIO, RSVP, Rusha, Blob, console */
/*global jIO, RSVP, Rusha, console, Blob */
(function (jIO, RSVP, Rusha, Blob, console) {
"use strict";
......@@ -16,19 +16,8 @@
*
* {
* "type": "replicatedopml",
* "opml_storage_list": [
* {
* "type": "opml",
* "url": "http://some.url.com",
* "sub_storage_list": [
* {"type": "rss", "url_attribute": "xmlUrl"},
* {"type": "webdav": "url_path": "/"},
* {"type": "webdav", "url_path": "/data/"},
* {"type": "webhttp": "url_path": "/storage"}
* ],
* "basic_login": "XUSODISOIDOISUJDD=="
* }
* ],
* "remote_storage_unreachable_status": "WARNING",
* "remote_opml_check_time_interval": 86400000,
* local_sub_storage: {
* type: "query",
* sub_storage: {
......@@ -41,7 +30,7 @@
*/
var rusha = new Rusha(),
KNOWN_SUB_STORAGE_LIST = ['rss', 'dav', 'webhttp'];
OPML_ATTACHMENT_NAME = "__opml__";
function generateHash(str) {
return rusha.digestFromString(str);
......@@ -61,48 +50,28 @@
* @constructor
*/
function ReplicatedOPMLStorage(spec) {
var i,
j;
function checkSubStorage(sub_storage_spec) {
if (typeof sub_storage_spec.url !== 'string' &&
typeof sub_storage_spec.url_path !== 'string' &&
typeof sub_storage_spec.url_attribute !== 'string') {
throw new TypeError("one or more OPML sub storage(s) has no 'url' set");
}
if (typeof sub_storage_spec.type !== 'string') {
throw new TypeError(
"one or more OPML sub storage(s) has no attribute 'type' set"
);
}
}
if (typeof spec.opml_storage_list !== 'object') {
throw new TypeError("ReplicatedOPMLStorage 'opml_storage_list' " +
"is not of type object");
if (typeof spec.type !== 'string') {
throw new TypeError(
"ReplicatedOPMLStorage 'type' is not a string"
);
}
if (spec.local_sub_storage === undefined) {
throw new TypeError("ReplicatedOPMLStorage 'local_sub_storage' " +
"is not defined");
}
this._local_sub_storage = jIO.createJIO(spec.local_sub_storage);
this._remote_storage_unreachable_status =
spec.remote_storage_unreachable_status;
this._remote_storage_dict = {};
this._opml_storage_list = spec.opml_storage_list;
for (i = 0; i < this._opml_storage_list.length; i += 1) {
if (typeof this._opml_storage_list[i].url !== 'string') {
throw new TypeError("opml storage 'url' is not of type string");
}
if (this._opml_storage_list[i].sub_storage_list !== undefined) {
for (j = 0; j < this._opml_storage_list[i].sub_storage_list.length;
j += 1) {
checkSubStorage(this._opml_storage_list[i].sub_storage_list[j]);
}
}
/*if (spec.opml_storage_list[i].type !== 'opml') {
throw new TypeError("ReplicatedOPMLStorage 'type' should be 'opml'");
}*/
this._remote_parser_storage_type = spec.remote_parser_storage_type;
if (this._remote_parser_storage_type === undefined) {
this._remote_parser_storage_type = "parser";
}
this._remote_opml_check_time_interval =
spec.remote_opml_check_time_interval;
if (this._remote_opml_check_time_interval === undefined) {
// one day in miliseconds
this._remote_opml_check_time_interval = 86400000;
}
}
......@@ -116,10 +85,15 @@
arguments);
};
/*ReplicatedOPMLStorage.prototype.put = function () {
return this._local_sub_storage.put.apply(this._local_sub_storage,
arguments);
};*/
ReplicatedOPMLStorage.prototype.put = function (id, doc) {
if (!doc.hasOwnProperty('portal_type') || doc.portal_type !== 'opml') {
throw new TypeError("Cannot put object which portal_type is not 'opml'");
}
if (doc.active === undefined) {
doc.active = true;
}
return this._local_sub_storage.put(id, doc);
};
ReplicatedOPMLStorage.prototype.hasCapacity = function (capacity) {
if (capacity === 'include') {
......@@ -134,314 +108,497 @@
arguments);
};
ReplicatedOPMLStorage.prototype.remove = function (id) {
var storage = this._local_sub_storage;
return storage.get(id)
.push(function (doc) {
if (doc.portal_type !== 'opml') {
throw new TypeError("Object with portal_type" + doc.portal_type +
"is frozen, cannot remove it.");
}
function removeOPMLTree(url) {
var remove_id_list = [],
remove_signature_id_list = [];
remove_signature_id_list.push({
id: url,
name: url
});
return storage.allDocs({
select_list: ["xmlUrl", "url"],
query: '(portal_type:"opml-outline") AND (parent_url:"' + url + '")'
})
.push(function (document_result) {
var i,
query_list = [];
for (i = 0; i < document_result.data.total_rows; i += 1) {
query_list.push('(parent_id:"' +
document_result.data.rows[i].id + '")');
remove_id_list.push(document_result.data.rows[i].id);
remove_signature_id_list.push({
id: document_result.data.rows[i].id,
name: document_result.data.rows[i].value.xmlUrl
});
remove_signature_id_list.push({
id: document_result.data.rows[i].id,
name: document_result.data.rows[i].value.url
});
}
// cleanup all sub opml items
if (query_list.length > 0) {
return storage.allDocs({query: query_list.join(" OR ")});
}
return {data: {total_rows: 0}};
})
.push(function (sub_item_result) {
var j,
i,
k,
remove_queue = new RSVP.Queue();
function removeItem(key) {
remove_queue
.push(function () {
return storage.remove(key);
});
}
function removeAttachmentItem(id, name) {
remove_queue
.push(function () {
return storage.removeAttachment(id, name);
})
.push(undefined, function (error) {
if ((error instanceof jIO.util.jIOError) &&
(error.status_code === 404)) {
return undefined;
}
throw error;
});
}
// remove signatures
for (k = 0; k < remove_signature_id_list.length; k += 1) {
removeAttachmentItem(
remove_signature_id_list[k].id,
remove_signature_id_list[k].name
);
}
// remove opml-outline sub-items (rss)
for (j = 0; j < sub_item_result.data.total_rows; j += 1) {
removeItem(sub_item_result.data.rows[j].id);
}
// remove opml-outline
for (i = 0; i < remove_id_list.length; i += 1) {
removeItem(remove_id_list[i]);
}
return remove_queue;
})
.push(function () {
return storage.remove(url);
});
}
return removeOPMLTree(id);
});
};
ReplicatedOPMLStorage.prototype.allAttachments = function () {
return this._local_sub_storage.allAttachments.apply(this._local_sub_storage,
arguments);
};
function getSubOpmlStorageDescription(storage_spec, opml_doc, basic_login) {
if (storage_spec.basic_login === undefined && basic_login !== undefined) {
storage_spec.basic_login = basic_login;
}
if (KNOWN_SUB_STORAGE_LIST.indexOf(storage_spec.type) !== -1) {
if (storage_spec.url_attribute !== undefined &&
opml_doc.hasOwnProperty(storage_spec.url_attribute)) {
storage_spec.url = opml_doc[storage_spec.url_attribute];
} else if (storage_spec.url_path !== undefined) {
storage_spec.url_path = storage_spec.url_path.replace(
new RegExp("^[/]+"),
""
);
storage_spec.url = opml_doc.url.replace(
new RegExp("[/]+$"),
""
) + "/" + storage_spec.url_path;
}
// XXX - for compatibility, remove url with jio_private path
storage_spec.url = storage_spec.url.replace("jio_private", "private");
if (storage_spec.type === "dav") {
storage_spec = {
type: "query",
sub_storage: {
type: "drivetojiomapping",
sub_storage: storage_spec
}
};
}
}
return storage_spec;
}
function getStorageUrl(storage_spec) {
var spec = storage_spec;
while (spec !== undefined) {
if (spec.url !== undefined) {
return spec.url;
}
if (spec.document_id !== undefined) {
return spec.document_id;
}
spec = spec.sub_storage;
}
throw new Error("No url found on sub storage: " +
JSON.stringify(storage_spec));
}
function loadSubStorage(context, spec, parent_id, opml_doc, basic_login) {
function getDocumentAsAttachment(context, attachment_id, name) {
return context._local_sub_storage.getAttachment(attachment_id, name)
.push(undefined, function (error) {
if ((error instanceof jIO.util.jIOError) &&
(error.status_code === 404)) {
return undefined;
}
console.error(error);
})
.push(function (attachment) {
if (attachment) {
return jIO.util.readBlobAsText(attachment)
.then(function (evt) {
return JSON.parse(evt.target.result);
});
}
return {};
});
}
function updateSubStorageStatus(context, signature_dict, next_status) {
var key,
update_status_queue = new RSVP.Queue();
function updateStatus(id) {
update_status_queue
.push(function () {
return context._local_sub_storage.get(id);
})
.push(function (doc) {
if (doc.portal_type === "promise") {
doc.category = next_status;
return context._local_sub_storage.put(id, doc);
}
if (doc.status !== undefined) {
doc.status = next_status;
return context._local_sub_storage.put(id, doc);
}
});
}
for (key in signature_dict) {
if (signature_dict.hasOwnProperty(key)) {
if (signature_dict[key].status !== next_status) {
updateStatus(key);
signature_dict[key].status = next_status;
}
}
}
return update_status_queue
.push(function () {
return signature_dict;
});
}
function loadSubStorage(context, storage_spec, parent_id, type) {
var sub_storage,
storage_spec,
options = {},
result_dict,
storage_key,
url;
if (spec.has_include_docs === true) {
options = {include_docs: true};
}
try {
storage_spec = getSubOpmlStorageDescription(
spec,
opml_doc,
basic_login
);
url = getStorageUrl(storage_spec);
storage_key = generateHash(parent_id + url);
sub_storage = createStorage(context, storage_spec, storage_key);
} catch (error) {
console.log(error);
throw error;
}
url = getStorageUrl(storage_spec);
storage_key = generateHash(parent_id + url);
sub_storage = createStorage(context, storage_spec, storage_key);
result_dict = {
parent_id: parent_id,
parent_title: opml_doc.title,
opml_title: opml_doc.opml_title,
type: storage_spec.type,
current_doc: {},
result: {
data: {
total_rows: 0
}
},
type: type || storage_spec.type,
current_signature: {},
result: {data: {total_rows: 0}},
url: url
};
return sub_storage.allDocs(options)
return sub_storage.allDocs({include_docs: true})
.push(undefined, function (error) {
if ((error instanceof jIO.util.jIOError) &&
(error.status_code === 404)) {
console.log(error);
return undefined;
}
//throw error;
// throw will cancel all others allDocs, this is not wanted
console.log(error);
console.error(error);
return undefined;
})
.push(function (result) {
if (result === undefined) {
return {data: {total_rows: 0}};
if (context._remote_storage_unreachable_status !== undefined) {
// update status of local documents
// and set unreachable status
return getDocumentAsAttachment(context, parent_id, url)
.push(function (signature_document) {
return updateSubStorageStatus(
context,
signature_document,
context._remote_storage_unreachable_status
);
})
.push(function (signature_dict) {
return signature_dict;
});
}
return {};
}
result_dict.result = result;
return context._local_sub_storage.allDocs({
select_list: ["signature"],
query: '(parent_id: "' + parent_id + '") AND (url:"' + url + '")'
});
return getDocumentAsAttachment(
context,
parent_id,
url
);
})
.push(function (all_document) {
var i;
for (i = 0; i < all_document.data.total_rows; i += 1) {
result_dict.current_doc[
all_document.data.rows[i].id
] = all_document.data.rows[i].value;
}
.push(function (signature_document) {
result_dict.current_signature = signature_document;
return result_dict;
});
}
function getOpmlTree(context, opml_url, opml_spec) {
function getOpmlTree(context, opml_url, opml_spec, basic_login) {
var opml_storage,
opml_document_list = [],
document_attachment_dict = {},
delete_key_list = [],
current_opml_dict = {},
attachment_document_list = [],
opml_result_list,
current_signature_dict = {},
fetch_remote_opml = false,
id;
id = generateHash(opml_url);
opml_storage = createStorage(context, opml_spec, id);
return opml_storage.allDocs({include_docs: true})
.push(undefined, function (error) {
if ((error instanceof jIO.util.jIOError) &&
(error.status_code === 404)) {
return {data: {total_rows: 0}};
return getDocumentAsAttachment(context, opml_url, OPML_ATTACHMENT_NAME)
.push(function (opml_doc) {
var current_time = new Date().getTime();
if (opml_doc.expire_time !== undefined) {
fetch_remote_opml = (opml_doc.expire_time - current_time) < 0;
} else {
fetch_remote_opml = true;
}
//throw error;
// throw will cancel all remaning tasks
return {data: {total_rows: 0}};
})
.push(function (opml_result) {
return RSVP.all([
opml_result,
context._local_sub_storage.allDocs({
select_list: ["signature"],
query: '(parent_id: "' + id + '")'
})
]);
})
.push(function (result_list) {
var i;
for (i = 0; i < result_list[1].data.total_rows; i += 1) {
current_opml_dict[
result_list[1].data.rows[i].id
] = result_list[1].data.rows[i].value;
if (fetch_remote_opml) {
return opml_storage.allDocs({include_docs: true})
.push(undefined, function (error) {
if ((error instanceof jIO.util.jIOError) &&
(error.status_code === 404)) {
return {data: {total_rows: 0}};
}
//throw error;
console.error(error);
return {data: {total_rows: 0}};
})
.push(function (opml_result) {
opml_result_list = opml_result;
if (opml_result.data.total_rows > 0) {
attachment_document_list.push({
id: opml_url,
name: OPML_ATTACHMENT_NAME,
doc: {
expire_time: new Date().getTime() +
context._remote_opml_check_time_interval,
data: opml_result
}
});
return getDocumentAsAttachment(
context,
id,
opml_url
);
}
return {};
})
.push(function (signature_dict) {
current_signature_dict = signature_dict;
});
}
return result_list[0];
opml_result_list = opml_doc.data;
})
.push(function (opml_result_list) {
var j,
i,
.push(function () {
var i,
item,
signature,
doc_signature_dict = {},
skip_add = false,
id_hash,
result_list = [];
result_list = [],
header_dict = {};
if (opml_result_list.data.total_rows > 0 && fetch_remote_opml) {
header_dict = {
dateCreated: opml_result_list.data.rows[0].doc.dateCreated,
dateModified: opml_result_list.data.rows[0].doc.dateModified,
opml_title: opml_result_list.data.rows[0].doc.title
};
}
for (i = 0; i < opml_result_list.data.total_rows; i += 1) {
for (i = 1; i < opml_result_list.data.total_rows; i += 1) {
item = opml_result_list.data.rows[i];
id_hash = generateHash(id + item.id);
signature = generateHash(JSON.stringify(item.doc));
for (j = 0; j < opml_spec.sub_storage_list.length; j += 1) {
if (item.doc.xmlUrl !== undefined) {
id_hash = generateHash(id + item.id);
result_list.push(loadSubStorage(
context,
opml_spec.sub_storage_list[j],
{
type: context._remote_parser_storage_type,
document_id: item.doc.xmlUrl,
attachment_id: 'enclosure',
parser: 'rss',
sub_storage: {
type: "http"
}
},
id_hash,
item.doc,
opml_spec.basic_login
'promise'
));
}
if (current_opml_dict.hasOwnProperty(id_hash)) {
if (current_opml_dict[id_hash].signature === signature) {
// the document was not modified, delete and skip add
delete current_opml_dict[id_hash];
skip_add = true;
// Load private docs
if (item.doc.url !== undefined) {
result_list.push(loadSubStorage(
context,
{
type: 'webhttp',
url: item.doc.url.replace('jio_private', 'private'),
basic_login: basic_login
},
id_hash
));
}
delete current_opml_dict[id_hash];
}
if (!skip_add) {
opml_document_list.push({
id: id_hash,
doc: {
type: "opml-item",
name: item.id,
reference: id_hash,
parent_id: id,
creation_date: item.doc.created_date,
url: opml_url,
title: item.doc.title,
parent_title: undefined,
opml_title: item.doc.opml_title,
status: undefined,
if (fetch_remote_opml) {
// Append this document signature to the list
signature = generateHash(JSON.stringify(item.doc));
doc_signature_dict[id_hash] = {
signature: signature
};
if (current_signature_dict.hasOwnProperty(id_hash)) {
if (current_signature_dict[id_hash].signature === signature) {
// remote document was not modified, delete and skip add
delete current_signature_dict[id_hash];
skip_add = true;
}
delete current_signature_dict[id_hash];
}
});
document_attachment_dict[id_hash] = {
name: item.id,
doc: item.doc
};
Object.assign(item.doc, {
portal_type: "opml-outline",
parent_id: id,
parent_url: opml_url,
reference: id_hash,
active: true
});
Object.assign(item.doc, header_dict);
if (!skip_add) {
opml_document_list.push({
id: id_hash,
doc: item.doc
});
}
}
}
}
delete_key_list.push.apply(delete_key_list,
Object.keys(current_opml_dict));
if (fetch_remote_opml && Object.keys(doc_signature_dict).length > 0) {
attachment_document_list.push({
id: opml_url,
name: opml_url,
doc: doc_signature_dict
});
delete_key_list.push.apply(delete_key_list,
Object.keys(current_signature_dict));
}
return RSVP.all(result_list);
})
.push(function (result_list) {
var i,
j;
j,
start,
extra_dict,
item_signature_dict = {};
function applyItemToTree(item, item_result) {
function applyItemToTree(item, item_result, portal_type, extra_dict) {
var id_hash,
element,
signature;
element = item.doc,
signature,
item_id = item.guid || item.id,
status = (element.status || element.category);
id_hash = generateHash(item_result.parent_id +
item_result.url + item.id);
if (item.doc !== undefined) {
element = item.doc;
} else {
element = item.value;
}
item_result.url + item_id);
if (extra_dict !== undefined) {
Object.assign(element, extra_dict);
}
// Generating document signature
signature = generateHash(JSON.stringify(element));
item_signature_dict[id_hash] = {
signature: signature,
status: status
};
if (item_result.current_doc.hasOwnProperty(id_hash)) {
if (item_result.current_doc[id_hash].signature === signature) {
// the document was not modified delete and return
delete item_result.current_doc[id_hash];
if (item_result.current_signature.hasOwnProperty(id_hash)) {
if (item_result.current_signature[id_hash].signature
=== signature) {
// the document was not modified return
delete item_result.current_signature[id_hash];
return;
}
// the document exists and has changed
delete item_result.current_doc[id_hash];
delete item_result.current_signature[id_hash];
}
Object.assign(element, {
parent_id: item_result.parent_id,
portal_type: portal_type || element.type || item_result.type,
status: status,
reference: id_hash,
active: true
});
opml_document_list.push({
id: id_hash,
doc: {
parent_id: item_result.parent_id,
name: item.id,
type: (element.type || item_result.type + "-item"),
reference: id_hash,
creation_date: element.date || element["start-date"],
url: item_result.url,
status: (element.status || element.category),
title: (element.source || element.title),
parent_title: item_result.parent_title,
opml_title: item_result.opml_title,
signature: signature
}
});
document_attachment_dict[id_hash] = {
name: item.id,
doc: element
};
});
}
for (i = 0; i < result_list.length; i += 1) {
for (j = 0; j < result_list[i].result.data.total_rows; j += 1) {
applyItemToTree(
result_list[i].result.data.rows[j],
result_list[i]
extra_dict = undefined;
start = 0;
if (result_list[i].result.data.total_rows > 0) {
if (result_list[i].type === "promise") {
// the first element of rss is the header
extra_dict = {
lastBuildDate: result_list[i].result.data.rows[0].doc
.lastBuildDate,
channel: result_list[i].result.data.rows[0].doc.description,
channel_item: result_list[i].result.data.rows[0].doc.title
};
applyItemToTree(
result_list[i].result.data.rows[0],
result_list[i],
"rss"
);
start = 1;
}
for (j = start; j < result_list[i].result.data.total_rows; j += 1) {
applyItemToTree(
result_list[i].result.data.rows[j],
result_list[i],
undefined,
extra_dict
);
}
attachment_document_list.push({
id: result_list[i].parent_id,
name: result_list[i].url,
doc: item_signature_dict
});
item_signature_dict = {};
delete_key_list.push.apply(
delete_key_list,
Object.keys(result_list[i].current_signature)
);
} else if (Object.keys(result_list[i].current_signature).length > 0) {
// if the remote data is empty and current_signature is not empty,
// push to storage in case the status was changed
// this help for speed optimisation
attachment_document_list.push({
id: result_list[i].parent_id,
name: result_list[i].url,
doc: result_list[i].current_signature
});
}
delete_key_list.push.apply(delete_key_list,
Object.keys(result_list[i].current_doc));
}
return [opml_document_list, document_attachment_dict, delete_key_list];
return [opml_document_list, delete_key_list, attachment_document_list];
});
}
function pushDocumentToStorage(context, document_list, attachment_dict,
delete_key_list) {
function pushDocumentToStorage(context, document_list, delete_key_list,
attachment_document_list) {
var document_queue = new RSVP.Queue(),
i;
function pushDocument(id, element, attachment) {
function pushDocument(id, element) {
document_queue
.push(function () {
return context._local_sub_storage.put(id, element);
})
.push(function () {
return context._local_sub_storage.putAttachment(
id,
attachment.name,
new Blob([JSON.stringify(attachment.doc)])
);
});
}
for (i = 0; i < document_list.length; i += 1) {
pushDocument(
document_list[i].id,
document_list[i].doc,
attachment_dict[document_list[i].id]
document_list[i].doc
);
}
return document_queue
......@@ -452,18 +609,6 @@
// remove all document which were not updated
function removeDocument(key) {
remove_queue
.push(function () {
return context._local_sub_storage.get(key);
})
.push(undefined, function (error) {
throw error;
})
.push(function (element) {
return context._local_sub_storage.removeAttachment(
key,
element.name
);
})
.push(function () {
return context._local_sub_storage.remove(key);
})
......@@ -480,32 +625,79 @@
removeDocument(delete_key_list[k]);
}
return remove_queue;
})
.push(function () {
var j,
signature_queue = new RSVP.Queue();
function pushAttachment(id, name, element) {
signature_queue
.push(function () {
return context._local_sub_storage.putAttachment(
id,
name,
new Blob([JSON.stringify(element)], {type : 'application/json'})
);
})
.push(undefined, function (error) {
console.error(error);
});
}
for (j = 0; j < attachment_document_list.length; j += 1) {
pushAttachment(
attachment_document_list[j].id,
attachment_document_list[j].name,
attachment_document_list[j].doc
);
}
});
}
function syncOpmlStorage(context) {
var i,
opml_queue = new RSVP.Queue();
return context._local_sub_storage.allDocs({
query: '(portal_type:"opml") AND (active:true)',
select_list: ["url", "basic_login"]
})
.push(function (storage_result) {
var i,
opml_queue = new RSVP.Queue();
function syncFullOpml(url, storage_spec) {
return opml_queue
.push(function () {
return getOpmlTree(context, url, storage_spec);
})
.push(function (result_list) {
return pushDocumentToStorage(
context,
result_list[0],
result_list[1],
result_list[2]
);
});
}
for (i = 0; i < context._opml_storage_list.length; i += 1) {
syncFullOpml(context._opml_storage_list[i].url,
context._opml_storage_list[i]);
}
return opml_queue;
function syncFullOpml(storage_spec) {
opml_queue
.push(function () {
return getOpmlTree(
context,
storage_spec.url,
{
type: context._remote_parser_storage_type,
document_id: storage_spec.url,
attachment_id: 'enclosure',
parser: 'opml',
sub_storage: {
type: "http"
}
},
storage_spec.basic_login
);
})
.push(function (result_list) {
return pushDocumentToStorage(
context,
result_list[0],
result_list[1],
result_list[2]
);
});
}
for (i = 0; i < storage_result.data.total_rows; i += 1) {
syncFullOpml({
type: storage_result.data.rows[i].value.type,
url: storage_result.data.rows[i].value.url,
basic_login: storage_result.data.rows[i].value.basic_login
});
}
return opml_queue;
});
}
ReplicatedOPMLStorage.prototype.repair = function () {
......
src/jio.storage/webhttpstorage.js
View file @ 39feb6e5
......@@ -39,7 +39,7 @@
throw new jIO.util.jIOError("id " + id + " is forbidden (no begin /)",
400);
}
if (id.lastIndexOf("/") !== (id.length - 1)) {
if (id.lastIndexOf("/") === (id.length - 1)) {
throw new jIO.util.jIOError("id " + id + " is forbidden (no end /)",
400);
}
......@@ -84,14 +84,19 @@
}
WEBHTTPStorage.prototype.get = function (id) {
var context = this,
element;
var context = this;
id = restrictDocumentId(id);
element = getJsonDocument(context, id);
if (element !== undefined) {
return element.doc;
}
return getJsonDocument(context, id)
.push(function (element) {
return element.doc;
}, function (error) {
if ((error.target !== undefined) &&
(error.target.status === 404)) {
throw new jIO.util.jIOError("Cannot find document", 404);
}
throw error;
});
};
WEBHTTPStorage.prototype.hasCapacity = function (capacity) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7