Skip to content

jio_mebibou
jio_mebibou
Commit 3d97883d authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

replicated opml now check signature and use new parser storage

parent 134f4eda
Show whitespace changes
Inline Side-by-side
Showing with 997 additions and 1028 deletions
Gruntfile.js
View file @ 3d97883d
......@@ -182,11 +182,7 @@ module.exports = function (grunt) {
'src/jio.storage/localstorage.js',
'src/jio.storage/indexeddbstorage.js',
'src/jio.storage/cryptstorage.js',
'src/jio.storage/websqlstorage.js',
'src/jio.storage/rssfeedstorage.js',
'src/jio.storage/opmlstorage.js',
'src/jio.storage/replicatedopmltreestorage.js',
'src/jio.storage/webhttpstorage.js'
'src/jio.storage/websqlstorage.js'
],
dest: 'dist/<%= pkg.name %>-<%= pkg.version %>.js'
// dest: 'jio.js'
......
examples/scenario_monitor.html
View file @ 3d97883d
......@@ -7,6 +7,8 @@
<script src="../node_modules/rsvp/dist/rsvp-2.0.4.js"></script>
<script src="../dist/jio-latest.js"></script>
<script src="../src/jio.storage/replicatedopmltreestorage.js"></script>
<script src="../src/jio.storage/webhttpstorage.js"></script>
<link rel="stylesheet" href="../node_modules/grunt-contrib-qunit/test/libs/qunit.css" type="text/css" media="screen"/>
<script src="../node_modules/grunt-contrib-qunit/test/libs/qunit.js" type="text/javascript"></script>
......
examples/scenario_monitor.js
View file @ 3d97883d
......@@ -12,6 +12,10 @@
module = QUnit.module,
rusha = new Rusha(),
i,
opml_mock_options,
rss_mock_options,
opml_mock_options2 = {},
rss_mock_options2 = {},
name_list = ['get', 'put', 'remove', 'buildQuery',
'putAttachment', 'getAttachment', 'allAttachments'];
......@@ -24,66 +28,41 @@
}
}
function RSSMockStorage(spec) {
this._rss_storage = jIO.createJIO({
type: "rss",
url: "http://example.com/rss.xml"
function ParserMockStorage(spec) {
this._sub_storage = jIO.createJIO({
type: "parser",
document_id: spec.document_id,
attachment_id: spec.attachment_id,
parser: spec.parser,
sub_storage: spec.sub_storage
});
this._options = spec.options;
resetCount(spec.options.count);
if (spec.parser === "opml") {
if (spec.document_id === "http://example2.com/opml.xml") {
this._options = opml_mock_options2;
} else {
this._options = opml_mock_options;
}
RSSMockStorage.prototype.hasCapacity = function (name) {
return this._rss_storage.hasCapacity(name);
};
function WEBMockStorage(spec) {
this._web_storage = jIO.createJIO({
type: "webhttp",
url: "http://example.com/"
});
this._options = spec.options;
resetCount(spec.options.count);
} else if (spec.parser === "rss") {
if (spec.document_id === "http://example2.com/rss.xml") {
this._options = rss_mock_options2;
} else {
this._options = rss_mock_options;
}
WEBMockStorage.prototype.hasCapacity = function (name) {
return this._web_storage.hasCapacity(name);
};
function OPMLMockStorage(spec) {
this._opml_storage = jIO.createJIO({
type: "opml",
url: "http://example.com/opml.xml"
});
this._options = spec.options;
resetCount(spec.options.count);
}
resetCount(this._options.count);
}
OPMLMockStorage.prototype.hasCapacity = function (name) {
return this._opml_storage.hasCapacity(name);
ParserMockStorage.prototype.hasCapacity = function (name) {
return this._sub_storage.hasCapacity(name);
};
function mockFunction(name) {
WEBMockStorage.prototype[name] = function () {
this._options.count[name] += 1;
if (this._options.mock.hasOwnProperty(name)) {
return this._options.mock[name].apply(this, arguments);
}
return this._web_storage[name].apply(this._web_storage, arguments);
};
RSSMockStorage.prototype[name] = function () {
ParserMockStorage.prototype[name] = function () {
this._options.count[name] += 1;
if (this._options.mock.hasOwnProperty(name)) {
return this._options.mock[name].apply(this, arguments);
}
return this._rss_storage[name].apply(this._rss_storage, arguments);
};
OPMLMockStorage.prototype[name] = function () {
this._options.count[name] += 1;
if (this._options.mock.hasOwnProperty(name)) {
return this._options.mock[name].apply(this, arguments);
}
return this._opml_storage[name].apply(this._opml_storage, arguments);
return this._sub_storage[name].apply(this._sub_storage, arguments);
};
}
......@@ -91,9 +70,7 @@
mockFunction(name_list[i]);
}
jIO.addStorage('opmlmock', OPMLMockStorage);
jIO.addStorage('rssmock', RSSMockStorage);
jIO.addStorage('webmock', WEBMockStorage);
jIO.addStorage('parsermock', ParserMockStorage);
///////////////////////////////////////////////////////
// Helpers
......@@ -110,9 +87,7 @@
for (i = 0; i < result.data.rows.length; i += 1) {
promise_list.push(RSVP.all([
result.data.rows[i].id,
storage.get(result.data.rows[i].id),
storage.getAttachment(result.data.rows[i].id,
result.data.rows[i].doc.name)
storage.get(result.data.rows[i].id)
]));
}
return RSVP.all(promise_list);
......@@ -146,82 +121,9 @@
this.server.autoRespond = true;
this.server.autoRespondAfter = 5;
this.rss_mock_options = {
mock: {
remove: function () {
throw new Error('remove not supported');
},
removeAttachment: function () {
throw new Error('removeAttachment not supported');
},
allAttachments: function () {
return {data: null};
},
putAttachment: function () {
throw new Error('putAttachment not supported');
}
},
count: {}
};
this.opml_mock_options = {
mock: {
remove: function () {
throw new Error('remove not supported');
},
removeAttachment: function () {
throw new Error('removeAttachment not supported');
},
allAttachments: function () {
return {data: null};
},
putAttachment: function () {
throw new Error('putAttachment not supported');
}
},
count: {}
};
this.web_mock_options = {
mock: {
remove: function () {
throw new Error('remove not supported');
},
removeAttachment: function () {
throw new Error('removeAttachment not supported');
},
allAttachments: function () {
return {data: null};
},
putAttachment: function () {
throw new Error('putAttachment not supported');
}
},
count: {}
};
this.sub_opml_storage = {
type: "opmlmock",
options: this.opml_mock_options,
url: "http://example.com/opml.xml",
sub_storage_list: [
{
type: "rssmock",
url: "http://example.com/rss.xml",
has_include_docs: true,
options: this.rss_mock_options
},
{
type: "webmock",
url: "http://example.com/",
has_include_docs: true,
options: this.web_mock_options
}
],
basic_login: "YWRtaW46endfEzrJUZGw="
};
this.jio = jIO.createJIO({
type: "replicatedopml",
opml_storage_list: [
this.sub_opml_storage
],
remote_parser_storage_type: "parsermock",
local_sub_storage: {
type: "query",
sub_storage: {
......@@ -243,9 +145,28 @@
'</head>' +
'<body>' +
'<outline text="OPML Item List">' +
'<outline text="instance foo" type="link" url="http://example.com/' +
'<outline text="instance foo" xmlUrl="http://example.com/' +
'rss.xml" dateCreated="Thu, 12 Sep 2003 23:35:52 GMT" ' +
'htmlUrl="http://example.com/" title="opml item foo" />' +
'htmlUrl="http://example.com/" title="opml item foo" type="link"/>' +
'</outline>' +
'</body>' +
'</opml>'
]);
this.server.respondWith("GET", "http://example2.com/opml.xml", [200,
{ "Content-Type": "text/xml" },
'<?xml version="1.0" encoding="ISO-8859-1"?>' +
'<opml version="1.0">' +
'<head>' +
'<title>opml foo</title>' +
'<dateCreated>Thu, 12 Sep 2003 23:35:52 GMT</dateCreated>' +
'<dateModified>Fri, 12 Sep 2003 23:45:37 GMT</dateModified>' +
'</head>' +
'<body>' +
'<outline text="OPML Item List">' +
'<outline text="instance foo" xmlUrl="http://example2.com/' +
'rss.xml" dateCreated="Thu, 12 Sep 2003 23:35:52 GMT" ' +
'htmlUrl="http://example2.com/" title="opml item foo" type="link"/>' +
'</outline>' +
'</body>' +
'</opml>'
......@@ -273,21 +194,64 @@
'</rss>'
]);
this.server.respondWith("GET", "http://example.com/_document_list", [200,
{ "Content-Type": "text/plain" },
'monitor.status'
this.server.respondWith("GET", "http://example2.com/rss.xml", [200,
{ "Content-Type": "text/xml" },
'<?xml version="1.0" encoding="UTF-8" ?>' +
'<rss version="2.0">' +
'<channel>' +
'<title>instance foo</title>' +
'<description>This is an example of an RSS feed</description>' +
'<link>http://www.domain.com/link.htm</link>' +
'<lastBuildDate>Mon, 28 Aug 2006 11:12:55 -0400 </lastBuildDate>' +
'<pubDate>Tue, 29 Aug 2006 09:00:00 -0400</pubDate>' +
'<item>' +
'<title>Item Example</title>' +
'<category>ERROR</category>' +
'<description>This is an example of an Item</description>' +
'<link>http://www.domain.com/link.htm</link>' +
'<guid isPermaLink="false">11026875</guid>' +
'<pubDate>Tue, 29 Aug 2006 09:00:00 -0400</pubDate>' +
'</item>' +
'</channel>' +
'</rss>'
]);
this.server.respondWith(
"GET",
"http://example.com/monitor.status.json",
[200,
{ "Content-Type": "application/json" },
'{"title": "document fooo", "status": "ERROR",' +
'"date": "Tue, 29 Aug 2006 09:00:00 -0400",' +
'"type": "global", "foo_p": "fooo parameter",' +
'"bar_p": "bar parameter", "total_error": 12345}']
);
opml_mock_options = {
mock: {
remove: function () {
throw new Error('remove not supported');
},
removeAttachment: function () {
throw new Error('removeAttachment not supported');
},
allAttachments: function () {
return {data: null};
},
putAttachment: function () {
throw new Error('putAttachment not supported');
}
},
count: {}
};
rss_mock_options = {
mock: {
remove: function () {
throw new Error('remove not supported');
},
removeAttachment: function () {
throw new Error('removeAttachment not supported');
},
allAttachments: function () {
return {data: null};
},
putAttachment: function () {
throw new Error('putAttachment not supported');
}
},
count: {}
};
console.log(new Blob([JSON.stringify({toto: ""})]));
},
teardown: function () {
this.server.restore();
......@@ -303,24 +267,19 @@
stop();
var test = this;
test.opml_mock_options.mock.buildQuery = function () {
return [];
};
this.jio.repair()
.then(function () {
return RSVP.all([
isEmptyStorage(test.jio),
deepEqual(test.jio.__storage._remote_storage_dict,
{},
'SubStorage empty'),
equalsubStorageCallCount(
test.opml_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.rss_mock_options.count,
opml_mock_options.count,
{}
),
equalsubStorageCallCount(
test.web_mock_options.count,
rss_mock_options.count,
{}
)
]);
......@@ -333,113 +292,90 @@
});
});
///////////////////////////////////////////////////////
// complete sync - one opml, 2 sub storages
// complete sync - one opml, one sub storages
///////////////////////////////////////////////////////
test("complete storage sync", function () {
expect(4);
expect(3);
stop();
var test = this,
doc_id = "http://example.com/rss.xml",
doc = {
opml_doc = {
title: "opml item foo",
htmlurl: "http://example.com/",
url: "http://example.com/rss.xml",
url: "http://example.com/opml.xml",
portal_type: "opml",
basic_login: "cred foo",
active: true
},
opml_id = generateHash(opml_doc.url),
opml_outline_id = "/1/0/0",
opml_outline = {
opml_title: "opml foo",
dateCreated: "Thu, 12 Sep 2003 23:35:52 GMT",
dateModified: "Fri, 12 Sep 2003 23:45:37 GMT",
text: "instance foo",
type: "link",
opml_title: "opml foo",
created_date: "Thu, 12 Sep 2003 23:35:52 GMT",
modified_date: "Fri, 12 Sep 2003 23:45:37 GMT"
},
parent_id = generateHash(test.sub_opml_storage.url),
opml_item_id = generateHash(parent_id + doc_id),
opml_item = {
name: doc_id,
opml_title: doc.opml_title,
parent_id: parent_id,
reference: generateHash(parent_id + doc_id),
creation_date: doc.created_date,
title: doc.title,
type: "opml-item",
url: test.sub_opml_storage.url,
signature: generateHash(JSON.stringify(doc))
},
full_opml = new Blob([JSON.stringify(doc)]),
rss_id = "1102345",
rss_doc = {
"link": "http://www.domain.com/link.htm",
"date": "Tue, 29 Aug 2006 09:00:00 -0400",
"title": "Item Example",
"category": "ERROR",
"description": "This is an example of an Item",
"guid": "1102345",
"siteTitle": "instance foo",
"reference": "This is an example of an RSS feed",
"siteLink": "http://www.domain.com/link.htm",
"lastBuildDate": "Mon, 28 Aug 2006 11:12:55 -0400 "
htmlUrl: "http://example.com/",
xmlUrl: "http://example.com/rss.xml",
title: "opml item foo",
portal_type: "opml-outline",
parent_id: opml_id,
parent_url: opml_doc.url,
reference: generateHash(opml_id + opml_outline_id),
active: true
},
promise_id = generateHash(opml_outline.reference + opml_outline.xmlUrl +
"/0/0"),
promise_item = {
link: "http://www.domain.com/link.htm",
title: "Item Example",
category: "ERROR",
description: "This is an example of an Item",
guid: "1102345",
guid_isPermaLink: "false",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
channel: "This is an example of an RSS feed",
channel_item: "instance foo",
parent_id: opml_outline.reference,
reference: promise_id,
status: "ERROR",
portal_type: "promise",
active: true
},
// Sub OPML document (rss)
rss_feed_url = "http://example.com/rss.xml",
rss_item_id = generateHash(opml_item.reference + rss_feed_url + rss_id),
rss_item_id = generateHash(opml_outline.reference + opml_outline.xmlUrl +
"/0"),
rss_item = {
name: rss_id,
opml_title: opml_item.opml_title,
parent_title: opml_item.title,
parent_id: opml_item.reference,
title: "instance foo",
description: "This is an example of an RSS feed",
link: "http://www.domain.com/link.htm",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
parent_id: opml_outline.reference,
reference: rss_item_id,
title: rss_doc.title,
type: rss_doc.type || "rssmock-item",
url: rss_feed_url,
status: rss_doc.category,
creation_date: rss_doc.date,
signature: generateHash(JSON.stringify(rss_doc))
},
full_rss = new Blob([JSON.stringify(rss_doc)]),
json_id = "monitor.status",
json_doc = {
title: "document fooo",
status: "ERROR",
date: "Tue, 29 Aug 2006 09:00:00 -0400",
type: "global",
foo_p: "fooo parameter",
bar_p: "bar parameter",
total_error: 12345
},
// Sub OPML document (webhttp)
http_url = "http://example.com/",
json_item_id = generateHash(opml_item.reference + http_url + json_id),
json_item = {
name: json_id,
opml_title: opml_item.opml_title,
parent_title: opml_item.title,
parent_id: opml_item.reference,
reference: json_item_id,
title: json_doc.title,
type: json_doc.type,
url: http_url,
status: json_doc.status,
creation_date: json_doc.date,
signature: generateHash(JSON.stringify(json_doc))
},
full_json = new Blob([JSON.stringify(json_doc)]);
portal_type: "rss",
active: true
};
test.jio.repair()
test.jio.put(opml_doc.url, opml_doc)
.then(function () {
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[opml_item_id, opml_item, full_opml],
[rss_item_id, rss_item, full_rss],
[json_item_id, json_item, full_json]]),
equalsubStorageCallCount(
test.opml_mock_options.count,
{buildQuery: 1}
),
equalStorage(test.jio, [
[opml_doc.url, opml_doc],
[opml_outline.reference, opml_outline],
[rss_item_id, rss_item],
[promise_id, promise_item]
]),
equalsubStorageCallCount(
test.rss_mock_options.count,
opml_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.web_mock_options.count,
rss_mock_options.count,
{buildQuery: 1}
)
]);
......@@ -453,130 +389,125 @@
});
///////////////////////////////////////////////////////
// document update
// remote document update
///////////////////////////////////////////////////////
test("remote document modified", function () {
expect(4);
expect(3);
stop();
var test = this,
doc_id = "http://example.com/rss.xml",
doc = {
opml_doc = {
title: "opml item foo",
htmlurl: "http://example.com/",
url: "http://example.com/rss.xml",
url: "http://example.com/opml.xml",
portal_type: "opml",
basic_login: "cred foo",
active: true
},
opml_id = generateHash(opml_doc.url),
opml_outline_id = "/1/0/0",
opml_outline = {
opml_title: "opml foo",
dateCreated: "Thu, 12 Sep 2003 23:35:52 GMT",
dateModified: "Fri, 12 Sep 2003 23:45:37 GMT",
text: "instance foo",
type: "link",
opml_title: "opml foo",
created_date: "Thu, 12 Sep 2003 23:35:52 GMT",
modified_date: "Fri, 12 Sep 2003 23:45:37 GMT"
htmlUrl: "http://example.com/",
xmlUrl: "http://example.com/rss.xml",
title: "opml item foo",
portal_type: "opml-outline",
parent_id: opml_id,
parent_url: opml_doc.url,
reference: generateHash(opml_id + opml_outline_id),
active: true
},
promise_id = generateHash(opml_outline.reference + opml_outline.xmlUrl +
"/0/0"),
promise_item = {
link: "http://www.domain.com/link.htm",
title: "Item Example",
category: "ERROR",
description: "This is an example of an Item",
guid: "1102345",
guid_isPermaLink: "false",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
channel: "This is an example of an RSS feed",
channel_item: "instance foo",
parent_id: opml_outline.reference,
reference: promise_id,
status: "ERROR",
portal_type: "promise",
active: true
},
rss_item_id = generateHash(opml_outline.reference + opml_outline.xmlUrl +
"/0"),
rss_item = {
title: "instance foo",
description: "This is an example of an RSS feed",
link: "http://www.domain.com/link.htm",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
parent_id: opml_outline.reference,
reference: rss_item_id,
portal_type: "rss",
active: true
},
parent_id = generateHash(test.sub_opml_storage.url),
opml_item_id = generateHash(parent_id + doc_id),
opml_item = {
name: doc_id,
opml_title: doc.opml_title,
parent_id: parent_id,
reference: generateHash(parent_id + doc_id),
creation_date: doc.created_date,
title: doc.title,
type: "opml-item",
url: test.sub_opml_storage.url,
signature: generateHash(JSON.stringify(doc))
updated_promise = JSON.parse(JSON.stringify(promise_item));
test.jio.put(opml_doc.url, opml_doc)
.then(function () {
return test.jio.repair();
})
.then(function () {
rss_mock_options.mock.buildQuery = function () {
return [
{
"id": "/0",
"value": {},
"doc": {
"title": "instance foo",
"description": "This is an example of an RSS feed",
"link": "http://www.domain.com/link.htm",
"lastBuildDate": "Mon, 28 Aug 2006 11:12:55 -0400 ",
"pubDate": "Tue, 29 Aug 2006 09:00:00 -0400"
}
},
full_opml = new Blob([JSON.stringify(doc)]),
rss_id = "1102345",
rss_doc = {
{
"id": "/0/0",
"value": {},
"doc": {
"link": "http://www.domain.com/link.htm",
"date": "Tue, 29 Aug 2006 09:00:00 -0400",
"pubDate": "Tue, 29 Aug 2006 10:00:00 -0400", //Changed
"title": "Item Example",
"category": "ERROR",
"category": "OK", // changed to OK
"description": "This is an example of an Item",
"guid": "1102345",
"siteTitle": "instance foo",
"reference": "This is an example of an RSS feed",
"siteLink": "http://www.domain.com/link.htm",
"lastBuildDate": "Mon, 28 Aug 2006 11:12:55 -0400 "
},
// Sub OPML document (rss)
rss_feed_url = test.sub_opml_storage.sub_storage_list[0].url,
rss_item_id = generateHash(opml_item.reference + rss_feed_url + rss_id),
rss_item2 = {
name: rss_id,
opml_title: opml_item.opml_title,
parent_title: opml_item.title,
parent_id: opml_item.reference,
reference: rss_item_id,
title: rss_doc.title,
type: rss_doc.type || "rssmock-item",
url: rss_feed_url,
status: rss_doc.category,
creation_date: rss_doc.date
},
rss_doc2 = JSON.parse(JSON.stringify(rss_doc)),
full_rss2,
json_id = "monitor.status",
json_doc = {
title: "document fooo",
status: "ERROR",
date: "Tue, 29 Aug 2006 09:00:00 -0400",
type: "global",
foo_p: "fooo parameter",
bar_p: "bar parameter",
total_error: 12345
},
// Sub OPML document (webhttp)
http_url = "http://example.com/",
json_item_id = generateHash(opml_item.reference + http_url + json_id),
json_item = {
name: json_id,
opml_title: opml_item.opml_title,
parent_title: opml_item.title,
parent_id: opml_item.reference,
reference: json_item_id,
title: json_doc.title,
type: json_doc.type,
url: http_url,
status: json_doc.status,
creation_date: json_doc.date,
signature: generateHash(JSON.stringify(json_doc))
},
full_json = new Blob([JSON.stringify(json_doc)]);
/* Update rss document */
rss_doc2.date = "new rss date";
// new signature
rss_item2.signature = generateHash(JSON.stringify(rss_doc2));
// modified date
rss_item2.creation_date = rss_doc2.date;
// get the full rss item
full_rss2 = new Blob([JSON.stringify(rss_doc2)]);
test.jio.repair()
.then(function () {
test.rss_mock_options.mock.buildQuery = function () {
return [{id: rss_id, doc: rss_doc2, value: {}}];
"guid_isPermaLink": "false"
}
}
];
};
resetCount(test.opml_mock_options.count);
resetCount(test.rss_mock_options.count);
resetCount(test.web_mock_options.count);
updated_promise.pubDate = "Tue, 29 Aug 2006 10:00:00 -0400";
updated_promise.category = "OK";
updated_promise.status = "OK";
resetCount(opml_mock_options.count);
resetCount(rss_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[opml_item_id, opml_item, full_opml],
[rss_item_id, rss_item2, full_rss2],
[json_item_id, json_item, full_json]]),
equalStorage(test.jio, [
[opml_doc.url, opml_doc],
[opml_outline.reference, opml_outline],
[rss_item_id, rss_item],
[promise_id, updated_promise]
]),
equalsubStorageCallCount(
test.opml_mock_options.count,
opml_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.rss_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.web_mock_options.count,
rss_mock_options.count,
{buildQuery: 1}
)
]);
......@@ -594,127 +525,112 @@
// remote document deleted - non exist in result
///////////////////////////////////////////////////////
test("remote document deleted: empty result", function () {
expect(5);
expect(4);
stop();
var test = this,
doc_id = "http://example.com/rss.xml",
doc = {
opml_doc = {
title: "opml item foo",
htmlurl: "http://example.com/",
url: "http://example.com/rss.xml",
url: "http://example.com/opml.xml",
portal_type: "opml",
basic_login: "cred foo",
active: true
},
opml_id = generateHash(opml_doc.url),
opml_outline_id = "/1/0/0",
opml_outline = {
opml_title: "opml foo",
dateCreated: "Thu, 12 Sep 2003 23:35:52 GMT",
dateModified: "Fri, 12 Sep 2003 23:45:37 GMT",
text: "instance foo",
type: "link",
opml_title: "opml foo",
created_date: "Thu, 12 Sep 2003 23:35:52 GMT",
modified_date: "Fri, 12 Sep 2003 23:45:37 GMT"
},
parent_id = generateHash(test.sub_opml_storage.url),
opml_item_id = generateHash(parent_id + doc_id),
opml_item = {
name: doc_id,
opml_title: doc.opml_title,
parent_id: parent_id,
reference: generateHash(parent_id + doc_id),
creation_date: doc.created_date,
title: doc.title,
type: "opml-item",
url: test.sub_opml_storage.url,
signature: generateHash(JSON.stringify(doc))
},
full_opml = new Blob([JSON.stringify(doc)]),
rss_id = "1102345",
rss_doc = {
"link": "http://www.domain.com/link.htm",
"date": "Tue, 29 Aug 2006 09:00:00 -0400",
"title": "Item Example",
"category": "ERROR",
"description": "This is an example of an Item",
"guid": "1102345",
"siteTitle": "instance foo",
"reference": "This is an example of an RSS feed",
"siteLink": "http://www.domain.com/link.htm",
"lastBuildDate": "Mon, 28 Aug 2006 11:12:55 -0400 "
htmlUrl: "http://example.com/",
xmlUrl: "http://example.com/rss.xml",
title: "opml item foo",
portal_type: "opml-outline",
parent_id: opml_id,
parent_url: opml_doc.url,
reference: generateHash(opml_id + opml_outline_id),
active: true
},
promise_id = generateHash(opml_outline.reference + opml_outline.xmlUrl +
"/0/0"),
promise_item = {
link: "http://www.domain.com/link.htm",
title: "Item Example",
category: "ERROR",
description: "This is an example of an Item",
guid: "1102345",
guid_isPermaLink: "false",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
channel: "This is an example of an RSS feed",
channel_item: "instance foo",
parent_id: opml_outline.reference,
reference: promise_id,
status: "ERROR",
portal_type: "promise",
active: true
},
// Sub OPML document (rss)
rss_feed_url = "http://example.com/rss.xml",
rss_item_id = generateHash(opml_item.reference + rss_feed_url + rss_id),
rss_item_id = generateHash(opml_outline.reference + opml_outline.xmlUrl +
"/0"),
rss_item = {
name: rss_id,
opml_title: opml_item.opml_title,
parent_title: opml_item.title,
parent_id: opml_item.reference,
title: "instance foo",
description: "This is an example of an RSS feed",
link: "http://www.domain.com/link.htm",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
parent_id: opml_outline.reference,
reference: rss_item_id,
title: rss_doc.title,
type: rss_doc.type || "rssmock-item",
url: rss_feed_url,
status: rss_doc.category,
creation_date: rss_doc.date,
signature: generateHash(JSON.stringify(rss_doc))
},
full_rss = new Blob([JSON.stringify(rss_doc)]),
json_id = "monitor.status",
json_doc = {
title: "document fooo",
status: "ERROR",
date: "Tue, 29 Aug 2006 09:00:00 -0400",
type: "global",
foo_p: "fooo parameter",
bar_p: "bar parameter",
total_error: 12345
},
// Sub OPML document (webhttp)
http_url = "http://example.com/",
json_item_id = generateHash(opml_item.reference + http_url + json_id),
json_item = {
name: json_id,
opml_title: opml_item.opml_title,
parent_title: opml_item.title,
parent_id: opml_item.reference,
reference: json_item_id,
title: json_doc.title,
type: json_doc.type,
url: http_url,
status: json_doc.status,
creation_date: json_doc.date,
signature: generateHash(JSON.stringify(json_doc))
},
full_json = new Blob([JSON.stringify(json_doc)]);
portal_type: "rss",
active: true
};
new RSVP.Queue()
test.jio.put(opml_doc.url, opml_doc)
.then(function () {
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[opml_item_id, opml_item, full_opml],
[rss_item_id, rss_item, full_rss],
[json_item_id, json_item, full_json]])
equalStorage(test.jio, [
[opml_doc.url, opml_doc],
[opml_outline.reference, opml_outline],
[rss_item_id, rss_item],
[promise_id, promise_item]
])
]);
})
.then(function () {
test.rss_mock_options.mock.buildQuery = function () {
return [];
rss_mock_options.mock.buildQuery = function () {
return [{
"id": "/0",
"value": {},
"doc": {
"title": "instance foo",
"description": "This is an example of an RSS feed",
"link": "http://www.domain.com/link.htm",
"lastBuildDate": "Mon, 28 Aug 2006 11:12:55 -0400 ",
"pubDate": "Tue, 29 Aug 2006 09:00:00 -0400"
}
}];
};
resetCount(test.opml_mock_options.count);
resetCount(test.rss_mock_options.count);
resetCount(test.web_mock_options.count);
resetCount(opml_mock_options.count);
resetCount(rss_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[opml_item_id, opml_item, full_opml],
[json_item_id, json_item, full_json]]),
equalStorage(test.jio, [
[opml_doc.url, opml_doc],
[opml_outline.reference, opml_outline],
[rss_item_id, rss_item]
]),
equalsubStorageCallCount(
test.opml_mock_options.count,
opml_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.rss_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.web_mock_options.count,
rss_mock_options.count,
{buildQuery: 1}
)
]);
......@@ -727,336 +643,194 @@
});
});
///////////////////////////////////////////////////////
// some document remove - id has changed
///////////////////////////////////////////////////////
test("remote document removed", function () {
//////////////////////////////////////////////////////////
// complete sync - more than one opml with sub storages
//////////////////////////////////////////////////////////
test("multi opml storage sync", function () {
expect(5);
stop();
var test = this,
doc_id = "http://example.com/rss.xml",
doc = {
opml_doc = {
title: "opml item foo",
htmlurl: "http://example.com/",
url: "http://example.com/rss.xml",
url: "http://example.com/opml.xml",
portal_type: "opml",
basic_login: "cred foo",
active: true
},
opml_id = generateHash(opml_doc.url),
opml_outline_id = "/1/0/0",
opml_outline = {
opml_title: "opml foo",
dateCreated: "Thu, 12 Sep 2003 23:35:52 GMT",
dateModified: "Fri, 12 Sep 2003 23:45:37 GMT",
text: "instance foo",
type: "link",
opml_title: "opml foo",
created_date: "Thu, 12 Sep 2003 23:35:52 GMT",
modified_date: "Fri, 12 Sep 2003 23:45:37 GMT"
},
parent_id = generateHash(test.sub_opml_storage.url),
opml_item_id = generateHash(parent_id + doc_id),
opml_item = {
name: doc_id,
opml_title: doc.opml_title,
parent_id: parent_id,
reference: generateHash(parent_id + doc_id),
creation_date: doc.created_date,
title: doc.title,
type: "opml-item",
url: test.sub_opml_storage.url,
signature: generateHash(JSON.stringify(doc))
},
full_opml = new Blob([JSON.stringify(doc)]),
rss_id = "1102345",
rss_doc = {
"link": "http://www.domain.com/link.htm",
"date": "Tue, 29 Aug 2006 09:00:00 -0400",
"title": "Item Example",
"category": "ERROR",
"description": "This is an example of an Item",
"guid": "1102345",
"siteTitle": "instance foo",
"reference": "This is an example of an RSS feed",
"siteLink": "http://www.domain.com/link.htm",
"lastBuildDate": "Mon, 28 Aug 2006 11:12:55 -0400 "
htmlUrl: "http://example.com/",
xmlUrl: "http://example.com/rss.xml",
title: "opml item foo",
portal_type: "opml-outline",
parent_id: opml_id,
parent_url: opml_doc.url,
reference: generateHash(opml_id + opml_outline_id),
active: true
},
promise_id = generateHash(opml_outline.reference + opml_outline.xmlUrl +
"/0/0"),
promise_item = {
link: "http://www.domain.com/link.htm",
title: "Item Example",
category: "ERROR",
description: "This is an example of an Item",
guid: "1102345",
guid_isPermaLink: "false",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
channel: "This is an example of an RSS feed",
channel_item: "instance foo",
parent_id: opml_outline.reference,
reference: promise_id,
status: "ERROR",
portal_type: "promise",
active: true
},
// Sub OPML document (rss)
rss_feed_url = "http://example.com/rss.xml",
rss_item_id = generateHash(opml_item.reference + rss_feed_url + rss_id),
rss_item_id = generateHash(opml_outline.reference + opml_outline.xmlUrl +
"/0"),
rss_item = {
name: rss_id,
opml_title: opml_item.opml_title,
parent_title: opml_item.title,
parent_id: opml_item.reference,
title: "instance foo",
description: "This is an example of an RSS feed",
link: "http://www.domain.com/link.htm",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
parent_id: opml_outline.reference,
reference: rss_item_id,
title: rss_doc.title,
type: rss_doc.type || "rssmock-item",
url: rss_feed_url,
status: rss_doc.category,
creation_date: rss_doc.date,
signature: generateHash(JSON.stringify(rss_doc))
},
full_rss = new Blob([JSON.stringify(rss_doc)]),
json_id = "monitor.status",
json_doc = {
title: "document fooo",
status: "ERROR",
date: "Tue, 29 Aug 2006 09:00:00 -0400",
type: "global",
foo_p: "fooo parameter",
bar_p: "bar parameter",
total_error: 12345
portal_type: "rss",
active: true
},
// Sub OPML document (webhttp)
http_url = "http://example.com/",
json_item_id = generateHash(opml_item.reference + http_url + json_id),
json_item = {
name: json_id,
opml_title: opml_item.opml_title,
parent_title: opml_item.title,
parent_id: opml_item.reference,
reference: json_item_id,
title: json_doc.title,
type: json_doc.type,
url: http_url,
status: json_doc.status,
creation_date: json_doc.date,
signature: generateHash(JSON.stringify(json_doc))
opml_doc2 = {
title: "opml item bar",
url: "http://example2.com/opml.xml",
portal_type: "opml",
basic_login: "cred bar",
active: true
},
full_json = new Blob([JSON.stringify(json_doc)]),
/* rss doc 2 with different id */
rss_id2 = "1102345-new",
rss_item2_id = generateHash(opml_item.reference + rss_feed_url + rss_id2),
rss_item2 = JSON.parse(JSON.stringify(rss_item));
rss_item2.name = rss_id2;
rss_item2.reference = rss_item2_id;
test.jio.repair()
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[opml_item_id, opml_item, full_opml],
[rss_item_id, rss_item, full_rss],
[json_item_id, json_item, full_json]])
]);
})
.then(function () {
test.rss_mock_options.mock.buildQuery = function () {
// return a different document
return [{id: rss_id2, doc: rss_doc, value: {}}];
};
resetCount(test.opml_mock_options.count);
resetCount(test.rss_mock_options.count);
resetCount(test.web_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[opml_item_id, opml_item, full_opml],
[json_item_id, json_item, full_json],
[rss_item2_id, rss_item2, full_rss]]),
equalsubStorageCallCount(
test.opml_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.rss_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.web_mock_options.count,
{buildQuery: 1}
)
]);
})
.fail(function (error) {
console.log(error);
ok(false, error);
})
.always(function () {
start();
});
});
///////////////////////////////////////////////////////
// complete sync - 2 opmls, 3 sub storages
///////////////////////////////////////////////////////
test("multi opml storage sync", function () {
expect(6);
stop();
var test = this,
doc_id = "http://example.com/rss.xml",
doc = {
title: "opml item foo",
htmlurl: "http://example.com/",
url: "http://example.com/rss.xml",
opml_id2 = generateHash(opml_doc2.url),
opml_outline2 = {
opml_title: "opml foo",
dateCreated: "Thu, 12 Sep 2003 23:35:52 GMT",
dateModified: "Fri, 12 Sep 2003 23:45:37 GMT",
text: "instance foo",
type: "link",
opml_title: "opml foo",
created_date: "Thu, 12 Sep 2003 23:35:52 GMT",
modified_date: "Fri, 12 Sep 2003 23:45:37 GMT"
},
full_opml = new Blob([JSON.stringify(doc)]),
rss_id = "1102345",
rss_doc = {
"link": "http://www.domain.com/link.htm",
"date": "Tue, 29 Aug 2006 09:00:00 -0400",
"title": "Item Example",
"category": "ERROR",
"description": "This is an example of an Item",
"guid": "1102345",
"siteTitle": "instance foo",
"reference": "This is an example of an RSS feed",
"siteLink": "http://www.domain.com/link.htm",
"lastBuildDate": "Mon, 28 Aug 2006 11:12:55 -0400 "
},
full_rss = new Blob([JSON.stringify(rss_doc)]),
json_id = "monitor.status",
json_doc = {
title: "document fooo",
htmlUrl: "http://example2.com/",
xmlUrl: "http://example2.com/rss.xml",
title: "opml item foo",
portal_type: "opml-outline",
parent_id: opml_id2,
parent_url: opml_doc2.url,
reference: generateHash(opml_id2 + opml_outline_id),
active: true
},
promise_id2 = generateHash(opml_outline2.reference +
opml_outline2.xmlUrl + "/0/0"),
promise_item2 = {
link: "http://www.domain.com/link.htm",
title: "Item Example",
category: "ERROR",
description: "This is an example of an Item",
guid: "11026875",
guid_isPermaLink: "false",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
channel: "This is an example of an RSS feed",
channel_item: "instance foo",
parent_id: opml_outline2.reference,
reference: promise_id2,
status: "ERROR",
date: "Tue, 29 Aug 2006 09:00:00 -0400",
type: "global",
foo_p: "fooo parameter",
bar_p: "bar parameter",
total_error: 12345
portal_type: "promise",
active: true
},
full_json = new Blob([JSON.stringify(json_doc)]),
item_dict = {},
rss_url = "http://example.com/rss.xml",
rss2_url = "http://example2.com/rss.xml",
http_url = "http://example.com/";
rss_item_id2 = generateHash(opml_outline2.reference +
opml_outline2.xmlUrl + "/0"),
rss_item2 = {
title: "instance foo",
description: "This is an example of an RSS feed",
link: "http://www.domain.com/link.htm",
lastBuildDate: "Mon, 28 Aug 2006 11:12:55 -0400 ",
pubDate: "Tue, 29 Aug 2006 09:00:00 -0400",
parent_id: opml_outline2.reference,
reference: rss_item_id2,
portal_type: "rss",
active: true
};
// update storage with 2 opmls
// opml2 has only rss feed substorage
this.sub_opml_storage2 = {
type: "opmlmock",
options: {
opml_mock_options2 = {
mock: {
remove: function () {
throw new Error('remove not supported');
},
count: {}
removeAttachment: function () {
throw new Error('removeAttachment not supported');
},
url: "http://example2.com/opml.xml",
sub_storage_list: [
{
type: "rssmock",
url: "http://example2.com/rss.xml",
has_include_docs: true,
options: {
mock: {
allAttachments: function () {
return {data: null};
},
count: {}
}
putAttachment: function () {
throw new Error('putAttachment not supported');
}
]
},
count: {}
};
this.jio = jIO.createJIO({
type: "replicatedopml",
opml_storage_list: [
this.sub_opml_storage,
this.sub_opml_storage2
],
local_sub_storage: {
type: "query",
sub_storage: {
type: "uuid",
sub_storage: {
type: "memory"
}
}
rss_mock_options2 = {
mock: {
remove: function () {
throw new Error('remove not supported');
},
removeAttachment: function () {
throw new Error('removeAttachment not supported');
},
allAttachments: function () {
return {data: null};
},
putAttachment: function () {
throw new Error('putAttachment not supported');
}
});
/* Expected item in indexeddb*/
item_dict.opml = {
parent_id: generateHash(test.sub_opml_storage.url),
reference: generateHash(generateHash(test.sub_opml_storage.url) + doc_id),
name: doc_id,
opml_title: doc.opml_title,
creation_date: doc.created_date,
title: doc.title,
type: "opml-item",
url: test.sub_opml_storage.url,
signature: generateHash(JSON.stringify(doc))
};
item_dict.opml2 = {
parent_id: generateHash(test.sub_opml_storage2.url),
reference: generateHash(
generateHash(test.sub_opml_storage2.url) + doc_id
),
name: doc_id,
opml_title: doc.opml_title,
creation_date: doc.created_date,
title: doc.title,
type: "opml-item",
url: test.sub_opml_storage2.url,
signature: generateHash(JSON.stringify(doc))
};
item_dict.rss = {
name: rss_id,
opml_title: item_dict.opml.opml_title,
parent_title: item_dict.opml.title,
parent_id: item_dict.opml.reference,
reference: generateHash(item_dict.opml.reference + rss_url + rss_id),
title: rss_doc.title,
type: rss_doc.type || "rssmock-item",
url: rss_url,
status: rss_doc.category,
creation_date: rss_doc.date,
signature: generateHash(JSON.stringify(rss_doc))
};
item_dict.rss2 = {
name: rss_id,
opml_title: item_dict.opml2.opml_title,
parent_title: item_dict.opml2.title,
parent_id: item_dict.opml2.reference,
reference: generateHash(item_dict.opml2.reference + rss2_url + rss_id),
title: rss_doc.title,
type: "rssmock-item",
url: rss2_url,
status: rss_doc.category,
creation_date: rss_doc.date,
signature: generateHash(JSON.stringify(rss_doc))
};
item_dict.json = {
name: json_id,
opml_title: item_dict.opml.opml_title,
parent_title: item_dict.opml.title,
parent_id: item_dict.opml.reference,
reference: generateHash(item_dict.opml.reference + http_url + json_id),
title: json_doc.title,
type: json_doc.type,
url: http_url,
status: json_doc.status,
creation_date: json_doc.date,
signature: generateHash(JSON.stringify(json_doc))
},
count: {}
};
new RSVP.Queue()
test.jio.put(opml_doc.url, opml_doc)
.then(function () {
return test.jio.put(opml_doc2.url, opml_doc2);
})
.then(function () {
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio,
[[item_dict.opml.reference, item_dict.opml, full_opml],
[item_dict.rss.reference, item_dict.rss, full_rss],
[item_dict.json.reference, item_dict.json, full_json],
[item_dict.opml2.reference, item_dict.opml2, full_opml],
[item_dict.rss2.reference, item_dict.rss2, full_rss]]
),
equalsubStorageCallCount(
test.opml_mock_options.count,
{buildQuery: 1}
),
equalStorage(test.jio, [
[opml_doc.url, opml_doc],
[opml_doc2.url, opml_doc2],
[opml_outline.reference, opml_outline],
[rss_item_id, rss_item],
[promise_id, promise_item],
[opml_outline2.reference, opml_outline2],
[rss_item_id2, rss_item2],
[promise_id2, promise_item2]
]),
equalsubStorageCallCount(
test.rss_mock_options.count,
opml_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.web_mock_options.count,
rss_mock_options.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.sub_opml_storage2.options.count,
opml_mock_options2.count,
{buildQuery: 1}
),
equalsubStorageCallCount(
test.sub_opml_storage2.sub_storage_list[0].options.count,
rss_mock_options2.count,
{buildQuery: 1}
)
]);
......
src/jio.storage/replicatedopmltreestorage.js
View file @ 3d97883d
......@@ -5,7 +5,7 @@
*/
/*jslint nomen: true */
/*global jIO, RSVP, Rusha, Blob, console */
/*global jIO, RSVP, Rusha, console, Blob */
(function (jIO, RSVP, Rusha, Blob, console) {
"use strict";
......@@ -16,19 +16,8 @@
*
* {
* "type": "replicatedopml",
* "opml_storage_list": [
* {
* "type": "opml",
* "url": "http://some.url.com",
* "sub_storage_list": [
* {"type": "rss", "url_attribute": "xmlUrl"},
* {"type": "webdav": "url_path": "/"},
* {"type": "webdav", "url_path": "/data/"},
* {"type": "webhttp": "url_path": "/storage"}
* ],
* "basic_login": "XUSODISOIDOISUJDD=="
* }
* ],
* "remote_storage_unreachable_status": "WARNING",
* "remote_opml_check_time_interval": 86400000,
* local_sub_storage: {
* type: "query",
* sub_storage: {
......@@ -41,7 +30,7 @@
*/
var rusha = new Rusha(),
KNOWN_SUB_STORAGE_LIST = ['rss', 'dav', 'webhttp'];
OPML_ATTACHMENT_NAME = "__opml__";
function generateHash(str) {
return rusha.digestFromString(str);
......@@ -61,48 +50,28 @@
* @constructor
*/
function ReplicatedOPMLStorage(spec) {
var i,
j;
function checkSubStorage(sub_storage_spec) {
if (typeof sub_storage_spec.url !== 'string' &&
typeof sub_storage_spec.url_path !== 'string' &&
typeof sub_storage_spec.url_attribute !== 'string') {
throw new TypeError("one or more OPML sub storage(s) has no 'url' set");
}
if (typeof sub_storage_spec.type !== 'string') {
if (typeof spec.type !== 'string') {
throw new TypeError(
"one or more OPML sub storage(s) has no attribute 'type' set"
"ReplicatedOPMLStorage 'type' is not a string"
);
}
}
if (typeof spec.opml_storage_list !== 'object') {
throw new TypeError("ReplicatedOPMLStorage 'opml_storage_list' " +
"is not of type object");
}
if (spec.local_sub_storage === undefined) {
throw new TypeError("ReplicatedOPMLStorage 'local_sub_storage' " +
"is not defined");
}
this._local_sub_storage = jIO.createJIO(spec.local_sub_storage);
this._remote_storage_unreachable_status =
spec.remote_storage_unreachable_status;
this._remote_storage_dict = {};
this._opml_storage_list = spec.opml_storage_list;
for (i = 0; i < this._opml_storage_list.length; i += 1) {
if (typeof this._opml_storage_list[i].url !== 'string') {
throw new TypeError("opml storage 'url' is not of type string");
}
if (this._opml_storage_list[i].sub_storage_list !== undefined) {
for (j = 0; j < this._opml_storage_list[i].sub_storage_list.length;
j += 1) {
checkSubStorage(this._opml_storage_list[i].sub_storage_list[j]);
this._remote_parser_storage_type = spec.remote_parser_storage_type;
if (this._remote_parser_storage_type === undefined) {
this._remote_parser_storage_type = "parser";
}
}
/*if (spec.opml_storage_list[i].type !== 'opml') {
throw new TypeError("ReplicatedOPMLStorage 'type' should be 'opml'");
}*/
this._remote_opml_check_time_interval =
spec.remote_opml_check_time_interval;
if (this._remote_opml_check_time_interval === undefined) {
// one day in miliseconds
this._remote_opml_check_time_interval = 86400000;
}
}
......@@ -116,10 +85,15 @@
arguments);
};
/*ReplicatedOPMLStorage.prototype.put = function () {
return this._local_sub_storage.put.apply(this._local_sub_storage,
arguments);
};*/
ReplicatedOPMLStorage.prototype.put = function (id, doc) {
if (!doc.hasOwnProperty('portal_type') || doc.portal_type !== 'opml') {
throw new TypeError("Cannot put object which portal_type is not 'opml'");
}
if (doc.active === undefined) {
doc.active = true;
}
return this._local_sub_storage.put(id, doc);
};
ReplicatedOPMLStorage.prototype.hasCapacity = function (capacity) {
if (capacity === 'include') {
......@@ -134,44 +108,105 @@
arguments);
};
ReplicatedOPMLStorage.prototype.allAttachments = function () {
return this._local_sub_storage.allAttachments.apply(this._local_sub_storage,
arguments);
};
ReplicatedOPMLStorage.prototype.remove = function (id) {
var storage = this._local_sub_storage;
return storage.get(id)
.push(function (doc) {
if (doc.portal_type !== 'opml') {
throw new TypeError("Object with portal_type" + doc.portal_type +
"is frozen, cannot remove it.");
}
function removeOPMLTree(url) {
var remove_id_list = [],
remove_signature_id_list = [];
remove_signature_id_list.push({
id: url,
name: url
});
return storage.allDocs({
select_list: ["xmlUrl", "url"],
query: '(portal_type:"opml-outline") AND (parent_url:"' + url + '")'
})
.push(function (document_result) {
var i,
query_list = [];
for (i = 0; i < document_result.data.total_rows; i += 1) {
query_list.push('(parent_id:"' +
document_result.data.rows[i].id + '")');
function getSubOpmlStorageDescription(storage_spec, opml_doc, basic_login) {
remove_id_list.push(document_result.data.rows[i].id);
remove_signature_id_list.push({
id: document_result.data.rows[i].id,
name: document_result.data.rows[i].value.xmlUrl
});
remove_signature_id_list.push({
id: document_result.data.rows[i].id,
name: document_result.data.rows[i].value.url
});
}
// cleanup all sub opml items
if (query_list.length > 0) {
return storage.allDocs({query: query_list.join(" OR ")});
}
return {data: {total_rows: 0}};
})
.push(function (sub_item_result) {
var j,
i,
k,
remove_queue = new RSVP.Queue();
if (storage_spec.basic_login === undefined && basic_login !== undefined) {
storage_spec.basic_login = basic_login;
function removeItem(key) {
remove_queue
.push(function () {
return storage.remove(key);
});
}
if (KNOWN_SUB_STORAGE_LIST.indexOf(storage_spec.type) !== -1) {
if (storage_spec.url_attribute !== undefined &&
opml_doc.hasOwnProperty(storage_spec.url_attribute)) {
storage_spec.url = opml_doc[storage_spec.url_attribute];
} else if (storage_spec.url_path !== undefined) {
storage_spec.url_path = storage_spec.url_path.replace(
new RegExp("^[/]+"),
""
function removeAttachmentItem(id, name) {
remove_queue
.push(function () {
return storage.removeAttachment(id, name);
})
.push(undefined, function (error) {
if ((error instanceof jIO.util.jIOError) &&
(error.status_code === 404)) {
return undefined;
}
throw error;
});
}
// remove signatures
for (k = 0; k < remove_signature_id_list.length; k += 1) {
removeAttachmentItem(
remove_signature_id_list[k].id,
remove_signature_id_list[k].name
);
storage_spec.url = opml_doc.url.replace(
new RegExp("[/]+$"),
""
) + "/" + storage_spec.url_path;
}
// XXX - for compatibility, remove url with jio_private path
storage_spec.url = storage_spec.url.replace("jio_private", "private");
if (storage_spec.type === "dav") {
storage_spec = {
type: "query",
sub_storage: {
type: "drivetojiomapping",
sub_storage: storage_spec
}
};
// remove opml-outline sub-items (rss)
for (j = 0; j < sub_item_result.data.total_rows; j += 1) {
removeItem(sub_item_result.data.rows[j].id);
}
// remove opml-outline
for (i = 0; i < remove_id_list.length; i += 1) {
removeItem(remove_id_list[i]);
}
return storage_spec;
return remove_queue;
})
.push(function () {
return storage.remove(url);
});
}
return removeOPMLTree(id);
});
};
ReplicatedOPMLStorage.prototype.allAttachments = function () {
return this._local_sub_storage.allAttachments.apply(this._local_sub_storage,
arguments);
};
function getStorageUrl(storage_spec) {
var spec = storage_spec;
......@@ -179,93 +214,146 @@
if (spec.url !== undefined) {
return spec.url;
}
if (spec.document_id !== undefined) {
return spec.document_id;
}
spec = spec.sub_storage;
}
throw new Error("No url found on sub storage: " +
JSON.stringify(storage_spec));
}
function loadSubStorage(context, spec, parent_id, opml_doc, basic_login) {
function getDocumentAsAttachment(context, attachment_id, name) {
return context._local_sub_storage.getAttachment(attachment_id, name)
.push(undefined, function (error) {
if ((error instanceof jIO.util.jIOError) &&
(error.status_code === 404)) {
return undefined;
}
console.error(error);
})
.push(function (attachment) {
if (attachment) {
return jIO.util.readBlobAsText(attachment)
.then(function (evt) {
return JSON.parse(evt.target.result);
});
}
return {};
});
}
function updateSubStorageStatus(context, signature_dict, next_status) {
var key,
update_status_queue = new RSVP.Queue();
function updateStatus(id) {
update_status_queue
.push(function () {
return context._local_sub_storage.get(id);
})
.push(function (doc) {
if (doc.portal_type === "promise") {
doc.category = next_status;
return context._local_sub_storage.put(id, doc);
}
if (doc.status !== undefined) {
doc.status = next_status;
return context._local_sub_storage.put(id, doc);
}
});
}
for (key in signature_dict) {
if (signature_dict.hasOwnProperty(key)) {
if (signature_dict[key].status !== next_status) {
updateStatus(key);
signature_dict[key].status = next_status;
}
}
}
return update_status_queue
.push(function () {
return signature_dict;
});
}
function loadSubStorage(context, storage_spec, parent_id, type) {
var sub_storage,
storage_spec,
options = {},
result_dict,
storage_key,
url;
if (spec.has_include_docs === true) {
options = {include_docs: true};
}
try {
storage_spec = getSubOpmlStorageDescription(
spec,
opml_doc,
basic_login
);
url = getStorageUrl(storage_spec);
storage_key = generateHash(parent_id + url);
sub_storage = createStorage(context, storage_spec, storage_key);
} catch (error) {
console.log(error);
throw error;
}
result_dict = {
parent_id: parent_id,
parent_title: opml_doc.title,
opml_title: opml_doc.opml_title,
type: storage_spec.type,
current_doc: {},
result: {
data: {
total_rows: 0
}
},
type: type || storage_spec.type,
current_signature: {},
result: {data: {total_rows: 0}},
url: url
};
return sub_storage.allDocs(options)
return sub_storage.allDocs({include_docs: true})
.push(undefined, function (error) {
if ((error instanceof jIO.util.jIOError) &&
(error.status_code === 404)) {
console.log(error);
return undefined;
}
//throw error;
// throw will cancel all others allDocs, this is not wanted
console.log(error);
console.error(error);
return undefined;
})
.push(function (result) {
if (result === undefined) {
return {data: {total_rows: 0}};
if (context._remote_storage_unreachable_status !== undefined) {
// update status of local documents
// and set unreachable status
return getDocumentAsAttachment(context, parent_id, url)
.push(function (signature_document) {
return updateSubStorageStatus(
context,
signature_document,
context._remote_storage_unreachable_status
);
})
.push(function (signature_dict) {
return signature_dict;
});
}
return {};
}
result_dict.result = result;
return context._local_sub_storage.allDocs({
select_list: ["signature"],
query: '(parent_id: "' + parent_id + '") AND (url:"' + url + '")'
});
return getDocumentAsAttachment(
context,
parent_id,
url
);
})
.push(function (all_document) {
var i;
for (i = 0; i < all_document.data.total_rows; i += 1) {
result_dict.current_doc[
all_document.data.rows[i].id
] = all_document.data.rows[i].value;
}
.push(function (signature_document) {
result_dict.current_signature = signature_document;
return result_dict;
});
}
function getOpmlTree(context, opml_url, opml_spec) {
function getOpmlTree(context, opml_url, opml_spec, basic_login) {
var opml_storage,
opml_document_list = [],
document_attachment_dict = {},
delete_key_list = [],
current_opml_dict = {},
attachment_document_list = [],
opml_result_list,
current_signature_dict = {},
fetch_remote_opml = false,
id;
id = generateHash(opml_url);
opml_storage = createStorage(context, opml_spec, id);
return getDocumentAsAttachment(context, opml_url, OPML_ATTACHMENT_NAME)
.push(function (opml_doc) {
var current_time = new Date().getTime();
if (opml_doc.expire_time !== undefined) {
fetch_remote_opml = (opml_doc.expire_time - current_time) < 0;
} else {
fetch_remote_opml = true;
}
if (fetch_remote_opml) {
return opml_storage.allDocs({include_docs: true})
.push(undefined, function (error) {
if ((error instanceof jIO.util.jIOError) &&
......@@ -273,175 +361,244 @@
return {data: {total_rows: 0}};
}
//throw error;
// throw will cancel all remaning tasks
console.error(error);
return {data: {total_rows: 0}};
})
.push(function (opml_result) {
return RSVP.all([
opml_result,
context._local_sub_storage.allDocs({
select_list: ["signature"],
query: '(parent_id: "' + id + '")'
})
]);
opml_result_list = opml_result;
if (opml_result.data.total_rows > 0) {
attachment_document_list.push({
id: opml_url,
name: OPML_ATTACHMENT_NAME,
doc: {
expire_time: new Date().getTime() +
context._remote_opml_check_time_interval,
data: opml_result
}
});
return getDocumentAsAttachment(
context,
id,
opml_url
);
}
return {};
})
.push(function (result_list) {
var i;
for (i = 0; i < result_list[1].data.total_rows; i += 1) {
current_opml_dict[
result_list[1].data.rows[i].id
] = result_list[1].data.rows[i].value;
.push(function (signature_dict) {
current_signature_dict = signature_dict;
});
}
return result_list[0];
opml_result_list = opml_doc.data;
})
.push(function (opml_result_list) {
var j,
i,
.push(function () {
var i,
item,
signature,
doc_signature_dict = {},
skip_add = false,
id_hash,
result_list = [];
result_list = [],
header_dict = {};
if (opml_result_list.data.total_rows > 0 && fetch_remote_opml) {
header_dict = {
dateCreated: opml_result_list.data.rows[0].doc.dateCreated,
dateModified: opml_result_list.data.rows[0].doc.dateModified,
opml_title: opml_result_list.data.rows[0].doc.title
};
}
for (i = 0; i < opml_result_list.data.total_rows; i += 1) {
for (i = 1; i < opml_result_list.data.total_rows; i += 1) {
item = opml_result_list.data.rows[i];
if (item.doc.xmlUrl !== undefined) {
id_hash = generateHash(id + item.id);
signature = generateHash(JSON.stringify(item.doc));
for (j = 0; j < opml_spec.sub_storage_list.length; j += 1) {
result_list.push(loadSubStorage(
context,
opml_spec.sub_storage_list[j],
{
type: context._remote_parser_storage_type,
document_id: item.doc.xmlUrl,
attachment_id: 'enclosure',
parser: 'rss',
sub_storage: {
type: "http"
}
},
id_hash,
item.doc,
opml_spec.basic_login
'promise'
));
// Load private docs
if (item.doc.url !== undefined) {
result_list.push(loadSubStorage(
context,
{
type: 'webhttp',
url: item.doc.url.replace('jio_private', 'private'),
basic_login: basic_login
},
id_hash
));
}
if (current_opml_dict.hasOwnProperty(id_hash)) {
if (current_opml_dict[id_hash].signature === signature) {
// the document was not modified, delete and skip add
delete current_opml_dict[id_hash];
if (fetch_remote_opml) {
// Append this document signature to the list
signature = generateHash(JSON.stringify(item.doc));
doc_signature_dict[id_hash] = {
signature: signature
};
if (current_signature_dict.hasOwnProperty(id_hash)) {
if (current_signature_dict[id_hash].signature === signature) {
// remote document was not modified, delete and skip add
delete current_signature_dict[id_hash];
skip_add = true;
}
delete current_opml_dict[id_hash];
delete current_signature_dict[id_hash];
}
Object.assign(item.doc, {
portal_type: "opml-outline",
parent_id: id,
parent_url: opml_url,
reference: id_hash,
active: true
});
Object.assign(item.doc, header_dict);
if (!skip_add) {
opml_document_list.push({
id: id_hash,
doc: {
type: "opml-item",
name: item.id,
reference: id_hash,
parent_id: id,
creation_date: item.doc.created_date,
url: opml_url,
title: item.doc.title,
parent_title: undefined,
opml_title: item.doc.opml_title,
status: undefined,
signature: signature
}
});
document_attachment_dict[id_hash] = {
name: item.id,
doc: item.doc
};
});
}
}
}
}
if (fetch_remote_opml && Object.keys(doc_signature_dict).length > 0) {
attachment_document_list.push({
id: opml_url,
name: opml_url,
doc: doc_signature_dict
});
delete_key_list.push.apply(delete_key_list,
Object.keys(current_opml_dict));
Object.keys(current_signature_dict));
}
return RSVP.all(result_list);
})
.push(function (result_list) {
var i,
j;
j,
start,
extra_dict,
item_signature_dict = {};
function applyItemToTree(item, item_result) {
function applyItemToTree(item, item_result, portal_type, extra_dict) {
var id_hash,
element,
signature;
element = item.doc,
signature,
item_id = item.guid || item.id,
status = (element.status || element.category);
id_hash = generateHash(item_result.parent_id +
item_result.url + item.id);
if (item.doc !== undefined) {
element = item.doc;
} else {
element = item.value;
}
item_result.url + item_id);
if (extra_dict !== undefined) {
Object.assign(element, extra_dict);
}
// Generating document signature
signature = generateHash(JSON.stringify(element));
item_signature_dict[id_hash] = {
signature: signature,
status: status
};
if (item_result.current_doc.hasOwnProperty(id_hash)) {
if (item_result.current_doc[id_hash].signature === signature) {
// the document was not modified delete and return
delete item_result.current_doc[id_hash];
if (item_result.current_signature.hasOwnProperty(id_hash)) {
if (item_result.current_signature[id_hash].signature
=== signature) {
// the document was not modified return
delete item_result.current_signature[id_hash];
return;
}
// the document exists and has changed
delete item_result.current_doc[id_hash];
delete item_result.current_signature[id_hash];
}
opml_document_list.push({
id: id_hash,
doc: {
Object.assign(element, {
parent_id: item_result.parent_id,
name: item.id,
type: (element.type || item_result.type + "-item"),
portal_type: portal_type || element.type || item_result.type,
status: status,
reference: id_hash,
creation_date: element.date || element["start-date"],
url: item_result.url,
status: (element.status || element.category),
title: (element.source || element.title),
parent_title: item_result.parent_title,
opml_title: item_result.opml_title,
signature: signature
}
active: true
});
document_attachment_dict[id_hash] = {
name: item.id,
opml_document_list.push({
id: id_hash,
doc: element
};
});
}
for (i = 0; i < result_list.length; i += 1) {
for (j = 0; j < result_list[i].result.data.total_rows; j += 1) {
extra_dict = undefined;
start = 0;
if (result_list[i].result.data.total_rows > 0) {
if (result_list[i].type === "promise") {
// the first element of rss is the header
extra_dict = {
lastBuildDate: result_list[i].result.data.rows[0].doc
.lastBuildDate,
channel: result_list[i].result.data.rows[0].doc.description,
channel_item: result_list[i].result.data.rows[0].doc.title
};
applyItemToTree(
result_list[i].result.data.rows[0],
result_list[i],
"rss"
);
start = 1;
}
for (j = start; j < result_list[i].result.data.total_rows; j += 1) {
applyItemToTree(
result_list[i].result.data.rows[j],
result_list[i]
result_list[i],
undefined,
extra_dict
);
}
delete_key_list.push.apply(delete_key_list,
Object.keys(result_list[i].current_doc));
attachment_document_list.push({
id: result_list[i].parent_id,
name: result_list[i].url,
doc: item_signature_dict
});
item_signature_dict = {};
delete_key_list.push.apply(
delete_key_list,
Object.keys(result_list[i].current_signature)
);
} else if (Object.keys(result_list[i].current_signature).length > 0) {
// if the remote data is empty and current_signature is not empty,
// push to storage in case the status was changed
// this help for speed optimisation
attachment_document_list.push({
id: result_list[i].parent_id,
name: result_list[i].url,
doc: result_list[i].current_signature
});
}
}
return [opml_document_list, document_attachment_dict, delete_key_list];
return [opml_document_list, delete_key_list, attachment_document_list];
});
}
function pushDocumentToStorage(context, document_list, attachment_dict,
delete_key_list) {
function pushDocumentToStorage(context, document_list, delete_key_list,
attachment_document_list) {
var document_queue = new RSVP.Queue(),
i;
function pushDocument(id, element, attachment) {
function pushDocument(id, element) {
document_queue
.push(function () {
return context._local_sub_storage.put(id, element);
})
.push(function () {
return context._local_sub_storage.putAttachment(
id,
attachment.name,
new Blob([JSON.stringify(attachment.doc)])
);
});
}
for (i = 0; i < document_list.length; i += 1) {
pushDocument(
document_list[i].id,
document_list[i].doc,
attachment_dict[document_list[i].id]
document_list[i].doc
);
}
return document_queue
......@@ -452,18 +609,6 @@
// remove all document which were not updated
function removeDocument(key) {
remove_queue
.push(function () {
return context._local_sub_storage.get(key);
})
.push(undefined, function (error) {
throw error;
})
.push(function (element) {
return context._local_sub_storage.removeAttachment(
key,
element.name
);
})
.push(function () {
return context._local_sub_storage.remove(key);
})
......@@ -480,17 +625,60 @@
removeDocument(delete_key_list[k]);
}
return remove_queue;
})
.push(function () {
var j,
signature_queue = new RSVP.Queue();
function pushAttachment(id, name, element) {
signature_queue
.push(function () {
return context._local_sub_storage.putAttachment(
id,
name,
new Blob([JSON.stringify(element)], {type : 'application/json'})
);
})
.push(undefined, function (error) {
console.error(error);
});
}
for (j = 0; j < attachment_document_list.length; j += 1) {
pushAttachment(
attachment_document_list[j].id,
attachment_document_list[j].name,
attachment_document_list[j].doc
);
}
});
}
function syncOpmlStorage(context) {
return context._local_sub_storage.allDocs({
query: '(portal_type:"opml") AND (active:true)',
select_list: ["url", "basic_login"]
})
.push(function (storage_result) {
var i,
opml_queue = new RSVP.Queue();
function syncFullOpml(url, storage_spec) {
return opml_queue
function syncFullOpml(storage_spec) {
opml_queue
.push(function () {
return getOpmlTree(context, url, storage_spec);
return getOpmlTree(
context,
storage_spec.url,
{
type: context._remote_parser_storage_type,
document_id: storage_spec.url,
attachment_id: 'enclosure',
parser: 'opml',
sub_storage: {
type: "http"
}
},
storage_spec.basic_login
);
})
.push(function (result_list) {
return pushDocumentToStorage(
......@@ -501,11 +689,15 @@
);
});
}
for (i = 0; i < context._opml_storage_list.length; i += 1) {
syncFullOpml(context._opml_storage_list[i].url,
context._opml_storage_list[i]);
for (i = 0; i < storage_result.data.total_rows; i += 1) {
syncFullOpml({
type: storage_result.data.rows[i].value.type,
url: storage_result.data.rows[i].value.url,
basic_login: storage_result.data.rows[i].value.basic_login
});
}
return opml_queue;
});
}
ReplicatedOPMLStorage.prototype.repair = function () {
......
src/jio.storage/webhttpstorage.js
View file @ 3d97883d
......@@ -39,7 +39,7 @@
throw new jIO.util.jIOError("id " + id + " is forbidden (no begin /)",
400);
}
if (id.lastIndexOf("/") !== (id.length - 1)) {
if (id.lastIndexOf("/") === (id.length - 1)) {
throw new jIO.util.jIOError("id " + id + " is forbidden (no end /)",
400);
}
......@@ -84,14 +84,19 @@
}
WEBHTTPStorage.prototype.get = function (id) {
var context = this,
element;
var context = this;
id = restrictDocumentId(id);
element = getJsonDocument(context, id);
if (element !== undefined) {
return getJsonDocument(context, id)
.push(function (element) {
return element.doc;
}, function (error) {
if ((error.target !== undefined) &&
(error.target.status === 404)) {
throw new jIO.util.jIOError("Cannot find document", 404);
}
throw error;
});
};
WEBHTTPStorage.prototype.hasCapacity = function (capacity) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7