Commit e6b9cc10 authored by Alain Takoudjou's avatar Alain Takoudjou

erp5_web_mynij_search: Fix downloading index from torrent, load source now...

erp5_web_mynij_search: Fix downloading index from torrent, load source now check if there is a torrent for download

When loading new source from store link, check if there is attached torrent and try to download it. If not possible to download after a timeout, register a build for the source.
parent 44352741
......@@ -17,6 +17,7 @@
<script src="handlebars.js" type="text/javascript"></script>
<script src="webtorrent.min.js" type="text/javascript"></script>
<script src="jszip.min.js" type="text/javascript"></script>
<script src="tv4.js" type="text/javascript"></script>
<script src="gadget_erp5_mynij_webtorrent.js" type="text/javascript"></script>
<script id="torrent-info-template" type="text/x-handlebars-template">
......
......@@ -246,7 +246,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>989.27299.4752.580</string> </value>
<value> <string>989.32963.49230.49595</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -264,7 +264,7 @@
</tuple>
<state>
<tuple>
<float>1611306465.06</float>
<float>1632909569.13</float>
<string>UTC</string>
</tuple>
</state>
......
/*jslint nomen: true, indent: 2, maxerr: 3, maxlen: 80*/
/*global window, RSVP, rJS, WebTorrent, JSZip*/
(function (window, RSVP, rJS, document, crypto, WebTorrent, JSZip) {
/*global window, RSVP, rJS, WebTorrent, JSZip, tv4*/
(function (window, RSVP, rJS, document, crypto, WebTorrent, JSZip, tv4) {
"use strict";
var gadget_klass = rJS(window),
......@@ -113,6 +113,9 @@
.declareMethod("render", function (options) {
var gadget = this;
return gadget.changeState({
service: options.service || true
});
})
.declareMethod("digestMessage", function digestMessage(text, algorithm) {
......@@ -130,37 +133,6 @@
});
})
.declareMethod("getRtcconfig", function () {
var default_rtcconfig = {
// XXX - set configurable stun address
iceServers: [
{urls: [
"stun:turn.api.nexedi.net:3478",
"stun:stun.l.google.com:19302",
"stun:global.stun.twilio.com:3478"
]}
]
};
return new RSVP.Queue()
.push(function () {
return jIO.util.ajax({
url : "mynij-config.json"
});
})
.push(undefined, function (error) {
//no rtcconfig file found, use the default
console.log(error);
return undefined;
})
.push(function (response) {
var rtc_config;
if (!response)
return default_rtcconfig;
rtc_config = JSON.parse(response.target.responseText);
return rtc_config.rtcConfig;
});
})
.declareMethod("index_to_file", function (index_name) {
var gadget = this,
index_id = "index-" + index_name;
......@@ -313,7 +285,8 @@
ext,
i,
insert_blob,
msg_html;
msg_html,
schema;
insert_blob = function (index_id, type, data) {
return gadget.state.model_gadget.putAttachment(
......@@ -353,12 +326,20 @@
else if (ext === "zip")
zipped_blob = blob_list[i];
}
return new RSVP.Queue()
.push(function () {
return gadget.state.model_gadget.getSourceSchema()
.push(function (s) {
schema = s;
return jIO.util.readBlobAsText(doc_blob);
})
.push(function (evt) {
doc = JSON.parse(evt.target.result);
try {
doc = JSON.parse(evt.target.result);
if (!tv4.validate([doc], schema))
return false;
} catch (e) {
console.error(e);
return false;
}
if (msg_html)
msg_html.innerText = "Uncompressing index data...";
// extract zipped data here
......@@ -380,38 +361,50 @@
return RSVP.all(promise_list);
});
})
.push(function () {
.push(function (state) {
if (state === false) {
return gadget.notifySubmitted({
message: "Error, the torrent contains invalid data!",
status: "error"
})
.push(function () {
return gadget.redirect({command: 'display', options: {
page: "mynij_search"
}});
});
}
if (msg_html)
msg_html.innerText = "Creating index document...";
return gadget.state.model_gadget.createIndexDoc({
title: doc.title,
links: doc.links,
description: doc.description,
magnet_uri: torrent.magnetURI,
status: "built"
});
})
.push(function () {
return gadget.state.model_gadget.pushIndex(index_name);
})
.push(function () {
gadget.notifySubmitted({
message: "Index '" + index_name + "' Saved.",
status: "success"
})
})
.push(function () {
// stop seeding the torrent
gadget.state.client.destroy(function (err) {
if (err) {
console.error(err.stack || err.message || err);
.push(function () {
return gadget.state.model_gadget.pushIndex(index_name);
})
.push(function () {
gadget.notifySubmitted({
message: "Index '" + index_name + "' Saved.",
status: "success"
})
})
.push(function () {
// stop seeding the torrent
gadget.state.client.destroy(function (err) {
if (err) {
console.error(err.stack || err.message || err);
}
});
if (gadget.state.redirect) {
return gadget.redirect({command: 'display', options: {
page: gadget.state.redirect
}});
}
gadget.state.redirect = "";
});
if (gadget.state.redirect) {
return gadget.redirect({command: 'display', options: {
page: gadget.state.redirect
}});
}
gadget.state.redirect = "";
});
})
......@@ -423,7 +416,7 @@
element,
i,
key_list = [];
if (!container)
if (!container || !gadget.state.service)
return;
info = container.querySelector('h2.info')
seed_element = container.querySelector('.torrent-seed');
......@@ -530,7 +523,9 @@
return gadget.index_to_file(index_doc.title);
})
.push(function (index_blob) {
var doc = {title: index_doc.title, links: index_doc.links},
var doc = {title: index_doc.title,
links: index_doc.links,
description: index_doc.description},
index_file_list = [index_blob];
// Add index document to blob
index_file_list.push(new File(
......@@ -599,7 +594,7 @@
});
}
return gadget.getRtcconfig()
return gadget.state.model_gadget.getRtcconfig()
.push(function (config) {
gadget.state.client = createWebTorrent(config);
return seedTorrent();
......@@ -614,9 +609,10 @@
if (!torrent_id) {
return;
}
torrent_id = torrent_id.trim();
gadget.state.redirect = redirect;
container.classList.remove('hidden');
return gadget.getRtcconfig()
return gadget.state.model_gadget.getRtcconfig()
.push(function (config) {
gadget.state.client = createWebTorrent(config);
return gadget.notifySubmitting();
......@@ -664,7 +660,17 @@
onProgress(gadget, torrent, element);
});
})
.push(function (torrent) {
torrent.on('error', function (err) {
clearInterval(gadget.state.interval);
console.error(err.stack || err.message || err);
return gadget.notifySubmitted({
message: "Error: " + err.message || err,
status: "error"
});
});
});
});
}(window, RSVP, rJS, document, crypto, WebTorrent, JSZip));
\ No newline at end of file
}(window, RSVP, rJS, document, crypto, WebTorrent, JSZip, tv4));
\ No newline at end of file
......@@ -242,7 +242,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>990.12632.20339.25224</string> </value>
<value> <string>995.7322.44137.26811</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -260,7 +260,7 @@
</tuple>
<state>
<tuple>
<float>1630505542.2</float>
<float>1632931116.23</float>
<string>UTC</string>
</tuple>
</state>
......
......@@ -5,6 +5,7 @@
<script src="rsvp.js" type="text/javascript"></script>
<script src="renderjs.js" type="text/javascript"></script>
<script src="tv4.js" type="text/javascript"></script>
<script src="webtorrent.min.js" type="text/javascript"></script>
<script src="gadget_erp5_page_load_source.js" type="text/javascript"></script>
<link rel="stylesheet" type="text/css" href="mynij-style.css">
</head>
......
......@@ -242,7 +242,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>994.43649.32057.29422</string> </value>
<value> <string>994.64296.3731.46916</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -260,7 +260,7 @@
</tuple>
<state>
<tuple>
<float>1632415203.4</float>
<float>1632928886.44</float>
<string>UTC</string>
</tuple>
</state>
......
/*jslint nomen: true, indent: 2, maxerr: 3, maxlen: 80*/
/*global window, RSVP, rJS, document, tv4*/
(function (window, document, rJS, RSVP) {
var gadget;
/*global window, RSVP, rJS, document, tv4, WebTorrent*/
(function (window, document, rJS, RSVP, tv4, WebTorrent) {
function createWebTorrent(rtcconfig) {
return new WebTorrent({
tracker: {
rtcConfig: rtcconfig
}
});
}
function checkDownloadTorrent(gadget, index_dict) {
var client,
counter = 0,
downloading,
magnet;
return gadget.state.model_gadget.getRtcconfig()
.push(function (rtcconfig) {
magnet = "magnet:?xt=urn:btih:";
if (!index_dict.infohash || ! index_dict.namehash)
return;
// XXX - hardcoded tracker uri
magnet += index_dict.infohash + "&dn=" + index_dict.namehash +
"&tr=wss://tracker.openwebtorrent.com&tr=" +
"wss://softinst140244.host.vifib.net";
client = createWebTorrent(rtcconfig);
// XXX - find a better way to check if torrent is downloadable
return client.add(magnet, function (torrent) {
torrent.on('error', function (err) {
console.error(err);
});
});
})
.push(function (torrent) {
function checkReady() {
if (torrent.ready)
return true;
if (counter >= 7)
return false;
return RSVP.delay(1000)
.then(function () {
counter += 1;
return checkReady();
});
}
return new RSVP.Queue()
.push(function () {
return checkReady();
})
.push(function (status) {
// cancel torrent
torrent.destroy(function (err) {
console.error(err);
});
if (status)
// torrent will download...
return gadget.redirect({command: 'change', options: {
page : "mynij_torrent",
magnet_uri: magnet
}});
});
});
}
rJS(window)
.setState({
torrent_gadget : null
})
.declareAcquiredMethod("redirect", "redirect")
.declareAcquiredMethod("getUrlParameter", "getUrlParameter")
.declareAcquiredMethod("getSetting", "getSetting")
.declareAcquiredMethod("setSetting", "setSetting")
.declareAcquiredMethod("notifySubmitted", "notifySubmitted")
.declareAcquiredMethod("updateHeader", "updateHeader")
.ready(function () {
var gadget = this;
......@@ -25,48 +84,18 @@
})
.onStateChange(function () {
})
.declareMethod("render", function (options) {
var gadget = this;
return new RSVP.Queue()
.push(function () {
return gadget.updateHeader({
page_title: "JSON Source import"
});
});
})
.declareMethod("loadSource", function (url) {
return new RSVP.Queue()
.push(function () {
return jIO.util.ajax({
url : url
});
})
.push(undefined, function (error) {
console.log(error);
return undefined;
})
.push(function (response) {
if (!response)
return undefined;
return response.target.responseText;
});
})
.declareMethod("render", function (options) {
var gadget = this,
doc,
promise_list = [],
schema,
i;
if (!gadget.state.source_url)
return;
return gadget.state.model_gadget.getSourceSchema()
.push(function (s) {
schema = s;
return gadget.loadSource(options.url);
return gadget.loadSource(gadget.state.source_url);
})
.push(function (source_string) {
var source_list,
......@@ -77,6 +106,7 @@
source_list = JSON.parse(source_string);
valid = tv4.validate(source_list, schema);
} catch (e) {
console.error(e);
a_json = false;
}
if (!source_string || !a_json || !valid) {
......@@ -92,50 +122,80 @@
page : "mynij_search"
}});
});
} else {
for (i = 0; i < source_list.length; i += 1) {
doc = {};
doc.title = source_list[i].title;
doc.links = source_list[i].links;
doc.status = "not built";
doc.description = source_list[i].description || "";
promise_list.push(gadget.state.model_gadget.createIndexDoc(doc));
}
return new RSVP.Queue()
}
for (i = 0; i < source_list.length; i += 1) {
doc = {};
doc.title = source_list[i].title;
doc.links = source_list[i].links;
doc.status = "not built";
doc.description = source_list[i].description || "";
promise_list.push(gadget.state.model_gadget.createIndexDoc(doc));
}
return new RSVP.Queue()
.push(function () {
return RSVP.all(promise_list);
})
.push(function (result) {
var count = result.length;
return new RSVP.Queue()
.push(function () {
return RSVP.all(promise_list);
return checkDownloadTorrent(gadget, source_list[0]);
})
.push(function (result) {
var count = result.length;
return gadget.getSetting("build_dict", {})
.push(function (build_dict) {
var i;
.push(function () {
return gadget.getSetting("build_dict", {});
})
.push(function (build_dict) {
var i;
for (i = 0; i < result.length; i += 1) {
build_dict[result[i]] = {
links: source_list[i].links,
index_name: source_list[i].title,
jio_key: result[i],
id: Date.now()
};
}
return gadget.setSetting("build_dict", build_dict);
})
.push(function () {
return gadget.notifySubmitted({
message: count + " Source(s) correctly imported!\n" +
"Imported source(s) will now build!",
status: "success"
});
})
.push(function () {
var options = {page : "mynij_search"};
return gadget.redirect({command: 'display', options: options});
});
for (i = 0; i < result.length; i += 1) {
build_dict[result[i]] = {
links: source_list[i].links,
index_name: source_list[i].title,
jio_key: result[i],
id: Date.now(),
infohash: source_list[i].infohash,
namehash: source_list[i].namehash
};
}
return gadget.setSetting("build_dict", build_dict);
})
.push(function () {
var options = {page : "mynij_search",
msg: count + " Source(s) correctly imported!\n" +
"Imported source(s) will now build!"};
return gadget.redirect({command: 'display', options: options});
});
}
});
});
})
.declareMethod("loadSource", function (url) {
return new RSVP.Queue()
.push(function () {
return jIO.util.ajax({
url : url
});
})
.push(undefined, function (error) {
console.log(error);
return undefined;
})
.push(function (response) {
if (!response)
return undefined;
return response.target.responseText;
});
})
.declareMethod("render", function (options) {
var gadget = this;
return gadget.updateHeader({page_title: "JSON Source import"})
.push(function () {
return gadget.changeState({
source_url: options.url
});
});
});
}(window, document, rJS, RSVP, tv4));
\ No newline at end of file
}(window, document, rJS, RSVP, tv4, WebTorrent));
\ No newline at end of file
......@@ -242,7 +242,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>994.64336.47707.63931</string> </value>
<value> <string>995.7472.37563.34030</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -260,7 +260,7 @@
</tuple>
<state>
<tuple>
<float>1632417697.12</float>
<float>1632938785.95</float>
<string>UTC</string>
</tuple>
</state>
......
......@@ -14,6 +14,7 @@
.declareAcquiredMethod("redirect", "redirect")
.declareAcquiredMethod("getSetting", "getSetting")
.declareAcquiredMethod("updatePanel", "updatePanel")
.declareAcquiredMethod("notifySubmitted", "notifySubmitted")
.ready(function () {
var model_gadget,
......@@ -46,6 +47,16 @@
})
.onStateChange(function () {
var gadget = this,
msg;
if (this.state.notify) {
msg = gadget.state.notify;
gadget.state.notify = undefined;
return gadget.notifySubmitted({
message: msg,
status: "success"
});
}
})
.declareMethod("render", function (options) {
......@@ -53,7 +64,9 @@
index_list,
drop_list = gadget.element.querySelector("#select_index");
return new RSVP.Queue()
return gadget.changeState({
notify: options.msg
})
.push(function () {
return RSVP.all([
gadget.getUrlFor({command: 'history_previous'}),
......
......@@ -242,7 +242,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>995.3972.54835.4522</string> </value>
<value> <string>995.7469.40491.12902</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -260,7 +260,7 @@
</tuple>
<state>
<tuple>
<float>1632754684.96</float>
<float>1632937826.03</float>
<string>UTC</string>
</tuple>
</state>
......
......@@ -271,6 +271,37 @@
};
})
.declareMethod("getRtcconfig", function () {
var default_rtcconfig = {
// XXX - set configurable stun address
iceServers: [
{urls: [
"stun:turn-paris-rapidspace.api.nexedi.net:3478",
"stun:stun.l.google.com:19302",
"stun:global.stun.twilio.com:3478"
]}
]
};
return new RSVP.Queue()
.push(function () {
return jIO.util.ajax({
url : "mynij-config.json"
});
})
.push(undefined, function (error) {
//no rtcconfig file found, use the default
console.log(error);
return undefined;
})
.push(function (response) {
var rtc_config;
if (!response)
return default_rtcconfig;
rtc_config = JSON.parse(response.target.responseText);
return rtc_config.rtcConfig;
});
})
.declareMethod("getConfig", function () {
return new RSVP.Queue()
.push(function () {
......
......@@ -242,7 +242,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>994.64287.12898.50995</string> </value>
<value> <string>995.5735.8910.39321</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -260,7 +260,7 @@
</tuple>
<state>
<tuple>
<float>1632415269.6</float>
<float>1632938866.29</float>
<string>UTC</string>
</tuple>
</state>
......
......@@ -31,6 +31,7 @@
<section class="section-result">
<ul id="searx-results">
</ul>
<div class="searx-more"><p><a class="searx-morelink" href="#">More results</a></p></div>
<div id="searx-loading" class="loading" style="display: none"><div class="loader"></div></div>
</section>
</div>
......
......@@ -92,6 +92,15 @@ div[data-gadget-scope='header'] .ui-header .ui-controlgroup-controls button[data
max-width: 50em;
}
.searx-more {
margin: 20px 10px;
display: none;
}
.searx-morelink {
color: #3c459e;
}
.body {
color : #444444;
max-width: 50em;
......@@ -567,6 +576,15 @@ div[data-gadget-scope='panel'] ul li a:hover, div[data-gadget-scope='panel'] ul
background-color: #2f3147;
}
div[data-gadget-scope='panel'] ul li a.upload, div[data-gadget-scope='panel'] ul li a.store {
background: #272a44;
color: #bfbedc;
}
div[data-gadget-scope='panel'] ul li a.upload:hover, div[data-gadget-scope='panel'] ul li a.store:hover {
color: #deddec;
}
div[data-gadget-scope='panel'] dl dd a:hover, div[data-gadget-scope='panel'] dl dd a:active {
background-color: rgb(255 255 255 / 55%);
}
......@@ -630,6 +648,7 @@ div[data-gadget-scope='header'] .ui-header .ui-subheader {
.submit-wrap {
width: 10%;
min-width: 6em;
}
......@@ -681,6 +700,11 @@ select {
border-radius: 0;
}
.add-source {
font-weight: 600;
font-size: .9em;
}
option {
margin: 0.6em 0;
}
......
......@@ -242,7 +242,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>991.44537.19963.26094</string> </value>
<value> <string>994.45131.22698.21435</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -260,7 +260,7 @@
</tuple>
<state>
<tuple>
<float>1619607458.67</float>
<float>1631265938.19</float>
<string>UTC</string>
</tuple>
</state>
......
......@@ -128,7 +128,7 @@
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Import index from .zip file</string> </value>
<value> <string>Import .zip or .json index file</string> </value>
</item>
</dictionary>
</value>
......
(function (window, rJS, RSVP, jIO, File, Blob, document, JSZip, tv4) {
"use strict";
function getInfoHash(index) {
var param_list,
infohash,
name,
i,
l;
if (index.magnet_uri) {
//remove magnet:?
param_list = index.magnet_uri.slice(8, index.length).split("&");
//param_list = l[0].split(':');
for (i = 0; i < param_list.length; i += 1) {
if (param_list[i].indexOf('btih:') !== -1) {
infohash = param_list[i].split('btih:')[1];
}
if (param_list[i].indexOf('dn=') !== -1) {
name = param_list[i].split('dn=')[1];
}
}
return {infohash: infohash, namehash: name};
}
return {infohash: "", namehash: ""};
}
rJS(window)
.declareAcquiredMethod("notifySubmitting", "notifySubmitting")
.declareAcquiredMethod("notifySubmitted", "notifySubmitted")
......@@ -260,6 +283,7 @@
file_blob,
built = false,
index = parent_options.doc,
info_dict,
redirect = {
command: "display",
options: {page : "ojs_local_controller", jio_key: jio_key}
......@@ -271,10 +295,13 @@
if (content_dict.export_collection === "on") {
if (index) {
info_dict = getInfoHash(index);
file_blob = new Blob([JSON.stringify([{
title: index.title,
description: index.description || "",
links: index.links}])], {type : "application/json"});
links: index.links,
infohash: info_dict.infohash,
namehash: info_dict.namehash}])], {type : "application/json"});
return gadget.downloadBlob(index.title + ".json", file_blob)
.push(function () {
return {
......@@ -293,9 +320,13 @@
file_blob,
index_list = [];
for (i = 0; i < results.data.rows.length; i += 1) {
info_dict = getInfoHash(results.data.rows[i].value);
index_list.push({
title: results.data.rows[i].value.title,
links: results.data.rows[i].value.links
links: results.data.rows[i].value.links,
description: results.data.rows[i].value.description || "",
infohash: info_dict.infohash,
namehash: info_dict.namehash
});
}
file_blob = new Blob([JSON.stringify(index_list)], {type : "application/json"});
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment