Skip to content

W
wendelin
Commit 448c9c9d authored by Ivan Tyagov's avatar Ivan Tyagov
Browse files
Options

Data lake UI functional tests 

See merge request nexedi/wendelin!53
parents 237ac09a 24511e08
Show whitespace changes
Inline Side-by-side
Showing with 758 additions and 267 deletions
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_getUserName.xml 0 → 100644
View file @ 448c9c9d
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_getUserName</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_stopIngestionList.py
View file @ 448c9c9d
......@@ -93,6 +93,16 @@ for data_ingestion in portal_catalog(portal_type = "Data Ingestion",
ingestion.stop()
else:
ingestion.deliver()
#link split datastreams
related_split_streams = portal_catalog(portal_type = "Data Stream",
reference = data_ingestion.getReference(),
sort_on=[('creation_date', 'ascending')])
predecessor = None
for stream in related_split_streams:
if predecessor:
predecessor.setSuccessorValue(stream)
stream.setPredecessorValue(predecessor)
predecessor = stream
except Exception as e:
context.log("ERROR handling split data streams for ingestion: %s - reference: %s." % (data_ingestion.getId(), data_ingestion.getReference()))
context.log(e)
......
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.py
View file @ 448c9c9d
......@@ -75,7 +75,8 @@ class TestDataIngestion(SecurityTestCase):
def getDataStreamChunkList(self, reference):
data_stream_list = self.portal.portal_catalog(
portal_type = 'Data Stream',
reference = reference)
reference = reference,
sort_on=[('creation_date', 'ascending')])
return data_stream_list
def ingestRequest(self, reference, eof, data_chunk, ingestion_policy):
......@@ -189,6 +190,23 @@ class TestDataIngestion(SecurityTestCase):
#all data streams are validated
self.assertSameSet(['validated' for x in data_stream_list],
[x.getValidationState() for x in data_stream_list])
#data streams are linked
data_stream_1 = data_stream_list[0].getObject()
data_stream_2 = data_stream_list[1].getObject()
data_stream_3 = data_stream_list[2].getObject()
data_stream_4 = data_stream_list[3].getObject()
# test successor
self.assertSameSet(data_stream_2.getRecursiveSuccessorValueList(), \
[data_stream_3, data_stream_4])
self.assertSameSet(data_stream_4.getRecursiveSuccessorValueList(), \
[])
# test predecessor
self.assertSameSet(data_stream_1.getRecursivePredecessorValueList(), \
[])
self.assertSameSet(data_stream_2.getRecursivePredecessorValueList(), \
[data_stream_1])
self.assertSameSet(data_stream_4.getRecursivePredecessorValueList(), \
[data_stream_3, data_stream_2, data_stream_1])
def test_03_DefaultWendelinConfigurationExistency(self):
"""
......
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.xml
View file @ 448c9c9d
......@@ -46,8 +46,8 @@
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple>
<string>W:104, 34: Unused variable \'i\' (unused-variable)</string>
<string>W:104, 76: Unused variable \'j\' (unused-variable)</string>
<string>W:105, 34: Unused variable \'i\' (unused-variable)</string>
<string>W:105, 76: Unused variable \'j\' (unused-variable)</string>
</tuple>
</value>
</item>
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/portal_tests/wendelin_data_lake_ui_zuite.xml 0 → 100644
View file @ 448c9c9d
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Zuite" module="Products.Zelenium.zuite"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_objects</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>wendelin_data_lake_ui_zuite</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/portal_tests/wendelin_data_lake_ui_zuite/testSiteBasicNavigation.xml 0 → 100644
View file @ 448c9c9d
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ZopePageTemplate" module="Products.PageTemplates.ZopePageTemplate"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>content_type</string> </key>
<value> <string>text/html</string> </value>
</item>
<item>
<key> <string>expand</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>testSiteBasicNavigation</string> </value>
</item>
<item>
<key> <string>output_encoding</string> </key>
<value> <string>utf-8</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <unicode></unicode> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/portal_tests/wendelin_data_lake_ui_zuite/testSiteBasicNavigation.zpt 0 → 100644
View file @ 448c9c9d
<html xmlns:tal="http://xml.zope.org/namespaces/tal"
xmlns:metal="http://xml.zope.org/namespaces/metal">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<title>Test RenderJS UI</title>
</head>
<body>
<table cellpadding="1" cellspacing="1" border="1">
<thead>
<tr><td rowspan="1" colspan="3">Test RenderJS UI</td></tr>
</thead><tbody>
<tal:block metal:use-macro="here/Zuite_CommonTemplate/macros/init" />
<!-- Clean Up -->
<tr>
<td>open</td>
<td>${base_url}/bar_module/ListBoxZuite_reset</td>
<td></td>
</tr>
<tr>
<td>assertTextPresent</td>
<td>Reset Successfully.</td>
<td></td>
</tr>
<!-- Initialize
<tr>
<td>open</td>
<td>${base_url}/portal_skins/erp5_wendelin_data_lake_test/ERP5Site_createTestDataSet</td>
<td></td>
</tr>
<tr>
<td>assertTextPresent</td>
<td>Dataset Created</td>
<td></td>
</tr>
<tal:block metal:use-macro="here/Zuite_CommonTemplate/macros/wait_for_activities" /> -->
<tr>
<td>open</td>
<td>${base_url}/web_site_module/default_wendelin_data_lake/</td>
<td></td>
</tr>
<tr>
<td>waitForElementPresent</td>
<td>//div[@data-gadget-url='${base_url}/web_site_module/default_wendelin_data_lake/gadget_erp5_page_home.html']</td>
<td></td>
</tr>
<!-- Check panel -->
<tr>
<td>waitForElementPresent</td>
<td>//div[@data-gadget-url='${base_url}/web_site_module/default_wendelin_data_lake/fif_gadget_erp5_panel.html']//a[text()='Home']</td>
<td></td>
</tr>
<tr>
<td>waitForElementPresent</td>
<td>//div[@data-gadget-url='${base_url}/web_site_module/default_wendelin_data_lake/fif_gadget_erp5_panel.html']//a[text()='Data Sets' and contains(@href, '#!display') and contains(@href, 'n.page=fifdata')]</td>
<td></td>
</tr>
<tr>
<td>waitForElementPresent</td>
<td>//div[@data-gadget-url='${base_url}/web_site_module/default_wendelin_data_lake/fif_gadget_erp5_panel.html']//a[text()='Register' and contains(@href, '#!display') and contains(@href, 'n.page=register')]</td>
<td></td>
</tr>
<tr>
<td>waitForElementPresent</td>
<td>//div[@data-gadget-url='${base_url}/web_site_module/default_wendelin_data_lake/fif_gadget_erp5_panel.html']//a[text()='Download Ebulk' and contains(@href, '#!display') and contains(@href, 'n.page=download')]</td>
<td></td>
</tr>
<tr>
<td>waitForElementPresent</td>
<td>//div[@data-gadget-url='${base_url}/web_site_module/default_wendelin_data_lake/fif_gadget_erp5_panel.html']//a[text()='Contact us' and contains(@href, '#!display') and contains(@href, 'n.page=contact')]</td>
<td></td>
</tr>
<tr>
<td>assertElementPresent</td>
<td>//div[@data-gadget-url='${base_url}/web_site_module/default_wendelin_data_lake/fif_gadget_erp5_panel.html']//a[text()='Logout' and contains(@href, '#!display') and contains(@href, 'n.page=logout')]</td>
<td></td>
</tr>
<!-- Check links -->
<tr>
<td>waitForElementPresent</td>
<td>//div[@data-gadget-url='${base_url}/web_site_module/default_wendelin_data_lake/gadget_erp5_page_home.html']//a[@id='home_link' and contains(@href, '#!display') and contains(@href, 'p.page=home')]</td>
<td></td>
</tr>
<tr>
<td>waitForElementPresent</td>
<td>//div[@data-gadget-url='${base_url}/web_site_module/default_wendelin_data_lake/gadget_erp5_page_home.html']//a[@id='download_link' and contains(@href, '#!display') and contains(@href, 'n.page=download')]</td>
<td></td>
</tr>
<tr>
<td>waitForElementPresent</td>
<td>//div[@data-gadget-url='${base_url}/web_site_module/default_wendelin_data_lake/gadget_erp5_page_home.html']//a[@id='download_ebulk_link' and contains(@href, '#!display') and contains(@href, 'n.page=download')]</td>
<td></td>
</tr>
<tr>
<td>waitForElementPresent</td>
<td>//div[@data-gadget-url='${base_url}/web_site_module/default_wendelin_data_lake/gadget_erp5_page_home.html']//a[@id='dataset_link' and contains(@href, '#!display') and contains(@href, 'n.page=fifdata')]</td>
<td></td>
</tr>
<tr>
<td>waitForElementPresent</td>
<td>//div[@data-gadget-url='${base_url}/web_site_module/default_wendelin_data_lake/gadget_erp5_page_home.html']//a[@id='register_link' and contains(@href, '#!display') and contains(@href, 'n.page=register')]</td>
<td></td>
</tr>
<tr>
<td>waitForElementPresent</td>
<td>//div[@data-gadget-url='${base_url}/web_site_module/default_wendelin_data_lake/gadget_erp5_page_home.html']//a[@id='documentation_link' and contains(@href, '#!display') and contains(@href, 'n.page=ebulk_doc')]</td>
<td></td>
</tr>
</tbody></table>
</body>
</html>
\ No newline at end of file
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_download_access_html.html
View file @ 448c9c9d
......@@ -10,8 +10,7 @@
<!-- renderjs -->
<script src="rsvp.js" type="text/javascript"></script>
<script src="renderjs.js" type="text/javascript"></script>
<!-- custom script
<script src="gadget_global.js" type="text/javascript"></script> -->
<!-- custom script -->
<script src="gadget_erp5_download_access.js" type="text/javascript"></script>
</head>
......@@ -19,15 +18,14 @@
<div class="download-access">
<form name='download-tool'>
<p class="bottom"><a href="#/?page=download">Download ebulk</a></p>
<p class="bottom"><a id="download_link">Download ebulk</a></p>
<p class="bottom"><b><a href="#/?page=register">Register</a></b> to get a fully functional user!</p>
<p class="bottom"><b><a id="register_link">Register</a></b> to get a fully functional user!</p>
<p class="bottom"> and run command: <b class="ident"> ebulk <span id="dataset_reference"/>push <i>my-data-set</i></b></p>
</form>
</div>
</body>
</html>
\ No newline at end of file
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_download_access_html.xml
View file @ 448c9c9d
......@@ -240,7 +240,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>977.1424.28928.61559</string> </value>
<value> <string>985.7487.30380.53572</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -258,7 +258,7 @@
</tuple>
<state>
<tuple>
<float>1562919500.08</float>
<float>1594222417.38</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_download_access_js.js
View file @ 448c9c9d
......@@ -3,22 +3,15 @@
(function (window, document, RSVP, rJS,
XMLHttpRequest, location, console) {
"use strict";
function download_tool(context, evt) {
var link = document.createElement('a');
link.href = window.location.origin + "/erp5/web_site_module/fif_data_runner/#/?page=download";
link.click();
}
rJS(window)
.declareAcquiredMethod("getUrlForList", "getUrlForList")
.allowPublicAcquisition('setFillStyle', function () {
return {
height: '100%',
width: '100%'
};
})
.declareJob('download_tool', function (evt) {
return download_tool(this, evt);
})
.declareMethod("render", function (reference) {
var html = "pull <i>" + reference + "</i>";
if (reference === undefined)
......@@ -26,14 +19,25 @@
return this.changeState({"dataset_reference" : html});
})
.declareService(function () {
var gadget = this,
url_parameter_list = [];
document.getElementById("dataset_reference").innerHTML = this.state.dataset_reference;
url_parameter_list.push({
command: 'display_stored_state',
options: {page: 'download'}
});
url_parameter_list.push({
command: 'display_stored_state',
options: {page: 'register'}
});
return gadget.getUrlForList(url_parameter_list)
.push(function (url_list) {
document.querySelector("#download_link").href = url_list[0];
document.querySelector("#register_link").href = url_list[1];
})
.onEvent('submit', function (evt) {
if (evt.target.name === 'download-tool') {
return this.download_tool(evt);
} else {
throw new Error('Unknown form');
}
.push(undefined, function (error) {
throw error;
});
});
}(window, document, RSVP, rJS,
XMLHttpRequest, location, console));
\ No newline at end of file
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_download_access_js.xml
View file @ 448c9c9d
......@@ -71,7 +71,9 @@
</item>
<item>
<key> <string>content_type</string> </key>
<value> <string>text/javascript</string> </value>
<value>
<none/>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
......@@ -240,7 +242,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>970.10968.63773.24814</string> </value>
<value> <string>985.7487.14593.50210</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -258,7 +260,7 @@
</tuple>
<state>
<tuple>
<float>1536571801.32</float>
<float>1594222385.74</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_header_html.html
View file @ 448c9c9d
......@@ -45,6 +45,11 @@
<h1 class="ui-title"></h1>
<div id="logged_in_div" class="ui-screen-hidden" style="padding-left: 24pt;">
<span>Logged In as :</span>
<span id="logged_in_user"/>
</div>
<div class="ui-controlgroup ui-controlgroup-horizontal ui-btn-right">
<div class="ui-controlgroup-controls">
</div>
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_header_html.xml
View file @ 448c9c9d
......@@ -240,7 +240,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>972.3959.36662.38195</string> </value>
<value> <string>985.18880.54279.1672</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -258,7 +258,7 @@
</tuple>
<state>
<tuple>
<float>1543842714.3</float>
<float>1594905686.52</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_header_js.js
View file @ 448c9c9d
/*jslint nomen: true, indent: 2, maxerr: 3 */
/*global window, rJS, Handlebars, document, RSVP */
/*global window, rJS, Handlebars, document, RSVP, jIO, URL */
(function (window, rJS, Handlebars, document, RSVP) {
"use strict";
......@@ -139,11 +139,7 @@
title_button_name: options.action
});
})
/*
.declareMethod('notifyUpdate', function () {
return this.render(this.stats.options);
})
*/
.declareMethod('render', function render(options) {
var state = {
error: false,
......@@ -161,6 +157,7 @@
right_button_icon: undefined,
right_button_name: undefined
},
gadget = this,
klass,
sub_header_list = [],
i;
......@@ -232,7 +229,17 @@
}
state.sub_header_list = sub_header_list;
return this.changeState(state);
return new RSVP.Queue()
.push(function () {
return jIO.util.ajax({
type: "GET",
url: new URL('./ERP5Site_getUserName', window.location.href)
});
})
.push(function (result) {
state.user = result.target.response;
return gadget.changeState(state);
});
})
.onStateChange(function onStateChange(modification_dict) {
......@@ -353,6 +360,12 @@
} else {
promise_list.push(null);
}
if (gadget.state.user) {
var logged_in_div = document.querySelector("#logged_in_div"),
logged_in_user = document.querySelector("#logged_in_user");
logged_in_div.classList.remove("ui-screen-hidden");
logged_in_user.innerHTML = gadget.state.user;
}
return new RSVP.Queue()
.push(function () {
......@@ -385,4 +398,4 @@
throw new Error("Unsupported button " + name);
});
}(window, rJS, Handlebars, document, RSVP));
\ No newline at end of file
}(window, rJS, Handlebars, document, RSVP, jIO, URL));
\ No newline at end of file
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_header_js.xml
View file @ 448c9c9d
......@@ -236,7 +236,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>969.59527.36796.52394</string> </value>
<value> <string>985.18881.59706.59153</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -254,7 +254,7 @@
</tuple>
<state>
<tuple>
<float>1536569733.07</float>
<float>1594905727.52</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_page_data_set_js.js
View file @ 448c9c9d
......@@ -25,10 +25,10 @@
.allowPublicAcquisition('updateHeader', function (argument_list) {
var header_dict = {
page_title: "Data Set : " + this.state.document_title,
selection_url: argument_list[0].selection_url,
selection_url: argument_list[0].selection_url
//next_url: argument_list[0].next_url,
//previous_url: argument_list[0].previous_url,
actions_url: argument_list[0].actions_url
//actions_url: argument_list[0].actions_url
};
return this.updateHeader(header_dict);
})
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_page_data_set_js.xml
View file @ 448c9c9d
......@@ -71,7 +71,9 @@
</item>
<item>
<key> <string>content_type</string> </key>
<value> <string>text/javascript</string> </value>
<value>
<none/>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
......@@ -244,7 +246,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>969.16316.40169.55057</string> </value>
<value> <string>983.63603.62266.8260</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -262,7 +264,7 @@
</tuple>
<state>
<tuple>
<float>1532962171.47</float>
<float>1594223527.37</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_page_file_js.js
View file @ 448c9c9d
......@@ -18,7 +18,7 @@
/////////////////////////////////////////////////////////////////
.allowPublicAcquisition('getUrlFor', function (argument_list) {
if (argument_list[0].command === 'change') {
if (argument_list[0].options.page == "action") {
if (argument_list[0].options.page === "action") {
return this.getUrlFor({command: 'change', options: {page: "fif_action"}});
}
}
......@@ -27,15 +27,24 @@
.allowPublicAcquisition('updateHeader', function (argument_list) {
var header_dict = {
page_title: "File : " + this.state.document_title,
selection_url: argument_list[0].selection_url,
selection_url: argument_list[0].selection_url
//next_url: argument_list[0].next_url,
//previous_url: argument_list[0].previous_url,
actions_url: argument_list[0].actions_url
//actions_url: argument_list[0].actions_url
};
return this.updateHeader(header_dict);
})
.onStateChange(function () {
var gadget = this;
var gadget = this,
akey,
table,
array,
i,
j,
newRow,
cell,
nkey,
side;
return gadget.getDeclaredGadget('form_view')
.push(function (form_gadget) {
var view_dict = {},
......@@ -54,30 +63,28 @@
};
group_list.push(["right", [["my_reference"]]]);
if (metadata !== undefined) {
if (Object.keys(metadata).length == 1) {
var akey = Object.keys(metadata)[0];
if (akey == "csv") {
var table = document.getElementById("csv_table"),
if (Object.keys(metadata).length === 1) {
akey = Object.keys(metadata)[0];
if (akey === "csv") {
table = document.getElementById("csv_table");
array = metadata[akey];
document.getElementById("text_content_title").innerHTML = "CSV file content sample:";
for (var i = 0; i < array.length; i++) {
var newRow = table.insertRow(table.length);
for (var j = 0; j < array[i].length; j++) {
var cell = newRow.insertCell(j);
for (i = 0; i < array.length; i++) {
newRow = table.insertRow(table.length);
for (j = 0; j < array[i].length; j++) {
cell = newRow.insertCell(j);
cell.innerHTML = array[i][j];
}
}
}
else {
} else {
document.getElementById("text_content_title").innerHTML = akey;
document.getElementById("text_content").innerHTML = metadata[akey];
}
}
else {
var nkey = 0;
} else {
nkey = 0;
for (var key in metadata) {
if (metadata.hasOwnProperty(key)) {
var side = (nkey % 2 === 0) ? "left" : "right";
side = (nkey % 2 === 0) ? "left" : "right";
view_dict[key] = {
"title": key,
"default": metadata[key],
......@@ -90,12 +97,12 @@
}
}
else {
view_dict["my_metadata"] = {
view_dict.my_metadata = {
"title": "Metadata",
"default": "Could not find metadata for this file",
"key": "field_my_metadata"
};
group_list.push(["left",[["my_metadata"]]])
group_list.push(["left", [["my_metadata"]]]);
}
return form_gadget.render({
erp5_document: {"_embedded": {"_view": view_dict},
......@@ -119,16 +126,17 @@
xmlHttp.send(null);
return xmlHttp.responseText;
}
catch(err) {
console.log("URL error: " + err)
catch (err) {
console.log("URL error: " + err);
return "";
}
})
.declareMethod("render", function (options) {
var gadget = this;
var gadget = this,
file_info;
return gadget.jio_get(options.jio_key)
.push(function (result) {
var file_info = result;
file_info = result;
return gadget.getDescriptorContent(file_info.reference)
.push(function (htmlContent) {
return gadget.changeState({"document_title" : file_info.title,
......@@ -139,13 +147,13 @@
})
.declareService(function () {
try {
var json_dict = JSON.parse(this.state.textcontent)
var json_dict = JSON.parse(this.state.textcontent);
return this.changeState({"metadata" : json_dict });
}
catch(err) {
console.log("Error reading Data Descriptor JSON: " + err)
return
catch (err) {
console.log("Error reading Data Descriptor JSON: " + err);
return;
}
return this.changeState({"metadata" : json_dict });
});
}(window, rJS, document, RSVP, calculatePageTitle, jIO));
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_page_file_js.xml
View file @ 448c9c9d
......@@ -71,7 +71,9 @@
</item>
<item>
<key> <string>content_type</string> </key>
<value> <string>text/javascript</string> </value>
<value>
<none/>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
......@@ -238,7 +240,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>984.982.33861.30037</string> </value>
<value> <string>985.7595.39398.9506</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -256,7 +258,7 @@
</tuple>
<state>
<tuple>
<float>1589986197.54</float>
<float>1594228616.34</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_page_home_html.html
View file @ 448c9c9d
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Content-type" content="text/html; charset=utf-8" />
<meta name="viewport" content="width=device-width, user-scalable=no" />
<title>BIG DATA SHARING PLATFORM</title>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>WENDELIN DATA LAKE SHARING PLATFORM - HOME</title>
<!-- styles -->
<link rel="stylesheet" href="fif_gadget_erp5.css">
<link rel="stylesheet" type="text/css" href="fif_gadget_erp5.css">
<!-- renderjs -->
<script src="rsvp.js" type="text/javascript"></script>
<script src="renderjs.js" type="text/javascript"></script>
<script src="rsvp.js"></script>
<script src="renderjs.js"></script>
<!-- custom script -->
<script src="gadget_global.js" type="text/javascript"></script>
<script src="gadget_erp5_page_home.js" type="text/javascript"></script>
<script src="gadget_erp5_page_home.js"></script>
</head>
</head>
<body>
<body>
<div class="homepage">
<div class="header">
<a href="#/?page=home">
<a id="home_link">
<img src="telecom-nexedi-logos?display=svg" alt="BigData Home">
</a>
<p>Wendelin Data Lake Sharing Platform</p>
......@@ -41,19 +39,19 @@
</div>
<h1>Data lake</h1>
<p>Dozens of public and private big data sets are available in the platform, terabytes of data of any kind, including binaries like medical images, ndarrays and more. Do you want to download data sets or share your data? <a href="#/?page=download">Download</a> our Ebulk tool to transfer big data! </p>
<p>Dozens of public and private big data sets are available in the platform, terabytes of data of any kind, including binaries like medical images, ndarrays and more. Do you want to download data sets or share your data? <a id="download_link">Download</a> our Ebulk tool to transfer big data! </p>
<div class="half vertical-child">
<p>See our full data set list!</p>
<a href="#/?page=fifdata">
<p><a id="dataset_link">See our full data set list!</a></p>
<a id="dataset_link_img">
<img src="dataset-icon?display=svg" alt="dataset icon">
</a>
<p><a href="#/?page=register">Register to get full functionality.</a></p>
<p><a id="register_link">Register to get full functionality.</a></p>
<p><span class="contact-link"></span></p>
</div>
<h1>Ebulk tool</h1>
<p>Ebulk tool is a wrapper for <a target="_blank" href="http://www.embulk.org/docs/">Embulk</a>, an open-source bulk data loader that helps data transfer between various databases, storages, file formats, and cloud services. It supports any kind of input file formats, parallel and distributed execution to deal with big data sets, transaction control to guarantee All-or-Nothing file transfer, and operation resuming. Ebulk is as easy as git to use, allowing the big data transfering to be done by using very few commands. Please, <a href="#/?page=download">download</a> Ebulk and check the <a href="#/?page=ebulk_doc">documentation</a>.</p>
<p>Ebulk tool is a wrapper for <a target="_blank" href="http://www.embulk.org/docs/">Embulk</a>, an open-source bulk data loader that helps data transfer between various databases, storages, file formats, and cloud services. It supports any kind of input file formats, parallel and distributed execution to deal with big data sets, transaction control to guarantee All-or-Nothing file transfer, and operation resuming. Ebulk is as easy as git to use, allowing the big data transfering to be done by using very few commands. Please, <a id="download_ebulk_link">download</a> Ebulk and check the <a id="documentation_link">documentation</a>.</p>
<h1>Wendelin</h1>
<p><a target="_blank" href="https://wendelin.nexedi.com/">Wendelin</a> is a big data framework designed for industrial applications based on python, NumPy, Scipy and other NumPy based libraries. It uses at its core the NEO distributed transactional NoSQL database to store petabytes of binary data. Wendelin combines the performance of scikit-learn machine learning with NEO distributed storage in order to provide out-of-core processing of large data sets. Its goal is to bring the best open source, big data engine based on Numpy python technologies and gather a wide community of contributors of new data analytics algorithms.</p>
......@@ -68,5 +66,5 @@
</ul>
</div>
</body>
</body>
</html>
\ No newline at end of file
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_page_home_html.xml
View file @ 448c9c9d
......@@ -238,7 +238,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>977.26217.31457.36573</string> </value>
<value> <string>985.7476.59155.53896</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -256,7 +256,7 @@
</tuple>
<state>
<tuple>
<float>1564407041.42</float>
<float>1594221622.21</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_page_home_js.js
View file @ 448c9c9d
/*global window, rJS, RSVP, URI */
/*global window, rJS, URI, document */
/*jslint nomen: true, indent: 2, maxerr: 3 */
(function (window, rJS, RSVP, document) {
(function (window, rJS, document) {
"use strict";
function data_lake(context, evt) {
var link = document.createElement('a');
link.href = window.location.origin + "/erp5/web_site_module/fif_data_runner/#/?page=fifdata";
link.click();
}
rJS(window)
.declareAcquiredMethod("updateHeader", "updateHeader")
.declareJob('data_lake', function (evt) {
return data_lake(this, evt);
.declareAcquiredMethod("getUrlForList", "getUrlForList")
.declareMethod("render", function (options) {
return this.changeState(options);
})
.declareMethod("render", function () {
var gadget = this;
return new RSVP.Queue()
.push(function () {
return gadget.updateHeader({
.onStateChange(function () {
return this.updateHeader({
page_title: 'Wendelin Data Lake Sharing Platform'
});
})
.declareService(function () {
var gadget = this,
url_parameter_list = [];
url_parameter_list.push({
command: 'display'
});
url_parameter_list.push({
command: 'display_stored_state',
options: {page: 'download'}
});
url_parameter_list.push({
command: 'display_stored_state',
options: {page: 'fifdata'}
});
url_parameter_list.push({
command: 'display_stored_state',
options: {page: 'register'}
});
url_parameter_list.push({
command: 'display_stored_state',
options: {page: 'ebulk_doc'}
});
return gadget.getUrlForList(url_parameter_list)
.push(function (url_list) {
document.querySelector("#home_link").href = url_list[0];
document.querySelector("#download_link").href = url_list[1];
document.querySelector("#download_ebulk_link").href = url_list[1];
document.querySelector("#dataset_link").href = url_list[2];
document.querySelector("#dataset_link_img").href = url_list[2];
document.querySelector("#register_link").href = url_list[3];
document.querySelector("#documentation_link").href = url_list[4];
})
.push(undefined, function (error) {
throw error;
});
})
.onEvent('submit', function (evt) {
if (evt.target.name === 'data-lake') {
return this.data_lake(evt);
} else {
throw new Error('Unknown form');
}
});
}(window, rJS, RSVP, document));
\ No newline at end of file
});
}(window, rJS, document));
\ No newline at end of file
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_page_home_js.xml
View file @ 448c9c9d
......@@ -234,7 +234,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>976.34131.824.20292</string> </value>
<value> <string>985.7597.28546.5683</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -252,7 +252,7 @@
</tuple>
<state>
<tuple>
<float>1561465018.46</float>
<float>1594228670.03</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_page_main_html.html
View file @ 448c9c9d
......@@ -3,7 +3,7 @@
<head>
<meta http-equiv="Content-type" content="text/html; charset=utf-8" />
<meta name="viewport" content="width=device-width, user-scalable=no" />
<title>TELECOM NEXEDI WENDELIN IA PROJECT</title>
<title>WENDELIN DATA LAKE SHARING PLATFORM - DATASET LIST</title>
<!-- renderjs -->
<script src="rsvp.js" type="text/javascript"></script>
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_page_main_html.xml
View file @ 448c9c9d
......@@ -240,7 +240,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>971.52031.55611.27340</string> </value>
<value> <string>985.7478.10187.22835</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -258,7 +258,7 @@
</tuple>
<state>
<tuple>
<float>1542881345.5</float>
<float>1594221604.04</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_page_main_js.js
View file @ 448c9c9d
......@@ -9,14 +9,6 @@
.declareMethod("render", function () {
var gadget = this;
return new RSVP.Queue()
/*.push(function () {
return gadget.jio_allDocs({
query: 'portal_type:"Data Array"' +
' AND validation_state:"validated"',
select_list: ["title", "reference"],
limit: [0, 1000000]
});
})*/
.push(function () {
return gadget.getDeclaredGadget("gadget_fif_page_list_dataset");
})
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_page_main_js.xml
View file @ 448c9c9d
......@@ -236,7 +236,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>976.41250.17185.10854</string> </value>
<value> <string>983.63603.62266.8260</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -254,7 +254,7 @@
</tuple>
<state>
<tuple>
<float>1561463216.84</float>
<float>1594221507.22</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_register_html.html
View file @ 448c9c9d
......@@ -4,7 +4,7 @@
<head>
<meta http-equiv="Content-type" content="text/html; charset=utf-8" />
<meta name="viewport" content="width=device-width, user-scalable=no" />
<title>Register</title>
<title>WENDELIN DATA LAKE SHARING PLATFORM - REGISTER</title>
<!-- styles -->
<link rel="stylesheet" href="fif_gadget_erp5.css">
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_register_html.xml
View file @ 448c9c9d
......@@ -238,7 +238,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>977.10261.58373.1809</string> </value>
<value> <string>985.7500.2038.6246</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -256,7 +256,7 @@
</tuple>
<state>
<tuple>
<float>1563449676.86</float>
<float>1594222815.79</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_register_js.rst
View file @ 448c9c9d
......@@ -21,9 +21,14 @@
// declared methods
/////////////////////////////////////////////////////////////////
//.declareMethod("render", function (options) {
//})
.declareMethod("render", function (options) {
return this.changeState(options);
})
.onStateChange(function () {
return this.updateHeader({
page_title: 'Register'
});
})
.onEvent('submit', function (evt) {
var gadget = this,
first_name = document.getElementById("first_name").value,
......@@ -34,7 +39,7 @@
confirm_password = document.getElementById("confirm_password").value;
if ((validateEmail(email)) && (first_name) && (last_name) && (reference) && (password) && (confirm_password) && (password === confirm_password) ) {
if ((validateEmail(email)) && (first_name) && (last_name) && (reference) && (password) && (confirm_password) && (password === confirm_password)) {
/* send to server */
var url = "ERP5Site_newCredentialRequest?batch_mode=1&reference=" + reference + "&default_email_text=" + email + "&first_name=" + first_name + "&last_name=" + last_name + "&password=" + password;
......@@ -66,7 +71,7 @@
page: "home"
}
});
})
});
} else {
return gadget.notifySubmitted({message: 'Misformatted email and / or passwords not matching. Please note that all fields are required!', status: 'error'});
}
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/fif_gadget_erp5_register_js.xml
View file @ 448c9c9d
......@@ -238,7 +238,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>977.10298.62565.9932</string> </value>
<value> <string>985.7506.43884.63931</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -256,7 +256,7 @@
</tuple>
<state>
<tuple>
<float>1563451951.09</float>
<float>1594223341.64</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/gadget_fif_page_list_dataset_html.html
View file @ 448c9c9d
......@@ -6,7 +6,7 @@
<head>
<meta http-equiv="Content-type" content="text/html; charset=utf-8" />
<meta name="viewport" content="width=device-width, user-scalable=no" />
<title>Fif Page List Data Set</title>
<title>WENDELIN DATA LAKE SHARING PLATFORM - DATASET LIST</title>
<!-- renderjs -->
<script src="rsvp.js" type="text/javascript"></script>
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/gadget_fif_page_list_dataset_html.xml
View file @ 448c9c9d
......@@ -240,7 +240,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>972.5384.26701.42410</string> </value>
<value> <string>983.63603.62266.8260</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -258,7 +258,7 @@
</tuple>
<state>
<tuple>
<float>1543928481.09</float>
<float>1594221691.76</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/gadget_fif_page_list_dataset_js.xml
View file @ 448c9c9d
......@@ -236,7 +236,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>984.959.26850.28842</string> </value>
<value> <string>984.52806.51617.21333</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -254,7 +254,7 @@
</tuple>
<state>
<tuple>
<float>1593095447.25</float>
<float>1594229222.71</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin_data_lake_ui/PathTemplateItem/web_page_module/gadget_fif_page_list_file_js.js
View file @ 448c9c9d
......@@ -41,7 +41,7 @@
})
.declareMethod("render", function (options) {
var gadget = this,
dataset = (options) ? options.reference : "",
dataset = options ? options.reference : "",
header_dict = {
page_title: 'Files',
filter_action: true
......
bt5/erp5_wendelin_data_lake_ui/TestTemplateItem/portal_components/test.erp5.testFunctionalDataLakeSite.py 0 → 100644
View file @ 448c9c9d
##############################################################################
#
# Copyright (c) 2002-2020 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
##############################################################################
import unittest
from Products.ERP5Type.tests.ERP5TypeFunctionalTestCase import ERP5TypeFunctionalTestCase
class TestDataLakeSite(ERP5TypeFunctionalTestCase):
foreground = 0
run_only = "wendelin_data_lake_ui_zuite"
def getBusinessTemplateList(self):
return (
'erp5_wendelin_data_lake_ui',
'erp5_ui_test_core',
'erp5_accounting',
'erp5_test_result',
)
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestDataLakeSite))
return suite
bt5/erp5_wendelin_data_lake_ui/TestTemplateItem/portal_components/test.erp5.testFunctionalDataLakeSite.xml 0 → 100644
View file @ 448c9c9d
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Test Component" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>default_reference</string> </key>
<value> <string>testFunctionalDataLakeSite</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>test.erp5.testFunctionalDataLakeSite</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Test Component</string> </value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>erp5</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_log</string> </key>
<value>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_data_lake_ui/bt/template_path_list
View file @ 448c9c9d
image_module/fif_*
portal_tests/wendelin_data_lake_ui_zuite
portal_tests/wendelin_data_lake_ui_zuite/**
web_page_module/*_wendelin_*
web_page_module/*fif_*
web_page_module/fif_*
......
bt5/erp5_wendelin_data_lake_ui/bt/template_test_id_list 0 → 100644
View file @ 448c9c9d
test.erp5.testFunctionalDataLakeSite
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7