Commit 1fbc4096 authored by Nikola Balog's avatar Nikola Balog

Added notebook for visualization of data

parent ac51e636
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Notebook" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Access_contents_information_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Change_local_roles_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>_View_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>content_md5</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>1</string> </value>
</item>
<item>
<key> <string>short_title</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content</string> </key>
<value> <string encoding="cdata"><![CDATA[
%% md\n
# Drone Simulation Results (best current simulation)\n
<div class="plot_div" id="plot_div"/>\n
\n
\n
%% fetch\n
js: jio.js\n
js: ndarray_bundle.js\n
js: wendelin.js\n
js: https://cdn.plot.ly/plotly-latest.min.js\n
js: https://cdn.jsdelivr.net/pyodide/v0.23.4/full/pyodide.js\n
\n
%% js\n
hateoas_url = "https://softinst195526.host.vifib.net/erp5/web_site_module/renderjs_runner/hateoas/";\n
jio = jIO.createJIO({\n
type: "erp5",\n
url: hateoas_url,\n
default_view_reference: "view"\n
});\n
\n
gadget = {\n
getSetting: function(property) {\n
return new RSVP.Queue()\n
.push(function () {\n
if (property == "hateoas_url") {\n
return hateoas_url;\n
}\n
return;\n
});\n
},\n
jio_getAttachment: function(id, url, parameter_dict) {\n
return jio.getAttachment(id, url, parameter_dict);\n
},\n
jio_get: function(id) {return jio.get(id);}\n
}\n
\n
\n
var graph = document.getElementById(\'plot_div\'),\n
graph_data = [],\n
label_list = ["ASML",\t"climb_rate",\t"distance",\t"ground_speed",\t"time"]; \n
\n
\n
\n
\n
function unpack(rows, key) {\n
return rows.map(function(row) { return row[key]; });\n
}\n
\n
// TODO we still need to manually type in the index range for the iteration we are interested in, because for some reason the data array lines are not ploted\n
function getData(){\n
var graph_data=[],\n
ratio = screen.availWidth / 1024,\n
interval,\n
frequency,\n
start_index = 0,\n
stop_index = 10000,\n
array_id;\n
\n
\n
return jio.allDocs({\n
query: \'portal_type:"Data Array" AND \' +\n
\'title: "Create Plot Datas"\'\n
})\n
.push(function (result) {\n
array_id = result.data.rows[0].id;\n
return wendelin.getArrayRawSlice(gadget, array_id, start_index, stop_index);\n
})\n
.push(function(result) {\n
for (i = 0; i < label_list.length; i += 1) {\n
graph_data = graph_data.concat(nj.unpack(result.pick( null, label_list[i])));\n
}\n
\n
return graph_data\n
})\n
.push(function (graph_data) {\n
var trace1 = {\n
type: "scattergl",\n
mode: "markers",\n
name: "distance",\n
x: unpack(graph_data, label_list.indexOf("time")),\n
y: unpack(graph_data, label_list.indexOf("distance")),\n
line: {color: \'#1f77b4\'}\n
};\n
\n
var trace2 = {\n
type: "scattergl",\n
mode: "markers",\n
name: \'ASML\',\n
x: unpack(graph_data, label_list.indexOf("time")),\n
y: unpack(graph_data, label_list.indexOf("ASML")),\n
line: {color: \'#ff7f0e\'}\n
}\n
\n
var data = [trace1, trace2];\n
return data\n
});\n
}\n
\n
\n
function plot () {\n
return getData()\n
.push(function (data) {\n
Plotly.newPlot(\n
graph,\n
data,\n
{ \'title\' :\'Distance and ASML over time\'\n
});\n
graph.on(\'plotly_relayout\', function(eventdata) {\n
var start_date = new Date(eventdata["xaxis.range[0]"]),\n
stop_date = new Date(eventdata["xaxis.range[1]"]);\n
return update_graph(start_date, stop_date);\n
});\n
});\n
}\n
\n
\n
plot();\n
\n
%% md\n
<div class="plot_div2" id="plot_div2"/>\n
\n
\n
%% js\n
\n
var graph = document.getElementById(\'plot_div2\');\n
function getData(){\n
var graph_data=[],\n
ratio = screen.availWidth / 1024,\n
interval,\n
frequency,\n
start_index,\n
stop_index,\n
array_id;\n
\n
return jio.allDocs({\n
query: \'portal_type:"Data Array" AND \' +\n
\'title: "Create Plot Datas"\'\n
})\n
.push(function (result) {\n
array_id = result.data.rows[0].id;\n
console.log(array_id);\n
return wendelin.getArrayRawSlice(gadget, array_id, 0, 1);\n
})\n
.push(function (result) {\n
start_index = 0,\n
stop_index = 100000;\n
\n
return wendelin.getArrayRawSlice(gadget, array_id, start_index, stop_index);\n
})\n
.push(function(result) {\n
for (i = 0; i < label_list.length; i += 1) {\n
graph_data = graph_data.concat(nj.unpack(result.pick( null, label_list[i])));\n
}\n
return graph_data\n
})\n
.push(function (graph_data) {\n
\n
var trace1 = {\n
type: "scatter",\n
mode: "markers",\n
name: "ground speed",\n
x: unpack(graph_data, label_list.indexOf("time")),\n
y: unpack(graph_data, label_list.indexOf("ground_speed")),\n
line: {color: \'#1f77b4\'}\n
};\n
\n
var trace2 = {\n
type: "scatter",\n
mode: "markers",\n
name: \'climb rate\',\n
x: unpack(graph_data, label_list.indexOf("time")),\n
y: unpack(graph_data, label_list.indexOf("climb_rate")),\n
line: {color: \'#ff7f0e\'}\n
}\n
\n
var data = [trace1, trace2];\n
return data\n
});\n
}\n
\n
\n
\n
function plot () {\n
return getData()\n
.push(function (data) {\n
Plotly.newPlot(\n
graph,\n
data,\n
{ \'title\' :\'Ground speed and climb rate over time\'\n
});\n
graph.on(\'plotly_relayout\', function(eventdata) {\n
var start_date = new Date(eventdata["xaxis.range[0]"]),\n
stop_date = new Date(eventdata["xaxis.range[1]"]);\n
return update_graph(start_date, stop_date);\n
});\n
});\n
}\n
\n
\n
plot();
]]></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>DroneSim</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
......@@ -46,23 +46,29 @@ def create_scorelist(scores):
def create_score_dataframe(scores, mean_lists, iteration=0):
#df = pd.DataFrame(columns=['Iteration', 'Sim Name', 'Parameters', 'Score', 'Distance mean', 'ASML mean', 'Ground Speed mean', 'Climb Rate mean'])
df = pd.DataFrame(columns=['Iteration', 'Parameters1', 'Parameters2', 'Score', 'Distance mean', 'ASML mean', 'Ground Speed mean', 'Climb Rate mean'])
def create_score_dataframe(scores, mean_lists):
df = pd.DataFrame(columns=['Sim_name', 'Parameters1', 'Parameters2', 'Score', 'Distance mean', 'ASML mean', 'Ground Speed mean', 'Climb Rate mean'])
for score in scores:
row=[int(iteration)]
row.extend([score[2][0],score[2][1], int(score[3])])
row=[]
row.extend([score[1],score[2][0],score[2][1], int(score[3])])
for data in mean_lists:
for i in data:
#print(i)
if i[1] == score[0]:
row.append(i[0])
break
context.log(row)
new_row = pd.DataFrame([row], columns=df.columns)
df = pd.concat([df, new_row], ignore_index=True)
return df
def create_plot_dataframe(plot_data):
df = pd.DataFrame({
"time": plot_data[0],
"distance":plot_data[1],
"ASML": plot_data[2],
"ground_speed": plot_data[3],
"climb_rate": plot_data[4]
})
return df
real_array = input_array_real["Data Array"]
sim_array = input_array_sim["Data Array"]
......@@ -179,21 +185,13 @@ timestamp_list = simulated_flights_value_dict_list[to_plot_index]["timestamp"]
score_list2 = create_scorelist(sorted_score_list)
zbigarray = out_array_scores["Data Array"].getArray()
if zbigarray is None:
iteration = 1
else:
try:
iteration = zbigarray[-1][1] + 1 # Look at the last iteration number
except:
iteration = 1
score_names = list(out_array_scores["Data Array"])
df = create_score_dataframe(score_list2, mean_list_list, iteration)
dtypes = {'index': 'i8', 'Iteration': 'i8', 'Parameters1': 'f8', 'Parameters2': 'f8',
df = create_score_dataframe(score_list2, mean_list_list)
dtypes = {'index': 'i8','Sim_name':'S256', 'Parameters1': 'f16', 'Parameters2': 'f16',
'Score': 'f8', 'Distance_mean': 'f8', 'ASML_mean': 'f8',
'Ground_Speed_mean': 'f8', 'Climb_Rate_mean': 'f8'}
......@@ -205,10 +203,105 @@ ndarray = df.to_records(column_dtypes=dtypes, index = False)
if zbigarray is None:
zbigarray = out_array_scores["Data Array"].initArray(shape=(0,), dtype=ndarray.dtype.fields)
score_array_start_idx = zbigarray.shape[0]
zbigarray.append(ndarray)
new_key = None
try:
max_nr = 0
for key in score_names:
old_key_nr = int(key.split("_")[1])
if old_key_nr>max_nr:
max_nr=old_key_nr
new_key = "Iteration_" + str(max_nr+1)
except:
new_key = "Iteration_1"
data_array_line_score = out_array_scores.get(new_key)
if data_array_line_score is None:
data_array_line_score = out_array_scores["Data Array"].newContent(id=new_key,
portal_type="Data Array Line")
data_array_line_score.edit(reference=new_key,
index_expression="%s:%s" %(score_array_start_idx, zbigarray.shape[0])
)
zbigarray2 = out_array_plot_data["Data Array"].getArray()
#Uncomment this to set the array to empty
#out_array_plot_data["Data Array"] = out_array_plot_data["Data Array"].initArray(shape=(0,), dtype=ndarray.dtype.fields)
things = [timestamp_list,
simulated_flights_value_dict_list[to_plot_index]["distance"],
simulated_flights_value_dict_list[to_plot_index]["ASML"],
simulated_flights_value_dict_list[to_plot_index]["ground speed"],
simulated_flights_value_dict_list[to_plot_index]["climb rate"]]
df = create_plot_dataframe(things)
dtypes = {'time': 'f8', 'distance': 'f8', 'ASML': 'f8',
'ground_speed': 'f8', 'climb_rate': 'f8'}
ndarray2 = df.to_records(column_dtypes=dtypes, index = False)
context.log(ndarray2)
if zbigarray2 is None:
context.log("zbigarray2 is none")
zbigarray2 = out_array_plot_data["Data Array"].initArray(shape=(0,), dtype=ndarray2.dtype.fields)
plot_array_start_idx = zbigarray2.shape[0]
zbigarray2.append(ndarray2)
new_key_plot = "PlotIteration_1"
plot_names = list(out_array_plot_data["Data Array"])
context.log(plot_names)
try:
context.log("OK")
max_nr = 0
for key in plot_names:
old_key_nr = int(key.split("_")[1])
if old_key_nr>max_nr:
max_nr=old_key_nr
new_key_plot = "PlotIteration_" + str(max_nr+1)
except:
context.log("h")
new_key_plot = "PlotIteration_1"
context.log(new_key_plot)
data_array_line_plot = out_array_plot_data.get(new_key_plot)
if data_array_line_plot is None:
data_array_line_plot = out_array_plot_data["Data Array"].newContent(id=new_key_plot,
portal_type="Data Array Line")
data_array_line_plot.edit(reference=new_key_plot,
index_expression="%s:%s" %(plot_array_start_idx, zbigarray2.shape[0])
)
if end > start:
progress_indicator_sim.setIntOffsetIndex(end)
......
This business template shows how to ingest the bucket stream data we get from the drone simulations.
\ No newline at end of file
Ingest simulation drone data, score the simulations and display some relevant information.
\ No newline at end of file
......@@ -12,6 +12,7 @@ data_supply_module/4
data_transformation_module/4
data_transformation_module/5
data_transformation_module/6
notebook_module/1
portal_callables/DataIngestionline_writeFluentdIngestionToDataBucketStream
portal_callables/convert_log_array_script
portal_callables/fitness_score_drones
\ No newline at end of file
......@@ -12,6 +12,7 @@ data_supply_module/4
data_transformation_module/4
data_transformation_module/5
data_transformation_module/6
notebook_module/1
portal_callables/DataIngestionline_writeFluentdIngestionToDataBucketStream
portal_callables/convert_log_array_script
portal_callables/fitness_score_drones
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment