erp5_wendelin_drone: automatically fetch the amount of parameters

bt5/erp5_wendelin_drone/PathTemplateItem/portal_callables/DataAnalysisLine_convertDroneDataArrayToScoreArray.py

@@ -74,32 +74,6 @@ seen_sims = progress_indicator_sim.getStringOffsetIndex()
 new_seen = ""
 if seen_sims is None:
   seen_sims = ""
-  score_dtypes = {'name': 'S256', 'Param1': 'f16', 'Param2': 'f16',
-          'distance_reciprocal': 'f8', 'ASML_reciprocal': 'f8',
-          'ground_speed_reciprocal': 'f8', 'climb_rate_reciprocal': 'f8',
-          'score_reciprocal': 'f16', 'score_cosine_row': 'f16',
-          'score_cosine_column': 'f16'}
-
-  _ = out_array_scores["Data Array"].initArray(shape=(0,), dtype=list(score_dtypes.items()))
-
-  plot_dtypes = {
-    'name': 'S256',
-    'Param1': 'f8',
-    'Param2': 'f8',
-    'timestamp': 'f8',
-    'distance_diff': 'f8',
-    'ASML_diff': 'f8',
-    'ground_speed_diff': 'f8',
-    'climb_rate_diff': 'f8',
-    'distance_reciprocal': 'f8',
-    'ASML_reciprocal': 'f8',
-    'ground_speed_reciprocal': 'f8',
-    'climb_rate_reciprocal': 'f8',
-    'score_reciprocal': 'f16',
-    'score_cosine_row': 'f16',
-    'score_cosine_column': 'f16'
-  }
-  _ = out_array_plot_data["Data Array"].initArray(shape=(0,), dtype=list(plot_dtypes.items()))
 
 simulated_flight_list = []
 simulated_flights_value_dict_list = []
@@ -135,11 +109,11 @@ for name in not_seen_list:
         break
 
       try:
-        under_timestamp = simulated_flight[simulated_flight['timestamp (ms)'] - timestamp_diff < row['timestamp (ms)']].tail(1)
+        under_timestamp = simulated_flight[simulated_flight['timestamp (ms)'] - timestamp_diff < row['timestamp (ms)']].tail(1).to_dict('records')[0]
       except IndexError:
         continue
-      over_timestamp = simulated_flight[simulated_flight['timestamp (ms)'] - timestamp_diff >= row['timestamp (ms)']].head(1)
-      rate = (over_timestamp["timestamp (ms)"] - timestamp_diff - row["timestamp (ms)"])/(over_timestamp["timestamp (ms)"] - under_timestamp["timestamp (ms)"])
+      over_timestamp = simulated_flight[simulated_flight['timestamp (ms)'] - timestamp_diff >= row['timestamp (ms)']].head(1).to_dict('records')[0]
+      rate = (over_timestamp['timestamp (ms)'] - timestamp_diff - row['timestamp (ms)'])/(over_timestamp['timestamp (ms)'] - under_timestamp['timestamp (ms)'])
 
       for criterion in distance_criterion_tuple:
         tmp_sim[criterion].append(rated_value(over_timestamp[criterion], under_timestamp[criterion], rate))
@@ -154,8 +128,9 @@ for name in not_seen_list:
             ),
             tmp_sim['AMSL (m)'][-1],
             tmp_sim['ground speed (m/s)'][-1],
-        tmp_real['climb rate (m/s)'][-1],
+            tmp_real['climb rate (m/s)'][-1]
         )):
+
         distance_list_tuple[index].append(value)
         # If we have at least 100 entries (timestamps) analysed, add the filename to the list
 
@@ -163,25 +138,19 @@ for name in not_seen_list:
     if len(distance_list_tuple[0]) > 0:
       # Add it to the dictionary
       # A list of simulations, and their values over time
-      pattern = r'\((.*?)\)'
-      match = re.search(pattern, name)
-      if match:
-        values_str = match.group(1)
-        parameters = [float(val.strip()) for val in values_str.split(',')]
-
+      parameters = {k: float(v) for _, k, v in re.findall(r'[_-](([a-zA-Z_]+)_(\d+))', name)}
+      if parameters:
         reciprocal_of_difference = [1/(1+mean_operation(list(map(abs, x)))) for x in distance_list_tuple[1:]]
         score_reciprocal = sum(reciprocal_of_difference)
 
         tmp_df_sim = pd.DataFrame(tmp_sim)
         tmp_df_real = pd.DataFrame(tmp_real)
         # Uncomment once you can actually use np..norm
-      score_cosine_row =mean_cosine_similarity_custom(tmp_df_real, tmp_df_sim)# cosine_similarity_rows(tmp_df_real, tmp_df_sim).mean()
+        score_cosine_row = mean_cosine_similarity_custom(tmp_df_real, tmp_df_sim)# cosine_similarity_rows(tmp_df_real, tmp_df_sim).mean()
         score_cosine_column = mean_cosine_similarity_custom(tmp_df_real.T, tmp_df_sim.T)#cosine_similarity_rows(tmp_df_real.T, tmp_df_sim.T).mean()
 
-      simulated_flights_value_dict_list.append({
+        flights_value_dict = {
           "name": name,
-            "Param1" : parameters[0],
-            "Param2": parameters[1],
           "timestamp": distance_list_tuple[0],
           "distance_diff": distance_list_tuple[1],
           "ASML_diff" : distance_list_tuple[2],
@@ -193,21 +162,24 @@ for name in not_seen_list:
           "climb_rate_reciprocal" : reciprocal_of_difference[3],
           "score_reciprocal": score_reciprocal/4,
           "score_cosine_row" : score_cosine_row,
-            "score_cosine_column" : score_cosine_column
-        })
+          "score_cosine_column" : score_cosine_column,
+        }
 
-      selected_simulation_data_dict_list.append({
+        simulation_data_dict = {
           "name": name,
-            "Param1" : parameters[0],
-            "Param2": parameters[1],
           "distance_reciprocal": reciprocal_of_difference[0],
           "ASML_reciprocal" : reciprocal_of_difference[1],
           "ground_speed_reciprocal" : reciprocal_of_difference[2],
           "climb_rate_reciprocal" : reciprocal_of_difference[3],
           "score_reciprocal": score_reciprocal/4,
           "score_cosine_row" : score_cosine_row,
-            "score_cosine_column" : score_cosine_column
-        })
+        }
+
+        for flight_parameter, value in parameters.items():
+          flights_value_dict[flight_parameter] = value
+          simulation_data_dict[flight_parameter] = value
+        simulated_flights_value_dict_list.append(flights_value_dict)
+        selected_simulation_data_dict_list.append(simulation_data_dict)
 
 
 combined_data = pd.DataFrame()  # Initialize an empty DataFrame
@@ -217,26 +189,7 @@ for data_dict in simulated_flights_value_dict_list:
     tmp = pd.DataFrame(data_dict)  # Wrap data_dict in a list and create a DataFrame
     combined_data = pd.concat([combined_data, tmp], ignore_index=True)
 
-
-
-plot_dtypes = {
-    'name': 'S256',
-    'Param1': 'float64',
-    'Param2': 'float64',
-    'timestamp': 'float64',
-    'distance_diff': 'float64',
-    'ASML_diff': 'float64',
-    'ground_speed_diff': 'float64',
-    'climb_rate_diff': 'float64',
-    'distance_reciprocal': 'float64',
-    'ASML_reciprocal': 'float64',
-    'ground_speed_reciprocal': 'float64',
-    'climb_rate_reciprocal': 'float64',
-    'score_reciprocal': 'float64',
-    'score_cosine_row': 'float64',
-    'score_cosine_column': 'float64',
-}
-
+plot_dtypes = {k: 'float64' if k != 'name' else 'S256' for k in simulated_flights_value_dict_list[0]}
 
 plots_df = combined_data
 if plots_df.empty:
@@ -261,11 +214,7 @@ for data_dict in selected_simulation_data_dict_list:
 
 
 
-score_dtypes = {'name': 'S256', 'Param1': 'float64', 'Param2': 'float64',
-          'distance_reciprocal': 'float64', 'ASML_reciprocal': 'float64',
-          'ground_speed_reciprocal': 'float64', 'climb_rate_reciprocal': 'float64',
-          'score_reciprocal': 'float64', 'score_cosine_row': 'float64',
-          'score_cosine_column': 'float64'}
+score_dtypes = {k: 'float64' if k != 'name' else 'S256' for k in selected_simulation_data_dict_list[0]}
 scores_df = pd.DataFrame(combined_data)
 scores_df = scores_df.astype(score_dtypes)
 ###############################
@@ -273,6 +222,9 @@ scores_df = scores_df.astype(score_dtypes)
 
 
 zbigarray_scores = out_array_scores["Data Array"].getArray()
+if zbigarray_scores is None:
+  out_array_scores["Data Array"].initArray(shape=(0,), dtype=list(score_dtypes.items()))
+  out_array_plot_data["Data Array"].initArray(shape=(0,), dtype=list(plot_dtypes.items()))
 
 score_names = list(out_array_scores["Data Array"])
 
@@ -281,7 +233,7 @@ score_names = list(out_array_scores["Data Array"])
 
 
 scores_ndarray = scores_df.to_records(index = False)
-scores_ndarray = scores_ndarray.astype([("name", "S256"),("Param1", "f8"), ("Param2", "f8"), ("distance_reciprocal", "f8"), ("ASML_reciprocal", "f8"), ("ground_speed_reciprocal", "f8"), ('climb_rate_reciprocal', 'f8'), ("score_reciprocal", "f8"), ("score_cosine_column", "f8"), ("score_cosine_row", "f8")])
+scores_ndarray = scores_ndarray.astype([(k, 'f8' if k != 'name' else 'S256') for k in score_dtypes])
 
 
 if zbigarray_scores is None:
@@ -330,7 +282,7 @@ zbigarray_plots = out_array_plot_data["Data Array"].getArray()
 
 
 plots_ndarray = plots_df.to_records(index = False)
-plots_ndarray = plots_ndarray.astype([('name', 'S256'), ('Param1', 'f8'), ('Param2', 'f8'), ('timestamp', 'f8'), ('distance_diff', 'f8'), ('ASML_diff', 'f8'), ('ground_speed_diff', 'f8'), ('climb_rate_diff', 'f8'), ('distance_reciprocal', 'f8'), ('ASML_reciprocal', 'f8'), ('ground_speed_reciprocal', 'f8'), ('climb_rate_reciprocal', 'f8'), ('score_reciprocal', 'f8'), ('score_cosine_row', 'f8'), ('score_cosine_column', 'f8')])
+plots_ndarray = plots_ndarray.astype([(k, 'f8' if k != 'name' else 'S256') for k in plot_dtypes])
 
 if zbigarray_plots is None:
   zbigarray_plots = out_array_plot_data["Data Array"].initArray(shape=(0,), dtype=plots_ndarray.dtype.fields)

bt5/erp5_wendelin_drone/PathTemplateItem/portal_callables/DataAnalysisLine_recalculateScore.py

@@ -8,148 +8,64 @@ import re
 # Now, to save space, we just overwrite the old array with the new one, that contains the new scores
 # This new array will give us the final overview of the ranking of the parameters. This can be used by our genetic algorithm to decide when we can stop and which parameters worked the best.
 
-
-score_dtypes = {'name': 'S256', 'Param1': 'f8', 'Param2': 'f8', 
-          'distance_reciprocal': 'f8', 'ASML_reciprocal': 'f8', 
-          'ground_speed_reciprocal': 'f8', 'climb_rate_reciprocal': 'f8', 
-          'score_reciprocal': 'f8', 'score_cosine_row': 'f8', 
-          'score_cosine_column': 'f8'}
-
-new_score_dtypes= {'name': 'S256', 'Param1': 'f8', 'Param2': 'f8', 
-          'distance_reciprocal': 'f8', 'ASML_reciprocal': 'f8', 
-          'ground_speed_reciprocal': 'f8', 'climb_rate_reciprocal': 'f8', 
-          'score_reciprocal': 'f8', 'score_cosine_row': 'f8', 
-          'score_cosine_column': 'f8',
-          'iteration': 'i8'}
-          
-    
-plot_dtypes = {
-    'name': 'S256',
-    'Param1': 'f8',
-    'Param2': 'f8',
-    'timestamp': 'f8',
-    'distance_diff': 'f8',
-    'ASML_diff': 'f8',
-    'ground_speed_diff': 'f8',
-    'climb_rate_diff': 'f8',
-    'distance_reciprocal': 'f8',
-    'ASML_reciprocal': 'f8',
-    'ground_speed_reciprocal': 'f8',
-    'climb_rate_reciprocal': 'f8',
-    'score_reciprocal': 'f8',
-    'score_cosine_row': 'f8',
-    'score_cosine_column': 'f8',
-    'iteration': 'i8'
-}
-
-new_plot_dtypes = {
-    'Param1': 'f8',
-    'Param2': 'f8',
-    'timestamp': 'f8',
-    'distance_diff': 'f8',
-    'ASML_diff': 'f8',
-    'ground_speed_diff': 'f8',
-    'climb_rate_diff': 'f8',
-    'distance_reciprocal': 'f8',
-    'ASML_reciprocal': 'f8',
-    'ground_speed_reciprocal': 'f8',
-    'climb_rate_reciprocal': 'f8',
-    'score_reciprocal': 'f8',
-    'score_cosine_row': 'f8',
-    'score_cosine_column': 'f8',
-    'iteration': 'i8'
-}
-
-
-score_array = input_array_scores["Data Array"]
-new_score_array = out_array_scores["Data Array"]
-
-plot_array = input_array_plot["Data Array"]
-new_plot_array = out_array_plots["Data Array"]
-
-# Should only look at the newest few
+score_array = input_array_scores['Data Array']
 score_nparray = score_array.getArray()
 if score_nparray is None:
-  score_nparray = out_array_scores["Data Array"].initArray(shape=(0,), dtype=list(new_score_dtypes.items()))
+  return
 
+plot_array = input_array_plot['Data Array']
 plot_nparray = plot_array.getArray()
 if plot_nparray is None:
-  plot_nparray = out_array_plots["Data Array"].initArray(shape=(0,), dtype=list(new_plot_dtypes.items()))
+  return
+
+new_score_dtypes = [(k, v[0]) for k, v in dict(score_nparray.dtype.fields).items()]
+new_score_dtypes.append(('iteration', 'i8'))
+new_score_array = out_array_scores['Data Array']
+new_score_nparray = new_score_array.getArray()
+if new_score_nparray is None:
+  new_score_array.initArray(shape=(0,), dtype=new_score_dtypes)
+
+new_plot_dtypes = [(k, v[0]) for k, v in dict(plot_nparray.dtype.fields).items()]
+new_plot_dtypes.append(('iteration', 'i8'))
+new_plot_array = out_array_plots['Data Array']
+new_plot_nparray = new_plot_array.getArray()
+if new_plot_nparray is None:
+  new_plot_array.initArray(shape=(0,), dtype=new_plot_dtypes)
 
 old_score_df = pd.DataFrame.from_records(score_nparray[:].copy())
 old_plot_df = pd.DataFrame.from_records(plot_nparray[:].copy())
 
-progress_indicator = input_array_scores["Progress Indicator"]
+progress_indicator = input_array_scores['Progress Indicator']
 seen_sims = progress_indicator.getStringOffsetIndex()
 
 if seen_sims is None:
   seen_sims = ""
 
-sim_flight_names = list(old_score_df["name"])
+sim_flight_names = list(old_score_df['name'])
 
 # We will only continue if there is new data available.
 if len([x for x in sim_flight_names if x not in seen_sims]) == 0:
   return
 
-
-
-
-
-
-
-
-
-new_score_nparray = new_score_array.getArray()
-if new_score_nparray is None:
-  new_score_nparray = out_array_scores["Data Array"].initArray(shape=(0,), dtype=list(new_score_dtypes.items()))
-
-
-
-new_plot_nparray = new_plot_array.getArray()
-if new_plot_nparray is None:
-  new_plot_nparray = out_array_plots["Data Array"].initArray(shape=(0,), dtype=list(new_plot_dtypes.items()))
-
-
-
-
 new_score_df = pd.DataFrame.from_records(new_score_nparray[:].copy())
-
 new_plot_df = pd.DataFrame.from_records(new_plot_nparray[:].copy())
 
-
-
-
-
-
-
-new_score_iteration = new_score_df["iteration"].max()
+new_score_iteration = new_score_df['iteration'].max()
 if math.isnan(new_score_iteration):
   new_score_iteration = 0
 
-
-new_score_df = new_score_df.drop(["iteration"],axis=1)
-
-
+new_score_df = new_score_df.drop(['iteration'], axis=1)
 new_plot_iteration = new_plot_df["iteration"].max()
 if math.isnan(new_plot_iteration):
   new_plot_iteration = 0
 
-#new_plot_df = new_plot_df.drop(columns=["iteration"])
-
-
-answer_scores = pd.concat([old_score_df, new_score_df]).drop_duplicates(subset=['Param1','Param2'], keep='last')
-answer_plots = pd.concat([old_plot_df, new_plot_df]).drop_duplicates(subset=['Param1','Param2'], keep='last')
-
-answer_scores["score_reciprocal"] = answer_scores["score_reciprocal"].astype('float64')
-
-
-
-answer_scores = answer_scores.nlargest(5, "score_reciprocal")
-answer_plots = old_plot_df[old_plot_df["name"].isin(list(answer_scores["name"]))]
-
-
-
+param_subset = [k for k in old_score_df.dtypes.keys().values if not 'reciprocal' in k and not 'score' in k and k != 'name']
+answer_scores = pd.concat([old_score_df, new_score_df]).drop_duplicates(subset=param_subset, keep='last')
+answer_plots = pd.concat([old_plot_df, new_plot_df]).drop_duplicates(subset=param_subset, keep='last')
 
+answer_scores['score_reciprocal'] = answer_scores['score_reciprocal'].astype('float64')
+answer_scores = answer_scores.nlargest(5, 'score_reciprocal')
+answer_plots = old_plot_df[old_plot_df['name'].isin(list(answer_scores['name']))]
 
 new_score_iteration = new_score_iteration + 1
 new_plot_iteration = new_plot_iteration + 1
@@ -158,24 +74,14 @@ answer_scores["iteration"] = new_score_iteration
 answer_plots["iteration"] = new_plot_iteration
 
 # We will remove all the data from the data arrays before we append the new data. Essentially we will be left with only the best few iterations
-new_score_nparray = out_array_scores["Data Array"].initArray(shape=(0,), dtype=list(new_score_dtypes.items()))
-
-new_plot_nparray = out_array_plots["Data Array"].initArray(shape=(0,), dtype=list(new_plot_dtypes.items()))
+new_score_nparray = new_score_array.initArray(shape=(0,), dtype=new_score_dtypes)
+new_plot_nparray = new_plot_array.initArray(shape=(0,), dtype=new_plot_dtypes)
 
 new_score_nparray.append(answer_scores.to_records(index = False))
 
 
-
-
-
-
-
-
-
-
-
 # Group the DataFrame by the 'name' column
-grouped = answer_plots.groupby(['Param1', 'Param2'])
+grouped = answer_plots.groupby(param_subset)
 
 # Initialize a list to store the resulting DataFrames
 resulting_dfs = []