WIP: rewrite and clean up code

packet-exchange/build/Makefile

@@ -5,12 +5,12 @@ SRCDIR = ../src
 SERVER_SRCS  = server.c
 SERVER_SRCS  += recv_packet.c
 SERVER_SRCS  += send_packet.c
-SERVER_SRCS  += utilities.c
+SERVER_SRCS  += common.c
 
 CLIENT_SRCS  = client.c
 CLIENT_SRCS += recv_packet.c
 CLIENT_SRCS += send_packet.c
-CLIENT_SRCS += utilities.c
+CLIENT_SRCS += common.c
 
 SERVER_OBJS = $(SERVER_SRCS:%.c=%.o)
 CLIENT_OBJS = $(CLIENT_SRCS:%.c=%.o)

packet-exchange/src/client.c

@@ -20,20 +20,17 @@
 
 #include "recv_packet.h"
 #include "send_packet.h"
-#include "utilities.h"
-
-#define MAX_KERNEL_LATENCY 1000
-#define MAX_RTT_LATENCY 1000
+#include "common.h"
 
 // Structs
 
-typedef struct egress_stat {
-  int nb_cycles;
-  uint64_t high_kernel_latency;
-  uint64_t invalid_parameter;
-  uint64_t missed_deadline;
-  char data[MAX_BUFFER_SIZE];
-} thread_stat_t;
+typedef struct rtt_stat {
+
+  int min_rtt;
+  int avg_rtt;
+  int max_rtt;
+
+} rtt_stat_t;
 
 typedef struct thread_param {
   int interval;
@@ -57,14 +54,16 @@ static void sighand(int sig_num);
 
 // Static variables
 
-static int64_t kernel_latencies[MAX_KERNEL_LATENCY]++;
-static int64_t rtt_latencies[MAX_RTT_LATENCY]++;
+static int64_t kernel_latency_hist[MAX_KERNEL_LATENCY]++;
+static int64_t rtt_hist[MAX_RTT_LATENCY]++;
+
+static int nb_cycles;
 
 static main_param_t main_params;
 static thread_param_t thread_params;
-
-static egress_stat * egress_stats;
-static egress_param * egress_params;
+static rtt_stat_t rtt_stats;
+static egress_stat_t * egress_stats;
+static egress_param_t * egress_params;
 
 static int enable_histograms;
 static int enable_affinity;
@@ -131,12 +130,12 @@ static void *packet_sending_thread(void *p) {
   clock_gettime(CLOCK_MONOTONIC, &next);
   clock_gettime(CLOCK_MONOTONIC, &measures_start);
   // Packet sending loop
-  for (egress_stats->nb_cycles = 0;; egress_stats->nb_cycles++) {
+  for (nb_cycles = 0;; nb_cycles++) {
     if (thread_params.max_cycles)
-      if (egress_stats->nb_cycles >= thread_params.max_cycles)
+      if (nb_cycles >= thread_params.max_cycles)
         break;
 
-    sprintf(send_data, "%d", egress_stats->nb_cycles % 1000);
+    sprintf(send_data, "%d", nb_cycles % 1000);
     do_tsn_task(send_data, next_txtime);
 
     add_ns(&next, thread_params.interval);
@@ -188,41 +187,41 @@ int main(int argc, char *argv[]) {
   init_signals(sighand, enable_histograms);
 
   // Initialize the UDP packet sending socket
-  init_udp_send(enable_etf, enable_timestamps,
-                network_config.packet_priority,
-                network_config.network_if,
-                network_config.tx_buffer_len);
+  init_udp_send(egress_params,
+                thread_params,
+                kernel_latency_hist);
 
   // Initialize the UDP packet receiving socket if RTT is measured
   if (tsn_task == RTT_TASK)
     init_udp_recv(0, network_config.network_if);
 
   // Create the real time thread
-  if (pthread_create(&thread, NULL, packet_sending_thread, (void *)param))
+  if (pthread_create(&thread, NULL, packet_sending_thread, NULL));
     error(EXIT_FAILURE, errno, "Couldn't create thread");
 
   // Verbose loop
   for (;;) {
-    usleep(main_param.refresh_rate);
+    usleep(main_params.refresh_rate);
 
-    if (main_param.verbose) {
+    if (main_params.verbose) {
       if (tsn_task == RTT_TASK) {
 
-        printf("%*d:   RTT: %*" PRIu64 "\n", 10, stats->nb_cycles, 10, stats->rtt);
+        printf("%10d:   RTT: %4d %4d %4d\n", nb_cycles, rtt_stats.min_rtt,
+                                                        rtt_stats.avg_rtt,
+                                                        rtt_stats.max_rtt);
         printf("\033[%dA", 1);
       } else if (enable_timestamps) {
 
-        int64_t user_space_time = stats->packet_info.userspace_exit_ts - stats->packet_info.userspace_enter_ts;
-        int64_t kernel_space_time = stats->packet_info.kernelspace_ts - stats->packet_info.userspace_exit_ts;
-
-        printf("%*d:   U: %*" PRIi64 ",   K: %*" PRIi64 "\n",
-               10, stats->nb_cycles, 10, user_space_time, 10, kernel_space_time);
+        printf("%10d:   K: %4d %4d %4d\n",
+               nb_cycles, egress_stats->min_kernel_latency,
+                          egress_stats->avg_kernel_latency,
+                          egress_stats->max_kernel_latency);
         printf("\033[%dA", 1);
       }
     }
 
-    if (param->max_cycles)
-      if (param->max_cycles == stats->nb_cycles)
+    if (thread_params.max_cycles)
+      if (thread_params.max_cycles == nb_cycles)
         break;
   }
 
@@ -233,35 +232,33 @@ int main(int argc, char *argv[]) {
 }
 
 // Critical TSN task
-static void do_tsn_task(struct thread_param *param, char *data, uint64_t next_txtime) {
+static void do_tsn_task(char *data, uint64_t next_txtime) {
   struct timespec t1, t2;
   int rtt_us;
 
   // One way packet sending
   if (tsn_task == SEND_PACKET_TASK) {
 
-    param->stats.packet_info = send_udp_packet(
-        enable_etf, enable_timestamps, data, next_txtime,
-        network_config.ip_address, histograms);
+    send_udp_packet(data, next_txtime);
 
     // Round Trip Time measurement
   } else if (tsn_task == RTT_TASK) {
 
     clock_gettime(CLOCK_MONOTONIC, &t1);
-    send_udp_packet(0, 0, data, next_txtime,
-                    network_config.ip_address, NULL);
+    send_udp_packet(data, next_txtime);
     recv_udp_packet(0, 0, NULL);
     clock_gettime(CLOCK_MONOTONIC, &t2);
 
-    param->stats.rtt = calcdiff_ns(t2, t1);
+    rtt_us = calcdiff_ns(t2, t1) / 1000;
 
-    if (enable_histograms) {
-      rtt_us = param->stats.rtt / 1000;
+    rtt_stats.min_rtt = min(rtt_us, rtt_stats.min_rtt);
+    rtt_stats.max_rtt = max(rtt_us, rtt_stats.max_rtt);
+    rtt_stats.avg_rtt = (((uint64_t)rtt_stats.avg_rtt) * (nb_cycles-1) + rtt_us) / nb_cycles;
 
-      if (rtt_us > MAX_HIST_VAL)
-        fprintf(stderr, "RTT value higher than MAX_HIST_VAL : %d ( > %d)\n", rtt_us, MAX_HIST_VAL);
+      if (rtt_us > MAX_RTT_VAL)
+        fprintf(stderr, "RTT value higher than MAX_RTT_VAL : %d ( > %d)\n", rtt_us, MAX_RTT_VAL);
       else
-        histograms[0][rtt_us]++;
+        rtt_hist[rtt_us]++;
     }
   }
 }
@@ -272,6 +269,7 @@ static void print_histograms() {
   uint64_t duration;
   int duration_hour, duration_minutes, interval;
   int max_hist_val, nb_hists;
+  uint64_t * histogram;
 
   clock_gettime(CLOCK_MONOTONIC, &measures_end);
 
@@ -291,7 +289,9 @@ static void print_histograms() {
            "\"i\": \"%dus\", \"duration\": \"%dh%d\", \"etf_offset\": \"%dus\","
            "},"
            "\"props\": [",
-           interval, duration_hour, duration_minutes, param->etf_offset);
+           interval, duration_hour, duration_minutes, thread_params.etf_offset);
+      histogram = kernel_latency_hist;
+
   } else if (tsn_task == RTT_TASK) {
     printf("{\"measure_sets\": [{"
            "\"measure_type\": \"packet_rtt\","
@@ -302,23 +302,20 @@ static void print_histograms() {
            "\"i\": \"%dus\", \"duration\": \"%dh%d\", \"etf_offset\": \"%dus\","
            "},"
            "\"props\": [",
-           interval, duration_hour, duration_minutes, param->etf_offset);
-  }
+           interval, duration_hour, duration_minutes, thread_params.etf_offset);
+      histogram = rtt_hist;
 
-  nb_hists = tsn_task == SEND_PACKET_TASK ? 2 : 1;
+  }
 
   max_hist_val = 0;
-  for (int i = 0; i < nb_hists; i++)
-    for (int j = 0; j < MAX_HIST_VAL; j++)
-      if (histograms[i][j])
-        max_hist_val = j > max_hist_val ? j : max_hist_val;
-
-  for (int i = 0; i < nb_hists; i++) {
-    printf("[");
-    for (int j = 0; j < max_hist_val; j++)
-      printf("%" PRIi64 "%s", histograms[i][j], (j + 1 < max_hist_val ? ", " : ""));
-    printf("%s", (i + 1 < nb_hists ? "], " : "]"));
-  }
+  for (int j = 0; j < MAX_HIST_VAL; j++)
+    if (histogram[j])
+      max_hist_val = j > max_hist_val ? j : max_hist_val;
+
+  printf("[");
+  for (int j = 0; j < max_hist_val; j++)
+    printf("%" PRIi64 "%s", histogram[j], (j + 1 < max_hist_val ? ", " : ""));
+  printf("%s]");
 
   printf("]}]}\n");
 }
@@ -348,19 +345,19 @@ static void process_options(int argc, char *argv[]) {
       tsn_task = RTT_TASK;
       break;
     case 'd':
-      network_config.tx_buffer_len = atoi(optarg);
-      if (network_config.tx_buffer_len < 1) {
+      egress_params->tx_buffer_len = atoi(optarg);
+      if (egress_params->tx_buffer_len < 1) {
         fprintf(stderr, "BUF_LEN should be greater than 1\n");
         exit(EXIT_FAILURE);
       }
       break;
     case 'e':
       enable_etf = 1;
-      param->etf_offset = atoi(optarg) * 1000;
+      thread_params.etf_offset = atoi(optarg) * 1000;
       break;
     case 'f':
       network_if_specified = 1;
-      strcpy(network_config.network_if, optarg);
+      strcpy(egress_params->network_if, optarg);
       break;
     case 'g':
       enable_histograms = 1;
@@ -370,25 +367,25 @@ static void process_options(int argc, char *argv[]) {
       exit(EXIT_SUCCESS);
       break;
     case 'i':
-      param->interval = atoi(optarg) * 1000;
+      thread_params.interval = atoi(optarg) * 1000;
       break;
     case 'l':
-      param->max_cycles = atoi(optarg);
+      thread_params.max_cycles = atoi(optarg);
       break;
     case 'p':
-      param->priority = atoi(optarg);
+      thread_params.priority = atoi(optarg);
       break;
     case 'q':
-      network_config.packet_priority = atoi(optarg);
+      egress_params->packet_priority = atoi(optarg);
       break;
     case 'r':
-      main_param.refresh_rate = atoi(optarg);
+      main_params.refresh_rate = atoi(optarg);
       break;
     case 't':
       enable_timestamps = 1;
       break;
     case 'v':
-      main_param.verbose = 1;
+      main_params.verbose = 1;
       break;
     }
   }
@@ -407,5 +404,5 @@ static void process_options(int argc, char *argv[]) {
     help(argv);
     exit(EXIT_FAILURE);
   }
-  strcpy(network_config.ip_address, argv[optind]);
+  strcpy(egress_params->ip_address, argv[optind]);
 }

packet-exchange/src/utilities.c → packet-exchange/src/common.c

@@ -7,7 +7,7 @@
 #include <time.h>
 #include <unistd.h>
 
-#include "utilities.h"
+#include "common.h"
 
 void (*previous_handlers[NSIG])(int);
 static void (*sighand)(int);

packet-exchange/src/utilities.h → packet-exchange/src/common.h

@@ -11,8 +11,10 @@
 #define NSEC_PER_SEC UINT64_C(1000000000)
 #define SERVER_PORT "50000"
 #define SERVER_PORT_INT 50000
-#define MAX_HIST_VAL 2000
-#define NB_HISTOGRAMS 3
+
+#define MAX_KERNEL_LATENCY 1000
+#define MAX_RTT_LATENCY 1000
+
 #define MAX_BUFFER_SIZE 1024
 
 typedef struct packet_info {

packet-exchange/src/recv_packet.c

@@ -26,9 +26,7 @@
 #include <unistd.h>
 
 #include "send_packet.h"
-#include "utilities.h"
-
-static void fill_histograms(packet_info_t *packet_info, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]);
+#include "common.h"
 
 static char rx_buffer[MAX_BUFFER_SIZE];
 static int sock_fd;
@@ -37,11 +35,11 @@ static int so_timestamping_flags =
     SOF_TIMESTAMPING_RX_SOFTWARE | SOF_TIMESTAMPING_SOFTWARE;
 
 // Sets the interface
-static int set_if(char *network_if) {
+static int set_if() {
   struct ifreq ifreq;
 
   memset(&ifreq, 0, sizeof(ifreq));
-  strncpy(ifreq.ifr_name, network_if, sizeof(ifreq.ifr_name) - 1);
+  strncpy(ifreq.ifr_name, ingress_params->network_if, sizeof(ifreq.ifr_name) - 1);
 
   if (ioctl(sock_fd, SIOCGIFINDEX, &ifreq))
     error(EXIT_FAILURE, errno, "ioctl SIOCGIFINDEX failed\n");
@@ -49,11 +47,17 @@ static int set_if(char *network_if) {
   return ifreq.ifr_ifindex;
 }
 
-int init_udp_recv(int use_timestamps, char *network_if) {
+void init_udp_recv(struct ingress_param * _params,
+              struct thread_param * _thread_params,
+              uint64_t * _kernel_latency_hist) {
   int getaddrinfo_err;
   int set_if_err;
   struct addrinfo hints, *servinfo, *servinfo_it;
 
+  params = _params; 
+  thread_params = _thread_params; 
+  kernel_latency_hist = _kernel_latency_hist; 
+
   memset(&hints, 0, sizeof hints);
   hints.ai_family = AF_UNSPEC;
   hints.ai_socktype = SOCK_DGRAM;
@@ -82,21 +86,20 @@ int init_udp_recv(int use_timestamps, char *network_if) {
   if (sock_fd == -1)
     error(EXIT_FAILURE, errno, "Couldn't create receive socket");
 
-  set_if_err = set_if(network_if);
+  set_if_err = set_if(params->network_if);
   if (set_if_err < 0)
     error(EXIT_FAILURE, errno, "Couldn't set interface\n");
 
-  if (setsockopt(sock_fd, SOL_SOCKET, SO_BINDTODEVICE, network_if,
-                 strlen(network_if)))
+  if (setsockopt(sock_fd, SOL_SOCKET, SO_BINDTODEVICE, params->network_if,
+                 strlen(params->network_if)))
     error(EXIT_FAILURE, errno, "setsockopt SO_BINDTODEVICE failed\n");
 
-  if (use_timestamps) {
+  if (params->use_timestamps) {
     if (setsockopt(sock_fd, SOL_SOCKET, SO_TIMESTAMPING, &so_timestamping_flags,
                    sizeof(so_timestamping_flags)))
       error(EXIT_FAILURE, errno, "setsockopt SO_TIMESTAMPING failed\n");
   }
 
-  return sock_fd;
 }
 
 /*

packet-exchange/src/recv_packet.h

@@ -6,4 +6,17 @@
 int init_udp_recv(int use_timestamps, char *network_if);
 packet_info_t recv_udp_packet(int use_timestamps, int use_histograms, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]);
 
+void init_udp_recv(ingress_param_t * params,
+              thread_param_t * thread_params,
+              uint64_t * kernel_latency_hist);
+void recv_udp_packet(int nb_cycles);
+
+
+typedef struct ingress_param {
+  char network_if[16];
+
+  int use_timestamps;
+
+} ingress_param_t;
+
 #endif

packet-exchange/src/send_packet.c

@@ -35,7 +35,7 @@
 #include <unistd.h>
 
 #include "send_packet.h"
-#include "utilities.h"
+#include "common.h"
 
 static void * poll_thread(void *p);
 static void process_error_queue();
@@ -46,7 +46,7 @@ static int set_if();
 static int so_timestamping_flags =
     SOF_TIMESTAMPING_TX_SOFTWARE | SOF_TIMESTAMPING_SOFTWARE;
 
-struct egress_param * params;
+egress_param_t * params;
 struct thread_param * thread_params;
 uint64_t * kernel_latency_hist;
 int use_histogram;
@@ -62,17 +62,17 @@ static int ts_buf_write_index = 0;
 /*
  * Init UDP socket
  */
-init_udp_send(struct egress_param * _params,
-              struct thread_param * _thread_params,
+void init_udp_send(egress_param_t * _params,
+              thread_param_t * _thread_params,
               int _use_histogram,
-              uint64_t * _kernel_latency) {
+              uint64_t * _kernel_latency_hist) {
 
   int set_if_err;
   pthread_t thread;
 
   params = _params;
   thread_params = _thread_params;
-  kernel_latency = _kernel_latency;
+  kernel_latency_hist = _kernel_latency_hist;
   use_histogram = _use_histogram
 
   init_tx_buffer();
@@ -116,7 +116,7 @@ init_udp_send(struct egress_param * _params,
  * Sends udp packets
  */
 void send_udp_packet(char *data,
-                     uint64_t txtime) {
+                     uint64_t txtime, int nb_cycles) {
 
   struct msghdr msg;                             // Message hardware, sent to the socket
   struct cmsghdr *cmsg;                          // Control message hardware, for txtime
@@ -126,7 +126,7 @@ void send_udp_packet(char *data,
   struct sockaddr_in sin; // Server address
   struct timespec ts;     // timestamp for userspace timestamping
 
-  if (use_timestamps) {
+  if (params->use_timestamps) {
     clock_gettime(CLOCK_REALTIME, &ts);
     timestamps_buffer[ts_buf_write_index++] = ts_to_uint(ts);
   }
@@ -139,7 +139,7 @@ void send_udp_packet(char *data,
   sin.sin_port = htons(SERVER_PORT_INT);
 
   iov.iov_base = tx_buffer;
-  iov.iov_len = tx_buffer_len;
+  iov.iov_len = params->tx_buffer_len;
 
   memset(&msg, 0, sizeof(msg));
   msg.msg_name = &sin;
@@ -147,7 +147,7 @@ void send_udp_packet(char *data,
   msg.msg_iov = &iov;
   msg.msg_iovlen = 1;
 
-  if (use_etf) {
+  if (params->use_etf) {
     // We specify the transmission time in the CMSG.
     msg.msg_control = control;
     msg.msg_controllen = sizeof(control);
@@ -176,12 +176,12 @@ static void * poll_thread(void *p) {
 
     ret = poll(&poll_fd, 1, -1);
     if (ret == 1 && p_fd.revents & POLLERR) {
-      process_error_queue();
+      process_error_queue(nb_cycles);
     }
   }
 }
 
-static void process_error_queue() {
+static void process_error_queue(int nb_cycles) {
   // IO vector
   unsigned char data_buffer[256]; // Buffer in io vector
   struct iovec iov = {
@@ -216,10 +216,16 @@ static void process_error_queue() {
       uint64_t kernel_latency = ts_to_uint(stamp) - timestamps_buffer[ts_buf_read_index++];
       kernel_latency /= 1000u;
 
-      if (kernel_latency > MAX_HIST_VAL)
-        stats.high_kernel_latency++;
-      else
-        kernel_latencies[kernel_latency]++;
+      egress_stats->min_kernel_latency = min(kernel_latency, egress_stats->min_kernel_latency);
+      egress_stats->max_kernel_latency = max(kernel_latency, egress_stats->max_kernel_latency);
+      egress_stats->avg_kernel_latency = (egress_stats->max_kernel_latency * (nb_cycles-1) + kernel_latency) / nb_cycles;
+
+      if (use_histogram) {
+        if (kernel_latency > MAX_KERNEL_LATENCY)
+          stats.high_kernel_latency++;
+        else
+          kernel_latency_hist[kernel_latency]++;
+      }
 
     }
     // If an error was received

packet-exchange/src/send_packet.h

 #ifndef SEND_PACKET_H
 #define SEND_PACKET_H
 
-#include "utilities.h"
+#include "common.h"
 
-init_udp_send(struct egress_param * _params,
-              struct thread_param * _thread_params,
-              uint64_t kernel_latency[MAX_KERNEL_LATENCY]);
-void send_udp_packet(char *data, uint64_t txtime);
+void init_udp_send(egress_param_t * params,
+              thread_param_t * thread_params,
+              uint64_t * kernel_latency_hist);
+void send_udp_packet(char *data, uint64_t txtime, int nb_cycles);
 
-struct egress_param {
+typedef struct egress_param {
   int packet_priority;
   size_t tx_buffer_len;
   char server_ip[45];
@@ -17,9 +17,19 @@ struct egress_param {
   int use_etf;
   int use_timestamps;
 
+} egress_param_t;
+
+typedef struct egress_stat {
+
+  uint64_t high_kernel_latency;
+  uint64_t invalid_parameter;
+  uint64_t missed_deadline;
+
   int min_kernel_latency;
   int avg_kernel_latency;
   int max_kernel_latency;
-};
+
+  char data[MAX_BUFFER_SIZE];
+} thread_stat_t;
 
 #endif

packet-exchange/src/server.c

@@ -26,7 +26,7 @@
 
 #include "recv_packet.h"
 #include "send_packet.h"
-#include "utilities.h"
+#include "common.h"
 
 // Structs