Add trace-cmd tracing in the client

Don't clear trace at the beginning
Fix warnings and stop tracing when deadline is missed
Update run-client script to include trace-cmd
Remove useless latency threshold in client
Stop the client when deadline is missed
Add tracemark to client
Replace obsolete -G option with -P for the client

packet-exchange/src/client.c

@@ -41,7 +41,6 @@ typedef struct thread_param {
   unsigned int max_cycles;
   int priority;
   int etf_offset;
-  uint64_t latency_threshold;
 
 } thread_param_t;
 
@@ -92,7 +91,7 @@ static char send_data[MAX_BUFFER_SIZE];
 static void help(char *argv[]) {
   printf(
       "Usage: %s -f IF  [-abthgv] [-e ETF_OFFSET] [-d BUF_LEN] [-i USEC] [-l N]"
-      "[-p PRIO] [-q PACKET_PRIO] [-r USEC] [-T LATENCY_THRESHOLD -G]\n\n",
+      "[-p PRIO] [-q PACKET_PRIO] [-r USEC] [-T LATENCY_THRESHOLD -G]\n\n"
       "       -a                    Run the real time thread on CPU1\n"
       "       -b                    Measure RTT\n"
       "       -d BUF_LEN            Set the length of tx buffer\n"
@@ -112,10 +111,8 @@ static void help(char *argv[]) {
       "every USEC microseconds (Default: 50ms)\n"
       "       -t                    Enable timestamps\n"
       "       -v                    Verbose\n"
-      "       -T LATENCY_THRESHOLD  Enable tracing until LATENCY_THRESHOLD is "
-      "reached\n"
-      "       -G                    Enable function_graph tracer, used with "
-      "-T\n"
+      "       -T                    Enable tracing until deadline is "
+      "missed\n"
       "\n",
       argv[0]);
 }
@@ -146,9 +143,6 @@ static void *packet_sending_thread(void *p) {
     next_txtime = 0;
   }
 
-  // Start tracing
-  if (main_params.enable_tracing) tracing(1);
-
   clock_gettime(CLOCK_MONOTONIC, &next);
   clock_gettime(CLOCK_MONOTONIC, &measures_start);
 
@@ -177,13 +171,6 @@ static void *packet_sending_thread(void *p) {
           (nb_cycles + 1);
     }
 
-    // If the latency hits the tracing threshold, stop tracing
-    if (main_params.enable_tracing &&
-        (egress_stats.max_interval > thread_params.latency_threshold)) {
-      tracing(0);
-      break;
-    }
-
     previous = current;
 
     clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &next, NULL);
@@ -240,14 +227,14 @@ int main(int argc, char *argv[]) {
 
   if (main_params.enable_tracing) {
     // Enable ftrace
-    setup_tracer(main_params.enable_graph);
+    open_fds();
   }
 
   // Catch breaks with sighand to print the histograms
   init_signals(sighand, enable_histograms);
 
   // Initialize the UDP packet sending socket
-  init_udp_send(&egress_params, &egress_stats, enable_histograms,
+  init_udp_send(&egress_params, &egress_stats, enable_histograms, main_params.enable_tracing,
                 kernel_latency_hist);
 
   // Initialize the UDP packet receiving socket if RTT is measured
@@ -405,7 +392,7 @@ static void process_options(int argc, char *argv[]) {
   int network_if_specified = 0;
 
   for (;;) {
-    int c = getopt(argc, argv, "abd:e:f:ghi:l:p:q:r:tvT:G");
+    int c = getopt(argc, argv, "abd:e:f:ghi:l:p:q:r:tvT");
 
     if (c == -1) break;
 
@@ -461,10 +448,6 @@ static void process_options(int argc, char *argv[]) {
         break;
       case 'T':
         main_params.enable_tracing = 1;
-        thread_params.latency_threshold = atoi(optarg);
-        break;
-      case 'G':
-        main_params.enable_graph = 1;
         break;
     }
   }

packet-exchange/src/send_packet.c

@@ -37,6 +37,7 @@
 #include <unistd.h>
 
 #include "common.h"
+#include "tracer.h"
 
 static void *poll_thread(void *p);
 static void process_error_queue();
@@ -51,6 +52,7 @@ static egress_param_t *params;
 static egress_stat_t *stats;
 static uint64_t *kernel_latency_hist;
 static int use_histogram;
+static int stop_tracing;
 static uint64_t packets_sent = 0;
 
 static struct sock_txtime sk_txtime;
@@ -65,7 +67,7 @@ static int ts_buf_write_index = 0;
  * Init UDP socket
  */
 void init_udp_send(egress_param_t *_params, egress_stat_t *_stats,
-                   int _use_histogram, uint64_t *_kernel_latency_hist) {
+                   int _use_histogram, int _stop_tracing, uint64_t *_kernel_latency_hist) {
   int set_if_err;
   pthread_t thread;
 
@@ -73,6 +75,7 @@ void init_udp_send(egress_param_t *_params, egress_stat_t *_stats,
   stats = _stats;
   kernel_latency_hist = _kernel_latency_hist;
   use_histogram = _use_histogram;
+  stop_tracing = _stop_tracing;
 
   init_tx_buffer();
 
@@ -245,6 +248,11 @@ static void process_error_queue() {
           stats->invalid_parameter++;
           break;
         case SO_EE_CODE_TXTIME_MISSED:
+          if(stop_tracing) {
+            tracemark("Deadline missed\n");
+            tracing(0);
+            exit(EXIT_FAILURE);
+          }
           stats->missed_deadline++;
           break;
         default:

packet-exchange/src/send_packet.h

@@ -32,7 +32,7 @@ typedef struct egress_stat {
 
 void init_udp_send(egress_param_t *_params,
                    egress_stat_t *_stats,
-                   int _use_histogram,
+                   int _use_histogram, int _stop_tracing,
                    uint64_t *_kernel_latency_hist);
 
 void send_udp_packet(char *data, uint64_t txtime);

packet-exchange/src/server.c

@@ -80,7 +80,7 @@ static void help(char *argv[]) {
   printf(
       "Usage: %s [-aghtv] [-b CLIENT_IP] [-d BUF_LEN] [-f IF] [-i USEC] [-p "
       "PRIO] [-T LATENCY_THRESHOLD -G]"
-      " [-r USEC]\n\n",
+      " [-r USEC]\n\n"
       "       -a                    Run the real time thread on CPU1\n"
       "       -b CLIENT_IP          Server side RTT\n"
       "       -d BUF_LEN            Set the length of tx buffer\n"
@@ -165,7 +165,7 @@ static void *packet_receiving_thread(void *p) {
 
       // If the latency hits the tracing threshold, stop tracing
       if (main_params.enable_tracing &&
-          (ingress_stats.max_interval > thread_params.latency_threshold)) {
+          (ingress_stats.max_interval > ((int)thread_params.latency_threshold))) {
         sprintf(tracemark_message, "Jitter threshold hit: %dus\n", ingress_stats.max_interval);
         tracemark(tracemark_message);
         tracing(0);
@@ -225,7 +225,7 @@ int main(int argc, char *argv[]) {
 
   if (main_params.enable_tracing) {
     // Enable ftrace
-    setup_tracer(main_params.enable_graph);
+    open_fds();
   }
 
   // Catch breaks with sighand to print the histograms
@@ -237,7 +237,7 @@ int main(int argc, char *argv[]) {
 
   // Initialize the UDP packet sending socket if RTT is measured
   if (tsn_task == RTT_TASK)
-    init_udp_send(&egress_params, &egress_stats, 0, NULL);
+    init_udp_send(&egress_params, &egress_stats, 0, 0, NULL);
 
   /* Initialize pthread attributes (default values) */
   if (pthread_attr_init(&attr)) {
@@ -297,7 +297,7 @@ int main(int argc, char *argv[]) {
     }
 
     if (main_params.enable_tracing &&
-        (ingress_stats.max_interval >= thread_params.latency_threshold))
+        (ingress_stats.max_interval >= ((int)thread_params.latency_threshold)))
       break;
   }
 }

packet-exchange/src/tracer.c

@@ -75,22 +75,13 @@ static void setkernvar(const char *name, char *value) {
     fprintf(stderr, "could not set %s to %s\n", name, value);
 }
 
-void setup_tracer(int enable_graph) {
+void open_fds() {
+
+  fileprefix = debugfileprefix;
 
   char trace_path[MAX_PATH];
   char tracemark_path[MAX_PATH];
 
-  fileprefix = procfileprefix;
-  setkernvar("ftrace_enabled", "1");
-
-  fileprefix = debugfileprefix;
-  // Clear old traces by setting tracer to nop first
-  setkernvar("current_tracer", "nop");
-  if(enable_graph)
-    setkernvar("current_tracer", "function_graph");
-  else
-    setkernvar("current_tracer", "function");
-
   // Open tracing_on file
   strcpy(trace_path, fileprefix);
   strcat(trace_path, "tracing_on");
@@ -103,6 +94,23 @@ void setup_tracer(int enable_graph) {
   if ((tracemark_fd = open(tracemark_path, O_WRONLY)) == -1)
     printf("unable to open %s for tracing", tracemark_path);
 
+}
+
+void setup_tracer(int enable_graph) {
+
+  fileprefix = procfileprefix;
+  setkernvar("ftrace_enabled", "1");
+
+  fileprefix = debugfileprefix;
+  // Clear old traces by setting tracer to nop first
+  setkernvar("current_tracer", "nop");
+  if(enable_graph)
+    setkernvar("current_tracer", "function_graph");
+  else
+    setkernvar("current_tracer", "function");
+
+  open_fds();
+
   tracing(0);
 }
 

packet-exchange/src/tracer.h

@@ -2,6 +2,7 @@
 #define TRACER_H
 
 void setup_tracer(int enable_graph);
+void open_fds();
 void tracing(int on);
 void tracemark(char * s);
 

scripts/run-client

@@ -3,7 +3,16 @@
 script_dir=$(dirname $(realpath $0))
 
 usage() {
-  echo "Usage: $0 (-e delta [-o etf_offset] | -p) [-bgt] [-i INTERVAL] [-d TX_BUFFER_LEN] [-T LATENCY_THRESHOLD -G] [emerald|slate|onyx]" 1>&2;
+  cat << ENDUSAGE
+Usage: $0 QDISC_OPT [CLIENT_OPTS] BOARD
+   QDISC_OPTS: (-e delta [-o etf_offset] | -p)
+   CLIENT_OPTS: -bgt -i INTERVAL -d TX_BUF_LEN [TRACE_OPTS]
+   TRACE_OPTS: [-T -P TRACER -E EVENTS]
+   default tracer opts: irq, sched, net_dev_start_xmit, 
+                        net_dev_xmit, net_dev_xmit_timeout
+ENDUSAGE
+1>&2;
+
   exit 1;
 }
 
@@ -13,10 +22,11 @@ interval=100000
 # Default options
 client_options="-a -p 99 -f eth0"
 qdisc_options=""
-ip="10.100.21."
 etf_offset=500
+tracecmd_events="-e irq -e sched -e net_dev_start_xmit -e net_dev_xmit -e net_dev_xmit_timeout"
+tracecmd_opts=""
 
-while getopts "bd:e:o:ghi:ptT:G" opt; do
+while getopts "bd:e:o:ghi:ptTP:E:" opt; do
   case "${opt}" in
     b )
         client_options+=" -b"
@@ -53,10 +63,14 @@ while getopts "bd:e:o:ghi:ptT:G" opt; do
         client_options+=" -t"
       ;;
     T )
-        client_options+=" -T ${OPTARG}"
+        use_tracer=1
+        client_options+=" -T"
+      ;;
+    P )
+        tracecmd_opts+=" -p ${OPTARG}"
       ;;
-    G )
-        client_options+=" -G"
+    E )
+        tracecmd_events=${OPTARG}
       ;;
     * )
         usage
@@ -112,11 +126,14 @@ cd $script_dir/../packet-exchange/build;
 make client;
 cd $script_dir;
 
-if [ -z "${use_histogram}" ]; then
-  echo "client $client_options $board_ip";
-  $script_dir/../packet-exchange/build/client $client_options $board_ip;
-else
+if [ -n "${use_histogram}" ]; then
   echo "client $client_options $board_ip > $output;mv $output ~/";
   $script_dir/../packet-exchange/build/client $client_options $board_ip > $output;
   mv $output ~/;
+elif [ -n "${use_tracer}" ]; then
+  echo "trace-cmd record $tracecmd_opts $tracecmd_events ./client $client_options $board_ip";
+  trace-cmd record $tracecmd_opts $tracecmd_events $script_dir/../packet-exchange/build/client $client_options $board_ip;
+else
+  echo "client $client_options $board_ip";
+  $script_dir/../packet-exchange/build/client $client_options $board_ip;
 fi

scripts/run-server

@@ -38,10 +38,10 @@ while getopts "b:htd:i:g:T:G" opt; do
         server_options+=" -t"
         ;;
     T )
-        client_options+=" -T ${OPTARG}"
+        server_options+=" -T ${OPTARG}"
       ;;
     G )
-        client_options+=" -G"
+        server_options+=" -G"
       ;;
     * )
         usage