Add test-board-synchro

Clean up client and add comments
add send_tx_data option
Clean up server and add emit signal option
Delete line breaks between static variables
Update ptp script
Add run-phc2sys script
Add gettime program
Add test-board-synchro script

gettime/build/Makefile

+PROG = ptptime
+SRCDIR = ../src
+
+SRCS  = main.c
+
+OBJS = $(SRCS:%.c=%.o)
+
+ifeq ($(DEBUG),)
+CFLAGS = -O2
+else
+CFLAGS = -Og -g -Wall -Wextra
+endif
+
+CFLAGS += -MD -MP
+CFLAGS += -I $(SRCDIR)
+CFLAGS += -std=gnu99
+
+LLIBS = -pthread
+
+vpath %.c $(SRCDIR)
+
+$(PROG): $(OBJS)
+	$(CC) $(LDFLAGS) $(LDIRS) $^ $(LLIBS) -o $@
+
+-include $(subst .c,.d,$(SRCS))
+
+clean:
+	$(RM) $(OBJS) $(PROG) $(subst .c,.d,$(SRCS))
+
+.PHONY: clean

gettime/src/main.c

+/*
+ * Get PTP time
+ *
+ * Large portions taken from cyclictest
+ *
+ */
+
+#define _GNU_SOURCE
+#include <errno.h>
+#include <error.h>
+#include <inttypes.h>
+#include <limits.h>
+#include <pthread.h>
+#include <sched.h>
+#include <signal.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <time.h>
+#include <unistd.h>
+
+#define NSEC_PER_SEC UINT64_C(1000000000)
+
+// Structs
+
+typedef struct thread_param {
+  int interval;
+  int priority;
+
+} thread_param_t;
+
+// Static functions
+
+static void process_options(int argc, char *argv[]);
+static void add_ns(struct timespec *t, uint64_t ns);
+static uint64_t ts_to_uint(struct timespec t);
+
+// Static variables
+
+static thread_param_t thread_params;
+static uint64_t ptptime;
+static int quit;
+static clockid_t clock_id;
+
+static void help(char *argv[]) {
+  printf(
+      "Usage: %s [-h] [-i USEC]\n"
+      "       -h                 Show help\n"
+      "       -m                 Get time from CLOCK_MONOTONIC\n"
+      "       -r                 Get time from CLOCK_REALTIME (default)\n"
+      "       -t                 Get time from CLOCK_TAI\n"
+      "       -q                 Get time then quit\n"
+      "       -i USEC            RT thread wake-up interval (default: 10ms)\n"
+      "\n",
+      argv[0]);
+}
+
+/*
+ * Real-time thread: Sends packets at a regular intervall
+ */
+static void *packet_sending_thread(void *p) {
+  (void)p;
+  struct timespec next, ptptime_ts;
+  cpu_set_t mask;
+
+  // Set thread CPU affinity
+  CPU_ZERO(&mask);
+  CPU_SET(1, &mask);
+  if (sched_setaffinity(0, sizeof(mask), &mask))
+    error(EXIT_FAILURE, errno, "Could not set CPU affinity to CPU #1\n");
+
+  clock_gettime(CLOCK_MONOTONIC, &next);
+
+  // Packet sending loop
+  for (;;) {
+    clock_gettime(clock_id, &ptptime_ts);
+    ptptime = ts_to_uint(ptptime_ts);
+
+    add_ns(&next, thread_params.interval);
+
+    clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &next, NULL);
+  }
+
+  return NULL;
+}
+
+/*
+ * Main thread, has non-real time priority
+ * Handles the IO and creates the real time thread
+ */
+int main(int argc, char *argv[]) {
+  pthread_t thread;
+  struct sched_param param;
+  pthread_attr_t attr;
+
+  // Default configuration values
+  thread_params.interval = 100000 * 1000;
+  thread_params.priority = 99;
+  clock_id = CLOCK_REALTIME;
+
+  /* Lock all current and future pages from preventing of being paged to
+   * swap */
+  if (mlockall(MCL_CURRENT | MCL_FUTURE)) {
+    perror("mlockall failed");
+    /* exit(-1) or do error handling */
+  }
+
+  // Process bash options
+  process_options(argc, argv);
+
+  if (quit) {
+    struct timespec ts;
+    clock_gettime(clock_id, &ts);
+    printf("%" PRIu64 "\n", ts_to_uint(ts));
+    exit(EXIT_SUCCESS);
+  }
+
+  /* Initialize pthread attributes (default values) */
+  if (pthread_attr_init(&attr)) {
+    fprintf(stderr, "init pthread attributes failed\n");
+    exit(EXIT_FAILURE);
+  }
+  /* Set a specific stack size  */
+  if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN)) {
+    fprintf(stderr, "pthread setstacksize failed\n");
+    exit(EXIT_FAILURE);
+  }
+  /* Set scheduler policy and priority of pthread */
+  if (pthread_attr_setschedpolicy(&attr, SCHED_FIFO)) {
+    fprintf(stderr, "pthread setschedpolicy failed\n");
+    exit(EXIT_FAILURE);
+  }
+  param.sched_priority = thread_params.priority;
+  if (pthread_attr_setschedparam(&attr, &param)) {
+    fprintf(stderr, "pthread setschedparam failed\n");
+    exit(EXIT_FAILURE);
+  }
+  /* Use scheduling parameters of attr */
+  if (pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED)) {
+    fprintf(stderr, "pthread setinheritsched failed\n");
+    exit(EXIT_FAILURE);
+  }
+
+  // Create the real time thread
+  if (pthread_create(&thread, &attr, packet_sending_thread, NULL))
+    error(EXIT_FAILURE, errno, "Couldn't create packet sending thread");
+
+  // Verbose loop
+  for (;;) {
+    usleep(100000);
+    printf("%9" PRIu64 ": \n", ptptime);
+    printf("\033[%dA", 1);
+  }
+
+  exit(EXIT_SUCCESS);
+}
+
+// Process bash options
+static void process_options(int argc, char *argv[]) {
+
+  for (;;) {
+    int c = getopt(argc, argv, "hi:qmrt");
+
+    if (c == -1) break;
+
+    switch (c) {
+      case 'm':
+        clock_id = CLOCK_MONOTONIC;
+        break;
+      case 'r':
+        clock_id = CLOCK_REALTIME;
+        break;
+      case 't':
+        clock_id = CLOCK_TAI;
+        break;
+      case 'h':
+        help(argv);
+        exit(EXIT_SUCCESS);
+        break;
+      case 'i':
+        thread_params.interval = atoi(optarg) * 1000;
+        break;
+      case 'q':
+        quit = 1;
+        break;
+    }
+  }
+}
+
+static void add_ns(struct timespec *t, uint64_t ns) {
+  t->tv_nsec += ns;
+
+  while ((unsigned int)t->tv_nsec >= NSEC_PER_SEC) {
+    t->tv_sec += 1;
+    t->tv_nsec -= NSEC_PER_SEC;
+  }
+}
+static uint64_t ts_to_uint(struct timespec t) {
+  return t.tv_sec * NSEC_PER_SEC + t.tv_nsec;
+}

packet-exchange/build/Makefile

@@ -7,6 +7,7 @@ SERVER_SRCS  += recv_packet.c
 SERVER_SRCS  += send_packet.c
 SERVER_SRCS  += common.c
 SERVER_SRCS  += tracer.c
+SERVER_SRCS  += gpio.c
 
 CLIENT_SRCS  = client.c
 CLIENT_SRCS += recv_packet.c

packet-exchange/src/common.c

@@ -13,61 +13,66 @@ void (*previous_handlers[NSIG])(int);
 static void (*sighand)(int);
 
 uint64_t ts_to_uint(struct timespec t) {
-	return t.tv_sec * NSEC_PER_SEC + t.tv_nsec;
+  return t.tv_sec * NSEC_PER_SEC + t.tv_nsec;
+}
+struct timespec uint_to_ts(uint64_t t) {
+  struct timespec ts;
+  ts.tv_sec = t / NSEC_PER_SEC;
+  ts.tv_nsec = t - (ts.tv_sec * NSEC_PER_SEC);
+  return ts;
 }
 
 void add_ns(struct timespec *t, uint64_t ns) {
-	t->tv_nsec += ns;
+  t->tv_nsec += ns;
 
-	while ((unsigned int)t->tv_nsec >= NSEC_PER_SEC) {
-		t->tv_sec += 1;
-		t->tv_nsec -= NSEC_PER_SEC;
-	}
+  while ((unsigned int)t->tv_nsec >= NSEC_PER_SEC) {
+    t->tv_sec += 1;
+    t->tv_nsec -= NSEC_PER_SEC;
+  }
 }
 
 uint64_t calcdiff_ns(struct timespec t1, struct timespec t2) {
-	uint64_t diff;
-	diff = NSEC_PER_SEC * (uint64_t)((int)t1.tv_sec - (int)t2.tv_sec);
-	diff += ((int)t1.tv_nsec - (int)t2.tv_nsec);
-	return diff;
+  uint64_t diff;
+  diff = NSEC_PER_SEC * (uint64_t)((int)t1.tv_sec - (int)t2.tv_sec);
+  diff += ((int)t1.tv_nsec - (int)t2.tv_nsec);
+  return diff;
 }
 
 int _max_(int a, int b) { return a > b ? a : b; }
 int _min_(int a, int b) { return a < b ? a : b; }
 
 int histogram_min(uint64_t *histogram, int max_value) {
-	int ret = max_value;
-	for (int i = max_value; i >= 0; i--) ret = histogram[i] ? i : ret;
-	return ret;
+  int ret = max_value;
+  for (int i = max_value; i >= 0; i--) ret = histogram[i] ? i : ret;
+  return ret;
 }
 int histogram_max(uint64_t *histogram, int max_value) {
-	int ret = 0;
-	for (int i = 0; i <= max_value; i++) ret = histogram[i] ? i : ret;
-	return ret;
+  int ret = 0;
+  for (int i = 0; i <= max_value; i++) ret = histogram[i] ? i : ret;
+  return ret;
 }
 
 static void sighand_wrapper(int sig) {
-	// If we get un unexpected signal, report it, if not print the histogram
-	if (sig == SIGINT || sig == SIGTERM)
-		(*sighand)(sig);  // Will print the histogram
-	else
-		printf("Uknown signal interrupt: %s (%d)\n", strsignal(sig),
-		       sig);
+  // If we get un unexpected signal, report it, if not print the histogram
+  if (sig == SIGINT || sig == SIGTERM)
+    (*sighand)(sig);  // Will print the histogram
+  else
+    printf("Uknown signal interrupt: %s (%d)\n", strsignal(sig), sig);
 
-	// Execute the default handler
-	if (previous_handlers[sig] == SIG_DFL) {
-		signal(sig, SIG_DFL);
-		raise(sig);
-	} else if (previous_handlers[sig] == SIG_IGN) {
-		return;
-	} else {
-		(*previous_handlers[sig])(sig);
-	}
+  // Execute the default handler
+  if (previous_handlers[sig] == SIG_DFL) {
+    signal(sig, SIG_DFL);
+    raise(sig);
+  } else if (previous_handlers[sig] == SIG_IGN) {
+    return;
+  } else {
+    (*previous_handlers[sig])(sig);
+  }
 }
 
 void init_signals(void (*_sighand)(int)) {
-	sighand = _sighand;
+  sighand = _sighand;
 
-	for (int i = 0; i < NSIG; i++) signal(i, sighand_wrapper);
+  for (int i = 0; i < NSIG; i++) signal(i, sighand_wrapper);
 }
 

packet-exchange/src/common.h

@@ -39,6 +39,7 @@
 #define err_errno(...) error(EXIT_FAILURE, errno, __VA_ARGS__);
 
 uint64_t ts_to_uint(struct timespec t);
+struct timespec uint_to_ts(uint64_t t);
 void add_ns(struct timespec *t, uint64_t ns);
 uint64_t calcdiff_ns(struct timespec t1, struct timespec t2);
 

packet-exchange/src/gpio.c

+#include "gpio.h"
+#include <stdlib.h>
+#include <stdio.h>
+
+static int gpio_state;
+static char cmd[128];
+
+void enable_gpio(int gpio_index) {
+  sprintf(cmd, "echo %d > /sys/class/gpio/export", gpio_index);
+  system(cmd);
+  sprintf(cmd, "echo out > /sys/class/gpio/gpio%d/direction", gpio_index);
+  system(cmd);
+}
+void toggle_gpio(int gpio_index) {
+  sprintf(cmd, "echo %d > /sys/class/gpio/gpio%d/value", gpio_state, gpio_index);
+  system(cmd);
+
+  gpio_state = !gpio_state;
+}

packet-exchange/src/gpio.h

+#ifndef GPIO_H
+#define GPIO_H
+
+void enable_gpio(int gpio_index);
+void toggle_gpio(int gpio_index);
+
+#endif

packet-exchange/src/recv_packet.h

@@ -26,7 +26,6 @@ typedef struct ingress_stat {
 	uint64_t packets_received;
 	uint64_t high_kernel_latency;
 	uint64_t high_jitter;
-	int lost_packets;
 
 	char data[MAX_BUFFER_SIZE];
 } ingress_stat_t;

scripts/run-client

@@ -5,8 +5,8 @@ script_dir=$(dirname $(realpath $0))
 usage() {
   cat << ENDUSAGE
 Usage: $0 QDISC_OPT [CLIENT_OPTS] BOARD
-   QDISC_OPTS: (-e delta [-o etf_offset] -H | -p)
-   CLIENT_OPTS: -bgt -i INTERVAL -I if -d TX_BUF_LEN [TRACE_OPTS]
+   QDISC_OPTS: (-e delta [-o etf_offset] -H | -p) [-q]
+   CLIENT_OPTS: -bgt -c DELAY -s NS -i INTERVAL -I if -a CPU -d TX_BUF_LEN [TRACE_OPTS]
    TRACE_OPTS: [-T -P TRACER -E EVENTS -B SIZE]
    default tracer opts: irq, sched, net_dev_start_xmit, 
                         net_dev_xmit, net_dev_xmit_timeout
@@ -21,17 +21,24 @@ interval=100000
 
 # Default options
 interface="eth0"
-client_options="-a -p 99"
+cpu="1"
+client_options="-p 99"
 qdisc_options=""
 etf_offset=500
 tracecmd_events="-e irq -e sched -e net_dev_start_xmit -e net_dev_xmit -e net_dev_xmit_timeout"
 tracecmd_opts=""
 
-while getopts "bd:e:o:ghi:ptB:E:I:HP:T" opt; do
+while getopts "a:bc:d:e:o:ghi:pqs:tB:E:I:HP:T" opt; do
   case "${opt}" in
+    a )
+        cpu=${OPTARG}
+        ;;
     b )
         client_options+=" -b"
         ;;
+    c )
+        client_options+=" -c ${OPTARG}"
+        ;;
     d )
         client_options+=" -d ${OPTARG}"
         ;;
@@ -59,6 +66,12 @@ while getopts "bd:e:o:ghi:ptB:E:I:HP:T" opt; do
         client_options+=" -q 1"
         qdisc_options+="-p"
       ;;
+    q )
+        dont_create_qdisc=1
+      ;;
+    s )
+        client_options+=" -s ${OPTARG}"
+      ;;
     t )
         use_timestamps=1
         client_options+=" -t"
@@ -88,14 +101,15 @@ while getopts "bd:e:o:ghi:ptB:E:I:HP:T" opt; do
   esac
 done
 
+
 shift $((OPTIND-1))
 
 if [ -z "$1" ]; then
   usage
 fi
 
-client_options+=" -f $interface"
 qdisc_options+=" -I $interface"
+client_options+=" -a $cpu"
 
 board_name=$1
 
@@ -131,8 +145,10 @@ fi
 
 client_options+=" -i $interval"
 
-echo "create-qdisc $qdisc_options";
-$script_dir/create-qdisc $qdisc_options;
+if [ -z "$dont_create_qdisc" ]; then
+  echo "create-qdisc $qdisc_options";
+  $script_dir/create-qdisc $qdisc_options;
+fi
 
 echo "make client";
 cd $script_dir/../packet-exchange/build;
@@ -140,13 +156,13 @@ make client;
 cd $script_dir;
 
 if [ -n "${use_histogram}" ]; then
-  echo "client $client_options $board_ip > $output;mv $output ~/";
-  $script_dir/../packet-exchange/build/client $client_options $board_ip > $output;
+  echo "client $client_options $interface $board_ip > $output;mv $output ~/";
+  $script_dir/../packet-exchange/build/client $client_options $interface $board_ip > $output;
   mv $output ~/;
 elif [ -n "${use_tracer}" ]; then
-  echo "trace-cmd record $tracecmd_opts $tracecmd_events ./client $client_options $board_ip";
-  trace-cmd record $tracecmd_opts $tracecmd_events $script_dir/../packet-exchange/build/client $client_options $board_ip;
+  echo "trace-cmd record $tracecmd_opts $tracecmd_events ./client $client_options $interface $board_ip";
+  trace-cmd record $tracecmd_opts $tracecmd_events $script_dir/../packet-exchange/build/client $client_options $interface $board_ip;
 else
-  echo "client $client_options $board_ip";
-  $script_dir/../packet-exchange/build/client $client_options $board_ip;
+  echo "client $client_options $interface $board_ip";
+  $script_dir/../packet-exchange/build/client $client_options $interface $board_ip;
 fi

scripts/run-phc2sys

+#!/bin/bash
+
+linuxptp_dir=/home/oli/linuxptp
+
+opts=""
+interface="eth0"
+
+usage() {
+  echo "Usage: $0 [-r -i IF -o OPTS]" 1>&2;
+  exit 1;
+}
+
+while getopts "i:o:r" opt; do
+  case "${opt}" in
+    i )
+        interface=${OPTARG}
+        ;;
+    o )
+        opts+=${OPTARG}
+        ;;
+    r )
+        reset=1
+        ;;
+    * )
+        usage
+        ;;
+  esac
+done
+
+shift $((OPTIND-1))
+
+opts+="-s $interface"
+
+log=phc2sys_${interface}_log
+
+if [ -n "$reset" ]; then
+  killall phc2sys;
+fi
+echo "phc2sys $opts -c CLOCK_REALTIME --step_threshold=1 --transportSpecific=1 -w -m &> ~/$log &";
+phc2sys $opts -c CLOCK_REALTIME --step_threshold=1 --transportSpecific=1 -w -m &> ~/$log &

scripts/run-ptp4l

@@ -6,20 +6,30 @@ telecom_profile="G.8265.1.cfg"
 gPTP_profile="gPTP.cfg"
 profile_name="none"
 opts=""
+interface="eth0"
 
 usage() {
-  echo "Usage: $0 [-p telecom|gPTP|OTHER -s -o OPTS]" 1>&2;
+  echo "Usage: $0 [-r -p telecom|gPTP|OTHER -s -H -i IF -o OPTS]" 1>&2;
   exit 1;
 }
 
-while getopts "o:p:s" opt; do
+while getopts "i:o:p:rsH" opt; do
   case "${opt}" in
+    i )
+        interface=${OPTARG}
+        ;;
+    H )
+        opts+=" -H "
+        ;;
     o )
         opts+=${OPTARG}
         ;;
     p )
         profile_name=${OPTARG}
         ;;
+    r )
+        reset=1
+        ;;
     s )
         opts+=" -s "
         ;;
@@ -31,6 +41,8 @@ done
 
 shift $((OPTIND-1))
 
+opts+="-i $interface"
+
 if [ $profile_name == "none" ]; then
   opts+=""
 elif [ $profile_name == "telecom" ]; then
@@ -41,6 +53,10 @@ else
   opts=" -f $profile_name "
 fi
 
-killall ptp4l;
-echo "ptp4l $opts -i eth0 --step_threshold=1 -m -S &> ~/ptp4l_log &";
-ptp4l $opts -i eth0 --step_threshold=1 -m -S &> ~/ptp4l_log &
+log=ptp4l_${interface}_log
+
+if [ -n "$reset" ]; then
+  killall ptp4l;
+fi
+echo "ptp4l $opts --step_threshold=1 -m -S &> ~/$log &";
+ptp4l $opts --step_threshold=1 -m -S &> ~/$log &

scripts/run-server

@@ -9,7 +9,7 @@ usage() {
 usage() {
   cat << ENDUSAGE
 Usage: $0 [-I if] SERVER | TCPDUMP [TRACE_OPTS]
-   SERVER: [-bt] [(-x | -X) POLL] [-g INTERVAL]
+   SERVER: [-bct] [(-x | -X) POLL] [-g INTERVAL] [-a CPU]
    TCPDUMP: -d NB_PACKETS [-i INTERVAL]
    TRACE_OPTS: (-T LATENCY_THRESHOLD -G) | -E LATENCY_THRESHOLD
 ENDUSAGE
@@ -20,18 +20,25 @@ ENDUSAGE
 
 # Default options
 interface="eth0"
-server_options="-a -p 99"
+server_options="-p 99"
 make_opts=""
 ip="10.100.21."
 tcpdump_interval=1000000
 tracecmd_events="-e irq -e sched -e xdp -e net -e page_pool -e preemptirq -e napi"
+cpu=1
 
-while getopts "b:htx:X:d:i:g:I:T:GE:" opt; do
+while getopts "a:b:chtx:X:d:i:g:I:T:GE:" opt; do
   case "${opt}" in
+    a )
+        cpu=${OPTARG}
+        ;;
     b )
         use_rtt=1
         board_name=${OPTARG}
         ;;
+    c )
+        server_options+=" -c"
+        ;;
     d )
         use_tcpdump=1
         nb_packets=${OPTARG}
@@ -80,7 +87,7 @@ done
 
 shift $((OPTIND-1))
 
-server_options+=" -f $interface"
+server_options+=" -f $interface -a $cpu"
 
 if [ -n "${use_rtt}" ]; then
   if [ $board_name != "emerald" ] && [ $board_name != "onyx" ] && [ $board_name != "slate" ]; then

scripts/test-board-synchro

+#!/bin/bash
+
+usage() {
+  echo "Usage: $0 [-r -i IF -o OPTS]" 1>&2;
+  exit 1;
+}
+
+while getopts "i:o:r" opt; do
+  case "${opt}" in
+    i )
+        interface=${OPTARG}
+        ;;
+    o )
+        opts+=${OPTARG}
+        ;;
+    r )
+        reset=1
+        ;;
+    * )
+        usage
+        ;;
+  esac
+done
+
+shift $((OPTIND-1))
+
+# Setup irqs
+
+ksoftirq1_pid=$(ps -eL -o pid,cmd | grep "ksoftirqd/1" | awk '{ print $1 }' | head -n1)
+ksoftirq2_pid=$(ps -eL -o pid,cmd | grep "ksoftirqd/2" | awk '{ print $1 }' | head -n1)
+
+# Set high priority for ksoftirq deamons
+chrt -f -p 97 $ksoftirq1_pid;
+chrt -f -p 97 $ksoftirq2_pid;
+
+# Run linux-ptp
+
+echo "Starting ptp4l on master";
+
+killall ptp4l;
+./run-ptp4l -i enp1s0 -H;
+./run-ptp4l -i enp2s0 -H;
+
+echo "Starting ptp4l on slaves";
+
+./sudossh slate "killall ptp4l";
+./sudossh onyx "killall ptp4l";
+./sudossh slate "run-ptp4l -s";
+./sudossh onyx "run-ptp4l -s";
+
+echo "Waiting for ptp to take effect...";
+
+sleep 10;
+
+# Run phc2sys
+
+./run-phc2sys -i enp1s0;
+./run-phc2sys -i enp2s0;
+
+echo "Waiting for phc2sys to take effect...";
+
+sleep 2;
+
+echo "Start servers on slave";
+
+./sudossh slate "setup_irqs";
+./sudossh onyx "setup_irqs";
+
+./sudossh slate "killall server";
+./sudossh onyx "killall server";
+
+./exec-ssh-nohup slate "run-server -c";
+./exec-ssh-nohup onyx "run-server -c";
+
+cd ../gettime/build;
+make;
+
+time=$(./gettime -r -q)
+
+echo "CLOCK_REALTIME time is $time";
+
+time+=15000000000
+
+echo "Start clients on master";
+
+cd ../../scripts;
+
+./run-client -a 1 -q -i 1000000 -p -I enp1s0 -c 500000 -s $time slate &> client_enp1s0_log&
+./run-client -a 2 -q -i 1000000 -p -I enp2s0 -c 500000 -s $time onyx &> client_enp2s0_log&