Skip to content

T
tsn-measures
Commit d495022a authored by Joanne Hugé's avatar Joanne Hugé
Browse files
Options

Setup the RT thread differently 

Setup the RT thread's priority, scheduling, etc... as suggested on the RT wiki
Add multissh and sudossh scripts
Add high jitter packets to lost packets when printing histogram
Fix wrapper scripts using obsolete script name

Pass attributes to the pthread
Change interval unit display
Add ftrace option
Add interval computations in client
Update wrapper scripts
Remove DEBUG_TS
parent ded97060
Hide whitespace changes
Inline Side-by-side
Showing with 520 additions and 207 deletions
img-kernel-utils/scripts/configure-board
View file @ d495022a
......@@ -7,7 +7,7 @@ usage() {
partition_number=2
while getopts "h" opt; do
while getopts "hp:" opt; do
case "${opt}" in
h )
usage
......
packet-exchange/build/Makefile
View file @ d495022a
......@@ -6,11 +6,13 @@ SERVER_SRCS = server.c
SERVER_SRCS += recv_packet.c
SERVER_SRCS += send_packet.c
SERVER_SRCS += common.c
SERVER_SRCS += tracer.c
CLIENT_SRCS = client.c
CLIENT_SRCS += recv_packet.c
CLIENT_SRCS += send_packet.c
CLIENT_SRCS += common.c
CLIENT_SRCS += tracer.c
SERVER_OBJS = $(SERVER_SRCS:%.c=%.o)
CLIENT_OBJS = $(CLIENT_SRCS:%.c=%.o)
......
packet-exchange/src/client.c
View file @ d495022a
......@@ -16,12 +16,14 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <time.h>
#include <unistd.h>
#include "common.h"
#include "recv_packet.h"
#include "send_packet.h"
#include "tracer.h"
// Structs
......@@ -39,12 +41,15 @@ typedef struct thread_param {
unsigned int max_cycles;
int priority;
int etf_offset;
uint64_t latency_threshold;
} thread_param_t;
typedef struct main_param {
int refresh_rate;
int verbose;
int enable_tracing;
int enable_graph;
} main_param_t;
// Static functions
......@@ -67,7 +72,8 @@ static egress_param_t egress_params;
static ingress_param_t ingress_params;
static rtt_stat_t rtt_stats = {.min_rtt = INT_MAX};
static egress_stat_t egress_stats = {.min_kernel_latency = INT_MAX};
static egress_stat_t egress_stats = {.min_kernel_latency = INT_MAX,
.min_interval = INT_MAX};
static ingress_stat_t ingress_stats = {.min_kernel_latency = INT_MAX,
.min_interval = INT_MAX};
......@@ -85,35 +91,33 @@ static char send_data[MAX_BUFFER_SIZE];
static void help(char *argv[]) {
printf(
"Usage: %s -f IF [-abthgv] [-e ETF_OFFSET] [-d BUF_LEN] [-i USEC] [-l "
"N] [-p PRIO] [-q PACKET_PRIO] [-r USEC]\n\n",
argv[0]);
printf(" -a Run the real time thread on CPU1\n");
printf(" -b Measure RTT\n");
printf(" -d BUF_LEN Set the length of tx buffer\n");
printf(
"Usage: %s -f IF [-abthgv] [-e ETF_OFFSET] [-d BUF_LEN] [-i USEC] [-l N]"
"[-p PRIO] [-q PACKET_PRIO] [-r USEC] [-T LATENCY_THRESHOLD -G]\n\n",
" -a Run the real time thread on CPU1\n"
" -b Measure RTT\n"
" -d BUF_LEN Set the length of tx buffer\n"
" -e ETF_OFFSET Set a txtime with an offset of ETF_OFFSET "
"us (to be used in an ETF qdisc)\n");
printf(" -f IF Set the network interface to be used\n");
printf(" -g Print histograms to sdtout on exit\n");
printf(" -h Show help\n");
printf(
"us (to be used in an ETF qdisc)\n"
" -f IF Set the network interface to be used\n"
" -g Print histograms to sdtout on exit\n"
" -h Show help\n"
" -i USEC Wake up the real time thread every USEC "
"microseconds (Default: 10ms)\n");
printf(
"microseconds (Default: 10ms)\n"
" -l N Wake up the real time thread N times "
"(Default: 0)\n");
printf(
"(Default: 0)\n"
" -p PRIO Run the real time thread at priority "
"PRIO\n");
printf(
" -q PACKET_PRIO Send packets with PACKET_PRIO priority\n");
printf(
"PRIO\n"
" -q PACKET_PRIO Send packets with PACKET_PRIO priority\n"
" -r USEC Refresh the non real time main thread "
"every USEC microseconds (Default: 50ms)\n");
printf(" -t Enable timestamps\n");
printf(" -v Verbose\n");
printf("\n");
"every USEC microseconds (Default: 50ms)\n"
" -t Enable timestamps\n"
" -v Verbose\n"
" -T LATENCY_THRESHOLD Enable tracing until LATENCY_THRESHOLD is "
"reached\n"
" -G Enable function_graph tracer, used with "
"-T\n"
"\n",
argv[0]);
}
/*
......@@ -121,9 +125,8 @@ static void help(char *argv[]) {
*/
static void *packet_sending_thread(void *p) {
(void)p;
struct timespec next;
struct timespec next, current, previous;
uint64_t next_txtime;
struct sched_param priority;
cpu_set_t mask;
// Set thread CPU affinity
......@@ -134,11 +137,6 @@ static void *packet_sending_thread(void *p) {
error(EXIT_FAILURE, errno, "Could not set CPU affinity to CPU #1\n");
}
// Set thread priority
priority.sched_priority = thread_params.priority;
if (sched_setscheduler(0, SCHED_FIFO, &priority))
error(EXIT_FAILURE, errno, "Couldn't set priority");
if (enable_etf) {
// Measure from CLOCK_TAI to generate timestamp
clock_gettime(CLOCK_TAI, &next);
......@@ -148,6 +146,9 @@ static void *packet_sending_thread(void *p) {
next_txtime = 0;
}
// Start tracing
if (main_params.enable_tracing) tracing(1);
clock_gettime(CLOCK_MONOTONIC, &next);
clock_gettime(CLOCK_MONOTONIC, &measures_start);
......@@ -157,6 +158,8 @@ static void *packet_sending_thread(void *p) {
nb_cycles >= ((unsigned int)thread_params.max_cycles))
break;
clock_gettime(CLOCK_MONOTONIC, &current);
sprintf(send_data, "%d", (int)(nb_cycles % 1000));
do_tsn_task(send_data, next_txtime);
......@@ -164,6 +167,25 @@ static void *packet_sending_thread(void *p) {
if (enable_etf) next_txtime += thread_params.interval;
if (nb_cycles) {
int interval_us = calcdiff_ns(current, previous) / 1000;
egress_stats.min_interval = min(interval_us, egress_stats.min_interval);
egress_stats.max_interval = max(interval_us, egress_stats.max_interval);
egress_stats.avg_interval =
(egress_stats.avg_interval * nb_cycles + interval_us) /
(nb_cycles + 1);
}
// If the latency hits the tracing threshold, stop tracing
if (main_params.enable_tracing &&
(egress_stats.max_interval > thread_params.latency_threshold)) {
tracing(0);
break;
}
previous = current;
clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &next, NULL);
}
......@@ -176,6 +198,8 @@ static void *packet_sending_thread(void *p) {
*/
int main(int argc, char *argv[]) {
pthread_t thread;
struct sched_param param;
pthread_attr_t attr;
// Default configuration values
thread_params.interval = 100000 * 1000;
......@@ -184,6 +208,8 @@ int main(int argc, char *argv[]) {
main_params.refresh_rate = 50000;
main_params.verbose = 0;
main_params.enable_tracing = 0;
main_params.enable_graph = 0;
enable_affinity = 0;
enable_etf = 0;
......@@ -194,6 +220,12 @@ int main(int argc, char *argv[]) {
egress_params.packet_priority = 3;
egress_params.tx_buffer_len = 1024;
/* Lock all current and future pages from preventing of being paged to swap */
if (mlockall(MCL_CURRENT | MCL_FUTURE)) {
perror("mlockall failed");
/* exit(-1) or do error handling */
}
// Process bash options
process_options(argc, argv);
......@@ -206,6 +238,11 @@ int main(int argc, char *argv[]) {
memset(rtt_hist, 0, sizeof(rtt_hist));
}
if (main_params.enable_tracing) {
// Enable ftrace
setup_tracer(main_params.enable_graph);
}
// Catch breaks with sighand to print the histograms
init_signals(sighand, enable_histograms);
......@@ -218,8 +255,34 @@ int main(int argc, char *argv[]) {
init_udp_recv(&ingress_params, &ingress_stats, enable_histograms,
kernel_latency_hist);
/* Initialize pthread attributes (default values) */
if (pthread_attr_init(&attr)) {
fprintf(stderr, "init pthread attributes failed\n");
exit(EXIT_FAILURE);
}
/* Set a specific stack size */
if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN)) {
fprintf(stderr, "pthread setstacksize failed\n");
exit(EXIT_FAILURE);
}
/* Set scheduler policy and priority of pthread */
if (pthread_attr_setschedpolicy(&attr, SCHED_FIFO)) {
fprintf(stderr, "pthread setschedpolicy failed\n");
exit(EXIT_FAILURE);
}
param.sched_priority = thread_params.priority;
if (pthread_attr_setschedparam(&attr, &param)) {
fprintf(stderr, "pthread setschedparam failed\n");
exit(EXIT_FAILURE);
}
/* Use scheduling parameters of attr */
if (pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED)) {
fprintf(stderr, "pthread setinheritsched failed\n");
exit(EXIT_FAILURE);
}
// Create the real time thread
if (pthread_create(&thread, NULL, packet_sending_thread, NULL))
if (pthread_create(&thread, &attr, packet_sending_thread, NULL))
error(EXIT_FAILURE, errno, "Couldn't create packet sending thread");
// Verbose loop
......@@ -232,9 +295,11 @@ int main(int argc, char *argv[]) {
rtt_stats.avg_rtt, rtt_stats.max_rtt);
printf("\033[%dA", 1);
} else {
printf("%9" PRIu64 ": [%4d, %4d]", nb_cycles,
printf("%9" PRIu64 ": [%4d, %4d], I (10us): %3d %3d %3d", nb_cycles,
(int)egress_stats.invalid_parameter,
(int)egress_stats.missed_deadline);
(int)egress_stats.missed_deadline,
egress_stats.min_interval / 10, egress_stats.avg_interval / 10,
egress_stats.max_interval / 10);
if (enable_timestamps) {
printf(", K: %4d %4d %4d [%4d]\n", egress_stats.min_kernel_latency,
......@@ -340,7 +405,7 @@ static void process_options(int argc, char *argv[]) {
int network_if_specified = 0;
for (;;) {
int c = getopt(argc, argv, "abd:e:f:ghi:l:p:q:r:tv");
int c = getopt(argc, argv, "abd:e:f:ghi:l:p:q:r:tvT:G");
if (c == -1) break;
......@@ -394,6 +459,13 @@ static void process_options(int argc, char *argv[]) {
case 'v':
main_params.verbose = 1;
break;
case 'T':
main_params.enable_tracing = 1;
thread_params.latency_threshold = atoi(optarg);
break;
case 'G':
main_params.enable_graph = 1;
break;
}
}
......
packet-exchange/src/common.h
View file @ d495022a
......@@ -18,7 +18,7 @@
#define MAX_BUFFER_SIZE 1024
#define TIMESTAMP_BUFFER_SIZE 64
#define TIMESTAMP_BUFFER_SIZE 4096
uint64_t ts_to_uint(struct timespec t);
void add_ns(struct timespec *t, uint64_t ns);
......
packet-exchange/src/send_packet.c
View file @ d495022a
......@@ -7,6 +7,8 @@
*/
#define _GNU_SOURCE
#include "send_packet.h"
#include <arpa/inet.h>
#include <errno.h>
#include <error.h>
......@@ -35,7 +37,6 @@
#include <unistd.h>
#include "common.h"
#include "send_packet.h"
static void *poll_thread(void *p);
static void process_error_queue();
......@@ -56,18 +57,15 @@ static struct sock_txtime sk_txtime;
static char *tx_buffer;
static int sock_fd;
static uint64_t timestamps_buffer[TIMESTAMP_BUFFER_SIZE];
static int64_t timestamps_buffer[TIMESTAMP_BUFFER_SIZE];
static int ts_buf_read_index = 0;
static int ts_buf_write_index = 0;
/*
* Init UDP socket
*/
void init_udp_send(egress_param_t *_params,
egress_stat_t *_stats,
int _use_histogram,
uint64_t *_kernel_latency_hist) {
void init_udp_send(egress_param_t *_params, egress_stat_t *_stats,
int _use_histogram, uint64_t *_kernel_latency_hist) {
int set_if_err;
pthread_t thread;
......@@ -79,12 +77,10 @@ void init_udp_send(egress_param_t *_params,
init_tx_buffer();
sock_fd = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP);
if (sock_fd < 0)
error(EXIT_FAILURE, errno, "Socket creation failed\n");
if (sock_fd < 0) error(EXIT_FAILURE, errno, "Socket creation failed\n");
set_if_err = set_if();
if (set_if_err < 0)
error(EXIT_FAILURE, errno, "Couldn't set interface\n");
if (set_if_err < 0) error(EXIT_FAILURE, errno, "Couldn't set interface\n");
if (setsockopt(sock_fd, SOL_SOCKET, SO_PRIORITY, &params->packet_priority,
sizeof(params->packet_priority)))
......@@ -98,7 +94,8 @@ void init_udp_send(egress_param_t *_params,
sk_txtime.clockid = CLOCK_TAI;
sk_txtime.flags = SOF_TXTIME_REPORT_ERRORS;
if (setsockopt(sock_fd, SOL_SOCKET, SO_TXTIME, &sk_txtime, sizeof(sk_txtime)))
if (setsockopt(sock_fd, SOL_SOCKET, SO_TXTIME, &sk_txtime,
sizeof(sk_txtime)))
error(EXIT_FAILURE, errno, "setsockopt SO_TXTIME failed\n");
}
......@@ -113,19 +110,18 @@ void init_udp_send(egress_param_t *_params,
error(EXIT_FAILURE, errno, "Couldn't create poll thread");
}
/*
* Sends udp packets
*/
void send_udp_packet(char *data,
uint64_t txtime) {
struct msghdr msg; // Message hardware, sent to the socket
struct cmsghdr *cmsg; // Control message hardware, for txtime
char control[CMSG_SPACE(sizeof(txtime))] = {}; // Stores txtime
struct iovec iov; // The iovec structures stores the TX buffer
void send_udp_packet(char *data, uint64_t txtime) {
struct msghdr msg; // Message hardware, sent to the socket
struct cmsghdr *cmsg; // Control message hardware, for txtime
char control[CMSG_SPACE(sizeof(txtime))] = {}; // Stores txtime
struct iovec iov; // The iovec structures stores the TX buffer
int sendmsgerr;
struct sockaddr_in sin; // Server address
struct timespec ts; // timestamp for userspace timestamping
struct sockaddr_in sin; // Server address
struct timespec ts; // timestamp for userspace timestamping
if (params->use_timestamps) {
clock_gettime(CLOCK_REALTIME, &ts);
......@@ -170,7 +166,6 @@ void send_udp_packet(char *data,
}
static void *poll_thread(void *p) {
(void)p;
// Poll file descriptor
struct pollfd poll_fd = {.fd = sock_fd};
......@@ -191,20 +186,19 @@ static void process_error_queue() {
int recv_ret;
// IO vector
unsigned char data_buffer[256]; // Buffer in io vector
struct iovec iov = {
.iov_base = data_buffer,
.iov_len = sizeof(data_buffer)};
unsigned char data_buffer[256]; // Buffer in io vector
struct iovec iov = {.iov_base = data_buffer, .iov_len = sizeof(data_buffer)};
// Control data, will store error or timestamps
unsigned char msg_control[CMSG_SPACE(sizeof(struct sock_extended_err)) + CMSG_SPACE(sizeof(struct timespec))];
unsigned char msg_control[CMSG_SPACE(sizeof(struct sock_extended_err)) +
CMSG_SPACE(sizeof(struct timespec))];
// Message hardware structure, containts IO vector and control message hardware
struct msghdr msg = {
.msg_iov = &iov,
.msg_iovlen = 1,
.msg_control = msg_control,
.msg_controllen = sizeof(msg_control)};
// Message hardware structure, containts IO vector and control message
// hardware
struct msghdr msg = {.msg_iov = &iov,
.msg_iovlen = 1,
.msg_control = msg_control,
.msg_controllen = sizeof(msg_control)};
struct cmsghdr *cmsg;
......@@ -218,15 +212,19 @@ static void process_error_queue() {
for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
// If a timestamp was received
if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SO_TIMESTAMPING) {
struct timespec *stamp = (struct timespec *)CMSG_DATA(cmsg);
int kernel_latency = (ts_to_uint(*stamp) - timestamps_buffer[ts_buf_read_index]) / 1000;
int kernel_latency =
(ts_to_uint(*stamp) - timestamps_buffer[ts_buf_read_index]) / 1000;
ts_buf_read_index = (ts_buf_read_index + 1) % TIMESTAMP_BUFFER_SIZE;
stats->min_kernel_latency = min(kernel_latency, stats->min_kernel_latency);
stats->max_kernel_latency = max(kernel_latency, stats->max_kernel_latency);
stats->avg_kernel_latency = (stats->max_kernel_latency * packets_sent + kernel_latency) / (packets_sent + 1);
stats->min_kernel_latency =
min(kernel_latency, stats->min_kernel_latency);
stats->max_kernel_latency =
max(kernel_latency, stats->max_kernel_latency);
stats->avg_kernel_latency =
(stats->max_kernel_latency * packets_sent + kernel_latency) /
(packets_sent + 1);
if (use_histogram) {
if (kernel_latency > MAX_KERNEL_LATENCY)
......@@ -238,21 +236,19 @@ static void process_error_queue() {
}
// If an error was received
else {
struct sock_extended_err *serr = (void *)CMSG_DATA(cmsg);
if (serr->ee_origin != SO_EE_ORIGIN_TXTIME)
continue;
if (serr->ee_origin != SO_EE_ORIGIN_TXTIME) continue;
switch (serr->ee_code) {
case SO_EE_CODE_TXTIME_INVALID_PARAM:
stats->invalid_parameter++;
break;
case SO_EE_CODE_TXTIME_MISSED:
stats->missed_deadline++;
break;
default:
fprintf(stderr, "Uknown TxTime error\n");
case SO_EE_CODE_TXTIME_INVALID_PARAM:
stats->invalid_parameter++;
break;
case SO_EE_CODE_TXTIME_MISSED:
stats->missed_deadline++;
break;
default:
fprintf(stderr, "Uknown TxTime error\n");
}
}
}
......
packet-exchange/src/send_packet.h
View file @ d495022a
......@@ -24,6 +24,10 @@ typedef struct egress_stat {
int avg_kernel_latency;
int max_kernel_latency;
int min_interval;
int avg_interval;
int max_interval;
} egress_stat_t;
void init_udp_send(egress_param_t *_params,
......
packet-exchange/src/server.c
View file @ d495022a
......@@ -20,6 +20,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <time.h>
......@@ -28,6 +29,7 @@
#include "common.h"
#include "recv_packet.h"
#include "send_packet.h"
#include "tracer.h"
// Structs
......@@ -36,12 +38,15 @@ enum TSNTask { RECV_PACKET_TASK, RTT_TASK };
typedef struct thread_param {
int interval;
int priority;
uint64_t latency_threshold;
} thread_param_t;
typedef struct main_params {
int refresh_rate;
int verbose;
int enable_tracing;
int enable_graph;
} main_param_t;
static void process_options(int argc, char *argv[]);
......@@ -72,19 +77,29 @@ static struct timespec measures_start;
static struct timespec measures_end;
static void help(char *argv[]) {
printf("Usage: %s [-aghtv] [-b CLIENT_IP] [-d BUF_LEN] [-f IF] [-i USEC] [-p PRIO] [-r USEC]\n\n", argv[0]);
printf(" -a Run the real time thread on CPU1\n");
printf(" -b CLIENT_IP Server side RTT\n");
printf(" -d BUF_LEN Set the length of tx buffer\n");
printf(" -f IF Set the network interface to be used\n");
printf(" -g Print histograms to sdtout on exit\n");
printf(" -h Show help\n");
printf(" -i USEC Wake up the real time thread every USEC microseconds (Default: 10ms)\n");
printf(" -p PRIO Run the real time thread at priority PRIO\n");
printf(" -r USEC Refresh the non real time main thread every USEC microseconds\n");
printf(" -t Enable timestamps\n");
printf(" -v Verbose\n");
printf("\n");
printf(
"Usage: %s [-aghtv] [-b CLIENT_IP] [-d BUF_LEN] [-f IF] [-i USEC] [-p "
"PRIO] [-T LATENCY_THRESHOLD -G]"
" [-r USEC]\n\n",
" -a Run the real time thread on CPU1\n"
" -b CLIENT_IP Server side RTT\n"
" -d BUF_LEN Set the length of tx buffer\n"
" -f IF Set the network interface to be used\n"
" -g Print histograms to sdtout on exit\n"
" -h Show help\n"
" -i USEC Wake up the real time thread every USEC "
"microseconds (Default: 10ms)\n"
" -p PRIO Run the real time thread at priority PRIO\n"
" -r USEC Refresh the non real time main thread "
"every USEC microseconds\n"
" -t Enable timestamps\n"
" -v Verbose\n"
" -T LATENCY_THRESHOLD Enable tracing until LATENCY_THRESHOLD is "
"reached\n"
" -G Enable function_graph tracer, used with "
"-T\n"
"\n",
argv[0]);
}
// Real-time thread
......@@ -92,7 +107,6 @@ static void help(char *argv[]) {
static void *packet_receiving_thread(void *p) {
(void)p;
struct timespec current, previous;
struct sched_param priority;
cpu_set_t mask;
int prev_packet_id = 0;
......@@ -104,23 +118,18 @@ static void *packet_receiving_thread(void *p) {
error(EXIT_FAILURE, errno, "Could not set CPU affinity to CPU #1\n");
}
// Set thread priority
priority.sched_priority = thread_params.priority;
if (sched_setscheduler(0, SCHED_FIFO, &priority))
error(EXIT_FAILURE, errno, "Couldn't set priority");
// Start tracing
if (main_params.enable_tracing) tracing(1);
clock_gettime(CLOCK_MONOTONIC, &measures_start);
// Packet receiving loop
for (ingress_stats.packets_received = 0;; ingress_stats.packets_received++) {
if (tsn_task == RTT_TASK) {
recv_udp_packet();
send_udp_packet("", 0);
} else if (tsn_task == RECV_PACKET_TASK) {
int current_packet_id;
recv_udp_packet();
......@@ -132,12 +141,18 @@ static void *packet_receiving_thread(void *p) {
if (ingress_stats.packets_received) {
int interval_us = calcdiff_ns(current, previous) / 1000;
ingress_stats.min_interval = min(interval_us, ingress_stats.min_interval);
ingress_stats.max_interval = max(interval_us, ingress_stats.max_interval);
ingress_stats.avg_interval = (ingress_stats.avg_interval * ingress_stats.packets_received + interval_us) / (ingress_stats.packets_received + 1);
ingress_stats.min_interval =
min(interval_us, ingress_stats.min_interval);
ingress_stats.max_interval =
max(interval_us, ingress_stats.max_interval);
ingress_stats.avg_interval =
(ingress_stats.avg_interval * ingress_stats.packets_received +
interval_us) /
(ingress_stats.packets_received + 1);
// Check if packets were lost
ingress_stats.lost_packets += (current_packet_id - prev_packet_id - 1) % 1000;
ingress_stats.lost_packets +=
(current_packet_id - prev_packet_id - 1) % 1000;
if (enable_histograms) {
int dist_to_interval = interval_us - (thread_params.interval / 1000);
......@@ -150,6 +165,13 @@ static void *packet_receiving_thread(void *p) {
}
}
// If the latency hits the tracing threshold, stop tracing
if (main_params.enable_tracing &&
(ingress_stats.max_interval > thread_params.latency_threshold)) {
tracing(0);
break;
}
previous = current;
prev_packet_id = current_packet_id;
}
......@@ -162,6 +184,8 @@ static void *packet_receiving_thread(void *p) {
// Handles the IO and creates real time threads
int main(int argc, char *argv[]) {
pthread_t thread;
struct sched_param param;
pthread_attr_t attr;
ingress_stats.min_interval = INT_MAX;
ingress_stats.avg_interval = 0;
......@@ -178,6 +202,8 @@ int main(int argc, char *argv[]) {
thread_params.priority = 99;
main_params.refresh_rate = 50000;
main_params.verbose = 0;
main_params.enable_tracing = 0;
main_params.enable_graph = 0;
enable_affinity = 0;
enable_timestamps = 0;
......@@ -197,24 +223,50 @@ int main(int argc, char *argv[]) {
memset(jitter_hist, 0, sizeof(jitter_hist));
}
if (main_params.enable_tracing) {
// Enable ftrace
setup_tracer(main_params.enable_graph);
}
// Catch breaks with sighand to print the histograms
init_signals(sighand, enable_histograms);
// Initialize the UDP packet receiving socket
init_udp_recv(&ingress_params,
&ingress_stats,
enable_histograms,
init_udp_recv(&ingress_params, &ingress_stats, enable_histograms,
kernel_latency_hist);
// Initialize the UDP packet sending socket if RTT is measured
if (tsn_task == RTT_TASK)
init_udp_send(&egress_params,
&egress_stats,
0,
NULL);
init_udp_send(&egress_params, &egress_stats, 0, NULL);
/* Initialize pthread attributes (default values) */
if (pthread_attr_init(&attr)) {
fprintf(stderr, "init pthread attributes failed\n");
exit(EXIT_FAILURE);
}
/* Set a specific stack size */
if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN)) {
fprintf(stderr, "pthread setstacksize failed\n");
exit(EXIT_FAILURE);
}
/* Set scheduler policy and priority of pthread */
if (pthread_attr_setschedpolicy(&attr, SCHED_FIFO)) {
fprintf(stderr, "pthread setschedpolicy failed\n");
exit(EXIT_FAILURE);
}
param.sched_priority = thread_params.priority;
if (pthread_attr_setschedparam(&attr, &param)) {
fprintf(stderr, "pthread setschedparam failed\n");
exit(EXIT_FAILURE);
}
/* Use scheduling parameters of attr */
if (pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED)) {
fprintf(stderr, "pthread setinheritsched failed\n");
exit(EXIT_FAILURE);
}
// Create the real time thread
if (pthread_create(&thread, NULL, packet_receiving_thread, NULL))
if (pthread_create(&thread, &attr, packet_receiving_thread, NULL))
error(EXIT_FAILURE, errno, "Couldn't create packet receiving thread");
// Verbose loop
......@@ -222,27 +274,20 @@ int main(int argc, char *argv[]) {
usleep(main_params.refresh_rate);
if (main_params.verbose && ingress_stats.packets_received > 1) {
if (tsn_task == RECV_PACKET_TASK) {
int jitter = ingress_stats.max_interval - ingress_stats.min_interval;
printf("%9" PRIu64 ": J: %5d, I (ms): %3d %3d %3d [%3d,%3d]",
ingress_stats.packets_received,
jitter,
ingress_stats.min_interval / 1000,
ingress_stats.avg_interval / 1000,
ingress_stats.max_interval / 1000,
(int) ingress_stats.high_jitter,
(int) ingress_stats.lost_packets);
printf("%9" PRIu64 ": J: %5d, I (10us): %3d %3d %3d [%3d,%3d]",
ingress_stats.packets_received, jitter,
ingress_stats.min_interval / 10, ingress_stats.avg_interval / 10,
ingress_stats.max_interval / 10, (int)ingress_stats.high_jitter,
(int)ingress_stats.lost_packets);
if (enable_timestamps) {
printf(", K: %4d %4d %4d [%3d]\n",
ingress_stats.min_kernel_latency,
printf(", K: %4d %4d %4d [%3d]\n", ingress_stats.min_kernel_latency,
ingress_stats.avg_kernel_latency,
ingress_stats.max_kernel_latency,
(int) ingress_stats.high_kernel_latency);
(int)ingress_stats.high_kernel_latency);
} else {
printf("\n");
}
......@@ -250,12 +295,15 @@ int main(int argc, char *argv[]) {
printf("\033[%dA", 1);
}
}
if (main_params.enable_tracing &&
(ingress_stats.max_interval >= thread_params.latency_threshold))
break;
}
}
// Print histograms in .json format
static void print_histograms() {
uint64_t duration;
int duration_hour, duration_minutes, interval;
int max_latency, max_jitter, min_jitter;
......@@ -269,20 +317,23 @@ static void print_histograms() {
interval = thread_params.interval / 1000;
if (enable_timestamps) {
printf("{\"measure_sets\": [{"
"\"measure_type\": \"packet_recv_timestamps\","
"\"props_names\": [\"kernel_space\"],"
"\"units\": [\"us\"],"
"\"props_type\": \"histogram\","
"\"metadata\": {\"i\": \"%dus\", \"duration\": \"%dh%d\","
"\"lost_packets\": \"%d\"},"
"\"props\": [[",
interval, duration_hour, duration_minutes, ingress_stats.lost_packets);
printf(
"{\"measure_sets\": [{"
"\"measure_type\": \"packet_recv_timestamps\","
"\"props_names\": [\"kernel_space\"],"
"\"units\": [\"us\"],"
"\"props_type\": \"histogram\","
"\"metadata\": {\"i\": \"%dus\", \"duration\": \"%dh%d\","
"\"lost_packets\": \"%d\"},"
"\"props\": [[",
interval, duration_hour, duration_minutes,
ingress_stats.lost_packets + ((int)ingress_stats.high_jitter));
max_latency = histogram_max(kernel_latency_hist, MAX_KERNEL_LATENCY - 1);
for (int j = 0; j < max_latency; j++)
printf("%" PRIi64 "%s", kernel_latency_hist[j], (j + 1 < max_latency ? ", " : ""));
printf("%" PRIi64 "%s", kernel_latency_hist[j],
(j + 1 < max_latency ? ", " : ""));
printf("]]");
}
......@@ -291,16 +342,17 @@ static void print_histograms() {
printf("%s", enable_timestamps ? "}, {" : "{\"measure_sets\": [{");
printf("\"measure_type\": \"packet_jitter\","
"\"props_names\": [\"jitter\"],"
"\"units\": [\"us\"],"
"\"props_type\": \"histogram\","
"\"middle\": \"%d\","
"\"metadata\": {\"i\": \"%dus\", \"duration\": \"%dh%d\","
"\"lost_packets\": \"%d\"},"
"\"props\": [[",
MAX_JITTER / 2 - min_jitter, interval, duration_hour, duration_minutes,
ingress_stats.lost_packets);
printf(
"\"measure_type\": \"packet_jitter\","
"\"props_names\": [\"jitter\"],"
"\"units\": [\"us\"],"
"\"props_type\": \"histogram\","
"\"middle\": \"%d\","
"\"metadata\": {\"i\": \"%dus\", \"duration\": \"%dh%d\","
"\"lost_packets\": \"%d\"},"
"\"props\": [[",
MAX_JITTER / 2 - min_jitter, interval, duration_hour, duration_minutes,
ingress_stats.lost_packets + ((int)ingress_stats.high_jitter));
for (int j = min_jitter; j < max_jitter; j++)
printf("%" PRIi64 "%s", jitter_hist[j], (j + 1 < max_jitter ? ", " : ""));
......@@ -320,56 +372,61 @@ static void sighand(int sig_num) {
// Process bash options
static void process_options(int argc, char *argv[]) {
int network_if_specified = 0;
for (;;) {
int c = getopt(argc, argv, "ab:d:f:ghi:p:r:tv");
int c = getopt(argc, argv, "ab:d:f:ghi:p:r:tvT:G");
if (c == -1)
break;
if (c == -1) break;
switch (c) {
case 'a':
enable_affinity = 1;
break;
case 'b':
tsn_task = RTT_TASK;
strcpy(egress_params.server_ip, optarg);
break;
case 'd':
ingress_params.tx_buffer_len = atoi(optarg);
if (ingress_params.tx_buffer_len < 1) {
fprintf(stderr, "BUF_LEN should be greater than 1\n");
exit(EXIT_FAILURE);
}
break;
case 'f':
network_if_specified = 1;
strcpy(ingress_params.network_if, optarg);
break;
case 'h':
help(argv);
exit(EXIT_SUCCESS);
break;
case 'i':
thread_params.interval = atoi(optarg) * 1000;
break;
case 'g':
enable_histograms = 1;
break;
case 'p':
thread_params.priority = atoi(optarg);
break;
case 'r':
main_params.refresh_rate = atoi(optarg);
break;
case 't':
enable_timestamps = 1;
break;
case 'v':
main_params.verbose = 1;
break;
case 'a':
enable_affinity = 1;
break;
case 'b':
tsn_task = RTT_TASK;
strcpy(egress_params.server_ip, optarg);
break;
case 'd':
ingress_params.tx_buffer_len = atoi(optarg);
if (ingress_params.tx_buffer_len < 1) {
fprintf(stderr, "BUF_LEN should be greater than 1\n");
exit(EXIT_FAILURE);
}
break;
case 'f':
network_if_specified = 1;
strcpy(ingress_params.network_if, optarg);
break;
case 'h':
help(argv);
exit(EXIT_SUCCESS);
break;
case 'i':
thread_params.interval = atoi(optarg) * 1000;
break;
case 'g':
enable_histograms = 1;
break;
case 'p':
thread_params.priority = atoi(optarg);
break;
case 'r':
main_params.refresh_rate = atoi(optarg);
break;
case 't':
enable_timestamps = 1;
break;
case 'v':
main_params.verbose = 1;
break;
case 'T':
main_params.enable_tracing = 1;
thread_params.latency_threshold = atoi(optarg);
break;
case 'G':
main_params.enable_graph = 1;
break;
}
}
......
packet-exchange/src/tracer.c 0 → 100644
View file @ d495022a
#include "tracer.h"
#define _GNU_SOURCE
#include <errno.h>
#include <error.h>
#include <fcntl.h>
#include <getopt.h>
#include <limits.h>
#include <linux/unistd.h>
#include <pthread.h>
#include <sched.h>
#include <signal.h>
#include <stdarg.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/prctl.h>
#include <sys/resource.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include <sys/sysinfo.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/utsname.h>
#include <time.h>
#include <unistd.h>
/*
* From cyclictest code source
*/
#define KVARS 32
#define KVARNAMELEN 32
#define KVALUELEN 32
#define MAX_PATH 255
static char *fileprefix;
static char *procfileprefix = "/proc/sys/kernel/";
static char *debugfileprefix = "/sys/kernel/debug/tracing/";
static int trace_fd = -1;
static int tracemark_fd = -1;
static int kernvar(int mode, const char *name, char *value,
size_t sizeofvalue) {
char filename[128];
int retval = 1;
int path;
size_t len_prefix = strlen(fileprefix), len_name = strlen(name);
memcpy(filename, fileprefix, len_prefix);
memcpy(filename + len_prefix, name, len_name + 1);
path = open(filename, mode);
if (path >= 0) {
if (mode == O_RDONLY) {
int got;
if ((got = read(path, value, sizeofvalue)) > 0) {
retval = 0;
value[got - 1] = '\0';
}
} else if (mode == O_WRONLY) {
if (write(path, value, sizeofvalue) == sizeofvalue) retval = 0;
}
close(path);
}
return retval;
}
static void setkernvar(const char *name, char *value) {
if (kernvar(O_WRONLY, name, value, strlen(value)))
fprintf(stderr, "could not set %s to %s\n", name, value);
}
void setup_tracer(int enable_graph) {
char trace_path[MAX_PATH];
char tracemark_path[MAX_PATH];
fileprefix = procfileprefix;
setkernvar("ftrace_enabled", "1");
fileprefix = debugfileprefix;
// Clear old traces by setting tracer to nop first
setkernvar("current_tracer", "nop");
if(enable_graph)
setkernvar("current_tracer", "function_graph");
else
setkernvar("current_tracer", "function");
// Open tracing_on file
strcpy(trace_path, fileprefix);
strcat(trace_path, "tracing_on");
if ((trace_fd = open(trace_path, O_WRONLY)) == -1)
printf("unable to open %s for tracing", trace_path);
// Open trace mark file
strcpy(tracemark_path, fileprefix);
strcat(tracemark_path, "trace_marker");
if ((tracemark_fd = open(tracemark_path, O_WRONLY)) == -1)
printf("unable to open %s for tracing", tracemark_path);
tracing(0);
}
void tracing(int on) {
if (on)
write(trace_fd, "1", 1);
else
write(trace_fd, "0", 1);
}
void tracemark(char * s) {
write(tracemark_fd, s, strlen(s));
}
packet-exchange/src/tracer.h 0 → 100644
View file @ d495022a
#ifndef TRACER_H
#define TRACER_H
void setup_tracer(int enable_graph);
void tracing(int on);
void tracemark(char * s);
#endif
scripts/multissh 0 → 100755
View file @ d495022a
#!/bin/bash
usage() {
cat << ENDUSAGE
Usage: $0 CMD
$0 sudo CMD
ENDUSAGE
exit 1;
}
if [ -z "$1" ]; then
usage
fi
if [ $1 == "sudo" ]; then
if [ -z "$2" ]; then
usage
fi
./sudossh emerald "$2";
./sudossh slate "$2";
./sudossh onyx "$2";
else
ssh emerald $1&
ssh onyx $1&
ssh slate $1
fi
scripts/run-client
View file @ d495022a
......@@ -3,7 +3,7 @@
script_dir=$(dirname $(realpath $0))
usage() {
echo "Usage: $0 (-e delta [-o etf_offset] | -p) [-bgt] [-i INTERVAL] [-d TX_BUFFER_LEN] [emerald|slate|onyx]" 1>&2;
echo "Usage: $0 (-e delta [-o etf_offset] | -p) [-bgt] [-i INTERVAL] [-d TX_BUFFER_LEN] [-T LATENCY_THRESHOLD -G] [emerald|slate|onyx]" 1>&2;
exit 1;
}
......@@ -16,7 +16,7 @@ qdisc_options=""
ip="192.168.99."
etf_offset=500
while getopts "bd:e:o:ghi:pt" opt; do
while getopts "bd:e:o:ghi:ptT:G" opt; do
case "${opt}" in
b )
client_options+=" -b"
......@@ -52,6 +52,12 @@ while getopts "bd:e:o:ghi:pt" opt; do
use_timestamps=1
client_options+=" -t"
;;
T )
client_options+=" -T ${OPTARG}"
;;
G )
client_options+=" -G"
;;
* )
usage
;;
......@@ -102,8 +108,8 @@ fi
client_options+=" -i $interval"
echo "create_qdisc $qdisc_options";
$script_dir/create_qdisc $qdisc_options;
echo "create-qdisc $qdisc_options";
$script_dir/create-qdisc $qdisc_options;
echo "make client";
cd $script_dir/../packet-exchange/build;
......
scripts/run-server
View file @ d495022a
......@@ -3,7 +3,7 @@
script_dir=$(dirname $(realpath $0))
usage() {
echo "Usage: $0 [[-bt] [-g INTERVAL]] | (-d NB_PACKETS [-i INTERVAL])" 1>&2;
echo "Usage: $0 [[-bt] [-g INTERVAL]] | (-d NB_PACKETS [-i INTERVAL]) [-T LATENCY_THRESHOLD -G]" 1>&2;
exit 1;
}
......@@ -12,7 +12,7 @@ server_options="-a -p 99 -f eth0"
ip="192.168.99."
tcpdump_interval=1000000
while getopts "b:htd:i:g:" opt; do
while getopts "b:htd:i:g:T:G" opt; do
case "${opt}" in
b )
use_rtt=1
......@@ -37,6 +37,12 @@ while getopts "b:htd:i:g:" opt; do
t )
server_options+=" -t"
;;
T )
client_options+=" -T ${OPTARG}"
;;
G )
client_options+=" -G"
;;
* )
usage
;;
......@@ -73,8 +79,8 @@ if [ -n "${use_tcpdump}" ]; then
echo "tshark -r tmp.pcap --disable-protocol dcp-etsi --disable-protocol dcp-pft -t e -E separator=, -T fields -e frame.number -e frame.time_epoch -e data.data > tmp.out";
tshark -r server_stats_tmp.pcap --disable-protocol dcp-etsi --disable-protocol dcp-pft -t e -E separator=, -T fields -e frame.number -e frame.time_epoch -e data.data > server_stats_tmp.out;
echo "txtime_stats.py -f server_pcap_stats -i $tcpdump_interval";
$script_dir/txtime_stats.py -f server_stats_tmp.out -i $tcpdump_interval;
echo "txtime-stats.py -f server_pcap_stats -i $tcpdump_interval";
$script_dir/txtime-stats.py -f server_stats_tmp.out -i $tcpdump_interval;
else
echo "make server";
......
scripts/sudossh 0 → 100755
View file @ d495022a
#!/bin/expect
spawn ssh [lindex $argv 0]
expect "oli@"
send -- "sudo [lindex $argv 1]\r"
expect "assword"
send -- "olimex\r"
expect {
"Do you want to continue?" {
send "Y\r"
exp_continue
}
"oli@" {
exit
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7