First step towards TRX library

- Huge refactoring: functions fit in a new library, libcapulet
- Rewriting of the `rdma_standalone` example with new library
- Separate MR registration from other RDMA operations
- Allow passing custom access flags to QP

.gitignore

-bin/*
+out/*
+**/*.o

Makefile

-CFLAGS = -Wall -O3 -Iinclude/
+CFLAGS = -Wall -O3
+INCLUDE_FLAGS = -Ilibcapulet/include/
+LIB_FLAGS =-libverbs
 
-all: rdma_demo
+all: out/rdma_standalone
 
 clean:
-	rm bin/*
+	rm out/*
+	rm **/*.o
 
-rdma_demo: rdma_demo.c lib/soft_rdma.c lib/udp_rdma.c
-	gcc $(CFLAGS) -o bin/$@ $? -libverbs
+%.o: %.c
+	gcc -c $(CFLAGS) $(INCLUDE_FLAGS) $< -o $@
+
+out/libcapulet.a: libcapulet/net_udp.o libcapulet/rdma_ib.o
+	ar -rc $@ $^
+
+out/rdma_standalone: example/rdma_standalone.o out/libcapulet.a
+	gcc $(CFLAGS) -o $@ $^ $(LIB_FLAGS)
 

README.md

@@ -19,13 +19,13 @@ On one machine, run the server program, which will read content to be put in the
 
 ```bash
 make
-bin/rdma_demo < examples/rj
+out/rdma_standalone < README.md
 ```
 
 The other machine will act as a client, so we need to pass it the IP address of the server:
 
 ```bash
 make
-bin/rdma_demo -c 192.168.16.10
+out/rdma_standalone -c 192.168.16.10
 ```
 

example/rdma_standalone.c

+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+
+#include "net_udp.h"
+#include "rdma_ib.h"
+
+int main(int argc, char *argv[]) {
+	struct CapuletRdmaIbContext rdma_ctx;
+	struct CapuletNetUdpContext *udp_ctx;
+
+	struct ibv_mr *memory_buffer_mr;
+	struct ibv_wc poll_wc;
+
+	char remote_host[16];
+	char *memory_char_buffer;
+	bool result;
+	void *memory_buffer;
+
+	int allocated_size = 16384 * sizeof(char);
+	int page_size = sysconf(_SC_PAGESIZE);
+	bool is_client = false;
+
+	srand48(getpid() * time(NULL));
+
+	/******************************
+	 **** Arguments processing ****
+	 ******************************/
+	// Detect if a server was given; if any, act as a client
+	if(argc >= 3) {
+		if(strcmp(argv[1], "-c") == 0) {
+			strcpy((void *) remote_host, (void *) argv[2]);
+			is_client = true;
+		}
+	}
+
+	// Allocate memory and get user data from STDIN
+	memory_buffer = aligned_alloc(page_size, allocated_size);
+	if(!memory_buffer) {
+		fprintf(stderr, "Memory allocation failed (before registrering MR)\n");
+		return -1;
+	}
+
+	memset(memory_buffer, 0, allocated_size);
+	memory_char_buffer = (char *) memory_buffer;
+
+	if(!is_client) {
+		read(STDIN_FILENO, memory_char_buffer, allocated_size);
+	}
+
+	/******************************
+	 ***** RDMA initialization ****
+	 ******************************/
+	result = capulet_rdma_ib_initialize_device(&rdma_ctx, "rxe0");
+	if(!result) {
+		fprintf(stderr, "Device initialization failed\n");
+		return -1;
+	}
+
+	memory_buffer_mr = ibv_reg_mr(rdma_ctx.pd, memory_buffer, allocated_size, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ);
+	if(!memory_buffer_mr) {
+		fprintf(stderr, "Memory Region registration failed\n");
+		return -1;
+	}
+
+	result = capulet_rdma_ib_initialize_qp(&rdma_ctx, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ);
+	if(!result) {
+		fprintf(stderr, "Queue Pair initialization failed\n");
+		return -1;
+	}
+
+	result = capulet_rdma_ib_post_recv(&rdma_ctx, memory_buffer_mr, allocated_size);
+	if(!result) {
+		fprintf(stderr, "Posting Recv failed\n");
+		return -1;
+	}
+
+	/******************************
+	 **** Exchange informations ***
+	 ******************************/
+	udp_ctx = capulet_net_udp_initialize();
+	if(!udp_ctx) {
+		fprintf(stderr, "UDP context initialization failed\n");
+		return -1;
+	}
+
+	result = capulet_rdma_ib_fill_base_udp(&rdma_ctx, memory_buffer_mr, udp_ctx->local);
+	if(!result) {
+		fprintf(stderr, "Query port failed\n");
+		return -1;
+	}
+
+	capulet_net_udp_dump_packet(udp_ctx->local);
+
+	if(is_client) {
+		result = capulet_net_udp_connect(udp_ctx, remote_host);
+	} else {
+		result = capulet_net_udp_bind(udp_ctx);
+	}
+
+	if(!result) {
+		fprintf(stderr, "Connection to peer failed\n");
+		return -1;
+	}
+
+	capulet_net_udp_dump_packet(udp_ctx->remote);
+
+	/******************************
+	 ******** Read or wait ********
+	 ******************************/
+	result = capulet_rdma_ib_set_peer_from_udp(&rdma_ctx, udp_ctx);
+	if(!result) {
+		fprintf(stderr, "Setting state to RTS failed\n");
+		return -1;
+	}
+
+	if(is_client) {
+		result = capulet_rdma_ib_send_read(&rdma_ctx, udp_ctx->remote, memory_buffer_mr, allocated_size);
+		if(!result) {
+			fprintf(stderr, "Sending Read failed\n");
+			return -1;
+		}
+	}
+
+	do {
+		result = ibv_poll_cq(rdma_ctx.cq, 1, &poll_wc);
+	} while(result == 0);
+
+	if(result > 0 && poll_wc.status == IBV_WC_SUCCESS) {
+		if(is_client) {
+			printf("%s\n", memory_char_buffer);
+		}
+	} else {
+		printf("Failed: %s (WR %lu)\n", ibv_wc_status_str(poll_wc.status), poll_wc.wr_id);
+	}
+
+	/******************************
+	 ******* Global cleanup *******
+	 ******************************/
+	capulet_net_udp_free(udp_ctx);
+	ibv_dereg_mr(memory_buffer_mr);
+	capulet_rdma_ib_free(&rdma_ctx);
+	free(memory_buffer);
+
+	return 0;
+}
+

examples/rj

-Good-night, good-night! Parting is such sweet sorrow
-That I shall say good-night till it be morrow.

include/sdr_rdma.h

-#include <stdint.h>
-
-#include "soft_rdma.h"
-
-struct SDRMetadata {
-	uint8_t server_version;
-	uint16_t iq_input_length;
-	uint16_t iq_output_length;
-	uint8_t __unused[25];
-};
-
-struct SDRMemoryRegion {
-	struct SDRMetadata meta;
-	float iq_input[4096];
-	float iq_output[4096];
-};
-

include/soft_rdma.h

-#include <stdbool.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include <infiniband/verbs.h>
-
-struct ibv_context *initialize_device_context(const char *device_name);
-struct ibv_qp *initialize_queue_pair(struct ibv_pd *ibv_dev_pd, struct ibv_cq *ibv_dev_cq);
-bool set_peer_informations(struct ibv_qp *ibv_dev_qp, uint32_t local_psn, uint32_t remote_psn, uint32_t remote_lid, uint32_t remote_qpn, uint8_t *remote_gid);
-bool initialize_post_recv(struct ibv_qp *ibv_dev_qp, struct ibv_mr *ibv_dev_mr, int mr_size);
-bool send_rdma_read(struct ibv_qp *ibv_dev_qp, struct ibv_mr *ibv_dev_mr, int mr_size, void *remote_addr, uint32_t remote_key);
-

include/udp_rdma.h → libcapulet/include/net_udp.h

@@ -3,13 +3,14 @@
 #include <netdb.h>
 #include <stdbool.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
 
 #include <sys/types.h>
 #include <sys/socket.h>
 
-struct udp_rdma_parameters {
+struct CapuletNetUdpPacket {
 	uint16_t lid;
 	uint32_t qpn;
 	uint32_t psn;
@@ -19,7 +20,14 @@ struct udp_rdma_parameters {
 	uint32_t key;
 };
 
-bool udp_rdma_receive_parameters(struct udp_rdma_parameters *local_params, struct udp_rdma_parameters *remote_params);
-bool udp_rdma_send_parameters(const char *hostname, struct udp_rdma_parameters *local_params, struct udp_rdma_parameters *remote_params);
-void udp_rdma_dump_packet(struct udp_rdma_parameters *rdma_params);
+struct CapuletNetUdpContext {
+	struct CapuletNetUdpPacket *local;
+	struct CapuletNetUdpPacket *remote;
+};
+
+struct CapuletNetUdpContext *capulet_net_udp_initialize();
+bool capulet_net_udp_bind(struct CapuletNetUdpContext *ctx);
+bool capulet_net_udp_connect(struct CapuletNetUdpContext *ctx, const char *hostname);
+void capulet_net_udp_dump_packet(struct CapuletNetUdpPacket *packet);
+void capulet_net_udp_free(struct CapuletNetUdpContext *ctx);
 

libcapulet/include/rdma_ib.h

+#pragma once
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <infiniband/verbs.h>
+
+#include "net_udp.h"
+
+struct CapuletRdmaIbContext {
+	struct ibv_context *ib_ctx;
+	struct ibv_pd *pd;
+	struct ibv_cq *cq;
+	struct ibv_qp *qp;
+};
+
+bool capulet_rdma_ib_initialize_device(struct CapuletRdmaIbContext *ctx, const char *device_name);
+bool capulet_rdma_ib_initialize_qp(struct CapuletRdmaIbContext *ctx, int access);
+bool capulet_rdma_ib_fill_base_udp(struct CapuletRdmaIbContext *ctx, struct ibv_mr *mr, struct CapuletNetUdpPacket *own);
+bool capulet_rdma_ib_set_peer_from_udp(struct CapuletRdmaIbContext *ctx, struct CapuletNetUdpContext *udp_ctx);
+bool capulet_rdma_ib_post_recv(struct CapuletRdmaIbContext *ctx, struct ibv_mr *ibv_dev_mr, int mr_size);
+bool capulet_rdma_ib_send_read(struct CapuletRdmaIbContext *ctx, struct CapuletNetUdpPacket *peer, struct ibv_mr *ibv_dev_mr, int mr_size);
+void capulet_rdma_ib_free(struct CapuletRdmaIbContext *ctx);
+

lib/udp_rdma.c → libcapulet/net_udp.c

-#include "udp_rdma.h"
+#include "include/net_udp.h"
 
-bool udp_rdma_receive_parameters(struct udp_rdma_parameters *local_params, struct udp_rdma_parameters *remote_params) {
+struct CapuletNetUdpContext *capulet_net_udp_initialize() {
+	struct CapuletNetUdpContext *ctx;
+	void *raw_ptr;
+
+	size_t packet_size = sizeof(struct CapuletNetUdpPacket);
+	size_t ctx_size = sizeof(struct CapuletNetUdpContext);
+	size_t total_size = 2 * packet_size + ctx_size;
+
+	raw_ptr = malloc(total_size);
+	if(!raw_ptr) {
+		return NULL;
+	}
+
+	memset(raw_ptr, 0, total_size);
+
+	ctx = (struct CapuletNetUdpContext *) raw_ptr;
+	ctx->local = (struct CapuletNetUdpPacket *) (raw_ptr + ctx_size);
+	ctx->remote = (struct CapuletNetUdpPacket *) (raw_ptr + ctx_size + packet_size);
+
+	return ctx;	
+}
+
+bool capulet_net_udp_bind(struct CapuletNetUdpContext *ctx) {
 	struct addrinfo server_hints;
 	struct addrinfo *server_infos;
 	struct sockaddr client_addr;
 
+	int result;
+	int server_socket;
+
+	const size_t buffer_length = sizeof(struct CapuletNetUdpContext);
 	socklen_t client_addr_size = sizeof(struct sockaddr_storage);
-	const size_t buffer_length = sizeof(struct udp_rdma_parameters);
-	int result, server_socket;
 	char receive_buffer[buffer_length];
 
 	// Set up bind address
@@ -46,9 +70,9 @@ bool udp_rdma_receive_parameters(struct udp_rdma_parameters *local_params, struc
 		return false;
 	}
 
-	memcpy((void *) remote_params, (void *) receive_buffer, buffer_length);
+	memcpy((void *) ctx->remote, (void *) receive_buffer, buffer_length);
 
-	result = sendto(server_socket, (void *) local_params, buffer_length, MSG_CONFIRM, &client_addr, client_addr_size);
+	result = sendto(server_socket, (void *) ctx->local, buffer_length, MSG_CONFIRM, &client_addr, client_addr_size);
 	if(result == -1) {
 		perror("sendto error");
 		return false;
@@ -59,12 +83,14 @@ bool udp_rdma_receive_parameters(struct udp_rdma_parameters *local_params, struc
 	return true;
 }
 
-bool udp_rdma_send_parameters(const char *hostname, struct udp_rdma_parameters *local_params, struct udp_rdma_parameters *remote_params) {
+bool capulet_net_udp_connect(struct CapuletNetUdpContext *ctx, const char *hostname) {
 	struct addrinfo server_hints;
 	struct addrinfo *server_infos;
 
-	const size_t buffer_length = sizeof(struct udp_rdma_parameters);
-	int result, server_socket;
+	int result;
+	int server_socket;
+
+	const size_t buffer_length = sizeof(struct CapuletNetUdpContext);
 	char receive_buffer[buffer_length];
 
 	// Set up remote address
@@ -87,7 +113,7 @@ bool udp_rdma_send_parameters(const char *hostname, struct udp_rdma_parameters *
 	}
 
 	// Create and send a query with full capabilities
-	result = sendto(server_socket, (void *) local_params, buffer_length, 0, server_infos->ai_addr, server_infos->ai_addrlen);
+	result = sendto(server_socket, (void *) ctx->local, buffer_length, 0, server_infos->ai_addr, server_infos->ai_addrlen);
 	if(result == -1) {
 		perror("sendto error");
 		return false;
@@ -101,7 +127,7 @@ bool udp_rdma_send_parameters(const char *hostname, struct udp_rdma_parameters *
 		return false;
 	}
 
-	memcpy((void *) remote_params, (void *) receive_buffer, buffer_length);
+	memcpy((void *) ctx->remote, (void *) receive_buffer, buffer_length);
 
 	close(server_socket);
 	freeaddrinfo(server_infos);
@@ -109,13 +135,17 @@ bool udp_rdma_send_parameters(const char *hostname, struct udp_rdma_parameters *
 	return true;
 }
 
-void udp_rdma_dump_packet(struct udp_rdma_parameters *rdma_params) {
-	printf("LID: 0x%x | QPN: 0x%x | PSN: 0x%x | GID: ", rdma_params->lid, rdma_params->qpn, rdma_params->psn);
+void capulet_net_udp_dump_packet(struct CapuletNetUdpPacket *packet) {
+	printf("LID: 0x%x | QPN: 0x%x | PSN: 0x%x | GID: ", packet->lid, packet->qpn, packet->psn);
 
 	for(char i = 0; i < 8; i++) {
-		printf("%x%x ", rdma_params->gid[2 * i], rdma_params->gid[2 * i + 1]);
+		printf("%x%x ", packet->gid[2 * i], packet->gid[2 * i + 1]);
 	}
 
-	printf("\naddr: %p | size: %ld | key: 0x%x\n", rdma_params->addr, rdma_params->size, rdma_params->key);
+	printf("\naddr: %p | size: %ld | key: 0x%x\n", packet->addr, packet->size, packet->key);
+}
+
+void capulet_net_udp_free(struct CapuletNetUdpContext *ctx) {
+	free((void *) ctx);
 }
 

lib/soft_rdma.c → libcapulet/rdma_ib.c

-#include "soft_rdma.h"
+#include "include/rdma_ib.h"
 
-struct ibv_context *initialize_device_context(const char *device_name) {
+bool capulet_rdma_ib_initialize_device(struct CapuletRdmaIbContext *ctx, const char *device_name) {
 	struct ibv_device **dev_list;
 	struct ibv_device *ibv_dev = NULL;
-	struct ibv_context *ibv_dev_ctx;
 
 	int dev_list_length, dev_id;
 
@@ -11,7 +10,7 @@ struct ibv_context *initialize_device_context(const char *device_name) {
 	// and the name of each one is compared to the requested name.
 	dev_list = ibv_get_device_list(&dev_list_length);
 	if(!dev_list || dev_list_length < 1) {
-		return NULL;
+		return false;
 	}
 
 	for(dev_id = 0; dev_id < dev_list_length; dev_id++) {
@@ -22,24 +21,34 @@ struct ibv_context *initialize_device_context(const char *device_name) {
 	}
 	
 	if(!ibv_dev) {
-		return NULL;
+		return false;
 	}
 
 	// The devices list can be freed as soon as context was obtained.
-	ibv_dev_ctx = ibv_open_device(ibv_dev);
+	ctx->ib_ctx = ibv_open_device(ibv_dev);
 	ibv_free_device_list(dev_list);
 
-	// Returned pointer was allocated by ibv_open_device
-	return ibv_dev_ctx;
+	// XXX: for now, a global Protection Domain is allocated
+	ctx->pd = ibv_alloc_pd(ctx->ib_ctx);
+	if(!ctx->pd) {
+		return false;
+	}
+
+	return true;
 }
 
-struct ibv_qp *initialize_queue_pair(struct ibv_pd *ibv_dev_pd, struct ibv_cq *ibv_dev_cq) {
+bool capulet_rdma_ib_initialize_qp(struct CapuletRdmaIbContext *ctx, int access) {
 	struct ibv_qp_init_attr ibv_dev_qp_request;
 	struct ibv_qp_attr ibv_dev_qp_params;
 	struct ibv_qp *ibv_dev_qp;
 
 	int result;
 
+	ctx->cq = ibv_create_cq(ctx->ib_ctx, 2, NULL, NULL, 0);
+	if(!ctx->cq) {
+		return false;
+	}
+
 	memset(&ibv_dev_qp_request, 0, sizeof(struct ibv_qp_init_attr));
 	memset(&ibv_dev_qp_params, 0, sizeof(struct ibv_qp_attr));
 
@@ -47,59 +56,88 @@ struct ibv_qp *initialize_queue_pair(struct ibv_pd *ibv_dev_pd, struct ibv_cq *i
 	// initializing it.
 	// Creation of QP with initial parameters
 	ibv_dev_qp_request.qp_type = IBV_QPT_RC;
-	ibv_dev_qp_request.send_cq = ibv_dev_cq;
-	ibv_dev_qp_request.recv_cq = ibv_dev_cq;
+	ibv_dev_qp_request.send_cq = ctx->cq;
+	ibv_dev_qp_request.recv_cq = ctx->cq;
 
 	ibv_dev_qp_request.cap.max_send_wr = 1;
 	ibv_dev_qp_request.cap.max_recv_wr = 1;
 	ibv_dev_qp_request.cap.max_send_sge = 1;
 	ibv_dev_qp_request.cap.max_recv_sge = 1;
 
-	ibv_dev_qp = ibv_create_qp(ibv_dev_pd, &ibv_dev_qp_request);
+	ibv_dev_qp = ibv_create_qp(ctx->pd, &ibv_dev_qp_request);
 
 	if(!ibv_dev_qp) {
-		return NULL;
+		return false;
 	}
 
 	// XXX: added to try and make things work
-	ibv_query_qp(ibv_dev_qp, &ibv_dev_qp_params, IBV_QP_CAP, &ibv_dev_qp_request);
+	//ibv_query_qp(ibv_dev_qp, &ibv_dev_qp_params, IBV_QP_CAP, &ibv_dev_qp_request);
 
 	// Definition of QP state to "Initialized" and access control
 	ibv_dev_qp_params.qp_state = IBV_QPS_INIT;
 	ibv_dev_qp_params.pkey_index = 0;
 	ibv_dev_qp_params.port_num = 1;
-	ibv_dev_qp_params.qp_access_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ;
+	ibv_dev_qp_params.qp_access_flags = access;
 
 	result = ibv_modify_qp(ibv_dev_qp, &ibv_dev_qp_params, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS);
+	if(result) {
+		return false;
+	}
+
+	ctx->qp = ibv_dev_qp;
+
+	return true;
+}
 
+bool capulet_rdma_ib_fill_base_udp(struct CapuletRdmaIbContext *ctx, struct ibv_mr *mr, struct CapuletNetUdpPacket *own) {
+	struct ibv_port_attr ibv_dev_attr;
+	union ibv_gid ibv_dev_gid;
+
+	int result;
+
+	result = ibv_query_port(ctx->ib_ctx, 1, &ibv_dev_attr);
+	if(result) {
+		return false;
+	}
+
+	result = ibv_query_gid(ctx->ib_ctx, 1, 0, &ibv_dev_gid);
 	if(result) {
-		return NULL;
+		return false;
 	}
 
-	// Returned pointer was allocated by ibv_create_qp
-	return ibv_dev_qp;
+	memcpy((void *) own->gid, (void *) ibv_dev_gid.raw, 16);
+
+	own->lid = ibv_dev_attr.lid;
+	own->qpn = ctx->qp->qp_num;
+	own->psn = lrand48() & 0xffffff;
+
+	own->addr = mr->addr;
+	own->size = mr->length;
+	own->key = mr->rkey;
+
+	return true;	
 }
 
-bool set_peer_informations(struct ibv_qp *ibv_dev_qp, uint32_t local_psn, uint32_t remote_psn, uint32_t remote_lid, uint32_t remote_qpn, uint8_t *remote_gid) {
+bool capulet_rdma_ib_set_peer_from_udp(struct CapuletRdmaIbContext *ctx, struct CapuletNetUdpContext *udp_ctx) {
 	struct ibv_qp_attr ibv_dev_qp_params;
 	union ibv_gid ibv_dev_gid;
 
 	int result;
 
 	// Insert GID into IBv structure
-	memcpy((void *) ibv_dev_gid.raw, (void *) remote_gid, 16);
+	memcpy((void *) ibv_dev_gid.raw, (void *) udp_ctx->remote->gid, 16);
 
 	// Change state from Init to RTR (Ready To Receive)
 	memset(&ibv_dev_qp_params, 0, sizeof(struct ibv_qp_attr));
 	ibv_dev_qp_params.qp_state = IBV_QPS_RTR;
 	ibv_dev_qp_params.path_mtu = IBV_MTU_1024;
-	ibv_dev_qp_params.dest_qp_num = remote_qpn;
-	ibv_dev_qp_params.rq_psn = remote_psn;
+	ibv_dev_qp_params.dest_qp_num = udp_ctx->remote->qpn;
+	ibv_dev_qp_params.rq_psn = udp_ctx->remote->psn;
 	ibv_dev_qp_params.max_dest_rd_atomic = 1;
 	ibv_dev_qp_params.min_rnr_timer = 12;
 
 	ibv_dev_qp_params.ah_attr.is_global = 1;
-	ibv_dev_qp_params.ah_attr.dlid = remote_lid;
+	ibv_dev_qp_params.ah_attr.dlid = udp_ctx->remote->lid;
 	ibv_dev_qp_params.ah_attr.sl = 0;
 	ibv_dev_qp_params.ah_attr.src_path_bits = 0;
 	ibv_dev_qp_params.ah_attr.port_num = 1;
@@ -108,9 +146,8 @@ bool set_peer_informations(struct ibv_qp *ibv_dev_qp, uint32_t local_psn, uint32
 	ibv_dev_qp_params.ah_attr.grh.dgid = ibv_dev_gid;
 	ibv_dev_qp_params.ah_attr.grh.sgid_index = 0;
 
-	result = ibv_modify_qp(ibv_dev_qp, &ibv_dev_qp_params, IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER);
+	result = ibv_modify_qp(ctx->qp, &ibv_dev_qp_params, IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER);
 	if(result) {
-		fprintf(stderr, "Could not set queue state to RTR with remote parameters\n");
 		return false;
 	}
 
@@ -120,19 +157,18 @@ bool set_peer_informations(struct ibv_qp *ibv_dev_qp, uint32_t local_psn, uint32
 	ibv_dev_qp_params.timeout = 0x12;
 	ibv_dev_qp_params.retry_cnt = 7;
 	ibv_dev_qp_params.rnr_retry = 7;
-	ibv_dev_qp_params.sq_psn = local_psn;
+	ibv_dev_qp_params.sq_psn = udp_ctx->local->psn;
 	ibv_dev_qp_params.max_rd_atomic = 1;
 
-	result = ibv_modify_qp(ibv_dev_qp, &ibv_dev_qp_params, IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_SQ_PSN | IBV_QP_MAX_QP_RD_ATOMIC);
+	result = ibv_modify_qp(ctx->qp, &ibv_dev_qp_params, IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_SQ_PSN | IBV_QP_MAX_QP_RD_ATOMIC);
 	if(result) {
-		fprintf(stderr, "Could not set queue state to RTS\n");
 		return false;
 	}
 
 	return true;
 }
 
-bool initialize_post_recv(struct ibv_qp *ibv_dev_qp, struct ibv_mr *ibv_dev_mr, int mr_size) {
+bool capulet_rdma_ib_post_recv(struct CapuletRdmaIbContext *ctx, struct ibv_mr *ibv_dev_mr, int mr_size) {
 	struct ibv_sge ibv_dev_sge;
 	struct ibv_recv_wr ibv_dev_rdma_wr;
 	struct ibv_recv_wr *ibv_dev_bad_wr = NULL;
@@ -150,7 +186,7 @@ bool initialize_post_recv(struct ibv_qp *ibv_dev_qp, struct ibv_mr *ibv_dev_mr,
 	ibv_dev_rdma_wr.sg_list = &ibv_dev_sge;
 	ibv_dev_rdma_wr.num_sge = 1;
 
-	result = ibv_post_recv(ibv_dev_qp, &ibv_dev_rdma_wr, &ibv_dev_bad_wr);
+	result = ibv_post_recv(ctx->qp, &ibv_dev_rdma_wr, &ibv_dev_bad_wr);
 
 	if(result) {
 		perror("Could not post Recv request");
@@ -160,7 +196,7 @@ bool initialize_post_recv(struct ibv_qp *ibv_dev_qp, struct ibv_mr *ibv_dev_mr,
 	return ibv_dev_bad_wr == NULL;
 }
 
-bool send_rdma_read(struct ibv_qp *ibv_dev_qp, struct ibv_mr *ibv_dev_mr, int mr_size, void *remote_addr, uint32_t remote_key) {
+bool capulet_rdma_ib_send_read(struct CapuletRdmaIbContext *ctx, struct CapuletNetUdpPacket *peer, struct ibv_mr *ibv_dev_mr, int mr_size) {
 	struct ibv_send_wr ibv_dev_rdma_wr;
 	struct ibv_sge ibv_dev_sge;
 	struct ibv_send_wr *ibv_dev_bad_wr = NULL;
@@ -180,10 +216,10 @@ bool send_rdma_read(struct ibv_qp *ibv_dev_qp, struct ibv_mr *ibv_dev_mr, int mr
 	ibv_dev_rdma_wr.opcode = IBV_WR_RDMA_READ;
 	ibv_dev_rdma_wr.send_flags = IBV_SEND_SIGNALED;
 
-	ibv_dev_rdma_wr.wr.rdma.remote_addr = (uintptr_t) remote_addr;
-	ibv_dev_rdma_wr.wr.rdma.rkey = remote_key;
+	ibv_dev_rdma_wr.wr.rdma.remote_addr = (uintptr_t) peer->addr;
+	ibv_dev_rdma_wr.wr.rdma.rkey = peer->key;
 
-	result = ibv_post_send(ibv_dev_qp, &ibv_dev_rdma_wr, &ibv_dev_bad_wr);
+	result = ibv_post_send(ctx->qp, &ibv_dev_rdma_wr, &ibv_dev_bad_wr);
 
 	if(result) {
 		perror("Could not post Send request");
@@ -193,3 +229,13 @@ bool send_rdma_read(struct ibv_qp *ibv_dev_qp, struct ibv_mr *ibv_dev_mr, int mr
 	return ibv_dev_bad_wr == NULL;
 }
 
+void capulet_rdma_ib_free(struct CapuletRdmaIbContext *ctx) {
+	if(ctx->cq && ctx->qp) {
+		ibv_destroy_qp(ctx->qp);
+		ibv_destroy_cq(ctx->cq);
+	}
+
+	ibv_dealloc_pd(ctx->pd);
+	ibv_close_device(ctx->ib_ctx);
+}
+

rdma_demo.c

-#include <stdio.h>
-#include <stdlib.h>
-#include <time.h>
-
-#include <infiniband/verbs.h>
-
-#include "sdr_rdma.h"
-#include "soft_rdma.h"
-#include "udp_rdma.h"
-
-int main(int argc, char *argv[]) {
-	/******************************
-	 *** Variables declarations ***
-	 ******************************/
-	struct ibv_context *ibv_dev_ctx;
-	struct ibv_pd *ibv_dev_pd;
-	struct ibv_cq *ibv_dev_cq;
-	struct ibv_qp *ibv_dev_qp;
-	struct ibv_mr *ibv_dev_mr;
-	struct ibv_wc ibv_dev_wc;
-	struct ibv_port_attr ibv_dev_attr;
-	union ibv_gid ibv_dev_gid;
-
-	struct udp_rdma_parameters local_params;
-	struct udp_rdma_parameters remote_params;
-	struct SDRMemoryRegion *server_mr;
-
-	uint16_t i;
-	int result;
-	int mr_size = sizeof(struct SDRMemoryRegion);
-	int page_size = sysconf(_SC_PAGESIZE);
-	char remote_host[16];
-	bool is_client = false;
-
-	srand(time(NULL));
-
-	/******************************
-	 ****** Memory allocation *****
-	 ******************************/
-	// Detect if a server was given; if any, act as a client
-	if(argc >= 3) {
-		if(strcmp(argv[1], "-c") == 0) {
-			strcpy((void *) remote_host, (void *) argv[2]);
-			is_client = true;
-		}
-	}
-
-	server_mr = (struct SDRMemoryRegion *) aligned_alloc(page_size, mr_size);
-	if(server_mr == NULL) {
-		fprintf(stderr, "Memory allocation failed (before registrering MR)\n");
-		return -1;
-	}
-
-	memset((void *) server_mr, 0, mr_size);
-
-	// Fill output IQ with (predictable) dummy data
-	if(!is_client) {
-		server_mr->meta.server_version = 0x01;
-		server_mr->meta.iq_output_length = 4096;
-
-		for(i = 0; i < 4096; i++) {
-			// Should not exceed 1.0
-			server_mr->iq_output[i] = (float) i / 4096.0;
-		}
-	}
-
-	/******************************
-	 ***** RDMA initialization ****
-	 ******************************/
-	ibv_dev_ctx = initialize_device_context("rxe0");
-	if(!ibv_dev_ctx) {
-		fprintf(stderr, "Device initialization failed\n");
-		return -1;
-	}
-
-	ibv_dev_pd = ibv_alloc_pd(ibv_dev_ctx);
-	if(!ibv_dev_pd) {
-		fprintf(stderr, "Protection Domain (PD) allocation failed\n");
-		return -1;
-	}
-
-	ibv_dev_mr = ibv_reg_mr(ibv_dev_pd, (void *) server_mr, mr_size, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE);
-	if(!ibv_dev_mr) {
-		fprintf(stderr, "Memory Region registration failed\n");
-		return -1;
-	}
-
-	ibv_dev_cq = ibv_create_cq(ibv_dev_ctx, 2, NULL, NULL, 0);
-	if(!ibv_dev_cq) {
-		fprintf(stderr, "Completion Queue creation failed\n");
-		return -1;
-	}
-
-	ibv_dev_qp = initialize_queue_pair(ibv_dev_pd, ibv_dev_cq);
-	if(!ibv_dev_qp) {
-		fprintf(stderr, "Queue Pair initialization failed\n");
-		return -1;
-	}
-
-	// RX buffer of size 1 is filled with Recv requests
-	if(!initialize_post_recv(ibv_dev_qp, ibv_dev_mr, mr_size)) {
-		return -1;
-	}
-
-	/******************************
-	 **** Exchange informations ***
-	 ******************************/
-	result = ibv_query_port(ibv_dev_ctx, 1, &ibv_dev_attr);
-	if(result) {
-		perror("Query port failed");
-		return -1;
-	}
-
-	result = ibv_query_gid(ibv_dev_ctx, 1, 0, &ibv_dev_gid);
-	if(result) {
-		fprintf(stderr, "Query GID failed\n");
-		return -1;
-	}
-	memcpy((void *) local_params.gid, (void *) ibv_dev_gid.raw, 16);
-
-	local_params.lid = ibv_dev_attr.lid;
-	local_params.qpn = ibv_dev_qp->qp_num;
-	local_params.psn = rand() & 0xffffff;
-	local_params.addr = ibv_dev_mr->addr;
-	local_params.size = ibv_dev_mr->length;
-	local_params.key = ibv_dev_mr->rkey;
-
-	udp_rdma_dump_packet(&local_params);
-
-	if(is_client) {
-		if(!udp_rdma_send_parameters(remote_host, &local_params, &remote_params)) {
-			return -1;
-		}
-	} else {
-		if(!udp_rdma_receive_parameters(&local_params, &remote_params)) {
-			return -1;
-		}
-	}
-
-	udp_rdma_dump_packet(&remote_params);
-
-	/******************************
-	 ****** Prepare for read ******
-	 ******************************/
-	if(!set_peer_informations(ibv_dev_qp, local_params.psn, remote_params.psn, remote_params.lid, remote_params.qpn, remote_params.gid)) {
-		return -1;
-	}
-
-	if(is_client) {
-		if(!send_rdma_read(ibv_dev_qp, ibv_dev_mr, mr_size, remote_params.addr, remote_params.key)) {
-			return -1;
-		}
-	}
-
-	do {
-		result = ibv_poll_cq(ibv_dev_cq, 1, &ibv_dev_wc);
-	} while(result == 0);
-
-	if(result > 0 && ibv_dev_wc.status == IBV_WC_SUCCESS) {
-		if(is_client) {
-			printf("%d %f\n", server_mr->meta.server_version, server_mr->iq_output[2048]);
-		}
-	} else {
-		printf("Failed: %s (WR %lu)\n", ibv_wc_status_str(ibv_dev_wc.status), ibv_dev_wc.wr_id);
-	}
-
-	/******************************
-	 ******* Global cleanup *******
-	 ******************************/
-	ibv_destroy_qp(ibv_dev_qp);
-	ibv_destroy_cq(ibv_dev_cq);
-	ibv_dereg_mr(ibv_dev_mr);
-	ibv_dealloc_pd(ibv_dev_pd);
-	ibv_close_device(ibv_dev_ctx);
-	free((void *) server_mr);
-
-	return 0;
-}
-