RDMA initialization: base for client and server

Add an example file which creates an RDMA device, allocates a Memory Region
and prepares a Queue Pair for a client to read the MR.

Makefile

 CFLAGS = -Wall -O3
-LIB_FILES = udp_rdma.c
 
-all: udp_send udp_recv
+all: udp_send udp_recv rdma_server
 
-udp_send: udp_send.c
-	gcc $(CFLAGS) -o bin/$@ $? $(LIB_FILES)
+udp_send: udp_send.c udp_rdma.c
+	gcc $(CFLAGS) -o bin/$@ $?
 
-udp_recv: udp_recv.c
-	gcc $(CFLAGS) -o bin/$@ $? $(LIB_FILES)
+udp_recv: udp_recv.c udp_rdma.c
+	gcc $(CFLAGS) -o bin/$@ $?
+
+rdma_server: rdma_server.c udp_rdma.c soft_rdma.c
+	gcc $(CFLAGS) -o bin/$@ $? -libverbs
 

README.md

@@ -2,3 +2,14 @@
 
 Minimal working example for RDMA with multiple slaves: one machine serves a portion of memory and the others can read it.
 
+## Configuring the RDMA interface
+
+In order to run this RDMA example, you will need to create a RDMA interface on two virtual machines:
+
+```bash
+modprobe rdma_rxe
+rdma link add rxe0 type rxe netdev enp1s0
+rdma link show
+mkdir -p /sys/kernel/config/rdma_cm/rxe0
+```
+

rdma_server.c

+#include <stdio.h>
+#include <stdlib.h>
+
+#include <infiniband/verbs.h>
+
+#include "soft_rdma.h"
+#include "udp_rdma.h"
+
+int main(void) {
+	/******************************
+	 *** Variables declarations ***
+	 ******************************/
+	struct ibv_context *ibv_dev_ctx;
+	struct ibv_pd *ibv_dev_pd;
+	struct ibv_cq *ibv_dev_cq;
+	struct ibv_qp *ibv_dev_qp;
+	struct ibv_mr *ibv_dev_mr;
+
+	void *memory_buffer;
+	int allocated_size = 16384 * sizeof(char);
+	int page_size = sysconf(_SC_PAGESIZE);
+
+	/******************************
+	 *** RDMA initialization (1) **
+	 ******************************/
+	ibv_dev_ctx = initialize_device_context("rxe0");
+	if(!ibv_dev_ctx) {
+		fprintf(stderr, "Device initialization failed\n");
+		return -1;
+	}
+
+	ibv_dev_pd = ibv_alloc_pd(ibv_dev_ctx);
+	if(!ibv_dev_pd) {
+		fprintf(stderr, "Protection Domain (PD) allocation failed\n");
+		return -1;
+	}
+
+	/******************************
+	 ****** Memory allocation *****
+	 ******************************/
+	memory_buffer = aligned_alloc(page_size, allocated_size);
+	if(!memory_buffer) {
+		fprintf(stderr, "Memory allocation failed (before registrering MR)\n");
+		return -1;
+	}
+
+	memset(memory_buffer, 0, allocated_size);
+
+	ibv_dev_mr = ibv_reg_mr(ibv_dev_pd, memory_buffer, allocated_size, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ);
+	if(!ibv_dev_mr) {
+		fprintf(stderr, "Memory Region registration failed\n");
+		return -1;
+	}
+
+	/******************************
+	 *** RDMA initialization (2) **
+	 ******************************/
+	ibv_dev_cq = ibv_create_cq(ibv_dev_ctx, 1025, NULL, NULL, 0);
+	if(!ibv_dev_cq) {
+		fprintf(stderr, "Completion Queue creation failed\n");
+		return -1;
+	}
+
+	ibv_dev_qp = initialize_queue_pair(ibv_dev_pd, ibv_dev_cq);
+	if(!ibv_dev_qp) {
+		fprintf(stderr, "Queue Pair initialization failed\n");
+		return -1;
+	}
+
+	// Global cleanup
+	ibv_destroy_qp(ibv_dev_qp);
+	ibv_destroy_cq(ibv_dev_cq);
+	ibv_dereg_mr(ibv_dev_mr);
+	free(memory_buffer);
+	ibv_dealloc_pd(ibv_dev_pd);
+	ibv_close_device(ibv_dev_ctx);
+
+	return 0;
+}
+

soft_rdma.c

+#include "soft_rdma.h"
+
+struct ibv_context *initialize_device_context(const char *device_name) {
+	struct ibv_device **dev_list;
+	struct ibv_device *ibv_dev = NULL;
+	struct ibv_context *ibv_dev_ctx;
+
+	int dev_list_length, dev_id;
+
+	// Devices cannot be selected by name, so all devices are listed
+	// and the name of each one is compared to the requested name.
+	dev_list = ibv_get_device_list(&dev_list_length);
+	if(!dev_list || dev_list_length < 1) {
+		return NULL;
+	}
+
+	for(dev_id = 0; dev_id < dev_list_length; dev_id++) {
+		const char *current_name = ibv_get_device_name(dev_list[dev_id]);
+		if(strcmp(device_name, current_name) == 0) {
+			ibv_dev = dev_list[dev_id];
+		}
+	}
+	
+	if(!ibv_dev) {
+		return NULL;
+	}
+
+	// The devices list can be freed as soon as context was obtained.
+	ibv_dev_ctx = ibv_open_device(ibv_dev);
+	ibv_free_device_list(dev_list);
+
+	// Returned pointer was allocated by ibv_open_device
+	return ibv_dev_ctx;
+}
+
+struct ibv_qp *initialize_queue_pair(struct ibv_pd *ibv_dev_pd, struct ibv_cq *ibv_dev_cq) {
+	struct ibv_qp_init_attr ibv_dev_qp_request;
+	struct ibv_qp_attr ibv_dev_qp_params;
+	struct ibv_qp *ibv_dev_qp;
+
+	int result;
+
+	memset(&ibv_dev_qp_request, 0, sizeof(struct ibv_qp_init_attr));
+	memset(&ibv_dev_qp_params, 0, sizeof(struct ibv_qp_attr));
+
+	// Two steps are required: creating the queue pair and
+	// initializing it.
+	// Creation of QP with initial parameters
+	ibv_dev_qp_request.qp_type = IBV_QPT_RC;
+	ibv_dev_qp_request.send_cq = ibv_dev_cq;
+	ibv_dev_qp_request.recv_cq = ibv_dev_cq;
+
+	ibv_dev_qp_request.cap.max_send_wr = 1;
+	ibv_dev_qp_request.cap.max_recv_wr = 1025;
+	ibv_dev_qp_request.cap.max_send_sge = 1;
+	ibv_dev_qp_request.cap.max_recv_sge = 1;
+
+	ibv_dev_qp = ibv_create_qp(ibv_dev_pd, &ibv_dev_qp_request);
+
+	if(!ibv_dev_qp) {
+		return NULL;
+	}
+
+	// Definition of QP state to "Initialized" and access control
+	ibv_dev_qp_params.qp_state = IBV_QPS_INIT;
+	ibv_dev_qp_params.pkey_index = 0;
+	ibv_dev_qp_params.port_num = 1;
+	ibv_dev_qp_params.qp_access_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ;
+
+	result = ibv_modify_qp(ibv_dev_qp, &ibv_dev_qp_params, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS);
+
+	if(result) {
+		return NULL;
+	}
+
+	// Returned pointer was allocated by ibv_create_qp
+	return ibv_dev_qp;
+}
+

soft_rdma.h

+#include <stdbool.h>
+
+#include <infiniband/verbs.h>
+
+struct ibv_context *initialize_device_context(const char *device_name);
+struct ibv_qp *initialize_queue_pair(struct ibv_pd *ibv_dev_pd, struct ibv_cq *ibv_dev_cq);
+