Commit 6e7743f4 authored by Titouan Soulard's avatar Titouan Soulard

RDMA initialization: base for client and server

Add an example file which creates an RDMA device, allocates a Memory Region
and prepares a Queue Pair for a client to read the MR.
parent 48508daa
CFLAGS = -Wall -O3
LIB_FILES = udp_rdma.c
all: udp_send udp_recv
all: udp_send udp_recv rdma_server
udp_send: udp_send.c
gcc $(CFLAGS) -o bin/$@ $? $(LIB_FILES)
udp_send: udp_send.c udp_rdma.c
gcc $(CFLAGS) -o bin/$@ $?
udp_recv: udp_recv.c
gcc $(CFLAGS) -o bin/$@ $? $(LIB_FILES)
udp_recv: udp_recv.c udp_rdma.c
gcc $(CFLAGS) -o bin/$@ $?
rdma_server: rdma_server.c udp_rdma.c soft_rdma.c
gcc $(CFLAGS) -o bin/$@ $? -libverbs
......@@ -2,3 +2,14 @@
Minimal working example for RDMA with multiple slaves: one machine serves a portion of memory and the others can read it.
## Configuring the RDMA interface
In order to run this RDMA example, you will need to create a RDMA interface on two virtual machines:
```bash
modprobe rdma_rxe
rdma link add rxe0 type rxe netdev enp1s0
rdma link show
mkdir -p /sys/kernel/config/rdma_cm/rxe0
```
#include <stdio.h>
#include <stdlib.h>
#include <infiniband/verbs.h>
#include "soft_rdma.h"
#include "udp_rdma.h"
int main(void) {
/******************************
*** Variables declarations ***
******************************/
struct ibv_context *ibv_dev_ctx;
struct ibv_pd *ibv_dev_pd;
struct ibv_cq *ibv_dev_cq;
struct ibv_qp *ibv_dev_qp;
struct ibv_mr *ibv_dev_mr;
void *memory_buffer;
int allocated_size = 16384 * sizeof(char);
int page_size = sysconf(_SC_PAGESIZE);
/******************************
*** RDMA initialization (1) **
******************************/
ibv_dev_ctx = initialize_device_context("rxe0");
if(!ibv_dev_ctx) {
fprintf(stderr, "Device initialization failed\n");
return -1;
}
ibv_dev_pd = ibv_alloc_pd(ibv_dev_ctx);
if(!ibv_dev_pd) {
fprintf(stderr, "Protection Domain (PD) allocation failed\n");
return -1;
}
/******************************
****** Memory allocation *****
******************************/
memory_buffer = aligned_alloc(page_size, allocated_size);
if(!memory_buffer) {
fprintf(stderr, "Memory allocation failed (before registrering MR)\n");
return -1;
}
memset(memory_buffer, 0, allocated_size);
ibv_dev_mr = ibv_reg_mr(ibv_dev_pd, memory_buffer, allocated_size, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ);
if(!ibv_dev_mr) {
fprintf(stderr, "Memory Region registration failed\n");
return -1;
}
/******************************
*** RDMA initialization (2) **
******************************/
ibv_dev_cq = ibv_create_cq(ibv_dev_ctx, 1025, NULL, NULL, 0);
if(!ibv_dev_cq) {
fprintf(stderr, "Completion Queue creation failed\n");
return -1;
}
ibv_dev_qp = initialize_queue_pair(ibv_dev_pd, ibv_dev_cq);
if(!ibv_dev_qp) {
fprintf(stderr, "Queue Pair initialization failed\n");
return -1;
}
// Global cleanup
ibv_destroy_qp(ibv_dev_qp);
ibv_destroy_cq(ibv_dev_cq);
ibv_dereg_mr(ibv_dev_mr);
free(memory_buffer);
ibv_dealloc_pd(ibv_dev_pd);
ibv_close_device(ibv_dev_ctx);
return 0;
}
#include "soft_rdma.h"
struct ibv_context *initialize_device_context(const char *device_name) {
struct ibv_device **dev_list;
struct ibv_device *ibv_dev = NULL;
struct ibv_context *ibv_dev_ctx;
int dev_list_length, dev_id;
// Devices cannot be selected by name, so all devices are listed
// and the name of each one is compared to the requested name.
dev_list = ibv_get_device_list(&dev_list_length);
if(!dev_list || dev_list_length < 1) {
return NULL;
}
for(dev_id = 0; dev_id < dev_list_length; dev_id++) {
const char *current_name = ibv_get_device_name(dev_list[dev_id]);
if(strcmp(device_name, current_name) == 0) {
ibv_dev = dev_list[dev_id];
}
}
if(!ibv_dev) {
return NULL;
}
// The devices list can be freed as soon as context was obtained.
ibv_dev_ctx = ibv_open_device(ibv_dev);
ibv_free_device_list(dev_list);
// Returned pointer was allocated by ibv_open_device
return ibv_dev_ctx;
}
struct ibv_qp *initialize_queue_pair(struct ibv_pd *ibv_dev_pd, struct ibv_cq *ibv_dev_cq) {
struct ibv_qp_init_attr ibv_dev_qp_request;
struct ibv_qp_attr ibv_dev_qp_params;
struct ibv_qp *ibv_dev_qp;
int result;
memset(&ibv_dev_qp_request, 0, sizeof(struct ibv_qp_init_attr));
memset(&ibv_dev_qp_params, 0, sizeof(struct ibv_qp_attr));
// Two steps are required: creating the queue pair and
// initializing it.
// Creation of QP with initial parameters
ibv_dev_qp_request.qp_type = IBV_QPT_RC;
ibv_dev_qp_request.send_cq = ibv_dev_cq;
ibv_dev_qp_request.recv_cq = ibv_dev_cq;
ibv_dev_qp_request.cap.max_send_wr = 1;
ibv_dev_qp_request.cap.max_recv_wr = 1025;
ibv_dev_qp_request.cap.max_send_sge = 1;
ibv_dev_qp_request.cap.max_recv_sge = 1;
ibv_dev_qp = ibv_create_qp(ibv_dev_pd, &ibv_dev_qp_request);
if(!ibv_dev_qp) {
return NULL;
}
// Definition of QP state to "Initialized" and access control
ibv_dev_qp_params.qp_state = IBV_QPS_INIT;
ibv_dev_qp_params.pkey_index = 0;
ibv_dev_qp_params.port_num = 1;
ibv_dev_qp_params.qp_access_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ;
result = ibv_modify_qp(ibv_dev_qp, &ibv_dev_qp_params, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS);
if(result) {
return NULL;
}
// Returned pointer was allocated by ibv_create_qp
return ibv_dev_qp;
}
#include <stdbool.h>
#include <infiniband/verbs.h>
struct ibv_context *initialize_device_context(const char *device_name);
struct ibv_qp *initialize_queue_pair(struct ibv_pd *ibv_dev_pd, struct ibv_cq *ibv_dev_cq);
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment