Commit 562d2ad7 authored by Titouan Soulard's avatar Titouan Soulard

Add RDMA client file

While the RDMA server and client are very similar, two files are currently used
for an development purposes. The client acts as an UDP client and will try to
read available memory from it's peer.
parent b854a23f
CFLAGS = -Wall -O3 CFLAGS = -Wall -O3
all: udp_send udp_recv rdma_server all: udp_send udp_recv rdma_server rdma_client
clean: clean:
rm bin/* rm bin/*
...@@ -14,3 +14,6 @@ udp_recv: udp_recv.c udp_rdma.c ...@@ -14,3 +14,6 @@ udp_recv: udp_recv.c udp_rdma.c
rdma_server: rdma_server.c udp_rdma.c soft_rdma.c rdma_server: rdma_server.c udp_rdma.c soft_rdma.c
gcc $(CFLAGS) -o bin/$@ $? -libverbs gcc $(CFLAGS) -o bin/$@ $? -libverbs
rdma_client: rdma_client.c udp_rdma.c soft_rdma.c
gcc $(CFLAGS) -o bin/$@ $? -libverbs
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <infiniband/verbs.h>
#include "soft_rdma.h"
#include "udp_rdma.h"
int main(void) {
/******************************
*** Variables declarations ***
******************************/
struct ibv_context *ibv_dev_ctx;
struct ibv_pd *ibv_dev_pd;
struct ibv_cq *ibv_dev_cq;
struct ibv_qp *ibv_dev_qp;
struct ibv_mr *ibv_dev_mr;
struct ibv_port_attr ibv_dev_attr;
union ibv_gid ibv_dev_gid;
struct udp_rdma_parameters local_params;
struct udp_rdma_parameters remote_params;
const char remote_host[16] = "192.168.16.10";
void *memory_buffer;
int result;
int allocated_size = 16384 * sizeof(char);
int page_size = sysconf(_SC_PAGESIZE);
srand(time(NULL));
/******************************
*** RDMA initialization (1) **
******************************/
ibv_dev_ctx = initialize_device_context("rxe0");
if(!ibv_dev_ctx) {
fprintf(stderr, "Device initialization failed\n");
return -1;
}
ibv_dev_pd = ibv_alloc_pd(ibv_dev_ctx);
if(!ibv_dev_pd) {
fprintf(stderr, "Protection Domain (PD) allocation failed\n");
return -1;
}
/******************************
****** Memory allocation *****
******************************/
// XXX: should not be needed for client
memory_buffer = aligned_alloc(page_size, allocated_size);
if(!memory_buffer) {
fprintf(stderr, "Memory allocation failed (before registrering MR)\n");
return -1;
}
memset(memory_buffer, 0, allocated_size);
ibv_dev_mr = ibv_reg_mr(ibv_dev_pd, memory_buffer, allocated_size, IBV_ACCESS_LOCAL_WRITE);
if(!ibv_dev_mr) {
fprintf(stderr, "Memory Region registration failed\n");
return -1;
}
/******************************
*** RDMA initialization (2) **
******************************/
ibv_dev_cq = ibv_create_cq(ibv_dev_ctx, 1025, NULL, NULL, 0);
if(!ibv_dev_cq) {
fprintf(stderr, "Completion Queue creation failed\n");
return -1;
}
ibv_dev_qp = initialize_queue_pair(ibv_dev_pd, ibv_dev_cq);
if(!ibv_dev_qp) {
fprintf(stderr, "Queue Pair initialization failed\n");
return -1;
}
/******************************
**** Exchange informations ***
******************************/
result = ibv_query_port(ibv_dev_ctx, 1, &ibv_dev_attr);
if(result) {
perror("Query port failed");
return -1;
}
result = ibv_query_gid(ibv_dev_ctx, 1, 0, &ibv_dev_gid);
if(result) {
fprintf(stderr, "Query GID failed\n");
return -1;
}
memcpy((void *) local_params.gid, (void *) ibv_dev_gid.raw, 16);
// Send local parameters to a server
local_params.lid = ibv_dev_attr.lid;
local_params.qpn = ibv_dev_qp->qp_num;
local_params.psn = rand() & 0xffffff;
local_params.addr = ibv_dev_mr->addr;
local_params.size = ibv_dev_mr->length;
local_params.key = ibv_dev_mr->rkey;
udp_rdma_dump_packet(&local_params);
if(!udp_rdma_send_parameters(remote_host, &local_params, &remote_params)) {
return -1;
}
udp_rdma_dump_packet(&remote_params);
/******************************
******* RDMA connection ******
******************************/
if(!set_peer_informations(ibv_dev_qp, remote_params.psn, remote_params.lid, remote_params.qpn, remote_params.gid)) {
return -1;
}
/******************************
******* Global cleanup *******
******************************/
ibv_destroy_qp(ibv_dev_qp);
ibv_destroy_cq(ibv_dev_cq);
ibv_dereg_mr(ibv_dev_mr);
free(memory_buffer);
ibv_dealloc_pd(ibv_dev_pd);
ibv_close_device(ibv_dev_ctx);
return 0;
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment