libcapulet: add an MR manager

- Registers MR automatically
- Allows referencing by user name

Makefile

@@ -11,7 +11,7 @@ clean:
 %.o: %.c
 	gcc -c $(CFLAGS) $(INCLUDE_FLAGS) $< -o $@
 
-out/libcapulet.a: libcapulet/net_udp.o libcapulet/rdma_ib.o
+out/libcapulet.a: libcapulet/net_udp.o libcapulet/rdma_ib.o libcapulet/rdma_mr_mgr.o
 	ar -rc $@ $^
 
 out/rdma_standalone: example/rdma_standalone.o out/libcapulet.a

example/rdma_standalone.c

@@ -7,21 +7,20 @@
 
 #include "net_udp.h"
 #include "rdma_ib.h"
+#include "rdma_mr_mgr.h"
 
 int main(int argc, char *argv[]) {
 	struct CapuletRdmaIbContext rdma_ctx;
 	struct CapuletNetUdpContext *udp_ctx;
+	struct CapuletRdmaMrMgrElement *main_mr_el;
 
-	struct ibv_mr *memory_buffer_mr;
 	struct ibv_wc poll_wc;
 
 	char remote_host[16];
 	char *memory_char_buffer;
 	bool result;
-	void *memory_buffer;
 
 	int allocated_size = 16384 * sizeof(char);
-	int page_size = sysconf(_SC_PAGESIZE);
 	bool is_client = false;
 
 	srand48(getpid() * time(NULL));
@@ -37,20 +36,6 @@ int main(int argc, char *argv[]) {
 		}
 	}
 
-	// Allocate memory and get user data from STDIN
-	memory_buffer = aligned_alloc(page_size, allocated_size);
-	if(!memory_buffer) {
-		fprintf(stderr, "Memory allocation failed (before registrering MR)\n");
-		return -1;
-	}
-
-	memset(memory_buffer, 0, allocated_size);
-	memory_char_buffer = (char *) memory_buffer;
-
-	if(!is_client) {
-		read(STDIN_FILENO, memory_char_buffer, allocated_size);
-	}
-
 	/******************************
 	 ***** RDMA initialization ****
 	 ******************************/
@@ -60,19 +45,26 @@ int main(int argc, char *argv[]) {
 		return -1;
 	}
 
-	memory_buffer_mr = ibv_reg_mr(rdma_ctx.pd, memory_buffer, allocated_size, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ);
-	if(!memory_buffer_mr) {
-		fprintf(stderr, "Memory Region registration failed\n");
+	// Allocate memory and get user data from STDIN
+	main_mr_el = capulet_rdma_mr_mgr_register(&rdma_ctx, "main", allocated_size, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ);
+	if(!main_mr_el) {
+		fprintf(stderr, "Memory registration failed\n");
 		return -1;
 	}
 
+	memory_char_buffer = (char *) main_mr_el->mr->addr;
+
+	if(!is_client) {
+		read(STDIN_FILENO, memory_char_buffer, allocated_size);
+	}
+
 	result = capulet_rdma_ib_initialize_qp(&rdma_ctx, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ);
 	if(!result) {
 		fprintf(stderr, "Queue Pair initialization failed\n");
 		return -1;
 	}
 
-	result = capulet_rdma_ib_post_recv(&rdma_ctx, memory_buffer_mr, allocated_size);
+	result = capulet_rdma_ib_post_recv(&rdma_ctx, main_mr_el->mr, allocated_size);
 	if(!result) {
 		fprintf(stderr, "Posting Recv failed\n");
 		return -1;
@@ -87,7 +79,7 @@ int main(int argc, char *argv[]) {
 		return -1;
 	}
 
-	result = capulet_rdma_ib_fill_base_udp(&rdma_ctx, memory_buffer_mr, udp_ctx->local);
+	result = capulet_rdma_ib_fill_base_udp(&rdma_ctx, main_mr_el->mr, udp_ctx->local);
 	if(!result) {
 		fprintf(stderr, "Query port failed\n");
 		return -1;
@@ -118,7 +110,7 @@ int main(int argc, char *argv[]) {
 	}
 
 	if(is_client) {
-		result = capulet_rdma_ib_send_read(&rdma_ctx, udp_ctx->remote, memory_buffer_mr, allocated_size);
+		result = capulet_rdma_ib_send_read(&rdma_ctx, udp_ctx->remote, main_mr_el->mr, allocated_size);
 		if(!result) {
 			fprintf(stderr, "Sending Read failed\n");
 			return -1;
@@ -141,9 +133,7 @@ int main(int argc, char *argv[]) {
 	 ******* Global cleanup *******
 	 ******************************/
 	capulet_net_udp_free(udp_ctx);
-	ibv_dereg_mr(memory_buffer_mr);
 	capulet_rdma_ib_free(&rdma_ctx);
-	free(memory_buffer);
 
 	return 0;
 }

libcapulet/include/rdma_mr_mgr.h

+#pragma once
+
+#include <infiniband/verbs.h>
+
+#include "rdma_ib.h"
+
+struct CapuletRdmaMrMgrElement {
+	char *name;
+	struct ibv_mr *mr;
+	struct CapuletRdmaMrMgrElement *next;
+};
+
+struct CapuletRdmaMrMgrElement *capulet_rdma_mr_mgr_register(struct CapuletRdmaIbContext *ib_ctx, char *name, size_t size, int flags);
+struct CapuletRdmaMrMgrElement *capulet_rdma_mr_mgr_find(char *name);
+

libcapulet/rdma_mr_mgr.c

+#include "rdma_mr_mgr.h"
+
+static struct CapuletRdmaMrMgrElement *capulet_rdma_mr_mgr_hashtable[171];
+
+unsigned char _capulet_rdma_mr_mgr_compute_hash(char *str) {
+	unsigned char result = 0;
+
+	while(*str != '\0') {
+		result += (unsigned char) *str + 17 * result;
+		str++;
+	}
+
+	return result % 171;
+}
+
+struct CapuletRdmaMrMgrElement *capulet_rdma_mr_mgr_register(struct CapuletRdmaIbContext *ib_ctx, char *name, size_t size, int flags) {
+	struct ibv_mr *memory_buffer_mr;
+
+	struct CapuletRdmaMrMgrElement *el;
+
+	unsigned char hash;
+	void *memory_buffer;
+
+	int page_size = sysconf(_SC_PAGESIZE);
+
+	// Find if an element with the same name was already registered
+	el = capulet_rdma_mr_mgr_find(name);
+	if(el != NULL) {
+		return NULL;
+	}
+
+	// Allocate a memory buffer
+	memory_buffer = aligned_alloc(page_size, size);
+	if(!memory_buffer) {
+		return NULL;
+	}
+
+	memset(memory_buffer, 0, size);
+
+	// Register a RDMA MR
+	memory_buffer_mr = ibv_reg_mr(ib_ctx->pd, memory_buffer, size, flags);
+	if(!memory_buffer_mr) {
+		return NULL;
+	}
+
+	// Allocate a new element
+	el = (struct CapuletRdmaMrMgrElement *) malloc(sizeof(struct CapuletRdmaMrMgrElement));
+	if(!el) {
+		return NULL;
+	}
+
+	el->name = (char *) calloc(16, sizeof(char));
+	if(!el->name || !memcpy(el->name, name, 16)) {
+		return NULL;
+	}
+
+	// Store informations
+	hash = _capulet_rdma_mr_mgr_compute_hash(name);
+
+	el->mr = memory_buffer_mr;
+	el->next = capulet_rdma_mr_mgr_hashtable[hash];
+
+	capulet_rdma_mr_mgr_hashtable[hash] = el;
+
+	return el;
+}
+
+struct CapuletRdmaMrMgrElement *capulet_rdma_mr_mgr_find(char *name) {
+	struct CapuletRdmaMrMgrElement *el;
+	unsigned char hash;
+
+	hash = _capulet_rdma_mr_mgr_compute_hash(name);
+	el = capulet_rdma_mr_mgr_hashtable[hash];
+
+	while(el != NULL) {
+		if(strcmp(el->name, name) == 0) {
+			return el;
+		}
+
+		el = el->next;
+	}
+
+	return NULL;
+}
+