Add an RDMA Write method

README.md

@@ -28,7 +28,7 @@ The other machine will act as a client, so we need to pass it the IP address of
 
 ```bash
 make install
-rdma_standalone -c 192.168.16.10
+rdma_standalone -c 192.168.16.10 < Makefile
 ```
 
 ## TRX driver

example/rdma_standalone.c

@@ -14,7 +14,8 @@ int main(int argc, char *argv[]) {
 	struct CapuletNetUdpContext *udp_ctx;
 	struct CapuletRdmaMrMgrElement *in_mr_el;
 	struct CapuletRdmaMrMgrElement *out_mr_el;
-	struct CapuletNetUdpMrInfoPacket mr_info_packet;
+	struct CapuletNetUdpMrInfoPacket in_info_packet;
+	struct CapuletNetUdpMrInfoPacket out_info_packet;
 
 	struct ibv_wc poll_wc;
 	struct addrinfo server_hints;
@@ -22,7 +23,6 @@ int main(int argc, char *argv[]) {
 
 	int server_socket;
 	char remote_host[16];
-	char *memory_char_buffer;
 	bool result;
 
 	int allocated_size = 16384 * sizeof(char);
@@ -70,8 +70,11 @@ int main(int argc, char *argv[]) {
 		return -1;
 	}
 
-	memory_char_buffer = (char *) out_mr_el->mr->addr;
-	read(STDIN_FILENO, memory_char_buffer, allocated_size);
+	if(is_client) {
+		read(STDIN_FILENO, (char *) in_mr_el->mr->addr, allocated_size);
+	} else {
+		read(STDIN_FILENO, (char *) out_mr_el->mr->addr, allocated_size);
+	}
 
 	result = capulet_rdma_ib_initialize_qp(&rdma_ctx, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE);
 	if(!result) {
@@ -136,18 +139,30 @@ int main(int argc, char *argv[]) {
 		return -1;
 	}
 
-	result = capulet_net_udp_query_mr(server_socket, server_infos, "out", &mr_info_packet);
-	if(!result || (strcmp(mr_info_packet.name, "out") != 0)) {
-		fprintf(stderr, "Querying MR from server failed\n");
+	result = capulet_net_udp_query_mr(server_socket, server_infos, "in", &in_info_packet);
+	if(!result || (strcmp(in_info_packet.name, "in") != 0)) {
+		fprintf(stderr, "Querying MR (in) from server failed\n");
+		return -1;
+	}
+
+	result = capulet_net_udp_query_mr(server_socket, server_infos, "out", &out_info_packet);
+	if(!result || (strcmp(out_info_packet.name, "out") != 0)) {
+		fprintf(stderr, "Querying MR (out) from server failed\n");
 		return -1;
 	}
 
-	result = capulet_rdma_ib_send_read(&rdma_ctx, &mr_info_packet, out_mr_el->mr, allocated_size);
+	result = capulet_rdma_ib_send_read(&rdma_ctx, &out_info_packet, out_mr_el->mr, allocated_size);
 	if(!result) {
 		fprintf(stderr, "Sending Read failed\n");
 		return -1;
 	}
 
+	result = capulet_rdma_ib_send_write(&rdma_ctx, &in_info_packet, in_mr_el->mr, allocated_size);
+	if(!result) {
+		fprintf(stderr, "Sending Write failed\n");
+		return -1;
+	}
+
 	/******************************
 	 ****** Poll completion *******
 	 ******************************/
@@ -157,7 +172,7 @@ int main(int argc, char *argv[]) {
 
 	if(result > 0 && poll_wc.status == IBV_WC_SUCCESS) {
 		if(is_client) {
-			printf("%s\n", memory_char_buffer);
+			printf("%s\n", (char *) out_mr_el->mr->addr);
 		}
 	} else {
 		printf("Failed: %s (WR %lu)\n", ibv_wc_status_str(poll_wc.status), poll_wc.wr_id);

libcapulet/include/rdma_ib.h

@@ -21,5 +21,6 @@ bool capulet_rdma_ib_fill_base_udp(struct CapuletRdmaIbContext *ctx, struct Capu
 bool capulet_rdma_ib_set_peer_from_udp(struct CapuletRdmaIbContext *ctx, struct CapuletNetUdpContext *udp_ctx);
 bool capulet_rdma_ib_post_recv(struct CapuletRdmaIbContext *ctx, struct ibv_mr *ibv_dev_mr, int mr_size);
 bool capulet_rdma_ib_send_read(struct CapuletRdmaIbContext *ctx, struct CapuletNetUdpMrInfoPacket *remote_mr, struct ibv_mr *local_mr, int local_mr_size);
+bool capulet_rdma_ib_send_write(struct CapuletRdmaIbContext *ctx, struct CapuletNetUdpMrInfoPacket *remote_mr, struct ibv_mr *local_mr, int local_mr_size);
 void capulet_rdma_ib_free(struct CapuletRdmaIbContext *ctx);
 

libcapulet/net_udp.c

@@ -93,7 +93,7 @@ bool capulet_net_udp_serve_mr(int server_socket) {
 
 		mr_el = capulet_rdma_mr_mgr_find(mr_query_pkt.name);
 		if(mr_el) {
-			memcpy((void *) mr_info_pkt.name, (void *) mr_query_pkt.name, 16);
+			memcpy((void *) mr_info_pkt.name, (void *) mr_el->name, 16);
 			mr_info_pkt.addr = mr_el->mr->addr;
 			mr_info_pkt.size = mr_el->mr->length;
 			mr_info_pkt.key = mr_el->mr->rkey;

libcapulet/rdma_ib.c

@@ -175,7 +175,7 @@ bool capulet_rdma_ib_post_recv(struct CapuletRdmaIbContext *ctx, struct ibv_mr *
 	ibv_dev_sge.length = mr_size;
 	ibv_dev_sge.lkey = ibv_dev_mr->lkey;
 
-	ibv_dev_rdma_wr.wr_id = (uint64_t) rand();
+	ibv_dev_rdma_wr.wr_id = (uint64_t) lrand48();
 	ibv_dev_rdma_wr.sg_list = &ibv_dev_sge;
 	ibv_dev_rdma_wr.num_sge = 1;
 
@@ -203,7 +203,7 @@ bool capulet_rdma_ib_send_read(struct CapuletRdmaIbContext *ctx, struct CapuletN
 	ibv_dev_sge.length = local_mr_size;
 	ibv_dev_sge.lkey = local_mr->lkey;
 
-	ibv_dev_rdma_wr.wr_id = (uint64_t) rand();
+	ibv_dev_rdma_wr.wr_id = (uint64_t) lrand48();
 	ibv_dev_rdma_wr.sg_list = &ibv_dev_sge;
 	ibv_dev_rdma_wr.num_sge = 1;
 	ibv_dev_rdma_wr.opcode = IBV_WR_RDMA_READ;
@@ -222,6 +222,39 @@ bool capulet_rdma_ib_send_read(struct CapuletRdmaIbContext *ctx, struct CapuletN
 	return ibv_dev_bad_wr == NULL;
 }
 
+bool capulet_rdma_ib_send_write(struct CapuletRdmaIbContext *ctx, struct CapuletNetUdpMrInfoPacket *remote_mr, struct ibv_mr *local_mr, int local_mr_size) {
+	struct ibv_send_wr ibv_dev_rdma_wr;
+	struct ibv_sge ibv_dev_sge;
+	struct ibv_send_wr *ibv_dev_bad_wr = NULL;
+
+	int result;
+
+	memset(&ibv_dev_sge, 0, sizeof(struct ibv_sge));
+	memset(&ibv_dev_rdma_wr, 0, sizeof(struct ibv_send_wr));
+
+	ibv_dev_sge.addr = (uintptr_t) local_mr->addr;
+	ibv_dev_sge.length = local_mr_size;
+	ibv_dev_sge.lkey = local_mr->lkey;
+
+	ibv_dev_rdma_wr.wr_id = (uint64_t) lrand48();
+	ibv_dev_rdma_wr.sg_list = &ibv_dev_sge;
+	ibv_dev_rdma_wr.num_sge = 1;
+	ibv_dev_rdma_wr.opcode = IBV_WR_RDMA_WRITE;
+	ibv_dev_rdma_wr.send_flags = IBV_SEND_SIGNALED;
+
+	ibv_dev_rdma_wr.wr.rdma.remote_addr = (uintptr_t) remote_mr->addr;
+	ibv_dev_rdma_wr.wr.rdma.rkey = remote_mr->key;
+
+	result = ibv_post_send(ctx->qp, &ibv_dev_rdma_wr, &ibv_dev_bad_wr);
+
+	if(result) {
+		perror("Could not post Send request");
+		return false;
+	}
+
+	return ibv_dev_bad_wr == NULL;
+}
+
 void capulet_rdma_ib_free(struct CapuletRdmaIbContext *ctx) {
 	if(ctx->cq && ctx->qp) {
 		ibv_destroy_qp(ctx->qp);