libcapulet: allow adding a custom prefix to WR id

example/rdma_test.c

@@ -203,13 +203,13 @@ int main(int argc, char *argv[]) {
 			return -1;
 		}
 
-		bool_result = capulet_rdma_ib_post_recv(&rdma_ctx, in_mr, allocated_size);
+		bool_result = capulet_rdma_ib_post_recv(&rdma_ctx, in_mr, allocated_size, 0);
 		if(!bool_result) {
 			fprintf(stderr, "Posting Recv (in) failed\n");
 			return -1;
 		}
 
-		bool_result = capulet_rdma_ib_post_recv(&rdma_ctx, out_mr, allocated_size);
+		bool_result = capulet_rdma_ib_post_recv(&rdma_ctx, out_mr, allocated_size, 0);
 		if(!bool_result) {
 			fprintf(stderr, "Posting Recv (out) failed\n");
 			return -1;
@@ -233,13 +233,13 @@ int main(int argc, char *argv[]) {
 				return -1;
 			}
 
-			bool_result = capulet_rdma_ib_post_send(&rdma_ctx, IBV_WR_RDMA_READ, out_mr, allocated_size, &out_info_packet);
+			bool_result = capulet_rdma_ib_post_send(&rdma_ctx, IBV_WR_RDMA_READ, out_mr, allocated_size, 0, &out_info_packet);
 			if(!bool_result) {
 				fprintf(stderr, "Sending Read failed\n");
 				return -1;
 			}
 
-			bool_result = capulet_rdma_ib_post_send(&rdma_ctx, IBV_WR_RDMA_WRITE, in_mr, allocated_size, &in_info_packet);
+			bool_result = capulet_rdma_ib_post_send(&rdma_ctx, IBV_WR_RDMA_WRITE, in_mr, allocated_size, 0, &in_info_packet);
 			if(!bool_result) {
 				fprintf(stderr, "Sending Write failed\n");
 				return -1;
@@ -276,7 +276,7 @@ int main(int argc, char *argv[]) {
 	} else if(mode == RDMA_TEST_STRESS) {
 		// Fill the Recv queue
 		for(int i = 0; i < 16; i++) {
-			capulet_rdma_ib_post_recv(&rdma_ctx, in_mr, allocated_size);
+			capulet_rdma_ib_post_recv(&rdma_ctx, in_mr, allocated_size, 0);
 		}
 
 		// A small delay is needed to avoid sending anything before Recv requests
@@ -287,7 +287,7 @@ int main(int argc, char *argv[]) {
 			clock_gettime(CLOCK_MONOTONIC, &timestamps[0]);
 
 			// Wait for Send
-			capulet_rdma_ib_post_send(&rdma_ctx, IBV_WR_SEND, out_mr, allocated_size, NULL);
+			capulet_rdma_ib_post_send(&rdma_ctx, IBV_WR_SEND, out_mr, allocated_size, 0, NULL);
 			do {
 				poll_result = ibv_poll_cq(rdma_ctx.send_cq, 1, &poll_wc);
 			} while(poll_result == 0);
@@ -309,7 +309,7 @@ int main(int argc, char *argv[]) {
 
 			// The Recv queue should always be full for optimal performances: once
 			// an element is consumed, push back to it.
-			capulet_rdma_ib_post_recv(&rdma_ctx, in_mr, allocated_size);
+			capulet_rdma_ib_post_recv(&rdma_ctx, in_mr, allocated_size, 0);
 			clock_gettime(CLOCK_MONOTONIC, &timestamps[1]);
 
 			elapsed_time = (timestamps[1].tv_sec - timestamps[0].tv_sec) * 1000000000 + timestamps[1].tv_nsec - timestamps[0].tv_nsec;

include/libcapulet/rdma_ib.h

@@ -21,8 +21,8 @@ bool capulet_rdma_ib_initialize_qp(struct CapuletRdmaIbContext *ctx, int access)
 struct ibv_mr *capulet_rdma_ib_create_mr(struct CapuletRdmaIbContext *ib_ctx, size_t size, int flags);
 bool capulet_rdma_ib_fill_base_udp(struct CapuletRdmaIbContext *ctx, struct CapuletNetUdpQpInfoPacket *own);
 bool capulet_rdma_ib_set_peer_from_udp(struct CapuletRdmaIbContext *ctx, struct CapuletNetUdpContext *udp_ctx);
-bool capulet_rdma_ib_post_recv(struct CapuletRdmaIbContext *ctx, struct ibv_mr *ibv_dev_mr, int mr_size);
-bool capulet_rdma_ib_post_send(struct CapuletRdmaIbContext *ctx, enum ibv_wr_opcode opcode, struct ibv_mr *local_mr, int local_mr_size, struct CapuletNetUdpMrInfoPacket *remote_mr);
+bool capulet_rdma_ib_post_recv(struct CapuletRdmaIbContext *ctx, struct ibv_mr *ibv_dev_mr, int mr_size, uint32_t id_prefix);
+bool capulet_rdma_ib_post_send(struct CapuletRdmaIbContext *ctx, enum ibv_wr_opcode opcode, struct ibv_mr *local_mr, int local_mr_size, uint32_t id_prefix, struct CapuletNetUdpMrInfoPacket *remote_mr);
 void capulet_rdma_ib_free_mr(struct ibv_mr *mr);
 void capulet_rdma_ib_free(struct CapuletRdmaIbContext *ctx);
 

libcapulet/rdma_ib.c

@@ -187,7 +187,7 @@ bool capulet_rdma_ib_set_peer_from_udp(struct CapuletRdmaIbContext *ctx, struct
 	return true;
 }
 
-bool capulet_rdma_ib_post_recv(struct CapuletRdmaIbContext *ctx, struct ibv_mr *ibv_dev_mr, int mr_size) {
+bool capulet_rdma_ib_post_recv(struct CapuletRdmaIbContext *ctx, struct ibv_mr *ibv_dev_mr, int mr_size, uint32_t id_prefix) {
 	struct ibv_sge ibv_dev_sge;
 	struct ibv_recv_wr ibv_dev_rdma_wr;
 	struct ibv_recv_wr *ibv_dev_bad_wr = NULL;
@@ -201,12 +201,11 @@ bool capulet_rdma_ib_post_recv(struct CapuletRdmaIbContext *ctx, struct ibv_mr *
 	ibv_dev_sge.length = mr_size;
 	ibv_dev_sge.lkey = ibv_dev_mr->lkey;
 
-	ibv_dev_rdma_wr.wr_id = (uint64_t) lrand48();
+	ibv_dev_rdma_wr.wr_id = (((uint64_t) id_prefix) << 32) | ((uint64_t) lrand48());
 	ibv_dev_rdma_wr.sg_list = &ibv_dev_sge;
 	ibv_dev_rdma_wr.num_sge = 1;
 
 	result = ibv_post_recv(ctx->qp, &ibv_dev_rdma_wr, &ibv_dev_bad_wr);
-
 	if(result) {
 		perror("Could not post Recv request");
 		return false;
@@ -215,7 +214,7 @@ bool capulet_rdma_ib_post_recv(struct CapuletRdmaIbContext *ctx, struct ibv_mr *
 	return ibv_dev_bad_wr == NULL;
 }
 
-bool capulet_rdma_ib_post_send(struct CapuletRdmaIbContext *ctx, enum ibv_wr_opcode opcode, struct ibv_mr *local_mr, int local_mr_size, struct CapuletNetUdpMrInfoPacket *remote_mr) {
+bool capulet_rdma_ib_post_send(struct CapuletRdmaIbContext *ctx, enum ibv_wr_opcode opcode, struct ibv_mr *local_mr, int local_mr_size, uint32_t id_prefix, struct CapuletNetUdpMrInfoPacket *remote_mr) {
 	struct ibv_send_wr ibv_dev_rdma_wr;
 	struct ibv_sge ibv_dev_sge;
 	struct ibv_send_wr *ibv_dev_bad_wr = NULL;
@@ -229,7 +228,7 @@ bool capulet_rdma_ib_post_send(struct CapuletRdmaIbContext *ctx, enum ibv_wr_opc
 	ibv_dev_sge.length = local_mr_size;
 	ibv_dev_sge.lkey = local_mr->lkey;
 
-	ibv_dev_rdma_wr.wr_id = (uint64_t) lrand48();
+	ibv_dev_rdma_wr.wr_id = (((uint64_t) id_prefix) << 32) | ((uint64_t) lrand48());
 	ibv_dev_rdma_wr.sg_list = &ibv_dev_sge;
 	ibv_dev_rdma_wr.num_sge = 1;
 	ibv_dev_rdma_wr.opcode = opcode;