Allow connecting to remote Queue Pair

Last step before introducing a client is to allow server to connect to the client QP.
That is done when switching the state of the local QP from Init to RTR.

rdma_server.c

@@ -97,6 +97,13 @@ int main(void) {
 
 	udp_rdma_dump_packet(&remote_params);
 
+	/******************************
+	 ******* RDMA connection ******
+	 ******************************/
+	if(!set_peer_informations(ibv_dev_qp, local_params.psn, remote_params.lid, remote_params.qpn)) {
+		return -1;
+	}
+
 	/******************************
 	 ******* Global cleanup *******
 	 ******************************/

soft_rdma.c

@@ -65,7 +65,7 @@ struct ibv_qp *initialize_queue_pair(struct ibv_pd *ibv_dev_pd, struct ibv_cq *i
 	ibv_dev_qp_params.qp_state = IBV_QPS_INIT;
 	ibv_dev_qp_params.pkey_index = 0;
 	ibv_dev_qp_params.port_num = 1;
-	ibv_dev_qp_params.qp_access_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ;
+	ibv_dev_qp_params.qp_access_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE;
 
 	result = ibv_modify_qp(ibv_dev_qp, &ibv_dev_qp_params, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS);
 
@@ -77,3 +77,47 @@ struct ibv_qp *initialize_queue_pair(struct ibv_pd *ibv_dev_pd, struct ibv_cq *i
 	return ibv_dev_qp;
 }
 
+bool set_peer_informations(struct ibv_qp *ibv_dev_qp, uint32_t local_psn, uint32_t remote_lid, uint32_t remote_qpn) {
+	struct ibv_qp_attr ibv_dev_qp_params;
+
+	int result;
+
+	// Change state from Init to RTR (Ready To Receive)
+	memset(&ibv_dev_qp_params, 0, sizeof(struct ibv_qp_attr));
+	ibv_dev_qp_params.qp_state = IBV_QPS_RTR;
+	ibv_dev_qp_params.path_mtu = IBV_MTU_1024;
+	ibv_dev_qp_params.rq_psn = local_psn;
+	ibv_dev_qp_params.max_dest_rd_atomic = 1;
+	ibv_dev_qp_params.min_rnr_timer = 0x12;
+	ibv_dev_qp_params.dest_qp_num = remote_qpn;
+
+	ibv_dev_qp_params.ah_attr.is_global = 0;
+	ibv_dev_qp_params.ah_attr.sl = 0;
+	ibv_dev_qp_params.ah_attr.src_path_bits = 0;
+	ibv_dev_qp_params.ah_attr.port_num = 1;
+	ibv_dev_qp_params.ah_attr.dlid = remote_lid;
+
+	result = ibv_modify_qp(ibv_dev_qp, &ibv_dev_qp_params, IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER);
+	if(result) {
+		fprintf(stderr, "Could not set queue state to RTR with remote parameters\n");
+		return false;
+	}
+
+	// Change state from RTR to RTS (Ready To Send)
+	memset(&ibv_dev_qp_params, 0, sizeof(struct ibv_qp_attr));
+	ibv_dev_qp_params.qp_state = IBV_QPS_RTS;
+	ibv_dev_qp_params.timeout = 0x12;
+	ibv_dev_qp_params.retry_cnt = 7;
+	ibv_dev_qp_params.rnr_retry = 7;
+	ibv_dev_qp_params.sq_psn = 0;
+	ibv_dev_qp_params.max_rd_atomic = 1;
+
+	result = ibv_modify_qp(ibv_dev_qp, &ibv_dev_qp_params, IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_SQ_PSN | IBV_QP_MAX_QP_RD_ATOMIC);
+	if(result) {
+		fprintf(stderr, "Could not set queue state to RTS\n");
+		return false;
+	}
+
+	return true;
+}
+

soft_rdma.h

 #include <stdbool.h>
+#include <stdio.h>
 
 #include <infiniband/verbs.h>
 
 struct ibv_context *initialize_device_context(const char *device_name);
 struct ibv_qp *initialize_queue_pair(struct ibv_pd *ibv_dev_pd, struct ibv_cq *ibv_dev_cq);
+bool set_peer_informations(struct ibv_qp *ibv_dev_qp, uint32_t local_psn, uint32_t remote_lid, uint32_t remote_qpn);