Allow remote connections on IP

Add a Global IDentifier to the UDP packet in order to be able to communicate with a distant machine.

rdma_server.c

 #include <stdio.h>
 #include <stdlib.h>
+#include <time.h>
 
 #include <infiniband/verbs.h>
 
@@ -16,6 +17,7 @@ int main(void) {
 	struct ibv_qp *ibv_dev_qp;
 	struct ibv_mr *ibv_dev_mr;
 	struct ibv_port_attr ibv_dev_attr;
+	union ibv_gid ibv_dev_gid;
 
 	struct udp_rdma_parameters local_params;
 	struct udp_rdma_parameters remote_params;
@@ -25,6 +27,8 @@ int main(void) {
 	int allocated_size = 16384 * sizeof(char);
 	int page_size = sysconf(_SC_PAGESIZE);
 
+	srand(time(NULL));
+
 	/******************************
 	 *** RDMA initialization (1) **
 	 ******************************/
@@ -81,10 +85,17 @@ int main(void) {
 		return -1;
 	}
 
+	result = ibv_query_gid(ibv_dev_ctx, 1, 0, &ibv_dev_gid);
+	if(result) {
+		fprintf(stderr, "Query GID failed\n");
+		return -1;
+	}
+	memcpy((void *) local_params.gid, (void *) ibv_dev_gid.raw, 16);
+
 	// Setup a server for sending RDMA parameters
 	local_params.lid = ibv_dev_attr.lid;
 	local_params.qpn = ibv_dev_qp->qp_num;
-	local_params.psn = lrand48() & 0xffffff;
+	local_params.psn = rand() & 0xffffff;
 	local_params.addr = ibv_dev_mr->addr;
 	local_params.size = ibv_dev_mr->length;
 	local_params.key = ibv_dev_mr->rkey;
@@ -100,7 +111,7 @@ int main(void) {
 	/******************************
 	 ******* RDMA connection ******
 	 ******************************/
-	if(!set_peer_informations(ibv_dev_qp, local_params.psn, remote_params.lid, remote_params.qpn)) {
+	if(!set_peer_informations(ibv_dev_qp, remote_params.psn, remote_params.lid, remote_params.qpn, remote_params.gid)) {
 		return -1;
 	}
 

soft_rdma.c

@@ -77,25 +77,33 @@ struct ibv_qp *initialize_queue_pair(struct ibv_pd *ibv_dev_pd, struct ibv_cq *i
 	return ibv_dev_qp;
 }
 
-bool set_peer_informations(struct ibv_qp *ibv_dev_qp, uint32_t local_psn, uint32_t remote_lid, uint32_t remote_qpn) {
+bool set_peer_informations(struct ibv_qp *ibv_dev_qp, uint32_t remote_psn, uint32_t remote_lid, uint32_t remote_qpn, uint8_t *remote_gid) {
 	struct ibv_qp_attr ibv_dev_qp_params;
+	union ibv_gid ibv_dev_gid;
 
 	int result;
 
+	// Insert GID into IBv structure
+	memcpy((void *) ibv_dev_gid.raw, (void *) remote_gid, 16);
+
 	// Change state from Init to RTR (Ready To Receive)
 	memset(&ibv_dev_qp_params, 0, sizeof(struct ibv_qp_attr));
 	ibv_dev_qp_params.qp_state = IBV_QPS_RTR;
 	ibv_dev_qp_params.path_mtu = IBV_MTU_1024;
-	ibv_dev_qp_params.rq_psn = local_psn;
-	ibv_dev_qp_params.max_dest_rd_atomic = 1;
-	ibv_dev_qp_params.min_rnr_timer = 0x12;
 	ibv_dev_qp_params.dest_qp_num = remote_qpn;
+	ibv_dev_qp_params.rq_psn = remote_psn;
+	ibv_dev_qp_params.max_dest_rd_atomic = 1;
+	ibv_dev_qp_params.min_rnr_timer = 12;
 
-	ibv_dev_qp_params.ah_attr.is_global = 0;
+	ibv_dev_qp_params.ah_attr.is_global = 1;
+	ibv_dev_qp_params.ah_attr.dlid = remote_lid;
 	ibv_dev_qp_params.ah_attr.sl = 0;
 	ibv_dev_qp_params.ah_attr.src_path_bits = 0;
 	ibv_dev_qp_params.ah_attr.port_num = 1;
-	ibv_dev_qp_params.ah_attr.dlid = remote_lid;
+
+	ibv_dev_qp_params.ah_attr.grh.hop_limit = 1;
+	ibv_dev_qp_params.ah_attr.grh.dgid = ibv_dev_gid;
+	ibv_dev_qp_params.ah_attr.grh.sgid_index = 0;
 
 	result = ibv_modify_qp(ibv_dev_qp, &ibv_dev_qp_params, IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER);
 	if(result) {

soft_rdma.h

@@ -5,5 +5,5 @@
 
 struct ibv_context *initialize_device_context(const char *device_name);
 struct ibv_qp *initialize_queue_pair(struct ibv_pd *ibv_dev_pd, struct ibv_cq *ibv_dev_cq);
-bool set_peer_informations(struct ibv_qp *ibv_dev_qp, uint32_t local_psn, uint32_t remote_lid, uint32_t remote_qpn);
+bool set_peer_informations(struct ibv_qp *ibv_dev_qp, uint32_t remote_psn, uint32_t remote_lid, uint32_t remote_qpn, uint8_t *remote_gid);
 

udp_rdma.c

@@ -110,7 +110,12 @@ bool udp_rdma_send_parameters(const char *hostname, struct udp_rdma_parameters *
 }
 
 void udp_rdma_dump_packet(struct udp_rdma_parameters *rdma_params) {
-	printf("LID: 0x%x | QPN: 0x%x | PSN: 0x%x\n", rdma_params->lid, rdma_params->qpn, rdma_params->psn);
-	printf("addr: %p | size: %ld | key: 0x%x\n", rdma_params->addr, rdma_params->size, rdma_params->key);
+	printf("LID: 0x%x | QPN: 0x%x | PSN: 0x%x | GID: ", rdma_params->lid, rdma_params->qpn, rdma_params->psn);
+
+	for(char i = 0; i < 8; i++) {
+		printf("%x%x ", rdma_params->gid[2 * i], rdma_params->gid[2 * i + 1]);
+	}
+
+	printf("\naddr: %p | size: %ld | key: 0x%x\n", rdma_params->addr, rdma_params->size, rdma_params->key);
 }
 

udp_rdma.h

@@ -13,6 +13,7 @@ struct udp_rdma_parameters {
 	uint16_t lid;
 	uint32_t qpn;
 	uint32_t psn;
+	uint8_t gid[16];
 	void *addr;
 	size_t size;
 	uint32_t key;