UBUNTU: SAUCE: RDMA Infiniband for Windows Azure

BugLink: http://bugs.launchpad.net/bugs/1641139Signed-off-by: Tim Gardner <tim.gardner@canonical.com>
Acked-by: Brad Figg <brad.figg@canonical.com>
Signed-off-by: Luis Henriques <luis.henriques@canonical.com>

drivers/infiniband/hw/vmbus-rdma/Kconfig

+config HYPERV_INFINIBAND_ND 
+	tristate "Microsoft Hyper-V Network Direct"
+	depends on PCI && INET && INFINIBAND && HYPERV
+	---help---
+	  This is a low-level driver for Vmbus based NetworkDirect. 

drivers/infiniband/hw/vmbus-rdma/Makefile

+obj-$(CONFIG_PCMCIA)	+= hv_network_direct.o
+
+hv_network_direct-y := provider.o vmbus_rdma.o hvnd_addr.o

drivers/infiniband/hw/vmbus-rdma/hvnd_addr.c

+/*
+ * Copyright (c) 2014, Microsoft Corporation.
+ *
+ * Author:
+ *   K. Y. Srinivasan <kys@microsoft.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ * NON INFRINGEMENT.  See the GNU General Public License for more
+ * details.
+ *
+ * Bug fixes/enhancements: Long Li <longli@microsoft.com>
+ */
+
+#include <linux/completion.h>
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <linux/hyperv.h>
+#include <linux/efi.h>
+#include <linux/slab.h>
+#include <linux/cred.h>
+#include <linux/sched.h>
+#include <linux/types.h>
+#include <linux/completion.h>
+#include <linux/scatterlist.h>
+#include <rdma/ib_umem.h>
+#include <rdma/ib_addr.h>
+
+#include "vmbus_rdma.h"
+
+
+#include <linux/semaphore.h>
+#include <linux/fs.h>
+#include <linux/nls.h>
+#include <linux/workqueue.h>
+#include <linux/cdev.h>
+#include <linux/hyperv.h>
+#include <linux/sched.h>
+#include <linux/uaccess.h>
+#include <linux/miscdevice.h>
+#include <linux/hyperv.h>
+
+
+int hvnd_get_outgoing_rdma_addr(struct hvnd_dev *nd_dev, struct hvnd_ucontext *uctx,
+				union nd_sockaddr_inet *og_addr)
+{
+	int ret;
+	/*
+	 * Query the host and select the first address.
+	 */
+	struct pkt_query_addr_list pkt;
+
+	memset(&pkt, 0, sizeof(pkt)); //KYS try to avoid having to zero everything
+
+	hvnd_init_hdr(&pkt.hdr,
+		      (sizeof(pkt) -
+		      sizeof(struct ndv_packet_hdr_control_1)),
+		      uctx->create_pkt.handle.local,
+		      uctx->create_pkt.handle.remote,
+		      IOCTL_ND_ADAPTER_QUERY_ADDRESS_LIST, 0, 0, 0);
+
+	pkt.ioctl.in.version = ND_VERSION_1;
+	pkt.ioctl.in.reserved = 0;
+	pkt.ioctl.in.handle = uctx->adaptor_hdl;
+
+	ret = hvnd_send_ioctl_pkt(nd_dev, &pkt.hdr,
+				  sizeof(pkt), (u64)&pkt);
+
+	if (ret)
+		return ret;
+
+	/*
+	 * Copy the address out.
+	 */
+
+	memcpy(og_addr, &pkt.ioctl.out[0], sizeof(*og_addr));
+	return 0;
+
+}
+
+static struct rdma_addr_client self;
+
+struct resolve_cb_context {
+	struct rdma_dev_addr *addr;
+	struct completion comp;
+};
+
+void hvnd_addr_init(void)
+{
+	rdma_addr_register_client(&self);
+	return;
+}
+
+void hvnd_addr_deinit(void)
+{
+	rdma_addr_unregister_client(&self);
+	return;
+}
+
+static void resolve_cb(int status, struct sockaddr *src_addr,
+	     struct rdma_dev_addr *addr, void *context)
+{
+	memcpy(((struct resolve_cb_context *)context)->addr, addr, sizeof(struct
+				rdma_dev_addr));
+	complete(&((struct resolve_cb_context *)context)->comp);
+}
+
+int hvnd_get_neigh_mac_addr(struct sockaddr *local, struct sockaddr *remote, char *mac_addr)
+{
+	struct rdma_dev_addr dev_addr;
+	struct resolve_cb_context ctx;
+	int ret;
+
+	memset(&dev_addr, 0, sizeof(dev_addr));
+	dev_addr.net = &init_net;
+	ctx.addr = &dev_addr;
+	init_completion(&ctx.comp);
+
+	ret = rdma_resolve_ip(&self, local, remote, &dev_addr, 1000, resolve_cb, &ctx);
+
+	if (ret) {
+		hvnd_error("rdma_resolve_ip failed ret=%d\n", ret);
+		return ret;
+	}
+
+	wait_for_completion(&ctx.comp);
+	memcpy(mac_addr, dev_addr.dst_dev_addr, ETH_ALEN);
+	return ret;
+}

drivers/infiniband/hw/vmbus-rdma/mx_abi.h

+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Cisco Systems.  All rights reserved.
+ * Copyright (c) 2005 PathScale, Inc.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * KYS: made some modifications.
+ */
+
+#ifndef MX_ABI_H
+#define MX_ABI_H
+
+
+
+/*
+ * Make sure that all structs defined in this file remain laid out so
+ * that they pack the same way on 32-bit and 64-bit architectures (to
+ * avoid incompatibility between 32-bit userspace and 64-bit kernels).
+ * Specifically:
+ *  - Do not use pointer types -- pass pointers in UINT64 instead.
+ *  - Make sure that any structure larger than 4 bytes is padded to a
+ *    multiple of 8 bytes.  Otherwise the structure size will be
+ *    different between 32-bit and 64-bit architectures.
+ */
+
+enum ibv_get_context_mappings {
+	IBV_GET_CONTEXT_UAR,
+	IBV_GET_CONTEXT_BF,
+	IBV_GET_CONTEXT_MAPPING_MAX
+};
+
+struct ibv_get_context_req {
+
+	union nd_mapping mappings[IBV_GET_CONTEXT_MAPPING_MAX];
+};
+
+struct ibv_get_context_resp {
+
+	// mmap UAR and BF
+	struct nd_mapping_result  mapping_results[IBV_GET_CONTEXT_MAPPING_MAX];
+	
+	// mmap Blue Flame
+	int bf_buf_size;
+	int bf_offset;
+	
+	// mlx4_query_device result 
+	int max_qp_wr;
+	int max_sge;
+	int max_cqe;
+
+	// general parameters
+	u32 cqe_size;
+	u32 vend_id;
+	u16 dev_id;
+	u16 bf_reg_size;
+	u16 bf_regs_per_page;
+	u16 reserved1;
+
+	// ibv_cmd_get_context result 
+	u32 qp_tab_size;
+
+	u32 reserved2;
+};
+
+struct ibv_alloc_pd_resp {
+	u64 pd_handle;
+	u32 pdn;
+	u32 reserved;
+};
+
+struct ibv_reg_mr {
+	u64 start;
+	u64 length;
+	u64 hca_va;
+	u32 access_flags;
+	u32 pdn;
+	u64 pd_handle;
+};
+
+struct ibv_reg_mr_resp {
+	u64 mr_handle;
+	u32 lkey;
+	u32 rkey;
+};
+
+
+enum mlx4_ib_create_cq_mapping {
+	MLX4_IB_CREATE_CQ_BUF,
+	MLX4_IB_CREATE_CQ_DB,
+	MLX4_IB_CREATE_CQ_ARM_SN,   // Windows specific
+	MLX4_IB_CREATE_CQ_MAPPING_MAX
+};
+
+#define MLX4_CQ_FLAGS_ARM_IN_KERNEL     1
+
+struct ibv_create_cq {
+	union nd_mapping mappings[MLX4_IB_CREATE_CQ_MAPPING_MAX];
+	u32  flags;
+};
+
+struct ibv_create_cq_resp {
+	struct nd_mapping_result mapping_results[MLX4_IB_CREATE_CQ_MAPPING_MAX];
+	u32  cqn;
+	u32  cqe;
+};
+
+enum mlx4_ib_create_srq_mappings {
+	MLX4_IB_CREATE_SRQ_BUF,
+	MLX4_IB_CREATE_SRQ_DB,
+	MLX4_IB_CREATE_SRQ_MAPPINGS_MAX
+};
+
+struct ibv_create_srq {
+	union nd_mapping mappings[MLX4_IB_CREATE_SRQ_MAPPINGS_MAX];
+};
+
+struct ibv_create_srq_resp {
+	struct nd_mapping_result mapping_results[MLX4_IB_CREATE_SRQ_MAPPINGS_MAX];
+};
+
+enum mlx4_ib_create_qp_mappings {
+	MLX4_IB_CREATE_QP_BUF,
+	MLX4_IB_CREATE_QP_DB,
+	MLX4_IB_CREATE_QP_MAPPINGS_MAX
+};
+
+struct ibv_create_qp {
+	union nd_mapping mappings[MLX4_IB_CREATE_QP_MAPPINGS_MAX];
+	u8	log_sq_bb_count;
+	u8	log_sq_stride;
+	u8	sq_no_prefetch;
+	u8	reserved;
+};
+
+struct ibv_create_qp_resp {
+	struct nd_mapping_result mapping_results[MLX4_IB_CREATE_QP_MAPPINGS_MAX];
+	// struct ib_uverbs_create_qp_resp
+	u64 qp_handle;
+	u32 qpn;
+	u32 max_send_wr;
+	u32 max_recv_wr;
+	u32 max_send_sge;
+	u32 max_recv_sge;
+	u32 max_inline_data;
+};
+
+enum ibv_qp_attr_mask {
+	IBV_QP_STATE			= 1 << 0,
+	IBV_QP_CUR_STATE		= 1 << 1,
+	IBV_QP_EN_SQD_ASYNC_NOTIFY	= 1 << 2,
+	IBV_QP_ACCESS_FLAGS		= 1 << 3,
+	IBV_QP_PKEY_INDEX		= 1 << 4,
+	IBV_QP_PORT			= 1 << 5,
+	IBV_QP_QKEY			= 1 << 6,
+	IBV_QP_AV			= 1 << 7,
+	IBV_QP_PATH_MTU			= 1 << 8,
+	IBV_QP_TIMEOUT			= 1 << 9,
+	IBV_QP_RETRY_CNT		= 1 << 10,
+	IBV_QP_RNR_RETRY		= 1 << 11,
+	IBV_QP_RQ_PSN			= 1 << 12,
+	IBV_QP_MAX_QP_RD_ATOMIC		= 1 << 13,
+	IBV_QP_ALT_PATH			= 1 << 14,
+	IBV_QP_MIN_RNR_TIMER		= 1 << 15,
+	IBV_QP_SQ_PSN			= 1 << 16,
+	IBV_QP_MAX_DEST_RD_ATOMIC	= 1 << 17,
+	IBV_QP_PATH_MIG_STATE		= 1 << 18,
+	IBV_QP_CAP			= 1 << 19,
+	IBV_QP_DEST_QPN			= 1 << 20
+};
+
+enum ibv_qp_state {
+	IBV_QPS_RESET,
+	IBV_QPS_INIT,
+	IBV_QPS_RTR,
+	IBV_QPS_RTS,
+	IBV_QPS_SQD,
+	IBV_QPS_SQE,
+	IBV_QPS_ERR
+};
+
+
+struct ibv_modify_qp_resp {
+	enum ibv_qp_attr_mask attr_mask;
+	u8 qp_state;
+	u8 reserved[3];
+};
+
+struct ibv_create_ah_resp {
+	u64 start;
+};
+
+/*
+ * Some mlx4 specific kernel definitions. Perhaps could be in 
+ * separate file.
+ */
+
+struct mlx4_ib_user_db_page {
+	struct list_head        list;
+	struct ib_umem         *umem;
+	unsigned long           user_virt;
+	int                     refcnt;
+};
+
+
+#endif /* MX_ABI_H */

drivers/infiniband/hw/vmbus-rdma/user.h

+/*
+ * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef MLX4_IB_USER_H
+#define MLX4_IB_USER_H
+
+#include <linux/types.h>
+
+/*
+ * Increment this value if any changes that break userspace ABI
+ * compatibility are made.
+ */
+
+#define MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION	3
+#define MLX4_IB_UVERBS_ABI_VERSION		4
+
+/*
+ * Make sure that all structs defined in this file remain laid out so
+ * that they pack the same way on 32-bit and 64-bit architectures (to
+ * avoid incompatibility between 32-bit userspace and 64-bit kernels).
+ * In particular do not use pointer types -- pass pointers in __u64
+ * instead.
+ */
+
+struct mlx4_ib_alloc_ucontext_resp_v3 {
+	__u32	qp_tab_size;
+	__u16	bf_reg_size;
+	__u16	bf_regs_per_page;
+};
+
+struct mlx4_ib_alloc_ucontext_resp {
+	__u32	dev_caps;
+	__u32	qp_tab_size;
+	__u16	bf_reg_size;
+	__u16	bf_regs_per_page;
+	__u32	cqe_size;
+};
+
+struct mlx4_ib_alloc_pd_resp {
+	__u32	pdn;
+	__u32	reserved;
+};
+
+struct mlx4_ib_create_cq {
+	__u64	buf_addr;
+	__u64	db_addr;
+};
+
+struct mlx4_ib_create_cq_resp {
+	__u32	cqn;
+	__u32	reserved;
+};
+
+struct mlx4_ib_resize_cq {
+	__u64	buf_addr;
+};
+
+struct mlx4_ib_create_srq {
+	__u64	buf_addr;
+	__u64	db_addr;
+};
+
+struct mlx4_ib_create_srq_resp {
+	__u32	srqn;
+	__u32	reserved;
+};
+
+struct mlx4_ib_create_qp {
+	__u64	buf_addr;
+	__u64	db_addr;
+	__u8	log_sq_bb_count;
+	__u8	log_sq_stride;
+	__u8	sq_no_prefetch;
+	__u8	reserved[5];
+};
+
+#endif /* MLX4_IB_USER_H */