Commit 5802883d authored by oulijun's avatar oulijun Committed by Doug Ledford

IB/hns: Fix the bug of polling cq failed for loopback Qps

In hip06 SoC, RoCE driver creates 8 reserved loopback QPs to
ensure zero wqe when free mr. However, if the enabled phy
port number is less than 6, it will fail in polling cqe with
8 reserved loopback QPs.

In order to solve this problem, the number of loopback Qps
will be adjusted based on the number of enabled phy port.
Signed-off-by: default avatarShaobo Xu <xushaobo2@huawei.com>
Signed-off-by: default avatarLijun Ou <oulijun@huawei.com>
Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parent 56012e1c
...@@ -661,9 +661,11 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev) ...@@ -661,9 +661,11 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
union ib_gid dgid; union ib_gid dgid;
u64 subnet_prefix; u64 subnet_prefix;
int attr_mask = 0; int attr_mask = 0;
int i; int i, j;
int ret; int ret;
u8 queue_en[HNS_ROCE_V1_RESV_QP] = { 0 };
u8 phy_port; u8 phy_port;
u8 port = 0;
u8 sl; u8 sl;
priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv; priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
...@@ -709,11 +711,27 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev) ...@@ -709,11 +711,27 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
attr.rnr_retry = 7; attr.rnr_retry = 7;
attr.timeout = 0x12; attr.timeout = 0x12;
attr.path_mtu = IB_MTU_256; attr.path_mtu = IB_MTU_256;
attr.ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
rdma_ah_set_grh(&attr.ah_attr, NULL, 0, 0, 1, 0); rdma_ah_set_grh(&attr.ah_attr, NULL, 0, 0, 1, 0);
rdma_ah_set_static_rate(&attr.ah_attr, 3); rdma_ah_set_static_rate(&attr.ah_attr, 3);
subnet_prefix = cpu_to_be64(0xfe80000000000000LL); subnet_prefix = cpu_to_be64(0xfe80000000000000LL);
for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) { for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
phy_port = (i >= HNS_ROCE_MAX_PORTS) ? (i - 2) :
(i % HNS_ROCE_MAX_PORTS);
sl = i / HNS_ROCE_MAX_PORTS;
for (j = 0; j < caps->num_ports; j++) {
if (hr_dev->iboe.phy_port[j] == phy_port) {
queue_en[i] = 1;
port = j;
break;
}
}
if (!queue_en[i])
continue;
free_mr->mr_free_qp[i] = hns_roce_v1_create_lp_qp(hr_dev, pd); free_mr->mr_free_qp[i] = hns_roce_v1_create_lp_qp(hr_dev, pd);
if (IS_ERR(free_mr->mr_free_qp[i])) { if (IS_ERR(free_mr->mr_free_qp[i])) {
dev_err(dev, "Create loop qp failed!\n"); dev_err(dev, "Create loop qp failed!\n");
...@@ -721,15 +739,7 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev) ...@@ -721,15 +739,7 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
} }
hr_qp = free_mr->mr_free_qp[i]; hr_qp = free_mr->mr_free_qp[i];
sl = i / caps->num_ports; hr_qp->port = port;
if (caps->num_ports == HNS_ROCE_MAX_PORTS)
phy_port = (i >= HNS_ROCE_MAX_PORTS) ? (i - 2) :
(i % caps->num_ports);
else
phy_port = i % caps->num_ports;
hr_qp->port = phy_port + 1;
hr_qp->phy_port = phy_port; hr_qp->phy_port = phy_port;
hr_qp->ibqp.qp_type = IB_QPT_RC; hr_qp->ibqp.qp_type = IB_QPT_RC;
hr_qp->ibqp.device = &hr_dev->ib_dev; hr_qp->ibqp.device = &hr_dev->ib_dev;
...@@ -739,23 +749,22 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev) ...@@ -739,23 +749,22 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
hr_qp->ibqp.recv_cq = cq; hr_qp->ibqp.recv_cq = cq;
hr_qp->ibqp.send_cq = cq; hr_qp->ibqp.send_cq = cq;
rdma_ah_set_port_num(&attr.ah_attr, phy_port + 1); rdma_ah_set_port_num(&attr.ah_attr, port + 1);
rdma_ah_set_sl(&attr.ah_attr, phy_port + 1); rdma_ah_set_sl(&attr.ah_attr, sl);
attr.port_num = phy_port + 1; attr.port_num = port + 1;
attr.dest_qp_num = hr_qp->qpn; attr.dest_qp_num = hr_qp->qpn;
memcpy(rdma_ah_retrieve_dmac(&attr.ah_attr), memcpy(rdma_ah_retrieve_dmac(&attr.ah_attr),
hr_dev->dev_addr[phy_port], hr_dev->dev_addr[port],
MAC_ADDR_OCTET_NUM); MAC_ADDR_OCTET_NUM);
memcpy(&dgid.raw, &subnet_prefix, sizeof(u64)); memcpy(&dgid.raw, &subnet_prefix, sizeof(u64));
memcpy(&dgid.raw[8], hr_dev->dev_addr[phy_port], 3); memcpy(&dgid.raw[8], hr_dev->dev_addr[port], 3);
memcpy(&dgid.raw[13], hr_dev->dev_addr[phy_port] + 3, 3); memcpy(&dgid.raw[13], hr_dev->dev_addr[port] + 3, 3);
dgid.raw[11] = 0xff; dgid.raw[11] = 0xff;
dgid.raw[12] = 0xfe; dgid.raw[12] = 0xfe;
dgid.raw[8] ^= 2; dgid.raw[8] ^= 2;
rdma_ah_set_dgid_raw(&attr.ah_attr, dgid.raw); rdma_ah_set_dgid_raw(&attr.ah_attr, dgid.raw);
attr_mask |= IB_QP_PORT;
ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask, ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask,
IB_QPS_RESET, IB_QPS_INIT); IB_QPS_RESET, IB_QPS_INIT);
...@@ -812,6 +821,9 @@ static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev) ...@@ -812,6 +821,9 @@ static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev)
for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) { for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
hr_qp = free_mr->mr_free_qp[i]; hr_qp = free_mr->mr_free_qp[i];
if (!hr_qp)
continue;
ret = hns_roce_v1_destroy_qp(&hr_qp->ibqp); ret = hns_roce_v1_destroy_qp(&hr_qp->ibqp);
if (ret) if (ret)
dev_err(dev, "Destroy qp %d for mr free failed(%d)!\n", dev_err(dev, "Destroy qp %d for mr free failed(%d)!\n",
...@@ -963,7 +975,7 @@ static void hns_roce_v1_mr_free_work_fn(struct work_struct *work) ...@@ -963,7 +975,7 @@ static void hns_roce_v1_mr_free_work_fn(struct work_struct *work)
msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS) + jiffies; msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS) + jiffies;
int i; int i;
int ret; int ret;
int ne; int ne = 0;
mr_work = container_of(work, struct hns_roce_mr_free_work, work); mr_work = container_of(work, struct hns_roce_mr_free_work, work);
hr_mr = (struct hns_roce_mr *)mr_work->mr; hr_mr = (struct hns_roce_mr *)mr_work->mr;
...@@ -976,6 +988,10 @@ static void hns_roce_v1_mr_free_work_fn(struct work_struct *work) ...@@ -976,6 +988,10 @@ static void hns_roce_v1_mr_free_work_fn(struct work_struct *work)
for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) { for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
hr_qp = free_mr->mr_free_qp[i]; hr_qp = free_mr->mr_free_qp[i];
if (!hr_qp)
continue;
ne++;
ret = hns_roce_v1_send_lp_wqe(hr_qp); ret = hns_roce_v1_send_lp_wqe(hr_qp);
if (ret) { if (ret) {
dev_err(dev, dev_err(dev,
...@@ -985,7 +1001,6 @@ static void hns_roce_v1_mr_free_work_fn(struct work_struct *work) ...@@ -985,7 +1001,6 @@ static void hns_roce_v1_mr_free_work_fn(struct work_struct *work)
} }
} }
ne = HNS_ROCE_V1_RESV_QP;
do { do {
ret = hns_roce_v1_poll_cq(&mr_free_cq->ib_cq, ne, wc); ret = hns_roce_v1_poll_cq(&mr_free_cq->ib_cq, ne, wc);
if (ret < 0) { if (ret < 0) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment