Commit 95f7f3e7 authored by Karsten Graul's avatar Karsten Graul Committed by David S. Miller

net/smc: improved fix wait on already cleared link

Commit 8f3d65c1 ("net/smc: fix wait on already cleared link")
introduced link refcounting to avoid waits on already cleared links.
This patch extents and improves the refcounting to cover all
remaining possible cases for this kind of error situation.

Fixes: 15e1b99a ("net/smc: no WR buffer wait for terminating link group")
Signed-off-by: default avatarKarsten Graul <kgraul@linux.ibm.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 097657c9
...@@ -150,9 +150,11 @@ static int smcr_cdc_get_slot_and_msg_send(struct smc_connection *conn) ...@@ -150,9 +150,11 @@ static int smcr_cdc_get_slot_and_msg_send(struct smc_connection *conn)
again: again:
link = conn->lnk; link = conn->lnk;
if (!smc_wr_tx_link_hold(link))
return -ENOLINK;
rc = smc_cdc_get_free_slot(conn, link, &wr_buf, NULL, &pend); rc = smc_cdc_get_free_slot(conn, link, &wr_buf, NULL, &pend);
if (rc) if (rc)
return rc; goto put_out;
spin_lock_bh(&conn->send_lock); spin_lock_bh(&conn->send_lock);
if (link != conn->lnk) { if (link != conn->lnk) {
...@@ -160,6 +162,7 @@ static int smcr_cdc_get_slot_and_msg_send(struct smc_connection *conn) ...@@ -160,6 +162,7 @@ static int smcr_cdc_get_slot_and_msg_send(struct smc_connection *conn)
spin_unlock_bh(&conn->send_lock); spin_unlock_bh(&conn->send_lock);
smc_wr_tx_put_slot(link, smc_wr_tx_put_slot(link,
(struct smc_wr_tx_pend_priv *)pend); (struct smc_wr_tx_pend_priv *)pend);
smc_wr_tx_link_put(link);
if (again) if (again)
return -ENOLINK; return -ENOLINK;
again = true; again = true;
...@@ -167,6 +170,8 @@ static int smcr_cdc_get_slot_and_msg_send(struct smc_connection *conn) ...@@ -167,6 +170,8 @@ static int smcr_cdc_get_slot_and_msg_send(struct smc_connection *conn)
} }
rc = smc_cdc_msg_send(conn, wr_buf, pend); rc = smc_cdc_msg_send(conn, wr_buf, pend);
spin_unlock_bh(&conn->send_lock); spin_unlock_bh(&conn->send_lock);
put_out:
smc_wr_tx_link_put(link);
return rc; return rc;
} }
......
...@@ -949,7 +949,7 @@ struct smc_link *smc_switch_conns(struct smc_link_group *lgr, ...@@ -949,7 +949,7 @@ struct smc_link *smc_switch_conns(struct smc_link_group *lgr,
to_lnk = &lgr->lnk[i]; to_lnk = &lgr->lnk[i];
break; break;
} }
if (!to_lnk) { if (!to_lnk || !smc_wr_tx_link_hold(to_lnk)) {
smc_lgr_terminate_sched(lgr); smc_lgr_terminate_sched(lgr);
return NULL; return NULL;
} }
...@@ -981,24 +981,26 @@ struct smc_link *smc_switch_conns(struct smc_link_group *lgr, ...@@ -981,24 +981,26 @@ struct smc_link *smc_switch_conns(struct smc_link_group *lgr,
read_unlock_bh(&lgr->conns_lock); read_unlock_bh(&lgr->conns_lock);
/* pre-fetch buffer outside of send_lock, might sleep */ /* pre-fetch buffer outside of send_lock, might sleep */
rc = smc_cdc_get_free_slot(conn, to_lnk, &wr_buf, NULL, &pend); rc = smc_cdc_get_free_slot(conn, to_lnk, &wr_buf, NULL, &pend);
if (rc) { if (rc)
smcr_link_down_cond_sched(to_lnk); goto err_out;
return NULL;
}
/* avoid race with smcr_tx_sndbuf_nonempty() */ /* avoid race with smcr_tx_sndbuf_nonempty() */
spin_lock_bh(&conn->send_lock); spin_lock_bh(&conn->send_lock);
smc_switch_link_and_count(conn, to_lnk); smc_switch_link_and_count(conn, to_lnk);
rc = smc_switch_cursor(smc, pend, wr_buf); rc = smc_switch_cursor(smc, pend, wr_buf);
spin_unlock_bh(&conn->send_lock); spin_unlock_bh(&conn->send_lock);
sock_put(&smc->sk); sock_put(&smc->sk);
if (rc) { if (rc)
smcr_link_down_cond_sched(to_lnk); goto err_out;
return NULL;
}
goto again; goto again;
} }
read_unlock_bh(&lgr->conns_lock); read_unlock_bh(&lgr->conns_lock);
smc_wr_tx_link_put(to_lnk);
return to_lnk; return to_lnk;
err_out:
smcr_link_down_cond_sched(to_lnk);
smc_wr_tx_link_put(to_lnk);
return NULL;
} }
static void smcr_buf_unuse(struct smc_buf_desc *rmb_desc, static void smcr_buf_unuse(struct smc_buf_desc *rmb_desc,
......
...@@ -383,9 +383,11 @@ int smc_llc_send_confirm_link(struct smc_link *link, ...@@ -383,9 +383,11 @@ int smc_llc_send_confirm_link(struct smc_link *link,
struct smc_wr_buf *wr_buf; struct smc_wr_buf *wr_buf;
int rc; int rc;
if (!smc_wr_tx_link_hold(link))
return -ENOLINK;
rc = smc_llc_add_pending_send(link, &wr_buf, &pend); rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
if (rc) if (rc)
return rc; goto put_out;
confllc = (struct smc_llc_msg_confirm_link *)wr_buf; confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
memset(confllc, 0, sizeof(*confllc)); memset(confllc, 0, sizeof(*confllc));
confllc->hd.common.type = SMC_LLC_CONFIRM_LINK; confllc->hd.common.type = SMC_LLC_CONFIRM_LINK;
...@@ -402,6 +404,8 @@ int smc_llc_send_confirm_link(struct smc_link *link, ...@@ -402,6 +404,8 @@ int smc_llc_send_confirm_link(struct smc_link *link,
confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS;
/* send llc message */ /* send llc message */
rc = smc_wr_tx_send(link, pend); rc = smc_wr_tx_send(link, pend);
put_out:
smc_wr_tx_link_put(link);
return rc; return rc;
} }
...@@ -415,9 +419,11 @@ static int smc_llc_send_confirm_rkey(struct smc_link *send_link, ...@@ -415,9 +419,11 @@ static int smc_llc_send_confirm_rkey(struct smc_link *send_link,
struct smc_link *link; struct smc_link *link;
int i, rc, rtok_ix; int i, rc, rtok_ix;
if (!smc_wr_tx_link_hold(send_link))
return -ENOLINK;
rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend); rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend);
if (rc) if (rc)
return rc; goto put_out;
rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf; rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
memset(rkeyllc, 0, sizeof(*rkeyllc)); memset(rkeyllc, 0, sizeof(*rkeyllc));
rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY; rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY;
...@@ -444,6 +450,8 @@ static int smc_llc_send_confirm_rkey(struct smc_link *send_link, ...@@ -444,6 +450,8 @@ static int smc_llc_send_confirm_rkey(struct smc_link *send_link,
(u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl)); (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl));
/* send llc message */ /* send llc message */
rc = smc_wr_tx_send(send_link, pend); rc = smc_wr_tx_send(send_link, pend);
put_out:
smc_wr_tx_link_put(send_link);
return rc; return rc;
} }
...@@ -456,9 +464,11 @@ static int smc_llc_send_delete_rkey(struct smc_link *link, ...@@ -456,9 +464,11 @@ static int smc_llc_send_delete_rkey(struct smc_link *link,
struct smc_wr_buf *wr_buf; struct smc_wr_buf *wr_buf;
int rc; int rc;
if (!smc_wr_tx_link_hold(link))
return -ENOLINK;
rc = smc_llc_add_pending_send(link, &wr_buf, &pend); rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
if (rc) if (rc)
return rc; goto put_out;
rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf; rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
memset(rkeyllc, 0, sizeof(*rkeyllc)); memset(rkeyllc, 0, sizeof(*rkeyllc));
rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY; rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY;
...@@ -467,6 +477,8 @@ static int smc_llc_send_delete_rkey(struct smc_link *link, ...@@ -467,6 +477,8 @@ static int smc_llc_send_delete_rkey(struct smc_link *link,
rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey); rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
/* send llc message */ /* send llc message */
rc = smc_wr_tx_send(link, pend); rc = smc_wr_tx_send(link, pend);
put_out:
smc_wr_tx_link_put(link);
return rc; return rc;
} }
...@@ -480,9 +492,11 @@ int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], ...@@ -480,9 +492,11 @@ int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
struct smc_wr_buf *wr_buf; struct smc_wr_buf *wr_buf;
int rc; int rc;
if (!smc_wr_tx_link_hold(link))
return -ENOLINK;
rc = smc_llc_add_pending_send(link, &wr_buf, &pend); rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
if (rc) if (rc)
return rc; goto put_out;
addllc = (struct smc_llc_msg_add_link *)wr_buf; addllc = (struct smc_llc_msg_add_link *)wr_buf;
memset(addllc, 0, sizeof(*addllc)); memset(addllc, 0, sizeof(*addllc));
...@@ -504,6 +518,8 @@ int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], ...@@ -504,6 +518,8 @@ int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
} }
/* send llc message */ /* send llc message */
rc = smc_wr_tx_send(link, pend); rc = smc_wr_tx_send(link, pend);
put_out:
smc_wr_tx_link_put(link);
return rc; return rc;
} }
...@@ -517,9 +533,11 @@ int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id, ...@@ -517,9 +533,11 @@ int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id,
struct smc_wr_buf *wr_buf; struct smc_wr_buf *wr_buf;
int rc; int rc;
if (!smc_wr_tx_link_hold(link))
return -ENOLINK;
rc = smc_llc_add_pending_send(link, &wr_buf, &pend); rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
if (rc) if (rc)
return rc; goto put_out;
delllc = (struct smc_llc_msg_del_link *)wr_buf; delllc = (struct smc_llc_msg_del_link *)wr_buf;
memset(delllc, 0, sizeof(*delllc)); memset(delllc, 0, sizeof(*delllc));
...@@ -536,6 +554,8 @@ int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id, ...@@ -536,6 +554,8 @@ int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id,
delllc->reason = htonl(reason); delllc->reason = htonl(reason);
/* send llc message */ /* send llc message */
rc = smc_wr_tx_send(link, pend); rc = smc_wr_tx_send(link, pend);
put_out:
smc_wr_tx_link_put(link);
return rc; return rc;
} }
...@@ -547,9 +567,11 @@ static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) ...@@ -547,9 +567,11 @@ static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
struct smc_wr_buf *wr_buf; struct smc_wr_buf *wr_buf;
int rc; int rc;
if (!smc_wr_tx_link_hold(link))
return -ENOLINK;
rc = smc_llc_add_pending_send(link, &wr_buf, &pend); rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
if (rc) if (rc)
return rc; goto put_out;
testllc = (struct smc_llc_msg_test_link *)wr_buf; testllc = (struct smc_llc_msg_test_link *)wr_buf;
memset(testllc, 0, sizeof(*testllc)); memset(testllc, 0, sizeof(*testllc));
testllc->hd.common.type = SMC_LLC_TEST_LINK; testllc->hd.common.type = SMC_LLC_TEST_LINK;
...@@ -557,6 +579,8 @@ static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) ...@@ -557,6 +579,8 @@ static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
/* send llc message */ /* send llc message */
rc = smc_wr_tx_send(link, pend); rc = smc_wr_tx_send(link, pend);
put_out:
smc_wr_tx_link_put(link);
return rc; return rc;
} }
...@@ -567,13 +591,16 @@ static int smc_llc_send_message(struct smc_link *link, void *llcbuf) ...@@ -567,13 +591,16 @@ static int smc_llc_send_message(struct smc_link *link, void *llcbuf)
struct smc_wr_buf *wr_buf; struct smc_wr_buf *wr_buf;
int rc; int rc;
if (!smc_link_usable(link)) if (!smc_wr_tx_link_hold(link))
return -ENOLINK; return -ENOLINK;
rc = smc_llc_add_pending_send(link, &wr_buf, &pend); rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
if (rc) if (rc)
return rc; goto put_out;
memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
return smc_wr_tx_send(link, pend); rc = smc_wr_tx_send(link, pend);
put_out:
smc_wr_tx_link_put(link);
return rc;
} }
/* schedule an llc send on link, may wait for buffers, /* schedule an llc send on link, may wait for buffers,
...@@ -586,13 +613,16 @@ static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf) ...@@ -586,13 +613,16 @@ static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf)
struct smc_wr_buf *wr_buf; struct smc_wr_buf *wr_buf;
int rc; int rc;
if (!smc_link_usable(link)) if (!smc_wr_tx_link_hold(link))
return -ENOLINK; return -ENOLINK;
rc = smc_llc_add_pending_send(link, &wr_buf, &pend); rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
if (rc) if (rc)
return rc; goto put_out;
memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
return smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME); rc = smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME);
put_out:
smc_wr_tx_link_put(link);
return rc;
} }
/********************************* receive ***********************************/ /********************************* receive ***********************************/
...@@ -672,9 +702,11 @@ static int smc_llc_add_link_cont(struct smc_link *link, ...@@ -672,9 +702,11 @@ static int smc_llc_add_link_cont(struct smc_link *link,
struct smc_buf_desc *rmb; struct smc_buf_desc *rmb;
u8 n; u8 n;
if (!smc_wr_tx_link_hold(link))
return -ENOLINK;
rc = smc_llc_add_pending_send(link, &wr_buf, &pend); rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
if (rc) if (rc)
return rc; goto put_out;
addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf; addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf;
memset(addc_llc, 0, sizeof(*addc_llc)); memset(addc_llc, 0, sizeof(*addc_llc));
...@@ -706,7 +738,10 @@ static int smc_llc_add_link_cont(struct smc_link *link, ...@@ -706,7 +738,10 @@ static int smc_llc_add_link_cont(struct smc_link *link,
addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont); addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont);
if (lgr->role == SMC_CLNT) if (lgr->role == SMC_CLNT)
addc_llc->hd.flags |= SMC_LLC_FLAG_RESP; addc_llc->hd.flags |= SMC_LLC_FLAG_RESP;
return smc_wr_tx_send(link, pend); rc = smc_wr_tx_send(link, pend);
put_out:
smc_wr_tx_link_put(link);
return rc;
} }
static int smc_llc_cli_rkey_exchange(struct smc_link *link, static int smc_llc_cli_rkey_exchange(struct smc_link *link,
......
...@@ -496,7 +496,7 @@ static int smc_tx_rdma_writes(struct smc_connection *conn, ...@@ -496,7 +496,7 @@ static int smc_tx_rdma_writes(struct smc_connection *conn,
/* Wakeup sndbuf consumers from any context (IRQ or process) /* Wakeup sndbuf consumers from any context (IRQ or process)
* since there is more data to transmit; usable snd_wnd as max transmit * since there is more data to transmit; usable snd_wnd as max transmit
*/ */
static int _smcr_tx_sndbuf_nonempty(struct smc_connection *conn) static int smcr_tx_sndbuf_nonempty(struct smc_connection *conn)
{ {
struct smc_cdc_producer_flags *pflags = &conn->local_tx_ctrl.prod_flags; struct smc_cdc_producer_flags *pflags = &conn->local_tx_ctrl.prod_flags;
struct smc_link *link = conn->lnk; struct smc_link *link = conn->lnk;
...@@ -505,8 +505,11 @@ static int _smcr_tx_sndbuf_nonempty(struct smc_connection *conn) ...@@ -505,8 +505,11 @@ static int _smcr_tx_sndbuf_nonempty(struct smc_connection *conn)
struct smc_wr_buf *wr_buf; struct smc_wr_buf *wr_buf;
int rc; int rc;
if (!link || !smc_wr_tx_link_hold(link))
return -ENOLINK;
rc = smc_cdc_get_free_slot(conn, link, &wr_buf, &wr_rdma_buf, &pend); rc = smc_cdc_get_free_slot(conn, link, &wr_buf, &wr_rdma_buf, &pend);
if (rc < 0) { if (rc < 0) {
smc_wr_tx_link_put(link);
if (rc == -EBUSY) { if (rc == -EBUSY) {
struct smc_sock *smc = struct smc_sock *smc =
container_of(conn, struct smc_sock, conn); container_of(conn, struct smc_sock, conn);
...@@ -547,22 +550,7 @@ static int _smcr_tx_sndbuf_nonempty(struct smc_connection *conn) ...@@ -547,22 +550,7 @@ static int _smcr_tx_sndbuf_nonempty(struct smc_connection *conn)
out_unlock: out_unlock:
spin_unlock_bh(&conn->send_lock); spin_unlock_bh(&conn->send_lock);
return rc; smc_wr_tx_link_put(link);
}
static int smcr_tx_sndbuf_nonempty(struct smc_connection *conn)
{
struct smc_link *link = conn->lnk;
int rc = -ENOLINK;
if (!link)
return rc;
atomic_inc(&link->wr_tx_refcnt);
if (smc_link_usable(link))
rc = _smcr_tx_sndbuf_nonempty(conn);
if (atomic_dec_and_test(&link->wr_tx_refcnt))
wake_up_all(&link->wr_tx_wait);
return rc; return rc;
} }
......
...@@ -60,6 +60,20 @@ static inline void smc_wr_tx_set_wr_id(atomic_long_t *wr_tx_id, long val) ...@@ -60,6 +60,20 @@ static inline void smc_wr_tx_set_wr_id(atomic_long_t *wr_tx_id, long val)
atomic_long_set(wr_tx_id, val); atomic_long_set(wr_tx_id, val);
} }
static inline bool smc_wr_tx_link_hold(struct smc_link *link)
{
if (!smc_link_usable(link))
return false;
atomic_inc(&link->wr_tx_refcnt);
return true;
}
static inline void smc_wr_tx_link_put(struct smc_link *link)
{
if (atomic_dec_and_test(&link->wr_tx_refcnt))
wake_up_all(&link->wr_tx_wait);
}
static inline void smc_wr_wakeup_tx_wait(struct smc_link *lnk) static inline void smc_wr_wakeup_tx_wait(struct smc_link *lnk)
{ {
wake_up_all(&lnk->wr_tx_wait); wake_up_all(&lnk->wr_tx_wait);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment