Commit bac6de7b authored by Stefan Raspl's avatar Stefan Raspl Committed by David S. Miller

net/smc: eliminate cursor read and write calls

The functions to read and write cursors are exclusively used to copy
cursors. Therefore switch to a respective function instead.
Signed-off-by: default avatarStefan Raspl <raspl@linux.ibm.com>
Signed-off-by: default avatarUrsula Braun <ubraun@linux.ibm.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent c601171d
...@@ -1755,12 +1755,8 @@ static int smc_ioctl(struct socket *sock, unsigned int cmd, ...@@ -1755,12 +1755,8 @@ static int smc_ioctl(struct socket *sock, unsigned int cmd,
smc->sk.sk_state == SMC_CLOSED) { smc->sk.sk_state == SMC_CLOSED) {
answ = 0; answ = 0;
} else { } else {
smc_curs_write(&cons, smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
smc_curs_read(&conn->local_tx_ctrl.cons, conn), smc_curs_copy(&urg, &conn->urg_curs, conn);
conn);
smc_curs_write(&urg,
smc_curs_read(&conn->urg_curs, conn),
conn);
answ = smc_curs_diff(conn->rmb_desc->len, answ = smc_curs_diff(conn->rmb_desc->len,
&cons, &urg) == 1; &cons, &urg) == 1;
} }
......
...@@ -34,14 +34,15 @@ static void smc_cdc_tx_handler(struct smc_wr_tx_pend_priv *pnd_snd, ...@@ -34,14 +34,15 @@ static void smc_cdc_tx_handler(struct smc_wr_tx_pend_priv *pnd_snd,
enum ib_wc_status wc_status) enum ib_wc_status wc_status)
{ {
struct smc_cdc_tx_pend *cdcpend = (struct smc_cdc_tx_pend *)pnd_snd; struct smc_cdc_tx_pend *cdcpend = (struct smc_cdc_tx_pend *)pnd_snd;
struct smc_connection *conn = cdcpend->conn;
struct smc_sock *smc; struct smc_sock *smc;
int diff; int diff;
if (!cdcpend->conn) if (!conn)
/* already dismissed */ /* already dismissed */
return; return;
smc = container_of(cdcpend->conn, struct smc_sock, conn); smc = container_of(conn, struct smc_sock, conn);
bh_lock_sock(&smc->sk); bh_lock_sock(&smc->sk);
if (!wc_status) { if (!wc_status) {
diff = smc_curs_diff(cdcpend->conn->sndbuf_desc->len, diff = smc_curs_diff(cdcpend->conn->sndbuf_desc->len,
...@@ -52,9 +53,7 @@ static void smc_cdc_tx_handler(struct smc_wr_tx_pend_priv *pnd_snd, ...@@ -52,9 +53,7 @@ static void smc_cdc_tx_handler(struct smc_wr_tx_pend_priv *pnd_snd,
atomic_add(diff, &cdcpend->conn->sndbuf_space); atomic_add(diff, &cdcpend->conn->sndbuf_space);
/* guarantee 0 <= sndbuf_space <= sndbuf_desc->len */ /* guarantee 0 <= sndbuf_space <= sndbuf_desc->len */
smp_mb__after_atomic(); smp_mb__after_atomic();
smc_curs_write(&cdcpend->conn->tx_curs_fin, smc_curs_copy(&conn->tx_curs_fin, &cdcpend->cursor, conn);
smc_curs_read(&cdcpend->cursor, cdcpend->conn),
cdcpend->conn);
} }
smc_tx_sndbuf_nonfull(smc); smc_tx_sndbuf_nonfull(smc);
bh_unlock_sock(&smc->sk); bh_unlock_sock(&smc->sk);
...@@ -110,9 +109,8 @@ int smc_cdc_msg_send(struct smc_connection *conn, ...@@ -110,9 +109,8 @@ int smc_cdc_msg_send(struct smc_connection *conn,
&conn->local_tx_ctrl, conn); &conn->local_tx_ctrl, conn);
rc = smc_wr_tx_send(link, (struct smc_wr_tx_pend_priv *)pend); rc = smc_wr_tx_send(link, (struct smc_wr_tx_pend_priv *)pend);
if (!rc) if (!rc)
smc_curs_write(&conn->rx_curs_confirmed, smc_curs_copy(&conn->rx_curs_confirmed,
smc_curs_read(&conn->local_tx_ctrl.cons, conn), &conn->local_tx_ctrl.cons, conn);
conn);
return rc; return rc;
} }
...@@ -194,8 +192,8 @@ int smcd_cdc_msg_send(struct smc_connection *conn) ...@@ -194,8 +192,8 @@ int smcd_cdc_msg_send(struct smc_connection *conn)
rc = smcd_tx_ism_write(conn, &cdc, sizeof(cdc), 0, 1); rc = smcd_tx_ism_write(conn, &cdc, sizeof(cdc), 0, 1);
if (rc) if (rc)
return rc; return rc;
smc_curs_write(&conn->rx_curs_confirmed, smc_curs_copy(&conn->rx_curs_confirmed, &conn->local_tx_ctrl.cons,
smc_curs_read(&conn->local_tx_ctrl.cons, conn), conn); conn);
/* Calculate transmitted data and increment free send buffer space */ /* Calculate transmitted data and increment free send buffer space */
diff = smc_curs_diff(conn->sndbuf_desc->len, &conn->tx_curs_fin, diff = smc_curs_diff(conn->sndbuf_desc->len, &conn->tx_curs_fin,
&conn->tx_curs_sent); &conn->tx_curs_sent);
...@@ -204,8 +202,7 @@ int smcd_cdc_msg_send(struct smc_connection *conn) ...@@ -204,8 +202,7 @@ int smcd_cdc_msg_send(struct smc_connection *conn)
atomic_add(diff, &conn->sndbuf_space); atomic_add(diff, &conn->sndbuf_space);
/* guarantee 0 <= sndbuf_space <= sndbuf_desc->len */ /* guarantee 0 <= sndbuf_space <= sndbuf_desc->len */
smp_mb__after_atomic(); smp_mb__after_atomic();
smc_curs_write(&conn->tx_curs_fin, smc_curs_copy(&conn->tx_curs_fin, &conn->tx_curs_sent, conn);
smc_curs_read(&conn->tx_curs_sent, conn), conn);
smc_tx_sndbuf_nonfull(smc); smc_tx_sndbuf_nonfull(smc);
return rc; return rc;
...@@ -225,9 +222,7 @@ static void smc_cdc_handle_urg_data_arrival(struct smc_sock *smc, ...@@ -225,9 +222,7 @@ static void smc_cdc_handle_urg_data_arrival(struct smc_sock *smc,
char *base; char *base;
/* new data included urgent business */ /* new data included urgent business */
smc_curs_write(&conn->urg_curs, smc_curs_copy(&conn->urg_curs, &conn->local_rx_ctrl.prod, conn);
smc_curs_read(&conn->local_rx_ctrl.prod, conn),
conn);
conn->urg_state = SMC_URG_VALID; conn->urg_state = SMC_URG_VALID;
if (!sock_flag(&smc->sk, SOCK_URGINLINE)) if (!sock_flag(&smc->sk, SOCK_URGINLINE))
/* we'll skip the urgent byte, so don't account for it */ /* we'll skip the urgent byte, so don't account for it */
...@@ -247,12 +242,8 @@ static void smc_cdc_msg_recv_action(struct smc_sock *smc, ...@@ -247,12 +242,8 @@ static void smc_cdc_msg_recv_action(struct smc_sock *smc,
struct smc_connection *conn = &smc->conn; struct smc_connection *conn = &smc->conn;
int diff_cons, diff_prod; int diff_cons, diff_prod;
smc_curs_write(&prod_old, smc_curs_copy(&prod_old, &conn->local_rx_ctrl.prod, conn);
smc_curs_read(&conn->local_rx_ctrl.prod, conn), smc_curs_copy(&cons_old, &conn->local_rx_ctrl.cons, conn);
conn);
smc_curs_write(&cons_old,
smc_curs_read(&conn->local_rx_ctrl.cons, conn),
conn);
smc_cdc_msg_to_host(&conn->local_rx_ctrl, cdc, conn); smc_cdc_msg_to_host(&conn->local_rx_ctrl, cdc, conn);
diff_cons = smc_curs_diff(conn->peer_rmbe_size, &cons_old, diff_cons = smc_curs_diff(conn->peer_rmbe_size, &cons_old,
......
...@@ -104,47 +104,34 @@ static inline u64 smc_curs_read(union smc_host_cursor *curs, ...@@ -104,47 +104,34 @@ static inline u64 smc_curs_read(union smc_host_cursor *curs,
#endif #endif
} }
static inline u64 smc_curs_read_net(union smc_cdc_cursor *curs, /* Copy cursor src into tgt */
struct smc_connection *conn) static inline void smc_curs_copy(union smc_host_cursor *tgt,
{ union smc_host_cursor *src,
#ifndef KERNEL_HAS_ATOMIC64
unsigned long flags;
u64 ret;
spin_lock_irqsave(&conn->acurs_lock, flags);
ret = curs->acurs;
spin_unlock_irqrestore(&conn->acurs_lock, flags);
return ret;
#else
return atomic64_read(&curs->acurs);
#endif
}
static inline void smc_curs_write(union smc_host_cursor *curs, u64 val,
struct smc_connection *conn) struct smc_connection *conn)
{ {
#ifndef KERNEL_HAS_ATOMIC64 #ifndef KERNEL_HAS_ATOMIC64
unsigned long flags; unsigned long flags;
spin_lock_irqsave(&conn->acurs_lock, flags); spin_lock_irqsave(&conn->acurs_lock, flags);
curs->acurs = val; tgt->acurs = src->acurs;
spin_unlock_irqrestore(&conn->acurs_lock, flags); spin_unlock_irqrestore(&conn->acurs_lock, flags);
#else #else
atomic64_set(&curs->acurs, val); atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
#endif #endif
} }
static inline void smc_curs_write_net(union smc_cdc_cursor *curs, u64 val, static inline void smc_curs_copy_net(union smc_cdc_cursor *tgt,
union smc_cdc_cursor *src,
struct smc_connection *conn) struct smc_connection *conn)
{ {
#ifndef KERNEL_HAS_ATOMIC64 #ifndef KERNEL_HAS_ATOMIC64
unsigned long flags; unsigned long flags;
spin_lock_irqsave(&conn->acurs_lock, flags); spin_lock_irqsave(&conn->acurs_lock, flags);
curs->acurs = val; tgt->acurs = src->acurs;
spin_unlock_irqrestore(&conn->acurs_lock, flags); spin_unlock_irqrestore(&conn->acurs_lock, flags);
#else #else
atomic64_set(&curs->acurs, val); atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
#endif #endif
} }
...@@ -179,7 +166,7 @@ static inline void smc_host_cursor_to_cdc(union smc_cdc_cursor *peer, ...@@ -179,7 +166,7 @@ static inline void smc_host_cursor_to_cdc(union smc_cdc_cursor *peer,
{ {
union smc_host_cursor temp; union smc_host_cursor temp;
smc_curs_write(&temp, smc_curs_read(local, conn), conn); smc_curs_copy(&temp, local, conn);
peer->count = htonl(temp.count); peer->count = htonl(temp.count);
peer->wrap = htons(temp.wrap); peer->wrap = htons(temp.wrap);
/* peer->reserved = htons(0); must be ensured by caller */ /* peer->reserved = htons(0); must be ensured by caller */
...@@ -206,8 +193,8 @@ static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local, ...@@ -206,8 +193,8 @@ static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local,
union smc_host_cursor temp, old; union smc_host_cursor temp, old;
union smc_cdc_cursor net; union smc_cdc_cursor net;
smc_curs_write(&old, smc_curs_read(local, conn), conn); smc_curs_copy(&old, local, conn);
smc_curs_write_net(&net, smc_curs_read_net(peer, conn), conn); smc_curs_copy_net(&net, peer, conn);
temp.count = ntohl(net.count); temp.count = ntohl(net.count);
temp.wrap = ntohs(net.wrap); temp.wrap = ntohs(net.wrap);
if ((old.wrap > temp.wrap) && temp.wrap) if ((old.wrap > temp.wrap) && temp.wrap)
...@@ -215,7 +202,7 @@ static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local, ...@@ -215,7 +202,7 @@ static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local,
if ((old.wrap == temp.wrap) && if ((old.wrap == temp.wrap) &&
(old.count > temp.count)) (old.count > temp.count))
return; return;
smc_curs_write(local, smc_curs_read(&temp, conn), conn); smc_curs_copy(local, &temp, conn);
} }
static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local, static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local,
......
...@@ -82,8 +82,7 @@ static int smc_rx_update_consumer(struct smc_sock *smc, ...@@ -82,8 +82,7 @@ static int smc_rx_update_consumer(struct smc_sock *smc,
} }
} }
smc_curs_write(&conn->local_tx_ctrl.cons, smc_curs_read(&cons, conn), smc_curs_copy(&conn->local_tx_ctrl.cons, &cons, conn);
conn);
/* send consumer cursor update if required */ /* send consumer cursor update if required */
/* similar to advertising new TCP rcv_wnd if required */ /* similar to advertising new TCP rcv_wnd if required */
...@@ -97,8 +96,7 @@ static void smc_rx_update_cons(struct smc_sock *smc, size_t len) ...@@ -97,8 +96,7 @@ static void smc_rx_update_cons(struct smc_sock *smc, size_t len)
struct smc_connection *conn = &smc->conn; struct smc_connection *conn = &smc->conn;
union smc_host_cursor cons; union smc_host_cursor cons;
smc_curs_write(&cons, smc_curs_read(&conn->local_tx_ctrl.cons, conn), smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
conn);
smc_rx_update_consumer(smc, cons, len); smc_rx_update_consumer(smc, cons, len);
} }
...@@ -245,10 +243,7 @@ static int smc_rx_recv_urg(struct smc_sock *smc, struct msghdr *msg, int len, ...@@ -245,10 +243,7 @@ static int smc_rx_recv_urg(struct smc_sock *smc, struct msghdr *msg, int len,
if (!(flags & MSG_TRUNC)) if (!(flags & MSG_TRUNC))
rc = memcpy_to_msg(msg, &conn->urg_rx_byte, 1); rc = memcpy_to_msg(msg, &conn->urg_rx_byte, 1);
len = 1; len = 1;
smc_curs_write(&cons, smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
smc_curs_read(&conn->local_tx_ctrl.cons,
conn),
conn);
if (smc_curs_diff(conn->rmb_desc->len, &cons, if (smc_curs_diff(conn->rmb_desc->len, &cons,
&conn->urg_curs) > 1) &conn->urg_curs) > 1)
conn->urg_rx_skip_pend = true; conn->urg_rx_skip_pend = true;
...@@ -370,9 +365,7 @@ int smc_rx_recvmsg(struct smc_sock *smc, struct msghdr *msg, ...@@ -370,9 +365,7 @@ int smc_rx_recvmsg(struct smc_sock *smc, struct msghdr *msg,
continue; continue;
} }
smc_curs_write(&cons, smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
smc_curs_read(&conn->local_tx_ctrl.cons, conn),
conn);
/* subsequent splice() calls pick up where previous left */ /* subsequent splice() calls pick up where previous left */
if (splbytes) if (splbytes)
smc_curs_add(conn->rmb_desc->len, &cons, splbytes); smc_curs_add(conn->rmb_desc->len, &cons, splbytes);
......
...@@ -181,9 +181,7 @@ int smc_tx_sendmsg(struct smc_sock *smc, struct msghdr *msg, size_t len) ...@@ -181,9 +181,7 @@ int smc_tx_sendmsg(struct smc_sock *smc, struct msghdr *msg, size_t len)
copylen = min_t(size_t, send_remaining, writespace); copylen = min_t(size_t, send_remaining, writespace);
/* determine start of sndbuf */ /* determine start of sndbuf */
sndbuf_base = conn->sndbuf_desc->cpu_addr; sndbuf_base = conn->sndbuf_desc->cpu_addr;
smc_curs_write(&prep, smc_curs_copy(&prep, &conn->tx_curs_prep, conn);
smc_curs_read(&conn->tx_curs_prep, conn),
conn);
tx_cnt_prep = prep.count; tx_cnt_prep = prep.count;
/* determine chunks where to write into sndbuf */ /* determine chunks where to write into sndbuf */
/* either unwrapped case, or 1st chunk of wrapped case */ /* either unwrapped case, or 1st chunk of wrapped case */
...@@ -214,9 +212,7 @@ int smc_tx_sendmsg(struct smc_sock *smc, struct msghdr *msg, size_t len) ...@@ -214,9 +212,7 @@ int smc_tx_sendmsg(struct smc_sock *smc, struct msghdr *msg, size_t len)
smc_sndbuf_sync_sg_for_device(conn); smc_sndbuf_sync_sg_for_device(conn);
/* update cursors */ /* update cursors */
smc_curs_add(conn->sndbuf_desc->len, &prep, copylen); smc_curs_add(conn->sndbuf_desc->len, &prep, copylen);
smc_curs_write(&conn->tx_curs_prep, smc_curs_copy(&conn->tx_curs_prep, &prep, conn);
smc_curs_read(&prep, conn),
conn);
/* increased in send tasklet smc_cdc_tx_handler() */ /* increased in send tasklet smc_cdc_tx_handler() */
smp_mb__before_atomic(); smp_mb__before_atomic();
atomic_sub(copylen, &conn->sndbuf_space); atomic_sub(copylen, &conn->sndbuf_space);
...@@ -417,8 +413,8 @@ static int smc_tx_rdma_writes(struct smc_connection *conn) ...@@ -417,8 +413,8 @@ static int smc_tx_rdma_writes(struct smc_connection *conn)
int rc; int rc;
/* source: sndbuf */ /* source: sndbuf */
smc_curs_write(&sent, smc_curs_read(&conn->tx_curs_sent, conn), conn); smc_curs_copy(&sent, &conn->tx_curs_sent, conn);
smc_curs_write(&prep, smc_curs_read(&conn->tx_curs_prep, conn), conn); smc_curs_copy(&prep, &conn->tx_curs_prep, conn);
/* cf. wmem_alloc - (snd_max - snd_una) */ /* cf. wmem_alloc - (snd_max - snd_una) */
to_send = smc_curs_diff(conn->sndbuf_desc->len, &sent, &prep); to_send = smc_curs_diff(conn->sndbuf_desc->len, &sent, &prep);
if (to_send <= 0) if (to_send <= 0)
...@@ -429,12 +425,8 @@ static int smc_tx_rdma_writes(struct smc_connection *conn) ...@@ -429,12 +425,8 @@ static int smc_tx_rdma_writes(struct smc_connection *conn)
rmbespace = atomic_read(&conn->peer_rmbe_space); rmbespace = atomic_read(&conn->peer_rmbe_space);
if (rmbespace <= 0) if (rmbespace <= 0)
return 0; return 0;
smc_curs_write(&prod, smc_curs_copy(&prod, &conn->local_tx_ctrl.prod, conn);
smc_curs_read(&conn->local_tx_ctrl.prod, conn), smc_curs_copy(&cons, &conn->local_rx_ctrl.cons, conn);
conn);
smc_curs_write(&cons,
smc_curs_read(&conn->local_rx_ctrl.cons, conn),
conn);
/* if usable snd_wnd closes ask peer to advertise once it opens again */ /* if usable snd_wnd closes ask peer to advertise once it opens again */
pflags = &conn->local_tx_ctrl.prod_flags; pflags = &conn->local_tx_ctrl.prod_flags;
...@@ -481,14 +473,9 @@ static int smc_tx_rdma_writes(struct smc_connection *conn) ...@@ -481,14 +473,9 @@ static int smc_tx_rdma_writes(struct smc_connection *conn)
pflags->urg_data_present = 1; pflags->urg_data_present = 1;
smc_tx_advance_cursors(conn, &prod, &sent, len); smc_tx_advance_cursors(conn, &prod, &sent, len);
/* update connection's cursors with advanced local cursors */ /* update connection's cursors with advanced local cursors */
smc_curs_write(&conn->local_tx_ctrl.prod, smc_curs_copy(&conn->local_tx_ctrl.prod, &prod, conn);
smc_curs_read(&prod, conn),
conn);
/* dst: peer RMBE */ /* dst: peer RMBE */
smc_curs_write(&conn->tx_curs_sent, smc_curs_copy(&conn->tx_curs_sent, &sent, conn);/* src: local sndbuf */
smc_curs_read(&sent, conn),
conn);
/* src: local sndbuf */
return 0; return 0;
} }
...@@ -606,17 +593,11 @@ void smc_tx_consumer_update(struct smc_connection *conn, bool force) ...@@ -606,17 +593,11 @@ void smc_tx_consumer_update(struct smc_connection *conn, bool force)
int sender_free = conn->rmb_desc->len; int sender_free = conn->rmb_desc->len;
int to_confirm; int to_confirm;
smc_curs_write(&cons, smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
smc_curs_read(&conn->local_tx_ctrl.cons, conn), smc_curs_copy(&cfed, &conn->rx_curs_confirmed, conn);
conn);
smc_curs_write(&cfed,
smc_curs_read(&conn->rx_curs_confirmed, conn),
conn);
to_confirm = smc_curs_diff(conn->rmb_desc->len, &cfed, &cons); to_confirm = smc_curs_diff(conn->rmb_desc->len, &cfed, &cons);
if (to_confirm > conn->rmbe_update_limit) { if (to_confirm > conn->rmbe_update_limit) {
smc_curs_write(&prod, smc_curs_copy(&prod, &conn->local_rx_ctrl.prod, conn);
smc_curs_read(&conn->local_rx_ctrl.prod, conn),
conn);
sender_free = conn->rmb_desc->len - sender_free = conn->rmb_desc->len -
smc_curs_diff(conn->rmb_desc->len, &prod, &cfed); smc_curs_diff(conn->rmb_desc->len, &prod, &cfed);
} }
...@@ -632,9 +613,8 @@ void smc_tx_consumer_update(struct smc_connection *conn, bool force) ...@@ -632,9 +613,8 @@ void smc_tx_consumer_update(struct smc_connection *conn, bool force)
SMC_TX_WORK_DELAY); SMC_TX_WORK_DELAY);
return; return;
} }
smc_curs_write(&conn->rx_curs_confirmed, smc_curs_copy(&conn->rx_curs_confirmed,
smc_curs_read(&conn->local_tx_ctrl.cons, conn), &conn->local_tx_ctrl.cons, conn);
conn);
conn->local_rx_ctrl.prod_flags.cons_curs_upd_req = 0; conn->local_rx_ctrl.prod_flags.cons_curs_upd_req = 0;
} }
if (conn->local_rx_ctrl.prod_flags.write_blocked && if (conn->local_rx_ctrl.prod_flags.write_blocked &&
......
...@@ -22,8 +22,8 @@ static inline int smc_tx_prepared_sends(struct smc_connection *conn) ...@@ -22,8 +22,8 @@ static inline int smc_tx_prepared_sends(struct smc_connection *conn)
{ {
union smc_host_cursor sent, prep; union smc_host_cursor sent, prep;
smc_curs_write(&sent, smc_curs_read(&conn->tx_curs_sent, conn), conn); smc_curs_copy(&sent, &conn->tx_curs_sent, conn);
smc_curs_write(&prep, smc_curs_read(&conn->tx_curs_prep, conn), conn); smc_curs_copy(&prep, &conn->tx_curs_prep, conn);
return smc_curs_diff(conn->sndbuf_desc->len, &sent, &prep); return smc_curs_diff(conn->sndbuf_desc->len, &sent, &prep);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment