Commit b223feb9 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

tcp: tsq: add shortcut in tcp_tasklet_func()

Under high stress, I've seen tcp_tasklet_func() consuming
~700 usec, handling ~150 tcp sockets.

By setting TCP_TSQ_DEFERRED in tcp_wfree(), we give a chance
for other cpus/threads entering tcp_write_xmit() to grab it,
allowing tcp_tasklet_func() to skip sockets that already did
an xmit cycle.

In the future, we might give to ACK processing an increased
budget to reduce even more tcp_tasklet_func() amount of work.
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 408f0a6c
...@@ -767,19 +767,19 @@ static void tcp_tasklet_func(unsigned long data) ...@@ -767,19 +767,19 @@ static void tcp_tasklet_func(unsigned long data)
list_for_each_safe(q, n, &list) { list_for_each_safe(q, n, &list) {
tp = list_entry(q, struct tcp_sock, tsq_node); tp = list_entry(q, struct tcp_sock, tsq_node);
list_del(&tp->tsq_node); list_del(&tp->tsq_node);
clear_bit(TSQ_QUEUED, &tp->tsq_flags);
sk = (struct sock *)tp; sk = (struct sock *)tp;
bh_lock_sock(sk); if (!sk->sk_lock.owned &&
test_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags)) {
if (!sock_owned_by_user(sk)) { bh_lock_sock(sk);
tcp_tsq_handler(sk); if (!sock_owned_by_user(sk)) {
} else { clear_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
/* defer the work to tcp_release_cb() */ tcp_tsq_handler(sk);
set_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags); }
bh_unlock_sock(sk);
} }
bh_unlock_sock(sk);
clear_bit(TSQ_QUEUED, &tp->tsq_flags);
sk_free(sk); sk_free(sk);
} }
} }
...@@ -884,7 +884,7 @@ void tcp_wfree(struct sk_buff *skb) ...@@ -884,7 +884,7 @@ void tcp_wfree(struct sk_buff *skb)
if (!(oval & TSQF_THROTTLED) || (oval & TSQF_QUEUED)) if (!(oval & TSQF_THROTTLED) || (oval & TSQF_QUEUED))
goto out; goto out;
nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED; nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED | TCPF_TSQ_DEFERRED;
nval = cmpxchg(&tp->tsq_flags, oval, nval); nval = cmpxchg(&tp->tsq_flags, oval, nval);
if (nval != oval) if (nval != oval)
continue; continue;
...@@ -2229,6 +2229,8 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, ...@@ -2229,6 +2229,8 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
unlikely(tso_fragment(sk, skb, limit, mss_now, gfp))) unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
break; break;
if (test_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags))
clear_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
if (tcp_small_queue_check(sk, skb, 0)) if (tcp_small_queue_check(sk, skb, 0))
break; break;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment