Commit 29f3490b authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

net: use indirect call wrappers for skb_copy_datagram_iter()

TCP recvmsg() calls skb_copy_datagram_iter(), which
calls an indirect function (cb pointing to simple_copy_to_iter())
for every MSS (fragment) present in the skb.

CONFIG_RETPOLINE=y forces a very expensive operation
that we can avoid thanks to indirect call wrappers.

This patch gives a 13% increase of performance on
a single flow, if the bottleneck is the thread reading
the TCP socket.

Fixes: 950fcaec ("datagram: consolidate datagram copy to iter helpers")
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Acked-by: default avatarPaolo Abeni <pabeni@redhat.com>
Acked-by: default avatarWillem de Bruijn <willemb@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent fab90c82
...@@ -51,6 +51,7 @@ ...@@ -51,6 +51,7 @@
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/pagemap.h> #include <linux/pagemap.h>
#include <linux/uio.h> #include <linux/uio.h>
#include <linux/indirect_call_wrapper.h>
#include <net/protocol.h> #include <net/protocol.h>
#include <linux/skbuff.h> #include <linux/skbuff.h>
...@@ -403,6 +404,11 @@ int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags) ...@@ -403,6 +404,11 @@ int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags)
} }
EXPORT_SYMBOL(skb_kill_datagram); EXPORT_SYMBOL(skb_kill_datagram);
INDIRECT_CALLABLE_DECLARE(static size_t simple_copy_to_iter(const void *addr,
size_t bytes,
void *data __always_unused,
struct iov_iter *i));
static int __skb_datagram_iter(const struct sk_buff *skb, int offset, static int __skb_datagram_iter(const struct sk_buff *skb, int offset,
struct iov_iter *to, int len, bool fault_short, struct iov_iter *to, int len, bool fault_short,
size_t (*cb)(const void *, size_t, void *, size_t (*cb)(const void *, size_t, void *,
...@@ -416,7 +422,8 @@ static int __skb_datagram_iter(const struct sk_buff *skb, int offset, ...@@ -416,7 +422,8 @@ static int __skb_datagram_iter(const struct sk_buff *skb, int offset,
if (copy > 0) { if (copy > 0) {
if (copy > len) if (copy > len)
copy = len; copy = len;
n = cb(skb->data + offset, copy, data, to); n = INDIRECT_CALL_1(cb, simple_copy_to_iter,
skb->data + offset, copy, data, to);
offset += n; offset += n;
if (n != copy) if (n != copy)
goto short_copy; goto short_copy;
...@@ -438,7 +445,8 @@ static int __skb_datagram_iter(const struct sk_buff *skb, int offset, ...@@ -438,7 +445,8 @@ static int __skb_datagram_iter(const struct sk_buff *skb, int offset,
if (copy > len) if (copy > len)
copy = len; copy = len;
n = cb(vaddr + skb_frag_off(frag) + offset - start, n = INDIRECT_CALL_1(cb, simple_copy_to_iter,
vaddr + skb_frag_off(frag) + offset - start,
copy, data, to); copy, data, to);
kunmap(page); kunmap(page);
offset += n; offset += n;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment