Commit a6024562 authored by Tom Herbert's avatar Tom Herbert Committed by David S. Miller

udp: Add GRO functions to UDP socket

This patch adds GRO functions (gro_receive and gro_complete) to UDP
sockets. udp_gro_receive is changed to perform socket lookup on a
packet. If a socket is found the related GRO functions are called.

This features obsoletes using UDP offload infrastructure for GRO
(udp_offload). This has the advantage of not being limited to provide
offload on a per port basis, GRO is now applied to whatever individual
UDP sockets are bound to.  This also allows the possbility of
"application defined GRO"-- that is we can attach something like
a BPF program to a UDP socket to perfrom GRO on an application
layer protocol.
Signed-off-by: default avatarTom Herbert <tom@herbertland.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 63058308
...@@ -71,6 +71,14 @@ struct udp_sock { ...@@ -71,6 +71,14 @@ struct udp_sock {
*/ */
int (*encap_rcv)(struct sock *sk, struct sk_buff *skb); int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
void (*encap_destroy)(struct sock *sk); void (*encap_destroy)(struct sock *sk);
/* GRO functions for UDP socket */
struct sk_buff ** (*gro_receive)(struct sock *sk,
struct sk_buff **head,
struct sk_buff *skb);
int (*gro_complete)(struct sock *sk,
struct sk_buff *skb,
int nhoff);
}; };
static inline struct udp_sock *udp_sk(const struct sock *sk) static inline struct udp_sock *udp_sk(const struct sock *sk)
......
...@@ -167,9 +167,12 @@ static inline void udp_csum_pull_header(struct sk_buff *skb) ...@@ -167,9 +167,12 @@ static inline void udp_csum_pull_header(struct sk_buff *skb)
UDP_SKB_CB(skb)->cscov -= sizeof(struct udphdr); UDP_SKB_CB(skb)->cscov -= sizeof(struct udphdr);
} }
typedef struct sock *(*udp_lookup_t)(struct sk_buff *skb, __be16 sport,
__be16 dport);
struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb, struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb,
struct udphdr *uh); struct udphdr *uh, udp_lookup_t lookup);
int udp_gro_complete(struct sk_buff *skb, int nhoff); int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup);
static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb) static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb)
{ {
......
...@@ -179,6 +179,7 @@ struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb, ...@@ -179,6 +179,7 @@ struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb,
return segs; return segs;
} }
EXPORT_SYMBOL(skb_udp_tunnel_segment);
static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb, static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb,
netdev_features_t features) netdev_features_t features)
...@@ -304,13 +305,13 @@ void udp_del_offload(struct udp_offload *uo) ...@@ -304,13 +305,13 @@ void udp_del_offload(struct udp_offload *uo)
EXPORT_SYMBOL(udp_del_offload); EXPORT_SYMBOL(udp_del_offload);
struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb, struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb,
struct udphdr *uh) struct udphdr *uh, udp_lookup_t lookup)
{ {
struct udp_offload_priv *uo_priv;
struct sk_buff *p, **pp = NULL; struct sk_buff *p, **pp = NULL;
struct udphdr *uh2; struct udphdr *uh2;
unsigned int off = skb_gro_offset(skb); unsigned int off = skb_gro_offset(skb);
int flush = 1; int flush = 1;
struct sock *sk;
if (NAPI_GRO_CB(skb)->encap_mark || if (NAPI_GRO_CB(skb)->encap_mark ||
(skb->ip_summed != CHECKSUM_PARTIAL && (skb->ip_summed != CHECKSUM_PARTIAL &&
...@@ -322,13 +323,11 @@ struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb, ...@@ -322,13 +323,11 @@ struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb,
NAPI_GRO_CB(skb)->encap_mark = 1; NAPI_GRO_CB(skb)->encap_mark = 1;
rcu_read_lock(); rcu_read_lock();
uo_priv = rcu_dereference(udp_offload_base); sk = (*lookup)(skb, uh->source, uh->dest);
for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) {
if (net_eq(read_pnet(&uo_priv->net), dev_net(skb->dev)) && if (sk && udp_sk(sk)->gro_receive)
uo_priv->offload->port == uh->dest && goto unflush;
uo_priv->offload->callbacks.gro_receive)
goto unflush;
}
goto out_unlock; goto out_unlock;
unflush: unflush:
...@@ -352,9 +351,7 @@ struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb, ...@@ -352,9 +351,7 @@ struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb,
skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */ skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */
skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr)); skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr));
NAPI_GRO_CB(skb)->proto = uo_priv->offload->ipproto; pp = udp_sk(sk)->gro_receive(sk, head, skb);
pp = uo_priv->offload->callbacks.gro_receive(head, skb,
uo_priv->offload);
out_unlock: out_unlock:
rcu_read_unlock(); rcu_read_unlock();
...@@ -362,6 +359,7 @@ struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb, ...@@ -362,6 +359,7 @@ struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb,
NAPI_GRO_CB(skb)->flush |= flush; NAPI_GRO_CB(skb)->flush |= flush;
return pp; return pp;
} }
EXPORT_SYMBOL(udp_gro_receive);
static struct sk_buff **udp4_gro_receive(struct sk_buff **head, static struct sk_buff **udp4_gro_receive(struct sk_buff **head,
struct sk_buff *skb) struct sk_buff *skb)
...@@ -383,39 +381,28 @@ static struct sk_buff **udp4_gro_receive(struct sk_buff **head, ...@@ -383,39 +381,28 @@ static struct sk_buff **udp4_gro_receive(struct sk_buff **head,
inet_gro_compute_pseudo); inet_gro_compute_pseudo);
skip: skip:
NAPI_GRO_CB(skb)->is_ipv6 = 0; NAPI_GRO_CB(skb)->is_ipv6 = 0;
return udp_gro_receive(head, skb, uh); return udp_gro_receive(head, skb, uh, udp4_lib_lookup_skb);
flush: flush:
NAPI_GRO_CB(skb)->flush = 1; NAPI_GRO_CB(skb)->flush = 1;
return NULL; return NULL;
} }
int udp_gro_complete(struct sk_buff *skb, int nhoff) int udp_gro_complete(struct sk_buff *skb, int nhoff,
udp_lookup_t lookup)
{ {
struct udp_offload_priv *uo_priv;
__be16 newlen = htons(skb->len - nhoff); __be16 newlen = htons(skb->len - nhoff);
struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
int err = -ENOSYS; int err = -ENOSYS;
struct sock *sk;
uh->len = newlen; uh->len = newlen;
rcu_read_lock(); rcu_read_lock();
sk = (*lookup)(skb, uh->source, uh->dest);
uo_priv = rcu_dereference(udp_offload_base); if (sk && udp_sk(sk)->gro_complete)
for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) { err = udp_sk(sk)->gro_complete(sk, skb,
if (net_eq(read_pnet(&uo_priv->net), dev_net(skb->dev)) && nhoff + sizeof(struct udphdr));
uo_priv->offload->port == uh->dest &&
uo_priv->offload->callbacks.gro_complete)
break;
}
if (uo_priv) {
NAPI_GRO_CB(skb)->proto = uo_priv->offload->ipproto;
err = uo_priv->offload->callbacks.gro_complete(skb,
nhoff + sizeof(struct udphdr),
uo_priv->offload);
}
rcu_read_unlock(); rcu_read_unlock();
if (skb->remcsum_offload) if (skb->remcsum_offload)
...@@ -426,6 +413,7 @@ int udp_gro_complete(struct sk_buff *skb, int nhoff) ...@@ -426,6 +413,7 @@ int udp_gro_complete(struct sk_buff *skb, int nhoff)
return err; return err;
} }
EXPORT_SYMBOL(udp_gro_complete);
static int udp4_gro_complete(struct sk_buff *skb, int nhoff) static int udp4_gro_complete(struct sk_buff *skb, int nhoff)
{ {
...@@ -440,7 +428,7 @@ static int udp4_gro_complete(struct sk_buff *skb, int nhoff) ...@@ -440,7 +428,7 @@ static int udp4_gro_complete(struct sk_buff *skb, int nhoff)
skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
} }
return udp_gro_complete(skb, nhoff); return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb);
} }
static const struct net_offload udpv4_offload = { static const struct net_offload udpv4_offload = {
......
...@@ -8,9 +8,10 @@ ipv6-objs := af_inet6.o anycast.o ip6_output.o ip6_input.o addrconf.o \ ...@@ -8,9 +8,10 @@ ipv6-objs := af_inet6.o anycast.o ip6_output.o ip6_input.o addrconf.o \
addrlabel.o \ addrlabel.o \
route.o ip6_fib.o ipv6_sockglue.o ndisc.o udp.o udplite.o \ route.o ip6_fib.o ipv6_sockglue.o ndisc.o udp.o udplite.o \
raw.o icmp.o mcast.o reassembly.o tcp_ipv6.o ping.o \ raw.o icmp.o mcast.o reassembly.o tcp_ipv6.o ping.o \
exthdrs.o datagram.o ip6_flowlabel.o inet6_connection_sock.o exthdrs.o datagram.o ip6_flowlabel.o inet6_connection_sock.o \
udp_offload.o
ipv6-offload := ip6_offload.o tcpv6_offload.o udp_offload.o exthdrs_offload.o ipv6-offload := ip6_offload.o tcpv6_offload.o exthdrs_offload.o
ipv6-$(CONFIG_SYSCTL) = sysctl_net_ipv6.o ipv6-$(CONFIG_SYSCTL) = sysctl_net_ipv6.o
ipv6-$(CONFIG_IPV6_MROUTE) += ip6mr.o ipv6-$(CONFIG_IPV6_MROUTE) += ip6mr.o
......
...@@ -64,6 +64,8 @@ ...@@ -64,6 +64,8 @@
#include <asm/uaccess.h> #include <asm/uaccess.h>
#include <linux/mroute6.h> #include <linux/mroute6.h>
#include "ip6_offload.h"
MODULE_AUTHOR("Cast of dozens"); MODULE_AUTHOR("Cast of dozens");
MODULE_DESCRIPTION("IPv6 protocol stack for Linux"); MODULE_DESCRIPTION("IPv6 protocol stack for Linux");
MODULE_LICENSE("GPL"); MODULE_LICENSE("GPL");
...@@ -959,6 +961,10 @@ static int __init inet6_init(void) ...@@ -959,6 +961,10 @@ static int __init inet6_init(void)
if (err) if (err)
goto udplitev6_fail; goto udplitev6_fail;
err = udpv6_offload_init();
if (err)
goto udpv6_offload_fail;
err = tcpv6_init(); err = tcpv6_init();
if (err) if (err)
goto tcpv6_fail; goto tcpv6_fail;
...@@ -988,6 +994,8 @@ static int __init inet6_init(void) ...@@ -988,6 +994,8 @@ static int __init inet6_init(void)
ipv6_packet_fail: ipv6_packet_fail:
tcpv6_exit(); tcpv6_exit();
tcpv6_fail: tcpv6_fail:
udpv6_offload_exit();
udpv6_offload_fail:
udplitev6_exit(); udplitev6_exit();
udplitev6_fail: udplitev6_fail:
udpv6_exit(); udpv6_exit();
......
...@@ -325,8 +325,6 @@ static int __init ipv6_offload_init(void) ...@@ -325,8 +325,6 @@ static int __init ipv6_offload_init(void)
if (tcpv6_offload_init() < 0) if (tcpv6_offload_init() < 0)
pr_crit("%s: Cannot add TCP protocol offload\n", __func__); pr_crit("%s: Cannot add TCP protocol offload\n", __func__);
if (udp_offload_init() < 0)
pr_crit("%s: Cannot add UDP protocol offload\n", __func__);
if (ipv6_exthdrs_offload_init() < 0) if (ipv6_exthdrs_offload_init() < 0)
pr_crit("%s: Cannot add EXTHDRS protocol offload\n", __func__); pr_crit("%s: Cannot add EXTHDRS protocol offload\n", __func__);
......
...@@ -12,7 +12,8 @@ ...@@ -12,7 +12,8 @@
#define __ip6_offload_h #define __ip6_offload_h
int ipv6_exthdrs_offload_init(void); int ipv6_exthdrs_offload_init(void);
int udp_offload_init(void); int udpv6_offload_init(void);
int udpv6_offload_exit(void);
int tcpv6_offload_init(void); int tcpv6_offload_init(void);
#endif #endif
...@@ -153,7 +153,7 @@ static struct sk_buff **udp6_gro_receive(struct sk_buff **head, ...@@ -153,7 +153,7 @@ static struct sk_buff **udp6_gro_receive(struct sk_buff **head,
skip: skip:
NAPI_GRO_CB(skb)->is_ipv6 = 1; NAPI_GRO_CB(skb)->is_ipv6 = 1;
return udp_gro_receive(head, skb, uh); return udp_gro_receive(head, skb, uh, udp6_lib_lookup_skb);
flush: flush:
NAPI_GRO_CB(skb)->flush = 1; NAPI_GRO_CB(skb)->flush = 1;
...@@ -173,7 +173,7 @@ static int udp6_gro_complete(struct sk_buff *skb, int nhoff) ...@@ -173,7 +173,7 @@ static int udp6_gro_complete(struct sk_buff *skb, int nhoff)
skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
} }
return udp_gro_complete(skb, nhoff); return udp_gro_complete(skb, nhoff, udp6_lib_lookup_skb);
} }
static const struct net_offload udpv6_offload = { static const struct net_offload udpv6_offload = {
...@@ -184,7 +184,12 @@ static const struct net_offload udpv6_offload = { ...@@ -184,7 +184,12 @@ static const struct net_offload udpv6_offload = {
}, },
}; };
int __init udp_offload_init(void) int udpv6_offload_init(void)
{ {
return inet6_add_offload(&udpv6_offload, IPPROTO_UDP); return inet6_add_offload(&udpv6_offload, IPPROTO_UDP);
} }
int udpv6_offload_exit(void)
{
return inet6_del_offload(&udpv6_offload, IPPROTO_UDP);
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment