Commit a5b1cf28 authored by Herbert Xu's avatar Herbert Xu Committed by David S. Miller

gro: Avoid unnecessary comparison after skb_gro_header

For the overwhelming majority of cases, skb_gro_header's return
value cannot be NULL.  Yet we must check it because of its current
form.  This patch splits it up into multiple functions in order
to avoid this.
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 7489594c
...@@ -1132,18 +1132,23 @@ static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len) ...@@ -1132,18 +1132,23 @@ static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)
NAPI_GRO_CB(skb)->data_offset += len; NAPI_GRO_CB(skb)->data_offset += len;
} }
static inline void *skb_gro_header(struct sk_buff *skb, unsigned int hlen) static inline void *skb_gro_header_fast(struct sk_buff *skb,
unsigned int offset)
{ {
unsigned int offset = skb_gro_offset(skb); return NAPI_GRO_CB(skb)->frag0 + offset;
}
static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen)
{
return NAPI_GRO_CB(skb)->frag0_len < hlen;
}
hlen += offset; static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
if (NAPI_GRO_CB(skb)->frag0_len < hlen) { unsigned int offset)
{
NAPI_GRO_CB(skb)->frag0 = NULL; NAPI_GRO_CB(skb)->frag0 = NULL;
NAPI_GRO_CB(skb)->frag0_len = 0; NAPI_GRO_CB(skb)->frag0_len = 0;
return pskb_may_pull(skb, hlen) ? skb->data + offset : NULL; return pskb_may_pull(skb, hlen) ? skb->data + offset : NULL;
}
return NAPI_GRO_CB(skb)->frag0 + offset;
} }
static inline void *skb_gro_mac_header(struct sk_buff *skb) static inline void *skb_gro_mac_header(struct sk_buff *skb)
......
...@@ -2590,18 +2590,25 @@ struct sk_buff *napi_frags_skb(struct napi_struct *napi) ...@@ -2590,18 +2590,25 @@ struct sk_buff *napi_frags_skb(struct napi_struct *napi)
{ {
struct sk_buff *skb = napi->skb; struct sk_buff *skb = napi->skb;
struct ethhdr *eth; struct ethhdr *eth;
unsigned int hlen;
unsigned int off;
napi->skb = NULL; napi->skb = NULL;
skb_reset_mac_header(skb); skb_reset_mac_header(skb);
skb_gro_reset_offset(skb); skb_gro_reset_offset(skb);
eth = skb_gro_header(skb, sizeof(*eth)); off = skb_gro_offset(skb);
if (!eth) { hlen = off + sizeof(*eth);
eth = skb_gro_header_fast(skb, off);
if (skb_gro_header_hard(skb, hlen)) {
eth = skb_gro_header_slow(skb, hlen, off);
if (unlikely(!eth)) {
napi_reuse_skb(napi, skb); napi_reuse_skb(napi, skb);
skb = NULL; skb = NULL;
goto out; goto out;
} }
}
skb_gro_pull(skb, sizeof(*eth)); skb_gro_pull(skb, sizeof(*eth));
......
...@@ -1246,13 +1246,20 @@ static struct sk_buff **inet_gro_receive(struct sk_buff **head, ...@@ -1246,13 +1246,20 @@ static struct sk_buff **inet_gro_receive(struct sk_buff **head,
struct sk_buff **pp = NULL; struct sk_buff **pp = NULL;
struct sk_buff *p; struct sk_buff *p;
struct iphdr *iph; struct iphdr *iph;
unsigned int hlen;
unsigned int off;
int flush = 1; int flush = 1;
int proto; int proto;
int id; int id;
iph = skb_gro_header(skb, sizeof(*iph)); off = skb_gro_offset(skb);
hlen = off + sizeof(*iph);
iph = skb_gro_header_fast(skb, off);
if (skb_gro_header_hard(skb, hlen)) {
iph = skb_gro_header_slow(skb, hlen, off);
if (unlikely(!iph)) if (unlikely(!iph))
goto out; goto out;
}
proto = iph->protocol & (MAX_INET_PROTOS - 1); proto = iph->protocol & (MAX_INET_PROTOS - 1);
......
...@@ -2518,20 +2518,30 @@ struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb) ...@@ -2518,20 +2518,30 @@ struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb)
unsigned int thlen; unsigned int thlen;
unsigned int flags; unsigned int flags;
unsigned int mss = 1; unsigned int mss = 1;
unsigned int hlen;
unsigned int off;
int flush = 1; int flush = 1;
int i; int i;
th = skb_gro_header(skb, sizeof(*th)); off = skb_gro_offset(skb);
hlen = off + sizeof(*th);
th = skb_gro_header_fast(skb, off);
if (skb_gro_header_hard(skb, hlen)) {
th = skb_gro_header_slow(skb, hlen, off);
if (unlikely(!th)) if (unlikely(!th))
goto out; goto out;
}
thlen = th->doff * 4; thlen = th->doff * 4;
if (thlen < sizeof(*th)) if (thlen < sizeof(*th))
goto out; goto out;
th = skb_gro_header(skb, thlen); hlen = off + thlen;
if (skb_gro_header_hard(skb, hlen)) {
th = skb_gro_header_slow(skb, hlen, off);
if (unlikely(!th)) if (unlikely(!th))
goto out; goto out;
}
skb_gro_pull(skb, thlen); skb_gro_pull(skb, thlen);
......
...@@ -817,13 +817,20 @@ static struct sk_buff **ipv6_gro_receive(struct sk_buff **head, ...@@ -817,13 +817,20 @@ static struct sk_buff **ipv6_gro_receive(struct sk_buff **head,
struct sk_buff *p; struct sk_buff *p;
struct ipv6hdr *iph; struct ipv6hdr *iph;
unsigned int nlen; unsigned int nlen;
unsigned int hlen;
unsigned int off;
int flush = 1; int flush = 1;
int proto; int proto;
__wsum csum; __wsum csum;
iph = skb_gro_header(skb, sizeof(*iph)); off = skb_gro_offset(skb);
hlen = off + sizeof(*iph);
iph = skb_gro_header_fast(skb, off);
if (skb_gro_header_hard(skb, hlen)) {
iph = skb_gro_header_slow(skb, hlen, off);
if (unlikely(!iph)) if (unlikely(!iph))
goto out; goto out;
}
skb_gro_pull(skb, sizeof(*iph)); skb_gro_pull(skb, sizeof(*iph));
skb_set_transport_header(skb, skb_gro_offset(skb)); skb_set_transport_header(skb, skb_gro_offset(skb));
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment