Commit 5e10da53 authored by Paolo Abeni's avatar Paolo Abeni Committed by David S. Miller

skbuff: allow 'slow_gro' for skb carring sock reference

This change leverages the infrastructure introduced by the previous
patches to allow soft devices passing to the GRO engine owned skbs
without impacting the fast-path.

It's up to the GRO caller ensuring the slow_gro bit validity before
invoking the GRO engine. The new helper skb_prepare_for_gro() is
introduced for that goal.

On slow_gro, skbs are aggregated only with equal sk.
Additionally, skb truesize on GRO recycle and free is correctly
updated so that sk wmem is not changed by the GRO processing.

rfc-> v1:
 - fixed bad truesize on dev_gro_receive NAPI_FREE
 - use the existing state bit
Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 9efb4b5b
...@@ -2249,6 +2249,15 @@ static inline __must_check bool skb_set_owner_sk_safe(struct sk_buff *skb, struc ...@@ -2249,6 +2249,15 @@ static inline __must_check bool skb_set_owner_sk_safe(struct sk_buff *skb, struc
return false; return false;
} }
static inline void skb_prepare_for_gro(struct sk_buff *skb)
{
if (skb->destructor != sock_wfree) {
skb_orphan(skb);
return;
}
skb->slow_gro = 1;
}
void sk_reset_timer(struct sock *sk, struct timer_list *timer, void sk_reset_timer(struct sock *sk, struct timer_list *timer,
unsigned long expires); unsigned long expires);
......
...@@ -6034,6 +6034,7 @@ static void gro_list_prepare(const struct list_head *head, ...@@ -6034,6 +6034,7 @@ static void gro_list_prepare(const struct list_head *head,
struct tc_skb_ext *p_ext; struct tc_skb_ext *p_ext;
#endif #endif
diffs |= p->sk != skb->sk;
diffs |= skb_metadata_dst_cmp(p, skb); diffs |= skb_metadata_dst_cmp(p, skb);
diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb); diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb);
...@@ -6311,6 +6312,7 @@ static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) ...@@ -6311,6 +6312,7 @@ static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
skb_shinfo(skb)->gso_type = 0; skb_shinfo(skb)->gso_type = 0;
skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
if (unlikely(skb->slow_gro)) { if (unlikely(skb->slow_gro)) {
skb_orphan(skb);
skb_ext_reset(skb); skb_ext_reset(skb);
nf_reset_ct(skb); nf_reset_ct(skb);
skb->slow_gro = 0; skb->slow_gro = 0;
......
...@@ -958,6 +958,7 @@ void napi_skb_free_stolen_head(struct sk_buff *skb) ...@@ -958,6 +958,7 @@ void napi_skb_free_stolen_head(struct sk_buff *skb)
nf_reset_ct(skb); nf_reset_ct(skb);
skb_dst_drop(skb); skb_dst_drop(skb);
skb_ext_put(skb); skb_ext_put(skb);
skb_orphan(skb);
skb->slow_gro = 0; skb->slow_gro = 0;
} }
napi_skb_cache_put(skb); napi_skb_cache_put(skb);
...@@ -3892,6 +3893,9 @@ int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb) ...@@ -3892,6 +3893,9 @@ int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)
NAPI_GRO_CB(p)->last = skb; NAPI_GRO_CB(p)->last = skb;
NAPI_GRO_CB(p)->count++; NAPI_GRO_CB(p)->count++;
p->data_len += skb->len; p->data_len += skb->len;
/* sk owenrship - if any - completely transferred to the aggregated packet */
skb->destructor = NULL;
p->truesize += skb->truesize; p->truesize += skb->truesize;
p->len += skb->len; p->len += skb->len;
...@@ -4259,6 +4263,7 @@ int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) ...@@ -4259,6 +4263,7 @@ int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)
unsigned int headlen = skb_headlen(skb); unsigned int headlen = skb_headlen(skb);
unsigned int len = skb_gro_len(skb); unsigned int len = skb_gro_len(skb);
unsigned int delta_truesize; unsigned int delta_truesize;
unsigned int new_truesize;
struct sk_buff *lp; struct sk_buff *lp;
if (unlikely(p->len + len >= 65536 || NAPI_GRO_CB(skb)->flush)) if (unlikely(p->len + len >= 65536 || NAPI_GRO_CB(skb)->flush))
...@@ -4290,10 +4295,10 @@ int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) ...@@ -4290,10 +4295,10 @@ int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)
skb_frag_size_sub(frag, offset); skb_frag_size_sub(frag, offset);
/* all fragments truesize : remove (head size + sk_buff) */ /* all fragments truesize : remove (head size + sk_buff) */
delta_truesize = skb->truesize - new_truesize = SKB_TRUESIZE(skb_end_offset(skb));
SKB_TRUESIZE(skb_end_offset(skb)); delta_truesize = skb->truesize - new_truesize;
skb->truesize -= skb->data_len; skb->truesize = new_truesize;
skb->len -= skb->data_len; skb->len -= skb->data_len;
skb->data_len = 0; skb->data_len = 0;
...@@ -4322,12 +4327,16 @@ int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) ...@@ -4322,12 +4327,16 @@ int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)
memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags); memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
/* We dont need to clear skbinfo->nr_frags here */ /* We dont need to clear skbinfo->nr_frags here */
delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); new_truesize = SKB_TRUESIZE(sizeof(struct sk_buff));
delta_truesize = skb->truesize - new_truesize;
skb->truesize = new_truesize;
NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD; NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
goto done; goto done;
} }
merge: merge:
/* sk owenrship - if any - completely transferred to the aggregated packet */
skb->destructor = NULL;
delta_truesize = skb->truesize; delta_truesize = skb->truesize;
if (offset > headlen) { if (offset > headlen) {
unsigned int eat = offset - headlen; unsigned int eat = offset - headlen;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment