Commit c4777efa authored by Eric Dumazet's avatar Eric Dumazet Committed by Jakub Kicinski

net: add and use skb_unclone_keeptruesize() helper

While commit 097b9146 ("net: fix up truesize of cloned
skb in skb_prepare_for_shift()") fixed immediate issues found
when KFENCE was enabled/tested, there are still similar issues,
when tcp_trim_head() hits KFENCE while the master skb
is cloned.

This happens under heavy networking TX workloads,
when the TX completion might be delayed after incoming ACK.

This patch fixes the WARNING in sk_stream_kill_queues
when sk->sk_mem_queued/sk->sk_forward_alloc are not zero.

Fixes: d3fb45f3 ("mm, kfence: insert KFENCE hooks for SLAB")
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Acked-by: default avatarMarco Elver <elver@google.com>
Link: https://lore.kernel.org/r/20211102004555.1359210-1-eric.dumazet@gmail.comSigned-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 236f57fe
...@@ -1675,6 +1675,22 @@ static inline int skb_unclone(struct sk_buff *skb, gfp_t pri) ...@@ -1675,6 +1675,22 @@ static inline int skb_unclone(struct sk_buff *skb, gfp_t pri)
return 0; return 0;
} }
/* This variant of skb_unclone() makes sure skb->truesize is not changed */
static inline int skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri)
{
might_sleep_if(gfpflags_allow_blocking(pri));
if (skb_cloned(skb)) {
unsigned int save = skb->truesize;
int res;
res = pskb_expand_head(skb, 0, 0, pri);
skb->truesize = save;
return res;
}
return 0;
}
/** /**
* skb_header_cloned - is the header a clone * skb_header_cloned - is the header a clone
* @skb: buffer to check * @skb: buffer to check
......
...@@ -3449,19 +3449,7 @@ EXPORT_SYMBOL(skb_split); ...@@ -3449,19 +3449,7 @@ EXPORT_SYMBOL(skb_split);
*/ */
static int skb_prepare_for_shift(struct sk_buff *skb) static int skb_prepare_for_shift(struct sk_buff *skb)
{ {
int ret = 0; return skb_unclone_keeptruesize(skb, GFP_ATOMIC);
if (skb_cloned(skb)) {
/* Save and restore truesize: pskb_expand_head() may reallocate
* memory where ksize(kmalloc(S)) != ksize(kmalloc(S)), but we
* cannot change truesize at this point.
*/
unsigned int save_truesize = skb->truesize;
ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
skb->truesize = save_truesize;
}
return ret;
} }
/** /**
......
...@@ -1559,7 +1559,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, ...@@ -1559,7 +1559,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
return -ENOMEM; return -ENOMEM;
} }
if (skb_unclone(skb, gfp)) if (skb_unclone_keeptruesize(skb, gfp))
return -ENOMEM; return -ENOMEM;
/* Get a new skb... force flag on. */ /* Get a new skb... force flag on. */
...@@ -1667,7 +1667,7 @@ int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len) ...@@ -1667,7 +1667,7 @@ int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)
{ {
u32 delta_truesize; u32 delta_truesize;
if (skb_unclone(skb, GFP_ATOMIC)) if (skb_unclone_keeptruesize(skb, GFP_ATOMIC))
return -ENOMEM; return -ENOMEM;
delta_truesize = __pskb_trim_head(skb, len); delta_truesize = __pskb_trim_head(skb, len);
...@@ -3166,7 +3166,7 @@ int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs) ...@@ -3166,7 +3166,7 @@ int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
cur_mss, GFP_ATOMIC)) cur_mss, GFP_ATOMIC))
return -ENOMEM; /* We'll try again later. */ return -ENOMEM; /* We'll try again later. */
} else { } else {
if (skb_unclone(skb, GFP_ATOMIC)) if (skb_unclone_keeptruesize(skb, GFP_ATOMIC))
return -ENOMEM; return -ENOMEM;
diff = tcp_skb_pcount(skb); diff = tcp_skb_pcount(skb);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment