Commit be5dba25 authored by Jesper Dangaard Brouer's avatar Jesper Dangaard Brouer Committed by Linus Torvalds

net: page_pool: use alloc_pages_bulk in refill code path

There are cases where the page_pool need to refill with pages from the
page allocator.  Some workloads cause the page_pool to release pages
instead of recycling these pages.

For these workload it can improve performance to bulk alloc pages from the
page-allocator to refill the alloc cache.

For XDP-redirect workload with 100G mlx5 driver (that use page_pool)
redirecting xdp_frame packets into a veth, that does XDP_PASS to create an
SKB from the xdp_frame, which then cannot return the page to the
page_pool.

Performance results under GitHub xdp-project[1]:
 [1] https://github.com/xdp-project/xdp-project/blob/master/areas/mem/page_pool06_alloc_pages_bulk.org

Mel: The patch "net: page_pool: convert to use alloc_pages_bulk_array
variant" was squashed with this patch. From the test page, the array
variant was superior with one of the test results as follows.

	Kernel		XDP stats       CPU     pps           Delta
	Baseline	XDP-RX CPU      total   3,771,046       n/a
	List		XDP-RX CPU      total   3,940,242    +4.49%
	Array		XDP-RX CPU      total   4,249,224   +12.68%

Link: https://lkml.kernel.org/r/20210325114228.27719-10-mgorman@techsingularity.netSigned-off-by: default avatarJesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: default avatarMel Gorman <mgorman@techsingularity.net>
Reviewed-by: default avatarAlexander Lobakin <alobakin@pm.me>
Cc: Alexander Duyck <alexander.duyck@gmail.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Chuck Lever <chuck.lever@oracle.com>
Cc: David Miller <davem@davemloft.net>
Cc: Ilias Apalodimas <ilias.apalodimas@linaro.org>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent dfa59717
...@@ -65,7 +65,7 @@ ...@@ -65,7 +65,7 @@
#define PP_ALLOC_CACHE_REFILL 64 #define PP_ALLOC_CACHE_REFILL 64
struct pp_alloc_cache { struct pp_alloc_cache {
u32 count; u32 count;
void *cache[PP_ALLOC_CACHE_SIZE]; struct page *cache[PP_ALLOC_CACHE_SIZE];
}; };
struct page_pool_params { struct page_pool_params {
......
...@@ -203,46 +203,76 @@ static bool page_pool_dma_map(struct page_pool *pool, struct page *page) ...@@ -203,46 +203,76 @@ static bool page_pool_dma_map(struct page_pool *pool, struct page *page)
return true; return true;
} }
static struct page *__page_pool_alloc_page_order(struct page_pool *pool,
gfp_t gfp)
{
struct page *page;
gfp |= __GFP_COMP;
page = alloc_pages_node(pool->p.nid, gfp, pool->p.order);
if (unlikely(!page))
return NULL;
if ((pool->p.flags & PP_FLAG_DMA_MAP) &&
unlikely(!page_pool_dma_map(pool, page))) {
put_page(page);
return NULL;
}
/* Track how many pages are held 'in-flight' */
pool->pages_state_hold_cnt++;
trace_page_pool_state_hold(pool, page, pool->pages_state_hold_cnt);
return page;
}
/* slow path */ /* slow path */
noinline noinline
static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool, static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,
gfp_t _gfp) gfp_t gfp)
{ {
const int bulk = PP_ALLOC_CACHE_REFILL;
unsigned int pp_flags = pool->p.flags; unsigned int pp_flags = pool->p.flags;
unsigned int pp_order = pool->p.order;
struct page *page; struct page *page;
gfp_t gfp = _gfp; int i, nr_pages;
/* We could always set __GFP_COMP, and avoid this branch, as /* Don't support bulk alloc for high-order pages */
* prep_new_page() can handle order-0 with __GFP_COMP. if (unlikely(pp_order))
*/ return __page_pool_alloc_page_order(pool, gfp);
if (pool->p.order)
gfp |= __GFP_COMP;
/* FUTURE development: /* Unnecessary as alloc cache is empty, but guarantees zero count */
* if (unlikely(pool->alloc.count > 0))
* Current slow-path essentially falls back to single page return pool->alloc.cache[--pool->alloc.count];
* allocations, which doesn't improve performance. This code
* need bulk allocation support from the page allocator code.
*/
/* Cache was empty, do real allocation */ /* Mark empty alloc.cache slots "empty" for alloc_pages_bulk_array */
#ifdef CONFIG_NUMA memset(&pool->alloc.cache, 0, sizeof(void *) * bulk);
page = alloc_pages_node(pool->p.nid, gfp, pool->p.order);
#else nr_pages = alloc_pages_bulk_array(gfp, bulk, pool->alloc.cache);
page = alloc_pages(gfp, pool->p.order); if (unlikely(!nr_pages))
#endif
if (!page)
return NULL; return NULL;
/* Pages have been filled into alloc.cache array, but count is zero and
* page element have not been (possibly) DMA mapped.
*/
for (i = 0; i < nr_pages; i++) {
page = pool->alloc.cache[i];
if ((pp_flags & PP_FLAG_DMA_MAP) && if ((pp_flags & PP_FLAG_DMA_MAP) &&
unlikely(!page_pool_dma_map(pool, page))) { unlikely(!page_pool_dma_map(pool, page))) {
put_page(page); put_page(page);
return NULL; continue;
} }
pool->alloc.cache[pool->alloc.count++] = page;
/* Track how many pages are held 'in-flight' */ /* Track how many pages are held 'in-flight' */
pool->pages_state_hold_cnt++; pool->pages_state_hold_cnt++;
trace_page_pool_state_hold(pool, page, pool->pages_state_hold_cnt); trace_page_pool_state_hold(pool, page,
pool->pages_state_hold_cnt);
}
/* Return last page */
if (likely(pool->alloc.count > 0))
page = pool->alloc.cache[--pool->alloc.count];
else
page = NULL;
/* When page just alloc'ed is should/must have refcnt 1. */ /* When page just alloc'ed is should/must have refcnt 1. */
return page; return page;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment