Commit c1d50836 authored by Christoph Lameter's avatar Christoph Lameter Committed by Pekka Enberg

slub: Move gfpflag masking out of the hotpath

Move the gfpflags masking into the hooks for checkers and into the slowpaths.
gfpflag masking requires access to a global variable and thus adds an
additional cacheline reference to the hotpaths.

If no hooks are active then the gfpflag masking will result in
code that the compiler can toss out.
Acked-by: default avatarDavid Rientjes <rientjes@google.com>
Signed-off-by: default avatarChristoph Lameter <cl@linux-foundation.org>
Signed-off-by: default avatarPekka Enberg <penberg@kernel.org>
parent c016b0bd
...@@ -796,6 +796,7 @@ static void trace(struct kmem_cache *s, struct page *page, void *object, ...@@ -796,6 +796,7 @@ static void trace(struct kmem_cache *s, struct page *page, void *object,
*/ */
static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags) static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
{ {
flags &= gfp_allowed_mask;
lockdep_trace_alloc(flags); lockdep_trace_alloc(flags);
might_sleep_if(flags & __GFP_WAIT); might_sleep_if(flags & __GFP_WAIT);
...@@ -804,6 +805,7 @@ static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags) ...@@ -804,6 +805,7 @@ static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, void *object) static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, void *object)
{ {
flags &= gfp_allowed_mask;
kmemcheck_slab_alloc(s, flags, object, s->objsize); kmemcheck_slab_alloc(s, flags, object, s->objsize);
kmemleak_alloc_recursive(object, s->objsize, 1, s->flags, flags); kmemleak_alloc_recursive(object, s->objsize, 1, s->flags, flags);
} }
...@@ -1677,6 +1679,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, ...@@ -1677,6 +1679,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
goto load_freelist; goto load_freelist;
} }
gfpflags &= gfp_allowed_mask;
if (gfpflags & __GFP_WAIT) if (gfpflags & __GFP_WAIT)
local_irq_enable(); local_irq_enable();
...@@ -1725,8 +1728,6 @@ static __always_inline void *slab_alloc(struct kmem_cache *s, ...@@ -1725,8 +1728,6 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
struct kmem_cache_cpu *c; struct kmem_cache_cpu *c;
unsigned long flags; unsigned long flags;
gfpflags &= gfp_allowed_mask;
if (slab_pre_alloc_hook(s, gfpflags)) if (slab_pre_alloc_hook(s, gfpflags))
return NULL; return NULL;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment