Commit 9353ffa6 authored by Andrey Konovalov's avatar Andrey Konovalov Committed by Linus Torvalds

kasan, page_alloc: allow skipping memory init for HW_TAGS

Add a new GFP flag __GFP_SKIP_ZERO that allows to skip memory
initialization.  The flag is only effective with HW_TAGS KASAN.

This flag will be used by vmalloc code for page_alloc allocations backing
vmalloc() mappings in a following patch.  The reason to skip memory
initialization for these pages in page_alloc is because vmalloc code will
be initializing them instead.

With the current implementation, when __GFP_SKIP_ZERO is provided,
__GFP_ZEROTAGS is ignored.  This doesn't matter, as these two flags are
never provided at the same time.  However, if this is changed in the
future, this particular implementation detail can be changed as well.

Link: https://lkml.kernel.org/r/0d53efeff345de7d708e0baa0d8829167772521e.1643047180.git.andreyknvl@google.comSigned-off-by: default avatarAndrey Konovalov <andreyknvl@google.com>
Acked-by: default avatarMarco Elver <elver@google.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Evgenii Stepanov <eugenis@google.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Peter Collingbourne <pcc@google.com>
Cc: Vincenzo Frascino <vincenzo.frascino@arm.com>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 53ae233c
...@@ -55,14 +55,16 @@ struct vm_area_struct; ...@@ -55,14 +55,16 @@ struct vm_area_struct;
#define ___GFP_ACCOUNT 0x400000u #define ___GFP_ACCOUNT 0x400000u
#define ___GFP_ZEROTAGS 0x800000u #define ___GFP_ZEROTAGS 0x800000u
#ifdef CONFIG_KASAN_HW_TAGS #ifdef CONFIG_KASAN_HW_TAGS
#define ___GFP_SKIP_KASAN_UNPOISON 0x1000000u #define ___GFP_SKIP_ZERO 0x1000000u
#define ___GFP_SKIP_KASAN_POISON 0x2000000u #define ___GFP_SKIP_KASAN_UNPOISON 0x2000000u
#define ___GFP_SKIP_KASAN_POISON 0x4000000u
#else #else
#define ___GFP_SKIP_ZERO 0
#define ___GFP_SKIP_KASAN_UNPOISON 0 #define ___GFP_SKIP_KASAN_UNPOISON 0
#define ___GFP_SKIP_KASAN_POISON 0 #define ___GFP_SKIP_KASAN_POISON 0
#endif #endif
#ifdef CONFIG_LOCKDEP #ifdef CONFIG_LOCKDEP
#define ___GFP_NOLOCKDEP 0x4000000u #define ___GFP_NOLOCKDEP 0x8000000u
#else #else
#define ___GFP_NOLOCKDEP 0 #define ___GFP_NOLOCKDEP 0
#endif #endif
...@@ -239,9 +241,10 @@ struct vm_area_struct; ...@@ -239,9 +241,10 @@ struct vm_area_struct;
* %__GFP_ZERO returns a zeroed page on success. * %__GFP_ZERO returns a zeroed page on success.
* *
* %__GFP_ZEROTAGS zeroes memory tags at allocation time if the memory itself * %__GFP_ZEROTAGS zeroes memory tags at allocation time if the memory itself
* is being zeroed (either via __GFP_ZERO or via init_on_alloc). This flag is * is being zeroed (either via __GFP_ZERO or via init_on_alloc, provided that
* intended for optimization: setting memory tags at the same time as zeroing * __GFP_SKIP_ZERO is not set). This flag is intended for optimization: setting
* memory has minimal additional performace impact. * memory tags at the same time as zeroing memory has minimal additional
* performace impact.
* *
* %__GFP_SKIP_KASAN_UNPOISON makes KASAN skip unpoisoning on page allocation. * %__GFP_SKIP_KASAN_UNPOISON makes KASAN skip unpoisoning on page allocation.
* Only effective in HW_TAGS mode. * Only effective in HW_TAGS mode.
...@@ -253,6 +256,7 @@ struct vm_area_struct; ...@@ -253,6 +256,7 @@ struct vm_area_struct;
#define __GFP_COMP ((__force gfp_t)___GFP_COMP) #define __GFP_COMP ((__force gfp_t)___GFP_COMP)
#define __GFP_ZERO ((__force gfp_t)___GFP_ZERO) #define __GFP_ZERO ((__force gfp_t)___GFP_ZERO)
#define __GFP_ZEROTAGS ((__force gfp_t)___GFP_ZEROTAGS) #define __GFP_ZEROTAGS ((__force gfp_t)___GFP_ZEROTAGS)
#define __GFP_SKIP_ZERO ((__force gfp_t)___GFP_SKIP_ZERO)
#define __GFP_SKIP_KASAN_UNPOISON ((__force gfp_t)___GFP_SKIP_KASAN_UNPOISON) #define __GFP_SKIP_KASAN_UNPOISON ((__force gfp_t)___GFP_SKIP_KASAN_UNPOISON)
#define __GFP_SKIP_KASAN_POISON ((__force gfp_t)___GFP_SKIP_KASAN_POISON) #define __GFP_SKIP_KASAN_POISON ((__force gfp_t)___GFP_SKIP_KASAN_POISON)
...@@ -261,7 +265,7 @@ struct vm_area_struct; ...@@ -261,7 +265,7 @@ struct vm_area_struct;
/* Room for N __GFP_FOO bits */ /* Room for N __GFP_FOO bits */
#define __GFP_BITS_SHIFT (24 + \ #define __GFP_BITS_SHIFT (24 + \
2 * IS_ENABLED(CONFIG_KASAN_HW_TAGS) + \ 3 * IS_ENABLED(CONFIG_KASAN_HW_TAGS) + \
IS_ENABLED(CONFIG_LOCKDEP)) IS_ENABLED(CONFIG_LOCKDEP))
#define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
......
...@@ -53,6 +53,7 @@ ...@@ -53,6 +53,7 @@
#ifdef CONFIG_KASAN_HW_TAGS #ifdef CONFIG_KASAN_HW_TAGS
#define __def_gfpflag_names_kasan , \ #define __def_gfpflag_names_kasan , \
{(unsigned long)__GFP_SKIP_ZERO, "__GFP_SKIP_ZERO"}, \
{(unsigned long)__GFP_SKIP_KASAN_POISON, "__GFP_SKIP_KASAN_POISON"}, \ {(unsigned long)__GFP_SKIP_KASAN_POISON, "__GFP_SKIP_KASAN_POISON"}, \
{(unsigned long)__GFP_SKIP_KASAN_UNPOISON, "__GFP_SKIP_KASAN_UNPOISON"} {(unsigned long)__GFP_SKIP_KASAN_UNPOISON, "__GFP_SKIP_KASAN_UNPOISON"}
#else #else
......
...@@ -2363,10 +2363,21 @@ static inline bool should_skip_kasan_unpoison(gfp_t flags, bool init_tags) ...@@ -2363,10 +2363,21 @@ static inline bool should_skip_kasan_unpoison(gfp_t flags, bool init_tags)
return init_tags || (flags & __GFP_SKIP_KASAN_UNPOISON); return init_tags || (flags & __GFP_SKIP_KASAN_UNPOISON);
} }
static inline bool should_skip_init(gfp_t flags)
{
/* Don't skip, if hardware tag-based KASAN is not enabled. */
if (!kasan_hw_tags_enabled())
return false;
/* For hardware tag-based KASAN, skip if requested. */
return (flags & __GFP_SKIP_ZERO);
}
inline void post_alloc_hook(struct page *page, unsigned int order, inline void post_alloc_hook(struct page *page, unsigned int order,
gfp_t gfp_flags) gfp_t gfp_flags)
{ {
bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags); bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) &&
!should_skip_init(gfp_flags);
bool init_tags = init && (gfp_flags & __GFP_ZEROTAGS); bool init_tags = init && (gfp_flags & __GFP_ZEROTAGS);
set_page_private(page, 0); set_page_private(page, 0);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment