Commit ea1f5f37 authored by Pavel Tatashin's avatar Pavel Tatashin Committed by Linus Torvalds

mm: define memblock_virt_alloc_try_nid_raw

* A new variant of memblock_virt_alloc_* allocations:
memblock_virt_alloc_try_nid_raw()
    - Does not zero the allocated memory
    - Does not panic if request cannot be satisfied

* optimize early system hash allocations

Clients can call alloc_large_system_hash() with flag: HASH_ZERO to
specify that memory that was allocated for system hash needs to be
zeroed, otherwise the memory does not need to be zeroed, and client will
initialize it.

If memory does not need to be zero'd, call the new
memblock_virt_alloc_raw() interface, and thus improve the boot
performance.

* debug for raw alloctor

When CONFIG_DEBUG_VM is enabled, this patch sets all the memory that is
returned by memblock_virt_alloc_try_nid_raw() to ones to ensure that no
places excpect zeroed memory.

Link: http://lkml.kernel.org/r/20171013173214.27300-6-pasha.tatashin@oracle.comSigned-off-by: default avatarPavel Tatashin <pasha.tatashin@oracle.com>
Reviewed-by: default avatarSteven Sistare <steven.sistare@oracle.com>
Reviewed-by: default avatarDaniel Jordan <daniel.m.jordan@oracle.com>
Reviewed-by: default avatarBob Picco <bob.picco@oracle.com>
Tested-by: default avatarBob Picco <bob.picco@oracle.com>
Acked-by: default avatarMichal Hocko <mhocko@suse.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Ryabinin <aryabinin@virtuozzo.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christian Borntraeger <borntraeger@de.ibm.com>
Cc: David S. Miller <davem@davemloft.net>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Sam Ravnborg <sam@ravnborg.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent df8ee578
...@@ -161,6 +161,9 @@ extern void *__alloc_bootmem_low_node(pg_data_t *pgdat, ...@@ -161,6 +161,9 @@ extern void *__alloc_bootmem_low_node(pg_data_t *pgdat,
#define BOOTMEM_ALLOC_ANYWHERE (~(phys_addr_t)0) #define BOOTMEM_ALLOC_ANYWHERE (~(phys_addr_t)0)
/* FIXME: Move to memblock.h at a point where we remove nobootmem.c */ /* FIXME: Move to memblock.h at a point where we remove nobootmem.c */
void *memblock_virt_alloc_try_nid_raw(phys_addr_t size, phys_addr_t align,
phys_addr_t min_addr,
phys_addr_t max_addr, int nid);
void *memblock_virt_alloc_try_nid_nopanic(phys_addr_t size, void *memblock_virt_alloc_try_nid_nopanic(phys_addr_t size,
phys_addr_t align, phys_addr_t min_addr, phys_addr_t align, phys_addr_t min_addr,
phys_addr_t max_addr, int nid); phys_addr_t max_addr, int nid);
...@@ -177,6 +180,14 @@ static inline void * __init memblock_virt_alloc( ...@@ -177,6 +180,14 @@ static inline void * __init memblock_virt_alloc(
NUMA_NO_NODE); NUMA_NO_NODE);
} }
static inline void * __init memblock_virt_alloc_raw(
phys_addr_t size, phys_addr_t align)
{
return memblock_virt_alloc_try_nid_raw(size, align, BOOTMEM_LOW_LIMIT,
BOOTMEM_ALLOC_ACCESSIBLE,
NUMA_NO_NODE);
}
static inline void * __init memblock_virt_alloc_nopanic( static inline void * __init memblock_virt_alloc_nopanic(
phys_addr_t size, phys_addr_t align) phys_addr_t size, phys_addr_t align)
{ {
...@@ -258,6 +269,14 @@ static inline void * __init memblock_virt_alloc( ...@@ -258,6 +269,14 @@ static inline void * __init memblock_virt_alloc(
return __alloc_bootmem(size, align, BOOTMEM_LOW_LIMIT); return __alloc_bootmem(size, align, BOOTMEM_LOW_LIMIT);
} }
static inline void * __init memblock_virt_alloc_raw(
phys_addr_t size, phys_addr_t align)
{
if (!align)
align = SMP_CACHE_BYTES;
return __alloc_bootmem_nopanic(size, align, BOOTMEM_LOW_LIMIT);
}
static inline void * __init memblock_virt_alloc_nopanic( static inline void * __init memblock_virt_alloc_nopanic(
phys_addr_t size, phys_addr_t align) phys_addr_t size, phys_addr_t align)
{ {
...@@ -310,6 +329,14 @@ static inline void * __init memblock_virt_alloc_try_nid(phys_addr_t size, ...@@ -310,6 +329,14 @@ static inline void * __init memblock_virt_alloc_try_nid(phys_addr_t size,
min_addr); min_addr);
} }
static inline void * __init memblock_virt_alloc_try_nid_raw(
phys_addr_t size, phys_addr_t align,
phys_addr_t min_addr, phys_addr_t max_addr, int nid)
{
return ___alloc_bootmem_node_nopanic(NODE_DATA(nid), size, align,
min_addr, max_addr);
}
static inline void * __init memblock_virt_alloc_try_nid_nopanic( static inline void * __init memblock_virt_alloc_try_nid_nopanic(
phys_addr_t size, phys_addr_t align, phys_addr_t size, phys_addr_t align,
phys_addr_t min_addr, phys_addr_t max_addr, int nid) phys_addr_t min_addr, phys_addr_t max_addr, int nid)
......
...@@ -1327,7 +1327,6 @@ static void * __init memblock_virt_alloc_internal( ...@@ -1327,7 +1327,6 @@ static void * __init memblock_virt_alloc_internal(
return NULL; return NULL;
done: done:
ptr = phys_to_virt(alloc); ptr = phys_to_virt(alloc);
memset(ptr, 0, size);
/* /*
* The min_count is set to 0 so that bootmem allocated blocks * The min_count is set to 0 so that bootmem allocated blocks
...@@ -1340,6 +1339,45 @@ static void * __init memblock_virt_alloc_internal( ...@@ -1340,6 +1339,45 @@ static void * __init memblock_virt_alloc_internal(
return ptr; return ptr;
} }
/**
* memblock_virt_alloc_try_nid_raw - allocate boot memory block without zeroing
* memory and without panicking
* @size: size of memory block to be allocated in bytes
* @align: alignment of the region and block's size
* @min_addr: the lower bound of the memory region from where the allocation
* is preferred (phys address)
* @max_addr: the upper bound of the memory region from where the allocation
* is preferred (phys address), or %BOOTMEM_ALLOC_ACCESSIBLE to
* allocate only from memory limited by memblock.current_limit value
* @nid: nid of the free area to find, %NUMA_NO_NODE for any node
*
* Public function, provides additional debug information (including caller
* info), if enabled. Does not zero allocated memory, does not panic if request
* cannot be satisfied.
*
* RETURNS:
* Virtual address of allocated memory block on success, NULL on failure.
*/
void * __init memblock_virt_alloc_try_nid_raw(
phys_addr_t size, phys_addr_t align,
phys_addr_t min_addr, phys_addr_t max_addr,
int nid)
{
void *ptr;
memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=0x%llx max_addr=0x%llx %pF\n",
__func__, (u64)size, (u64)align, nid, (u64)min_addr,
(u64)max_addr, (void *)_RET_IP_);
ptr = memblock_virt_alloc_internal(size, align,
min_addr, max_addr, nid);
#ifdef CONFIG_DEBUG_VM
if (ptr && size > 0)
memset(ptr, 0xff, size);
#endif
return ptr;
}
/** /**
* memblock_virt_alloc_try_nid_nopanic - allocate boot memory block * memblock_virt_alloc_try_nid_nopanic - allocate boot memory block
* @size: size of memory block to be allocated in bytes * @size: size of memory block to be allocated in bytes
...@@ -1351,8 +1389,8 @@ static void * __init memblock_virt_alloc_internal( ...@@ -1351,8 +1389,8 @@ static void * __init memblock_virt_alloc_internal(
* allocate only from memory limited by memblock.current_limit value * allocate only from memory limited by memblock.current_limit value
* @nid: nid of the free area to find, %NUMA_NO_NODE for any node * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
* *
* Public version of _memblock_virt_alloc_try_nid_nopanic() which provides * Public function, provides additional debug information (including caller
* additional debug information (including caller info), if enabled. * info), if enabled. This function zeroes the allocated memory.
* *
* RETURNS: * RETURNS:
* Virtual address of allocated memory block on success, NULL on failure. * Virtual address of allocated memory block on success, NULL on failure.
...@@ -1362,11 +1400,17 @@ void * __init memblock_virt_alloc_try_nid_nopanic( ...@@ -1362,11 +1400,17 @@ void * __init memblock_virt_alloc_try_nid_nopanic(
phys_addr_t min_addr, phys_addr_t max_addr, phys_addr_t min_addr, phys_addr_t max_addr,
int nid) int nid)
{ {
void *ptr;
memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=0x%llx max_addr=0x%llx %pF\n", memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=0x%llx max_addr=0x%llx %pF\n",
__func__, (u64)size, (u64)align, nid, (u64)min_addr, __func__, (u64)size, (u64)align, nid, (u64)min_addr,
(u64)max_addr, (void *)_RET_IP_); (u64)max_addr, (void *)_RET_IP_);
return memblock_virt_alloc_internal(size, align, min_addr,
max_addr, nid); ptr = memblock_virt_alloc_internal(size, align,
min_addr, max_addr, nid);
if (ptr)
memset(ptr, 0, size);
return ptr;
} }
/** /**
...@@ -1380,7 +1424,7 @@ void * __init memblock_virt_alloc_try_nid_nopanic( ...@@ -1380,7 +1424,7 @@ void * __init memblock_virt_alloc_try_nid_nopanic(
* allocate only from memory limited by memblock.current_limit value * allocate only from memory limited by memblock.current_limit value
* @nid: nid of the free area to find, %NUMA_NO_NODE for any node * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
* *
* Public panicking version of _memblock_virt_alloc_try_nid_nopanic() * Public panicking version of memblock_virt_alloc_try_nid_nopanic()
* which provides debug information (including caller info), if enabled, * which provides debug information (including caller info), if enabled,
* and panics if the request can not be satisfied. * and panics if the request can not be satisfied.
* *
...@@ -1399,8 +1443,10 @@ void * __init memblock_virt_alloc_try_nid( ...@@ -1399,8 +1443,10 @@ void * __init memblock_virt_alloc_try_nid(
(u64)max_addr, (void *)_RET_IP_); (u64)max_addr, (void *)_RET_IP_);
ptr = memblock_virt_alloc_internal(size, align, ptr = memblock_virt_alloc_internal(size, align,
min_addr, max_addr, nid); min_addr, max_addr, nid);
if (ptr) if (ptr) {
memset(ptr, 0, size);
return ptr; return ptr;
}
panic("%s: Failed to allocate %llu bytes align=0x%llx nid=%d from=0x%llx max_addr=0x%llx\n", panic("%s: Failed to allocate %llu bytes align=0x%llx nid=%d from=0x%llx max_addr=0x%llx\n",
__func__, (u64)size, (u64)align, nid, (u64)min_addr, __func__, (u64)size, (u64)align, nid, (u64)min_addr,
......
...@@ -7313,18 +7313,17 @@ void *__init alloc_large_system_hash(const char *tablename, ...@@ -7313,18 +7313,17 @@ void *__init alloc_large_system_hash(const char *tablename,
log2qty = ilog2(numentries); log2qty = ilog2(numentries);
/*
* memblock allocator returns zeroed memory already, so HASH_ZERO is
* currently not used when HASH_EARLY is specified.
*/
gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC;
do { do {
size = bucketsize << log2qty; size = bucketsize << log2qty;
if (flags & HASH_EARLY) if (flags & HASH_EARLY) {
table = memblock_virt_alloc_nopanic(size, 0); if (flags & HASH_ZERO)
else if (hashdist) table = memblock_virt_alloc_nopanic(size, 0);
else
table = memblock_virt_alloc_raw(size, 0);
} else if (hashdist) {
table = __vmalloc(size, gfp_flags, PAGE_KERNEL); table = __vmalloc(size, gfp_flags, PAGE_KERNEL);
else { } else {
/* /*
* If bucketsize is not a power-of-two, we may free * If bucketsize is not a power-of-two, we may free
* some pages at the end of hash table which * some pages at the end of hash table which
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment