Commit 0415b00d authored by Tejun Heo's avatar Tejun Heo

percpu: Always align percpu output section to PAGE_SIZE

Percpu allocator honors alignment request upto PAGE_SIZE and both the
percpu addresses in the percpu address space and the translated kernel
addresses should be aligned accordingly.  The calculation of the
former depends on the alignment of percpu output section in the kernel
image.

The linker script macros PERCPU_VADDR() and PERCPU() are used to
define this output section and the latter takes @align parameter.
Several architectures are using @align smaller than PAGE_SIZE breaking
percpu memory alignment.

This patch removes @align parameter from PERCPU(), renames it to
PERCPU_SECTION() and makes it always align to PAGE_SIZE.  While at it,
add PCPU_SETUP_BUG_ON() checks such that alignment problems are
reliably detected and remove percpu alignment comment recently added
in workqueue.c as the condition would trigger BUG way before reaching
there.

For um, this patch raises the alignment of percpu area.  As the area
is in .init, there shouldn't be any noticeable difference.

This problem was discovered by David Howells while debugging boot
failure on mn10300.
Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Acked-by: default avatarMike Frysinger <vapier@gentoo.org>
Cc: uclinux-dist-devel@blackfin.uclinux.org
Cc: David Howells <dhowells@redhat.com>
Cc: Jeff Dike <jdike@addtoit.com>
Cc: user-mode-linux-devel@lists.sourceforge.net
parent 6c510389
...@@ -39,7 +39,7 @@ SECTIONS ...@@ -39,7 +39,7 @@ SECTIONS
__init_begin = ALIGN(PAGE_SIZE); __init_begin = ALIGN(PAGE_SIZE);
INIT_TEXT_SECTION(PAGE_SIZE) INIT_TEXT_SECTION(PAGE_SIZE)
INIT_DATA_SECTION(16) INIT_DATA_SECTION(16)
PERCPU(L1_CACHE_BYTES, PAGE_SIZE) PERCPU_SECTION(L1_CACHE_BYTES)
/* Align to THREAD_SIZE rather than PAGE_SIZE here so any padding page /* Align to THREAD_SIZE rather than PAGE_SIZE here so any padding page
needed for the THREAD_SIZE aligned init_task gets freed after init */ needed for the THREAD_SIZE aligned init_task gets freed after init */
. = ALIGN(THREAD_SIZE); . = ALIGN(THREAD_SIZE);
......
...@@ -82,7 +82,7 @@ SECTIONS ...@@ -82,7 +82,7 @@ SECTIONS
#endif #endif
} }
PERCPU(32, PAGE_SIZE) PERCPU_SECTION(32)
#ifndef CONFIG_XIP_KERNEL #ifndef CONFIG_XIP_KERNEL
. = ALIGN(PAGE_SIZE); . = ALIGN(PAGE_SIZE);
......
...@@ -136,7 +136,7 @@ SECTIONS ...@@ -136,7 +136,7 @@ SECTIONS
. = ALIGN(16); . = ALIGN(16);
INIT_DATA_SECTION(16) INIT_DATA_SECTION(16)
PERCPU(32, PAGE_SIZE) PERCPU_SECTION(32)
.exit.data : .exit.data :
{ {
......
...@@ -102,7 +102,7 @@ SECTIONS ...@@ -102,7 +102,7 @@ SECTIONS
#endif #endif
__vmlinux_end = .; /* Last address of the physical file. */ __vmlinux_end = .; /* Last address of the physical file. */
#ifdef CONFIG_ETRAX_ARCH_V32 #ifdef CONFIG_ETRAX_ARCH_V32
PERCPU(32, PAGE_SIZE) PERCPU_SECTION(32)
.init.ramfs : { .init.ramfs : {
INIT_RAM_FS INIT_RAM_FS
......
...@@ -37,7 +37,7 @@ SECTIONS ...@@ -37,7 +37,7 @@ SECTIONS
_einittext = .; _einittext = .;
INIT_DATA_SECTION(8) INIT_DATA_SECTION(8)
PERCPU(L1_CACHE_BYTES, 4096) PERCPU_SECTION(L1_CACHE_BYTES)
. = ALIGN(PAGE_SIZE); . = ALIGN(PAGE_SIZE);
__init_end = .; __init_end = .;
......
...@@ -53,7 +53,7 @@ SECTIONS ...@@ -53,7 +53,7 @@ SECTIONS
__init_begin = .; __init_begin = .;
INIT_TEXT_SECTION(PAGE_SIZE) INIT_TEXT_SECTION(PAGE_SIZE)
INIT_DATA_SECTION(16) INIT_DATA_SECTION(16)
PERCPU(32, PAGE_SIZE) PERCPU_SECTION(32)
. = ALIGN(PAGE_SIZE); . = ALIGN(PAGE_SIZE);
__init_end = .; __init_end = .;
/* freed after init ends here */ /* freed after init ends here */
......
...@@ -115,7 +115,7 @@ SECTIONS ...@@ -115,7 +115,7 @@ SECTIONS
EXIT_DATA EXIT_DATA
} }
PERCPU(1 << CONFIG_MIPS_L1_CACHE_SHIFT, PAGE_SIZE) PERCPU_SECTION(1 << CONFIG_MIPS_L1_CACHE_SHIFT)
. = ALIGN(PAGE_SIZE); . = ALIGN(PAGE_SIZE);
__init_end = .; __init_end = .;
/* freed after init ends here */ /* freed after init ends here */
......
...@@ -70,7 +70,7 @@ SECTIONS ...@@ -70,7 +70,7 @@ SECTIONS
.exit.text : { EXIT_TEXT; } .exit.text : { EXIT_TEXT; }
.exit.data : { EXIT_DATA; } .exit.data : { EXIT_DATA; }
PERCPU(32, PAGE_SIZE) PERCPU_SECTION(32)
. = ALIGN(PAGE_SIZE); . = ALIGN(PAGE_SIZE);
__init_end = .; __init_end = .;
/* freed after init ends here */ /* freed after init ends here */
......
...@@ -145,7 +145,7 @@ SECTIONS ...@@ -145,7 +145,7 @@ SECTIONS
EXIT_DATA EXIT_DATA
} }
PERCPU(L1_CACHE_BYTES, PAGE_SIZE) PERCPU_SECTION(L1_CACHE_BYTES)
. = ALIGN(PAGE_SIZE); . = ALIGN(PAGE_SIZE);
__init_end = .; __init_end = .;
/* freed after init ends here */ /* freed after init ends here */
......
...@@ -160,7 +160,7 @@ SECTIONS ...@@ -160,7 +160,7 @@ SECTIONS
INIT_RAM_FS INIT_RAM_FS
} }
PERCPU(L1_CACHE_BYTES, PAGE_SIZE) PERCPU_SECTION(L1_CACHE_BYTES)
. = ALIGN(8); . = ALIGN(8);
.machine.desc : AT(ADDR(.machine.desc) - LOAD_OFFSET) { .machine.desc : AT(ADDR(.machine.desc) - LOAD_OFFSET) {
......
...@@ -77,7 +77,7 @@ SECTIONS ...@@ -77,7 +77,7 @@ SECTIONS
. = ALIGN(PAGE_SIZE); . = ALIGN(PAGE_SIZE);
INIT_DATA_SECTION(0x100) INIT_DATA_SECTION(0x100)
PERCPU(0x100, PAGE_SIZE) PERCPU_SECTION(0x100)
. = ALIGN(PAGE_SIZE); . = ALIGN(PAGE_SIZE);
__init_end = .; /* freed after init ends here */ __init_end = .; /* freed after init ends here */
......
...@@ -66,7 +66,7 @@ SECTIONS ...@@ -66,7 +66,7 @@ SECTIONS
__machvec_end = .; __machvec_end = .;
} }
PERCPU(L1_CACHE_BYTES, PAGE_SIZE) PERCPU_SECTION(L1_CACHE_BYTES)
/* /*
* .exit.text is discarded at runtime, not link time, to deal with * .exit.text is discarded at runtime, not link time, to deal with
......
...@@ -108,7 +108,7 @@ SECTIONS ...@@ -108,7 +108,7 @@ SECTIONS
__sun4v_2insn_patch_end = .; __sun4v_2insn_patch_end = .;
} }
PERCPU(SMP_CACHE_BYTES, PAGE_SIZE) PERCPU_SECTION(SMP_CACHE_BYTES)
. = ALIGN(PAGE_SIZE); . = ALIGN(PAGE_SIZE);
__init_end = .; __init_end = .;
......
...@@ -60,7 +60,7 @@ SECTIONS ...@@ -60,7 +60,7 @@ SECTIONS
. = ALIGN(PAGE_SIZE); . = ALIGN(PAGE_SIZE);
VMLINUX_SYMBOL(_sinitdata) = .; VMLINUX_SYMBOL(_sinitdata) = .;
INIT_DATA_SECTION(16) :data =0 INIT_DATA_SECTION(16) :data =0
PERCPU(L2_CACHE_BYTES, PAGE_SIZE) PERCPU_SECTION(L2_CACHE_BYTES)
. = ALIGN(PAGE_SIZE); . = ALIGN(PAGE_SIZE);
VMLINUX_SYMBOL(_einitdata) = .; VMLINUX_SYMBOL(_einitdata) = .;
......
...@@ -42,7 +42,7 @@ ...@@ -42,7 +42,7 @@
INIT_SETUP(0) INIT_SETUP(0)
} }
PERCPU(32, 32) PERCPU_SECTION(32)
.initcall.init : { .initcall.init : {
INIT_CALLS INIT_CALLS
......
...@@ -319,7 +319,7 @@ SECTIONS ...@@ -319,7 +319,7 @@ SECTIONS
} }
#if !defined(CONFIG_X86_64) || !defined(CONFIG_SMP) #if !defined(CONFIG_X86_64) || !defined(CONFIG_SMP)
PERCPU(INTERNODE_CACHE_BYTES, PAGE_SIZE) PERCPU_SECTION(INTERNODE_CACHE_BYTES)
#endif #endif
. = ALIGN(PAGE_SIZE); . = ALIGN(PAGE_SIZE);
......
...@@ -155,7 +155,7 @@ SECTIONS ...@@ -155,7 +155,7 @@ SECTIONS
INIT_RAM_FS INIT_RAM_FS
} }
PERCPU(XCHAL_ICACHE_LINESIZE, PAGE_SIZE) PERCPU_SECTION(XCHAL_ICACHE_LINESIZE)
/* We need this dummy segment here */ /* We need this dummy segment here */
......
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
* HEAD_TEXT_SECTION * HEAD_TEXT_SECTION
* INIT_TEXT_SECTION(PAGE_SIZE) * INIT_TEXT_SECTION(PAGE_SIZE)
* INIT_DATA_SECTION(...) * INIT_DATA_SECTION(...)
* PERCPU(CACHELINE_SIZE, PAGE_SIZE) * PERCPU_SECTION(CACHELINE_SIZE)
* __init_end = .; * __init_end = .;
* *
* _stext = .; * _stext = .;
...@@ -709,7 +709,7 @@ ...@@ -709,7 +709,7 @@
* *
* Note that this macros defines __per_cpu_load as an absolute symbol. * Note that this macros defines __per_cpu_load as an absolute symbol.
* If there is no need to put the percpu section at a predetermined * If there is no need to put the percpu section at a predetermined
* address, use PERCPU(). * address, use PERCPU_SECTION.
*/ */
#define PERCPU_VADDR(cacheline, vaddr, phdr) \ #define PERCPU_VADDR(cacheline, vaddr, phdr) \
VMLINUX_SYMBOL(__per_cpu_load) = .; \ VMLINUX_SYMBOL(__per_cpu_load) = .; \
...@@ -729,20 +729,19 @@ ...@@ -729,20 +729,19 @@
. = VMLINUX_SYMBOL(__per_cpu_load) + SIZEOF(.data..percpu); . = VMLINUX_SYMBOL(__per_cpu_load) + SIZEOF(.data..percpu);
/** /**
* PERCPU - define output section for percpu area, simple version * PERCPU_SECTION - define output section for percpu area, simple version
* @cacheline: cacheline size * @cacheline: cacheline size
* @align: required alignment
* *
* Align to @align and outputs output section for percpu area. This macro * Align to PAGE_SIZE and outputs output section for percpu area. This
* doesn't manipulate @vaddr or @phdr and __per_cpu_load and * macro doesn't manipulate @vaddr or @phdr and __per_cpu_load and
* __per_cpu_start will be identical. * __per_cpu_start will be identical.
* *
* This macro is equivalent to ALIGN(@align); PERCPU_VADDR(@cacheline,,) * This macro is equivalent to ALIGN(PAGE_SIZE); PERCPU_VADDR(@cacheline,,)
* except that __per_cpu_load is defined as a relative symbol against * except that __per_cpu_load is defined as a relative symbol against
* .data..percpu which is required for relocatable x86_32 configuration. * .data..percpu which is required for relocatable x86_32 configuration.
*/ */
#define PERCPU(cacheline, align) \ #define PERCPU_SECTION(cacheline) \
. = ALIGN(align); \ . = ALIGN(PAGE_SIZE); \
.data..percpu : AT(ADDR(.data..percpu) - LOAD_OFFSET) { \ .data..percpu : AT(ADDR(.data..percpu) - LOAD_OFFSET) { \
VMLINUX_SYMBOL(__per_cpu_load) = .; \ VMLINUX_SYMBOL(__per_cpu_load) = .; \
VMLINUX_SYMBOL(__per_cpu_start) = .; \ VMLINUX_SYMBOL(__per_cpu_start) = .; \
......
...@@ -2860,9 +2860,7 @@ static int alloc_cwqs(struct workqueue_struct *wq) ...@@ -2860,9 +2860,7 @@ static int alloc_cwqs(struct workqueue_struct *wq)
} }
} }
/* just in case, make sure it's actually aligned /* just in case, make sure it's actually aligned */
* - this is affected by PERCPU() alignment in vmlinux.lds.S
*/
BUG_ON(!IS_ALIGNED(wq->cpu_wq.v, align)); BUG_ON(!IS_ALIGNED(wq->cpu_wq.v, align));
return wq->cpu_wq.v ? 0 : -ENOMEM; return wq->cpu_wq.v ? 0 : -ENOMEM;
} }
......
...@@ -1216,8 +1216,10 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, ...@@ -1216,8 +1216,10 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
PCPU_SETUP_BUG_ON(ai->nr_groups <= 0); PCPU_SETUP_BUG_ON(ai->nr_groups <= 0);
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
PCPU_SETUP_BUG_ON(!ai->static_size); PCPU_SETUP_BUG_ON(!ai->static_size);
PCPU_SETUP_BUG_ON((unsigned long)__per_cpu_start & ~PAGE_MASK);
#endif #endif
PCPU_SETUP_BUG_ON(!base_addr); PCPU_SETUP_BUG_ON(!base_addr);
PCPU_SETUP_BUG_ON((unsigned long)base_addr & ~PAGE_MASK);
PCPU_SETUP_BUG_ON(ai->unit_size < size_sum); PCPU_SETUP_BUG_ON(ai->unit_size < size_sum);
PCPU_SETUP_BUG_ON(ai->unit_size & ~PAGE_MASK); PCPU_SETUP_BUG_ON(ai->unit_size & ~PAGE_MASK);
PCPU_SETUP_BUG_ON(ai->unit_size < PCPU_MIN_UNIT_SIZE); PCPU_SETUP_BUG_ON(ai->unit_size < PCPU_MIN_UNIT_SIZE);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment