Commit 63cc8c75 authored by Eric Dumazet's avatar Eric Dumazet Committed by Thomas Gleixner

percpu: introduce DEFINE_PER_CPU_PAGE_ALIGNED() macro

While examining holes in percpu section I found this :

c05f5000 D per_cpu__current_task
c05f5000 D __per_cpu_start
c05f5004 D per_cpu__cpu_number
c05f5008 D per_cpu__irq_regs
c05f500c d per_cpu__cpu_devices
c05f5040 D per_cpu__cyc2ns

<Big Hole of about 4000 bytes>

c05f6000 d per_cpu__cpuid4_info
c05f6004 d per_cpu__cache_kobject
c05f6008 d per_cpu__index_kobject

<Big Hole of about 4000 bytes>

c05f7000 D per_cpu__gdt_page

This is because gdt_page is a percpu variable, defined with
a page alignement, and linker is doing its job, two times because of .o
nesting in the build process.

I introduced a new macro DEFINE_PER_CPU_PAGE_ALIGNED() to avoid
wasting this space. All page aligned variables (only one at this time)
are put in a separate
subsection .data.percpu.page_aligned, at the very begining of percpu zone.

Before patch , on a x86_32 machine :

.data.percpu                30232   3227471872
.data.percpu                22168   3227471872

Thats 8064 bytes saved for each CPU.
Signed-off-by: default avatarEric Dumazet <dada1@cosmosbay.com>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
parent 75d3bce2
...@@ -21,7 +21,7 @@ ...@@ -21,7 +21,7 @@
#include "cpu.h" #include "cpu.h"
DEFINE_PER_CPU(struct gdt_page, gdt_page) = { .gdt = { DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
[GDT_ENTRY_KERNEL_CS] = { { { 0x0000ffff, 0x00cf9a00 } } }, [GDT_ENTRY_KERNEL_CS] = { { { 0x0000ffff, 0x00cf9a00 } } },
[GDT_ENTRY_KERNEL_DS] = { { { 0x0000ffff, 0x00cf9200 } } }, [GDT_ENTRY_KERNEL_DS] = { { { 0x0000ffff, 0x00cf9200 } } },
[GDT_ENTRY_DEFAULT_USER_CS] = { { { 0x0000ffff, 0x00cffa00 } } }, [GDT_ENTRY_DEFAULT_USER_CS] = { { { 0x0000ffff, 0x00cffa00 } } },
......
...@@ -189,6 +189,7 @@ SECTIONS ...@@ -189,6 +189,7 @@ SECTIONS
. = ALIGN(PAGE_SIZE); . = ALIGN(PAGE_SIZE);
.data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) { .data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) {
__per_cpu_start = .; __per_cpu_start = .;
*(.data.percpu.page_aligned)
*(.data.percpu) *(.data.percpu)
*(.data.percpu.shared_aligned) *(.data.percpu.shared_aligned)
__per_cpu_end = .; __per_cpu_end = .;
......
...@@ -348,6 +348,7 @@ ...@@ -348,6 +348,7 @@
. = ALIGN(align); \ . = ALIGN(align); \
__per_cpu_start = .; \ __per_cpu_start = .; \
.data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) { \ .data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) { \
*(.data.percpu.page_aligned) \
*(.data.percpu) \ *(.data.percpu) \
*(.data.percpu.shared_aligned) \ *(.data.percpu.shared_aligned) \
} \ } \
......
...@@ -23,12 +23,19 @@ ...@@ -23,12 +23,19 @@
__attribute__((__section__(SHARED_ALIGNED_SECTION))) \ __attribute__((__section__(SHARED_ALIGNED_SECTION))) \
PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name \ PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name \
____cacheline_aligned_in_smp ____cacheline_aligned_in_smp
#define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \
__attribute__((__section__(".data.percpu.page_aligned"))) \
PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
#else #else
#define DEFINE_PER_CPU(type, name) \ #define DEFINE_PER_CPU(type, name) \
PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \
DEFINE_PER_CPU(type, name) DEFINE_PER_CPU(type, name)
#define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \
DEFINE_PER_CPU(type, name)
#endif #endif
#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var) #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment