Commit 99dcc3e5 authored by Christoph Lameter's avatar Christoph Lameter Committed by Tejun Heo

this_cpu: Page allocator conversion

Use the per cpu allocator functionality to avoid per cpu arrays in struct zone.

This drastically reduces the size of struct zone for systems with large
amounts of processors and allows placement of critical variables of struct
zone in one cacheline even on very large systems.

Another effect is that the pagesets of one processor are placed near one
another. If multiple pagesets from different zones fit into one cacheline
then additional cacheline fetches can be avoided on the hot paths when
allocating memory from multiple zones.

Bootstrap becomes simpler if we use the same scheme for UP, SMP, NUMA. #ifdefs
are reduced and we can drop the zone_pcp macro.

Hotplug handling is also simplified since cpu alloc can bring up and
shut down cpu areas for a specific cpu as a whole. So there is no need to
allocate or free individual pagesets.

V7-V8:
- Explain chicken egg dilemmna with percpu allocator.

V4-V5:
- Fix up cases where per_cpu_ptr is called before irq disable
- Integrate the bootstrap logic that was separate before.

tj: Build failure in pageset_cpuup_callback() due to missing ret
    variable fixed.
Reviewed-by: default avatarMel Gorman <mel@csn.ul.ie>
Signed-off-by: default avatarChristoph Lameter <cl@linux-foundation.org>
Signed-off-by: default avatarTejun Heo <tj@kernel.org>
parent 5917dae8
...@@ -1079,11 +1079,7 @@ extern void si_meminfo(struct sysinfo * val); ...@@ -1079,11 +1079,7 @@ extern void si_meminfo(struct sysinfo * val);
extern void si_meminfo_node(struct sysinfo *val, int nid); extern void si_meminfo_node(struct sysinfo *val, int nid);
extern int after_bootmem; extern int after_bootmem;
#ifdef CONFIG_NUMA
extern void setup_per_cpu_pageset(void); extern void setup_per_cpu_pageset(void);
#else
static inline void setup_per_cpu_pageset(void) {}
#endif
extern void zone_pcp_update(struct zone *zone); extern void zone_pcp_update(struct zone *zone);
......
...@@ -184,13 +184,7 @@ struct per_cpu_pageset { ...@@ -184,13 +184,7 @@ struct per_cpu_pageset {
s8 stat_threshold; s8 stat_threshold;
s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS]; s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS];
#endif #endif
} ____cacheline_aligned_in_smp; };
#ifdef CONFIG_NUMA
#define zone_pcp(__z, __cpu) ((__z)->pageset[(__cpu)])
#else
#define zone_pcp(__z, __cpu) (&(__z)->pageset[(__cpu)])
#endif
#endif /* !__GENERATING_BOUNDS.H */ #endif /* !__GENERATING_BOUNDS.H */
...@@ -306,10 +300,8 @@ struct zone { ...@@ -306,10 +300,8 @@ struct zone {
*/ */
unsigned long min_unmapped_pages; unsigned long min_unmapped_pages;
unsigned long min_slab_pages; unsigned long min_slab_pages;
struct per_cpu_pageset *pageset[NR_CPUS];
#else
struct per_cpu_pageset pageset[NR_CPUS];
#endif #endif
struct per_cpu_pageset *pageset;
/* /*
* free areas of different sizes * free areas of different sizes
*/ */
......
...@@ -1008,10 +1008,10 @@ static void drain_pages(unsigned int cpu) ...@@ -1008,10 +1008,10 @@ static void drain_pages(unsigned int cpu)
struct per_cpu_pageset *pset; struct per_cpu_pageset *pset;
struct per_cpu_pages *pcp; struct per_cpu_pages *pcp;
pset = zone_pcp(zone, cpu); local_irq_save(flags);
pset = per_cpu_ptr(zone->pageset, cpu);
pcp = &pset->pcp; pcp = &pset->pcp;
local_irq_save(flags);
free_pcppages_bulk(zone, pcp->count, pcp); free_pcppages_bulk(zone, pcp->count, pcp);
pcp->count = 0; pcp->count = 0;
local_irq_restore(flags); local_irq_restore(flags);
...@@ -1095,7 +1095,6 @@ static void free_hot_cold_page(struct page *page, int cold) ...@@ -1095,7 +1095,6 @@ static void free_hot_cold_page(struct page *page, int cold)
arch_free_page(page, 0); arch_free_page(page, 0);
kernel_map_pages(page, 1, 0); kernel_map_pages(page, 1, 0);
pcp = &zone_pcp(zone, get_cpu())->pcp;
migratetype = get_pageblock_migratetype(page); migratetype = get_pageblock_migratetype(page);
set_page_private(page, migratetype); set_page_private(page, migratetype);
local_irq_save(flags); local_irq_save(flags);
...@@ -1118,6 +1117,7 @@ static void free_hot_cold_page(struct page *page, int cold) ...@@ -1118,6 +1117,7 @@ static void free_hot_cold_page(struct page *page, int cold)
migratetype = MIGRATE_MOVABLE; migratetype = MIGRATE_MOVABLE;
} }
pcp = &this_cpu_ptr(zone->pageset)->pcp;
if (cold) if (cold)
list_add_tail(&page->lru, &pcp->lists[migratetype]); list_add_tail(&page->lru, &pcp->lists[migratetype]);
else else
...@@ -1130,7 +1130,6 @@ static void free_hot_cold_page(struct page *page, int cold) ...@@ -1130,7 +1130,6 @@ static void free_hot_cold_page(struct page *page, int cold)
out: out:
local_irq_restore(flags); local_irq_restore(flags);
put_cpu();
} }
void free_hot_page(struct page *page) void free_hot_page(struct page *page)
...@@ -1180,17 +1179,15 @@ struct page *buffered_rmqueue(struct zone *preferred_zone, ...@@ -1180,17 +1179,15 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
unsigned long flags; unsigned long flags;
struct page *page; struct page *page;
int cold = !!(gfp_flags & __GFP_COLD); int cold = !!(gfp_flags & __GFP_COLD);
int cpu;
again: again:
cpu = get_cpu();
if (likely(order == 0)) { if (likely(order == 0)) {
struct per_cpu_pages *pcp; struct per_cpu_pages *pcp;
struct list_head *list; struct list_head *list;
pcp = &zone_pcp(zone, cpu)->pcp;
list = &pcp->lists[migratetype];
local_irq_save(flags); local_irq_save(flags);
pcp = &this_cpu_ptr(zone->pageset)->pcp;
list = &pcp->lists[migratetype];
if (list_empty(list)) { if (list_empty(list)) {
pcp->count += rmqueue_bulk(zone, 0, pcp->count += rmqueue_bulk(zone, 0,
pcp->batch, list, pcp->batch, list,
...@@ -1231,7 +1228,6 @@ struct page *buffered_rmqueue(struct zone *preferred_zone, ...@@ -1231,7 +1228,6 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
__count_zone_vm_events(PGALLOC, zone, 1 << order); __count_zone_vm_events(PGALLOC, zone, 1 << order);
zone_statistics(preferred_zone, zone); zone_statistics(preferred_zone, zone);
local_irq_restore(flags); local_irq_restore(flags);
put_cpu();
VM_BUG_ON(bad_range(zone, page)); VM_BUG_ON(bad_range(zone, page));
if (prep_new_page(page, order, gfp_flags)) if (prep_new_page(page, order, gfp_flags))
...@@ -1240,7 +1236,6 @@ struct page *buffered_rmqueue(struct zone *preferred_zone, ...@@ -1240,7 +1236,6 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
failed: failed:
local_irq_restore(flags); local_irq_restore(flags);
put_cpu();
return NULL; return NULL;
} }
...@@ -2179,7 +2174,7 @@ void show_free_areas(void) ...@@ -2179,7 +2174,7 @@ void show_free_areas(void)
for_each_online_cpu(cpu) { for_each_online_cpu(cpu) {
struct per_cpu_pageset *pageset; struct per_cpu_pageset *pageset;
pageset = zone_pcp(zone, cpu); pageset = per_cpu_ptr(zone->pageset, cpu);
printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n", printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n",
cpu, pageset->pcp.high, cpu, pageset->pcp.high,
...@@ -2744,10 +2739,29 @@ static void build_zonelist_cache(pg_data_t *pgdat) ...@@ -2744,10 +2739,29 @@ static void build_zonelist_cache(pg_data_t *pgdat)
#endif /* CONFIG_NUMA */ #endif /* CONFIG_NUMA */
/*
* Boot pageset table. One per cpu which is going to be used for all
* zones and all nodes. The parameters will be set in such a way
* that an item put on a list will immediately be handed over to
* the buddy list. This is safe since pageset manipulation is done
* with interrupts disabled.
*
* The boot_pagesets must be kept even after bootup is complete for
* unused processors and/or zones. They do play a role for bootstrapping
* hotplugged processors.
*
* zoneinfo_show() and maybe other functions do
* not check if the processor is online before following the pageset pointer.
* Other parts of the kernel may not check if the zone is available.
*/
static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
/* return values int ....just for stop_machine() */ /* return values int ....just for stop_machine() */
static int __build_all_zonelists(void *dummy) static int __build_all_zonelists(void *dummy)
{ {
int nid; int nid;
int cpu;
#ifdef CONFIG_NUMA #ifdef CONFIG_NUMA
memset(node_load, 0, sizeof(node_load)); memset(node_load, 0, sizeof(node_load));
...@@ -2758,6 +2772,23 @@ static int __build_all_zonelists(void *dummy) ...@@ -2758,6 +2772,23 @@ static int __build_all_zonelists(void *dummy)
build_zonelists(pgdat); build_zonelists(pgdat);
build_zonelist_cache(pgdat); build_zonelist_cache(pgdat);
} }
/*
* Initialize the boot_pagesets that are going to be used
* for bootstrapping processors. The real pagesets for
* each zone will be allocated later when the per cpu
* allocator is available.
*
* boot_pagesets are used also for bootstrapping offline
* cpus if the system is already booted because the pagesets
* are needed to initialize allocators on a specific cpu too.
* F.e. the percpu allocator needs the page allocator which
* needs the percpu allocator in order to allocate its pagesets
* (a chicken-egg dilemma).
*/
for_each_possible_cpu(cpu)
setup_pageset(&per_cpu(boot_pageset, cpu), 0);
return 0; return 0;
} }
...@@ -3095,121 +3126,33 @@ static void setup_pagelist_highmark(struct per_cpu_pageset *p, ...@@ -3095,121 +3126,33 @@ static void setup_pagelist_highmark(struct per_cpu_pageset *p,
pcp->batch = PAGE_SHIFT * 8; pcp->batch = PAGE_SHIFT * 8;
} }
#ifdef CONFIG_NUMA
/* /*
* Boot pageset table. One per cpu which is going to be used for all * Allocate per cpu pagesets and initialize them.
* zones and all nodes. The parameters will be set in such a way * Before this call only boot pagesets were available.
* that an item put on a list will immediately be handed over to * Boot pagesets will no longer be used by this processorr
* the buddy list. This is safe since pageset manipulation is done * after setup_per_cpu_pageset().
* with interrupts disabled.
*
* Some NUMA counter updates may also be caught by the boot pagesets.
*
* The boot_pagesets must be kept even after bootup is complete for
* unused processors and/or zones. They do play a role for bootstrapping
* hotplugged processors.
*
* zoneinfo_show() and maybe other functions do
* not check if the processor is online before following the pageset pointer.
* Other parts of the kernel may not check if the zone is available.
*/ */
static struct per_cpu_pageset boot_pageset[NR_CPUS]; void __init setup_per_cpu_pageset(void)
/*
* Dynamically allocate memory for the
* per cpu pageset array in struct zone.
*/
static int __cpuinit process_zones(int cpu)
{ {
struct zone *zone, *dzone; struct zone *zone;
int node = cpu_to_node(cpu); int cpu;
node_set_state(node, N_CPU); /* this node has a cpu */
for_each_populated_zone(zone) { for_each_populated_zone(zone) {
zone_pcp(zone, cpu) = kmalloc_node(sizeof(struct per_cpu_pageset), zone->pageset = alloc_percpu(struct per_cpu_pageset);
GFP_KERNEL, node);
if (!zone_pcp(zone, cpu))
goto bad;
setup_pageset(zone_pcp(zone, cpu), zone_batchsize(zone)); for_each_possible_cpu(cpu) {
struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
if (percpu_pagelist_fraction)
setup_pagelist_highmark(zone_pcp(zone, cpu),
(zone->present_pages / percpu_pagelist_fraction));
}
return 0; setup_pageset(pcp, zone_batchsize(zone));
bad:
for_each_zone(dzone) {
if (!populated_zone(dzone))
continue;
if (dzone == zone)
break;
kfree(zone_pcp(dzone, cpu));
zone_pcp(dzone, cpu) = &boot_pageset[cpu];
}
return -ENOMEM;
}
static inline void free_zone_pagesets(int cpu) if (percpu_pagelist_fraction)
{ setup_pagelist_highmark(pcp,
struct zone *zone; (zone->present_pages /
percpu_pagelist_fraction));
for_each_zone(zone) {
struct per_cpu_pageset *pset = zone_pcp(zone, cpu);
/* Free per_cpu_pageset if it is slab allocated */
if (pset != &boot_pageset[cpu])
kfree(pset);
zone_pcp(zone, cpu) = &boot_pageset[cpu];
} }
}
static int __cpuinit pageset_cpuup_callback(struct notifier_block *nfb,
unsigned long action,
void *hcpu)
{
int cpu = (long)hcpu;
int ret = NOTIFY_OK;
switch (action) {
case CPU_UP_PREPARE:
case CPU_UP_PREPARE_FROZEN:
if (process_zones(cpu))
ret = NOTIFY_BAD;
break;
case CPU_UP_CANCELED:
case CPU_UP_CANCELED_FROZEN:
case CPU_DEAD:
case CPU_DEAD_FROZEN:
free_zone_pagesets(cpu);
break;
default:
break;
} }
return ret;
}
static struct notifier_block __cpuinitdata pageset_notifier =
{ &pageset_cpuup_callback, NULL, 0 };
void __init setup_per_cpu_pageset(void)
{
int err;
/* Initialize per_cpu_pageset for cpu 0.
* A cpuup callback will do this for every cpu
* as it comes online
*/
err = process_zones(smp_processor_id());
BUG_ON(err);
register_cpu_notifier(&pageset_notifier);
} }
#endif
static noinline __init_refok static noinline __init_refok
int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages) int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
{ {
...@@ -3263,7 +3206,7 @@ static int __zone_pcp_update(void *data) ...@@ -3263,7 +3206,7 @@ static int __zone_pcp_update(void *data)
struct per_cpu_pageset *pset; struct per_cpu_pageset *pset;
struct per_cpu_pages *pcp; struct per_cpu_pages *pcp;
pset = zone_pcp(zone, cpu); pset = per_cpu_ptr(zone->pageset, cpu);
pcp = &pset->pcp; pcp = &pset->pcp;
local_irq_save(flags); local_irq_save(flags);
...@@ -3281,21 +3224,17 @@ void zone_pcp_update(struct zone *zone) ...@@ -3281,21 +3224,17 @@ void zone_pcp_update(struct zone *zone)
static __meminit void zone_pcp_init(struct zone *zone) static __meminit void zone_pcp_init(struct zone *zone)
{ {
int cpu; /*
unsigned long batch = zone_batchsize(zone); * per cpu subsystem is not up at this point. The following code
* relies on the ability of the linker to provide the
* offset of a (static) per cpu variable into the per cpu area.
*/
zone->pageset = &boot_pageset;
for (cpu = 0; cpu < NR_CPUS; cpu++) {
#ifdef CONFIG_NUMA
/* Early boot. Slab allocator not functional yet */
zone_pcp(zone, cpu) = &boot_pageset[cpu];
setup_pageset(&boot_pageset[cpu],0);
#else
setup_pageset(zone_pcp(zone,cpu), batch);
#endif
}
if (zone->present_pages) if (zone->present_pages)
printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%lu\n", printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
zone->name, zone->present_pages, batch); zone->name, zone->present_pages,
zone_batchsize(zone));
} }
__meminit int init_currently_empty_zone(struct zone *zone, __meminit int init_currently_empty_zone(struct zone *zone,
...@@ -4809,10 +4748,11 @@ int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write, ...@@ -4809,10 +4748,11 @@ int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write,
if (!write || (ret == -EINVAL)) if (!write || (ret == -EINVAL))
return ret; return ret;
for_each_populated_zone(zone) { for_each_populated_zone(zone) {
for_each_online_cpu(cpu) { for_each_possible_cpu(cpu) {
unsigned long high; unsigned long high;
high = zone->present_pages / percpu_pagelist_fraction; high = zone->present_pages / percpu_pagelist_fraction;
setup_pagelist_highmark(zone_pcp(zone, cpu), high); setup_pagelist_highmark(
per_cpu_ptr(zone->pageset, cpu), high);
} }
} }
return 0; return 0;
......
...@@ -139,7 +139,8 @@ static void refresh_zone_stat_thresholds(void) ...@@ -139,7 +139,8 @@ static void refresh_zone_stat_thresholds(void)
threshold = calculate_threshold(zone); threshold = calculate_threshold(zone);
for_each_online_cpu(cpu) for_each_online_cpu(cpu)
zone_pcp(zone, cpu)->stat_threshold = threshold; per_cpu_ptr(zone->pageset, cpu)->stat_threshold
= threshold;
} }
} }
...@@ -149,7 +150,8 @@ static void refresh_zone_stat_thresholds(void) ...@@ -149,7 +150,8 @@ static void refresh_zone_stat_thresholds(void)
void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item, void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
int delta) int delta)
{ {
struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id()); struct per_cpu_pageset *pcp = this_cpu_ptr(zone->pageset);
s8 *p = pcp->vm_stat_diff + item; s8 *p = pcp->vm_stat_diff + item;
long x; long x;
...@@ -202,7 +204,7 @@ EXPORT_SYMBOL(mod_zone_page_state); ...@@ -202,7 +204,7 @@ EXPORT_SYMBOL(mod_zone_page_state);
*/ */
void __inc_zone_state(struct zone *zone, enum zone_stat_item item) void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
{ {
struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id()); struct per_cpu_pageset *pcp = this_cpu_ptr(zone->pageset);
s8 *p = pcp->vm_stat_diff + item; s8 *p = pcp->vm_stat_diff + item;
(*p)++; (*p)++;
...@@ -223,7 +225,7 @@ EXPORT_SYMBOL(__inc_zone_page_state); ...@@ -223,7 +225,7 @@ EXPORT_SYMBOL(__inc_zone_page_state);
void __dec_zone_state(struct zone *zone, enum zone_stat_item item) void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
{ {
struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id()); struct per_cpu_pageset *pcp = this_cpu_ptr(zone->pageset);
s8 *p = pcp->vm_stat_diff + item; s8 *p = pcp->vm_stat_diff + item;
(*p)--; (*p)--;
...@@ -300,7 +302,7 @@ void refresh_cpu_vm_stats(int cpu) ...@@ -300,7 +302,7 @@ void refresh_cpu_vm_stats(int cpu)
for_each_populated_zone(zone) { for_each_populated_zone(zone) {
struct per_cpu_pageset *p; struct per_cpu_pageset *p;
p = zone_pcp(zone, cpu); p = per_cpu_ptr(zone->pageset, cpu);
for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
if (p->vm_stat_diff[i]) { if (p->vm_stat_diff[i]) {
...@@ -741,7 +743,7 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, ...@@ -741,7 +743,7 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
for_each_online_cpu(i) { for_each_online_cpu(i) {
struct per_cpu_pageset *pageset; struct per_cpu_pageset *pageset;
pageset = zone_pcp(zone, i); pageset = per_cpu_ptr(zone->pageset, i);
seq_printf(m, seq_printf(m,
"\n cpu: %i" "\n cpu: %i"
"\n count: %i" "\n count: %i"
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment