Commit 3bdafaa1 authored by Andi Kleen's avatar Andi Kleen Committed by Linus Torvalds

[PATCH] Use numa policy API for boot time policy

Suggested by Manfred Spraul.

__get_free_pages had a hack to do node interleaving allocation at boot
time.  This patch sets an interleave process policy using the NUMA API for
init and the idle threads instead.  Before entering the user space init the
policy is reset to default again.  Result is the same.

Advantage is less code and removing of a check from a fast path.

Removes more code than it adds.

I verified that the memory distribution after boot is roughly the same.
Signed-off-by: default avatarAndrew Morton <akpm@osdl.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent e7f32b95
...@@ -153,6 +153,9 @@ void mpol_free_shared_policy(struct shared_policy *p); ...@@ -153,6 +153,9 @@ void mpol_free_shared_policy(struct shared_policy *p);
struct mempolicy *mpol_shared_policy_lookup(struct shared_policy *sp, struct mempolicy *mpol_shared_policy_lookup(struct shared_policy *sp,
unsigned long idx); unsigned long idx);
extern void numa_default_policy(void);
extern void numa_policy_init(void);
#else #else
struct mempolicy {}; struct mempolicy {};
...@@ -215,6 +218,14 @@ mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx) ...@@ -215,6 +218,14 @@ mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
#define vma_policy(vma) NULL #define vma_policy(vma) NULL
#define vma_set_policy(vma, pol) do {} while(0) #define vma_set_policy(vma, pol) do {} while(0)
static inline void numa_policy_init(void)
{
}
static inline void numa_default_policy(void)
{
}
#endif /* CONFIG_NUMA */ #endif /* CONFIG_NUMA */
#endif /* __KERNEL__ */ #endif /* __KERNEL__ */
......
...@@ -43,6 +43,7 @@ ...@@ -43,6 +43,7 @@
#include <linux/efi.h> #include <linux/efi.h>
#include <linux/unistd.h> #include <linux/unistd.h>
#include <linux/rmap.h> #include <linux/rmap.h>
#include <linux/mempolicy.h>
#include <asm/io.h> #include <asm/io.h>
#include <asm/bugs.h> #include <asm/bugs.h>
...@@ -385,6 +386,7 @@ static void __init smp_init(void) ...@@ -385,6 +386,7 @@ static void __init smp_init(void)
static void noinline rest_init(void) static void noinline rest_init(void)
{ {
kernel_thread(init, NULL, CLONE_FS | CLONE_SIGHAND); kernel_thread(init, NULL, CLONE_FS | CLONE_SIGHAND);
numa_default_policy();
unlock_kernel(); unlock_kernel();
cpu_idle(); cpu_idle();
} }
...@@ -456,6 +458,7 @@ asmlinkage void __init start_kernel(void) ...@@ -456,6 +458,7 @@ asmlinkage void __init start_kernel(void)
#endif #endif
mem_init(); mem_init();
kmem_cache_init(); kmem_cache_init();
numa_policy_init();
if (late_time_init) if (late_time_init)
late_time_init(); late_time_init();
calibrate_delay(); calibrate_delay();
...@@ -645,6 +648,7 @@ static int init(void * unused) ...@@ -645,6 +648,7 @@ static int init(void * unused)
free_initmem(); free_initmem();
unlock_kernel(); unlock_kernel();
system_state = SYSTEM_RUNNING; system_state = SYSTEM_RUNNING;
numa_default_policy();
if (sys_open((const char __user *) "/dev/console", O_RDWR, 0) < 0) if (sys_open((const char __user *) "/dev/console", O_RDWR, 0) < 0)
printk("Warning: unable to open an initial console.\n"); printk("Warning: unable to open an initial console.\n");
......
...@@ -1002,7 +1002,8 @@ void mpol_free_shared_policy(struct shared_policy *p) ...@@ -1002,7 +1002,8 @@ void mpol_free_shared_policy(struct shared_policy *p)
up(&p->sem); up(&p->sem);
} }
static __init int numa_policy_init(void) /* assumes fs == KERNEL_DS */
void __init numa_policy_init(void)
{ {
policy_cache = kmem_cache_create("numa_policy", policy_cache = kmem_cache_create("numa_policy",
sizeof(struct mempolicy), sizeof(struct mempolicy),
...@@ -1011,6 +1012,17 @@ static __init int numa_policy_init(void) ...@@ -1011,6 +1012,17 @@ static __init int numa_policy_init(void)
sn_cache = kmem_cache_create("shared_policy_node", sn_cache = kmem_cache_create("shared_policy_node",
sizeof(struct sp_node), sizeof(struct sp_node),
0, SLAB_PANIC, NULL, NULL); 0, SLAB_PANIC, NULL, NULL);
return 0;
/* Set interleaving policy for system init. This way not all
the data structures allocated at system boot end up in node zero. */
if (sys_set_mempolicy(MPOL_INTERLEAVE, node_online_map, MAX_NUMNODES) < 0)
printk("numa_policy_init: interleaving failed\n");
}
/* Reset policy of current process to default.
* Assumes fs == KERNEL_DS */
void __init numa_default_policy(void)
{
sys_set_mempolicy(MPOL_DEFAULT, NULL, 0);
} }
module_init(numa_policy_init);
...@@ -732,53 +732,12 @@ __alloc_pages(unsigned int gfp_mask, unsigned int order, ...@@ -732,53 +732,12 @@ __alloc_pages(unsigned int gfp_mask, unsigned int order,
EXPORT_SYMBOL(__alloc_pages); EXPORT_SYMBOL(__alloc_pages);
#ifdef CONFIG_NUMA
/* Early boot: Everything is done by one cpu, but the data structures will be
* used by all cpus - spread them on all nodes.
*/
static __init unsigned long get_boot_pages(unsigned int gfp_mask, unsigned int order)
{
static int nodenr;
int i = nodenr;
struct page *page;
for (;;) {
if (i > nodenr + numnodes)
return 0;
if (node_present_pages(i%numnodes)) {
struct zone **z;
/* The node contains memory. Check that there is
* memory in the intended zonelist.
*/
z = NODE_DATA(i%numnodes)->node_zonelists[gfp_mask & GFP_ZONEMASK].zones;
while (*z) {
if ( (*z)->free_pages > (1UL<<order))
goto found_node;
z++;
}
}
i++;
}
found_node:
nodenr = i+1;
page = alloc_pages_node(i%numnodes, gfp_mask, order);
if (!page)
return 0;
return (unsigned long) page_address(page);
}
#endif
/* /*
* Common helper functions. * Common helper functions.
*/ */
fastcall unsigned long __get_free_pages(unsigned int gfp_mask, unsigned int order) fastcall unsigned long __get_free_pages(unsigned int gfp_mask, unsigned int order)
{ {
struct page * page; struct page * page;
#ifdef CONFIG_NUMA
if (unlikely(system_state == SYSTEM_BOOTING))
return get_boot_pages(gfp_mask, order);
#endif
page = alloc_pages(gfp_mask, order); page = alloc_pages(gfp_mask, order);
if (!page) if (!page)
return 0; return 0;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment