Commit cf4b1f75 authored by Martin J. Bligh's avatar Martin J. Bligh Committed by Linus Torvalds

[PATCH] Move node pgdat into node's own memory

Patch from William Lee Irwin

This moves the node's pgdat into the node's own local memory area along
with the lmem_map.

It only touches arch/i386/mm/discontig.c - ie can only affect NUMA
machines.  Has been tested in my tree for over a month on UP, SMP, and
NUMA and compile tested against a variety of different configs.
parent 9f58fa60
...@@ -48,6 +48,14 @@ extern unsigned long max_low_pfn; ...@@ -48,6 +48,14 @@ extern unsigned long max_low_pfn;
extern unsigned long totalram_pages; extern unsigned long totalram_pages;
extern unsigned long totalhigh_pages; extern unsigned long totalhigh_pages;
#define LARGE_PAGE_BYTES (PTRS_PER_PTE * PAGE_SIZE)
unsigned long node_remap_start_pfn[MAX_NUMNODES];
unsigned long node_remap_size[MAX_NUMNODES];
unsigned long node_remap_offset[MAX_NUMNODES];
void *node_remap_start_vaddr[MAX_NUMNODES];
void set_pmd_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags);
/* /*
* Find the highest page frame number we have available for the node * Find the highest page frame number we have available for the node
*/ */
...@@ -65,12 +73,13 @@ static void __init find_max_pfn_node(int nid) ...@@ -65,12 +73,13 @@ static void __init find_max_pfn_node(int nid)
*/ */
static void __init allocate_pgdat(int nid) static void __init allocate_pgdat(int nid)
{ {
unsigned long node_datasz; if (nid)
NODE_DATA(nid) = (pg_data_t *)node_remap_start_vaddr[nid];
node_datasz = PFN_UP(sizeof(struct pglist_data)); else {
NODE_DATA(nid) = (pg_data_t *)(__va(min_low_pfn << PAGE_SHIFT)); NODE_DATA(nid) = (pg_data_t *)(__va(min_low_pfn << PAGE_SHIFT));
min_low_pfn += node_datasz; min_low_pfn += PFN_UP(sizeof(pg_data_t));
memset(NODE_DATA(nid), 0, sizeof(struct pglist_data)); memset(NODE_DATA(nid), 0, sizeof(pg_data_t));
}
} }
/* /*
...@@ -113,14 +122,6 @@ static void __init register_bootmem_low_pages(unsigned long system_max_low_pfn) ...@@ -113,14 +122,6 @@ static void __init register_bootmem_low_pages(unsigned long system_max_low_pfn)
} }
} }
#define LARGE_PAGE_BYTES (PTRS_PER_PTE * PAGE_SIZE)
unsigned long node_remap_start_pfn[MAX_NUMNODES];
unsigned long node_remap_size[MAX_NUMNODES];
unsigned long node_remap_offset[MAX_NUMNODES];
void *node_remap_start_vaddr[MAX_NUMNODES];
extern void set_pmd_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags);
void __init remap_numa_kva(void) void __init remap_numa_kva(void)
{ {
void *vaddr; void *vaddr;
...@@ -145,7 +146,7 @@ static unsigned long calculate_numa_remap_pages(void) ...@@ -145,7 +146,7 @@ static unsigned long calculate_numa_remap_pages(void)
for (nid = 1; nid < numnodes; nid++) { for (nid = 1; nid < numnodes; nid++) {
/* calculate the size of the mem_map needed in bytes */ /* calculate the size of the mem_map needed in bytes */
size = (node_end_pfn[nid] - node_start_pfn[nid] + 1) size = (node_end_pfn[nid] - node_start_pfn[nid] + 1)
* sizeof(struct page); * sizeof(struct page) + sizeof(pg_data_t);
/* convert size to large (pmd size) pages, rounding up */ /* convert size to large (pmd size) pages, rounding up */
size = (size + LARGE_PAGE_BYTES - 1) / LARGE_PAGE_BYTES; size = (size + LARGE_PAGE_BYTES - 1) / LARGE_PAGE_BYTES;
/* now the roundup is correct, convert to PAGE_SIZE pages */ /* now the roundup is correct, convert to PAGE_SIZE pages */
...@@ -195,9 +196,9 @@ unsigned long __init setup_memory(void) ...@@ -195,9 +196,9 @@ unsigned long __init setup_memory(void)
printk("Low memory ends at vaddr %08lx\n", printk("Low memory ends at vaddr %08lx\n",
(ulong) pfn_to_kaddr(max_low_pfn)); (ulong) pfn_to_kaddr(max_low_pfn));
for (nid = 0; nid < numnodes; nid++) { for (nid = 0; nid < numnodes; nid++) {
allocate_pgdat(nid);
node_remap_start_vaddr[nid] = pfn_to_kaddr( node_remap_start_vaddr[nid] = pfn_to_kaddr(
highstart_pfn - node_remap_offset[nid]); highstart_pfn - node_remap_offset[nid]);
allocate_pgdat(nid);
printk ("node %d will remap to vaddr %08lx - %08lx\n", nid, printk ("node %d will remap to vaddr %08lx - %08lx\n", nid,
(ulong) node_remap_start_vaddr[nid], (ulong) node_remap_start_vaddr[nid],
(ulong) pfn_to_kaddr(highstart_pfn (ulong) pfn_to_kaddr(highstart_pfn
...@@ -251,13 +252,6 @@ unsigned long __init setup_memory(void) ...@@ -251,13 +252,6 @@ unsigned long __init setup_memory(void)
*/ */
find_smp_config(); find_smp_config();
/*insert other nodes into pgdat_list*/
for (nid = 1; nid < numnodes; nid++){
NODE_DATA(nid)->pgdat_next = pgdat_list;
pgdat_list = NODE_DATA(nid);
}
#ifdef CONFIG_BLK_DEV_INITRD #ifdef CONFIG_BLK_DEV_INITRD
if (LOADER_TYPE && INITRD_START) { if (LOADER_TYPE && INITRD_START) {
if (INITRD_START + INITRD_SIZE <= (system_max_low_pfn << PAGE_SHIFT)) { if (INITRD_START + INITRD_SIZE <= (system_max_low_pfn << PAGE_SHIFT)) {
...@@ -282,6 +276,18 @@ void __init zone_sizes_init(void) ...@@ -282,6 +276,18 @@ void __init zone_sizes_init(void)
{ {
int nid; int nid;
/*
* Insert nodes into pgdat_list backward so they appear in order.
* Clobber node 0's links and NULL out pgdat_list before starting.
*/
pgdat_list = NULL;
for (nid = numnodes - 1; nid >= 0; nid--) {
if (nid)
memset(NODE_DATA(nid), 0, sizeof(pg_data_t));
NODE_DATA(nid)->pgdat_next = pgdat_list;
pgdat_list = NODE_DATA(nid);
}
for (nid = 0; nid < numnodes; nid++) { for (nid = 0; nid < numnodes; nid++) {
unsigned long zones_size[MAX_NR_ZONES] = {0, 0, 0}; unsigned long zones_size[MAX_NR_ZONES] = {0, 0, 0};
unsigned int max_dma; unsigned int max_dma;
...@@ -312,13 +318,18 @@ void __init zone_sizes_init(void) ...@@ -312,13 +318,18 @@ void __init zone_sizes_init(void)
* normal bootmem allocator, but other nodes come from the * normal bootmem allocator, but other nodes come from the
* remapped KVA area - mbligh * remapped KVA area - mbligh
*/ */
if (nid) if (!nid)
free_area_init_node(nid, NODE_DATA(nid),
node_remap_start_vaddr[nid], zones_size,
start, 0);
else
free_area_init_node(nid, NODE_DATA(nid), 0, free_area_init_node(nid, NODE_DATA(nid), 0,
zones_size, start, 0); zones_size, start, 0);
else {
unsigned long lmem_map;
lmem_map = (unsigned long)node_remap_start_vaddr[nid];
lmem_map += sizeof(pg_data_t) + PAGE_SIZE - 1;
lmem_map &= PAGE_MASK;
free_area_init_node(nid, NODE_DATA(nid),
(struct page *)lmem_map, zones_size,
start, 0);
}
} }
return; return;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment