Commit 79e626e1 authored by Andrew Morton's avatar Andrew Morton Committed by James Bottomley

[PATCH] bootmem speedup from the IA64 tree

From: Christoph Hellwig <hch@lst.de>

This patch is from the IA64 tree, with some minor cleanups by me.
David described it as:

  This is a performance speed up and some minor indendation fixups.

  The problem is that the bootmem code is (a) hugely slow and (b) has
  execution that grow quadratically with the size of the bootmap bitmap.
  This causes noticable slowdowns, especially on machines with (relatively)
  large holes in the physical memory map.  Issue (b) is addressed by
  maintaining the "last_success" cache, so that we start the next search
  from the place where we last found some memory (this part of the patch
  could stand additional reviewing/testing).  Issue (a) is addressed by
  using find_next_zero_bit() instead of the slow bit-by-bit testing.
parent a413a276
...@@ -32,6 +32,8 @@ typedef struct bootmem_data { ...@@ -32,6 +32,8 @@ typedef struct bootmem_data {
void *node_bootmem_map; void *node_bootmem_map;
unsigned long last_offset; unsigned long last_offset;
unsigned long last_pos; unsigned long last_pos;
unsigned long last_success; /* Previous allocation point. To speed
* up searching */
} bootmem_data_t; } bootmem_data_t;
extern unsigned long __init bootmem_bootmap_pages (unsigned long); extern unsigned long __init bootmem_bootmap_pages (unsigned long);
......
...@@ -115,6 +115,9 @@ static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr, ...@@ -115,6 +115,9 @@ static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr,
if (end > bdata->node_low_pfn) if (end > bdata->node_low_pfn)
BUG(); BUG();
if (addr < bdata->last_success)
bdata->last_success = addr;
/* /*
* Round up the beginning of the address. * Round up the beginning of the address.
*/ */
...@@ -135,26 +138,23 @@ static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr, ...@@ -135,26 +138,23 @@ static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr,
* is not a problem. * is not a problem.
* *
* On low memory boxes we get it right in 100% of the cases. * On low memory boxes we get it right in 100% of the cases.
*/ *
/*
* alignment has to be a power of 2 value. * alignment has to be a power of 2 value.
*
* NOTE: This function is _not_ reenetrant.
*/ */
static void * __init __alloc_bootmem_core (bootmem_data_t *bdata, static void * __init
unsigned long size, unsigned long align, unsigned long goal) __alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
unsigned long align, unsigned long goal)
{ {
unsigned long i, start = 0; unsigned long offset, remaining_size, areasize, preferred;
unsigned long i, start = 0, incr, eidx;
void *ret; void *ret;
unsigned long offset, remaining_size;
unsigned long areasize, preferred, incr;
unsigned long eidx = bdata->node_low_pfn - (bdata->node_boot_start >>
PAGE_SHIFT);
if (!size) BUG(); BUG_ON(!size);
BUG_ON(align & (align-1));
if (align & (align-1))
BUG();
eidx = bdata->node_low_pfn - (bdata->node_boot_start >> PAGE_SHIFT);
offset = 0; offset = 0;
if (align && if (align &&
(bdata->node_boot_start & (align - 1UL)) != 0) (bdata->node_boot_start & (align - 1UL)) != 0)
...@@ -166,8 +166,11 @@ static void * __init __alloc_bootmem_core (bootmem_data_t *bdata, ...@@ -166,8 +166,11 @@ static void * __init __alloc_bootmem_core (bootmem_data_t *bdata,
* first, then we try to allocate lower pages. * first, then we try to allocate lower pages.
*/ */
if (goal && (goal >= bdata->node_boot_start) && if (goal && (goal >= bdata->node_boot_start) &&
((goal >> PAGE_SHIFT) < bdata->node_low_pfn)) { ((goal >> PAGE_SHIFT) < bdata->node_low_pfn)) {
preferred = goal - bdata->node_boot_start; preferred = goal - bdata->node_boot_start;
if (bdata->last_success >= preferred)
preferred = bdata->last_success;
} else } else
preferred = 0; preferred = 0;
...@@ -179,6 +182,8 @@ static void * __init __alloc_bootmem_core (bootmem_data_t *bdata, ...@@ -179,6 +182,8 @@ static void * __init __alloc_bootmem_core (bootmem_data_t *bdata,
restart_scan: restart_scan:
for (i = preferred; i < eidx; i += incr) { for (i = preferred; i < eidx; i += incr) {
unsigned long j; unsigned long j;
i = find_next_zero_bit(bdata->node_bootmem_map, eidx, i);
i = (i + incr - 1) & -incr;
if (test_bit(i, bdata->node_bootmem_map)) if (test_bit(i, bdata->node_bootmem_map))
continue; continue;
for (j = i + 1; j < i + areasize; ++j) { for (j = i + 1; j < i + areasize; ++j) {
...@@ -189,31 +194,33 @@ static void * __init __alloc_bootmem_core (bootmem_data_t *bdata, ...@@ -189,31 +194,33 @@ static void * __init __alloc_bootmem_core (bootmem_data_t *bdata,
} }
start = i; start = i;
goto found; goto found;
fail_block:; fail_block:
;
} }
if (preferred) { if (preferred) {
preferred = offset; preferred = offset;
goto restart_scan; goto restart_scan;
} }
return NULL; return NULL;
found: found:
if (start >= eidx) bdata->last_success = start << PAGE_SHIFT;
BUG(); BUG_ON(start >= eidx);
/* /*
* Is the next page of the previous allocation-end the start * Is the next page of the previous allocation-end the start
* of this allocation's buffer? If yes then we can 'merge' * of this allocation's buffer? If yes then we can 'merge'
* the previous partial page with this allocation. * the previous partial page with this allocation.
*/ */
if (align < PAGE_SIZE if (align < PAGE_SIZE &&
&& bdata->last_offset && bdata->last_pos+1 == start) { bdata->last_offset && bdata->last_pos+1 == start) {
offset = (bdata->last_offset+align-1) & ~(align-1); offset = (bdata->last_offset+align-1) & ~(align-1);
if (offset > PAGE_SIZE) BUG_ON(offset > PAGE_SIZE);
BUG();
remaining_size = PAGE_SIZE-offset; remaining_size = PAGE_SIZE-offset;
if (size < remaining_size) { if (size < remaining_size) {
areasize = 0; areasize = 0;
// last_pos unchanged /* last_pos unchanged */
bdata->last_offset = offset+size; bdata->last_offset = offset+size;
ret = phys_to_virt(bdata->last_pos*PAGE_SIZE + offset + ret = phys_to_virt(bdata->last_pos*PAGE_SIZE + offset +
bdata->node_boot_start); bdata->node_boot_start);
...@@ -231,11 +238,12 @@ static void * __init __alloc_bootmem_core (bootmem_data_t *bdata, ...@@ -231,11 +238,12 @@ static void * __init __alloc_bootmem_core (bootmem_data_t *bdata,
bdata->last_offset = size & ~PAGE_MASK; bdata->last_offset = size & ~PAGE_MASK;
ret = phys_to_virt(start * PAGE_SIZE + bdata->node_boot_start); ret = phys_to_virt(start * PAGE_SIZE + bdata->node_boot_start);
} }
/* /*
* Reserve the area now: * Reserve the area now:
*/ */
for (i = start; i < start+areasize; i++) for (i = start; i < start+areasize; i++)
if (test_and_set_bit(i, bdata->node_bootmem_map)) if (unlikely(test_and_set_bit(i, bdata->node_bootmem_map)))
BUG(); BUG();
memset(ret, 0, size); memset(ret, 0, size);
return ret; return ret;
...@@ -256,21 +264,21 @@ static unsigned long __init free_all_bootmem_core(pg_data_t *pgdat) ...@@ -256,21 +264,21 @@ static unsigned long __init free_all_bootmem_core(pg_data_t *pgdat)
map = bdata->node_bootmem_map; map = bdata->node_bootmem_map;
for (i = 0; i < idx; ) { for (i = 0; i < idx; ) {
unsigned long v = ~map[i / BITS_PER_LONG]; unsigned long v = ~map[i / BITS_PER_LONG];
if (v) { if (v) {
unsigned long m; unsigned long m;
for (m = 1; m && i < idx; m<<=1, page++, i++) { for (m = 1; m && i < idx; m<<=1, page++, i++) {
if (v & m) { if (v & m) {
count++; count++;
ClearPageReserved(page); ClearPageReserved(page);
set_page_count(page, 1); set_page_count(page, 1);
__free_page(page); __free_page(page);
} }
} }
} else { } else {
i+=BITS_PER_LONG; i+=BITS_PER_LONG;
page+=BITS_PER_LONG; page += BITS_PER_LONG;
} }
} }
total += count; total += count;
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment