Commit e084b2d9 authored by Mel Gorman's avatar Mel Gorman Committed by Linus Torvalds

page-allocator: preserve PFN ordering when __GFP_COLD is set

Fix a post-2.6.24 performace regression caused by
3dfa5721 ("page-allocator: preserve PFN
ordering when __GFP_COLD is set").

Narayanan reports "The regression is around 15%.  There is no disk controller
as our setup is based on Samsung OneNAND used as a memory mapped device on a
OMAP2430 based board."

The page allocator tries to preserve contiguous PFN ordering when returning
pages such that repeated callers to the allocator have a strong chance of
getting physically contiguous pages, particularly when external fragmentation
is low.  However, of the bulk of the allocations have __GFP_COLD set as they
are due to aio_read() for example, then the PFNs are in reverse PFN order.
This can cause performance degration when used with IO controllers that could
have merged the requests.

This patch attempts to preserve the contiguous ordering of PFNs for users of
__GFP_COLD.
Signed-off-by: default avatarMel Gorman <mel@csn.ul.ie>
Reported-by: default avatarNarayananu Gopalakrishnan <narayanan.g@samsung.com>
Tested-by: default avatarNarayanan Gopalakrishnan <narayanan.g@samsung.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: <stable@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 51fbb4ba
...@@ -882,7 +882,7 @@ static struct page *__rmqueue(struct zone *zone, unsigned int order, ...@@ -882,7 +882,7 @@ static struct page *__rmqueue(struct zone *zone, unsigned int order,
*/ */
static int rmqueue_bulk(struct zone *zone, unsigned int order, static int rmqueue_bulk(struct zone *zone, unsigned int order,
unsigned long count, struct list_head *list, unsigned long count, struct list_head *list,
int migratetype) int migratetype, int cold)
{ {
int i; int i;
...@@ -901,7 +901,10 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, ...@@ -901,7 +901,10 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
* merge IO requests if the physical pages are ordered * merge IO requests if the physical pages are ordered
* properly. * properly.
*/ */
if (likely(cold == 0))
list_add(&page->lru, list); list_add(&page->lru, list);
else
list_add_tail(&page->lru, list);
set_page_private(page, migratetype); set_page_private(page, migratetype);
list = &page->lru; list = &page->lru;
} }
...@@ -1119,7 +1122,8 @@ struct page *buffered_rmqueue(struct zone *preferred_zone, ...@@ -1119,7 +1122,8 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
local_irq_save(flags); local_irq_save(flags);
if (!pcp->count) { if (!pcp->count) {
pcp->count = rmqueue_bulk(zone, 0, pcp->count = rmqueue_bulk(zone, 0,
pcp->batch, &pcp->list, migratetype); pcp->batch, &pcp->list,
migratetype, cold);
if (unlikely(!pcp->count)) if (unlikely(!pcp->count))
goto failed; goto failed;
} }
...@@ -1138,7 +1142,8 @@ struct page *buffered_rmqueue(struct zone *preferred_zone, ...@@ -1138,7 +1142,8 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
/* Allocate more to the pcp list if necessary */ /* Allocate more to the pcp list if necessary */
if (unlikely(&page->lru == &pcp->list)) { if (unlikely(&page->lru == &pcp->list)) {
pcp->count += rmqueue_bulk(zone, 0, pcp->count += rmqueue_bulk(zone, 0,
pcp->batch, &pcp->list, migratetype); pcp->batch, &pcp->list,
migratetype, cold);
page = list_entry(pcp->list.next, struct page, lru); page = list_entry(pcp->list.next, struct page, lru);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment