Commit 9a76db09 authored by Lee Schermerhorn's avatar Lee Schermerhorn Committed by Linus Torvalds

hugetlb: rework hstate_next_node_* functions

Modify the hstate_next_node* functions to allow them to be called to
obtain the "start_nid".  Then, whereas prior to this patch we
unconditionally called hstate_next_node_to_{alloc|free}(), whether or not
we successfully allocated/freed a huge page on the node, now we only call
these functions on failure to alloc/free to advance to next allowed node.

Factor out the next_node_allowed() function to handle wrap at end of
node_online_map.  In this version, the allowed nodes include all of the
online nodes.
Signed-off-by: default avatarLee Schermerhorn <lee.schermerhorn@hp.com>
Reviewed-by: default avatarMel Gorman <mel@csn.ul.ie>
Acked-by: default avatarDavid Rientjes <rientjes@google.com>
Reviewed-by: default avatarAndi Kleen <andi@firstfloor.org>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Randy Dunlap <randy.dunlap@oracle.com>
Cc: Nishanth Aravamudan <nacc@us.ibm.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Adam Litke <agl@us.ibm.com>
Cc: Andy Whitcroft <apw@canonical.com>
Cc: Eric Whitney <eric.whitney@hp.com>
Cc: Christoph Lameter <cl@linux-foundation.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 4e7b8a6c
...@@ -621,6 +621,20 @@ static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid) ...@@ -621,6 +621,20 @@ static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
return page; return page;
} }
/*
* common helper function for hstate_next_node_to_{alloc|free}.
* return next node in node_online_map, wrapping at end.
*/
static int next_node_allowed(int nid)
{
nid = next_node(nid, node_online_map);
if (nid == MAX_NUMNODES)
nid = first_node(node_online_map);
VM_BUG_ON(nid >= MAX_NUMNODES);
return nid;
}
/* /*
* Use a helper variable to find the next node and then * Use a helper variable to find the next node and then
* copy it back to next_nid_to_alloc afterwards: * copy it back to next_nid_to_alloc afterwards:
...@@ -634,12 +648,12 @@ static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid) ...@@ -634,12 +648,12 @@ static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
*/ */
static int hstate_next_node_to_alloc(struct hstate *h) static int hstate_next_node_to_alloc(struct hstate *h)
{ {
int next_nid; int nid, next_nid;
next_nid = next_node(h->next_nid_to_alloc, node_online_map);
if (next_nid == MAX_NUMNODES) nid = h->next_nid_to_alloc;
next_nid = first_node(node_online_map); next_nid = next_node_allowed(nid);
h->next_nid_to_alloc = next_nid; h->next_nid_to_alloc = next_nid;
return next_nid; return nid;
} }
static int alloc_fresh_huge_page(struct hstate *h) static int alloc_fresh_huge_page(struct hstate *h)
...@@ -649,15 +663,17 @@ static int alloc_fresh_huge_page(struct hstate *h) ...@@ -649,15 +663,17 @@ static int alloc_fresh_huge_page(struct hstate *h)
int next_nid; int next_nid;
int ret = 0; int ret = 0;
start_nid = h->next_nid_to_alloc; start_nid = hstate_next_node_to_alloc(h);
next_nid = start_nid; next_nid = start_nid;
do { do {
page = alloc_fresh_huge_page_node(h, next_nid); page = alloc_fresh_huge_page_node(h, next_nid);
if (page) if (page) {
ret = 1; ret = 1;
break;
}
next_nid = hstate_next_node_to_alloc(h); next_nid = hstate_next_node_to_alloc(h);
} while (!page && next_nid != start_nid); } while (next_nid != start_nid);
if (ret) if (ret)
count_vm_event(HTLB_BUDDY_PGALLOC); count_vm_event(HTLB_BUDDY_PGALLOC);
...@@ -668,17 +684,19 @@ static int alloc_fresh_huge_page(struct hstate *h) ...@@ -668,17 +684,19 @@ static int alloc_fresh_huge_page(struct hstate *h)
} }
/* /*
* helper for free_pool_huge_page() - find next node * helper for free_pool_huge_page() - return the next node
* from which to free a huge page * from which to free a huge page. Advance the next node id
* whether or not we find a free huge page to free so that the
* next attempt to free addresses the next node.
*/ */
static int hstate_next_node_to_free(struct hstate *h) static int hstate_next_node_to_free(struct hstate *h)
{ {
int next_nid; int nid, next_nid;
next_nid = next_node(h->next_nid_to_free, node_online_map);
if (next_nid == MAX_NUMNODES) nid = h->next_nid_to_free;
next_nid = first_node(node_online_map); next_nid = next_node_allowed(nid);
h->next_nid_to_free = next_nid; h->next_nid_to_free = next_nid;
return next_nid; return nid;
} }
/* /*
...@@ -693,7 +711,7 @@ static int free_pool_huge_page(struct hstate *h, bool acct_surplus) ...@@ -693,7 +711,7 @@ static int free_pool_huge_page(struct hstate *h, bool acct_surplus)
int next_nid; int next_nid;
int ret = 0; int ret = 0;
start_nid = h->next_nid_to_free; start_nid = hstate_next_node_to_free(h);
next_nid = start_nid; next_nid = start_nid;
do { do {
...@@ -715,9 +733,10 @@ static int free_pool_huge_page(struct hstate *h, bool acct_surplus) ...@@ -715,9 +733,10 @@ static int free_pool_huge_page(struct hstate *h, bool acct_surplus)
} }
update_and_free_page(h, page); update_and_free_page(h, page);
ret = 1; ret = 1;
break;
} }
next_nid = hstate_next_node_to_free(h); next_nid = hstate_next_node_to_free(h);
} while (!ret && next_nid != start_nid); } while (next_nid != start_nid);
return ret; return ret;
} }
...@@ -1028,10 +1047,9 @@ int __weak alloc_bootmem_huge_page(struct hstate *h) ...@@ -1028,10 +1047,9 @@ int __weak alloc_bootmem_huge_page(struct hstate *h)
void *addr; void *addr;
addr = __alloc_bootmem_node_nopanic( addr = __alloc_bootmem_node_nopanic(
NODE_DATA(h->next_nid_to_alloc), NODE_DATA(hstate_next_node_to_alloc(h)),
huge_page_size(h), huge_page_size(h), 0); huge_page_size(h), huge_page_size(h), 0);
hstate_next_node_to_alloc(h);
if (addr) { if (addr) {
/* /*
* Use the beginning of the huge page to store the * Use the beginning of the huge page to store the
...@@ -1167,29 +1185,31 @@ static int adjust_pool_surplus(struct hstate *h, int delta) ...@@ -1167,29 +1185,31 @@ static int adjust_pool_surplus(struct hstate *h, int delta)
VM_BUG_ON(delta != -1 && delta != 1); VM_BUG_ON(delta != -1 && delta != 1);
if (delta < 0) if (delta < 0)
start_nid = h->next_nid_to_alloc; start_nid = hstate_next_node_to_alloc(h);
else else
start_nid = h->next_nid_to_free; start_nid = hstate_next_node_to_free(h);
next_nid = start_nid; next_nid = start_nid;
do { do {
int nid = next_nid; int nid = next_nid;
if (delta < 0) { if (delta < 0) {
next_nid = hstate_next_node_to_alloc(h);
/* /*
* To shrink on this node, there must be a surplus page * To shrink on this node, there must be a surplus page
*/ */
if (!h->surplus_huge_pages_node[nid]) if (!h->surplus_huge_pages_node[nid]) {
next_nid = hstate_next_node_to_alloc(h);
continue; continue;
}
} }
if (delta > 0) { if (delta > 0) {
next_nid = hstate_next_node_to_free(h);
/* /*
* Surplus cannot exceed the total number of pages * Surplus cannot exceed the total number of pages
*/ */
if (h->surplus_huge_pages_node[nid] >= if (h->surplus_huge_pages_node[nid] >=
h->nr_huge_pages_node[nid]) h->nr_huge_pages_node[nid]) {
next_nid = hstate_next_node_to_free(h);
continue; continue;
}
} }
h->surplus_huge_pages += delta; h->surplus_huge_pages += delta;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment