Commit 7be62de9 authored by Rik van Riel's avatar Rik van Riel Committed by Linus Torvalds

vmscan: kswapd carefully call compaction

With CONFIG_COMPACTION enabled, kswapd does not try to free contiguous
free pages, even when it is woken for a higher order request.

This could be bad for eg.  jumbo frame network allocations, which are done
from interrupt context and cannot compact memory themselves.  Higher than
before allocation failure rates in the network receive path have been
observed in kernels with compaction enabled.

Teach kswapd to defragment the memory zones in a node, but only if
required and compaction is not deferred in a zone.

[akpm@linux-foundation.org: reduce scope of zones_need_compaction]
Signed-off-by: default avatarRik van Riel <riel@redhat.com>
Acked-by: default avatarMel Gorman <mel@csn.ul.ie>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Hillf Danton <dhillf@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent fe2c2a10
...@@ -23,6 +23,7 @@ extern int fragmentation_index(struct zone *zone, unsigned int order); ...@@ -23,6 +23,7 @@ extern int fragmentation_index(struct zone *zone, unsigned int order);
extern unsigned long try_to_compact_pages(struct zonelist *zonelist, extern unsigned long try_to_compact_pages(struct zonelist *zonelist,
int order, gfp_t gfp_mask, nodemask_t *mask, int order, gfp_t gfp_mask, nodemask_t *mask,
bool sync); bool sync);
extern int compact_pgdat(pg_data_t *pgdat, int order);
extern unsigned long compaction_suitable(struct zone *zone, int order); extern unsigned long compaction_suitable(struct zone *zone, int order);
/* Do not skip compaction more than 64 times */ /* Do not skip compaction more than 64 times */
...@@ -62,6 +63,11 @@ static inline unsigned long try_to_compact_pages(struct zonelist *zonelist, ...@@ -62,6 +63,11 @@ static inline unsigned long try_to_compact_pages(struct zonelist *zonelist,
return COMPACT_CONTINUE; return COMPACT_CONTINUE;
} }
static inline int compact_pgdat(pg_data_t *pgdat, int order)
{
return COMPACT_CONTINUE;
}
static inline unsigned long compaction_suitable(struct zone *zone, int order) static inline unsigned long compaction_suitable(struct zone *zone, int order)
{ {
return COMPACT_SKIPPED; return COMPACT_SKIPPED;
......
...@@ -675,44 +675,61 @@ unsigned long try_to_compact_pages(struct zonelist *zonelist, ...@@ -675,44 +675,61 @@ unsigned long try_to_compact_pages(struct zonelist *zonelist,
/* Compact all zones within a node */ /* Compact all zones within a node */
static int compact_node(int nid) static int __compact_pgdat(pg_data_t *pgdat, struct compact_control *cc)
{ {
int zoneid; int zoneid;
pg_data_t *pgdat;
struct zone *zone; struct zone *zone;
if (nid < 0 || nid >= nr_node_ids || !node_online(nid))
return -EINVAL;
pgdat = NODE_DATA(nid);
/* Flush pending updates to the LRU lists */ /* Flush pending updates to the LRU lists */
lru_add_drain_all(); lru_add_drain_all();
for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
struct compact_control cc = {
.nr_freepages = 0,
.nr_migratepages = 0,
.order = -1,
.sync = true,
};
zone = &pgdat->node_zones[zoneid]; zone = &pgdat->node_zones[zoneid];
if (!populated_zone(zone)) if (!populated_zone(zone))
continue; continue;
cc.zone = zone; cc->nr_freepages = 0;
INIT_LIST_HEAD(&cc.freepages); cc->nr_migratepages = 0;
INIT_LIST_HEAD(&cc.migratepages); cc->zone = zone;
INIT_LIST_HEAD(&cc->freepages);
INIT_LIST_HEAD(&cc->migratepages);
compact_zone(zone, &cc); if (cc->order < 0 || !compaction_deferred(zone))
compact_zone(zone, cc);
VM_BUG_ON(!list_empty(&cc.freepages)); VM_BUG_ON(!list_empty(&cc->freepages));
VM_BUG_ON(!list_empty(&cc.migratepages)); VM_BUG_ON(!list_empty(&cc->migratepages));
} }
return 0; return 0;
} }
int compact_pgdat(pg_data_t *pgdat, int order)
{
struct compact_control cc = {
.order = order,
.sync = false,
};
return __compact_pgdat(pgdat, &cc);
}
static int compact_node(int nid)
{
pg_data_t *pgdat;
struct compact_control cc = {
.order = -1,
.sync = true,
};
if (nid < 0 || nid >= nr_node_ids || !node_online(nid))
return -EINVAL;
pgdat = NODE_DATA(nid);
return __compact_pgdat(pgdat, &cc);
}
/* Compact all nodes in the system */ /* Compact all nodes in the system */
static int compact_nodes(void) static int compact_nodes(void)
{ {
......
...@@ -2919,6 +2919,8 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order, ...@@ -2919,6 +2919,8 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order,
* and it is potentially going to sleep here. * and it is potentially going to sleep here.
*/ */
if (order) { if (order) {
int zones_need_compaction = 1;
for (i = 0; i <= end_zone; i++) { for (i = 0; i <= end_zone; i++) {
struct zone *zone = pgdat->node_zones + i; struct zone *zone = pgdat->node_zones + i;
...@@ -2939,9 +2941,17 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order, ...@@ -2939,9 +2941,17 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order,
goto loop_again; goto loop_again;
} }
/* Check if the memory needs to be defragmented. */
if (zone_watermark_ok(zone, order,
low_wmark_pages(zone), *classzone_idx, 0))
zones_need_compaction = 0;
/* If balanced, clear the congested flag */ /* If balanced, clear the congested flag */
zone_clear_flag(zone, ZONE_CONGESTED); zone_clear_flag(zone, ZONE_CONGESTED);
} }
if (zones_need_compaction)
compact_pgdat(pgdat, order);
} }
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment