Commit 78afd561 authored by Andi Kleen's avatar Andi Kleen Committed by Linus Torvalds

mm: add __GFP_OTHER_NODE flag

Add a new __GFP_OTHER_NODE flag to tell the low level numa statistics in
zone_statistics() that an allocation is on behalf of another thread.  This
way the local and remote counters can be still correct, even when
background daemons like khugepaged are changing memory mappings.

This only affects the accounting, but I think it's worth doing that right
to avoid confusing users.

I first tried to just pass down the right node, but this required a lot of
changes to pass down this parameter and at least one addition of a 10th
argument to a 9 argument function.  Using the flag is a lot less
intrusive.

Open: should be also used for migration?

[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: default avatarAndi Kleen <ak@linux.intel.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Reviewed-by: default avatarKAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 11bc82d6
...@@ -35,6 +35,7 @@ struct vm_area_struct; ...@@ -35,6 +35,7 @@ struct vm_area_struct;
#define ___GFP_NOTRACK 0 #define ___GFP_NOTRACK 0
#endif #endif
#define ___GFP_NO_KSWAPD 0x400000u #define ___GFP_NO_KSWAPD 0x400000u
#define ___GFP_OTHER_NODE 0x800000u
/* /*
* GFP bitmasks.. * GFP bitmasks..
...@@ -83,6 +84,7 @@ struct vm_area_struct; ...@@ -83,6 +84,7 @@ struct vm_area_struct;
#define __GFP_NOTRACK ((__force gfp_t)___GFP_NOTRACK) /* Don't track with kmemcheck */ #define __GFP_NOTRACK ((__force gfp_t)___GFP_NOTRACK) /* Don't track with kmemcheck */
#define __GFP_NO_KSWAPD ((__force gfp_t)___GFP_NO_KSWAPD) #define __GFP_NO_KSWAPD ((__force gfp_t)___GFP_NO_KSWAPD)
#define __GFP_OTHER_NODE ((__force gfp_t)___GFP_OTHER_NODE) /* On behalf of other node */
/* /*
* This may seem redundant, but it's a way of annotating false positives vs. * This may seem redundant, but it's a way of annotating false positives vs.
......
...@@ -220,12 +220,12 @@ static inline unsigned long node_page_state(int node, ...@@ -220,12 +220,12 @@ static inline unsigned long node_page_state(int node,
zone_page_state(&zones[ZONE_MOVABLE], item); zone_page_state(&zones[ZONE_MOVABLE], item);
} }
extern void zone_statistics(struct zone *, struct zone *); extern void zone_statistics(struct zone *, struct zone *, gfp_t gfp);
#else #else
#define node_page_state(node, item) global_page_state(item) #define node_page_state(node, item) global_page_state(item)
#define zone_statistics(_zl,_z) do { } while (0) #define zone_statistics(_zl, _z, gfp) do { } while (0)
#endif /* CONFIG_NUMA */ #endif /* CONFIG_NUMA */
......
...@@ -1337,7 +1337,7 @@ struct page *buffered_rmqueue(struct zone *preferred_zone, ...@@ -1337,7 +1337,7 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
} }
__count_zone_vm_events(PGALLOC, zone, 1 << order); __count_zone_vm_events(PGALLOC, zone, 1 << order);
zone_statistics(preferred_zone, zone); zone_statistics(preferred_zone, zone, gfp_flags);
local_irq_restore(flags); local_irq_restore(flags);
VM_BUG_ON(bad_range(zone, page)); VM_BUG_ON(bad_range(zone, page));
......
...@@ -500,8 +500,12 @@ void refresh_cpu_vm_stats(int cpu) ...@@ -500,8 +500,12 @@ void refresh_cpu_vm_stats(int cpu)
* z = the zone from which the allocation occurred. * z = the zone from which the allocation occurred.
* *
* Must be called with interrupts disabled. * Must be called with interrupts disabled.
*
* When __GFP_OTHER_NODE is set assume the node of the preferred
* zone is the local node. This is useful for daemons who allocate
* memory on behalf of other processes.
*/ */
void zone_statistics(struct zone *preferred_zone, struct zone *z) void zone_statistics(struct zone *preferred_zone, struct zone *z, gfp_t flags)
{ {
if (z->zone_pgdat == preferred_zone->zone_pgdat) { if (z->zone_pgdat == preferred_zone->zone_pgdat) {
__inc_zone_state(z, NUMA_HIT); __inc_zone_state(z, NUMA_HIT);
...@@ -509,7 +513,8 @@ void zone_statistics(struct zone *preferred_zone, struct zone *z) ...@@ -509,7 +513,8 @@ void zone_statistics(struct zone *preferred_zone, struct zone *z)
__inc_zone_state(z, NUMA_MISS); __inc_zone_state(z, NUMA_MISS);
__inc_zone_state(preferred_zone, NUMA_FOREIGN); __inc_zone_state(preferred_zone, NUMA_FOREIGN);
} }
if (z->node == numa_node_id()) if (z->node == ((flags & __GFP_OTHER_NODE) ?
preferred_zone->node : numa_node_id()))
__inc_zone_state(z, NUMA_LOCAL); __inc_zone_state(z, NUMA_LOCAL);
else else
__inc_zone_state(z, NUMA_OTHER); __inc_zone_state(z, NUMA_OTHER);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment