Commit 4518085e authored by Kemi Wang's avatar Kemi Wang Committed by Linus Torvalds

mm, sysctl: make NUMA stats configurable

This is the second step which introduces a tunable interface that allow
numa stats configurable for optimizing zone_statistics(), as suggested
by Dave Hansen and Ying Huang.

=========================================================================

When page allocation performance becomes a bottleneck and you can
tolerate some possible tool breakage and decreased numa counter
precision, you can do:

	echo 0 > /proc/sys/vm/numa_stat

In this case, numa counter update is ignored.  We can see about
*4.8%*(185->176) drop of cpu cycles per single page allocation and
reclaim on Jesper's page_bench01 (single thread) and *8.1%*(343->315)
drop of cpu cycles per single page allocation and reclaim on Jesper's
page_bench03 (88 threads) running on a 2-Socket Broadwell-based server
(88 threads, 126G memory).

Benchmark link provided by Jesper D Brouer (increase loop times to
10000000):

  https://github.com/netoptimizer/prototype-kernel/tree/master/kernel/mm/bench

=========================================================================

When page allocation performance is not a bottleneck and you want all
tooling to work, you can do:

	echo 1 > /proc/sys/vm/numa_stat

This is system default setting.

Many thanks to Michal Hocko, Dave Hansen, Ying Huang and Vlastimil Babka
for comments to help improve the original patch.

[keescook@chromium.org: make sure mutex is a global static]
  Link: http://lkml.kernel.org/r/20171107213809.GA4314@beast
Link: http://lkml.kernel.org/r/1508290927-8518-1-git-send-email-kemi.wang@intel.comSigned-off-by: default avatarKemi Wang <kemi.wang@intel.com>
Signed-off-by: default avatarKees Cook <keescook@chromium.org>
Reported-by: default avatarJesper Dangaard Brouer <brouer@redhat.com>
Suggested-by: default avatarDave Hansen <dave.hansen@intel.com>
Suggested-by: default avatarYing Huang <ying.huang@intel.com>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Acked-by: default avatarMichal Hocko <mhocko@suse.com>
Cc: "Luis R . Rodriguez" <mcgrof@kernel.org>
Cc: Kees Cook <keescook@chromium.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Christopher Lameter <cl@linux.com>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Andrey Ryabinin <aryabinin@virtuozzo.com>
Cc: Tim Chen <tim.c.chen@intel.com>
Cc: Andi Kleen <andi.kleen@intel.com>
Cc: Aaron Lu <aaron.lu@intel.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 9a8ec03e
...@@ -58,6 +58,7 @@ Currently, these files are in /proc/sys/vm: ...@@ -58,6 +58,7 @@ Currently, these files are in /proc/sys/vm:
- percpu_pagelist_fraction - percpu_pagelist_fraction
- stat_interval - stat_interval
- stat_refresh - stat_refresh
- numa_stat
- swappiness - swappiness
- user_reserve_kbytes - user_reserve_kbytes
- vfs_cache_pressure - vfs_cache_pressure
...@@ -799,6 +800,21 @@ with no ill effects: errors and warnings on these stats are suppressed.) ...@@ -799,6 +800,21 @@ with no ill effects: errors and warnings on these stats are suppressed.)
============================================================== ==============================================================
numa_stat
This interface allows runtime configuration of numa statistics.
When page allocation performance becomes a bottleneck and you can tolerate
some possible tool breakage and decreased numa counter precision, you can
do:
echo 0 > /proc/sys/vm/numa_stat
When page allocation performance is not a bottleneck and you want all
tooling to work, you can do:
echo 1 > /proc/sys/vm/numa_stat
==============================================================
swappiness swappiness
This control is used to define how aggressive the kernel will swap This control is used to define how aggressive the kernel will swap
......
...@@ -7,9 +7,19 @@ ...@@ -7,9 +7,19 @@
#include <linux/mmzone.h> #include <linux/mmzone.h>
#include <linux/vm_event_item.h> #include <linux/vm_event_item.h>
#include <linux/atomic.h> #include <linux/atomic.h>
#include <linux/static_key.h>
extern int sysctl_stat_interval; extern int sysctl_stat_interval;
#ifdef CONFIG_NUMA
#define ENABLE_NUMA_STAT 1
#define DISABLE_NUMA_STAT 0
extern int sysctl_vm_numa_stat;
DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
extern int sysctl_vm_numa_stat_handler(struct ctl_table *table,
int write, void __user *buffer, size_t *length, loff_t *ppos);
#endif
#ifdef CONFIG_VM_EVENT_COUNTERS #ifdef CONFIG_VM_EVENT_COUNTERS
/* /*
* Light weight per cpu counter implementation. * Light weight per cpu counter implementation.
......
...@@ -1356,6 +1356,15 @@ static struct ctl_table vm_table[] = { ...@@ -1356,6 +1356,15 @@ static struct ctl_table vm_table[] = {
.mode = 0644, .mode = 0644,
.proc_handler = &hugetlb_mempolicy_sysctl_handler, .proc_handler = &hugetlb_mempolicy_sysctl_handler,
}, },
{
.procname = "numa_stat",
.data = &sysctl_vm_numa_stat,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = sysctl_vm_numa_stat_handler,
.extra1 = &zero,
.extra2 = &one,
},
#endif #endif
{ {
.procname = "hugetlb_shm_group", .procname = "hugetlb_shm_group",
......
...@@ -1915,6 +1915,9 @@ static struct page *alloc_page_interleave(gfp_t gfp, unsigned order, ...@@ -1915,6 +1915,9 @@ static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
struct page *page; struct page *page;
page = __alloc_pages(gfp, order, nid); page = __alloc_pages(gfp, order, nid);
/* skip NUMA_INTERLEAVE_HIT counter update if numa stats is disabled */
if (!static_branch_likely(&vm_numa_stat_key))
return page;
if (page && page_to_nid(page) == nid) { if (page && page_to_nid(page) == nid) {
preempt_disable(); preempt_disable();
__inc_numa_state(page_zone(page), NUMA_INTERLEAVE_HIT); __inc_numa_state(page_zone(page), NUMA_INTERLEAVE_HIT);
......
...@@ -82,6 +82,8 @@ DEFINE_PER_CPU(int, numa_node); ...@@ -82,6 +82,8 @@ DEFINE_PER_CPU(int, numa_node);
EXPORT_PER_CPU_SYMBOL(numa_node); EXPORT_PER_CPU_SYMBOL(numa_node);
#endif #endif
DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);
#ifdef CONFIG_HAVE_MEMORYLESS_NODES #ifdef CONFIG_HAVE_MEMORYLESS_NODES
/* /*
* N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
...@@ -2777,6 +2779,10 @@ static inline void zone_statistics(struct zone *preferred_zone, struct zone *z) ...@@ -2777,6 +2779,10 @@ static inline void zone_statistics(struct zone *preferred_zone, struct zone *z)
#ifdef CONFIG_NUMA #ifdef CONFIG_NUMA
enum numa_stat_item local_stat = NUMA_LOCAL; enum numa_stat_item local_stat = NUMA_LOCAL;
/* skip numa counters update if numa stats is disabled */
if (!static_branch_likely(&vm_numa_stat_key))
return;
if (z->node != numa_node_id()) if (z->node != numa_node_id())
local_stat = NUMA_OTHER; local_stat = NUMA_OTHER;
......
...@@ -32,6 +32,77 @@ ...@@ -32,6 +32,77 @@
#define NUMA_STATS_THRESHOLD (U16_MAX - 2) #define NUMA_STATS_THRESHOLD (U16_MAX - 2)
#ifdef CONFIG_NUMA
int sysctl_vm_numa_stat = ENABLE_NUMA_STAT;
/* zero numa counters within a zone */
static void zero_zone_numa_counters(struct zone *zone)
{
int item, cpu;
for (item = 0; item < NR_VM_NUMA_STAT_ITEMS; item++) {
atomic_long_set(&zone->vm_numa_stat[item], 0);
for_each_online_cpu(cpu)
per_cpu_ptr(zone->pageset, cpu)->vm_numa_stat_diff[item]
= 0;
}
}
/* zero numa counters of all the populated zones */
static void zero_zones_numa_counters(void)
{
struct zone *zone;
for_each_populated_zone(zone)
zero_zone_numa_counters(zone);
}
/* zero global numa counters */
static void zero_global_numa_counters(void)
{
int item;
for (item = 0; item < NR_VM_NUMA_STAT_ITEMS; item++)
atomic_long_set(&vm_numa_stat[item], 0);
}
static void invalid_numa_statistics(void)
{
zero_zones_numa_counters();
zero_global_numa_counters();
}
static DEFINE_MUTEX(vm_numa_stat_lock);
int sysctl_vm_numa_stat_handler(struct ctl_table *table, int write,
void __user *buffer, size_t *length, loff_t *ppos)
{
int ret, oldval;
mutex_lock(&vm_numa_stat_lock);
if (write)
oldval = sysctl_vm_numa_stat;
ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
if (ret || !write)
goto out;
if (oldval == sysctl_vm_numa_stat)
goto out;
else if (sysctl_vm_numa_stat == ENABLE_NUMA_STAT) {
static_branch_enable(&vm_numa_stat_key);
pr_info("enable numa statistics\n");
} else {
static_branch_disable(&vm_numa_stat_key);
invalid_numa_statistics();
pr_info("disable numa statistics, and clear numa counters\n");
}
out:
mutex_unlock(&vm_numa_stat_lock);
return ret;
}
#endif
#ifdef CONFIG_VM_EVENT_COUNTERS #ifdef CONFIG_VM_EVENT_COUNTERS
DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}}; DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}};
EXPORT_PER_CPU_SYMBOL(vm_event_states); EXPORT_PER_CPU_SYMBOL(vm_event_states);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment