Commit bb2a0de9 authored by KAMEZAWA Hiroyuki's avatar KAMEZAWA Hiroyuki Committed by Linus Torvalds

memcg: consolidate memory cgroup lru stat functions

In mm/memcontrol.c, there are many lru stat functions as..

  mem_cgroup_zone_nr_lru_pages
  mem_cgroup_node_nr_file_lru_pages
  mem_cgroup_nr_file_lru_pages
  mem_cgroup_node_nr_anon_lru_pages
  mem_cgroup_nr_anon_lru_pages
  mem_cgroup_node_nr_unevictable_lru_pages
  mem_cgroup_nr_unevictable_lru_pages
  mem_cgroup_node_nr_lru_pages
  mem_cgroup_nr_lru_pages
  mem_cgroup_get_local_zonestat

Some of them are under #ifdef MAX_NUMNODES >1 and others are not.
This seems bad. This patch consolidates all functions into

  mem_cgroup_zone_nr_lru_pages()
  mem_cgroup_node_nr_lru_pages()
  mem_cgroup_nr_lru_pages()

For these functions, "which LRU?" information is passed by a mask.

example:
  mem_cgroup_nr_lru_pages(mem, BIT(LRU_ACTIVE_ANON))

And I added some macro as ALL_LRU, ALL_LRU_FILE, ALL_LRU_ANON.

example:
  mem_cgroup_nr_lru_pages(mem, ALL_LRU)

BTW, considering layout of NUMA memory placement of counters, this patch seems
to be better.

Now, when we gather all LRU information, we scan in following orer
    for_each_lru -> for_each_node -> for_each_zone.

This means we'll touch cache lines in different node in turn.

After patch, we'll scan
    for_each_node -> for_each_zone -> for_each_lru(mask)

Then, we'll gather information in the same cacheline at once.

[akpm@linux-foundation.org: fix warnigns, build error]
Signed-off-by: default avatarKAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Daisuke Nishimura <nishimura@mxp.nes.nec.co.jp>
Cc: Balbir Singh <bsingharora@gmail.com>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Ying Han <yinghan@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 1f4c025b
...@@ -111,8 +111,7 @@ int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg); ...@@ -111,8 +111,7 @@ int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg);
int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg); int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg);
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg); int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
unsigned long mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, unsigned long mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg,
struct zone *zone, int nid, int zid, unsigned int lrumask);
enum lru_list lru);
struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg, struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
struct zone *zone); struct zone *zone);
struct zone_reclaim_stat* struct zone_reclaim_stat*
...@@ -313,8 +312,8 @@ mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg) ...@@ -313,8 +312,8 @@ mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
} }
static inline unsigned long static inline unsigned long
mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, struct zone *zone, mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
enum lru_list lru) unsigned int lru_mask)
{ {
return 0; return 0;
} }
......
...@@ -158,6 +158,12 @@ static inline int is_unevictable_lru(enum lru_list l) ...@@ -158,6 +158,12 @@ static inline int is_unevictable_lru(enum lru_list l)
return (l == LRU_UNEVICTABLE); return (l == LRU_UNEVICTABLE);
} }
/* Mask used at gathering information at once (see memcontrol.c) */
#define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
#define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
#define LRU_ALL_EVICTABLE (LRU_ALL_FILE | LRU_ALL_ANON)
#define LRU_ALL ((1 << NR_LRU_LISTS) - 1)
enum zone_watermarks { enum zone_watermarks {
WMARK_MIN, WMARK_MIN,
WMARK_LOW, WMARK_LOW,
......
...@@ -636,27 +636,44 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, ...@@ -636,27 +636,44 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
preempt_enable(); preempt_enable();
} }
static unsigned long unsigned long
mem_cgroup_get_zonestat_node(struct mem_cgroup *mem, int nid, enum lru_list idx) mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *mem, int nid, int zid,
unsigned int lru_mask)
{ {
struct mem_cgroup_per_zone *mz; struct mem_cgroup_per_zone *mz;
enum lru_list l;
unsigned long ret = 0;
mz = mem_cgroup_zoneinfo(mem, nid, zid);
for_each_lru(l) {
if (BIT(l) & lru_mask)
ret += MEM_CGROUP_ZSTAT(mz, l);
}
return ret;
}
static unsigned long
mem_cgroup_node_nr_lru_pages(struct mem_cgroup *mem,
int nid, unsigned int lru_mask)
{
u64 total = 0; u64 total = 0;
int zid; int zid;
for (zid = 0; zid < MAX_NR_ZONES; zid++) { for (zid = 0; zid < MAX_NR_ZONES; zid++)
mz = mem_cgroup_zoneinfo(mem, nid, zid); total += mem_cgroup_zone_nr_lru_pages(mem, nid, zid, lru_mask);
total += MEM_CGROUP_ZSTAT(mz, idx);
}
return total; return total;
} }
static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem,
enum lru_list idx) static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *mem,
unsigned int lru_mask)
{ {
int nid; int nid;
u64 total = 0; u64 total = 0;
for_each_online_node(nid) for_each_node_state(nid, N_HIGH_MEMORY)
total += mem_cgroup_get_zonestat_node(mem, nid, idx); total += mem_cgroup_node_nr_lru_pages(mem, nid, lru_mask);
return total; return total;
} }
...@@ -1077,8 +1094,8 @@ static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_ ...@@ -1077,8 +1094,8 @@ static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_
unsigned long gb; unsigned long gb;
unsigned long inactive_ratio; unsigned long inactive_ratio;
inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON); inactive = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_INACTIVE_ANON));
active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON); active = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ACTIVE_ANON));
gb = (inactive + active) >> (30 - PAGE_SHIFT); gb = (inactive + active) >> (30 - PAGE_SHIFT);
if (gb) if (gb)
...@@ -1117,109 +1134,12 @@ int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg) ...@@ -1117,109 +1134,12 @@ int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
unsigned long active; unsigned long active;
unsigned long inactive; unsigned long inactive;
inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE); inactive = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_INACTIVE_FILE));
active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE); active = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ACTIVE_FILE));
return (active > inactive); return (active > inactive);
} }
unsigned long mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg,
struct zone *zone,
enum lru_list lru)
{
int nid = zone_to_nid(zone);
int zid = zone_idx(zone);
struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
return MEM_CGROUP_ZSTAT(mz, lru);
}
static unsigned long mem_cgroup_node_nr_file_lru_pages(struct mem_cgroup *memcg,
int nid)
{
unsigned long ret;
ret = mem_cgroup_get_zonestat_node(memcg, nid, LRU_INACTIVE_FILE) +
mem_cgroup_get_zonestat_node(memcg, nid, LRU_ACTIVE_FILE);
return ret;
}
static unsigned long mem_cgroup_node_nr_anon_lru_pages(struct mem_cgroup *memcg,
int nid)
{
unsigned long ret;
ret = mem_cgroup_get_zonestat_node(memcg, nid, LRU_INACTIVE_ANON) +
mem_cgroup_get_zonestat_node(memcg, nid, LRU_ACTIVE_ANON);
return ret;
}
#if MAX_NUMNODES > 1
static unsigned long mem_cgroup_nr_file_lru_pages(struct mem_cgroup *memcg)
{
u64 total = 0;
int nid;
for_each_node_state(nid, N_HIGH_MEMORY)
total += mem_cgroup_node_nr_file_lru_pages(memcg, nid);
return total;
}
static unsigned long mem_cgroup_nr_anon_lru_pages(struct mem_cgroup *memcg)
{
u64 total = 0;
int nid;
for_each_node_state(nid, N_HIGH_MEMORY)
total += mem_cgroup_node_nr_anon_lru_pages(memcg, nid);
return total;
}
static unsigned long
mem_cgroup_node_nr_unevictable_lru_pages(struct mem_cgroup *memcg, int nid)
{
return mem_cgroup_get_zonestat_node(memcg, nid, LRU_UNEVICTABLE);
}
static unsigned long
mem_cgroup_nr_unevictable_lru_pages(struct mem_cgroup *memcg)
{
u64 total = 0;
int nid;
for_each_node_state(nid, N_HIGH_MEMORY)
total += mem_cgroup_node_nr_unevictable_lru_pages(memcg, nid);
return total;
}
static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
int nid)
{
enum lru_list l;
u64 total = 0;
for_each_lru(l)
total += mem_cgroup_get_zonestat_node(memcg, nid, l);
return total;
}
static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg)
{
u64 total = 0;
int nid;
for_each_node_state(nid, N_HIGH_MEMORY)
total += mem_cgroup_node_nr_lru_pages(memcg, nid);
return total;
}
#endif /* CONFIG_NUMA */
struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg, struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
struct zone *zone) struct zone *zone)
{ {
...@@ -1576,11 +1496,11 @@ mem_cgroup_select_victim(struct mem_cgroup *root_mem) ...@@ -1576,11 +1496,11 @@ mem_cgroup_select_victim(struct mem_cgroup *root_mem)
static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *mem, static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *mem,
int nid, bool noswap) int nid, bool noswap)
{ {
if (mem_cgroup_node_nr_file_lru_pages(mem, nid)) if (mem_cgroup_node_nr_lru_pages(mem, nid, LRU_ALL_FILE))
return true; return true;
if (noswap || !total_swap_pages) if (noswap || !total_swap_pages)
return false; return false;
if (mem_cgroup_node_nr_anon_lru_pages(mem, nid)) if (mem_cgroup_node_nr_lru_pages(mem, nid, LRU_ALL_ANON))
return true; return true;
return false; return false;
...@@ -4151,15 +4071,15 @@ mem_cgroup_get_local_stat(struct mem_cgroup *mem, struct mcs_total_stat *s) ...@@ -4151,15 +4071,15 @@ mem_cgroup_get_local_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
s->stat[MCS_PGMAJFAULT] += val; s->stat[MCS_PGMAJFAULT] += val;
/* per zone stat */ /* per zone stat */
val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON); val = mem_cgroup_nr_lru_pages(mem, BIT(LRU_INACTIVE_ANON));
s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE; s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE;
val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON); val = mem_cgroup_nr_lru_pages(mem, BIT(LRU_ACTIVE_ANON));
s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE; s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE;
val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE); val = mem_cgroup_nr_lru_pages(mem, BIT(LRU_INACTIVE_FILE));
s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE; s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE;
val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE); val = mem_cgroup_nr_lru_pages(mem, BIT(LRU_ACTIVE_FILE));
s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE; s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE); val = mem_cgroup_nr_lru_pages(mem, BIT(LRU_UNEVICTABLE));
s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE; s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
} }
...@@ -4181,35 +4101,37 @@ static int mem_control_numa_stat_show(struct seq_file *m, void *arg) ...@@ -4181,35 +4101,37 @@ static int mem_control_numa_stat_show(struct seq_file *m, void *arg)
struct cgroup *cont = m->private; struct cgroup *cont = m->private;
struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont); struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
total_nr = mem_cgroup_nr_lru_pages(mem_cont); total_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL);
seq_printf(m, "total=%lu", total_nr); seq_printf(m, "total=%lu", total_nr);
for_each_node_state(nid, N_HIGH_MEMORY) { for_each_node_state(nid, N_HIGH_MEMORY) {
node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid); node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid, LRU_ALL);
seq_printf(m, " N%d=%lu", nid, node_nr); seq_printf(m, " N%d=%lu", nid, node_nr);
} }
seq_putc(m, '\n'); seq_putc(m, '\n');
file_nr = mem_cgroup_nr_file_lru_pages(mem_cont); file_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL_FILE);
seq_printf(m, "file=%lu", file_nr); seq_printf(m, "file=%lu", file_nr);
for_each_node_state(nid, N_HIGH_MEMORY) { for_each_node_state(nid, N_HIGH_MEMORY) {
node_nr = mem_cgroup_node_nr_file_lru_pages(mem_cont, nid); node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid,
LRU_ALL_FILE);
seq_printf(m, " N%d=%lu", nid, node_nr); seq_printf(m, " N%d=%lu", nid, node_nr);
} }
seq_putc(m, '\n'); seq_putc(m, '\n');
anon_nr = mem_cgroup_nr_anon_lru_pages(mem_cont); anon_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL_ANON);
seq_printf(m, "anon=%lu", anon_nr); seq_printf(m, "anon=%lu", anon_nr);
for_each_node_state(nid, N_HIGH_MEMORY) { for_each_node_state(nid, N_HIGH_MEMORY) {
node_nr = mem_cgroup_node_nr_anon_lru_pages(mem_cont, nid); node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid,
LRU_ALL_ANON);
seq_printf(m, " N%d=%lu", nid, node_nr); seq_printf(m, " N%d=%lu", nid, node_nr);
} }
seq_putc(m, '\n'); seq_putc(m, '\n');
unevictable_nr = mem_cgroup_nr_unevictable_lru_pages(mem_cont); unevictable_nr = mem_cgroup_nr_lru_pages(mem_cont, BIT(LRU_UNEVICTABLE));
seq_printf(m, "unevictable=%lu", unevictable_nr); seq_printf(m, "unevictable=%lu", unevictable_nr);
for_each_node_state(nid, N_HIGH_MEMORY) { for_each_node_state(nid, N_HIGH_MEMORY) {
node_nr = mem_cgroup_node_nr_unevictable_lru_pages(mem_cont, node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid,
nid); BIT(LRU_UNEVICTABLE));
seq_printf(m, " N%d=%lu", nid, node_nr); seq_printf(m, " N%d=%lu", nid, node_nr);
} }
seq_putc(m, '\n'); seq_putc(m, '\n');
......
...@@ -171,7 +171,8 @@ static unsigned long zone_nr_lru_pages(struct zone *zone, ...@@ -171,7 +171,8 @@ static unsigned long zone_nr_lru_pages(struct zone *zone,
struct scan_control *sc, enum lru_list lru) struct scan_control *sc, enum lru_list lru)
{ {
if (!scanning_global_lru(sc)) if (!scanning_global_lru(sc))
return mem_cgroup_zone_nr_lru_pages(sc->mem_cgroup, zone, lru); return mem_cgroup_zone_nr_lru_pages(sc->mem_cgroup,
zone_to_nid(zone), zone_idx(zone), BIT(lru));
return zone_page_state(zone, NR_LRU_BASE + lru); return zone_page_state(zone, NR_LRU_BASE + lru);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment