Commit 77d4459a authored by Yu Zhao's avatar Yu Zhao Committed by Andrew Morton

mm: multi-gen LRU: shuffle should_run_aging()

Move should_run_aging() next to its only caller left.

Link: https://lkml.kernel.org/r/20221222041905.2431096-6-yuzhao@google.comSigned-off-by: default avatarYu Zhao <yuzhao@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Michael Larabel <Michael@MichaelLarabel.com>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 7348cc91
...@@ -4467,68 +4467,6 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, ...@@ -4467,68 +4467,6 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
return true; return true;
} }
static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan)
{
int gen, type, zone;
unsigned long old = 0;
unsigned long young = 0;
unsigned long total = 0;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
DEFINE_MIN_SEQ(lruvec);
/* whether this lruvec is completely out of cold folios */
if (min_seq[!can_swap] + MIN_NR_GENS > max_seq) {
*nr_to_scan = 0;
return true;
}
for (type = !can_swap; type < ANON_AND_FILE; type++) {
unsigned long seq;
for (seq = min_seq[type]; seq <= max_seq; seq++) {
unsigned long size = 0;
gen = lru_gen_from_seq(seq);
for (zone = 0; zone < MAX_NR_ZONES; zone++)
size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
total += size;
if (seq == max_seq)
young += size;
else if (seq + MIN_NR_GENS == max_seq)
old += size;
}
}
/* try to scrape all its memory if this memcg was deleted */
*nr_to_scan = mem_cgroup_online(memcg) ? (total >> sc->priority) : total;
/*
* The aging tries to be lazy to reduce the overhead, while the eviction
* stalls when the number of generations reaches MIN_NR_GENS. Hence, the
* ideal number of generations is MIN_NR_GENS+1.
*/
if (min_seq[!can_swap] + MIN_NR_GENS < max_seq)
return false;
/*
* It's also ideal to spread pages out evenly, i.e., 1/(MIN_NR_GENS+1)
* of the total number of pages for each generation. A reasonable range
* for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The
* aging cares about the upper bound of hot pages, while the eviction
* cares about the lower bound of cold pages.
*/
if (young * MIN_NR_GENS > total)
return true;
if (old * (MIN_NR_GENS + 2) < total)
return true;
return false;
}
static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc) static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc)
{ {
int gen, type, zone; int gen, type, zone;
...@@ -5112,6 +5050,68 @@ static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swap ...@@ -5112,6 +5050,68 @@ static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swap
return scanned; return scanned;
} }
static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan)
{
int gen, type, zone;
unsigned long old = 0;
unsigned long young = 0;
unsigned long total = 0;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
DEFINE_MIN_SEQ(lruvec);
/* whether this lruvec is completely out of cold folios */
if (min_seq[!can_swap] + MIN_NR_GENS > max_seq) {
*nr_to_scan = 0;
return true;
}
for (type = !can_swap; type < ANON_AND_FILE; type++) {
unsigned long seq;
for (seq = min_seq[type]; seq <= max_seq; seq++) {
unsigned long size = 0;
gen = lru_gen_from_seq(seq);
for (zone = 0; zone < MAX_NR_ZONES; zone++)
size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
total += size;
if (seq == max_seq)
young += size;
else if (seq + MIN_NR_GENS == max_seq)
old += size;
}
}
/* try to scrape all its memory if this memcg was deleted */
*nr_to_scan = mem_cgroup_online(memcg) ? (total >> sc->priority) : total;
/*
* The aging tries to be lazy to reduce the overhead, while the eviction
* stalls when the number of generations reaches MIN_NR_GENS. Hence, the
* ideal number of generations is MIN_NR_GENS+1.
*/
if (min_seq[!can_swap] + MIN_NR_GENS < max_seq)
return false;
/*
* It's also ideal to spread pages out evenly, i.e., 1/(MIN_NR_GENS+1)
* of the total number of pages for each generation. A reasonable range
* for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The
* aging cares about the upper bound of hot pages, while the eviction
* cares about the lower bound of cold pages.
*/
if (young * MIN_NR_GENS > total)
return true;
if (old * (MIN_NR_GENS + 2) < total)
return true;
return false;
}
/* /*
* For future optimizations: * For future optimizations:
* 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment