Commit 31560180 authored by Minchan Kim's avatar Minchan Kim Committed by Linus Torvalds

mm: deactivate invalidated pages

Recently, there are reported problem about thrashing.
(http://marc.info/?l=rsync&m=128885034930933&w=2) It happens by backup
workloads(ex, nightly rsync).  That's because the workload makes just
use-once pages and touches pages twice.  It promotes the page into active
list so that it results in working set page eviction.

Some app developer want to support POSIX_FADV_NOREUSE.  But other OSes
don't support it, either.
(http://marc.info/?l=linux-mm&m=128928979512086&w=2)

By other approach, app developers use POSIX_FADV_DONTNEED.  But it has a
problem.  If kernel meets page is writing during invalidate_mapping_pages,
it can't work.  It makes for application programmer to use it since they
always have to sync data before calling fadivse(..POSIX_FADV_DONTNEED) to
make sure the pages could be discardable.  At last, they can't use
deferred write of kernel so that they could see performance loss.
(http://insights.oetiker.ch/linux/fadvise.html)

In fact, invalidation is very big hint to reclaimer.  It means we don't
use the page any more.  So let's move the writing page into inactive
list's head if we can't truncate it right now.

Why I move page to head of lru on this patch, Dirty/Writeback page would
be flushed sooner or later.  It can prevent writeout of pageout which is
less effective than flusher's writeout.

Originally, I reused lru_demote of Peter with some change so added his
Signed-off-by.
Signed-off-by: default avatarMinchan Kim <minchan.kim@gmail.com>
Reported-by: default avatarBen Gamari <bgamari.foss@gmail.com>
Signed-off-by: default avatarPeter Zijlstra <peterz@infradead.org>
Acked-by: default avatarRik van Riel <riel@redhat.com>
Acked-by: default avatarMel Gorman <mel@csn.ul.ie>
Reviewed-by: default avatarKOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Wu Fengguang <fengguang.wu@intel.com>
Acked-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Cc: Nick Piggin <npiggin@kernel.dk>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 481b4bb5
...@@ -215,6 +215,7 @@ extern void mark_page_accessed(struct page *); ...@@ -215,6 +215,7 @@ extern void mark_page_accessed(struct page *);
extern void lru_add_drain(void); extern void lru_add_drain(void);
extern int lru_add_drain_all(void); extern int lru_add_drain_all(void);
extern void rotate_reclaimable_page(struct page *page); extern void rotate_reclaimable_page(struct page *page);
extern void deactivate_page(struct page *page);
extern void swap_setup(void); extern void swap_setup(void);
extern void add_page_to_unevictable_list(struct page *page); extern void add_page_to_unevictable_list(struct page *page);
......
...@@ -39,6 +39,7 @@ int page_cluster; ...@@ -39,6 +39,7 @@ int page_cluster;
static DEFINE_PER_CPU(struct pagevec[NR_LRU_LISTS], lru_add_pvecs); static DEFINE_PER_CPU(struct pagevec[NR_LRU_LISTS], lru_add_pvecs);
static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs); static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
/* /*
* This path almost never happens for VM activity - pages are normally * This path almost never happens for VM activity - pages are normally
...@@ -346,6 +347,60 @@ void add_page_to_unevictable_list(struct page *page) ...@@ -346,6 +347,60 @@ void add_page_to_unevictable_list(struct page *page)
spin_unlock_irq(&zone->lru_lock); spin_unlock_irq(&zone->lru_lock);
} }
/*
* If the page can not be invalidated, it is moved to the
* inactive list to speed up its reclaim. It is moved to the
* head of the list, rather than the tail, to give the flusher
* threads some time to write it out, as this is much more
* effective than the single-page writeout from reclaim.
*/
static void lru_deactivate(struct page *page, struct zone *zone)
{
int lru, file;
if (!PageLRU(page) || !PageActive(page))
return;
/* Some processes are using the page */
if (page_mapped(page))
return;
file = page_is_file_cache(page);
lru = page_lru_base_type(page);
del_page_from_lru_list(zone, page, lru + LRU_ACTIVE);
ClearPageActive(page);
ClearPageReferenced(page);
add_page_to_lru_list(zone, page, lru);
__count_vm_event(PGDEACTIVATE);
update_page_reclaim_stat(zone, page, file, 0);
}
static void ____pagevec_lru_deactivate(struct pagevec *pvec)
{
int i;
struct zone *zone = NULL;
for (i = 0; i < pagevec_count(pvec); i++) {
struct page *page = pvec->pages[i];
struct zone *pagezone = page_zone(page);
if (pagezone != zone) {
if (zone)
spin_unlock_irq(&zone->lru_lock);
zone = pagezone;
spin_lock_irq(&zone->lru_lock);
}
lru_deactivate(page, zone);
}
if (zone)
spin_unlock_irq(&zone->lru_lock);
release_pages(pvec->pages, pvec->nr, pvec->cold);
pagevec_reinit(pvec);
}
/* /*
* Drain pages out of the cpu's pagevecs. * Drain pages out of the cpu's pagevecs.
* Either "cpu" is the current CPU, and preemption has already been * Either "cpu" is the current CPU, and preemption has already been
...@@ -372,6 +427,29 @@ static void drain_cpu_pagevecs(int cpu) ...@@ -372,6 +427,29 @@ static void drain_cpu_pagevecs(int cpu)
pagevec_move_tail(pvec); pagevec_move_tail(pvec);
local_irq_restore(flags); local_irq_restore(flags);
} }
pvec = &per_cpu(lru_deactivate_pvecs, cpu);
if (pagevec_count(pvec))
____pagevec_lru_deactivate(pvec);
}
/**
* deactivate_page - forcefully deactivate a page
* @page: page to deactivate
*
* This function hints the VM that @page is a good reclaim candidate,
* for example if its invalidation fails due to the page being dirty
* or under writeback.
*/
void deactivate_page(struct page *page)
{
if (likely(get_page_unless_zero(page))) {
struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);
if (!pagevec_add(pvec, page))
____pagevec_lru_deactivate(pvec);
put_cpu_var(lru_deactivate_pvecs);
}
} }
void lru_add_drain(void) void lru_add_drain(void)
......
...@@ -321,11 +321,12 @@ EXPORT_SYMBOL(truncate_inode_pages); ...@@ -321,11 +321,12 @@ EXPORT_SYMBOL(truncate_inode_pages);
* pagetables. * pagetables.
*/ */
unsigned long invalidate_mapping_pages(struct address_space *mapping, unsigned long invalidate_mapping_pages(struct address_space *mapping,
pgoff_t start, pgoff_t end) pgoff_t start, pgoff_t end)
{ {
struct pagevec pvec; struct pagevec pvec;
pgoff_t next = start; pgoff_t next = start;
unsigned long ret = 0; unsigned long ret;
unsigned long count = 0;
int i; int i;
pagevec_init(&pvec, 0); pagevec_init(&pvec, 0);
...@@ -352,9 +353,15 @@ unsigned long invalidate_mapping_pages(struct address_space *mapping, ...@@ -352,9 +353,15 @@ unsigned long invalidate_mapping_pages(struct address_space *mapping,
if (lock_failed) if (lock_failed)
continue; continue;
ret += invalidate_inode_page(page); ret = invalidate_inode_page(page);
unlock_page(page); unlock_page(page);
/*
* Invalidation is a hint that the page is no longer
* of interest and try to speed up its reclaim.
*/
if (!ret)
deactivate_page(page);
count += ret;
if (next > end) if (next > end)
break; break;
} }
...@@ -362,7 +369,7 @@ unsigned long invalidate_mapping_pages(struct address_space *mapping, ...@@ -362,7 +369,7 @@ unsigned long invalidate_mapping_pages(struct address_space *mapping,
mem_cgroup_uncharge_end(); mem_cgroup_uncharge_end();
cond_resched(); cond_resched();
} }
return ret; return count;
} }
EXPORT_SYMBOL(invalidate_mapping_pages); EXPORT_SYMBOL(invalidate_mapping_pages);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment