Commit 6a93ca8f authored by Johannes Weiner's avatar Johannes Weiner Committed by Linus Torvalds

mm: migrate: do not touch page->mem_cgroup of live pages

Changing a page's memcg association complicates dealing with the page,
so we want to limit this as much as possible.  Page migration e.g.  does
not have to do that.  Just like page cache replacement, it can forcibly
charge a replacement page, and then uncharge the old page when it gets
freed.  Temporarily overcharging the cgroup by a single page is not an
issue in practice, and charging is so cheap nowadays that this is much
preferrable to the headache of messing with live pages.

The only place that still changes the page->mem_cgroup binding of live
pages is when pages move along with a task to another cgroup.  But that
path isolates the page from the LRU, takes the page lock, and the move
lock (lock_page_memcg()).  That means page->mem_cgroup is always stable
in callers that have the page isolated from the LRU or locked.  Lighter
unlocked paths, like writeback accounting, can use lock_page_memcg().

[akpm@linux-foundation.org: fix build]
[vdavydov@virtuozzo.com: fix lockdep splat]
Signed-off-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Acked-by: default avatarVladimir Davydov <vdavydov@virtuozzo.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Greg Thelen <gthelen@google.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 23047a96
...@@ -300,7 +300,7 @@ void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg, ...@@ -300,7 +300,7 @@ void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
void mem_cgroup_uncharge(struct page *page); void mem_cgroup_uncharge(struct page *page);
void mem_cgroup_uncharge_list(struct list_head *page_list); void mem_cgroup_uncharge_list(struct list_head *page_list);
void mem_cgroup_replace_page(struct page *oldpage, struct page *newpage); void mem_cgroup_migrate(struct page *oldpage, struct page *newpage);
struct lruvec *mem_cgroup_zone_lruvec(struct zone *, struct mem_cgroup *); struct lruvec *mem_cgroup_zone_lruvec(struct zone *, struct mem_cgroup *);
struct lruvec *mem_cgroup_page_lruvec(struct page *, struct zone *); struct lruvec *mem_cgroup_page_lruvec(struct page *, struct zone *);
...@@ -580,7 +580,7 @@ static inline void mem_cgroup_uncharge_list(struct list_head *page_list) ...@@ -580,7 +580,7 @@ static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
{ {
} }
static inline void mem_cgroup_replace_page(struct page *old, struct page *new) static inline void mem_cgroup_migrate(struct page *old, struct page *new)
{ {
} }
......
...@@ -905,20 +905,11 @@ static inline struct mem_cgroup *page_memcg(struct page *page) ...@@ -905,20 +905,11 @@ static inline struct mem_cgroup *page_memcg(struct page *page)
{ {
return page->mem_cgroup; return page->mem_cgroup;
} }
static inline void set_page_memcg(struct page *page, struct mem_cgroup *memcg)
{
page->mem_cgroup = memcg;
}
#else #else
static inline struct mem_cgroup *page_memcg(struct page *page) static inline struct mem_cgroup *page_memcg(struct page *page)
{ {
return NULL; return NULL;
} }
static inline void set_page_memcg(struct page *page, struct mem_cgroup *memcg)
{
}
#endif #endif
/* /*
......
...@@ -577,7 +577,7 @@ int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask) ...@@ -577,7 +577,7 @@ int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
__inc_zone_page_state(new, NR_SHMEM); __inc_zone_page_state(new, NR_SHMEM);
spin_unlock_irqrestore(&mapping->tree_lock, flags); spin_unlock_irqrestore(&mapping->tree_lock, flags);
unlock_page_memcg(memcg); unlock_page_memcg(memcg);
mem_cgroup_replace_page(old, new); mem_cgroup_migrate(old, new);
radix_tree_preload_end(); radix_tree_preload_end();
if (freepage) if (freepage)
freepage(old); freepage(old);
......
...@@ -4457,7 +4457,7 @@ static int mem_cgroup_move_account(struct page *page, ...@@ -4457,7 +4457,7 @@ static int mem_cgroup_move_account(struct page *page,
VM_BUG_ON(compound && !PageTransHuge(page)); VM_BUG_ON(compound && !PageTransHuge(page));
/* /*
* Prevent mem_cgroup_replace_page() from looking at * Prevent mem_cgroup_migrate() from looking at
* page->mem_cgroup of its source page while we change it. * page->mem_cgroup of its source page while we change it.
*/ */
ret = -EBUSY; ret = -EBUSY;
...@@ -5486,16 +5486,17 @@ void mem_cgroup_uncharge_list(struct list_head *page_list) ...@@ -5486,16 +5486,17 @@ void mem_cgroup_uncharge_list(struct list_head *page_list)
} }
/** /**
* mem_cgroup_replace_page - migrate a charge to another page * mem_cgroup_migrate - charge a page's replacement
* @oldpage: currently charged page * @oldpage: currently circulating page
* @newpage: page to transfer the charge to * @newpage: replacement page
* *
* Migrate the charge from @oldpage to @newpage. * Charge @newpage as a replacement page for @oldpage. @oldpage will
* be uncharged upon free.
* *
* Both pages must be locked, @newpage->mapping must be set up. * Both pages must be locked, @newpage->mapping must be set up.
* Either or both pages might be on the LRU already. * Either or both pages might be on the LRU already.
*/ */
void mem_cgroup_replace_page(struct page *oldpage, struct page *newpage) void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
{ {
struct mem_cgroup *memcg; struct mem_cgroup *memcg;
unsigned int nr_pages; unsigned int nr_pages;
......
...@@ -326,12 +326,13 @@ int migrate_page_move_mapping(struct address_space *mapping, ...@@ -326,12 +326,13 @@ int migrate_page_move_mapping(struct address_space *mapping,
return -EAGAIN; return -EAGAIN;
/* No turning back from here */ /* No turning back from here */
set_page_memcg(newpage, page_memcg(page));
newpage->index = page->index; newpage->index = page->index;
newpage->mapping = page->mapping; newpage->mapping = page->mapping;
if (PageSwapBacked(page)) if (PageSwapBacked(page))
SetPageSwapBacked(newpage); SetPageSwapBacked(newpage);
mem_cgroup_migrate(page, newpage);
return MIGRATEPAGE_SUCCESS; return MIGRATEPAGE_SUCCESS;
} }
...@@ -373,7 +374,6 @@ int migrate_page_move_mapping(struct address_space *mapping, ...@@ -373,7 +374,6 @@ int migrate_page_move_mapping(struct address_space *mapping,
* Now we know that no one else is looking at the page: * Now we know that no one else is looking at the page:
* no turning back from here. * no turning back from here.
*/ */
set_page_memcg(newpage, page_memcg(page));
newpage->index = page->index; newpage->index = page->index;
newpage->mapping = page->mapping; newpage->mapping = page->mapping;
if (PageSwapBacked(page)) if (PageSwapBacked(page))
...@@ -428,6 +428,8 @@ int migrate_page_move_mapping(struct address_space *mapping, ...@@ -428,6 +428,8 @@ int migrate_page_move_mapping(struct address_space *mapping,
} }
local_irq_enable(); local_irq_enable();
mem_cgroup_migrate(page, newpage);
return MIGRATEPAGE_SUCCESS; return MIGRATEPAGE_SUCCESS;
} }
...@@ -458,9 +460,9 @@ int migrate_huge_page_move_mapping(struct address_space *mapping, ...@@ -458,9 +460,9 @@ int migrate_huge_page_move_mapping(struct address_space *mapping,
return -EAGAIN; return -EAGAIN;
} }
set_page_memcg(newpage, page_memcg(page));
newpage->index = page->index; newpage->index = page->index;
newpage->mapping = page->mapping; newpage->mapping = page->mapping;
get_page(newpage); get_page(newpage);
radix_tree_replace_slot(pslot, newpage); radix_tree_replace_slot(pslot, newpage);
...@@ -468,6 +470,9 @@ int migrate_huge_page_move_mapping(struct address_space *mapping, ...@@ -468,6 +470,9 @@ int migrate_huge_page_move_mapping(struct address_space *mapping,
page_unfreeze_refs(page, expected_count - 1); page_unfreeze_refs(page, expected_count - 1);
spin_unlock_irq(&mapping->tree_lock); spin_unlock_irq(&mapping->tree_lock);
mem_cgroup_migrate(page, newpage);
return MIGRATEPAGE_SUCCESS; return MIGRATEPAGE_SUCCESS;
} }
...@@ -775,7 +780,6 @@ static int move_to_new_page(struct page *newpage, struct page *page, ...@@ -775,7 +780,6 @@ static int move_to_new_page(struct page *newpage, struct page *page,
* page is freed; but stats require that PageAnon be left as PageAnon. * page is freed; but stats require that PageAnon be left as PageAnon.
*/ */
if (rc == MIGRATEPAGE_SUCCESS) { if (rc == MIGRATEPAGE_SUCCESS) {
set_page_memcg(page, NULL);
if (!PageAnon(page)) if (!PageAnon(page))
page->mapping = NULL; page->mapping = NULL;
} }
...@@ -1842,8 +1846,7 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm, ...@@ -1842,8 +1846,7 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm,
} }
mlock_migrate_page(new_page, page); mlock_migrate_page(new_page, page);
set_page_memcg(new_page, page_memcg(page)); mem_cgroup_migrate(page, new_page);
set_page_memcg(page, NULL);
page_remove_rmap(page, true); page_remove_rmap(page, true);
set_page_owner_migrate_reason(new_page, MR_NUMA_MISPLACED); set_page_owner_migrate_reason(new_page, MR_NUMA_MISPLACED);
......
...@@ -1116,7 +1116,7 @@ static int shmem_replace_page(struct page **pagep, gfp_t gfp, ...@@ -1116,7 +1116,7 @@ static int shmem_replace_page(struct page **pagep, gfp_t gfp,
*/ */
oldpage = newpage; oldpage = newpage;
} else { } else {
mem_cgroup_replace_page(oldpage, newpage); mem_cgroup_migrate(oldpage, newpage);
lru_cache_add_anon(newpage); lru_cache_add_anon(newpage);
*pagep = newpage; *pagep = newpage;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment