Commit 9a982250 authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Linus Torvalds

thp: introduce deferred_split_huge_page()

Currently we don't split huge page on partial unmap.  It's not an ideal
situation.  It can lead to memory overhead.

Furtunately, we can detect partial unmap on page_remove_rmap().  But we
cannot call split_huge_page() from there due to locking context.

It's also counterproductive to do directly from munmap() codepath: in
many cases we will hit this from exit(2) and splitting the huge page
just to free it up in small pages is not what we really want.

The patch introduce deferred_split_huge_page() which put the huge page
into queue for splitting.  The splitting itself will happen when we get
memory pressure via shrinker interface.  The page will be dropped from
list on freeing through compound page destructor.
Signed-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Tested-by: default avatarSasha Levin <sasha.levin@oracle.com>
Tested-by: default avatarAneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Acked-by: default avatarJerome Marchand <jmarchan@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Rik van Riel <riel@redhat.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Steve Capper <steve.capper@linaro.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Christoph Lameter <cl@linux.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 248db92d
...@@ -90,11 +90,15 @@ extern bool is_vma_temporary_stack(struct vm_area_struct *vma); ...@@ -90,11 +90,15 @@ extern bool is_vma_temporary_stack(struct vm_area_struct *vma);
extern unsigned long transparent_hugepage_flags; extern unsigned long transparent_hugepage_flags;
extern void prep_transhuge_page(struct page *page);
extern void free_transhuge_page(struct page *page);
int split_huge_page_to_list(struct page *page, struct list_head *list); int split_huge_page_to_list(struct page *page, struct list_head *list);
static inline int split_huge_page(struct page *page) static inline int split_huge_page(struct page *page)
{ {
return split_huge_page_to_list(page, NULL); return split_huge_page_to_list(page, NULL);
} }
void deferred_split_huge_page(struct page *page);
void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
unsigned long address); unsigned long address);
...@@ -170,6 +174,7 @@ static inline int split_huge_page(struct page *page) ...@@ -170,6 +174,7 @@ static inline int split_huge_page(struct page *page)
{ {
return 0; return 0;
} }
static inline void deferred_split_huge_page(struct page *page) {}
#define split_huge_pmd(__vma, __pmd, __address) \ #define split_huge_pmd(__vma, __pmd, __address) \
do { } while (0) do { } while (0)
static inline int hugepage_madvise(struct vm_area_struct *vma, static inline int hugepage_madvise(struct vm_area_struct *vma,
......
...@@ -507,6 +507,9 @@ enum compound_dtor_id { ...@@ -507,6 +507,9 @@ enum compound_dtor_id {
COMPOUND_PAGE_DTOR, COMPOUND_PAGE_DTOR,
#ifdef CONFIG_HUGETLB_PAGE #ifdef CONFIG_HUGETLB_PAGE
HUGETLB_PAGE_DTOR, HUGETLB_PAGE_DTOR,
#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
TRANSHUGE_PAGE_DTOR,
#endif #endif
NR_COMPOUND_DTORS, NR_COMPOUND_DTORS,
}; };
...@@ -537,6 +540,8 @@ static inline void set_compound_order(struct page *page, unsigned int order) ...@@ -537,6 +540,8 @@ static inline void set_compound_order(struct page *page, unsigned int order)
page[1].compound_order = order; page[1].compound_order = order;
} }
void free_compound_page(struct page *page);
#ifdef CONFIG_MMU #ifdef CONFIG_MMU
/* /*
* Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when
......
...@@ -55,6 +55,7 @@ struct page { ...@@ -55,6 +55,7 @@ struct page {
*/ */
void *s_mem; /* slab first object */ void *s_mem; /* slab first object */
atomic_t compound_mapcount; /* first tail page */ atomic_t compound_mapcount; /* first tail page */
/* page_deferred_list().next -- second tail page */
}; };
/* Second double word */ /* Second double word */
...@@ -62,6 +63,7 @@ struct page { ...@@ -62,6 +63,7 @@ struct page {
union { union {
pgoff_t index; /* Our offset within mapping. */ pgoff_t index; /* Our offset within mapping. */
void *freelist; /* sl[aou]b first free object */ void *freelist; /* sl[aou]b first free object */
/* page_deferred_list().prev -- second tail page */
}; };
union { union {
......
...@@ -135,6 +135,10 @@ static struct khugepaged_scan khugepaged_scan = { ...@@ -135,6 +135,10 @@ static struct khugepaged_scan khugepaged_scan = {
.mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head), .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head),
}; };
static DEFINE_SPINLOCK(split_queue_lock);
static LIST_HEAD(split_queue);
static unsigned long split_queue_len;
static struct shrinker deferred_split_shrinker;
static void set_recommended_min_free_kbytes(void) static void set_recommended_min_free_kbytes(void)
{ {
...@@ -667,6 +671,9 @@ static int __init hugepage_init(void) ...@@ -667,6 +671,9 @@ static int __init hugepage_init(void)
err = register_shrinker(&huge_zero_page_shrinker); err = register_shrinker(&huge_zero_page_shrinker);
if (err) if (err)
goto err_hzp_shrinker; goto err_hzp_shrinker;
err = register_shrinker(&deferred_split_shrinker);
if (err)
goto err_split_shrinker;
/* /*
* By default disable transparent hugepages on smaller systems, * By default disable transparent hugepages on smaller systems,
...@@ -684,6 +691,8 @@ static int __init hugepage_init(void) ...@@ -684,6 +691,8 @@ static int __init hugepage_init(void)
return 0; return 0;
err_khugepaged: err_khugepaged:
unregister_shrinker(&deferred_split_shrinker);
err_split_shrinker:
unregister_shrinker(&huge_zero_page_shrinker); unregister_shrinker(&huge_zero_page_shrinker);
err_hzp_shrinker: err_hzp_shrinker:
khugepaged_slab_exit(); khugepaged_slab_exit();
...@@ -740,6 +749,27 @@ static inline pmd_t mk_huge_pmd(struct page *page, pgprot_t prot) ...@@ -740,6 +749,27 @@ static inline pmd_t mk_huge_pmd(struct page *page, pgprot_t prot)
return entry; return entry;
} }
static inline struct list_head *page_deferred_list(struct page *page)
{
/*
* ->lru in the tail pages is occupied by compound_head.
* Let's use ->mapping + ->index in the second tail page as list_head.
*/
return (struct list_head *)&page[2].mapping;
}
void prep_transhuge_page(struct page *page)
{
/*
* we use page->mapping and page->indexlru in second tail page
* as list_head: assuming THP order >= 2
*/
BUILD_BUG_ON(HPAGE_PMD_ORDER < 2);
INIT_LIST_HEAD(page_deferred_list(page));
set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR);
}
static int __do_huge_pmd_anonymous_page(struct mm_struct *mm, static int __do_huge_pmd_anonymous_page(struct mm_struct *mm,
struct vm_area_struct *vma, struct vm_area_struct *vma,
unsigned long address, pmd_t *pmd, unsigned long address, pmd_t *pmd,
...@@ -896,6 +926,7 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -896,6 +926,7 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
count_vm_event(THP_FAULT_FALLBACK); count_vm_event(THP_FAULT_FALLBACK);
return VM_FAULT_FALLBACK; return VM_FAULT_FALLBACK;
} }
prep_transhuge_page(page);
return __do_huge_pmd_anonymous_page(mm, vma, address, pmd, page, gfp, return __do_huge_pmd_anonymous_page(mm, vma, address, pmd, page, gfp,
flags); flags);
} }
...@@ -1192,7 +1223,9 @@ int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -1192,7 +1223,9 @@ int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
} else } else
new_page = NULL; new_page = NULL;
if (unlikely(!new_page)) { if (likely(new_page)) {
prep_transhuge_page(new_page);
} else {
if (!page) { if (!page) {
split_huge_pmd(vma, pmd, address); split_huge_pmd(vma, pmd, address);
ret |= VM_FAULT_FALLBACK; ret |= VM_FAULT_FALLBACK;
...@@ -2109,6 +2142,7 @@ khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm, ...@@ -2109,6 +2142,7 @@ khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm,
return NULL; return NULL;
} }
prep_transhuge_page(*hpage);
count_vm_event(THP_COLLAPSE_ALLOC); count_vm_event(THP_COLLAPSE_ALLOC);
return *hpage; return *hpage;
} }
...@@ -2120,8 +2154,12 @@ static int khugepaged_find_target_node(void) ...@@ -2120,8 +2154,12 @@ static int khugepaged_find_target_node(void)
static inline struct page *alloc_hugepage(int defrag) static inline struct page *alloc_hugepage(int defrag)
{ {
return alloc_pages(alloc_hugepage_gfpmask(defrag, 0), struct page *page;
HPAGE_PMD_ORDER);
page = alloc_pages(alloc_hugepage_gfpmask(defrag, 0), HPAGE_PMD_ORDER);
if (page)
prep_transhuge_page(page);
return page;
} }
static struct page *khugepaged_alloc_hugepage(bool *wait) static struct page *khugepaged_alloc_hugepage(bool *wait)
...@@ -3098,7 +3136,7 @@ static int __split_huge_page_tail(struct page *head, int tail, ...@@ -3098,7 +3136,7 @@ static int __split_huge_page_tail(struct page *head, int tail,
set_page_idle(page_tail); set_page_idle(page_tail);
/* ->mapping in first tail page is compound_mapcount */ /* ->mapping in first tail page is compound_mapcount */
VM_BUG_ON_PAGE(tail != 1 && page_tail->mapping != TAIL_MAPPING, VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING,
page_tail); page_tail);
page_tail->mapping = head->mapping; page_tail->mapping = head->mapping;
...@@ -3207,12 +3245,20 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) ...@@ -3207,12 +3245,20 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
freeze_page(anon_vma, head); freeze_page(anon_vma, head);
VM_BUG_ON_PAGE(compound_mapcount(head), head); VM_BUG_ON_PAGE(compound_mapcount(head), head);
/* Prevent deferred_split_scan() touching ->_count */
spin_lock(&split_queue_lock);
count = page_count(head); count = page_count(head);
mapcount = total_mapcount(head); mapcount = total_mapcount(head);
if (mapcount == count - 1) { if (mapcount == count - 1) {
if (!list_empty(page_deferred_list(head))) {
split_queue_len--;
list_del(page_deferred_list(head));
}
spin_unlock(&split_queue_lock);
__split_huge_page(page, list); __split_huge_page(page, list);
ret = 0; ret = 0;
} else if (IS_ENABLED(CONFIG_DEBUG_VM) && mapcount > count - 1) { } else if (IS_ENABLED(CONFIG_DEBUG_VM) && mapcount > count - 1) {
spin_unlock(&split_queue_lock);
pr_alert("total_mapcount: %u, page_count(): %u\n", pr_alert("total_mapcount: %u, page_count(): %u\n",
mapcount, count); mapcount, count);
if (PageTail(page)) if (PageTail(page))
...@@ -3220,6 +3266,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) ...@@ -3220,6 +3266,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
dump_page(page, "total_mapcount(head) > page_count(head) - 1"); dump_page(page, "total_mapcount(head) > page_count(head) - 1");
BUG(); BUG();
} else { } else {
spin_unlock(&split_queue_lock);
unfreeze_page(anon_vma, head); unfreeze_page(anon_vma, head);
ret = -EBUSY; ret = -EBUSY;
} }
...@@ -3231,3 +3278,87 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) ...@@ -3231,3 +3278,87 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED);
return ret; return ret;
} }
void free_transhuge_page(struct page *page)
{
unsigned long flags;
spin_lock_irqsave(&split_queue_lock, flags);
if (!list_empty(page_deferred_list(page))) {
split_queue_len--;
list_del(page_deferred_list(page));
}
spin_unlock_irqrestore(&split_queue_lock, flags);
free_compound_page(page);
}
void deferred_split_huge_page(struct page *page)
{
unsigned long flags;
VM_BUG_ON_PAGE(!PageTransHuge(page), page);
spin_lock_irqsave(&split_queue_lock, flags);
if (list_empty(page_deferred_list(page))) {
list_add_tail(page_deferred_list(page), &split_queue);
split_queue_len++;
}
spin_unlock_irqrestore(&split_queue_lock, flags);
}
static unsigned long deferred_split_count(struct shrinker *shrink,
struct shrink_control *sc)
{
/*
* Split a page from split_queue will free up at least one page,
* at most HPAGE_PMD_NR - 1. We don't track exact number.
* Let's use HPAGE_PMD_NR / 2 as ballpark.
*/
return ACCESS_ONCE(split_queue_len) * HPAGE_PMD_NR / 2;
}
static unsigned long deferred_split_scan(struct shrinker *shrink,
struct shrink_control *sc)
{
unsigned long flags;
LIST_HEAD(list), *pos, *next;
struct page *page;
int split = 0;
spin_lock_irqsave(&split_queue_lock, flags);
list_splice_init(&split_queue, &list);
/* Take pin on all head pages to avoid freeing them under us */
list_for_each_safe(pos, next, &list) {
page = list_entry((void *)pos, struct page, mapping);
page = compound_head(page);
/* race with put_compound_page() */
if (!get_page_unless_zero(page)) {
list_del_init(page_deferred_list(page));
split_queue_len--;
}
}
spin_unlock_irqrestore(&split_queue_lock, flags);
list_for_each_safe(pos, next, &list) {
page = list_entry((void *)pos, struct page, mapping);
lock_page(page);
/* split_huge_page() removes page from list on success */
if (!split_huge_page(page))
split++;
unlock_page(page);
put_page(page);
}
spin_lock_irqsave(&split_queue_lock, flags);
list_splice_tail(&list, &split_queue);
spin_unlock_irqrestore(&split_queue_lock, flags);
return split * HPAGE_PMD_NR / 2;
}
static struct shrinker deferred_split_shrinker = {
.count_objects = deferred_split_count,
.scan_objects = deferred_split_scan,
.seeks = DEFAULT_SEEKS,
};
...@@ -1760,6 +1760,7 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm, ...@@ -1760,6 +1760,7 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm,
HPAGE_PMD_ORDER); HPAGE_PMD_ORDER);
if (!new_page) if (!new_page)
goto out_fail; goto out_fail;
prep_transhuge_page(new_page);
isolated = numamigrate_isolate_page(pgdat, page); isolated = numamigrate_isolate_page(pgdat, page);
if (!isolated) { if (!isolated) {
......
...@@ -222,13 +222,15 @@ static char * const zone_names[MAX_NR_ZONES] = { ...@@ -222,13 +222,15 @@ static char * const zone_names[MAX_NR_ZONES] = {
#endif #endif
}; };
static void free_compound_page(struct page *page);
compound_page_dtor * const compound_page_dtors[] = { compound_page_dtor * const compound_page_dtors[] = {
NULL, NULL,
free_compound_page, free_compound_page,
#ifdef CONFIG_HUGETLB_PAGE #ifdef CONFIG_HUGETLB_PAGE
free_huge_page, free_huge_page,
#endif #endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
free_transhuge_page,
#endif
}; };
int min_free_kbytes = 1024; int min_free_kbytes = 1024;
...@@ -450,7 +452,7 @@ static void bad_page(struct page *page, const char *reason, ...@@ -450,7 +452,7 @@ static void bad_page(struct page *page, const char *reason,
* This usage means that zero-order pages may not be compound. * This usage means that zero-order pages may not be compound.
*/ */
static void free_compound_page(struct page *page) void free_compound_page(struct page *page)
{ {
__free_pages_ok(page, compound_order(page)); __free_pages_ok(page, compound_order(page));
} }
...@@ -858,16 +860,27 @@ static int free_tail_pages_check(struct page *head_page, struct page *page) ...@@ -858,16 +860,27 @@ static int free_tail_pages_check(struct page *head_page, struct page *page)
ret = 0; ret = 0;
goto out; goto out;
} }
/* mapping in first tail page is used for compound_mapcount() */ switch (page - head_page) {
if (page - head_page == 1) { case 1:
/* the first tail page: ->mapping is compound_mapcount() */
if (unlikely(compound_mapcount(page))) { if (unlikely(compound_mapcount(page))) {
bad_page(page, "nonzero compound_mapcount", 0); bad_page(page, "nonzero compound_mapcount", 0);
goto out; goto out;
} }
} else if (page->mapping != TAIL_MAPPING) { break;
case 2:
/*
* the second tail page: ->mapping is
* page_deferred_list().next -- ignore value.
*/
break;
default:
if (page->mapping != TAIL_MAPPING) {
bad_page(page, "corrupted mapping in tail page", 0); bad_page(page, "corrupted mapping in tail page", 0);
goto out; goto out;
} }
break;
}
if (unlikely(!PageTail(page))) { if (unlikely(!PageTail(page))) {
bad_page(page, "PageTail not set", 0); bad_page(page, "PageTail not set", 0);
goto out; goto out;
......
...@@ -1282,8 +1282,10 @@ static void page_remove_anon_compound_rmap(struct page *page) ...@@ -1282,8 +1282,10 @@ static void page_remove_anon_compound_rmap(struct page *page)
nr = HPAGE_PMD_NR; nr = HPAGE_PMD_NR;
} }
if (nr) if (nr) {
__mod_zone_page_state(page_zone(page), NR_ANON_PAGES, -nr); __mod_zone_page_state(page_zone(page), NR_ANON_PAGES, -nr);
deferred_split_huge_page(page);
}
} }
/** /**
...@@ -1318,6 +1320,9 @@ void page_remove_rmap(struct page *page, bool compound) ...@@ -1318,6 +1320,9 @@ void page_remove_rmap(struct page *page, bool compound)
if (unlikely(PageMlocked(page))) if (unlikely(PageMlocked(page)))
clear_page_mlock(page); clear_page_mlock(page);
if (PageTransCompound(page))
deferred_split_huge_page(compound_head(page));
/* /*
* It would be tidy to reset the PageAnon mapping here, * It would be tidy to reset the PageAnon mapping here,
* but that might overwrite a racing page_add_anon_rmap * but that might overwrite a racing page_add_anon_rmap
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment