Commit ba988280 authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Linus Torvalds

thp: add option to setup migration entries during PMD split

We are going to use migration PTE entries to stabilize page counts.  If
the page is mapped with PMDs we need to split the PMD and setup
migration entries.  It's reasonable to combine these operations to avoid
double-scanning over the page table.
Signed-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Tested-by: default avatarSasha Levin <sasha.levin@oracle.com>
Tested-by: default avatarAneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Acked-by: default avatarJerome Marchand <jmarchan@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Rik van Riel <riel@redhat.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Steve Capper <steve.capper@linaro.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Christoph Lameter <cl@linux.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent eef1b3ba
...@@ -2695,7 +2695,7 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, ...@@ -2695,7 +2695,7 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
} }
static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
unsigned long haddr) unsigned long haddr, bool freeze)
{ {
struct mm_struct *mm = vma->vm_mm; struct mm_struct *mm = vma->vm_mm;
struct page *page; struct page *page;
...@@ -2739,12 +2739,18 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, ...@@ -2739,12 +2739,18 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
* transferred to avoid any possibility of altering * transferred to avoid any possibility of altering
* permissions across VMAs. * permissions across VMAs.
*/ */
if (freeze) {
swp_entry_t swp_entry;
swp_entry = make_migration_entry(page + i, write);
entry = swp_entry_to_pte(swp_entry);
} else {
entry = mk_pte(page + i, vma->vm_page_prot); entry = mk_pte(page + i, vma->vm_page_prot);
entry = maybe_mkwrite(pte_mkdirty(entry), vma); entry = maybe_mkwrite(pte_mkdirty(entry), vma);
if (!write) if (!write)
entry = pte_wrprotect(entry); entry = pte_wrprotect(entry);
if (!young) if (!young)
entry = pte_mkold(entry); entry = pte_mkold(entry);
}
pte = pte_offset_map(&_pmd, haddr); pte = pte_offset_map(&_pmd, haddr);
BUG_ON(!pte_none(*pte)); BUG_ON(!pte_none(*pte));
set_pte_at(mm, haddr, pte, entry); set_pte_at(mm, haddr, pte, entry);
...@@ -2785,7 +2791,7 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, ...@@ -2785,7 +2791,7 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE); mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE);
ptl = pmd_lock(mm, pmd); ptl = pmd_lock(mm, pmd);
if (likely(pmd_trans_huge(*pmd))) if (likely(pmd_trans_huge(*pmd)))
__split_huge_pmd_locked(vma, pmd, haddr); __split_huge_pmd_locked(vma, pmd, haddr, false);
spin_unlock(ptl); spin_unlock(ptl);
mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PMD_SIZE); mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PMD_SIZE);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment