Commit bd722058 authored by Vishal Moola (Oracle)'s avatar Vishal Moola (Oracle) Committed by Andrew Morton

hugetlb: convert hugetlb_wp() to use struct vm_fault

hugetlb_wp() can use the struct vm_fault passed in from hugetlb_fault().
This alleviates the stack by consolidating 5 variables into a single
struct.

[vishal.moola@gmail.com: simplify hugetlb_wp() arguments]
  Link: https://lkml.kernel.org/r/ZhQtoFNZBNwBCeXn@fedora
Link: https://lkml.kernel.org/r/20240401202651.31440-4-vishal.moola@gmail.comSigned-off-by: default avatarVishal Moola (Oracle) <vishal.moola@gmail.com>
Reviewed-by: default avatarOscar Salvador <osalvador@suse.de>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Muchun Song <muchun.song@linux.dev>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 7b6ec181
...@@ -5918,19 +5918,18 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -5918,19 +5918,18 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
* cannot race with other handlers or page migration. * cannot race with other handlers or page migration.
* Keep the pte_same checks anyway to make transition from the mutex easier. * Keep the pte_same checks anyway to make transition from the mutex easier.
*/ */
static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma, static vm_fault_t hugetlb_wp(struct folio *pagecache_folio,
unsigned long address, pte_t *ptep, unsigned int flags,
struct folio *pagecache_folio, spinlock_t *ptl,
struct vm_fault *vmf) struct vm_fault *vmf)
{ {
const bool unshare = flags & FAULT_FLAG_UNSHARE; struct vm_area_struct *vma = vmf->vma;
pte_t pte = huge_ptep_get(ptep); struct mm_struct *mm = vma->vm_mm;
const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
pte_t pte = huge_ptep_get(vmf->pte);
struct hstate *h = hstate_vma(vma); struct hstate *h = hstate_vma(vma);
struct folio *old_folio; struct folio *old_folio;
struct folio *new_folio; struct folio *new_folio;
int outside_reserve = 0; int outside_reserve = 0;
vm_fault_t ret = 0; vm_fault_t ret = 0;
unsigned long haddr = address & huge_page_mask(h);
struct mmu_notifier_range range; struct mmu_notifier_range range;
/* /*
...@@ -5953,7 +5952,7 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -5953,7 +5952,7 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,
/* Let's take out MAP_SHARED mappings first. */ /* Let's take out MAP_SHARED mappings first. */
if (vma->vm_flags & VM_MAYSHARE) { if (vma->vm_flags & VM_MAYSHARE) {
set_huge_ptep_writable(vma, haddr, ptep); set_huge_ptep_writable(vma, vmf->address, vmf->pte);
return 0; return 0;
} }
...@@ -5972,7 +5971,7 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -5972,7 +5971,7 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,
SetPageAnonExclusive(&old_folio->page); SetPageAnonExclusive(&old_folio->page);
} }
if (likely(!unshare)) if (likely(!unshare))
set_huge_ptep_writable(vma, haddr, ptep); set_huge_ptep_writable(vma, vmf->address, vmf->pte);
delayacct_wpcopy_end(); delayacct_wpcopy_end();
return 0; return 0;
...@@ -5999,8 +5998,8 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -5999,8 +5998,8 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,
* Drop page table lock as buddy allocator may be called. It will * Drop page table lock as buddy allocator may be called. It will
* be acquired again before returning to the caller, as expected. * be acquired again before returning to the caller, as expected.
*/ */
spin_unlock(ptl); spin_unlock(vmf->ptl);
new_folio = alloc_hugetlb_folio(vma, haddr, outside_reserve); new_folio = alloc_hugetlb_folio(vma, vmf->address, outside_reserve);
if (IS_ERR(new_folio)) { if (IS_ERR(new_folio)) {
/* /*
...@@ -6025,19 +6024,21 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -6025,19 +6024,21 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,
* *
* Reacquire both after unmap operation. * Reacquire both after unmap operation.
*/ */
idx = vma_hugecache_offset(h, vma, haddr); idx = vma_hugecache_offset(h, vma, vmf->address);
hash = hugetlb_fault_mutex_hash(mapping, idx); hash = hugetlb_fault_mutex_hash(mapping, idx);
hugetlb_vma_unlock_read(vma); hugetlb_vma_unlock_read(vma);
mutex_unlock(&hugetlb_fault_mutex_table[hash]); mutex_unlock(&hugetlb_fault_mutex_table[hash]);
unmap_ref_private(mm, vma, &old_folio->page, haddr); unmap_ref_private(mm, vma, &old_folio->page,
vmf->address);
mutex_lock(&hugetlb_fault_mutex_table[hash]); mutex_lock(&hugetlb_fault_mutex_table[hash]);
hugetlb_vma_lock_read(vma); hugetlb_vma_lock_read(vma);
spin_lock(ptl); spin_lock(vmf->ptl);
ptep = hugetlb_walk(vma, haddr, huge_page_size(h)); vmf->pte = hugetlb_walk(vma, vmf->address,
if (likely(ptep && huge_page_size(h));
pte_same(huge_ptep_get(ptep), pte))) if (likely(vmf->pte &&
pte_same(huge_ptep_get(vmf->pte), pte)))
goto retry_avoidcopy; goto retry_avoidcopy;
/* /*
* race occurs while re-acquiring page table * race occurs while re-acquiring page table
...@@ -6059,37 +6060,38 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -6059,37 +6060,38 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,
if (unlikely(ret)) if (unlikely(ret))
goto out_release_all; goto out_release_all;
if (copy_user_large_folio(new_folio, old_folio, address, vma)) { if (copy_user_large_folio(new_folio, old_folio, vmf->real_address, vma)) {
ret = VM_FAULT_HWPOISON_LARGE; ret = VM_FAULT_HWPOISON_LARGE;
goto out_release_all; goto out_release_all;
} }
__folio_mark_uptodate(new_folio); __folio_mark_uptodate(new_folio);
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, haddr, mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, vmf->address,
haddr + huge_page_size(h)); vmf->address + huge_page_size(h));
mmu_notifier_invalidate_range_start(&range); mmu_notifier_invalidate_range_start(&range);
/* /*
* Retake the page table lock to check for racing updates * Retake the page table lock to check for racing updates
* before the page tables are altered * before the page tables are altered
*/ */
spin_lock(ptl); spin_lock(vmf->ptl);
ptep = hugetlb_walk(vma, haddr, huge_page_size(h)); vmf->pte = hugetlb_walk(vma, vmf->address, huge_page_size(h));
if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) { if (likely(vmf->pte && pte_same(huge_ptep_get(vmf->pte), pte))) {
pte_t newpte = make_huge_pte(vma, &new_folio->page, !unshare); pte_t newpte = make_huge_pte(vma, &new_folio->page, !unshare);
/* Break COW or unshare */ /* Break COW or unshare */
huge_ptep_clear_flush(vma, haddr, ptep); huge_ptep_clear_flush(vma, vmf->address, vmf->pte);
hugetlb_remove_rmap(old_folio); hugetlb_remove_rmap(old_folio);
hugetlb_add_new_anon_rmap(new_folio, vma, haddr); hugetlb_add_new_anon_rmap(new_folio, vma, vmf->address);
if (huge_pte_uffd_wp(pte)) if (huge_pte_uffd_wp(pte))
newpte = huge_pte_mkuffd_wp(newpte); newpte = huge_pte_mkuffd_wp(newpte);
set_huge_pte_at(mm, haddr, ptep, newpte, huge_page_size(h)); set_huge_pte_at(mm, vmf->address, vmf->pte, newpte,
huge_page_size(h));
folio_set_hugetlb_migratable(new_folio); folio_set_hugetlb_migratable(new_folio);
/* Make the old page be freed below */ /* Make the old page be freed below */
new_folio = old_folio; new_folio = old_folio;
} }
spin_unlock(ptl); spin_unlock(vmf->ptl);
mmu_notifier_invalidate_range_end(&range); mmu_notifier_invalidate_range_end(&range);
out_release_all: out_release_all:
/* /*
...@@ -6097,12 +6099,12 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -6097,12 +6099,12 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,
* unshare) * unshare)
*/ */
if (new_folio != old_folio) if (new_folio != old_folio)
restore_reserve_on_error(h, vma, haddr, new_folio); restore_reserve_on_error(h, vma, vmf->address, new_folio);
folio_put(new_folio); folio_put(new_folio);
out_release_old: out_release_old:
folio_put(old_folio); folio_put(old_folio);
spin_lock(ptl); /* Caller expects lock to be held */ spin_lock(vmf->ptl); /* Caller expects lock to be held */
delayacct_wpcopy_end(); delayacct_wpcopy_end();
return ret; return ret;
...@@ -6369,8 +6371,7 @@ static vm_fault_t hugetlb_no_page(struct address_space *mapping, ...@@ -6369,8 +6371,7 @@ static vm_fault_t hugetlb_no_page(struct address_space *mapping,
hugetlb_count_add(pages_per_huge_page(h), mm); hugetlb_count_add(pages_per_huge_page(h), mm);
if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
/* Optimization, do the COW without a second fault */ /* Optimization, do the COW without a second fault */
ret = hugetlb_wp(mm, vma, vmf->real_address, vmf->pte, ret = hugetlb_wp(folio, vmf);
vmf->flags, folio, vmf->ptl, vmf);
} }
spin_unlock(vmf->ptl); spin_unlock(vmf->ptl);
...@@ -6583,8 +6584,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -6583,8 +6584,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
if (flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) { if (flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
if (!huge_pte_write(vmf.orig_pte)) { if (!huge_pte_write(vmf.orig_pte)) {
ret = hugetlb_wp(mm, vma, address, vmf.pte, flags, ret = hugetlb_wp(pagecache_folio, &vmf);
pagecache_folio, vmf.ptl, &vmf);
goto out_put_page; goto out_put_page;
} else if (likely(flags & FAULT_FLAG_WRITE)) { } else if (likely(flags & FAULT_FLAG_WRITE)) {
vmf.orig_pte = huge_pte_mkdirty(vmf.orig_pte); vmf.orig_pte = huge_pte_mkdirty(vmf.orig_pte);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment