Commit 53f5c3f4 authored by Jérôme Glisse's avatar Jérôme Glisse Committed by Linus Torvalds

mm/hmm: factor out pte and pmd handling to simplify hmm_vma_walk_pmd()

No functional change, just create one function to handle pmd and one to
handle pte (hmm_vma_handle_pmd() and hmm_vma_handle_pte()).

Link: http://lkml.kernel.org/r/20180323005527.758-14-jglisse@redhat.comSigned-off-by: default avatarJérôme Glisse <jglisse@redhat.com>
Reviewed-by: default avatarJohn Hubbard <jhubbard@nvidia.com>
Cc: Evgeny Baskakov <ebaskakov@nvidia.com>
Cc: Ralph Campbell <rcampbell@nvidia.com>
Cc: Mark Hairgrove <mhairgrove@nvidia.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 33cd47dc
......@@ -375,74 +375,45 @@ static int hmm_vma_walk_hole(unsigned long addr,
return hmm_vma_walk->fault ? -EAGAIN : 0;
}
static int hmm_vma_walk_pmd(pmd_t *pmdp,
unsigned long start,
static int hmm_vma_handle_pmd(struct mm_walk *walk,
unsigned long addr,
unsigned long end,
struct mm_walk *walk)
uint64_t *pfns,
pmd_t pmd)
{
struct hmm_vma_walk *hmm_vma_walk = walk->private;
struct hmm_range *range = hmm_vma_walk->range;
struct vm_area_struct *vma = walk->vma;
uint64_t *pfns = range->pfns;
unsigned long addr = start, i;
bool write_fault;
pte_t *ptep;
i = (addr - range->start) >> PAGE_SHIFT;
write_fault = hmm_vma_walk->fault & hmm_vma_walk->write;
again:
if (pmd_none(*pmdp))
return hmm_vma_walk_hole(start, end, walk);
if (pmd_huge(*pmdp) && vma->vm_flags & VM_HUGETLB)
return hmm_pfns_bad(start, end, walk);
if (pmd_devmap(*pmdp) || pmd_trans_huge(*pmdp)) {
unsigned long pfn;
unsigned long pfn, i;
uint64_t flag = 0;
pmd_t pmd;
/*
* No need to take pmd_lock here, even if some other threads
* is splitting the huge pmd we will get that event through
* mmu_notifier callback.
*
* So just read pmd value and check again its a transparent
* huge or device mapping one and compute corresponding pfn
* values.
*/
pmd = pmd_read_atomic(pmdp);
barrier();
if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))
goto again;
if (pmd_protnone(pmd))
return hmm_vma_walk_hole(start, end, walk);
return hmm_vma_walk_hole(addr, end, walk);
if (write_fault && !pmd_write(pmd))
return hmm_vma_walk_hole(start, end, walk);
if ((hmm_vma_walk->fault & hmm_vma_walk->write) && !pmd_write(pmd))
return hmm_vma_walk_hole(addr, end, walk);
pfn = pmd_pfn(pmd) + pte_index(addr);
flag |= pmd_write(pmd) ? HMM_PFN_WRITE : 0;
for (; addr < end; addr += PAGE_SIZE, i++, pfn++)
for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++)
pfns[i] = hmm_pfn_from_pfn(pfn) | flag;
hmm_vma_walk->last = end;
return 0;
}
if (pmd_bad(*pmdp))
return hmm_pfns_bad(start, end, walk);
}
ptep = pte_offset_map(pmdp, addr);
for (; addr < end; addr += PAGE_SIZE, ptep++, i++) {
static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
unsigned long end, pmd_t *pmdp, pte_t *ptep,
uint64_t *pfn)
{
struct hmm_vma_walk *hmm_vma_walk = walk->private;
struct vm_area_struct *vma = walk->vma;
pte_t pte = *ptep;
pfns[i] = 0;
*pfn = 0;
if (pte_none(pte)) {
pfns[i] = 0;
*pfn = 0;
if (hmm_vma_walk->fault)
goto fault;
continue;
return 0;
}
if (!pte_present(pte)) {
......@@ -451,7 +422,7 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp,
if (!non_swap_entry(entry)) {
if (hmm_vma_walk->fault)
goto fault;
continue;
return 0;
}
/*
......@@ -459,13 +430,16 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp,
* device and report anything else as error.
*/
if (is_device_private_entry(entry)) {
pfns[i] = hmm_pfn_from_pfn(swp_offset(entry));
*pfn = hmm_pfn_from_pfn(swp_offset(entry));
if (is_write_device_private_entry(entry)) {
pfns[i] |= HMM_PFN_WRITE;
} else if (write_fault)
*pfn |= HMM_PFN_WRITE;
} else if ((hmm_vma_walk->fault & hmm_vma_walk->write))
goto fault;
pfns[i] |= HMM_PFN_DEVICE_PRIVATE;
} else if (is_migration_entry(entry)) {
*pfn |= HMM_PFN_DEVICE_PRIVATE;
return 0;
}
if (is_migration_entry(entry)) {
if (hmm_vma_walk->fault) {
pte_unmap(ptep);
hmm_vma_walk->last = addr;
......@@ -473,28 +447,84 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp,
pmdp, addr);
return -EAGAIN;
}
continue;
} else {
/* Report error for everything else */
pfns[i] = HMM_PFN_ERROR;
return 0;
}
continue;
/* Report error for everything else */
*pfn = HMM_PFN_ERROR;
return -EFAULT;
}
if (write_fault && !pte_write(pte))
if ((hmm_vma_walk->fault & hmm_vma_walk->write) && !pte_write(pte))
goto fault;
pfns[i] = hmm_pfn_from_pfn(pte_pfn(pte));
pfns[i] |= pte_write(pte) ? HMM_PFN_WRITE : 0;
continue;
*pfn = hmm_pfn_from_pfn(pte_pfn(pte));
*pfn |= pte_write(pte) ? HMM_PFN_WRITE : 0;
return 0;
fault:
pte_unmap(ptep);
/* Fault any virtual address we were asked to fault */
return hmm_vma_walk_hole(addr, end, walk);
}
static int hmm_vma_walk_pmd(pmd_t *pmdp,
unsigned long start,
unsigned long end,
struct mm_walk *walk)
{
struct hmm_vma_walk *hmm_vma_walk = walk->private;
struct hmm_range *range = hmm_vma_walk->range;
uint64_t *pfns = range->pfns;
unsigned long addr = start, i;
pte_t *ptep;
i = (addr - range->start) >> PAGE_SHIFT;
again:
if (pmd_none(*pmdp))
return hmm_vma_walk_hole(start, end, walk);
if (pmd_huge(*pmdp) && (range->vma->vm_flags & VM_HUGETLB))
return hmm_pfns_bad(start, end, walk);
if (pmd_devmap(*pmdp) || pmd_trans_huge(*pmdp)) {
pmd_t pmd;
/*
* No need to take pmd_lock here, even if some other threads
* is splitting the huge pmd we will get that event through
* mmu_notifier callback.
*
* So just read pmd value and check again its a transparent
* huge or device mapping one and compute corresponding pfn
* values.
*/
pmd = pmd_read_atomic(pmdp);
barrier();
if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))
goto again;
return hmm_vma_handle_pmd(walk, addr, end, &pfns[i], pmd);
}
if (pmd_bad(*pmdp))
return hmm_pfns_bad(start, end, walk);
ptep = pte_offset_map(pmdp, addr);
for (; addr < end; addr += PAGE_SIZE, ptep++, i++) {
int r;
r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, &pfns[i]);
if (r) {
/* hmm_vma_handle_pte() did unmap pte directory */
hmm_vma_walk->last = addr;
return r;
}
}
pte_unmap(ptep - 1);
hmm_vma_walk->last = addr;
return 0;
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment