Commit 8782fb61 authored by Steven Price's avatar Steven Price Committed by Linus Torvalds

mm: pagewalk: Fix race between unmap and page walker

The mmap lock protects the page walker from changes to the page tables
during the walk.  However a read lock is insufficient to protect those
areas which don't have a VMA as munmap() detaches the VMAs before
downgrading to a read lock and actually tearing down PTEs/page tables.

For users of walk_page_range() the solution is to simply call pte_hole()
immediately without checking the actual page tables when a VMA is not
present. We now never call __walk_page_range() without a valid vma.

For walk_page_range_novma() the locking requirements are tightened to
require the mmap write lock to be taken, and then walking the pgd
directly with 'no_vma' set.

This in turn means that all page walkers either have a valid vma, or
it's that special 'novma' case for page table debugging.  As a result,
all the odd '(!walk->vma && !walk->no_vma)' tests can be removed.

Fixes: dd2283f2 ("mm: mmap: zap pages with read mmap_sem in munmap")
Reported-by: default avatarJann Horn <jannh@google.com>
Signed-off-by: default avatarSteven Price <steven.price@arm.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Thomas Hellström <thomas.hellstrom@linux.intel.com>
Cc: Konstantin Khlebnikov <koct9i@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent d895ec79
...@@ -118,10 +118,10 @@ static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask, ...@@ -118,10 +118,10 @@ static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask,
if (!numpages) if (!numpages)
return 0; return 0;
mmap_read_lock(&init_mm); mmap_write_lock(&init_mm);
ret = walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL, ret = walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL,
&masks); &masks);
mmap_read_unlock(&init_mm); mmap_write_unlock(&init_mm);
flush_tlb_kernel_range(start, end); flush_tlb_kernel_range(start, end);
......
...@@ -110,7 +110,7 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, ...@@ -110,7 +110,7 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
do { do {
again: again:
next = pmd_addr_end(addr, end); next = pmd_addr_end(addr, end);
if (pmd_none(*pmd) || (!walk->vma && !walk->no_vma)) { if (pmd_none(*pmd)) {
if (ops->pte_hole) if (ops->pte_hole)
err = ops->pte_hole(addr, next, depth, walk); err = ops->pte_hole(addr, next, depth, walk);
if (err) if (err)
...@@ -171,7 +171,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, ...@@ -171,7 +171,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
do { do {
again: again:
next = pud_addr_end(addr, end); next = pud_addr_end(addr, end);
if (pud_none(*pud) || (!walk->vma && !walk->no_vma)) { if (pud_none(*pud)) {
if (ops->pte_hole) if (ops->pte_hole)
err = ops->pte_hole(addr, next, depth, walk); err = ops->pte_hole(addr, next, depth, walk);
if (err) if (err)
...@@ -366,19 +366,19 @@ static int __walk_page_range(unsigned long start, unsigned long end, ...@@ -366,19 +366,19 @@ static int __walk_page_range(unsigned long start, unsigned long end,
struct vm_area_struct *vma = walk->vma; struct vm_area_struct *vma = walk->vma;
const struct mm_walk_ops *ops = walk->ops; const struct mm_walk_ops *ops = walk->ops;
if (vma && ops->pre_vma) { if (ops->pre_vma) {
err = ops->pre_vma(start, end, walk); err = ops->pre_vma(start, end, walk);
if (err) if (err)
return err; return err;
} }
if (vma && is_vm_hugetlb_page(vma)) { if (is_vm_hugetlb_page(vma)) {
if (ops->hugetlb_entry) if (ops->hugetlb_entry)
err = walk_hugetlb_range(start, end, walk); err = walk_hugetlb_range(start, end, walk);
} else } else
err = walk_pgd_range(start, end, walk); err = walk_pgd_range(start, end, walk);
if (vma && ops->post_vma) if (ops->post_vma)
ops->post_vma(walk); ops->post_vma(walk);
return err; return err;
...@@ -450,9 +450,13 @@ int walk_page_range(struct mm_struct *mm, unsigned long start, ...@@ -450,9 +450,13 @@ int walk_page_range(struct mm_struct *mm, unsigned long start,
if (!vma) { /* after the last vma */ if (!vma) { /* after the last vma */
walk.vma = NULL; walk.vma = NULL;
next = end; next = end;
if (ops->pte_hole)
err = ops->pte_hole(start, next, -1, &walk);
} else if (start < vma->vm_start) { /* outside vma */ } else if (start < vma->vm_start) { /* outside vma */
walk.vma = NULL; walk.vma = NULL;
next = min(end, vma->vm_start); next = min(end, vma->vm_start);
if (ops->pte_hole)
err = ops->pte_hole(start, next, -1, &walk);
} else { /* inside vma */ } else { /* inside vma */
walk.vma = vma; walk.vma = vma;
next = min(end, vma->vm_end); next = min(end, vma->vm_end);
...@@ -470,9 +474,8 @@ int walk_page_range(struct mm_struct *mm, unsigned long start, ...@@ -470,9 +474,8 @@ int walk_page_range(struct mm_struct *mm, unsigned long start,
} }
if (err < 0) if (err < 0)
break; break;
}
if (walk.vma || walk.ops->pte_hole)
err = __walk_page_range(start, next, &walk); err = __walk_page_range(start, next, &walk);
}
if (err) if (err)
break; break;
} while (start = next, start < end); } while (start = next, start < end);
...@@ -501,9 +504,9 @@ int walk_page_range_novma(struct mm_struct *mm, unsigned long start, ...@@ -501,9 +504,9 @@ int walk_page_range_novma(struct mm_struct *mm, unsigned long start,
if (start >= end || !walk.mm) if (start >= end || !walk.mm)
return -EINVAL; return -EINVAL;
mmap_assert_locked(walk.mm); mmap_assert_write_locked(walk.mm);
return __walk_page_range(start, end, &walk); return walk_pgd_range(start, end, &walk);
} }
int walk_page_vma(struct vm_area_struct *vma, const struct mm_walk_ops *ops, int walk_page_vma(struct vm_area_struct *vma, const struct mm_walk_ops *ops,
......
...@@ -152,13 +152,13 @@ void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd) ...@@ -152,13 +152,13 @@ void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd)
{ {
const struct ptdump_range *range = st->range; const struct ptdump_range *range = st->range;
mmap_read_lock(mm); mmap_write_lock(mm);
while (range->start != range->end) { while (range->start != range->end) {
walk_page_range_novma(mm, range->start, range->end, walk_page_range_novma(mm, range->start, range->end,
&ptdump_ops, pgd, st); &ptdump_ops, pgd, st);
range++; range++;
} }
mmap_read_unlock(mm); mmap_write_unlock(mm);
/* Flush out the last page */ /* Flush out the last page */
st->note_page(st, 0, -1, 0); st->note_page(st, 0, -1, 0);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment