Skip to content

L
linux
Commit 2f1c6611 authored by Lorenzo Stoakes's avatar Lorenzo Stoakes Committed by Andrew Morton
Browse files
Options

mm: introduce vma_merge_struct and abstract vma_merge(),vma_modify() 

Rather than passing around huge numbers of parameters to numerous helper
functions, abstract them into a single struct that we thread through the
operation, the vma_merge_struct ('vmg').

Adjust vma_merge() and vma_modify() to accept this parameter, as well as
predicate functions can_vma_merge_before(), can_vma_merge_after(), and the
vma_modify_...() helper functions.

Also introduce VMG_STATE() and VMG_VMA_STATE() helper macros to allow for
easy vmg declaration.

We additionally remove the requirement that vma_merge() is passed a VMA
object representing the candidate new VMA.  Previously it used this to
obtain the mm_struct, file and anon_vma properties of the proposed range
(a rather confusing state of affairs), which are now provided by the vmg
directly.

We also remove the pgoff calculation previously performed vma_modify(),
and instead calculate this in VMG_VMA_STATE() via the vma_pgoff_offset()
helper.

Link: https://lkml.kernel.org/r/a955aad09d81329f6fbeb636b2dd10cde7b73dab.1725040657.git.lorenzo.stoakes@oracle.comSigned-off-by: default avatarLorenzo Stoakes <lorenzo.stoakes@oracle.com>
Reviewed-by: default avatarLiam R. Howlett <Liam.Howlett@oracle.com>
Cc: Mark Brown <broonie@kernel.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Bert Karwatzki <spasswolf@web.de>
Cc: Jeff Xu <jeffxu@chromium.org>
Cc: Jiri Olsa <olsajiri@gmail.com>
Cc: Kees Cook <kees@kernel.org>
Cc: Lorenzo Stoakes <lstoakes@gmail.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: "Paul E. McKenney" <paulmck@kernel.org>
Cc: Paul Moore <paul@paul-moore.com>
Cc: Sidhartha Kumar <sidhartha.kumar@oracle.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 955db396
Hide whitespace changes
Inline Side-by-side
Showing with 246 additions and 207 deletions
mm/mmap.c
View file @ 2f1c6611
...@@ -1373,10 +1373,11 @@ unsigned long mmap_region(struct file *file, unsigned long addr, ...@@ -1373,10 +1373,11 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
unsigned long end = addr + len; unsigned long end = addr + len;
unsigned long merge_start = addr, merge_end = end; unsigned long merge_start = addr, merge_end = end;
bool writable_file_mapping = false; bool writable_file_mapping = false;
pgoff_t vm_pgoff;
int error = -ENOMEM; int error = -ENOMEM;
VMA_ITERATOR(vmi, mm, addr); VMA_ITERATOR(vmi, mm, addr);
VMG_STATE(vmg, mm, &vmi, addr, end, vm_flags, pgoff);
vmg.file = file;
/* Find the first overlapping VMA */ /* Find the first overlapping VMA */
vma = vma_find(&vmi, end); vma = vma_find(&vmi, end);
init_vma_munmap(&vms, &vmi, vma, addr, end, uf, /* unlock = */ false); init_vma_munmap(&vms, &vmi, vma, addr, end, uf, /* unlock = */ false);
...@@ -1389,12 +1390,12 @@ unsigned long mmap_region(struct file *file, unsigned long addr, ...@@ -1389,12 +1390,12 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
if (error) if (error)
goto gather_failed; goto gather_failed;
next = vms.next; next = vmg.next = vms.next;
prev = vms.prev; prev = vmg.prev = vms.prev;
vma = NULL; vma = NULL;
} else { } else {
next = vma_next(&vmi); next = vmg.next = vma_next(&vmi);
prev = vma_prev(&vmi); prev = vmg.prev = vma_prev(&vmi);
if (prev) if (prev)
vma_iter_next_range(&vmi); vma_iter_next_range(&vmi);
} }
...@@ -1414,6 +1415,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, ...@@ -1414,6 +1415,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
vms.nr_accounted = 0; vms.nr_accounted = 0;
vm_flags |= VM_ACCOUNT; vm_flags |= VM_ACCOUNT;
vmg.flags = vm_flags;
} }
if (vm_flags & VM_SPECIAL) if (vm_flags & VM_SPECIAL)
...@@ -1422,28 +1424,31 @@ unsigned long mmap_region(struct file *file, unsigned long addr, ...@@ -1422,28 +1424,31 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
/* Attempt to expand an old mapping */ /* Attempt to expand an old mapping */
/* Check next */ /* Check next */
if (next && next->vm_start == end && !vma_policy(next) && if (next && next->vm_start == end && !vma_policy(next) &&
can_vma_merge_before(next, vm_flags, NULL, file, pgoff+pglen, can_vma_merge_before(&vmg)) {
NULL_VM_UFFD_CTX, NULL)) {
merge_end = next->vm_end; merge_end = next->vm_end;
vma = next; vma = next;
vm_pgoff = next->vm_pgoff - pglen; vmg.pgoff = next->vm_pgoff - pglen;
/*
* We set this here so if we will merge with the previous VMA in
* the code below, can_vma_merge_after() ensures anon_vma
* compatibility between prev and next.
*/
vmg.anon_vma = vma->anon_vma;
vmg.uffd_ctx = vma->vm_userfaultfd_ctx;
} }
/* Check prev */ /* Check prev */
if (prev && prev->vm_end == addr && !vma_policy(prev) && if (prev && prev->vm_end == addr && !vma_policy(prev) &&
(vma ? can_vma_merge_after(prev, vm_flags, vma->anon_vma, file, can_vma_merge_after(&vmg)) {
pgoff, vma->vm_userfaultfd_ctx, NULL) :
can_vma_merge_after(prev, vm_flags, NULL, file, pgoff,
NULL_VM_UFFD_CTX, NULL))) {
merge_start = prev->vm_start; merge_start = prev->vm_start;
vma = prev; vma = prev;
vm_pgoff = prev->vm_pgoff; vmg.pgoff = prev->vm_pgoff;
vma_prev(&vmi); /* Equivalent to going to the previous range */ vma_prev(&vmi); /* Equivalent to going to the previous range */
} }
if (vma) { if (vma) {
/* Actually expand, if possible */ /* Actually expand, if possible */
if (!vma_expand(&vmi, vma, merge_start, merge_end, vm_pgoff, next)) { if (!vma_expand(&vmi, vma, merge_start, merge_end, vmg.pgoff, next)) {
khugepaged_enter_vma(vma, vm_flags); khugepaged_enter_vma(vma, vm_flags);
goto expanded; goto expanded;
} }
...@@ -1774,26 +1779,29 @@ static int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, ...@@ -1774,26 +1779,29 @@ static int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma,
* Expand the existing vma if possible; Note that singular lists do not * Expand the existing vma if possible; Note that singular lists do not
* occur after forking, so the expand will only happen on new VMAs. * occur after forking, so the expand will only happen on new VMAs.
*/ */
if (vma && vma->vm_end == addr && !vma_policy(vma) && if (vma && vma->vm_end == addr && !vma_policy(vma)) {
can_vma_merge_after(vma, flags, NULL, NULL, VMG_STATE(vmg, mm, vmi, addr, addr + len, flags, PHYS_PFN(addr));
addr >> PAGE_SHIFT, NULL_VM_UFFD_CTX, NULL)) {
vma_iter_config(vmi, vma->vm_start, addr + len); vmg.prev = vma;
if (vma_iter_prealloc(vmi, vma)) if (can_vma_merge_after(&vmg)) {
goto unacct_fail; vma_iter_config(vmi, vma->vm_start, addr + len);
if (vma_iter_prealloc(vmi, vma))
vma_start_write(vma); goto unacct_fail;
init_vma_prep(&vp, vma); vma_start_write(vma);
vma_prepare(&vp);
vma_adjust_trans_huge(vma, vma->vm_start, addr + len, 0); init_vma_prep(&vp, vma);
vma->vm_end = addr + len; vma_prepare(&vp);
vm_flags_set(vma, VM_SOFTDIRTY); vma_adjust_trans_huge(vma, vma->vm_start, addr + len, 0);
vma_iter_store(vmi, vma); vma->vm_end = addr + len;
vm_flags_set(vma, VM_SOFTDIRTY);
vma_complete(&vp, vmi, mm); vma_iter_store(vmi, vma);
validate_mm(mm);
khugepaged_enter_vma(vma, flags); vma_complete(&vp, vmi, mm);
goto out; validate_mm(mm);
khugepaged_enter_vma(vma, flags);
goto out;
}
} }
if (vma) if (vma)
......
mm/vma.c
View file @ 2f1c6611
...@@ -7,16 +7,18 @@ ...@@ -7,16 +7,18 @@
#include "vma_internal.h" #include "vma_internal.h"
#include "vma.h" #include "vma.h"
/* static inline bool is_mergeable_vma(struct vma_merge_struct *vmg, bool merge_next)
* If the vma has a ->close operation then the driver probably needs to release
* per-vma resources, so we don't attempt to merge those if the caller indicates
* the current vma may be removed as part of the merge.
*/
static inline bool is_mergeable_vma(struct vm_area_struct *vma,
struct file *file, unsigned long vm_flags,
struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
struct anon_vma_name *anon_name, bool may_remove_vma)
{ {
struct vm_area_struct *vma = merge_next ? vmg->next : vmg->prev;
/*
* If the vma has a ->close operation then the driver probably needs to
* release per-vma resources, so we don't attempt to merge those if the
* caller indicates the current vma may be removed as part of the merge,
* which is the case if we are attempting to merge the next VMA into
* this one.
*/
bool may_remove_vma = merge_next;
/* /*
* VM_SOFTDIRTY should not prevent from VMA merging, if we * VM_SOFTDIRTY should not prevent from VMA merging, if we
* match the flags but dirty bit -- the caller should mark * match the flags but dirty bit -- the caller should mark
...@@ -25,15 +27,15 @@ static inline bool is_mergeable_vma(struct vm_area_struct *vma, ...@@ -25,15 +27,15 @@ static inline bool is_mergeable_vma(struct vm_area_struct *vma,
* the kernel to generate new VMAs when old one could be * the kernel to generate new VMAs when old one could be
* extended instead. * extended instead.
*/ */
if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY) if ((vma->vm_flags ^ vmg->flags) & ~VM_SOFTDIRTY)
return false; return false;
if (vma->vm_file != file) if (vma->vm_file != vmg->file)
return false; return false;
if (may_remove_vma && vma->vm_ops && vma->vm_ops->close) if (may_remove_vma && vma->vm_ops && vma->vm_ops->close)
return false; return false;
if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx)) if (!is_mergeable_vm_userfaultfd_ctx(vma, vmg->uffd_ctx))
return false; return false;
if (!anon_vma_name_eq(anon_vma_name(vma), anon_name)) if (!anon_vma_name_eq(anon_vma_name(vma), vmg->anon_name))
return false; return false;
return true; return true;
} }
...@@ -94,16 +96,16 @@ static void init_multi_vma_prep(struct vma_prepare *vp, ...@@ -94,16 +96,16 @@ static void init_multi_vma_prep(struct vma_prepare *vp,
* We assume the vma may be removed as part of the merge. * We assume the vma may be removed as part of the merge.
*/ */
bool bool
can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags, can_vma_merge_before(struct vma_merge_struct *vmg)
struct anon_vma *anon_vma, struct file *file,
pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
struct anon_vma_name *anon_name)
{ {
if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, true) && pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start);
is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
if (vma->vm_pgoff == vm_pgoff) if (is_mergeable_vma(vmg, /* merge_next = */ true) &&
is_mergeable_anon_vma(vmg->anon_vma, vmg->next->anon_vma, vmg->next)) {
if (vmg->next->vm_pgoff == vmg->pgoff + pglen)
return true; return true;
} }
return false; return false;
} }
...@@ -116,18 +118,11 @@ can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags, ...@@ -116,18 +118,11 @@ can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
* *
* We assume that vma is not removed as part of the merge. * We assume that vma is not removed as part of the merge.
*/ */
bool bool can_vma_merge_after(struct vma_merge_struct *vmg)
can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
struct anon_vma *anon_vma, struct file *file,
pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
struct anon_vma_name *anon_name)
{ {
if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, false) && if (is_mergeable_vma(vmg, /* merge_next = */ false) &&
is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { is_mergeable_anon_vma(vmg->anon_vma, vmg->prev->anon_vma, vmg->prev)) {
pgoff_t vm_pglen; if (vmg->prev->vm_pgoff + vma_pages(vmg->prev) == vmg->pgoff)
vm_pglen = vma_pages(vma);
if (vma->vm_pgoff + vm_pglen == vm_pgoff)
return true; return true;
} }
return false; return false;
...@@ -1017,16 +1012,10 @@ int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, ...@@ -1017,16 +1012,10 @@ int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
* **** is not represented - it will be merged and the vma containing the * **** is not represented - it will be merged and the vma containing the
* area is returned, or the function will return NULL * area is returned, or the function will return NULL
*/ */
static struct vm_area_struct static struct vm_area_struct *vma_merge(struct vma_merge_struct *vmg)
*vma_merge(struct vma_iterator *vmi, struct vm_area_struct *prev,
struct vm_area_struct *src, unsigned long addr, unsigned long end,
unsigned long vm_flags, pgoff_t pgoff, struct mempolicy *policy,
struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
struct anon_vma_name *anon_name)
{ {
struct mm_struct *mm = src->vm_mm; struct mm_struct *mm = vmg->mm;
struct anon_vma *anon_vma = src->anon_vma; struct vm_area_struct *prev = vmg->prev;
struct file *file = src->vm_file;
struct vm_area_struct *curr, *next, *res; struct vm_area_struct *curr, *next, *res;
struct vm_area_struct *vma, *adjust, *remove, *remove2; struct vm_area_struct *vma, *adjust, *remove, *remove2;
struct vm_area_struct *anon_dup = NULL; struct vm_area_struct *anon_dup = NULL;
...@@ -1036,16 +1025,18 @@ static struct vm_area_struct ...@@ -1036,16 +1025,18 @@ static struct vm_area_struct
bool merge_prev = false; bool merge_prev = false;
bool merge_next = false; bool merge_next = false;
bool vma_expanded = false; bool vma_expanded = false;
unsigned long addr = vmg->start;
unsigned long end = vmg->end;
unsigned long vma_start = addr; unsigned long vma_start = addr;
unsigned long vma_end = end; unsigned long vma_end = end;
pgoff_t pglen = (end - addr) >> PAGE_SHIFT; pgoff_t pglen = PHYS_PFN(end - addr);
long adj_start = 0; long adj_start = 0;
/* /*
* We later require that vma->vm_flags == vm_flags, * We later require that vma->vm_flags == vm_flags,
* so this tests vma->vm_flags & VM_SPECIAL, too. * so this tests vma->vm_flags & VM_SPECIAL, too.
*/ */
if (vm_flags & VM_SPECIAL) if (vmg->flags & VM_SPECIAL)
return NULL; return NULL;
/* Does the input range span an existing VMA? (cases 5 - 8) */ /* Does the input range span an existing VMA? (cases 5 - 8) */
...@@ -1053,27 +1044,26 @@ static struct vm_area_struct ...@@ -1053,27 +1044,26 @@ static struct vm_area_struct
if (!curr || /* cases 1 - 4 */ if (!curr || /* cases 1 - 4 */
end == curr->vm_end) /* cases 6 - 8, adjacent VMA */ end == curr->vm_end) /* cases 6 - 8, adjacent VMA */
next = vma_lookup(mm, end); next = vmg->next = vma_lookup(mm, end);
else else
next = NULL; /* case 5 */ next = vmg->next = NULL; /* case 5 */
if (prev) { if (prev) {
vma_start = prev->vm_start; vma_start = prev->vm_start;
vma_pgoff = prev->vm_pgoff; vma_pgoff = prev->vm_pgoff;
/* Can we merge the predecessor? */ /* Can we merge the predecessor? */
if (addr == prev->vm_end && mpol_equal(vma_policy(prev), policy) if (addr == prev->vm_end && mpol_equal(vma_policy(prev), vmg->policy)
&& can_vma_merge_after(prev, vm_flags, anon_vma, file, && can_vma_merge_after(vmg)) {
pgoff, vm_userfaultfd_ctx, anon_name)) {
merge_prev = true; merge_prev = true;
vma_prev(vmi); vma_prev(vmg->vmi);
} }
} }
/* Can we merge the successor? */ /* Can we merge the successor? */
if (next && mpol_equal(policy, vma_policy(next)) && if (next && mpol_equal(vmg->policy, vma_policy(next)) &&
can_vma_merge_before(next, vm_flags, anon_vma, file, pgoff+pglen, can_vma_merge_before(vmg)) {
vm_userfaultfd_ctx, anon_name)) {
merge_next = true; merge_next = true;
} }
...@@ -1164,13 +1154,13 @@ static struct vm_area_struct ...@@ -1164,13 +1154,13 @@ static struct vm_area_struct
vma_expanded = true; vma_expanded = true;
if (vma_expanded) { if (vma_expanded) {
vma_iter_config(vmi, vma_start, vma_end); vma_iter_config(vmg->vmi, vma_start, vma_end);
} else { } else {
vma_iter_config(vmi, adjust->vm_start + adj_start, vma_iter_config(vmg->vmi, adjust->vm_start + adj_start,
adjust->vm_end); adjust->vm_end);
} }
if (vma_iter_prealloc(vmi, vma)) if (vma_iter_prealloc(vmg->vmi, vma))
goto prealloc_fail; goto prealloc_fail;
init_multi_vma_prep(&vp, vma, adjust, remove, remove2); init_multi_vma_prep(&vp, vma, adjust, remove, remove2);
...@@ -1182,20 +1172,20 @@ static struct vm_area_struct ...@@ -1182,20 +1172,20 @@ static struct vm_area_struct
vma_set_range(vma, vma_start, vma_end, vma_pgoff); vma_set_range(vma, vma_start, vma_end, vma_pgoff);
if (vma_expanded) if (vma_expanded)
vma_iter_store(vmi, vma); vma_iter_store(vmg->vmi, vma);
if (adj_start) { if (adj_start) {
adjust->vm_start += adj_start; adjust->vm_start += adj_start;
adjust->vm_pgoff += adj_start >> PAGE_SHIFT; adjust->vm_pgoff += adj_start >> PAGE_SHIFT;
if (adj_start < 0) { if (adj_start < 0) {
WARN_ON(vma_expanded); WARN_ON(vma_expanded);
vma_iter_store(vmi, next); vma_iter_store(vmg->vmi, next);
} }
} }
vma_complete(&vp, vmi, mm); vma_complete(&vp, vmg->vmi, mm);
validate_mm(mm); validate_mm(mm);
khugepaged_enter_vma(res, vm_flags); khugepaged_enter_vma(res, vmg->flags);
return res; return res;
prealloc_fail: prealloc_fail:
...@@ -1203,8 +1193,8 @@ static struct vm_area_struct ...@@ -1203,8 +1193,8 @@ static struct vm_area_struct
unlink_anon_vmas(anon_dup); unlink_anon_vmas(anon_dup);
anon_vma_fail: anon_vma_fail:
vma_iter_set(vmi, addr); vma_iter_set(vmg->vmi, addr);
vma_iter_load(vmi); vma_iter_load(vmg->vmi);
return NULL; return NULL;
} }
...@@ -1221,32 +1211,27 @@ static struct vm_area_struct ...@@ -1221,32 +1211,27 @@ static struct vm_area_struct
* The function returns either the merged VMA, the original VMA if a split was * The function returns either the merged VMA, the original VMA if a split was
* required instead, or an error if the split failed. * required instead, or an error if the split failed.
*/ */
struct vm_area_struct *vma_modify(struct vma_iterator *vmi, static struct vm_area_struct *vma_modify(struct vma_merge_struct *vmg)
struct vm_area_struct *prev,
struct vm_area_struct *vma,
unsigned long start, unsigned long end,
unsigned long vm_flags,
struct mempolicy *policy,
struct vm_userfaultfd_ctx uffd_ctx,
struct anon_vma_name *anon_name)
{ {
pgoff_t pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); struct vm_area_struct *vma = vmg->vma;
struct vm_area_struct *merged; struct vm_area_struct *merged;
merged = vma_merge(vmi, prev, vma, start, end, vm_flags, /* First, try to merge. */
pgoff, policy, uffd_ctx, anon_name); merged = vma_merge(vmg);
if (merged) if (merged)
return merged; return merged;
if (vma->vm_start < start) { /* Split any preceding portion of the VMA. */
int err = split_vma(vmi, vma, start, 1); if (vma->vm_start < vmg->start) {
int err = split_vma(vmg->vmi, vma, vmg->start, 1);
if (err) if (err)
return ERR_PTR(err); return ERR_PTR(err);
} }
if (vma->vm_end > end) { /* Split any trailing portion of the VMA. */
int err = split_vma(vmi, vma, end, 0); if (vma->vm_end > vmg->end) {
int err = split_vma(vmg->vmi, vma, vmg->end, 0);
if (err) if (err)
return ERR_PTR(err); return ERR_PTR(err);
...@@ -1255,6 +1240,65 @@ struct vm_area_struct *vma_modify(struct vma_iterator *vmi, ...@@ -1255,6 +1240,65 @@ struct vm_area_struct *vma_modify(struct vma_iterator *vmi,
return vma; return vma;
} }
struct vm_area_struct *vma_modify_flags(
struct vma_iterator *vmi, struct vm_area_struct *prev,
struct vm_area_struct *vma, unsigned long start, unsigned long end,
unsigned long new_flags)
{
VMG_VMA_STATE(vmg, vmi, prev, vma, start, end);
vmg.flags = new_flags;
return vma_modify(&vmg);
}
struct vm_area_struct
*vma_modify_flags_name(struct vma_iterator *vmi,
struct vm_area_struct *prev,
struct vm_area_struct *vma,
unsigned long start,
unsigned long end,
unsigned long new_flags,
struct anon_vma_name *new_name)
{
VMG_VMA_STATE(vmg, vmi, prev, vma, start, end);
vmg.flags = new_flags;
vmg.anon_name = new_name;
return vma_modify(&vmg);
}
struct vm_area_struct
*vma_modify_policy(struct vma_iterator *vmi,
struct vm_area_struct *prev,
struct vm_area_struct *vma,
unsigned long start, unsigned long end,
struct mempolicy *new_pol)
{
VMG_VMA_STATE(vmg, vmi, prev, vma, start, end);
vmg.policy = new_pol;
return vma_modify(&vmg);
}
struct vm_area_struct
*vma_modify_flags_uffd(struct vma_iterator *vmi,
struct vm_area_struct *prev,
struct vm_area_struct *vma,
unsigned long start, unsigned long end,
unsigned long new_flags,
struct vm_userfaultfd_ctx new_ctx)
{
VMG_VMA_STATE(vmg, vmi, prev, vma, start, end);
vmg.flags = new_flags;
vmg.uffd_ctx = new_ctx;
return vma_modify(&vmg);
}
/* /*
* Attempt to merge a newly mapped VMA with those adjacent to it. The caller * Attempt to merge a newly mapped VMA with those adjacent to it. The caller
* must ensure that [start, end) does not overlap any existing VMA. * must ensure that [start, end) does not overlap any existing VMA.
...@@ -1264,8 +1308,11 @@ struct vm_area_struct ...@@ -1264,8 +1308,11 @@ struct vm_area_struct
struct vm_area_struct *vma, unsigned long start, struct vm_area_struct *vma, unsigned long start,
unsigned long end, pgoff_t pgoff) unsigned long end, pgoff_t pgoff)
{ {
return vma_merge(vmi, prev, vma, start, end, vma->vm_flags, pgoff, VMG_VMA_STATE(vmg, vmi, prev, vma, start, end);
vma_policy(vma), vma->vm_userfaultfd_ctx, anon_vma_name(vma));
vmg.pgoff = pgoff;
return vma_merge(&vmg);
} }
/* /*
...@@ -1276,12 +1323,10 @@ struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi, ...@@ -1276,12 +1323,10 @@ struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
struct vm_area_struct *vma, struct vm_area_struct *vma,
unsigned long delta) unsigned long delta)
{ {
pgoff_t pgoff = vma->vm_pgoff + vma_pages(vma); VMG_VMA_STATE(vmg, vmi, vma, vma, vma->vm_end, vma->vm_end + delta);
/* vma is specified as prev, so case 1 or 2 will apply. */ /* vma is specified as prev, so case 1 or 2 will apply. */
return vma_merge(vmi, vma, vma, vma->vm_end, vma->vm_end + delta, return vma_merge(&vmg);
vma->vm_flags, pgoff, vma_policy(vma),
vma->vm_userfaultfd_ctx, anon_vma_name(vma));
} }
void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb) void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb)
......
mm/vma.h
View file @ 2f1c6611
...@@ -52,6 +52,59 @@ struct vma_munmap_struct { ...@@ -52,6 +52,59 @@ struct vma_munmap_struct {
unsigned long data_vm; unsigned long data_vm;
}; };
/* Represents a VMA merge operation. */
struct vma_merge_struct {
struct mm_struct *mm;
struct vma_iterator *vmi;
pgoff_t pgoff;
struct vm_area_struct *prev;
struct vm_area_struct *next; /* Modified by vma_merge(). */
struct vm_area_struct *vma; /* Either a new VMA or the one being modified. */
unsigned long start;
unsigned long end;
unsigned long flags;
struct file *file;
struct anon_vma *anon_vma;
struct mempolicy *policy;
struct vm_userfaultfd_ctx uffd_ctx;
struct anon_vma_name *anon_name;
};
/* Assumes addr >= vma->vm_start. */
static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma,
unsigned long addr)
{
return vma->vm_pgoff + PHYS_PFN(addr - vma->vm_start);
}
#define VMG_STATE(name, mm_, vmi_, start_, end_, flags_, pgoff_) \
struct vma_merge_struct name = { \
.mm = mm_, \
.vmi = vmi_, \
.start = start_, \
.end = end_, \
.flags = flags_, \
.pgoff = pgoff_, \
}
#define VMG_VMA_STATE(name, vmi_, prev_, vma_, start_, end_) \
struct vma_merge_struct name = { \
.mm = vma_->vm_mm, \
.vmi = vmi_, \
.prev = prev_, \
.next = NULL, \
.vma = vma_, \
.start = start_, \
.end = end_, \
.flags = vma_->vm_flags, \
.pgoff = vma_pgoff_offset(vma_, start_), \
.file = vma_->vm_file, \
.anon_vma = vma_->anon_vma, \
.policy = vma_policy(vma_), \
.uffd_ctx = vma_->vm_userfaultfd_ctx, \
.anon_name = anon_vma_name(vma_), \
}
#ifdef CONFIG_DEBUG_VM_MAPLE_TREE #ifdef CONFIG_DEBUG_VM_MAPLE_TREE
void validate_mm(struct mm_struct *mm); void validate_mm(struct mm_struct *mm);
#else #else
...@@ -212,80 +265,52 @@ void remove_vma(struct vm_area_struct *vma, bool unreachable, bool closed); ...@@ -212,80 +265,52 @@ void remove_vma(struct vm_area_struct *vma, bool unreachable, bool closed);
void unmap_region(struct ma_state *mas, struct vm_area_struct *vma, void unmap_region(struct ma_state *mas, struct vm_area_struct *vma,
struct vm_area_struct *prev, struct vm_area_struct *next); struct vm_area_struct *prev, struct vm_area_struct *next);
/* Required by mmap_region(). */ /*
bool * Can we merge the VMA described by vmg into the following VMA vmg->next?
can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags, *
struct anon_vma *anon_vma, struct file *file, * Required by mmap_region().
pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx, */
struct anon_vma_name *anon_name); bool can_vma_merge_before(struct vma_merge_struct *vmg);
/* Required by mmap_region() and do_brk_flags(). */ /*
bool * Can we merge the VMA described by vmg into the preceding VMA vmg->prev?
can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags, *
struct anon_vma *anon_vma, struct file *file, * Required by mmap_region() and do_brk_flags().
pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx, */
struct anon_vma_name *anon_name); bool can_vma_merge_after(struct vma_merge_struct *vmg);
struct vm_area_struct *vma_modify(struct vma_iterator *vmi,
struct vm_area_struct *prev,
struct vm_area_struct *vma,
unsigned long start, unsigned long end,
unsigned long vm_flags,
struct mempolicy *policy,
struct vm_userfaultfd_ctx uffd_ctx,
struct anon_vma_name *anon_name);
/* We are about to modify the VMA's flags. */ /* We are about to modify the VMA's flags. */
static inline struct vm_area_struct struct vm_area_struct *vma_modify_flags(struct vma_iterator *vmi,
*vma_modify_flags(struct vma_iterator *vmi, struct vm_area_struct *prev, struct vm_area_struct *vma,
struct vm_area_struct *prev, unsigned long start, unsigned long end,
struct vm_area_struct *vma, unsigned long new_flags);
unsigned long start, unsigned long end,
unsigned long new_flags)
{
return vma_modify(vmi, prev, vma, start, end, new_flags,
vma_policy(vma), vma->vm_userfaultfd_ctx,
anon_vma_name(vma));
}
/* We are about to modify the VMA's flags and/or anon_name. */ /* We are about to modify the VMA's flags and/or anon_name. */
static inline struct vm_area_struct struct vm_area_struct
*vma_modify_flags_name(struct vma_iterator *vmi, *vma_modify_flags_name(struct vma_iterator *vmi,
struct vm_area_struct *prev, struct vm_area_struct *prev,
struct vm_area_struct *vma, struct vm_area_struct *vma,
unsigned long start, unsigned long start,
unsigned long end, unsigned long end,
unsigned long new_flags, unsigned long new_flags,
struct anon_vma_name *new_name) struct anon_vma_name *new_name);
{
return vma_modify(vmi, prev, vma, start, end, new_flags,
vma_policy(vma), vma->vm_userfaultfd_ctx, new_name);
}
/* We are about to modify the VMA's memory policy. */ /* We are about to modify the VMA's memory policy. */
static inline struct vm_area_struct struct vm_area_struct
*vma_modify_policy(struct vma_iterator *vmi, *vma_modify_policy(struct vma_iterator *vmi,
struct vm_area_struct *prev, struct vm_area_struct *prev,
struct vm_area_struct *vma, struct vm_area_struct *vma,
unsigned long start, unsigned long end, unsigned long start, unsigned long end,
struct mempolicy *new_pol) struct mempolicy *new_pol);
{
return vma_modify(vmi, prev, vma, start, end, vma->vm_flags,
new_pol, vma->vm_userfaultfd_ctx, anon_vma_name(vma));
}
/* We are about to modify the VMA's flags and/or uffd context. */ /* We are about to modify the VMA's flags and/or uffd context. */
static inline struct vm_area_struct struct vm_area_struct
*vma_modify_flags_uffd(struct vma_iterator *vmi, *vma_modify_flags_uffd(struct vma_iterator *vmi,
struct vm_area_struct *prev, struct vm_area_struct *prev,
struct vm_area_struct *vma, struct vm_area_struct *vma,
unsigned long start, unsigned long end, unsigned long start, unsigned long end,
unsigned long new_flags, unsigned long new_flags,
struct vm_userfaultfd_ctx new_ctx) struct vm_userfaultfd_ctx new_ctx);
{
return vma_modify(vmi, prev, vma, start, end, new_flags,
vma_policy(vma), new_ctx, anon_vma_name(vma));
}
struct vm_area_struct struct vm_area_struct
*vma_merge_new_vma(struct vma_iterator *vmi, struct vm_area_struct *prev, *vma_merge_new_vma(struct vma_iterator *vmi, struct vm_area_struct *prev,
......
tools/testing/vma/vma.c
View file @ 2f1c6611
...@@ -22,26 +22,6 @@ static bool fail_prealloc; ...@@ -22,26 +22,6 @@ static bool fail_prealloc;
*/ */
#include "../../../mm/vma.c" #include "../../../mm/vma.c"
/*
* Temporarily forward-ported from a future in which vmg's are used for merging.
*/
struct vma_merge_struct {
struct mm_struct *mm;
struct vma_iterator *vmi;
pgoff_t pgoff;
struct vm_area_struct *prev;
struct vm_area_struct *next; /* Modified by vma_merge(). */
struct vm_area_struct *vma; /* Either a new VMA or the one being modified. */
unsigned long start;
unsigned long end;
unsigned long flags;
struct file *file;
struct anon_vma *anon_vma;
struct mempolicy *policy;
struct vm_userfaultfd_ctx uffd_ctx;
struct anon_vma_name *anon_name;
};
const struct vm_operations_struct vma_dummy_vm_ops; const struct vm_operations_struct vma_dummy_vm_ops;
static struct anon_vma dummy_anon_vma; static struct anon_vma dummy_anon_vma;
...@@ -115,14 +95,6 @@ static struct vm_area_struct *alloc_and_link_vma(struct mm_struct *mm, ...@@ -115,14 +95,6 @@ static struct vm_area_struct *alloc_and_link_vma(struct mm_struct *mm,
/* Helper function which provides a wrapper around a merge new VMA operation. */ /* Helper function which provides a wrapper around a merge new VMA operation. */
static struct vm_area_struct *merge_new(struct vma_merge_struct *vmg) static struct vm_area_struct *merge_new(struct vma_merge_struct *vmg)
{ {
/* vma_merge() needs a VMA to determine mm, anon_vma, and file. */
struct vm_area_struct dummy = {
.vm_mm = vmg->mm,
.vm_flags = vmg->flags,
.anon_vma = vmg->anon_vma,
.vm_file = vmg->file,
};
/* /*
* For convenience, get prev and next VMAs. Which the new VMA operation * For convenience, get prev and next VMAs. Which the new VMA operation
* requires. * requires.
...@@ -131,8 +103,7 @@ static struct vm_area_struct *merge_new(struct vma_merge_struct *vmg) ...@@ -131,8 +103,7 @@ static struct vm_area_struct *merge_new(struct vma_merge_struct *vmg)
vmg->prev = vma_prev(vmg->vmi); vmg->prev = vma_prev(vmg->vmi);
vma_iter_set(vmg->vmi, vmg->start); vma_iter_set(vmg->vmi, vmg->start);
return vma_merge_new_vma(vmg->vmi, vmg->prev, &dummy, vmg->start, return vma_merge(vmg);
vmg->end, vmg->pgoff);
} }
/* /*
...@@ -141,17 +112,7 @@ static struct vm_area_struct *merge_new(struct vma_merge_struct *vmg) ...@@ -141,17 +112,7 @@ static struct vm_area_struct *merge_new(struct vma_merge_struct *vmg)
*/ */
static struct vm_area_struct *merge_existing(struct vma_merge_struct *vmg) static struct vm_area_struct *merge_existing(struct vma_merge_struct *vmg)
{ {
/* vma_merge() needs a VMA to determine mm, anon_vma, and file. */ return vma_merge(vmg);
struct vm_area_struct dummy = {
.vm_mm = vmg->mm,
.vm_flags = vmg->flags,
.anon_vma = vmg->anon_vma,
.vm_file = vmg->file,
};
return vma_merge(vmg->vmi, vmg->prev, &dummy, vmg->start, vmg->end,
vmg->flags, vmg->pgoff, vmg->policy, vmg->uffd_ctx,
vmg->anon_name);
} }
/* /*
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7