Commit 7ed9e3d9 authored by Si-Wei Liu's avatar Si-Wei Liu Committed by Michael S. Tsirkin

vhost-vdpa: fix page pinning leakage in error path

Pinned pages are not properly accounted particularly when
mapping error occurs on IOTLB update. Clean up dangling
pinned pages for the error path. As the inflight pinned
pages, specifically for memory region that strides across
multiple chunks, would need more than one free page for
book keeping and accounting. For simplicity, pin pages
for all memory in the IOVA range in one go rather than
have multiple pin_user_pages calls to make up the entire
region. This way it's easier to track and account the
pages already mapped, particularly for clean-up in the
error path.

Fixes: 4c8cf318 ("vhost: introduce vDPA-based backend")
Signed-off-by: default avatarSi-Wei Liu <si-wei.liu@oracle.com>
Link: https://lore.kernel.org/r/1601701330-16837-3-git-send-email-si-wei.liu@oracle.comSigned-off-by: default avatarMichael S. Tsirkin <mst@redhat.com>
parent 1477c8ae
...@@ -595,21 +595,19 @@ static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v, ...@@ -595,21 +595,19 @@ static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v,
struct vhost_dev *dev = &v->vdev; struct vhost_dev *dev = &v->vdev;
struct vhost_iotlb *iotlb = dev->iotlb; struct vhost_iotlb *iotlb = dev->iotlb;
struct page **page_list; struct page **page_list;
unsigned long list_size = PAGE_SIZE / sizeof(struct page *); struct vm_area_struct **vmas;
unsigned int gup_flags = FOLL_LONGTERM; unsigned int gup_flags = FOLL_LONGTERM;
unsigned long npages, cur_base, map_pfn, last_pfn = 0; unsigned long map_pfn, last_pfn = 0;
unsigned long locked, lock_limit, pinned, i; unsigned long npages, lock_limit;
unsigned long i, nmap = 0;
u64 iova = msg->iova; u64 iova = msg->iova;
long pinned;
int ret = 0; int ret = 0;
if (vhost_iotlb_itree_first(iotlb, msg->iova, if (vhost_iotlb_itree_first(iotlb, msg->iova,
msg->iova + msg->size - 1)) msg->iova + msg->size - 1))
return -EEXIST; return -EEXIST;
page_list = (struct page **) __get_free_page(GFP_KERNEL);
if (!page_list)
return -ENOMEM;
if (msg->perm & VHOST_ACCESS_WO) if (msg->perm & VHOST_ACCESS_WO)
gup_flags |= FOLL_WRITE; gup_flags |= FOLL_WRITE;
...@@ -617,61 +615,86 @@ static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v, ...@@ -617,61 +615,86 @@ static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v,
if (!npages) if (!npages)
return -EINVAL; return -EINVAL;
page_list = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
vmas = kvmalloc_array(npages, sizeof(struct vm_area_struct *),
GFP_KERNEL);
if (!page_list || !vmas) {
ret = -ENOMEM;
goto free;
}
mmap_read_lock(dev->mm); mmap_read_lock(dev->mm);
locked = atomic64_add_return(npages, &dev->mm->pinned_vm);
lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
if (npages + atomic64_read(&dev->mm->pinned_vm) > lock_limit) {
ret = -ENOMEM;
goto unlock;
}
if (locked > lock_limit) { pinned = pin_user_pages(msg->uaddr & PAGE_MASK, npages, gup_flags,
page_list, vmas);
if (npages != pinned) {
if (pinned < 0) {
ret = pinned;
} else {
unpin_user_pages(page_list, pinned);
ret = -ENOMEM; ret = -ENOMEM;
goto out; }
goto unlock;
} }
cur_base = msg->uaddr & PAGE_MASK;
iova &= PAGE_MASK; iova &= PAGE_MASK;
while (npages) {
pinned = min_t(unsigned long, npages, list_size);
ret = pin_user_pages(cur_base, pinned,
gup_flags, page_list, NULL);
if (ret != pinned)
goto out;
if (!last_pfn)
map_pfn = page_to_pfn(page_list[0]); map_pfn = page_to_pfn(page_list[0]);
for (i = 0; i < ret; i++) { /* One more iteration to avoid extra vdpa_map() call out of loop. */
unsigned long this_pfn = page_to_pfn(page_list[i]); for (i = 0; i <= npages; i++) {
unsigned long this_pfn;
u64 csize; u64 csize;
if (last_pfn && (this_pfn != last_pfn + 1)) { /* The last chunk may have no valid PFN next to it */
this_pfn = i < npages ? page_to_pfn(page_list[i]) : -1UL;
if (last_pfn && (this_pfn == -1UL ||
this_pfn != last_pfn + 1)) {
/* Pin a contiguous chunk of memory */ /* Pin a contiguous chunk of memory */
csize = (last_pfn - map_pfn + 1) << PAGE_SHIFT; csize = last_pfn - map_pfn + 1;
if (vhost_vdpa_map(v, iova, csize, ret = vhost_vdpa_map(v, iova, csize << PAGE_SHIFT,
map_pfn << PAGE_SHIFT, map_pfn << PAGE_SHIFT,
msg->perm)) msg->perm);
if (ret) {
/*
* Unpin the rest chunks of memory on the
* flight with no corresponding vdpa_map()
* calls having been made yet. On the other
* hand, vdpa_unmap() in the failure path
* is in charge of accounting the number of
* pinned pages for its own.
* This asymmetrical pattern of accounting
* is for efficiency to pin all pages at
* once, while there is no other callsite
* of vdpa_map() than here above.
*/
unpin_user_pages(&page_list[nmap],
npages - nmap);
goto out; goto out;
}
atomic64_add(csize, &dev->mm->pinned_vm);
nmap += csize;
iova += csize << PAGE_SHIFT;
map_pfn = this_pfn; map_pfn = this_pfn;
iova += csize;
} }
last_pfn = this_pfn; last_pfn = this_pfn;
} }
cur_base += ret << PAGE_SHIFT; WARN_ON(nmap != npages);
npages -= ret;
}
/* Pin the rest chunk */
ret = vhost_vdpa_map(v, iova, (last_pfn - map_pfn + 1) << PAGE_SHIFT,
map_pfn << PAGE_SHIFT, msg->perm);
out: out:
if (ret) { if (ret)
vhost_vdpa_unmap(v, msg->iova, msg->size); vhost_vdpa_unmap(v, msg->iova, msg->size);
atomic64_sub(npages, &dev->mm->pinned_vm); unlock:
}
mmap_read_unlock(dev->mm); mmap_read_unlock(dev->mm);
free_page((unsigned long)page_list); free:
kvfree(vmas);
kvfree(page_list);
return ret; return ret;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment