Commit 64019a2e authored by Peter Xu's avatar Peter Xu Committed by Linus Torvalds

mm/gup: remove task_struct pointer for all gup code

After the cleanup of page fault accounting, gup does not need to pass
task_struct around any more.  Remove that parameter in the whole gup
stack.
Signed-off-by: default avatarPeter Xu <peterx@redhat.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Reviewed-by: default avatarJohn Hubbard <jhubbard@nvidia.com>
Link: http://lkml.kernel.org/r/20200707225021.200906-26-peterx@redhat.comSigned-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent a2beb5f1
...@@ -91,7 +91,7 @@ SYSCALL_DEFINE3(arc_usr_cmpxchg, int *, uaddr, int, expected, int, new) ...@@ -91,7 +91,7 @@ SYSCALL_DEFINE3(arc_usr_cmpxchg, int *, uaddr, int, expected, int, new)
goto fail; goto fail;
mmap_read_lock(current->mm); mmap_read_lock(current->mm);
ret = fixup_user_fault(current, current->mm, (unsigned long) uaddr, ret = fixup_user_fault(current->mm, (unsigned long) uaddr,
FAULT_FLAG_WRITE, NULL); FAULT_FLAG_WRITE, NULL);
mmap_read_unlock(current->mm); mmap_read_unlock(current->mm);
......
...@@ -2768,7 +2768,7 @@ static struct page *get_map_page(struct kvm *kvm, u64 uaddr) ...@@ -2768,7 +2768,7 @@ static struct page *get_map_page(struct kvm *kvm, u64 uaddr)
struct page *page = NULL; struct page *page = NULL;
mmap_read_lock(kvm->mm); mmap_read_lock(kvm->mm);
get_user_pages_remote(NULL, kvm->mm, uaddr, 1, FOLL_WRITE, get_user_pages_remote(kvm->mm, uaddr, 1, FOLL_WRITE,
&page, NULL, NULL); &page, NULL, NULL);
mmap_read_unlock(kvm->mm); mmap_read_unlock(kvm->mm);
return page; return page;
......
...@@ -1892,7 +1892,7 @@ static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) ...@@ -1892,7 +1892,7 @@ static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
r = set_guest_storage_key(current->mm, hva, keys[i], 0); r = set_guest_storage_key(current->mm, hva, keys[i], 0);
if (r) { if (r) {
r = fixup_user_fault(current, current->mm, hva, r = fixup_user_fault(current->mm, hva,
FAULT_FLAG_WRITE, &unlocked); FAULT_FLAG_WRITE, &unlocked);
if (r) if (r)
break; break;
......
...@@ -273,7 +273,7 @@ static int handle_iske(struct kvm_vcpu *vcpu) ...@@ -273,7 +273,7 @@ static int handle_iske(struct kvm_vcpu *vcpu)
rc = get_guest_storage_key(current->mm, vmaddr, &key); rc = get_guest_storage_key(current->mm, vmaddr, &key);
if (rc) { if (rc) {
rc = fixup_user_fault(current, current->mm, vmaddr, rc = fixup_user_fault(current->mm, vmaddr,
FAULT_FLAG_WRITE, &unlocked); FAULT_FLAG_WRITE, &unlocked);
if (!rc) { if (!rc) {
mmap_read_unlock(current->mm); mmap_read_unlock(current->mm);
...@@ -319,7 +319,7 @@ static int handle_rrbe(struct kvm_vcpu *vcpu) ...@@ -319,7 +319,7 @@ static int handle_rrbe(struct kvm_vcpu *vcpu)
mmap_read_lock(current->mm); mmap_read_lock(current->mm);
rc = reset_guest_reference_bit(current->mm, vmaddr); rc = reset_guest_reference_bit(current->mm, vmaddr);
if (rc < 0) { if (rc < 0) {
rc = fixup_user_fault(current, current->mm, vmaddr, rc = fixup_user_fault(current->mm, vmaddr,
FAULT_FLAG_WRITE, &unlocked); FAULT_FLAG_WRITE, &unlocked);
if (!rc) { if (!rc) {
mmap_read_unlock(current->mm); mmap_read_unlock(current->mm);
...@@ -390,7 +390,7 @@ static int handle_sske(struct kvm_vcpu *vcpu) ...@@ -390,7 +390,7 @@ static int handle_sske(struct kvm_vcpu *vcpu)
m3 & SSKE_MC); m3 & SSKE_MC);
if (rc < 0) { if (rc < 0) {
rc = fixup_user_fault(current, current->mm, vmaddr, rc = fixup_user_fault(current->mm, vmaddr,
FAULT_FLAG_WRITE, &unlocked); FAULT_FLAG_WRITE, &unlocked);
rc = !rc ? -EAGAIN : rc; rc = !rc ? -EAGAIN : rc;
} }
...@@ -1094,7 +1094,7 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) ...@@ -1094,7 +1094,7 @@ static int handle_pfmf(struct kvm_vcpu *vcpu)
rc = cond_set_guest_storage_key(current->mm, vmaddr, rc = cond_set_guest_storage_key(current->mm, vmaddr,
key, NULL, nq, mr, mc); key, NULL, nq, mr, mc);
if (rc < 0) { if (rc < 0) {
rc = fixup_user_fault(current, current->mm, vmaddr, rc = fixup_user_fault(current->mm, vmaddr,
FAULT_FLAG_WRITE, &unlocked); FAULT_FLAG_WRITE, &unlocked);
rc = !rc ? -EAGAIN : rc; rc = !rc ? -EAGAIN : rc;
} }
......
...@@ -649,7 +649,7 @@ int gmap_fault(struct gmap *gmap, unsigned long gaddr, ...@@ -649,7 +649,7 @@ int gmap_fault(struct gmap *gmap, unsigned long gaddr,
rc = vmaddr; rc = vmaddr;
goto out_up; goto out_up;
} }
if (fixup_user_fault(current, gmap->mm, vmaddr, fault_flags, if (fixup_user_fault(gmap->mm, vmaddr, fault_flags,
&unlocked)) { &unlocked)) {
rc = -EFAULT; rc = -EFAULT;
goto out_up; goto out_up;
...@@ -879,7 +879,7 @@ static int gmap_pte_op_fixup(struct gmap *gmap, unsigned long gaddr, ...@@ -879,7 +879,7 @@ static int gmap_pte_op_fixup(struct gmap *gmap, unsigned long gaddr,
BUG_ON(gmap_is_shadow(gmap)); BUG_ON(gmap_is_shadow(gmap));
fault_flags = (prot == PROT_WRITE) ? FAULT_FLAG_WRITE : 0; fault_flags = (prot == PROT_WRITE) ? FAULT_FLAG_WRITE : 0;
if (fixup_user_fault(current, mm, vmaddr, fault_flags, &unlocked)) if (fixup_user_fault(mm, vmaddr, fault_flags, &unlocked))
return -EFAULT; return -EFAULT;
if (unlocked) if (unlocked)
/* lost mmap_lock, caller has to retry __gmap_translate */ /* lost mmap_lock, caller has to retry __gmap_translate */
......
...@@ -469,7 +469,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work) ...@@ -469,7 +469,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
locked = 1; locked = 1;
} }
ret = pin_user_pages_remote ret = pin_user_pages_remote
(work->task, mm, (mm,
obj->userptr.ptr + pinned * PAGE_SIZE, obj->userptr.ptr + pinned * PAGE_SIZE,
npages - pinned, npages - pinned,
flags, flags,
......
...@@ -439,7 +439,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt, ...@@ -439,7 +439,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt,
* complex (and doesn't gain us much performance in most use * complex (and doesn't gain us much performance in most use
* cases). * cases).
*/ */
npages = get_user_pages_remote(owning_process, owning_mm, npages = get_user_pages_remote(owning_mm,
user_virt, gup_num_pages, user_virt, gup_num_pages,
flags, local_page_list, NULL, NULL); flags, local_page_list, NULL, NULL);
mmap_read_unlock(owning_mm); mmap_read_unlock(owning_mm);
......
...@@ -425,7 +425,7 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, ...@@ -425,7 +425,7 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm,
if (ret) { if (ret) {
bool unlocked = false; bool unlocked = false;
ret = fixup_user_fault(NULL, mm, vaddr, ret = fixup_user_fault(mm, vaddr,
FAULT_FLAG_REMOTE | FAULT_FLAG_REMOTE |
(write_fault ? FAULT_FLAG_WRITE : 0), (write_fault ? FAULT_FLAG_WRITE : 0),
&unlocked); &unlocked);
...@@ -453,7 +453,7 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, ...@@ -453,7 +453,7 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
flags |= FOLL_WRITE; flags |= FOLL_WRITE;
mmap_read_lock(mm); mmap_read_lock(mm);
ret = pin_user_pages_remote(NULL, mm, vaddr, 1, flags | FOLL_LONGTERM, ret = pin_user_pages_remote(mm, vaddr, 1, flags | FOLL_LONGTERM,
page, NULL, NULL); page, NULL, NULL);
if (ret == 1) { if (ret == 1) {
*pfn = page_to_pfn(page[0]); *pfn = page_to_pfn(page[0]);
......
...@@ -217,7 +217,7 @@ static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, ...@@ -217,7 +217,7 @@ static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos,
* We are doing an exec(). 'current' is the process * We are doing an exec(). 'current' is the process
* doing the exec and bprm->mm is the new process's mm. * doing the exec and bprm->mm is the new process's mm.
*/ */
ret = get_user_pages_remote(current, bprm->mm, pos, 1, gup_flags, ret = get_user_pages_remote(bprm->mm, pos, 1, gup_flags,
&page, NULL, NULL); &page, NULL, NULL);
if (ret <= 0) if (ret <= 0)
return NULL; return NULL;
......
...@@ -1661,7 +1661,7 @@ int invalidate_inode_page(struct page *page); ...@@ -1661,7 +1661,7 @@ int invalidate_inode_page(struct page *page);
extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma, extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
unsigned long address, unsigned int flags, unsigned long address, unsigned int flags,
struct pt_regs *regs); struct pt_regs *regs);
extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm, extern int fixup_user_fault(struct mm_struct *mm,
unsigned long address, unsigned int fault_flags, unsigned long address, unsigned int fault_flags,
bool *unlocked); bool *unlocked);
void unmap_mapping_pages(struct address_space *mapping, void unmap_mapping_pages(struct address_space *mapping,
...@@ -1677,8 +1677,7 @@ static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma, ...@@ -1677,8 +1677,7 @@ static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
BUG(); BUG();
return VM_FAULT_SIGBUS; return VM_FAULT_SIGBUS;
} }
static inline int fixup_user_fault(struct task_struct *tsk, static inline int fixup_user_fault(struct mm_struct *mm, unsigned long address,
struct mm_struct *mm, unsigned long address,
unsigned int fault_flags, bool *unlocked) unsigned int fault_flags, bool *unlocked)
{ {
/* should never happen if there's no MMU */ /* should never happen if there's no MMU */
...@@ -1704,11 +1703,11 @@ extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, ...@@ -1704,11 +1703,11 @@ extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
extern int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, extern int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
unsigned long addr, void *buf, int len, unsigned int gup_flags); unsigned long addr, void *buf, int len, unsigned int gup_flags);
long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, long get_user_pages_remote(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *locked); struct vm_area_struct **vmas, int *locked);
long pin_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, long pin_user_pages_remote(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *locked); struct vm_area_struct **vmas, int *locked);
......
...@@ -376,7 +376,7 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) ...@@ -376,7 +376,7 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d)
if (!vaddr || !d) if (!vaddr || !d)
return -EINVAL; return -EINVAL;
ret = get_user_pages_remote(NULL, mm, vaddr, 1, ret = get_user_pages_remote(mm, vaddr, 1,
FOLL_WRITE, &page, &vma, NULL); FOLL_WRITE, &page, &vma, NULL);
if (unlikely(ret <= 0)) { if (unlikely(ret <= 0)) {
/* /*
...@@ -477,7 +477,7 @@ int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm, ...@@ -477,7 +477,7 @@ int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm,
if (is_register) if (is_register)
gup_flags |= FOLL_SPLIT_PMD; gup_flags |= FOLL_SPLIT_PMD;
/* Read the page with vaddr into memory */ /* Read the page with vaddr into memory */
ret = get_user_pages_remote(NULL, mm, vaddr, 1, gup_flags, ret = get_user_pages_remote(mm, vaddr, 1, gup_flags,
&old_page, &vma, NULL); &old_page, &vma, NULL);
if (ret <= 0) if (ret <= 0)
return ret; return ret;
...@@ -2029,7 +2029,7 @@ static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr) ...@@ -2029,7 +2029,7 @@ static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr)
* but we treat this as a 'remote' access since it is * but we treat this as a 'remote' access since it is
* essentially a kernel access to the memory. * essentially a kernel access to the memory.
*/ */
result = get_user_pages_remote(NULL, mm, vaddr, 1, FOLL_FORCE, &page, result = get_user_pages_remote(mm, vaddr, 1, FOLL_FORCE, &page,
NULL, NULL); NULL, NULL);
if (result < 0) if (result < 0)
return result; return result;
......
...@@ -678,7 +678,7 @@ static int fault_in_user_writeable(u32 __user *uaddr) ...@@ -678,7 +678,7 @@ static int fault_in_user_writeable(u32 __user *uaddr)
int ret; int ret;
mmap_read_lock(mm); mmap_read_lock(mm);
ret = fixup_user_fault(current, mm, (unsigned long)uaddr, ret = fixup_user_fault(mm, (unsigned long)uaddr,
FAULT_FLAG_WRITE, NULL); FAULT_FLAG_WRITE, NULL);
mmap_read_unlock(mm); mmap_read_unlock(mm);
......
...@@ -859,7 +859,7 @@ static int get_gate_page(struct mm_struct *mm, unsigned long address, ...@@ -859,7 +859,7 @@ static int get_gate_page(struct mm_struct *mm, unsigned long address,
* does not include FOLL_NOWAIT, the mmap_lock may be released. If it * does not include FOLL_NOWAIT, the mmap_lock may be released. If it
* is, *@locked will be set to 0 and -EBUSY returned. * is, *@locked will be set to 0 and -EBUSY returned.
*/ */
static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma, static int faultin_page(struct vm_area_struct *vma,
unsigned long address, unsigned int *flags, int *locked) unsigned long address, unsigned int *flags, int *locked)
{ {
unsigned int fault_flags = 0; unsigned int fault_flags = 0;
...@@ -962,7 +962,6 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) ...@@ -962,7 +962,6 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
/** /**
* __get_user_pages() - pin user pages in memory * __get_user_pages() - pin user pages in memory
* @tsk: task_struct of target task
* @mm: mm_struct of target mm * @mm: mm_struct of target mm
* @start: starting user address * @start: starting user address
* @nr_pages: number of pages from start to pin * @nr_pages: number of pages from start to pin
...@@ -1021,7 +1020,7 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) ...@@ -1021,7 +1020,7 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
* instead of __get_user_pages. __get_user_pages should be used only if * instead of __get_user_pages. __get_user_pages should be used only if
* you need some special @gup_flags. * you need some special @gup_flags.
*/ */
static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, static long __get_user_pages(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *locked) struct vm_area_struct **vmas, int *locked)
...@@ -1103,8 +1102,7 @@ static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, ...@@ -1103,8 +1102,7 @@ static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
page = follow_page_mask(vma, start, foll_flags, &ctx); page = follow_page_mask(vma, start, foll_flags, &ctx);
if (!page) { if (!page) {
ret = faultin_page(tsk, vma, start, &foll_flags, ret = faultin_page(vma, start, &foll_flags, locked);
locked);
switch (ret) { switch (ret) {
case 0: case 0:
goto retry; goto retry;
...@@ -1178,8 +1176,6 @@ static bool vma_permits_fault(struct vm_area_struct *vma, ...@@ -1178,8 +1176,6 @@ static bool vma_permits_fault(struct vm_area_struct *vma,
/** /**
* fixup_user_fault() - manually resolve a user page fault * fixup_user_fault() - manually resolve a user page fault
* @tsk: the task_struct to use for page fault accounting, or
* NULL if faults are not to be recorded.
* @mm: mm_struct of target mm * @mm: mm_struct of target mm
* @address: user address * @address: user address
* @fault_flags:flags to pass down to handle_mm_fault() * @fault_flags:flags to pass down to handle_mm_fault()
...@@ -1207,7 +1203,7 @@ static bool vma_permits_fault(struct vm_area_struct *vma, ...@@ -1207,7 +1203,7 @@ static bool vma_permits_fault(struct vm_area_struct *vma,
* This function will not return with an unlocked mmap_lock. So it has not the * This function will not return with an unlocked mmap_lock. So it has not the
* same semantics wrt the @mm->mmap_lock as does filemap_fault(). * same semantics wrt the @mm->mmap_lock as does filemap_fault().
*/ */
int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm, int fixup_user_fault(struct mm_struct *mm,
unsigned long address, unsigned int fault_flags, unsigned long address, unsigned int fault_flags,
bool *unlocked) bool *unlocked)
{ {
...@@ -1256,8 +1252,7 @@ EXPORT_SYMBOL_GPL(fixup_user_fault); ...@@ -1256,8 +1252,7 @@ EXPORT_SYMBOL_GPL(fixup_user_fault);
* Please note that this function, unlike __get_user_pages will not * Please note that this function, unlike __get_user_pages will not
* return 0 for nr_pages > 0 without FOLL_NOWAIT * return 0 for nr_pages > 0 without FOLL_NOWAIT
*/ */
static __always_inline long __get_user_pages_locked(struct task_struct *tsk, static __always_inline long __get_user_pages_locked(struct mm_struct *mm,
struct mm_struct *mm,
unsigned long start, unsigned long start,
unsigned long nr_pages, unsigned long nr_pages,
struct page **pages, struct page **pages,
...@@ -1290,7 +1285,7 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk, ...@@ -1290,7 +1285,7 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk,
pages_done = 0; pages_done = 0;
lock_dropped = false; lock_dropped = false;
for (;;) { for (;;) {
ret = __get_user_pages(tsk, mm, start, nr_pages, flags, pages, ret = __get_user_pages(mm, start, nr_pages, flags, pages,
vmas, locked); vmas, locked);
if (!locked) if (!locked)
/* VM_FAULT_RETRY couldn't trigger, bypass */ /* VM_FAULT_RETRY couldn't trigger, bypass */
...@@ -1350,7 +1345,7 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk, ...@@ -1350,7 +1345,7 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk,
} }
*locked = 1; *locked = 1;
ret = __get_user_pages(tsk, mm, start, 1, flags | FOLL_TRIED, ret = __get_user_pages(mm, start, 1, flags | FOLL_TRIED,
pages, NULL, locked); pages, NULL, locked);
if (!*locked) { if (!*locked) {
/* Continue to retry until we succeeded */ /* Continue to retry until we succeeded */
...@@ -1437,7 +1432,7 @@ long populate_vma_page_range(struct vm_area_struct *vma, ...@@ -1437,7 +1432,7 @@ long populate_vma_page_range(struct vm_area_struct *vma,
* We made sure addr is within a VMA, so the following will * We made sure addr is within a VMA, so the following will
* not result in a stack expansion that recurses back here. * not result in a stack expansion that recurses back here.
*/ */
return __get_user_pages(current, mm, start, nr_pages, gup_flags, return __get_user_pages(mm, start, nr_pages, gup_flags,
NULL, NULL, locked); NULL, NULL, locked);
} }
...@@ -1521,7 +1516,7 @@ struct page *get_dump_page(unsigned long addr) ...@@ -1521,7 +1516,7 @@ struct page *get_dump_page(unsigned long addr)
struct vm_area_struct *vma; struct vm_area_struct *vma;
struct page *page; struct page *page;
if (__get_user_pages(current, current->mm, addr, 1, if (__get_user_pages(current->mm, addr, 1,
FOLL_FORCE | FOLL_DUMP | FOLL_GET, &page, &vma, FOLL_FORCE | FOLL_DUMP | FOLL_GET, &page, &vma,
NULL) < 1) NULL) < 1)
return NULL; return NULL;
...@@ -1530,8 +1525,7 @@ struct page *get_dump_page(unsigned long addr) ...@@ -1530,8 +1525,7 @@ struct page *get_dump_page(unsigned long addr)
} }
#endif /* CONFIG_ELF_CORE */ #endif /* CONFIG_ELF_CORE */
#else /* CONFIG_MMU */ #else /* CONFIG_MMU */
static long __get_user_pages_locked(struct task_struct *tsk, static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start,
struct mm_struct *mm, unsigned long start,
unsigned long nr_pages, struct page **pages, unsigned long nr_pages, struct page **pages,
struct vm_area_struct **vmas, int *locked, struct vm_area_struct **vmas, int *locked,
unsigned int foll_flags) unsigned int foll_flags)
...@@ -1596,8 +1590,7 @@ static bool check_dax_vmas(struct vm_area_struct **vmas, long nr_pages) ...@@ -1596,8 +1590,7 @@ static bool check_dax_vmas(struct vm_area_struct **vmas, long nr_pages)
} }
#ifdef CONFIG_CMA #ifdef CONFIG_CMA
static long check_and_migrate_cma_pages(struct task_struct *tsk, static long check_and_migrate_cma_pages(struct mm_struct *mm,
struct mm_struct *mm,
unsigned long start, unsigned long start,
unsigned long nr_pages, unsigned long nr_pages,
struct page **pages, struct page **pages,
...@@ -1675,7 +1668,7 @@ static long check_and_migrate_cma_pages(struct task_struct *tsk, ...@@ -1675,7 +1668,7 @@ static long check_and_migrate_cma_pages(struct task_struct *tsk,
* again migrating any new CMA pages which we failed to isolate * again migrating any new CMA pages which we failed to isolate
* earlier. * earlier.
*/ */
ret = __get_user_pages_locked(tsk, mm, start, nr_pages, ret = __get_user_pages_locked(mm, start, nr_pages,
pages, vmas, NULL, pages, vmas, NULL,
gup_flags); gup_flags);
...@@ -1689,8 +1682,7 @@ static long check_and_migrate_cma_pages(struct task_struct *tsk, ...@@ -1689,8 +1682,7 @@ static long check_and_migrate_cma_pages(struct task_struct *tsk,
return ret; return ret;
} }
#else #else
static long check_and_migrate_cma_pages(struct task_struct *tsk, static long check_and_migrate_cma_pages(struct mm_struct *mm,
struct mm_struct *mm,
unsigned long start, unsigned long start,
unsigned long nr_pages, unsigned long nr_pages,
struct page **pages, struct page **pages,
...@@ -1705,8 +1697,7 @@ static long check_and_migrate_cma_pages(struct task_struct *tsk, ...@@ -1705,8 +1697,7 @@ static long check_and_migrate_cma_pages(struct task_struct *tsk,
* __gup_longterm_locked() is a wrapper for __get_user_pages_locked which * __gup_longterm_locked() is a wrapper for __get_user_pages_locked which
* allows us to process the FOLL_LONGTERM flag. * allows us to process the FOLL_LONGTERM flag.
*/ */
static long __gup_longterm_locked(struct task_struct *tsk, static long __gup_longterm_locked(struct mm_struct *mm,
struct mm_struct *mm,
unsigned long start, unsigned long start,
unsigned long nr_pages, unsigned long nr_pages,
struct page **pages, struct page **pages,
...@@ -1731,7 +1722,7 @@ static long __gup_longterm_locked(struct task_struct *tsk, ...@@ -1731,7 +1722,7 @@ static long __gup_longterm_locked(struct task_struct *tsk,
flags = memalloc_nocma_save(); flags = memalloc_nocma_save();
} }
rc = __get_user_pages_locked(tsk, mm, start, nr_pages, pages, rc = __get_user_pages_locked(mm, start, nr_pages, pages,
vmas_tmp, NULL, gup_flags); vmas_tmp, NULL, gup_flags);
if (gup_flags & FOLL_LONGTERM) { if (gup_flags & FOLL_LONGTERM) {
...@@ -1745,7 +1736,7 @@ static long __gup_longterm_locked(struct task_struct *tsk, ...@@ -1745,7 +1736,7 @@ static long __gup_longterm_locked(struct task_struct *tsk,
goto out; goto out;
} }
rc = check_and_migrate_cma_pages(tsk, mm, start, rc, pages, rc = check_and_migrate_cma_pages(mm, start, rc, pages,
vmas_tmp, gup_flags); vmas_tmp, gup_flags);
out: out:
memalloc_nocma_restore(flags); memalloc_nocma_restore(flags);
...@@ -1756,22 +1747,20 @@ static long __gup_longterm_locked(struct task_struct *tsk, ...@@ -1756,22 +1747,20 @@ static long __gup_longterm_locked(struct task_struct *tsk,
return rc; return rc;
} }
#else /* !CONFIG_FS_DAX && !CONFIG_CMA */ #else /* !CONFIG_FS_DAX && !CONFIG_CMA */
static __always_inline long __gup_longterm_locked(struct task_struct *tsk, static __always_inline long __gup_longterm_locked(struct mm_struct *mm,
struct mm_struct *mm,
unsigned long start, unsigned long start,
unsigned long nr_pages, unsigned long nr_pages,
struct page **pages, struct page **pages,
struct vm_area_struct **vmas, struct vm_area_struct **vmas,
unsigned int flags) unsigned int flags)
{ {
return __get_user_pages_locked(tsk, mm, start, nr_pages, pages, vmas, return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
NULL, flags); NULL, flags);
} }
#endif /* CONFIG_FS_DAX || CONFIG_CMA */ #endif /* CONFIG_FS_DAX || CONFIG_CMA */
#ifdef CONFIG_MMU #ifdef CONFIG_MMU
static long __get_user_pages_remote(struct task_struct *tsk, static long __get_user_pages_remote(struct mm_struct *mm,
struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *locked) struct vm_area_struct **vmas, int *locked)
...@@ -1790,20 +1779,18 @@ static long __get_user_pages_remote(struct task_struct *tsk, ...@@ -1790,20 +1779,18 @@ static long __get_user_pages_remote(struct task_struct *tsk,
* This will check the vmas (even if our vmas arg is NULL) * This will check the vmas (even if our vmas arg is NULL)
* and return -ENOTSUPP if DAX isn't allowed in this case: * and return -ENOTSUPP if DAX isn't allowed in this case:
*/ */
return __gup_longterm_locked(tsk, mm, start, nr_pages, pages, return __gup_longterm_locked(mm, start, nr_pages, pages,
vmas, gup_flags | FOLL_TOUCH | vmas, gup_flags | FOLL_TOUCH |
FOLL_REMOTE); FOLL_REMOTE);
} }
return __get_user_pages_locked(tsk, mm, start, nr_pages, pages, vmas, return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
locked, locked,
gup_flags | FOLL_TOUCH | FOLL_REMOTE); gup_flags | FOLL_TOUCH | FOLL_REMOTE);
} }
/** /**
* get_user_pages_remote() - pin user pages in memory * get_user_pages_remote() - pin user pages in memory
* @tsk: the task_struct to use for page fault accounting, or
* NULL if faults are not to be recorded.
* @mm: mm_struct of target mm * @mm: mm_struct of target mm
* @start: starting user address * @start: starting user address
* @nr_pages: number of pages from start to pin * @nr_pages: number of pages from start to pin
...@@ -1862,7 +1849,7 @@ static long __get_user_pages_remote(struct task_struct *tsk, ...@@ -1862,7 +1849,7 @@ static long __get_user_pages_remote(struct task_struct *tsk,
* should use get_user_pages_remote because it cannot pass * should use get_user_pages_remote because it cannot pass
* FAULT_FLAG_ALLOW_RETRY to handle_mm_fault. * FAULT_FLAG_ALLOW_RETRY to handle_mm_fault.
*/ */
long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, long get_user_pages_remote(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *locked) struct vm_area_struct **vmas, int *locked)
...@@ -1874,13 +1861,13 @@ long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, ...@@ -1874,13 +1861,13 @@ long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
if (WARN_ON_ONCE(gup_flags & FOLL_PIN)) if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
return -EINVAL; return -EINVAL;
return __get_user_pages_remote(tsk, mm, start, nr_pages, gup_flags, return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
pages, vmas, locked); pages, vmas, locked);
} }
EXPORT_SYMBOL(get_user_pages_remote); EXPORT_SYMBOL(get_user_pages_remote);
#else /* CONFIG_MMU */ #else /* CONFIG_MMU */
long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, long get_user_pages_remote(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *locked) struct vm_area_struct **vmas, int *locked)
...@@ -1888,8 +1875,7 @@ long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, ...@@ -1888,8 +1875,7 @@ long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
return 0; return 0;
} }
static long __get_user_pages_remote(struct task_struct *tsk, static long __get_user_pages_remote(struct mm_struct *mm,
struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *locked) struct vm_area_struct **vmas, int *locked)
...@@ -1909,11 +1895,10 @@ static long __get_user_pages_remote(struct task_struct *tsk, ...@@ -1909,11 +1895,10 @@ static long __get_user_pages_remote(struct task_struct *tsk,
* @vmas: array of pointers to vmas corresponding to each page. * @vmas: array of pointers to vmas corresponding to each page.
* Or NULL if the caller does not require them. * Or NULL if the caller does not require them.
* *
* This is the same as get_user_pages_remote(), just with a * This is the same as get_user_pages_remote(), just with a less-flexible
* less-flexible calling convention where we assume that the task * calling convention where we assume that the mm being operated on belongs to
* and mm being operated on are the current task's and don't allow * the current task, and doesn't allow passing of a locked parameter. We also
* passing of a locked parameter. We also obviously don't pass * obviously don't pass FOLL_REMOTE in here.
* FOLL_REMOTE in here.
*/ */
long get_user_pages(unsigned long start, unsigned long nr_pages, long get_user_pages(unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, unsigned int gup_flags, struct page **pages,
...@@ -1926,7 +1911,7 @@ long get_user_pages(unsigned long start, unsigned long nr_pages, ...@@ -1926,7 +1911,7 @@ long get_user_pages(unsigned long start, unsigned long nr_pages,
if (WARN_ON_ONCE(gup_flags & FOLL_PIN)) if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
return -EINVAL; return -EINVAL;
return __gup_longterm_locked(current, current->mm, start, nr_pages, return __gup_longterm_locked(current->mm, start, nr_pages,
pages, vmas, gup_flags | FOLL_TOUCH); pages, vmas, gup_flags | FOLL_TOUCH);
} }
EXPORT_SYMBOL(get_user_pages); EXPORT_SYMBOL(get_user_pages);
...@@ -1936,7 +1921,7 @@ EXPORT_SYMBOL(get_user_pages); ...@@ -1936,7 +1921,7 @@ EXPORT_SYMBOL(get_user_pages);
* *
* mmap_read_lock(mm); * mmap_read_lock(mm);
* do_something() * do_something()
* get_user_pages(tsk, mm, ..., pages, NULL); * get_user_pages(mm, ..., pages, NULL);
* mmap_read_unlock(mm); * mmap_read_unlock(mm);
* *
* to: * to:
...@@ -1944,7 +1929,7 @@ EXPORT_SYMBOL(get_user_pages); ...@@ -1944,7 +1929,7 @@ EXPORT_SYMBOL(get_user_pages);
* int locked = 1; * int locked = 1;
* mmap_read_lock(mm); * mmap_read_lock(mm);
* do_something() * do_something()
* get_user_pages_locked(tsk, mm, ..., pages, &locked); * get_user_pages_locked(mm, ..., pages, &locked);
* if (locked) * if (locked)
* mmap_read_unlock(mm); * mmap_read_unlock(mm);
* *
...@@ -1982,7 +1967,7 @@ long get_user_pages_locked(unsigned long start, unsigned long nr_pages, ...@@ -1982,7 +1967,7 @@ long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
if (WARN_ON_ONCE(gup_flags & FOLL_PIN)) if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
return -EINVAL; return -EINVAL;
return __get_user_pages_locked(current, current->mm, start, nr_pages, return __get_user_pages_locked(current->mm, start, nr_pages,
pages, NULL, locked, pages, NULL, locked,
gup_flags | FOLL_TOUCH); gup_flags | FOLL_TOUCH);
} }
...@@ -1992,12 +1977,12 @@ EXPORT_SYMBOL(get_user_pages_locked); ...@@ -1992,12 +1977,12 @@ EXPORT_SYMBOL(get_user_pages_locked);
* get_user_pages_unlocked() is suitable to replace the form: * get_user_pages_unlocked() is suitable to replace the form:
* *
* mmap_read_lock(mm); * mmap_read_lock(mm);
* get_user_pages(tsk, mm, ..., pages, NULL); * get_user_pages(mm, ..., pages, NULL);
* mmap_read_unlock(mm); * mmap_read_unlock(mm);
* *
* with: * with:
* *
* get_user_pages_unlocked(tsk, mm, ..., pages); * get_user_pages_unlocked(mm, ..., pages);
* *
* It is functionally equivalent to get_user_pages_fast so * It is functionally equivalent to get_user_pages_fast so
* get_user_pages_fast should be used instead if specific gup_flags * get_user_pages_fast should be used instead if specific gup_flags
...@@ -2020,7 +2005,7 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, ...@@ -2020,7 +2005,7 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
return -EINVAL; return -EINVAL;
mmap_read_lock(mm); mmap_read_lock(mm);
ret = __get_user_pages_locked(current, mm, start, nr_pages, pages, NULL, ret = __get_user_pages_locked(mm, start, nr_pages, pages, NULL,
&locked, gup_flags | FOLL_TOUCH); &locked, gup_flags | FOLL_TOUCH);
if (locked) if (locked)
mmap_read_unlock(mm); mmap_read_unlock(mm);
...@@ -2665,7 +2650,7 @@ static int __gup_longterm_unlocked(unsigned long start, int nr_pages, ...@@ -2665,7 +2650,7 @@ static int __gup_longterm_unlocked(unsigned long start, int nr_pages,
*/ */
if (gup_flags & FOLL_LONGTERM) { if (gup_flags & FOLL_LONGTERM) {
mmap_read_lock(current->mm); mmap_read_lock(current->mm);
ret = __gup_longterm_locked(current, current->mm, ret = __gup_longterm_locked(current->mm,
start, nr_pages, start, nr_pages,
pages, NULL, gup_flags); pages, NULL, gup_flags);
mmap_read_unlock(current->mm); mmap_read_unlock(current->mm);
...@@ -2908,10 +2893,8 @@ int pin_user_pages_fast_only(unsigned long start, int nr_pages, ...@@ -2908,10 +2893,8 @@ int pin_user_pages_fast_only(unsigned long start, int nr_pages,
EXPORT_SYMBOL_GPL(pin_user_pages_fast_only); EXPORT_SYMBOL_GPL(pin_user_pages_fast_only);
/** /**
* pin_user_pages_remote() - pin pages of a remote process (task != current) * pin_user_pages_remote() - pin pages of a remote process
* *
* @tsk: the task_struct to use for page fault accounting, or
* NULL if faults are not to be recorded.
* @mm: mm_struct of target mm * @mm: mm_struct of target mm
* @start: starting user address * @start: starting user address
* @nr_pages: number of pages from start to pin * @nr_pages: number of pages from start to pin
...@@ -2932,7 +2915,7 @@ EXPORT_SYMBOL_GPL(pin_user_pages_fast_only); ...@@ -2932,7 +2915,7 @@ EXPORT_SYMBOL_GPL(pin_user_pages_fast_only);
* FOLL_PIN means that the pages must be released via unpin_user_page(). Please * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
* see Documentation/core-api/pin_user_pages.rst for details. * see Documentation/core-api/pin_user_pages.rst for details.
*/ */
long pin_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, long pin_user_pages_remote(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *locked) struct vm_area_struct **vmas, int *locked)
...@@ -2942,7 +2925,7 @@ long pin_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, ...@@ -2942,7 +2925,7 @@ long pin_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
return -EINVAL; return -EINVAL;
gup_flags |= FOLL_PIN; gup_flags |= FOLL_PIN;
return __get_user_pages_remote(tsk, mm, start, nr_pages, gup_flags, return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
pages, vmas, locked); pages, vmas, locked);
} }
EXPORT_SYMBOL(pin_user_pages_remote); EXPORT_SYMBOL(pin_user_pages_remote);
...@@ -2974,7 +2957,7 @@ long pin_user_pages(unsigned long start, unsigned long nr_pages, ...@@ -2974,7 +2957,7 @@ long pin_user_pages(unsigned long start, unsigned long nr_pages,
return -EINVAL; return -EINVAL;
gup_flags |= FOLL_PIN; gup_flags |= FOLL_PIN;
return __gup_longterm_locked(current, current->mm, start, nr_pages, return __gup_longterm_locked(current->mm, start, nr_pages,
pages, vmas, gup_flags); pages, vmas, gup_flags);
} }
EXPORT_SYMBOL(pin_user_pages); EXPORT_SYMBOL(pin_user_pages);
...@@ -3019,7 +3002,7 @@ long pin_user_pages_locked(unsigned long start, unsigned long nr_pages, ...@@ -3019,7 +3002,7 @@ long pin_user_pages_locked(unsigned long start, unsigned long nr_pages,
return -EINVAL; return -EINVAL;
gup_flags |= FOLL_PIN; gup_flags |= FOLL_PIN;
return __get_user_pages_locked(current, current->mm, start, nr_pages, return __get_user_pages_locked(current->mm, start, nr_pages,
pages, NULL, locked, pages, NULL, locked,
gup_flags | FOLL_TOUCH); gup_flags | FOLL_TOUCH);
} }
......
...@@ -4742,7 +4742,7 @@ int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, ...@@ -4742,7 +4742,7 @@ int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
void *maddr; void *maddr;
struct page *page = NULL; struct page *page = NULL;
ret = get_user_pages_remote(tsk, mm, addr, 1, ret = get_user_pages_remote(mm, addr, 1,
gup_flags, &page, &vma, NULL); gup_flags, &page, &vma, NULL);
if (ret <= 0) { if (ret <= 0) {
#ifndef CONFIG_HAVE_IOREMAP_PROT #ifndef CONFIG_HAVE_IOREMAP_PROT
......
...@@ -105,7 +105,7 @@ static int process_vm_rw_single_vec(unsigned long addr, ...@@ -105,7 +105,7 @@ static int process_vm_rw_single_vec(unsigned long addr,
* current/current->mm * current/current->mm
*/ */
mmap_read_lock(mm); mmap_read_lock(mm);
pinned_pages = pin_user_pages_remote(task, mm, pa, pinned_pages, pinned_pages = pin_user_pages_remote(mm, pa, pinned_pages,
flags, process_pages, flags, process_pages,
NULL, &locked); NULL, &locked);
if (locked) if (locked)
......
...@@ -914,7 +914,7 @@ bool tomoyo_dump_page(struct linux_binprm *bprm, unsigned long pos, ...@@ -914,7 +914,7 @@ bool tomoyo_dump_page(struct linux_binprm *bprm, unsigned long pos,
* (represented by bprm). 'current' is the process doing * (represented by bprm). 'current' is the process doing
* the execve(). * the execve().
*/ */
if (get_user_pages_remote(current, bprm->mm, pos, 1, if (get_user_pages_remote(bprm->mm, pos, 1,
FOLL_FORCE, &page, NULL, NULL) <= 0) FOLL_FORCE, &page, NULL, NULL) <= 0)
return false; return false;
#else #else
......
...@@ -61,7 +61,7 @@ static void async_pf_execute(struct work_struct *work) ...@@ -61,7 +61,7 @@ static void async_pf_execute(struct work_struct *work)
* access remotely. * access remotely.
*/ */
mmap_read_lock(mm); mmap_read_lock(mm);
get_user_pages_remote(NULL, mm, addr, 1, FOLL_WRITE, NULL, NULL, get_user_pages_remote(mm, addr, 1, FOLL_WRITE, NULL, NULL,
&locked); &locked);
if (locked) if (locked)
mmap_read_unlock(mm); mmap_read_unlock(mm);
......
...@@ -1893,7 +1893,7 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, ...@@ -1893,7 +1893,7 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma,
* not call the fault handler, so do it here. * not call the fault handler, so do it here.
*/ */
bool unlocked = false; bool unlocked = false;
r = fixup_user_fault(current, current->mm, addr, r = fixup_user_fault(current->mm, addr,
(write_fault ? FAULT_FLAG_WRITE : 0), (write_fault ? FAULT_FLAG_WRITE : 0),
&unlocked); &unlocked);
if (unlocked) if (unlocked)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment