Commit a90f590a authored by Dominik Brodowski's avatar Dominik Brodowski

mm: add ksys_mmap_pgoff() helper; remove in-kernel calls to sys_mmap_pgoff()

Using this helper allows us to avoid the in-kernel calls to the
sys_mmap_pgoff() syscall. The ksys_ prefix denotes that this function is
meant as a drop-in replacement for the syscall. In particular, it uses the
same calling convention as sys_mmap_pgoff().

This patch is part of a series which removes in-kernel calls to syscalls.
On this basis, the syscall entry path can be streamlined. For details, see
http://lkml.kernel.org/r/20180325162527.GA17492@light.dominikbrodowski.net

Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: linux-mm@kvack.org
Signed-off-by: default avatarDominik Brodowski <linux@dominikbrodowski.net>
parent 9d5b7c95
...@@ -189,7 +189,7 @@ SYSCALL_DEFINE6(osf_mmap, unsigned long, addr, unsigned long, len, ...@@ -189,7 +189,7 @@ SYSCALL_DEFINE6(osf_mmap, unsigned long, addr, unsigned long, len,
goto out; goto out;
if (off & ~PAGE_MASK) if (off & ~PAGE_MASK)
goto out; goto out;
ret = sys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT); ret = ksys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT);
out: out:
return ret; return ret;
} }
......
...@@ -34,7 +34,7 @@ asmlinkage long sys_mmap(unsigned long addr, unsigned long len, ...@@ -34,7 +34,7 @@ asmlinkage long sys_mmap(unsigned long addr, unsigned long len,
if (offset_in_page(off) != 0) if (offset_in_page(off) != 0)
return -EINVAL; return -EINVAL;
return sys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT); return ksys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT);
} }
SYSCALL_DEFINE1(arm64_personality, unsigned int, personality) SYSCALL_DEFINE1(arm64_personality, unsigned int, personality)
......
...@@ -139,7 +139,7 @@ int ia64_mmap_check(unsigned long addr, unsigned long len, ...@@ -139,7 +139,7 @@ int ia64_mmap_check(unsigned long addr, unsigned long len,
asmlinkage unsigned long asmlinkage unsigned long
sys_mmap2 (unsigned long addr, unsigned long len, int prot, int flags, int fd, long pgoff) sys_mmap2 (unsigned long addr, unsigned long len, int prot, int flags, int fd, long pgoff)
{ {
addr = sys_mmap_pgoff(addr, len, prot, flags, fd, pgoff); addr = ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff);
if (!IS_ERR((void *) addr)) if (!IS_ERR((void *) addr))
force_successful_syscall_return(); force_successful_syscall_return();
return addr; return addr;
...@@ -151,7 +151,7 @@ sys_mmap (unsigned long addr, unsigned long len, int prot, int flags, int fd, lo ...@@ -151,7 +151,7 @@ sys_mmap (unsigned long addr, unsigned long len, int prot, int flags, int fd, lo
if (offset_in_page(off) != 0) if (offset_in_page(off) != 0)
return -EINVAL; return -EINVAL;
addr = sys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT); addr = ksys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT);
if (!IS_ERR((void *) addr)) if (!IS_ERR((void *) addr))
force_successful_syscall_return(); force_successful_syscall_return();
return addr; return addr;
......
...@@ -46,7 +46,7 @@ asmlinkage long sys_mmap2(unsigned long addr, unsigned long len, ...@@ -46,7 +46,7 @@ asmlinkage long sys_mmap2(unsigned long addr, unsigned long len,
* so we need to shift the argument down by 1; m68k mmap64(3) * so we need to shift the argument down by 1; m68k mmap64(3)
* (in libc) expects the last argument of mmap2 in 4Kb units. * (in libc) expects the last argument of mmap2 in 4Kb units.
*/ */
return sys_mmap_pgoff(addr, len, prot, flags, fd, pgoff); return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff);
} }
/* Convert virtual (user) address VADDR to physical address PADDR */ /* Convert virtual (user) address VADDR to physical address PADDR */
......
...@@ -40,7 +40,7 @@ SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, ...@@ -40,7 +40,7 @@ SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len,
if (pgoff & ~PAGE_MASK) if (pgoff & ~PAGE_MASK)
return -EINVAL; return -EINVAL;
return sys_mmap_pgoff(addr, len, prot, flags, fd, pgoff >> PAGE_SHIFT); return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff >> PAGE_SHIFT);
} }
SYSCALL_DEFINE6(mmap2, unsigned long, addr, unsigned long, len, SYSCALL_DEFINE6(mmap2, unsigned long, addr, unsigned long, len,
...@@ -50,6 +50,6 @@ SYSCALL_DEFINE6(mmap2, unsigned long, addr, unsigned long, len, ...@@ -50,6 +50,6 @@ SYSCALL_DEFINE6(mmap2, unsigned long, addr, unsigned long, len,
if (pgoff & (~PAGE_MASK >> 12)) if (pgoff & (~PAGE_MASK >> 12))
return -EINVAL; return -EINVAL;
return sys_mmap_pgoff(addr, len, prot, flags, fd, return ksys_mmap_pgoff(addr, len, prot, flags, fd,
pgoff >> (PAGE_SHIFT - 12)); pgoff >> (PAGE_SHIFT - 12));
} }
...@@ -67,7 +67,7 @@ SYSCALL_DEFINE6(32_mmap2, unsigned long, addr, unsigned long, len, ...@@ -67,7 +67,7 @@ SYSCALL_DEFINE6(32_mmap2, unsigned long, addr, unsigned long, len,
{ {
if (pgoff & (~PAGE_MASK >> 12)) if (pgoff & (~PAGE_MASK >> 12))
return -EINVAL; return -EINVAL;
return sys_mmap_pgoff(addr, len, prot, flags, fd, return ksys_mmap_pgoff(addr, len, prot, flags, fd,
pgoff >> (PAGE_SHIFT-12)); pgoff >> (PAGE_SHIFT-12));
} }
......
...@@ -63,7 +63,8 @@ SYSCALL_DEFINE6(mips_mmap, unsigned long, addr, unsigned long, len, ...@@ -63,7 +63,8 @@ SYSCALL_DEFINE6(mips_mmap, unsigned long, addr, unsigned long, len,
{ {
if (offset & ~PAGE_MASK) if (offset & ~PAGE_MASK)
return -EINVAL; return -EINVAL;
return sys_mmap_pgoff(addr, len, prot, flags, fd, offset >> PAGE_SHIFT); return ksys_mmap_pgoff(addr, len, prot, flags, fd,
offset >> PAGE_SHIFT);
} }
SYSCALL_DEFINE6(mips_mmap2, unsigned long, addr, unsigned long, len, SYSCALL_DEFINE6(mips_mmap2, unsigned long, addr, unsigned long, len,
...@@ -73,7 +74,8 @@ SYSCALL_DEFINE6(mips_mmap2, unsigned long, addr, unsigned long, len, ...@@ -73,7 +74,8 @@ SYSCALL_DEFINE6(mips_mmap2, unsigned long, addr, unsigned long, len,
if (pgoff & (~PAGE_MASK >> 12)) if (pgoff & (~PAGE_MASK >> 12))
return -EINVAL; return -EINVAL;
return sys_mmap_pgoff(addr, len, prot, flags, fd, pgoff >> (PAGE_SHIFT-12)); return ksys_mmap_pgoff(addr, len, prot, flags, fd,
pgoff >> (PAGE_SHIFT - 12));
} }
save_static_function(sys_fork); save_static_function(sys_fork);
......
...@@ -270,7 +270,7 @@ asmlinkage unsigned long sys_mmap2(unsigned long addr, unsigned long len, ...@@ -270,7 +270,7 @@ asmlinkage unsigned long sys_mmap2(unsigned long addr, unsigned long len,
{ {
/* Make sure the shift for mmap2 is constant (12), no matter what PAGE_SIZE /* Make sure the shift for mmap2 is constant (12), no matter what PAGE_SIZE
we have. */ we have. */
return sys_mmap_pgoff(addr, len, prot, flags, fd, return ksys_mmap_pgoff(addr, len, prot, flags, fd,
pgoff >> (PAGE_SHIFT - 12)); pgoff >> (PAGE_SHIFT - 12));
} }
...@@ -279,7 +279,7 @@ asmlinkage unsigned long sys_mmap(unsigned long addr, unsigned long len, ...@@ -279,7 +279,7 @@ asmlinkage unsigned long sys_mmap(unsigned long addr, unsigned long len,
unsigned long offset) unsigned long offset)
{ {
if (!(offset & ~PAGE_MASK)) { if (!(offset & ~PAGE_MASK)) {
return sys_mmap_pgoff(addr, len, prot, flags, fd, return ksys_mmap_pgoff(addr, len, prot, flags, fd,
offset >> PAGE_SHIFT); offset >> PAGE_SHIFT);
} else { } else {
return -EINVAL; return -EINVAL;
......
...@@ -57,7 +57,7 @@ static inline long do_mmap2(unsigned long addr, size_t len, ...@@ -57,7 +57,7 @@ static inline long do_mmap2(unsigned long addr, size_t len,
off >>= shift; off >>= shift;
} }
ret = sys_mmap_pgoff(addr, len, prot, flags, fd, off); ret = ksys_mmap_pgoff(addr, len, prot, flags, fd, off);
out: out:
return ret; return ret;
} }
......
...@@ -24,7 +24,7 @@ static long riscv_sys_mmap(unsigned long addr, unsigned long len, ...@@ -24,7 +24,7 @@ static long riscv_sys_mmap(unsigned long addr, unsigned long len,
{ {
if (unlikely(offset & (~PAGE_MASK >> page_shift_offset))) if (unlikely(offset & (~PAGE_MASK >> page_shift_offset)))
return -EINVAL; return -EINVAL;
return sys_mmap_pgoff(addr, len, prot, flags, fd, return ksys_mmap_pgoff(addr, len, prot, flags, fd,
offset >> (PAGE_SHIFT - page_shift_offset)); offset >> (PAGE_SHIFT - page_shift_offset));
} }
......
...@@ -442,7 +442,7 @@ COMPAT_SYSCALL_DEFINE1(s390_old_mmap, struct mmap_arg_struct_emu31 __user *, arg ...@@ -442,7 +442,7 @@ COMPAT_SYSCALL_DEFINE1(s390_old_mmap, struct mmap_arg_struct_emu31 __user *, arg
return -EFAULT; return -EFAULT;
if (a.offset & ~PAGE_MASK) if (a.offset & ~PAGE_MASK)
return -EINVAL; return -EINVAL;
return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd, return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
a.offset >> PAGE_SHIFT); a.offset >> PAGE_SHIFT);
} }
...@@ -452,7 +452,7 @@ COMPAT_SYSCALL_DEFINE1(s390_mmap2, struct mmap_arg_struct_emu31 __user *, arg) ...@@ -452,7 +452,7 @@ COMPAT_SYSCALL_DEFINE1(s390_mmap2, struct mmap_arg_struct_emu31 __user *, arg)
if (copy_from_user(&a, arg, sizeof(a))) if (copy_from_user(&a, arg, sizeof(a)))
return -EFAULT; return -EFAULT;
return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd, a.offset); return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd, a.offset);
} }
COMPAT_SYSCALL_DEFINE3(s390_read, unsigned int, fd, char __user *, buf, compat_size_t, count) COMPAT_SYSCALL_DEFINE3(s390_read, unsigned int, fd, char __user *, buf, compat_size_t, count)
......
...@@ -53,7 +53,7 @@ SYSCALL_DEFINE1(mmap2, struct s390_mmap_arg_struct __user *, arg) ...@@ -53,7 +53,7 @@ SYSCALL_DEFINE1(mmap2, struct s390_mmap_arg_struct __user *, arg)
if (copy_from_user(&a, arg, sizeof(a))) if (copy_from_user(&a, arg, sizeof(a)))
goto out; goto out;
error = sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd, a.offset); error = ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd, a.offset);
out: out:
return error; return error;
} }
......
...@@ -35,7 +35,7 @@ asmlinkage int old_mmap(unsigned long addr, unsigned long len, ...@@ -35,7 +35,7 @@ asmlinkage int old_mmap(unsigned long addr, unsigned long len,
{ {
if (off & ~PAGE_MASK) if (off & ~PAGE_MASK)
return -EINVAL; return -EINVAL;
return sys_mmap_pgoff(addr, len, prot, flags, fd, off>>PAGE_SHIFT); return ksys_mmap_pgoff(addr, len, prot, flags, fd, off>>PAGE_SHIFT);
} }
asmlinkage long sys_mmap2(unsigned long addr, unsigned long len, asmlinkage long sys_mmap2(unsigned long addr, unsigned long len,
...@@ -51,7 +51,7 @@ asmlinkage long sys_mmap2(unsigned long addr, unsigned long len, ...@@ -51,7 +51,7 @@ asmlinkage long sys_mmap2(unsigned long addr, unsigned long len,
pgoff >>= PAGE_SHIFT - 12; pgoff >>= PAGE_SHIFT - 12;
return sys_mmap_pgoff(addr, len, prot, flags, fd, pgoff); return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff);
} }
/* sys_cacheflush -- flush (part of) the processor cache. */ /* sys_cacheflush -- flush (part of) the processor cache. */
......
...@@ -104,7 +104,7 @@ asmlinkage long sys_mmap2(unsigned long addr, unsigned long len, ...@@ -104,7 +104,7 @@ asmlinkage long sys_mmap2(unsigned long addr, unsigned long len,
{ {
/* Make sure the shift for mmap2 is constant (12), no matter what PAGE_SIZE /* Make sure the shift for mmap2 is constant (12), no matter what PAGE_SIZE
we have. */ we have. */
return sys_mmap_pgoff(addr, len, prot, flags, fd, return ksys_mmap_pgoff(addr, len, prot, flags, fd,
pgoff >> (PAGE_SHIFT - 12)); pgoff >> (PAGE_SHIFT - 12));
} }
...@@ -113,7 +113,7 @@ asmlinkage long sys_mmap(unsigned long addr, unsigned long len, ...@@ -113,7 +113,7 @@ asmlinkage long sys_mmap(unsigned long addr, unsigned long len,
unsigned long off) unsigned long off)
{ {
/* no alignment check? */ /* no alignment check? */
return sys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT); return ksys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT);
} }
long sparc_remap_file_pages(unsigned long start, unsigned long size, long sparc_remap_file_pages(unsigned long start, unsigned long size,
......
...@@ -458,7 +458,7 @@ SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, ...@@ -458,7 +458,7 @@ SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len,
goto out; goto out;
if (off & ~PAGE_MASK) if (off & ~PAGE_MASK)
goto out; goto out;
retval = sys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT); retval = ksys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT);
out: out:
return retval; return retval;
} }
......
...@@ -22,7 +22,7 @@ long old_mmap(unsigned long addr, unsigned long len, ...@@ -22,7 +22,7 @@ long old_mmap(unsigned long addr, unsigned long len,
if (offset & ~PAGE_MASK) if (offset & ~PAGE_MASK)
goto out; goto out;
err = sys_mmap_pgoff(addr, len, prot, flags, fd, offset >> PAGE_SHIFT); err = ksys_mmap_pgoff(addr, len, prot, flags, fd, offset >> PAGE_SHIFT);
out: out:
return err; return err;
} }
...@@ -166,7 +166,7 @@ COMPAT_SYSCALL_DEFINE1(x86_mmap, struct mmap_arg_struct32 __user *, arg) ...@@ -166,7 +166,7 @@ COMPAT_SYSCALL_DEFINE1(x86_mmap, struct mmap_arg_struct32 __user *, arg)
if (a.offset & ~PAGE_MASK) if (a.offset & ~PAGE_MASK)
return -EINVAL; return -EINVAL;
return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd, return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
a.offset>>PAGE_SHIFT); a.offset>>PAGE_SHIFT);
} }
......
...@@ -97,7 +97,7 @@ SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, ...@@ -97,7 +97,7 @@ SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len,
if (off & ~PAGE_MASK) if (off & ~PAGE_MASK)
goto out; goto out;
error = sys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT); error = ksys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT);
out: out:
return error; return error;
} }
......
...@@ -979,6 +979,9 @@ static inline int ksys_fadvise64_64(int fd, loff_t offset, loff_t len, ...@@ -979,6 +979,9 @@ static inline int ksys_fadvise64_64(int fd, loff_t offset, loff_t len,
return -EINVAL; return -EINVAL;
} }
#endif #endif
unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len,
unsigned long prot, unsigned long flags,
unsigned long fd, unsigned long pgoff);
/* /*
* The following kernel syscall equivalents are just wrappers to fs-internal * The following kernel syscall equivalents are just wrappers to fs-internal
......
...@@ -1488,9 +1488,9 @@ unsigned long do_mmap(struct file *file, unsigned long addr, ...@@ -1488,9 +1488,9 @@ unsigned long do_mmap(struct file *file, unsigned long addr,
return addr; return addr;
} }
SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len,
unsigned long, prot, unsigned long, flags, unsigned long prot, unsigned long flags,
unsigned long, fd, unsigned long, pgoff) unsigned long fd, unsigned long pgoff)
{ {
struct file *file = NULL; struct file *file = NULL;
unsigned long retval; unsigned long retval;
...@@ -1537,6 +1537,13 @@ SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, ...@@ -1537,6 +1537,13 @@ SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
return retval; return retval;
} }
SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
unsigned long, prot, unsigned long, flags,
unsigned long, fd, unsigned long, pgoff)
{
return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff);
}
#ifdef __ARCH_WANT_SYS_OLD_MMAP #ifdef __ARCH_WANT_SYS_OLD_MMAP
struct mmap_arg_struct { struct mmap_arg_struct {
unsigned long addr; unsigned long addr;
...@@ -1556,7 +1563,7 @@ SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg) ...@@ -1556,7 +1563,7 @@ SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
if (offset_in_page(a.offset)) if (offset_in_page(a.offset))
return -EINVAL; return -EINVAL;
return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd, return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
a.offset >> PAGE_SHIFT); a.offset >> PAGE_SHIFT);
} }
#endif /* __ARCH_WANT_SYS_OLD_MMAP */ #endif /* __ARCH_WANT_SYS_OLD_MMAP */
......
...@@ -1423,9 +1423,9 @@ unsigned long do_mmap(struct file *file, ...@@ -1423,9 +1423,9 @@ unsigned long do_mmap(struct file *file,
return -ENOMEM; return -ENOMEM;
} }
SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len,
unsigned long, prot, unsigned long, flags, unsigned long prot, unsigned long flags,
unsigned long, fd, unsigned long, pgoff) unsigned long fd, unsigned long pgoff)
{ {
struct file *file = NULL; struct file *file = NULL;
unsigned long retval = -EBADF; unsigned long retval = -EBADF;
...@@ -1447,6 +1447,13 @@ SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, ...@@ -1447,6 +1447,13 @@ SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
return retval; return retval;
} }
SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
unsigned long, prot, unsigned long, flags,
unsigned long, fd, unsigned long, pgoff)
{
return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff);
}
#ifdef __ARCH_WANT_SYS_OLD_MMAP #ifdef __ARCH_WANT_SYS_OLD_MMAP
struct mmap_arg_struct { struct mmap_arg_struct {
unsigned long addr; unsigned long addr;
...@@ -1466,7 +1473,7 @@ SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg) ...@@ -1466,7 +1473,7 @@ SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
if (offset_in_page(a.offset)) if (offset_in_page(a.offset))
return -EINVAL; return -EINVAL;
return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd, return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
a.offset >> PAGE_SHIFT); a.offset >> PAGE_SHIFT);
} }
#endif /* __ARCH_WANT_SYS_OLD_MMAP */ #endif /* __ARCH_WANT_SYS_OLD_MMAP */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment