Commit 167fd210 authored by Thomas Gleixner's avatar Thomas Gleixner

x86/entry: Use generic syscall exit functionality

Replace the x86 variant with the generic version. Provide the relevant
architecture specific helper functions and defines.

Use a temporary define for idtentry_exit_user which will be cleaned up
seperately.
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Acked-by: default avatarKees Cook <keescook@chromium.org>
Link: https://lkml.kernel.org/r/20200722220520.494648601@linutronix.de
parent 27d6b4d1
...@@ -15,15 +15,8 @@ ...@@ -15,15 +15,8 @@
#include <linux/smp.h> #include <linux/smp.h>
#include <linux/errno.h> #include <linux/errno.h>
#include <linux/ptrace.h> #include <linux/ptrace.h>
#include <linux/tracehook.h>
#include <linux/audit.h>
#include <linux/signal.h>
#include <linux/export.h> #include <linux/export.h>
#include <linux/context_tracking.h>
#include <linux/user-return-notifier.h>
#include <linux/nospec.h> #include <linux/nospec.h>
#include <linux/uprobes.h>
#include <linux/livepatch.h>
#include <linux/syscalls.h> #include <linux/syscalls.h>
#include <linux/uaccess.h> #include <linux/uaccess.h>
...@@ -42,191 +35,6 @@ ...@@ -42,191 +35,6 @@
#include <asm/syscall.h> #include <asm/syscall.h>
#include <asm/irq_stack.h> #include <asm/irq_stack.h>
#include <trace/events/syscalls.h>
/**
* exit_to_user_mode - Fixup state when exiting to user mode
*
* Syscall exit enables interrupts, but the kernel state is interrupts
* disabled when this is invoked. Also tell RCU about it.
*
* 1) Trace interrupts on state
* 2) Invoke context tracking if enabled to adjust RCU state
* 3) Clear CPU buffers if CPU is affected by MDS and the migitation is on.
* 4) Tell lockdep that interrupts are enabled
*/
static __always_inline void exit_to_user_mode(void)
{
instrumentation_begin();
trace_hardirqs_on_prepare();
lockdep_hardirqs_on_prepare(CALLER_ADDR0);
instrumentation_end();
user_enter_irqoff();
mds_user_clear_cpu_buffers();
lockdep_hardirqs_on(CALLER_ADDR0);
}
#define EXIT_TO_USERMODE_LOOP_FLAGS \
(_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | \
_TIF_NEED_RESCHED | _TIF_PATCH_PENDING)
static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)
{
/*
* In order to return to user mode, we need to have IRQs off with
* none of EXIT_TO_USERMODE_LOOP_FLAGS set. Several of these flags
* can be set at any time on preemptible kernels if we have IRQs on,
* so we need to loop. Disabling preemption wouldn't help: doing the
* work to clear some of the flags can sleep.
*/
while (true) {
/* We have work to do. */
local_irq_enable();
if (cached_flags & _TIF_NEED_RESCHED)
schedule();
if (cached_flags & _TIF_UPROBE)
uprobe_notify_resume(regs);
if (cached_flags & _TIF_PATCH_PENDING)
klp_update_patch_state(current);
/* deal with pending signal delivery */
if (cached_flags & _TIF_SIGPENDING)
do_signal(regs);
if (cached_flags & _TIF_NOTIFY_RESUME) {
clear_thread_flag(TIF_NOTIFY_RESUME);
tracehook_notify_resume(regs);
rseq_handle_notify_resume(NULL, regs);
}
/* Disable IRQs and retry */
local_irq_disable();
cached_flags = READ_ONCE(current_thread_info()->flags);
if (!(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
break;
}
}
static void __prepare_exit_to_usermode(struct pt_regs *regs)
{
struct thread_info *ti = current_thread_info();
u32 cached_flags;
addr_limit_user_check();
lockdep_assert_irqs_disabled();
lockdep_sys_exit();
cached_flags = READ_ONCE(ti->flags);
if (unlikely(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
exit_to_usermode_loop(regs, cached_flags);
/* Reload ti->flags; we may have rescheduled above. */
cached_flags = READ_ONCE(ti->flags);
if (cached_flags & _TIF_USER_RETURN_NOTIFY)
fire_user_return_notifiers();
if (unlikely(cached_flags & _TIF_IO_BITMAP))
tss_update_io_bitmap();
fpregs_assert_state_consistent();
if (unlikely(cached_flags & _TIF_NEED_FPU_LOAD))
switch_fpu_return();
#ifdef CONFIG_COMPAT
/*
* Compat syscalls set TS_COMPAT. Make sure we clear it before
* returning to user mode. We need to clear it *after* signal
* handling, because syscall restart has a fixup for compat
* syscalls. The fixup is exercised by the ptrace_syscall_32
* selftest.
*
* We also need to clear TS_REGS_POKED_I386: the 32-bit tracer
* special case only applies after poking regs and before the
* very next return to user mode.
*/
ti->status &= ~(TS_COMPAT|TS_I386_REGS_POKED);
#endif
}
static noinstr void prepare_exit_to_usermode(struct pt_regs *regs)
{
instrumentation_begin();
__prepare_exit_to_usermode(regs);
instrumentation_end();
exit_to_user_mode();
}
#define SYSCALL_EXIT_WORK_FLAGS \
(_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
_TIF_SINGLESTEP | _TIF_SYSCALL_TRACEPOINT)
static void syscall_slow_exit_work(struct pt_regs *regs, u32 cached_flags)
{
bool step;
audit_syscall_exit(regs);
if (cached_flags & _TIF_SYSCALL_TRACEPOINT)
trace_sys_exit(regs, regs->ax);
/*
* If TIF_SYSCALL_EMU is set, we only get here because of
* TIF_SINGLESTEP (i.e. this is PTRACE_SYSEMU_SINGLESTEP).
* We already reported this syscall instruction in
* syscall_trace_enter().
*/
step = unlikely(
(cached_flags & (_TIF_SINGLESTEP | _TIF_SYSCALL_EMU))
== _TIF_SINGLESTEP);
if (step || cached_flags & _TIF_SYSCALL_TRACE)
tracehook_report_syscall_exit(regs, step);
}
static void __syscall_return_slowpath(struct pt_regs *regs)
{
struct thread_info *ti = current_thread_info();
u32 cached_flags = READ_ONCE(ti->flags);
CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
if (IS_ENABLED(CONFIG_PROVE_LOCKING) &&
WARN(irqs_disabled(), "syscall %ld left IRQs disabled", regs->orig_ax))
local_irq_enable();
rseq_syscall(regs);
/*
* First do one-time work. If these work items are enabled, we
* want to run them exactly once per syscall exit with IRQs on.
*/
if (unlikely(cached_flags & SYSCALL_EXIT_WORK_FLAGS))
syscall_slow_exit_work(regs, cached_flags);
local_irq_disable();
__prepare_exit_to_usermode(regs);
}
/*
* Called with IRQs on and fully valid regs. Returns with IRQs off in a
* state such that we can immediately switch to user mode.
*/
__visible noinstr void syscall_return_slowpath(struct pt_regs *regs)
{
instrumentation_begin();
__syscall_return_slowpath(regs);
instrumentation_end();
exit_to_user_mode();
}
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
__visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs) __visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs)
{ {
...@@ -245,7 +53,7 @@ __visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs) ...@@ -245,7 +53,7 @@ __visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs)
#endif #endif
} }
instrumentation_end(); instrumentation_end();
syscall_return_slowpath(regs); syscall_exit_to_user_mode(regs);
} }
#endif #endif
...@@ -284,7 +92,7 @@ __visible noinstr void do_int80_syscall_32(struct pt_regs *regs) ...@@ -284,7 +92,7 @@ __visible noinstr void do_int80_syscall_32(struct pt_regs *regs)
unsigned int nr = syscall_32_enter(regs); unsigned int nr = syscall_32_enter(regs);
do_syscall_32_irqs_on(regs, nr); do_syscall_32_irqs_on(regs, nr);
syscall_return_slowpath(regs); syscall_exit_to_user_mode(regs);
} }
static noinstr bool __do_fast_syscall_32(struct pt_regs *regs) static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
...@@ -310,13 +118,13 @@ static noinstr bool __do_fast_syscall_32(struct pt_regs *regs) ...@@ -310,13 +118,13 @@ static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
if (res) { if (res) {
/* User code screwed up. */ /* User code screwed up. */
regs->ax = -EFAULT; regs->ax = -EFAULT;
syscall_return_slowpath(regs); syscall_exit_to_user_mode(regs);
return false; return false;
} }
/* Now this is just like a normal syscall. */ /* Now this is just like a normal syscall. */
do_syscall_32_irqs_on(regs, nr); do_syscall_32_irqs_on(regs, nr);
syscall_return_slowpath(regs); syscall_exit_to_user_mode(regs);
return true; return true;
} }
...@@ -524,7 +332,7 @@ void noinstr idtentry_exit(struct pt_regs *regs, idtentry_state_t state) ...@@ -524,7 +332,7 @@ void noinstr idtentry_exit(struct pt_regs *regs, idtentry_state_t state)
/* Check whether this returns to user mode */ /* Check whether this returns to user mode */
if (user_mode(regs)) { if (user_mode(regs)) {
prepare_exit_to_usermode(regs); irqentry_exit_to_user_mode(regs);
} else if (regs->flags & X86_EFLAGS_IF) { } else if (regs->flags & X86_EFLAGS_IF) {
/* /*
* If RCU was not watching on entry this needs to be done * If RCU was not watching on entry this needs to be done
...@@ -555,25 +363,6 @@ void noinstr idtentry_exit(struct pt_regs *regs, idtentry_state_t state) ...@@ -555,25 +363,6 @@ void noinstr idtentry_exit(struct pt_regs *regs, idtentry_state_t state)
} }
} }
/**
* idtentry_exit_user - Handle return from exception to user mode
* @regs: Pointer to pt_regs (exception entry regs)
*
* Runs the necessary preemption and work checks and returns to the caller
* with interrupts disabled and no further work pending.
*
* This is the last action before returning to the low level ASM code which
* just needs to return to the appropriate context.
*
* Counterpart to idtentry_enter_user().
*/
void noinstr idtentry_exit_user(struct pt_regs *regs)
{
lockdep_assert_irqs_disabled();
prepare_exit_to_usermode(regs);
}
#ifdef CONFIG_XEN_PV #ifdef CONFIG_XEN_PV
#ifndef CONFIG_PREEMPTION #ifndef CONFIG_PREEMPTION
/* /*
......
...@@ -846,7 +846,7 @@ SYM_CODE_START(ret_from_fork) ...@@ -846,7 +846,7 @@ SYM_CODE_START(ret_from_fork)
2: 2:
/* When we fork, we trace the syscall return in the child, too. */ /* When we fork, we trace the syscall return in the child, too. */
movl %esp, %eax movl %esp, %eax
call syscall_return_slowpath call syscall_exit_to_user_mode
jmp .Lsyscall_32_done jmp .Lsyscall_32_done
/* kernel thread */ /* kernel thread */
......
...@@ -283,7 +283,7 @@ SYM_CODE_START(ret_from_fork) ...@@ -283,7 +283,7 @@ SYM_CODE_START(ret_from_fork)
2: 2:
UNWIND_HINT_REGS UNWIND_HINT_REGS
movq %rsp, %rdi movq %rsp, %rdi
call syscall_return_slowpath /* returns with IRQs disabled */ call syscall_exit_to_user_mode /* returns with IRQs disabled */
jmp swapgs_restore_regs_and_return_to_usermode jmp swapgs_restore_regs_and_return_to_usermode
1: 1:
......
...@@ -2,6 +2,12 @@ ...@@ -2,6 +2,12 @@
#ifndef _ASM_X86_ENTRY_COMMON_H #ifndef _ASM_X86_ENTRY_COMMON_H
#define _ASM_X86_ENTRY_COMMON_H #define _ASM_X86_ENTRY_COMMON_H
#include <linux/user-return-notifier.h>
#include <asm/nospec-branch.h>
#include <asm/io_bitmap.h>
#include <asm/fpu/api.h>
/* Check that the stack and regs on entry from user mode are sane. */ /* Check that the stack and regs on entry from user mode are sane. */
static __always_inline void arch_check_user_regs(struct pt_regs *regs) static __always_inline void arch_check_user_regs(struct pt_regs *regs)
{ {
...@@ -29,4 +35,42 @@ static __always_inline void arch_check_user_regs(struct pt_regs *regs) ...@@ -29,4 +35,42 @@ static __always_inline void arch_check_user_regs(struct pt_regs *regs)
} }
#define arch_check_user_regs arch_check_user_regs #define arch_check_user_regs arch_check_user_regs
#define ARCH_SYSCALL_EXIT_WORK (_TIF_SINGLESTEP)
static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
unsigned long ti_work)
{
if (ti_work & _TIF_USER_RETURN_NOTIFY)
fire_user_return_notifiers();
if (unlikely(ti_work & _TIF_IO_BITMAP))
tss_update_io_bitmap();
fpregs_assert_state_consistent();
if (unlikely(ti_work & _TIF_NEED_FPU_LOAD))
switch_fpu_return();
#ifdef CONFIG_COMPAT
/*
* Compat syscalls set TS_COMPAT. Make sure we clear it before
* returning to user mode. We need to clear it *after* signal
* handling, because syscall restart has a fixup for compat
* syscalls. The fixup is exercised by the ptrace_syscall_32
* selftest.
*
* We also need to clear TS_REGS_POKED_I386: the 32-bit tracer
* special case only applies after poking regs and before the
* very next return to user mode.
*/
current_thread_info()->status &= ~(TS_COMPAT | TS_I386_REGS_POKED);
#endif
}
#define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare
static __always_inline void arch_exit_to_user_mode(void)
{
mds_user_clear_cpu_buffers();
}
#define arch_exit_to_user_mode arch_exit_to_user_mode
#endif #endif
...@@ -13,8 +13,7 @@ ...@@ -13,8 +13,7 @@
/* Temporary define */ /* Temporary define */
#define idtentry_enter_user irqentry_enter_from_user_mode #define idtentry_enter_user irqentry_enter_from_user_mode
#define idtentry_exit_user irqentry_exit_to_user_mode
void idtentry_exit_user(struct pt_regs *regs);
typedef struct idtentry_state { typedef struct idtentry_state {
bool exit_rcu; bool exit_rcu;
......
...@@ -35,7 +35,6 @@ typedef sigset_t compat_sigset_t; ...@@ -35,7 +35,6 @@ typedef sigset_t compat_sigset_t;
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
#include <uapi/asm/signal.h> #include <uapi/asm/signal.h>
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
extern void do_signal(struct pt_regs *regs);
#define __ARCH_HAS_SA_RESTORER #define __ARCH_HAS_SA_RESTORER
......
...@@ -25,6 +25,7 @@ ...@@ -25,6 +25,7 @@
#include <linux/user-return-notifier.h> #include <linux/user-return-notifier.h>
#include <linux/uprobes.h> #include <linux/uprobes.h>
#include <linux/context_tracking.h> #include <linux/context_tracking.h>
#include <linux/entry-common.h>
#include <linux/syscalls.h> #include <linux/syscalls.h>
#include <asm/processor.h> #include <asm/processor.h>
...@@ -803,7 +804,7 @@ static inline unsigned long get_nr_restart_syscall(const struct pt_regs *regs) ...@@ -803,7 +804,7 @@ static inline unsigned long get_nr_restart_syscall(const struct pt_regs *regs)
* want to handle. Thus you cannot kill init even with a SIGKILL even by * want to handle. Thus you cannot kill init even with a SIGKILL even by
* mistake. * mistake.
*/ */
void do_signal(struct pt_regs *regs) void arch_do_signal(struct pt_regs *regs)
{ {
struct ksignal ksig; struct ksignal ksig;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment