x86/enter: Use IBRS on syscall and interrupts

CVE-2017-5715 (Spectre v2 Intel)

Set IBRS upon kernel entrance via syscall and interrupts. Clear it upon exit.
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Andy Whitcroft <apw@canonical.com>
(backported from commit bb6c1a01e82fb0eb14d1229fd71a99ed285d330d)
Signed-off-by: Andy Whitcroft <apw@canonical.com>
Acked-by: Colin Ian King <colin.king@canonical.com>
Acked-by: Kamal Mostafa <kamal@canonical.com>
Signed-off-by: Khalid Elmously <khalid.elmously@canonical.com>

arch/x86/entry/entry_64.S

@@ -37,6 +37,7 @@
 #include <asm/pgtable_types.h>
 #include <asm/kaiser.h>
 #include <asm/nospec-branch.h>
+#include <asm/spec_ctrl.h>
 #include <linux/err.h>
 
 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
@@ -174,6 +175,8 @@ GLOBAL(entry_SYSCALL_64_after_swapgs)
 	pushq	%r11				/* pt_regs->r11 */
 	sub	$(6*8), %rsp			/* pt_regs->bp, bx, r12-15 not saved */
 
+	ENABLE_IBRS
+
 	testl	$_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz	tracesys
 entry_SYSCALL_64_fastpath:
@@ -219,6 +222,9 @@ entry_SYSCALL_64_fastpath:
 	movq	RIP(%rsp), %rcx
 	movq	EFLAGS(%rsp), %r11
 	RESTORE_C_REGS_EXCEPT_RCX_R11
+
+	DISABLE_IBRS
+
 	/*
 	 * This opens a window where we have a user CR3, but are
 	 * running in the kernel.  This makes using the CS
@@ -367,6 +373,8 @@ GLOBAL(int_ret_from_sys_call)
 	 * perf profiles. Nothing jumps here.
 	 */
 syscall_return_via_sysret:
+	DISABLE_IBRS
+
 	/* rcx and r11 are already restored (see code above) */
 	RESTORE_C_REGS_EXCEPT_RCX_R11
 	/*
@@ -381,6 +389,7 @@ syscall_return_via_sysret:
 	USERGS_SYSRET64
 
 opportunistic_sysret_failed:
+	/* XXX: might we need a DISABLE_IBRS */
 	/*
 	 * This opens a window where we have a user CR3, but are
 	 * running in the kernel.  This makes using the CS
@@ -549,6 +558,8 @@ END(irq_entries_start)
 	SWAPGS
 	SWITCH_KERNEL_CR3
 
+	ENABLE_IBRS
+
 	/*
 	 * We need to tell lockdep that IRQs are off.  We can't do this until
 	 * we fix gsbase, and we should do it before enter_from_user_mode
@@ -607,6 +618,7 @@ GLOBAL(retint_user)
 	mov	%rsp,%rdi
 	call	prepare_exit_to_usermode
 	TRACE_IRQS_IRETQ
+	DISABLE_IBRS
 	SWITCH_USER_CR3
 	SWAPGS
 	jmp	restore_regs_and_iret
@@ -1073,6 +1085,8 @@ ENTRY(paranoid_entry)
 	movq	%rax, %cr3
 2:
 #endif
+	ENABLE_IBRS_CLOBBER
+
 	ret
 END(paranoid_entry)
 
@@ -1099,6 +1113,7 @@ ENTRY(paranoid_exit)
 	/* No ALTERNATIVE for X86_FEATURE_KAISER: paranoid_entry sets %ebx */
 	testl	$2, %ebx			/* SWITCH_USER_CR3 needed? */
 	jz	paranoid_exit_no_switch
+	DISABLE_IBRS
 	SWITCH_USER_CR3
 paranoid_exit_no_switch:
 #endif
@@ -1138,6 +1153,8 @@ ENTRY(error_entry)
 	 */
 	SWAPGS
 
+	ENABLE_IBRS
+
 .Lerror_entry_from_usermode_after_swapgs:
 	/*
 	 * We need to tell lockdep that IRQs are off.  We can't do this until
@@ -1188,6 +1205,7 @@ ENTRY(error_entry)
 	 * We came from an IRET to user mode, so we have user gsbase.
 	 * Switch to kernel gsbase:
 	 */
+	ENABLE_IBRS_CLOBBER
 	SWAPGS
 
 	/*
@@ -1338,6 +1356,7 @@ ENTRY(nmi)
 	movq	%rax, %cr3
 2:
 #endif
+	ENABLE_IBRS
 	call	do_nmi
 
 #ifdef CONFIG_KAISER

arch/x86/entry/entry_64_compat.S

@@ -15,6 +15,7 @@
 #include <asm/smap.h>
 #include <asm/pgtable_types.h>
 #include <asm/kaiser.h>
+#include <asm/spec_ctrl.h>
 #include <linux/linkage.h>
 #include <linux/err.h>
 
@@ -99,6 +100,8 @@ ENTRY(entry_SYSENTER_compat)
 	pushq   %r8                     /* pt_regs->r15 = 0 */
 	cld
 
+	ENABLE_IBRS
+
 	/*
 	 * Sysenter doesn't filter flags, so we need to clear NT
 	 * ourselves.  To save a few cycles, we can check whether
@@ -197,6 +200,8 @@ ENTRY(entry_SYSCALL_compat)
 	pushq   %r8                     /* pt_regs->r14 = 0 */
 	pushq   %r8                     /* pt_regs->r15 = 0 */
 
+	ENABLE_IBRS
+
 	/*
 	 * User mode is traced as though IRQs are on, and SYSENTER
 	 * turned them off.
@@ -212,6 +217,7 @@ ENTRY(entry_SYSCALL_compat)
 	/* Opportunistic SYSRET */
 sysret32_from_system_call:
 	TRACE_IRQS_ON			/* User mode traces as IRQs on. */
+	DISABLE_IBRS
 	SWITCH_USER_CR3
 	movq	RBX(%rsp), %rbx		/* pt_regs->rbx */
 	movq	RBP(%rsp), %rbp		/* pt_regs->rbp */
@@ -305,6 +311,8 @@ ENTRY(entry_INT80_compat)
 	pushq   %r15                    /* pt_regs->r15 */
 	cld
 
+	ENABLE_IBRS
+
 	/*
 	 * User mode is traced as though IRQs are on, and the interrupt
 	 * gate turned them off.