x86/enter: Use IBRS on syscall and interrupts

CVE-2017-5753
CVE-2017-5715

Set IBRS upon kernel entrance via syscall and interrupts. Clear it upon exit.
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Andy Whitcroft <apw@canonical.com>
(backported from commit bb6c1a01e82fb0eb14d1229fd71a99ed285d330d)
Signed-off-by: Andy Whitcroft <apw@canonical.com>

arch/x86/entry/entry_64.S

@@ -36,6 +36,7 @@
 #include <asm/smap.h>
 #include <asm/pgtable_types.h>
 #include <asm/kaiser.h>
+#include <asm/spec_ctrl.h>
 #include <linux/err.h>
 
 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
@@ -173,6 +174,8 @@ GLOBAL(entry_SYSCALL_64_after_swapgs)
 	pushq	%r11				/* pt_regs->r11 */
 	sub	$(6*8), %rsp			/* pt_regs->bp, bx, r12-15 not saved */
 
+	ENABLE_IBRS
+
 	testl	$_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz	tracesys
 entry_SYSCALL_64_fastpath:
@@ -212,6 +215,9 @@ entry_SYSCALL_64_fastpath:
 	movq	RIP(%rsp), %rcx
 	movq	EFLAGS(%rsp), %r11
 	RESTORE_C_REGS_EXCEPT_RCX_R11
+
+	DISABLE_IBRS
+
 	/*
 	 * This opens a window where we have a user CR3, but are
 	 * running in the kernel.  This makes using the CS
@@ -355,6 +361,8 @@ GLOBAL(int_ret_from_sys_call)
 	 * perf profiles. Nothing jumps here.
 	 */
 syscall_return_via_sysret:
+	DISABLE_IBRS
+
 	/* rcx and r11 are already restored (see code above) */
 	RESTORE_C_REGS_EXCEPT_RCX_R11
 	/*
@@ -369,6 +377,7 @@ syscall_return_via_sysret:
 	USERGS_SYSRET64
 
 opportunistic_sysret_failed:
+	/* XXX: might we need a DISABLE_IBRS */
 	/*
 	 * This opens a window where we have a user CR3, but are
 	 * running in the kernel.  This makes using the CS
@@ -537,6 +546,8 @@ END(irq_entries_start)
 	SWAPGS
 	SWITCH_KERNEL_CR3
 
+	ENABLE_IBRS
+
 	/*
 	 * We need to tell lockdep that IRQs are off.  We can't do this until
 	 * we fix gsbase, and we should do it before enter_from_user_mode
@@ -595,6 +606,7 @@ GLOBAL(retint_user)
 	mov	%rsp,%rdi
 	call	prepare_exit_to_usermode
 	TRACE_IRQS_IRETQ
+	DISABLE_IBRS
 	SWITCH_USER_CR3
 	SWAPGS
 	jmp	restore_regs_and_iret
@@ -1061,6 +1073,8 @@ ENTRY(paranoid_entry)
 	movq	%rax, %cr3
 2:
 #endif
+	ENABLE_IBRS_CLOBBER
+
 	ret
 END(paranoid_entry)
 
@@ -1087,6 +1101,7 @@ ENTRY(paranoid_exit)
 	/* No ALTERNATIVE for X86_FEATURE_KAISER: paranoid_entry sets %ebx */
 	testl	$2, %ebx			/* SWITCH_USER_CR3 needed? */
 	jz	paranoid_exit_no_switch
+	DISABLE_IBRS
 	SWITCH_USER_CR3
 paranoid_exit_no_switch:
 #endif
@@ -1126,6 +1141,8 @@ ENTRY(error_entry)
 	 */
 	SWAPGS
 
+	ENABLE_IBRS
+
 .Lerror_entry_from_usermode_after_swapgs:
 	/*
 	 * We need to tell lockdep that IRQs are off.  We can't do this until
@@ -1176,6 +1193,7 @@ ENTRY(error_entry)
 	 * We came from an IRET to user mode, so we have user gsbase.
 	 * Switch to kernel gsbase:
 	 */
+	ENABLE_IBRS_CLOBBER
 	SWAPGS
 
 	/*
@@ -1326,6 +1344,7 @@ ENTRY(nmi)
 	movq	%rax, %cr3
 2:
 #endif
+	ENABLE_IBRS
 	call	do_nmi
 
 #ifdef CONFIG_KAISER

arch/x86/entry/entry_64_compat.S

@@ -15,6 +15,7 @@
 #include <asm/smap.h>
 #include <asm/pgtable_types.h>
 #include <asm/kaiser.h>
+#include <asm/spec_ctrl.h>
 #include <linux/linkage.h>
 #include <linux/err.h>
 
@@ -99,6 +100,8 @@ ENTRY(entry_SYSENTER_compat)
 	pushq   %r8                     /* pt_regs->r15 = 0 */
 	cld
 
+	ENABLE_IBRS
+
 	/*
 	 * Sysenter doesn't filter flags, so we need to clear NT
 	 * ourselves.  To save a few cycles, we can check whether
@@ -197,6 +200,8 @@ ENTRY(entry_SYSCALL_compat)
 	pushq   %r8                     /* pt_regs->r14 = 0 */
 	pushq   %r8                     /* pt_regs->r15 = 0 */
 
+	ENABLE_IBRS
+
 	/*
 	 * User mode is traced as though IRQs are on, and SYSENTER
 	 * turned them off.
@@ -212,6 +217,7 @@ ENTRY(entry_SYSCALL_compat)
 	/* Opportunistic SYSRET */
 sysret32_from_system_call:
 	TRACE_IRQS_ON			/* User mode traces as IRQs on. */
+	DISABLE_IBRS
 	SWITCH_USER_CR3
 	movq	RBX(%rsp), %rbx		/* pt_regs->rbx */
 	movq	RBP(%rsp), %rbp		/* pt_regs->rbp */
@@ -305,6 +311,8 @@ ENTRY(entry_INT80_compat)
 	pushq   %r15                    /* pt_regs->r15 */
 	cld
 
+	ENABLE_IBRS
+
 	/*
 	 * User mode is traced as though IRQs are on, and the interrupt
 	 * gate turned them off.