UBUNTU: SAUCE: powerpc: Secure memory rfi flush

CVE-2017-5754

BugLink: http://bugs.launchpad.net/bugs/1742772

This puts a nop before each rfid/hrfid and patches in an L1-D
cache flush instruction where possible.

It provides /sys/devices/system/cpu/rfi_flush which can report and can
patch the rfi flushes at runtime.

This has some debug checking in the rfi instructions to make sure
we're returning to the context we think we are, so we can avoid
some flushes.

Includes support for querying the device tree, or hypervisor, to
determine the platform's capabilities and requirements. Also includes
an implementation of the hcall for KVM guests.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Signed-off-by: Michael Neuling <mikey@neuling.org>
Signed-off-by: Oliver O'Halloran <oohall@gmail.com>
Signed-off-by: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
Signed-off-by: Marcelo Henrique Cerri <marcelo.cerri@canonical.com>

arch/powerpc/include/asm/exception-64e.h

@@ -209,5 +209,11 @@ exc_##label##_book3e:
 	ori	r3,r3,vector_offset@l;		\
 	mtspr	SPRN_IVOR##vector_number,r3;
 
+#define RFI_TO_KERNEL							\
+	rfi
+
+#define RFI_TO_USER							\
+	rfi
+
 #endif /* _ASM_POWERPC_EXCEPTION_64E_H */
 

arch/powerpc/include/asm/exception-64s.h

@@ -35,6 +35,8 @@
  * implementations as possible.
  */
 
+#include <asm/bug.h>
+
 #define EX_R9		0
 #define EX_R10		8
 #define EX_R11		16
@@ -50,6 +52,72 @@
 #define EX_PPR		88	/* SMT thread status register (priority) */
 #define EX_CTR		96
 
+/*
+ * The nop instruction allows a secure memory protection instruction to be
+ * inserted with the rfi flush fixup.
+ */
+#define PREPARE_RFI_TO_USER						\
+	RFI_FLUSH_FIXUP_SECTION;					\
+	nop
+
+#define PREPARE_RFI_TO_GUEST						\
+	RFI_FLUSH_FIXUP_SECTION;					\
+	nop
+
+#define DEBUG_RFI
+
+#ifdef DEBUG_RFI
+#define CHECK_TARGET_MSR_PR(srr_reg, expected_pr)			\
+	SET_SCRATCH0(r3);						\
+	mfspr	r3,srr_reg;						\
+	extrdi	r3,r3,1,63-MSR_PR_LG;					\
+666:	tdnei	r3,expected_pr;						\
+	EMIT_BUG_ENTRY 666b,__FILE__,__LINE__,0;			\
+	GET_SCRATCH0(r3);
+#else
+#define CHECK_TARGET_MSR_PR(srr_reg, expected_pr)
+#endif
+
+#define RFI_TO_KERNEL							\
+	CHECK_TARGET_MSR_PR(SPRN_SRR1, 0);				\
+	rfid
+
+#define RFI_TO_USER							\
+	CHECK_TARGET_MSR_PR(SPRN_SRR1, 1);				\
+	PREPARE_RFI_TO_USER;						\
+	rfid;								\
+	b	rfi_flush_fallback
+
+#define RFI_TO_USER_OR_KERNEL						\
+	PREPARE_RFI_TO_USER;						\
+	rfid;								\
+	b	rfi_flush_fallback
+
+#define RFI_TO_GUEST							\
+	PREPARE_RFI_TO_GUEST;						\
+	rfid;								\
+	b	rfi_flush_fallback
+
+#define HRFI_TO_KERNEL							\
+	CHECK_TARGET_MSR_PR(SPRN_HSRR1, 0);				\
+	hrfid
+
+#define HRFI_TO_USER							\
+	CHECK_TARGET_MSR_PR(SPRN_HSRR1, 1);				\
+	PREPARE_RFI_TO_USER;						\
+	hrfid;								\
+	b	hrfi_flush_fallback
+
+#define HRFI_TO_USER_OR_KERNEL						\
+	PREPARE_RFI_TO_USER;						\
+	hrfid;								\
+	b	hrfi_flush_fallback
+
+#define HRFI_TO_GUEST							\
+	PREPARE_RFI_TO_GUEST;						\
+	hrfid;								\
+	b	hrfi_flush_fallback
+
 #ifdef CONFIG_RELOCATABLE
 #define __EXCEPTION_RELON_PROLOG_PSERIES_1(label, h)			\
 	ld	r12,PACAKBASE(r13);	/* get high part of &label */	\
@@ -191,7 +259,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 	mtspr	SPRN_##h##SRR0,r12;					\
 	mfspr	r12,SPRN_##h##SRR1;	/* and SRR1 */			\
 	mtspr	SPRN_##h##SRR1,r10;					\
-	h##rfid;							\
+	h##RFI_TO_KERNEL;						\
 	b	.	/* prevent speculative execution */
 #define EXCEPTION_PROLOG_PSERIES_1(label, h)				\
 	__EXCEPTION_PROLOG_PSERIES_1(label, h)

arch/powerpc/include/asm/feature-fixups.h

@@ -184,4 +184,20 @@ label##3:					       	\
 	FTR_ENTRY_OFFSET label##1b-label##3b;		\
 	.popsection;
 
+#define RFI_FLUSH_FIXUP_SECTION				\
+951:							\
+	.pushsection	__rfi_flush_fixup,"a";		\
+	.align 2;					\
+952:							\
+	FTR_ENTRY_OFFSET 951b-952b;			\
+	.popsection;
+
+#ifndef __ASSEMBLY__
+#include <linux/types.h>
+
+extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup;
+extern void do_rfi_flush_fixups(bool enable, unsigned int insn);
+
+#endif
+
 #endif /* __ASM_POWERPC_FEATURE_FIXUPS_H */

arch/powerpc/include/asm/hvcall.h

@@ -240,6 +240,7 @@
 #define H_GET_HCA_INFO          0x1B8
 #define H_GET_PERF_COUNT        0x1BC
 #define H_MANAGE_TRACE          0x1C0
+#define H_GET_CPU_CHARACTERISTICS 0x1C8
 #define H_FREE_LOGICAL_LAN_BUFFER 0x1D4
 #define H_QUERY_INT_STATE       0x1E4
 #define H_POLL_PENDING		0x1D8
@@ -306,6 +307,19 @@
 #define H_SET_MODE_RESOURCE_ADDR_TRANS_MODE	3
 #define H_SET_MODE_RESOURCE_LE			4
 
+/* H_GET_CPU_CHARACTERISTICS return values */
+#define H_GET_CPU_CHAR_CHAR_ORI31_SPEC_BAR	PPC_BIT(0)
+#define H_GET_CPU_CHAR_CHAR_BCCTR_SERIAL	PPC_BIT(1)
+#define H_GET_CPU_CHAR_CHAR_ORI30_L1_FLUSH	PPC_BIT(2)
+#define H_GET_CPU_CHAR_CHAR_MTTRIG2_L1_FLUSH	PPC_BIT(3)
+#define H_GET_CPU_CHAR_CHAR_L1D_PRIVATE		PPC_BIT(4)
+#define H_GET_CPU_CHAR_CHAR_BC_HINTS_HONORED	PPC_BIT(5)
+#define H_GET_CPU_CHAR_CHAR_MTTRID01_THR_CFG	PPC_BIT(6)
+
+#define H_GET_CPU_CHAR_BEHAV_FAV_SEC_VS_PERF	PPC_BIT(0)
+#define H_GET_CPU_CHAR_BEHAV_L1_FLUSH_LOW_PRIV	PPC_BIT(1)
+#define H_GET_CPU_CHAR_BEHAV_SPEC_BAR_BNDS_CHK	PPC_BIT(2)
+
 #ifndef __ASSEMBLY__
 
 /**

arch/powerpc/include/asm/paca.h

@@ -167,6 +167,8 @@ struct paca_struct {
 #endif
 
 #ifdef CONFIG_PPC_BOOK3S_64
+	void *rfi_flush_fallback_area;
+
 	/* Exclusive emergency stack pointer for machine check exception. */
 	void *mc_emergency_sp;
 	/*
@@ -201,6 +203,15 @@ struct paca_struct {
 #endif
 	struct kvmppc_host_state kvm_hstate;
 #endif
+#ifdef CONFIG_PPC_BOOK3S_64
+	/*
+	 * rfi fallback flush must be in its own cacheline to prevent
+	 * other paca data leaking into the L1d
+	 */
+	u64 exrfi[13] __aligned(0x80);
+	/* Number of consecutive 128 byte lines that must be loaded */
+	u64 l1d_flush_lines;
+#endif
 };
 
 extern struct paca_struct *paca;

arch/powerpc/include/asm/plpar_wrappers.h

@@ -323,4 +323,20 @@ static inline long plapr_set_watchpoint0(unsigned long dawr0, unsigned long dawr
 	return plpar_set_mode(0, H_SET_MODE_RESOURCE_SET_DAWR, dawr0, dawrx0);
 }
 
+static inline long plpar_get_cpu_characteristics(unsigned long *character,
+						 unsigned long *behavior)
+{
+	long rc;
+	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
+
+	rc = plpar_hcall(H_GET_CPU_CHARACTERISTICS, retbuf);
+
+	if (character)
+		*character = retbuf[0];
+	if (behavior)
+		*behavior = retbuf[1];
+
+	return rc;
+}
+
 #endif /* _ASM_POWERPC_PLPAR_WRAPPERS_H */

arch/powerpc/include/asm/setup.h

@@ -26,6 +26,17 @@ void initmem_init(void);
 void setup_panic(void);
 #define ARCH_PANIC_TIMEOUT 180
 
+void rfi_flush_enable(bool enable);
+
+enum l1d_flush_type {
+	L1D_FLUSH_NONE,
+	L1D_FLUSH_FALLBACK,
+	L1D_FLUSH_ORI,
+	L1D_FLUSH_MTTRIG,
+};
+
+void __init setup_rfi_flush(enum l1d_flush_type, bool enable);
+
 #endif /* !__ASSEMBLY__ */
 
 #endif	/* _ASM_POWERPC_SETUP_H */

arch/powerpc/kernel/asm-offsets.c

@@ -243,6 +243,9 @@ int main(void)
 #ifdef CONFIG_PPC_BOOK3S_64
 	DEFINE(PACAMCEMERGSP, offsetof(struct paca_struct, mc_emergency_sp));
 	DEFINE(PACA_IN_MCE, offsetof(struct paca_struct, in_mce));
+	OFFSET(PACA_RFI_FLUSH_FALLBACK_AREA, paca_struct, rfi_flush_fallback_area);
+	OFFSET(PACA_EXRFI, paca_struct, exrfi);
+	OFFSET(PACA_L1D_FLUSH_LINES, paca_struct, l1d_flush_lines);
 #endif
 	DEFINE(PACAHWCPUID, offsetof(struct paca_struct, hw_cpu_id));
 	DEFINE(PACAKEXECSTATE, offsetof(struct paca_struct, kexec_state));

arch/powerpc/kernel/entry_64.S

@@ -36,6 +36,11 @@
 #include <asm/hw_irq.h>
 #include <asm/context_tracking.h>
 #include <asm/tm.h>
+#ifdef CONFIG_PPC_BOOK3S
+#include <asm/exception-64s.h>
+#else
+#include <asm/exception-64e.h>
+#endif
 
 /*
  * System calls.
@@ -225,13 +230,23 @@ END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
 	ACCOUNT_CPU_USER_EXIT(r11, r12)
 	HMT_MEDIUM_LOW_HAS_PPR
 	ld	r13,GPR13(r1)	/* only restore r13 if returning to usermode */
+	ld	r2,GPR2(r1)
+	ld	r1,GPR1(r1)
+	mtlr	r4
+	mtcr	r5
+	mtspr	SPRN_SRR0,r7
+	mtspr	SPRN_SRR1,r8
+	RFI_TO_USER
+	b	.	/* prevent speculative execution */
+
+	/* exit to kernel */
 1:	ld	r2,GPR2(r1)
 	ld	r1,GPR1(r1)
 	mtlr	r4
 	mtcr	r5
 	mtspr	SPRN_SRR0,r7
 	mtspr	SPRN_SRR1,r8
-	RFI
+	RFI_TO_KERNEL
 	b	.	/* prevent speculative execution */
 
 syscall_error:	
@@ -353,8 +368,7 @@ tabort_syscall:
 	mtmsrd	r10, 1
 	mtspr	SPRN_SRR0, r11
 	mtspr	SPRN_SRR1, r12
-
-	rfid
+	RFI_TO_USER
 	b	.	/* prevent speculative execution */
 #endif
 
@@ -887,7 +901,7 @@ BEGIN_FTR_SECTION
 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
 	ACCOUNT_CPU_USER_EXIT(r2, r4)
 	REST_GPR(13, r1)
-1:
+
 	mtspr	SPRN_SRR1,r3
 
 	ld	r2,_CCR(r1)
@@ -900,8 +914,22 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
 	ld	r3,GPR3(r1)
 	ld	r4,GPR4(r1)
 	ld	r1,GPR1(r1)
+	RFI_TO_USER
+	b	.	/* prevent speculative execution */
 
-	rfid
+1:	mtspr	SPRN_SRR1,r3
+
+	ld	r2,_CCR(r1)
+	mtcrf	0xFF,r2
+	ld	r2,_NIP(r1)
+	mtspr	SPRN_SRR0,r2
+
+	ld	r0,GPR0(r1)
+	ld	r2,GPR2(r1)
+	ld	r3,GPR3(r1)
+	ld	r4,GPR4(r1)
+	ld	r1,GPR1(r1)
+	RFI_TO_KERNEL
 	b	.	/* prevent speculative execution */
 
 #endif /* CONFIG_PPC_BOOK3E */
@@ -1077,7 +1105,7 @@ _GLOBAL(enter_rtas)
 	
 	mtspr	SPRN_SRR0,r5
 	mtspr	SPRN_SRR1,r6
-	rfid
+	RFI_TO_KERNEL
 	b	.	/* prevent speculative execution */
 
 rtas_return_loc:
@@ -1102,7 +1130,7 @@ rtas_return_loc:
 
 	mtspr	SPRN_SRR0,r3
 	mtspr	SPRN_SRR1,r4
-	rfid
+	RFI_TO_KERNEL
 	b	.	/* prevent speculative execution */
 
 	.align	3
@@ -1173,7 +1201,7 @@ _GLOBAL(enter_prom)
 	LOAD_REG_IMMEDIATE(r12, MSR_SF | MSR_ISF | MSR_LE)
 	andc	r11,r11,r12
 	mtsrr1	r11
-	rfid
+	RFI_TO_KERNEL
 #endif /* CONFIG_PPC_BOOK3E */
 
 1:	/* Return from OF */

arch/powerpc/kernel/exceptions-64s.S

@@ -46,7 +46,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)				\
 	mtspr	SPRN_SRR0,r10 ; 				\
 	ld	r10,PACAKMSR(r13) ;				\
 	mtspr	SPRN_SRR1,r10 ; 				\
-	rfid ; 							\
+	RFI_TO_KERNEL ;					\
 	b	. ;	/* prevent speculative execution */
 
 #define SYSCALL_PSERIES_3					\
@@ -54,7 +54,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)				\
 1:	mfspr	r12,SPRN_SRR1 ;					\
 	xori	r12,r12,MSR_LE ;				\
 	mtspr	SPRN_SRR1,r12 ;					\
-	rfid ;		/* return to userspace */		\
+	RFI_TO_USER ;	/* return to userspace */		\
 	b	. ;	/* prevent speculative execution */
 
 #if defined(CONFIG_RELOCATABLE)
@@ -507,7 +507,7 @@ BEGIN_FTR_SECTION
 	LOAD_HANDLER(r12, machine_check_handle_early)
 1:	mtspr	SPRN_SRR0,r12
 	mtspr	SPRN_SRR1,r11
-	rfid
+	RFI_TO_KERNEL
 	b	.	/* prevent speculative execution */
 2:
 	/* Stack overflow. Stay on emergency stack and panic.
@@ -666,7 +666,7 @@ masked_##_H##interrupt:					\
 	ld	r10,PACA_EXGEN+EX_R10(r13);		\
 	ld	r11,PACA_EXGEN+EX_R11(r13);		\
 	GET_SCRATCH0(r13);				\
-	##_H##rfid;					\
+	##_H##RFI_TO_KERNEL;				\
 	b	.
 	
 	MASKED_INTERRUPT()
@@ -720,6 +720,72 @@ system_reset_fwnmi:
 
 #endif /* CONFIG_PPC_PSERIES */
 
+	.globl rfi_flush_fallback
+rfi_flush_fallback:
+	SET_SCRATCH0(r13);
+	GET_PACA(r13);
+	std	r9,PACA_EXRFI+EX_R9(r13)
+	std	r10,PACA_EXRFI+EX_R10(r13)
+	std	r11,PACA_EXRFI+EX_R11(r13)
+	mfctr	r9
+	ld	r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
+	ld	r11,PACA_L1D_FLUSH_LINES(r13)
+	srdi	r11,r11,2	/* Unrolled x4 */
+	mtctr	r11
+	DCBT_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
+	/* XXX: Should an instruction synchronizing operation be done here? */
+1:
+	/*
+	 * The load adresses are at staggered offsets within cachelines,
+	 * which suits some pipelines better (on others it should not
+	 * hurt.
+	 */
+	ld	r11,128*0+0(r10)
+	ld	r11,128*1+8(r10)
+	ld	r11,128*2+16(r10)
+	ld	r11,128*3+24(r10)
+	addi	r10,r10,(128 * 4)
+	bdnz	1b
+	mtctr	r9
+	ld	r9,PACA_EXRFI+EX_R9(r13)
+	ld	r10,PACA_EXRFI+EX_R10(r13)
+	ld	r11,PACA_EXRFI+EX_R11(r13)
+	GET_SCRATCH0(r13);
+	rfid
+
+	.globl hrfi_flush_fallback
+hrfi_flush_fallback:
+	SET_SCRATCH0(r13);
+	GET_PACA(r13);
+	std	r9,PACA_EXRFI+EX_R9(r13)
+	std	r10,PACA_EXRFI+EX_R10(r13)
+	std	r11,PACA_EXRFI+EX_R11(r13)
+	mfctr	r9
+	ld	r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
+	ld	r11,PACA_L1D_FLUSH_LINES(r13)
+	srdi	r11,r11,2	/* Unrolled x4 */
+	mtctr	r11
+	DCBT_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
+	/* XXX: Should an instruction synchronizing operation be done here? */
+1:
+	/*
+	 * The load adresses are at staggered offsets within cachelines,
+	 * which suits some pipelines better (on others it should not
+	 * hurt.
+	 */
+	ld	r11,128*0+0(r10)
+	ld	r11,128*1+8(r10)
+	ld	r11,128*2+16(r10)
+	ld	r11,128*3+24(r10)
+	addi	r10,r10,(128 * 4)
+	bdnz	1b
+	mtctr	r9
+	ld	r9,PACA_EXRFI+EX_R9(r13)
+	ld	r10,PACA_EXRFI+EX_R10(r13)
+	ld	r11,PACA_EXRFI+EX_R11(r13)
+	GET_SCRATCH0(r13);
+	hrfid
+
 #ifdef __DISABLED__
 /*
  * This is used for when the SLB miss handler has to go virtual,
@@ -744,7 +810,7 @@ slb_miss_user_pseries:
 	mtspr	SRR0,r12
 	mfspr	r12,SRR1			/* and SRR1 */
 	mtspr	SRR1,r10
-	rfid
+	rfid	/* This code is disabled, hence no change */
 	b	.				/* prevent spec. execution */
 #endif /* __DISABLED__ */
 
@@ -758,7 +824,7 @@ kvmppc_skip_interrupt:
 	addi	r13, r13, 4
 	mtspr	SPRN_SRR0, r13
 	GET_SCRATCH0(r13)
-	rfid
+	RFI_TO_GUEST
 	b	.
 
 kvmppc_skip_Hinterrupt:
@@ -770,7 +836,7 @@ kvmppc_skip_Hinterrupt:
 	addi	r13, r13, 4
 	mtspr	SPRN_HSRR0, r13
 	GET_SCRATCH0(r13)
-	hrfid
+	RFI_TO_GUEST
 	b	.
 #endif
 
@@ -1060,7 +1126,7 @@ slb_miss_user_common:
 	ld	r11,PACA_EXGEN+EX_R11(r13)
 	ld	r12,PACA_EXGEN+EX_R12(r13)
 	ld	r13,PACA_EXGEN+EX_R13(r13)
-	rfid
+	rfid	/* Disabled code. No need to change */
 	b	.
 
 slb_miss_fault:
@@ -1441,7 +1507,7 @@ machine_check_handle_early:
 	li	r3,MSR_ME
 	andc	r10,r10,r3		/* Turn off MSR_ME */
 	mtspr	SPRN_SRR1,r10
-	rfid
+	RFI_TO_KERNEL
 	b	.
 2:
 	/*
@@ -1459,7 +1525,7 @@ machine_check_handle_early:
 	 */
 	bl	machine_check_queue_event
 	MACHINE_CHECK_HANDLER_WINDUP
-	rfid
+	RFI_TO_USER_OR_KERNEL
 9:
 	/* Deliver the machine check to host kernel in V mode. */
 	MACHINE_CHECK_HANDLER_WINDUP
@@ -1506,6 +1572,9 @@ slb_miss_realmode:
 	andi.	r10,r12,MSR_RI	/* check for unrecoverable exception */
 	beq-	2f
 
+	andi.	r10,r12,MSR_PR	/* check for exception from userspace */
+	beq	1f	/* returning to kernel */
+
 .machine	push
 .machine	"power4"
 	mtcrf	0x80,r9
@@ -1518,7 +1587,22 @@ slb_miss_realmode:
 	ld	r11,PACA_EXSLB+EX_R11(r13)
 	ld	r12,PACA_EXSLB+EX_R12(r13)
 	ld	r13,PACA_EXSLB+EX_R13(r13)
-	rfid
+	RFI_TO_USER
+	b	.	/* prevent speculative execution */
+1:
+.machine	push
+.machine	"power4"
+	mtcrf	0x80,r9
+	mtcrf	0x01,r9		/* slb_allocate uses cr0 and cr7 */
+.machine	pop
+
+	RESTORE_PPR_PACA(PACA_EXSLB, r9)
+	ld	r9,PACA_EXSLB+EX_R9(r13)
+	ld	r10,PACA_EXSLB+EX_R10(r13)
+	ld	r11,PACA_EXSLB+EX_R11(r13)
+	ld	r12,PACA_EXSLB+EX_R12(r13)
+	ld	r13,PACA_EXSLB+EX_R13(r13)
+	RFI_TO_KERNEL
 	b	.	/* prevent speculative execution */
 
 2:	mfspr	r11,SPRN_SRR0
@@ -1527,7 +1611,7 @@ slb_miss_realmode:
 	mtspr	SPRN_SRR0,r10
 	ld	r10,PACAKMSR(r13)
 	mtspr	SPRN_SRR1,r10
-	rfid
+	RFI_TO_KERNEL
 	b	.
 
 unrecov_slb:

arch/powerpc/kernel/setup_64.c

@@ -835,3 +835,79 @@ static int __init disable_hardlockup_detector(void)
 }
 early_initcall(disable_hardlockup_detector);
 #endif
+#ifdef CONFIG_PPC_BOOK3S_64
+static enum l1d_flush_type l1d_flush_type;
+static void *l1d_flush_fallback_area;
+bool rfi_flush;
+
+static void do_nothing(void *unused)
+{
+	/*
+	 * We don't need to do the flush explicitly, just enter+exit kernel is
+	 * sufficient, the RFI exit handlers will do the right thing.
+	 */
+}
+
+void rfi_flush_enable(bool enable)
+{
+	unsigned int insn;
+
+	if (rfi_flush == enable)
+		return;
+
+	switch (l1d_flush_type) {
+	case L1D_FLUSH_NONE:
+		insn = 0x60000000; /* nop */
+		break;
+	case L1D_FLUSH_FALLBACK:
+		insn = 0x48000008; /* b .+8 to fallback flush */
+		pr_info("rfi-fixups: Using fallback displacement flush\n");
+		break;
+	case L1D_FLUSH_ORI:
+		insn = 0x63de0000;
+		pr_info("rfi-fixups: Using ori type flush\n");
+		break;
+	case L1D_FLUSH_MTTRIG:
+		insn = 0x7c12dba6;
+		pr_info("rfi-fixups: Using mttrig type flush\n");
+		break;
+	default:
+		printk("rfi-fixups: No flush type detected, system may be vulnerable, update firmware.\n");
+		return;
+	}
+
+	do_rfi_flush_fixups(enable, insn);
+
+	if (enable)
+		on_each_cpu(do_nothing, NULL, 1);
+
+	rfi_flush = enable;
+}
+
+void __init setup_rfi_flush(enum l1d_flush_type type, bool enable)
+{
+	if (type == L1D_FLUSH_FALLBACK) {
+		int cpu;
+		u64 l1d_size = ppc64_caches.dsize;
+		u64 limit = min(safe_stack_limit(), ppc64_rma_size);
+
+		/*
+		 * Align to L1d size, and size it at 2x L1d size, to
+		 * catch possible hardware prefetch runoff. We don't
+		 * have a recipe for load patterns to reliably avoid
+		 * the prefetcher.
+		 */
+		l1d_flush_fallback_area =
+			__va(memblock_alloc_base(l1d_size * 2, l1d_size, limit));
+		memset(l1d_flush_fallback_area, 0, l1d_size * 2);
+
+		for_each_possible_cpu(cpu) {
+			paca[cpu].rfi_flush_fallback_area = l1d_flush_fallback_area;
+			paca[cpu].l1d_flush_lines = l1d_size / 128;
+		}
+	}
+
+	l1d_flush_type = type;
+	rfi_flush_enable(enable);
+}
+#endif /* CONFIG_PPC_BOOK3S_64 */

arch/powerpc/kernel/sysfs.c

@@ -18,8 +18,10 @@
 #include <asm/smp.h>
 #include <asm/pmc.h>
 #include <asm/firmware.h>
+#include <asm/ppc_asm.h>
 
 #include "cacheinfo.h"
+#include "setup.h"
 
 #ifdef CONFIG_PPC64
 #include <asm/paca.h>
@@ -496,6 +498,43 @@ static DEVICE_ATTR(spurr, 0400, show_spurr, NULL);
 static DEVICE_ATTR(purr, 0400, show_purr, store_purr);
 static DEVICE_ATTR(pir, 0400, show_pir, NULL);
 
+#ifdef CONFIG_PPC_BOOK3S_64
+static ssize_t show_rfi_flush(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", rfi_flush ? 1 : 0);
+}
+
+static ssize_t __used store_rfi_flush(struct device *dev,
+		struct device_attribute *attr, const char *buf,
+		size_t count)
+{
+	int val;
+	int ret = 0;
+
+	ret = sscanf(buf, "%d", &val);
+	if (ret != 1)
+		return -EINVAL;
+
+	if (val == 1)
+		rfi_flush_enable(true);
+	else if (val == 0)
+		rfi_flush_enable(false);
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static DEVICE_ATTR(rfi_flush, 0600,
+		show_rfi_flush, store_rfi_flush);
+
+static void sysfs_create_rfi_flush(void)
+{
+	device_create_file(cpu_subsys.dev_root, &dev_attr_rfi_flush);
+}
+#endif /* CONFIG_PPC_BOOK3S_64 */
+
 /*
  * This is the system wide DSCR register default value. Any
  * change to this default value through the sysfs interface
@@ -1058,6 +1097,9 @@ static int __init topology_init(void)
 
 #ifdef CONFIG_PPC64
 	sysfs_create_dscr_default();
+#ifdef CONFIG_PPC_BOOK3S
+	sysfs_create_rfi_flush();
+#endif
 #endif /* CONFIG_PPC64 */
 
 	return 0;

arch/powerpc/kernel/vmlinux.lds.S

@@ -72,6 +72,15 @@ SECTIONS
 	/* Read-only data */
 	RODATA
 
+#ifdef CONFIG_PPC64
+	. = ALIGN(8);
+	__rfi_flush_fixup : AT(ADDR(__rfi_flush_fixup) - LOAD_OFFSET) {
+		__start___rfi_flush_fixup = .;
+		*(__rfi_flush_fixup)
+		__stop___rfi_flush_fixup = .;
+	}
+#endif
+
 	EXCEPTION_TABLE(0)
 
 	NOTES :kernel :notes

arch/powerpc/kvm/book3s_hv.c

@@ -683,6 +683,74 @@ static int kvmppc_get_yield_count(struct kvm_vcpu *vcpu)
 	return yield_count;
 }
 
+static unsigned long characteristics, behaviour;
+static bool have_characteristics;
+
+struct hdat_to_papr_mapping {
+	unsigned long bit;
+	char *name;
+	bool is_behaviour;
+};
+
+static const struct hdat_to_papr_mapping mapping[] = {
+  { .bit = H_GET_CPU_CHAR_CHAR_MTTRID01_THR_CFG,   .name = "inst-thread-reconfig-control-trig0-1", },
+  { .bit = H_GET_CPU_CHAR_CHAR_MTTRIG2_L1_FLUSH,   .name = "inst-l1d-flush-trig2", },
+  { .bit = H_GET_CPU_CHAR_CHAR_ORI30_L1_FLUSH,     .name = "inst-l1d-flush-ori30,30,0", },
+  { .bit = H_GET_CPU_CHAR_CHAR_ORI31_SPEC_BAR,     .name = "inst-spec-barrier-ori31,31,0", },
+  { .bit = H_GET_CPU_CHAR_CHAR_L1D_PRIVATE,        .name = "fw-l1d-thread-split", },
+  { .bit = H_GET_CPU_CHAR_CHAR_BCCTR_SERIAL,       .name = "fw-bcctrl-serialized", },
+//{ .bit = H_GET_CPU_CHAR_CHAR_BC_HINTS_HONORED,   .name = ???, // FIXME Unknown name },
+  { .bit = H_GET_CPU_CHAR_BEHAV_FAV_SEC_VS_PERF,   .name = "speculation-policy-favor-security",   .is_behaviour = true, },
+  { .bit = H_GET_CPU_CHAR_BEHAV_L1_FLUSH_LOW_PRIV, .name = "needs-l1d-flush-msr-hv-1-to-0",       .is_behaviour = true, },
+  { .bit = H_GET_CPU_CHAR_BEHAV_L1_FLUSH_LOW_PRIV, .name = "needs-l1d-flush-msr-pr-0-to-1",       .is_behaviour = true, },
+  { .bit = H_GET_CPU_CHAR_BEHAV_SPEC_BAR_BNDS_CHK, .name = "needs-spec-barrier-for-bound-checks", .is_behaviour = true, },
+};
+
+static void init_cpu_characteristics(void)
+{
+	struct device_node *np, *fw_features;
+	const struct hdat_to_papr_mapping *p;
+	int i;
+
+	np = of_find_node_by_name(NULL, "ibm,opal");
+	fw_features = of_get_child_by_name(np, "fw-features");
+	of_node_put(np);
+
+	if (!fw_features) {
+		have_characteristics = false;
+		return;
+	}
+
+	have_characteristics = true;
+
+	for (i = 0; i < ARRAY_SIZE(mapping); i++) {
+		p = &mapping[i];
+
+		np = of_get_child_by_name(fw_features, p->name);
+		if (np && of_property_read_bool(np, "enabled")) {
+			if (p->is_behaviour) {
+				behaviour |= p->bit;
+			} else {
+				characteristics |= p->bit;
+			}
+		}
+		of_node_put(np);
+	}
+
+	of_node_put(fw_features);
+}
+
+static int kvmppc_h_get_cpu_characteristics(struct kvm_vcpu *vcpu)
+{
+	if (!have_characteristics)
+		return H_FUNCTION;
+
+	kvmppc_set_gpr(vcpu, 4, characteristics);
+	kvmppc_set_gpr(vcpu, 5, behaviour);
+
+	return H_SUCCESS;
+}
+
 int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
 {
 	unsigned long req = kvmppc_get_gpr(vcpu, 3);
@@ -794,6 +862,7 @@ static int kvmppc_hcall_impl_hv(unsigned long cmd)
 	case H_SET_MODE:
 	case H_LOGICAL_CI_LOAD:
 	case H_LOGICAL_CI_STORE:
+	case H_GET_CPU_CHARACTERISTICS:
 #ifdef CONFIG_KVM_XICS
 	case H_XIRR:
 	case H_CPPR:
@@ -943,6 +1012,9 @@ static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
 		kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
 		r = RESUME_GUEST;
 		break;
+	case H_GET_CPU_CHARACTERISTICS:
+		ret = kvmppc_h_get_cpu_characteristics(vcpu);
+		break;
 	default:
 		kvmppc_dump_regs(vcpu);
 		printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
@@ -3212,6 +3284,7 @@ static unsigned int default_hcall_list[] = {
 	H_PROD,
 	H_CONFER,
 	H_REGISTER_VPA,
+	H_GET_CPU_CHARACTERISTICS,
 #ifdef CONFIG_KVM_XICS
 	H_EOI,
 	H_CPPR,
@@ -3319,6 +3392,7 @@ static int kvmppc_book3s_init_hv(void)
 	kvm_ops_hv.owner = THIS_MODULE;
 	kvmppc_hv_ops = &kvm_ops_hv;
 
+	init_cpu_characteristics();
 	init_default_hcalls();
 
 	init_vcore_lists();

arch/powerpc/kvm/book3s_hv_rmhandlers.S

@@ -65,7 +65,7 @@ _GLOBAL_TOC(kvmppc_hv_entry_trampoline)
 	mtmsrd	r0,1		/* clear RI in MSR */
 	mtsrr0	r5
 	mtsrr1	r6
-	RFI
+	RFI_TO_KERNEL
 
 kvmppc_call_hv_entry:
 	ld	r4, HSTATE_KVM_VCPU(r13)
@@ -171,7 +171,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
 	mtsrr0	r8
 	mtsrr1	r7
 	beq	cr1, 13f		/* machine check */
-	RFI
+	RFI_TO_KERNEL
 
 	/* On POWER7, we have external interrupts set to use HSRR0/1 */
 11:	mtspr	SPRN_HSRR0, r8
@@ -989,8 +989,7 @@ BEGIN_FTR_SECTION
 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
 	ld	r0, VCPU_GPR(R0)(r4)
 	ld	r4, VCPU_GPR(R4)(r4)
-
-	hrfid
+	HRFI_TO_GUEST
 	b	.
 
 secondary_too_late:

arch/powerpc/kvm/book3s_rmhandlers.S

@@ -141,7 +141,7 @@ kvmppc_handler_skip_ins:
 	GET_SCRATCH0(r13)
 
 	/* And get back into the code */
-	RFI
+	RFI_TO_GUEST
 #endif
 
 /*
@@ -164,6 +164,6 @@ _GLOBAL_TOC(kvmppc_entry_trampoline)
 	ori	r5, r5, MSR_EE
 	mtsrr0	r7
 	mtsrr1	r6
-	RFI
+	RFI_TO_KERNEL
 
 #include "book3s_segment.S"

arch/powerpc/kvm/trace_hv.h

@@ -95,6 +95,7 @@
 	{H_GET_HCA_INFO,		"H_GET_HCA_INFO"}, \
 	{H_GET_PERF_COUNT,		"H_GET_PERF_COUNT"}, \
 	{H_MANAGE_TRACE,		"H_MANAGE_TRACE"}, \
+	{H_GET_CPU_CHARACTERISTICS,	"H_GET_CPU_CHARACTERISTICS"}, \
 	{H_FREE_LOGICAL_LAN_BUFFER,	"H_FREE_LOGICAL_LAN_BUFFER"}, \
 	{H_QUERY_INT_STATE,		"H_QUERY_INT_STATE"}, \
 	{H_POLL_PENDING,		"H_POLL_PENDING"}, \

arch/powerpc/lib/feature-fixups.c

@@ -113,6 +113,33 @@ void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end)
 	}
 }
 
+#ifdef CONFIG_PPC_BOOK3S_64
+void do_rfi_flush_fixups(bool enable, unsigned int insn)
+{
+	long *start, *end;
+	unsigned int *dest;
+	int i;
+
+	start = PTRRELOC(&__start___rfi_flush_fixup),
+	end = PTRRELOC(&__stop___rfi_flush_fixup);
+
+	for (i = 0; start < end; start++, i++) {
+		dest = (void *)start + *start;
+
+		pr_devel("RFI FLUSH FIXUP %s %lx\n", enable ? "enable" : "disable", (unsigned long)start);
+		if (!enable) {
+			pr_devel("patching dest %lx\n", (unsigned long)dest);
+			patch_instruction(dest, PPC_INST_NOP);
+		} else {
+			pr_devel("patching dest %lx\n", (unsigned long)dest);
+			patch_instruction(dest, insn);
+		}
+	}
+
+	printk(KERN_DEBUG "rfi-fixups: patched %d locations\n", i);
+}
+#endif /* CONFIG_PPC_BOOK3S_64 */
+
 void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end)
 {
 	long *start, *end;

arch/powerpc/platforms/powernv/setup.c

@@ -35,13 +35,65 @@
 #include <asm/opal.h>
 #include <asm/kexec.h>
 #include <asm/smp.h>
+#include <asm/setup.h>
 
 #include "powernv.h"
 
+static void pnv_setup_rfi_flush(void)
+{
+	struct device_node *np, *fw_features;
+	enum l1d_flush_type type;
+	bool enable;
+
+	/* Default to fallback in case fw-features are not available */
+	type = L1D_FLUSH_FALLBACK;
+	enable = true;
+
+	np = of_find_node_by_name(NULL, "ibm,opal");
+	fw_features = of_get_child_by_name(np, "fw-features");
+	of_node_put(np);
+
+	if (fw_features) {
+		/* Default to no flush, unless firmware says otherwise */
+		type = L1D_FLUSH_NONE;
+
+		np = of_get_child_by_name(fw_features, "inst-l1d-flush-trig2");
+		if (np && of_property_read_bool(np, "enabled"))
+			type = L1D_FLUSH_MTTRIG;
+
+		of_node_put(np);
+
+		np = of_get_child_by_name(fw_features, "inst-l1d-flush-ori30,30,0");
+		if (np && of_property_read_bool(np, "enabled"))
+			type = L1D_FLUSH_ORI;
+
+		of_node_put(np);
+
+		/* Don't enable unless firmware says so */
+		enable = false;
+		np = of_get_child_by_name(fw_features, "needs-l1d-flush-msr-hv-1-to-0");
+		if (np && of_property_read_bool(np, "enabled"))
+			enable = true;
+
+		of_node_put(np);
+
+		np = of_get_child_by_name(fw_features, "needs-l1d-flush-msr-pr-0-to-1");
+		if (np && of_property_read_bool(np, "enabled"))
+			enable = true;
+
+		of_node_put(np);
+		of_node_put(fw_features);
+	}
+
+	setup_rfi_flush(type, enable);
+}
+
 static void __init pnv_setup_arch(void)
 {
 	set_arch_panic_timeout(10, ARCH_PANIC_TIMEOUT);
 
+	pnv_setup_rfi_flush();
+
 	/* Initialize SMP */
 	pnv_smp_init();
 

arch/powerpc/platforms/pseries/setup.c

@@ -499,6 +499,32 @@ static void __init find_and_init_phbs(void)
 	of_pci_check_probe_only();
 }
 
+static void pSeries_setup_rfi_flush(void)
+{
+	unsigned long character, behaviour, rc;
+	enum l1d_flush_type type;
+	bool enable;
+
+	/* Default to fallback in case hcall is not available */
+	type = L1D_FLUSH_FALLBACK;
+	enable = true;
+
+	rc = plpar_get_cpu_characteristics(&character, &behaviour);
+	if (rc == H_SUCCESS) {
+		/* Default to no flush, unless firmware says otherwise */
+		type = L1D_FLUSH_NONE;
+		if (character & H_GET_CPU_CHAR_CHAR_MTTRIG2_L1_FLUSH)
+			type = L1D_FLUSH_MTTRIG;
+		if (character & H_GET_CPU_CHAR_CHAR_ORI30_L1_FLUSH)
+			type = L1D_FLUSH_ORI;
+
+		if (!(behaviour & H_GET_CPU_CHAR_BEHAV_L1_FLUSH_LOW_PRIV))
+			enable = false;
+	}
+
+	setup_rfi_flush(type, enable);
+}
+
 static void __init pSeries_setup_arch(void)
 {
 	set_arch_panic_timeout(10, ARCH_PANIC_TIMEOUT);
@@ -515,6 +541,8 @@ static void __init pSeries_setup_arch(void)
 
 	fwnmi_init();
 
+	pSeries_setup_rfi_flush();
+
 	/* By default, only probe PCI (can be overriden by rtas_pci) */
 	pci_add_flags(PCI_PROBE_ONLY);