Commit 07f65a83 authored by Martin Schwidefsky's avatar Martin Schwidefsky Committed by Greg Kroah-Hartman

s390: add options to change branch prediction behaviour for the kernel

[ Upstream commit d768bd89 ]

Add the PPA instruction to the system entry and exit path to switch
the kernel to a different branch prediction behaviour. The instructions
are added via CPU alternatives and can be disabled with the "nospec"
or the "nobp=0" kernel parameter. If the default behaviour selected
with CONFIG_KERNEL_NOBP is set to "n" then the "nobp=1" parameter can be
used to enable the changed kernel branch prediction.
Acked-by: default avatarCornelia Huck <cohuck@redhat.com>
Signed-off-by: default avatarMartin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
parent 46bd2c0f
...@@ -704,6 +704,23 @@ config SECCOMP ...@@ -704,6 +704,23 @@ config SECCOMP
If unsure, say Y. If unsure, say Y.
config KERNEL_NOBP
def_bool n
prompt "Enable modified branch prediction for the kernel by default"
help
If this option is selected the kernel will switch to a modified
branch prediction mode if the firmware interface is available.
The modified branch prediction mode improves the behaviour in
regard to speculative execution.
With the option enabled the kernel parameter "nobp=0" or "nospec"
can be used to run the kernel in the normal branch prediction mode.
With the option disabled the modified branch prediction mode is
enabled with the "nobp=1" kernel parameter.
If unsure, say N.
endmenu endmenu
menu "Power Management" menu "Power Management"
......
...@@ -84,6 +84,7 @@ void cpu_detect_mhz_feature(void); ...@@ -84,6 +84,7 @@ void cpu_detect_mhz_feature(void);
extern const struct seq_operations cpuinfo_op; extern const struct seq_operations cpuinfo_op;
extern int sysctl_ieee_emulation_warnings; extern int sysctl_ieee_emulation_warnings;
extern void execve_tail(void); extern void execve_tail(void);
extern void __bpon(void);
/* /*
* User space process size: 2GB for 31 bit, 4TB or 8PT for 64 bit. * User space process size: 2GB for 31 bit, 4TB or 8PT for 64 bit.
......
...@@ -14,6 +14,29 @@ static int __init disable_alternative_instructions(char *str) ...@@ -14,6 +14,29 @@ static int __init disable_alternative_instructions(char *str)
early_param("noaltinstr", disable_alternative_instructions); early_param("noaltinstr", disable_alternative_instructions);
static int __init nobp_setup_early(char *str)
{
bool enabled;
int rc;
rc = kstrtobool(str, &enabled);
if (rc)
return rc;
if (enabled && test_facility(82))
__set_facility(82, S390_lowcore.alt_stfle_fac_list);
else
__clear_facility(82, S390_lowcore.alt_stfle_fac_list);
return 0;
}
early_param("nobp", nobp_setup_early);
static int __init nospec_setup_early(char *str)
{
__clear_facility(82, S390_lowcore.alt_stfle_fac_list);
return 0;
}
early_param("nospec", nospec_setup_early);
struct brcl_insn { struct brcl_insn {
u16 opc; u16 opc;
s32 disp; s32 disp;
......
...@@ -302,6 +302,8 @@ static noinline __init void setup_facility_list(void) ...@@ -302,6 +302,8 @@ static noinline __init void setup_facility_list(void)
memcpy(S390_lowcore.alt_stfle_fac_list, memcpy(S390_lowcore.alt_stfle_fac_list,
S390_lowcore.stfle_fac_list, S390_lowcore.stfle_fac_list,
sizeof(S390_lowcore.alt_stfle_fac_list)); sizeof(S390_lowcore.alt_stfle_fac_list));
if (!IS_ENABLED(CONFIG_KERNEL_NOBP))
__clear_facility(82, S390_lowcore.alt_stfle_fac_list);
} }
static __init void detect_diag9c(void) static __init void detect_diag9c(void)
......
...@@ -163,6 +163,34 @@ _PIF_WORK = (_PIF_PER_TRAP) ...@@ -163,6 +163,34 @@ _PIF_WORK = (_PIF_PER_TRAP)
tm off+\addr, \mask tm off+\addr, \mask
.endm .endm
.macro BPOFF
.pushsection .altinstr_replacement, "ax"
660: .long 0xb2e8c000
.popsection
661: .long 0x47000000
.pushsection .altinstructions, "a"
.long 661b - .
.long 660b - .
.word 82
.byte 4
.byte 4
.popsection
.endm
.macro BPON
.pushsection .altinstr_replacement, "ax"
662: .long 0xb2e8d000
.popsection
663: .long 0x47000000
.pushsection .altinstructions, "a"
.long 663b - .
.long 662b - .
.word 82
.byte 4
.byte 4
.popsection
.endm
.section .kprobes.text, "ax" .section .kprobes.text, "ax"
.Ldummy: .Ldummy:
/* /*
...@@ -175,6 +203,11 @@ _PIF_WORK = (_PIF_PER_TRAP) ...@@ -175,6 +203,11 @@ _PIF_WORK = (_PIF_PER_TRAP)
*/ */
nop 0 nop 0
ENTRY(__bpon)
.globl __bpon
BPON
br %r14
/* /*
* Scheduler resume function, called by switch_to * Scheduler resume function, called by switch_to
* gpr2 = (task_struct *) prev * gpr2 = (task_struct *) prev
...@@ -234,7 +267,10 @@ ENTRY(sie64a) ...@@ -234,7 +267,10 @@ ENTRY(sie64a)
jnz .Lsie_skip jnz .Lsie_skip
TSTMSK __LC_CPU_FLAGS,_CIF_FPU TSTMSK __LC_CPU_FLAGS,_CIF_FPU
jo .Lsie_skip # exit if fp/vx regs changed jo .Lsie_skip # exit if fp/vx regs changed
BPON
sie 0(%r14) sie 0(%r14)
.Lsie_exit:
BPOFF
.Lsie_skip: .Lsie_skip:
ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE
lctlg %c1,%c1,__LC_USER_ASCE # load primary asce lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
...@@ -286,6 +322,7 @@ ENTRY(system_call) ...@@ -286,6 +322,7 @@ ENTRY(system_call)
stpt __LC_SYNC_ENTER_TIMER stpt __LC_SYNC_ENTER_TIMER
.Lsysc_stmg: .Lsysc_stmg:
stmg %r8,%r15,__LC_SAVE_AREA_SYNC stmg %r8,%r15,__LC_SAVE_AREA_SYNC
BPOFF
lg %r10,__LC_LAST_BREAK lg %r10,__LC_LAST_BREAK
lg %r12,__LC_THREAD_INFO lg %r12,__LC_THREAD_INFO
lghi %r14,_PIF_SYSCALL lghi %r14,_PIF_SYSCALL
...@@ -332,6 +369,7 @@ ENTRY(system_call) ...@@ -332,6 +369,7 @@ ENTRY(system_call)
jnz .Lsysc_work # check for work jnz .Lsysc_work # check for work
TSTMSK __LC_CPU_FLAGS,_CIF_WORK TSTMSK __LC_CPU_FLAGS,_CIF_WORK
jnz .Lsysc_work jnz .Lsysc_work
BPON
.Lsysc_restore: .Lsysc_restore:
lg %r14,__LC_VDSO_PER_CPU lg %r14,__LC_VDSO_PER_CPU
lmg %r0,%r10,__PT_R0(%r11) lmg %r0,%r10,__PT_R0(%r11)
...@@ -492,6 +530,7 @@ ENTRY(kernel_thread_starter) ...@@ -492,6 +530,7 @@ ENTRY(kernel_thread_starter)
ENTRY(pgm_check_handler) ENTRY(pgm_check_handler)
stpt __LC_SYNC_ENTER_TIMER stpt __LC_SYNC_ENTER_TIMER
BPOFF
stmg %r8,%r15,__LC_SAVE_AREA_SYNC stmg %r8,%r15,__LC_SAVE_AREA_SYNC
lg %r10,__LC_LAST_BREAK lg %r10,__LC_LAST_BREAK
lg %r12,__LC_THREAD_INFO lg %r12,__LC_THREAD_INFO
...@@ -590,6 +629,7 @@ ENTRY(pgm_check_handler) ...@@ -590,6 +629,7 @@ ENTRY(pgm_check_handler)
ENTRY(io_int_handler) ENTRY(io_int_handler)
STCK __LC_INT_CLOCK STCK __LC_INT_CLOCK
stpt __LC_ASYNC_ENTER_TIMER stpt __LC_ASYNC_ENTER_TIMER
BPOFF
stmg %r8,%r15,__LC_SAVE_AREA_ASYNC stmg %r8,%r15,__LC_SAVE_AREA_ASYNC
lg %r10,__LC_LAST_BREAK lg %r10,__LC_LAST_BREAK
lg %r12,__LC_THREAD_INFO lg %r12,__LC_THREAD_INFO
...@@ -641,9 +681,13 @@ ENTRY(io_int_handler) ...@@ -641,9 +681,13 @@ ENTRY(io_int_handler)
lg %r14,__LC_VDSO_PER_CPU lg %r14,__LC_VDSO_PER_CPU
lmg %r0,%r10,__PT_R0(%r11) lmg %r0,%r10,__PT_R0(%r11)
mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
tm __PT_PSW+1(%r11),0x01 # returning to user ?
jno .Lio_exit_kernel
BPON
.Lio_exit_timer: .Lio_exit_timer:
stpt __LC_EXIT_TIMER stpt __LC_EXIT_TIMER
mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
.Lio_exit_kernel:
lmg %r11,%r15,__PT_R11(%r11) lmg %r11,%r15,__PT_R11(%r11)
lpswe __LC_RETURN_PSW lpswe __LC_RETURN_PSW
.Lio_done: .Lio_done:
...@@ -775,6 +819,7 @@ ENTRY(io_int_handler) ...@@ -775,6 +819,7 @@ ENTRY(io_int_handler)
ENTRY(ext_int_handler) ENTRY(ext_int_handler)
STCK __LC_INT_CLOCK STCK __LC_INT_CLOCK
stpt __LC_ASYNC_ENTER_TIMER stpt __LC_ASYNC_ENTER_TIMER
BPOFF
stmg %r8,%r15,__LC_SAVE_AREA_ASYNC stmg %r8,%r15,__LC_SAVE_AREA_ASYNC
lg %r10,__LC_LAST_BREAK lg %r10,__LC_LAST_BREAK
lg %r12,__LC_THREAD_INFO lg %r12,__LC_THREAD_INFO
...@@ -824,6 +869,7 @@ ENTRY(psw_idle) ...@@ -824,6 +869,7 @@ ENTRY(psw_idle)
.Lpsw_idle_stcctm: .Lpsw_idle_stcctm:
#endif #endif
oi __LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT oi __LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT
BPON
STCK __CLOCK_IDLE_ENTER(%r2) STCK __CLOCK_IDLE_ENTER(%r2)
stpt __TIMER_IDLE_ENTER(%r2) stpt __TIMER_IDLE_ENTER(%r2)
.Lpsw_idle_lpsw: .Lpsw_idle_lpsw:
...@@ -931,6 +977,7 @@ load_fpu_regs: ...@@ -931,6 +977,7 @@ load_fpu_regs:
*/ */
ENTRY(mcck_int_handler) ENTRY(mcck_int_handler)
STCK __LC_MCCK_CLOCK STCK __LC_MCCK_CLOCK
BPOFF
la %r1,4095 # revalidate r1 la %r1,4095 # revalidate r1
spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # revalidate cpu timer spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # revalidate cpu timer
lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# revalidate gprs lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# revalidate gprs
...@@ -997,6 +1044,7 @@ ENTRY(mcck_int_handler) ...@@ -997,6 +1044,7 @@ ENTRY(mcck_int_handler)
mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ? tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
jno 0f jno 0f
BPON
stpt __LC_EXIT_TIMER stpt __LC_EXIT_TIMER
mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
0: lmg %r11,%r15,__PT_R11(%r11) 0: lmg %r11,%r15,__PT_R11(%r11)
......
...@@ -563,6 +563,7 @@ static struct kset *ipl_kset; ...@@ -563,6 +563,7 @@ static struct kset *ipl_kset;
static void __ipl_run(void *unused) static void __ipl_run(void *unused)
{ {
__bpon();
diag308(DIAG308_LOAD_CLEAR, NULL); diag308(DIAG308_LOAD_CLEAR, NULL);
if (MACHINE_IS_VM) if (MACHINE_IS_VM)
__cpcmd("IPL", NULL, 0, NULL); __cpcmd("IPL", NULL, 0, NULL);
......
...@@ -304,6 +304,7 @@ static void pcpu_delegate(struct pcpu *pcpu, void (*func)(void *), ...@@ -304,6 +304,7 @@ static void pcpu_delegate(struct pcpu *pcpu, void (*func)(void *),
mem_assign_absolute(lc->restart_fn, (unsigned long) func); mem_assign_absolute(lc->restart_fn, (unsigned long) func);
mem_assign_absolute(lc->restart_data, (unsigned long) data); mem_assign_absolute(lc->restart_data, (unsigned long) data);
mem_assign_absolute(lc->restart_source, source_cpu); mem_assign_absolute(lc->restart_source, source_cpu);
__bpon();
asm volatile( asm volatile(
"0: sigp 0,%0,%2 # sigp restart to target cpu\n" "0: sigp 0,%0,%2 # sigp restart to target cpu\n"
" brc 2,0b # busy, try again\n" " brc 2,0b # busy, try again\n"
...@@ -877,6 +878,7 @@ void __cpu_die(unsigned int cpu) ...@@ -877,6 +878,7 @@ void __cpu_die(unsigned int cpu)
void __noreturn cpu_die(void) void __noreturn cpu_die(void)
{ {
idle_task_exit(); idle_task_exit();
__bpon();
pcpu_sigp_retry(pcpu_devices + smp_processor_id(), SIGP_STOP, 0); pcpu_sigp_retry(pcpu_devices + smp_processor_id(), SIGP_STOP, 0);
for (;;) ; for (;;) ;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment