Commit bc696ca0 authored by Borislav Petkov's avatar Borislav Petkov Committed by Ingo Molnar

x86/cpufeature: Replace the old static_cpu_has() with safe variant

So the old one didn't work properly before alternatives had run.
And it was supposed to provide an optimized JMP because the
assumption was that the offset it is jumping to is within a
signed byte and thus a two-byte JMP.

So I did an x86_64 allyesconfig build and dumped all possible
sites where static_cpu_has() was used. The optimization amounted
to all in all 12(!) places where static_cpu_has() had generated
a 2-byte JMP. Which has saved us a whopping 36 bytes!

This clearly is not worth the trouble so we can remove it. The
only place where the optimization might count - in __switch_to()
- we will handle differently. But that's not subject of this
patch.
Signed-off-by: default avatarBorislav Petkov <bp@suse.de>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/1453842730-28463-6-git-send-email-bp@alien8.deSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent cd4d09ec
...@@ -350,16 +350,6 @@ config DEBUG_IMR_SELFTEST ...@@ -350,16 +350,6 @@ config DEBUG_IMR_SELFTEST
If unsure say N here. If unsure say N here.
config X86_DEBUG_STATIC_CPU_HAS
bool "Debug alternatives"
depends on DEBUG_KERNEL
---help---
This option causes additional code to be generated which
fails if static_cpu_has() is used before alternatives have
run.
If unsure, say N.
config X86_DEBUG_FPU config X86_DEBUG_FPU
bool "Debug the x86 FPU code" bool "Debug the x86 FPU code"
depends on DEBUG_KERNEL depends on DEBUG_KERNEL
......
...@@ -125,103 +125,19 @@ extern const char * const x86_bug_flags[NBUGINTS*32]; ...@@ -125,103 +125,19 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
#define cpu_has_osxsave boot_cpu_has(X86_FEATURE_OSXSAVE) #define cpu_has_osxsave boot_cpu_has(X86_FEATURE_OSXSAVE)
#define cpu_has_hypervisor boot_cpu_has(X86_FEATURE_HYPERVISOR) #define cpu_has_hypervisor boot_cpu_has(X86_FEATURE_HYPERVISOR)
/* /*
* Do not add any more of those clumsy macros - use static_cpu_has_safe() for * Do not add any more of those clumsy macros - use static_cpu_has() for
* fast paths and boot_cpu_has() otherwise! * fast paths and boot_cpu_has() otherwise!
*/ */
#if __GNUC__ >= 4 && defined(CONFIG_X86_FAST_FEATURE_TESTS) #if __GNUC__ >= 4 && defined(CONFIG_X86_FAST_FEATURE_TESTS)
extern void warn_pre_alternatives(void); extern bool __static_cpu_has(u16 bit);
extern bool __static_cpu_has_safe(u16 bit);
/* /*
* Static testing of CPU features. Used the same as boot_cpu_has(). * Static testing of CPU features. Used the same as boot_cpu_has().
* These are only valid after alternatives have run, but will statically * These are only valid after alternatives have run, but will statically
* patch the target code for additional performance. * patch the target code for additional performance.
*/ */
static __always_inline __pure bool __static_cpu_has(u16 bit) static __always_inline __pure bool _static_cpu_has(u16 bit)
{
#ifdef CC_HAVE_ASM_GOTO
#ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS
/*
* Catch too early usage of this before alternatives
* have run.
*/
asm_volatile_goto("1: jmp %l[t_warn]\n"
"2:\n"
".section .altinstructions,\"a\"\n"
" .long 1b - .\n"
" .long 0\n" /* no replacement */
" .word %P0\n" /* 1: do replace */
" .byte 2b - 1b\n" /* source len */
" .byte 0\n" /* replacement len */
" .byte 0\n" /* pad len */
".previous\n"
/* skipping size check since replacement size = 0 */
: : "i" (X86_FEATURE_ALWAYS) : : t_warn);
#endif
asm_volatile_goto("1: jmp %l[t_no]\n"
"2:\n"
".section .altinstructions,\"a\"\n"
" .long 1b - .\n"
" .long 0\n" /* no replacement */
" .word %P0\n" /* feature bit */
" .byte 2b - 1b\n" /* source len */
" .byte 0\n" /* replacement len */
" .byte 0\n" /* pad len */
".previous\n"
/* skipping size check since replacement size = 0 */
: : "i" (bit) : : t_no);
return true;
t_no:
return false;
#ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS
t_warn:
warn_pre_alternatives();
return false;
#endif
#else /* CC_HAVE_ASM_GOTO */
u8 flag;
/* Open-coded due to __stringify() in ALTERNATIVE() */
asm volatile("1: movb $0,%0\n"
"2:\n"
".section .altinstructions,\"a\"\n"
" .long 1b - .\n"
" .long 3f - .\n"
" .word %P1\n" /* feature bit */
" .byte 2b - 1b\n" /* source len */
" .byte 4f - 3f\n" /* replacement len */
" .byte 0\n" /* pad len */
".previous\n"
".section .discard,\"aw\",@progbits\n"
" .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */
".previous\n"
".section .altinstr_replacement,\"ax\"\n"
"3: movb $1,%0\n"
"4:\n"
".previous\n"
: "=qm" (flag) : "i" (bit));
return flag;
#endif /* CC_HAVE_ASM_GOTO */
}
#define static_cpu_has(bit) \
( \
__builtin_constant_p(boot_cpu_has(bit)) ? \
boot_cpu_has(bit) : \
__builtin_constant_p(bit) ? \
__static_cpu_has(bit) : \
boot_cpu_has(bit) \
)
static __always_inline __pure bool _static_cpu_has_safe(u16 bit)
{ {
#ifdef CC_HAVE_ASM_GOTO #ifdef CC_HAVE_ASM_GOTO
asm_volatile_goto("1: jmp %l[t_dynamic]\n" asm_volatile_goto("1: jmp %l[t_dynamic]\n"
...@@ -255,7 +171,7 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit) ...@@ -255,7 +171,7 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit)
t_no: t_no:
return false; return false;
t_dynamic: t_dynamic:
return __static_cpu_has_safe(bit); return __static_cpu_has(bit);
#else #else
u8 flag; u8 flag;
/* Open-coded due to __stringify() in ALTERNATIVE() */ /* Open-coded due to __stringify() in ALTERNATIVE() */
...@@ -293,22 +209,21 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit) ...@@ -293,22 +209,21 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit)
".previous\n" ".previous\n"
: "=qm" (flag) : "=qm" (flag)
: "i" (bit), "i" (X86_FEATURE_ALWAYS)); : "i" (bit), "i" (X86_FEATURE_ALWAYS));
return (flag == 2 ? __static_cpu_has_safe(bit) : flag); return (flag == 2 ? __static_cpu_has(bit) : flag);
#endif /* CC_HAVE_ASM_GOTO */ #endif /* CC_HAVE_ASM_GOTO */
} }
#define static_cpu_has_safe(bit) \ #define static_cpu_has(bit) \
( \ ( \
__builtin_constant_p(boot_cpu_has(bit)) ? \ __builtin_constant_p(boot_cpu_has(bit)) ? \
boot_cpu_has(bit) : \ boot_cpu_has(bit) : \
_static_cpu_has_safe(bit) \ _static_cpu_has(bit) \
) )
#else #else
/* /*
* gcc 3.x is too stupid to do the static test; fall back to dynamic. * gcc 3.x is too stupid to do the static test; fall back to dynamic.
*/ */
#define static_cpu_has(bit) boot_cpu_has(bit) #define static_cpu_has(bit) boot_cpu_has(bit)
#define static_cpu_has_safe(bit) boot_cpu_has(bit)
#endif #endif
#define cpu_has_bug(c, bit) cpu_has(c, (bit)) #define cpu_has_bug(c, bit) cpu_has(c, (bit))
...@@ -316,7 +231,6 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit) ...@@ -316,7 +231,6 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit)
#define clear_cpu_bug(c, bit) clear_cpu_cap(c, (bit)) #define clear_cpu_bug(c, bit) clear_cpu_cap(c, (bit))
#define static_cpu_has_bug(bit) static_cpu_has((bit)) #define static_cpu_has_bug(bit) static_cpu_has((bit))
#define static_cpu_has_bug_safe(bit) static_cpu_has_safe((bit))
#define boot_cpu_has_bug(bit) cpu_has_bug(&boot_cpu_data, (bit)) #define boot_cpu_has_bug(bit) cpu_has_bug(&boot_cpu_data, (bit))
#define MAX_CPU_FEATURES (NCAPINTS * 32) #define MAX_CPU_FEATURES (NCAPINTS * 32)
......
...@@ -59,22 +59,22 @@ extern u64 fpu__get_supported_xfeatures_mask(void); ...@@ -59,22 +59,22 @@ extern u64 fpu__get_supported_xfeatures_mask(void);
*/ */
static __always_inline __pure bool use_eager_fpu(void) static __always_inline __pure bool use_eager_fpu(void)
{ {
return static_cpu_has_safe(X86_FEATURE_EAGER_FPU); return static_cpu_has(X86_FEATURE_EAGER_FPU);
} }
static __always_inline __pure bool use_xsaveopt(void) static __always_inline __pure bool use_xsaveopt(void)
{ {
return static_cpu_has_safe(X86_FEATURE_XSAVEOPT); return static_cpu_has(X86_FEATURE_XSAVEOPT);
} }
static __always_inline __pure bool use_xsave(void) static __always_inline __pure bool use_xsave(void)
{ {
return static_cpu_has_safe(X86_FEATURE_XSAVE); return static_cpu_has(X86_FEATURE_XSAVE);
} }
static __always_inline __pure bool use_fxsr(void) static __always_inline __pure bool use_fxsr(void)
{ {
return static_cpu_has_safe(X86_FEATURE_FXSR); return static_cpu_has(X86_FEATURE_FXSR);
} }
/* /*
...@@ -301,7 +301,7 @@ static inline void copy_xregs_to_kernel_booting(struct xregs_state *xstate) ...@@ -301,7 +301,7 @@ static inline void copy_xregs_to_kernel_booting(struct xregs_state *xstate)
WARN_ON(system_state != SYSTEM_BOOTING); WARN_ON(system_state != SYSTEM_BOOTING);
if (static_cpu_has_safe(X86_FEATURE_XSAVES)) if (static_cpu_has(X86_FEATURE_XSAVES))
XSTATE_OP(XSAVES, xstate, lmask, hmask, err); XSTATE_OP(XSAVES, xstate, lmask, hmask, err);
else else
XSTATE_OP(XSAVE, xstate, lmask, hmask, err); XSTATE_OP(XSAVE, xstate, lmask, hmask, err);
...@@ -323,7 +323,7 @@ static inline void copy_kernel_to_xregs_booting(struct xregs_state *xstate) ...@@ -323,7 +323,7 @@ static inline void copy_kernel_to_xregs_booting(struct xregs_state *xstate)
WARN_ON(system_state != SYSTEM_BOOTING); WARN_ON(system_state != SYSTEM_BOOTING);
if (static_cpu_has_safe(X86_FEATURE_XSAVES)) if (static_cpu_has(X86_FEATURE_XSAVES))
XSTATE_OP(XRSTORS, xstate, lmask, hmask, err); XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);
else else
XSTATE_OP(XRSTOR, xstate, lmask, hmask, err); XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);
...@@ -461,7 +461,7 @@ static inline void copy_kernel_to_fpregs(union fpregs_state *fpstate) ...@@ -461,7 +461,7 @@ static inline void copy_kernel_to_fpregs(union fpregs_state *fpstate)
* pending. Clear the x87 state here by setting it to fixed values. * pending. Clear the x87 state here by setting it to fixed values.
* "m" is a random variable that should be in L1. * "m" is a random variable that should be in L1.
*/ */
if (unlikely(static_cpu_has_bug_safe(X86_BUG_FXSAVE_LEAK))) { if (unlikely(static_cpu_has_bug(X86_BUG_FXSAVE_LEAK))) {
asm volatile( asm volatile(
"fnclex\n\t" "fnclex\n\t"
"emms\n\t" "emms\n\t"
......
...@@ -30,7 +30,7 @@ static unsigned int numachip1_get_apic_id(unsigned long x) ...@@ -30,7 +30,7 @@ static unsigned int numachip1_get_apic_id(unsigned long x)
unsigned long value; unsigned long value;
unsigned int id = (x >> 24) & 0xff; unsigned int id = (x >> 24) & 0xff;
if (static_cpu_has_safe(X86_FEATURE_NODEID_MSR)) { if (static_cpu_has(X86_FEATURE_NODEID_MSR)) {
rdmsrl(MSR_FAM10H_NODE_ID, value); rdmsrl(MSR_FAM10H_NODE_ID, value);
id |= (value << 2) & 0xff00; id |= (value << 2) & 0xff00;
} }
...@@ -178,7 +178,7 @@ static void fixup_cpu_id(struct cpuinfo_x86 *c, int node) ...@@ -178,7 +178,7 @@ static void fixup_cpu_id(struct cpuinfo_x86 *c, int node)
this_cpu_write(cpu_llc_id, node); this_cpu_write(cpu_llc_id, node);
/* Account for nodes per socket in multi-core-module processors */ /* Account for nodes per socket in multi-core-module processors */
if (static_cpu_has_safe(X86_FEATURE_NODEID_MSR)) { if (static_cpu_has(X86_FEATURE_NODEID_MSR)) {
rdmsrl(MSR_FAM10H_NODE_ID, val); rdmsrl(MSR_FAM10H_NODE_ID, val);
nodes = ((val >> 3) & 7) + 1; nodes = ((val >> 3) & 7) + 1;
} }
......
...@@ -1475,19 +1475,11 @@ void cpu_init(void) ...@@ -1475,19 +1475,11 @@ void cpu_init(void)
} }
#endif #endif
#ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS inline bool __static_cpu_has(u16 bit)
void warn_pre_alternatives(void)
{
WARN(1, "You're using static_cpu_has before alternatives have run!\n");
}
EXPORT_SYMBOL_GPL(warn_pre_alternatives);
#endif
inline bool __static_cpu_has_safe(u16 bit)
{ {
return boot_cpu_has(bit); return boot_cpu_has(bit);
} }
EXPORT_SYMBOL_GPL(__static_cpu_has_safe); EXPORT_SYMBOL_GPL(__static_cpu_has);
static void bsp_resume(void) static void bsp_resume(void)
{ {
......
...@@ -362,7 +362,7 @@ static long do_sys_vm86(struct vm86plus_struct __user *user_vm86, bool plus) ...@@ -362,7 +362,7 @@ static long do_sys_vm86(struct vm86plus_struct __user *user_vm86, bool plus)
/* make room for real-mode segments */ /* make room for real-mode segments */
tsk->thread.sp0 += 16; tsk->thread.sp0 += 16;
if (static_cpu_has_safe(X86_FEATURE_SEP)) if (static_cpu_has(X86_FEATURE_SEP))
tsk->thread.sysenter_cs = 0; tsk->thread.sysenter_cs = 0;
load_sp0(tss, &tsk->thread); load_sp0(tss, &tsk->thread);
......
...@@ -1431,7 +1431,7 @@ static int __init intel_pstate_init(void) ...@@ -1431,7 +1431,7 @@ static int __init intel_pstate_init(void)
if (!all_cpu_data) if (!all_cpu_data)
return -ENOMEM; return -ENOMEM;
if (static_cpu_has_safe(X86_FEATURE_HWP) && !no_hwp) { if (static_cpu_has(X86_FEATURE_HWP) && !no_hwp) {
pr_info("intel_pstate: HWP enabled\n"); pr_info("intel_pstate: HWP enabled\n");
hwp_active++; hwp_active++;
} }
......
...@@ -930,7 +930,7 @@ static int check_async_write(struct inode *inode, unsigned long bio_flags) ...@@ -930,7 +930,7 @@ static int check_async_write(struct inode *inode, unsigned long bio_flags)
if (bio_flags & EXTENT_BIO_TREE_LOG) if (bio_flags & EXTENT_BIO_TREE_LOG)
return 0; return 0;
#ifdef CONFIG_X86 #ifdef CONFIG_X86
if (static_cpu_has_safe(X86_FEATURE_XMM4_2)) if (static_cpu_has(X86_FEATURE_XMM4_2))
return 0; return 0;
#endif #endif
return 1; return 1;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment