Commit 4db73271 authored by Kevin Hao's avatar Kevin Hao Committed by Michael Ellerman

powerpc: Add option to use jump label for cpu_has_feature()

We do binary patching of asm code using CPU features, which is a
one-time operation, done during early boot. However checks of CPU
features in C code are currently done at run time, even though the set
of CPU features can never change after boot.

We can optimise this by using jump labels to implement cpu_has_feature(),
meaning checks in C code are binary patched into a single nop or branch.

For a C sequence along the lines of:

    if (cpu_has_feature(FOO))
         return 2;

The generated code before is roughly:

    ld      r9,-27640(r2)
    ld      r9,0(r9)
    lwz     r9,32(r9)
    cmpwi   cr7,r9,0
    bge     cr7, 1f
    li      r3,2
    blr
1:  ...

After (true):
    nop
    li      r3,2
    blr

After (false):
    b	1f
    li      r3,2
    blr
1:  ...

mpe: Rename MAX_CPU_FEATURES as we already have a #define with that
name, and define it simply as a constant, rather than doing tricks with
sizeof and NULL pointers. Rename the array to cpu_feature_keys. Use the
kconfig we added to guard it. Add BUILD_BUG_ON() if the feature is not a
compile time constant. Rewrite the change log.
Signed-off-by: default avatarKevin Hao <haokexin@gmail.com>
Signed-off-by: default avatarAneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent bfbfc8a4
...@@ -11,10 +11,34 @@ static inline bool early_cpu_has_feature(unsigned long feature) ...@@ -11,10 +11,34 @@ static inline bool early_cpu_has_feature(unsigned long feature)
(CPU_FTRS_POSSIBLE & cur_cpu_spec->cpu_features & feature)); (CPU_FTRS_POSSIBLE & cur_cpu_spec->cpu_features & feature));
} }
#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS
#include <linux/jump_label.h>
#define NUM_CPU_FTR_KEYS 64
extern struct static_key_true cpu_feature_keys[NUM_CPU_FTR_KEYS];
static __always_inline bool cpu_has_feature(unsigned long feature)
{
int i;
BUILD_BUG_ON(!__builtin_constant_p(feature));
if (CPU_FTRS_ALWAYS & feature)
return true;
if (!(CPU_FTRS_POSSIBLE & feature))
return false;
i = __builtin_ctzl(feature);
return static_branch_likely(&cpu_feature_keys[i]);
}
#else
static inline bool cpu_has_feature(unsigned long feature) static inline bool cpu_has_feature(unsigned long feature)
{ {
return early_cpu_has_feature(feature); return early_cpu_has_feature(feature);
} }
#endif
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
#endif /* __ASM_POWERPC_CPUFEATURE_H */ #endif /* __ASM_POWERPC_CPUFEATURE_H */
...@@ -123,6 +123,12 @@ extern void do_feature_fixups(unsigned long value, void *fixup_start, ...@@ -123,6 +123,12 @@ extern void do_feature_fixups(unsigned long value, void *fixup_start,
extern const char *powerpc_base_platform; extern const char *powerpc_base_platform;
#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS
extern void cpu_feature_keys_init(void);
#else
static inline void cpu_feature_keys_init(void) { }
#endif
/* TLB flush actions. Used as argument to cpu_spec.flush_tlb() hook */ /* TLB flush actions. Used as argument to cpu_spec.flush_tlb() hook */
enum { enum {
TLB_INVAL_SCOPE_GLOBAL = 0, /* invalidate all TLBs */ TLB_INVAL_SCOPE_GLOBAL = 0, /* invalidate all TLBs */
......
...@@ -112,6 +112,7 @@ ...@@ -112,6 +112,7 @@
#define MMU_FTRS_PA6T MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \ #define MMU_FTRS_PA6T MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
MMU_FTR_CI_LARGE_PAGE | MMU_FTR_NO_SLBIE_B MMU_FTR_CI_LARGE_PAGE | MMU_FTR_NO_SLBIE_B
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
#include <linux/bug.h>
#include <asm/cputable.h> #include <asm/cputable.h>
#ifdef CONFIG_PPC_FSL_BOOK3E #ifdef CONFIG_PPC_FSL_BOOK3E
......
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
#include <linux/threads.h> #include <linux/threads.h>
#include <linux/init.h> #include <linux/init.h>
#include <linux/export.h> #include <linux/export.h>
#include <linux/jump_label.h>
#include <asm/oprofile_impl.h> #include <asm/oprofile_impl.h>
#include <asm/cputable.h> #include <asm/cputable.h>
...@@ -2224,3 +2225,22 @@ struct cpu_spec * __init identify_cpu(unsigned long offset, unsigned int pvr) ...@@ -2224,3 +2225,22 @@ struct cpu_spec * __init identify_cpu(unsigned long offset, unsigned int pvr)
return NULL; return NULL;
} }
#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS
struct static_key_true cpu_feature_keys[NUM_CPU_FTR_KEYS] = {
[0 ... NUM_CPU_FTR_KEYS - 1] = STATIC_KEY_TRUE_INIT
};
EXPORT_SYMBOL_GPL(cpu_feature_keys);
void __init cpu_feature_keys_init(void)
{
int i;
for (i = 0; i < NUM_CPU_FTR_KEYS; i++) {
unsigned long f = 1ul << i;
if (!(cur_cpu_spec->cpu_features & f))
static_branch_disable(&cpu_feature_keys[i]);
}
}
#endif
...@@ -195,6 +195,7 @@ void __init apply_feature_fixups(void) ...@@ -195,6 +195,7 @@ void __init apply_feature_fixups(void)
* CPU/MMU features. * CPU/MMU features.
*/ */
jump_label_init(); jump_label_init();
cpu_feature_keys_init();
} }
static int __init check_features(void) static int __init check_features(void)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment