Commit 11276d53 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar

locking/static_keys: Add a new static_key interface

There are various problems and short-comings with the current
static_key interface:

 - static_key_{true,false}() read like a branch depending on the key
   value, instead of the actual likely/unlikely branch depending on
   init value.

 - static_key_{true,false}() are, as stated above, tied to the
   static_key init values STATIC_KEY_INIT_{TRUE,FALSE}.

 - we're limited to the 2 (out of 4) possible options that compile to
   a default NOP because that's what our arch_static_branch() assembly
   emits.

So provide a new static_key interface:

  DEFINE_STATIC_KEY_TRUE(name);
  DEFINE_STATIC_KEY_FALSE(name);

Which define a key of different types with an initial true/false
value.

Then allow:

   static_branch_likely()
   static_branch_unlikely()

to take a key of either type and emit the right instruction for the
case.

This means adding a second arch_static_branch_jump() assembly helper
which emits a JMP per default.

In order to determine the right instruction for the right state,
encode the branch type in the LSB of jump_entry::key.

This is the final step in removing the naming confusion that has led to
a stream of avoidable bugs such as:

  a833581e ("x86, perf: Fix static_key bug in load_mm_cr4()")

... but it also allows new static key combinations that will give us
performance enhancements in the subsequent patches.

Tested-by: Rabin Vincent <rabin@rab.in> # arm
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Michael Ellerman <mpe@ellerman.id.au> # ppc
Acked-by: Heiko Carstens <heiko.carstens@de.ibm.com> # s390
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 706249c2
...@@ -4,23 +4,32 @@ ...@@ -4,23 +4,32 @@
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
#include <linux/types.h> #include <linux/types.h>
#include <asm/unified.h>
#define JUMP_LABEL_NOP_SIZE 4 #define JUMP_LABEL_NOP_SIZE 4
#ifdef CONFIG_THUMB2_KERNEL static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
#define JUMP_LABEL_NOP "nop.w" {
#else asm_volatile_goto("1:\n\t"
#define JUMP_LABEL_NOP "nop" WASM(nop) "\n\t"
#endif ".pushsection __jump_table, \"aw\"\n\t"
".word 1b, %l[l_yes], %c0\n\t"
".popsection\n\t"
: : "i" (&((char *)key)[branch]) : : l_yes);
return false;
l_yes:
return true;
}
static __always_inline bool arch_static_branch(struct static_key *key) static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
{ {
asm_volatile_goto("1:\n\t" asm_volatile_goto("1:\n\t"
JUMP_LABEL_NOP "\n\t" WASM(b) " %l[l_yes]\n\t"
".pushsection __jump_table, \"aw\"\n\t" ".pushsection __jump_table, \"aw\"\n\t"
".word 1b, %l[l_yes], %c0\n\t" ".word 1b, %l[l_yes], %c0\n\t"
".popsection\n\t" ".popsection\n\t"
: : "i" (key) : : l_yes); : : "i" (&((char *)key)[branch]) : : l_yes);
return false; return false;
l_yes: l_yes:
......
...@@ -26,14 +26,28 @@ ...@@ -26,14 +26,28 @@
#define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE
static __always_inline bool arch_static_branch(struct static_key *key) static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
{ {
asm goto("1: nop\n\t" asm goto("1: nop\n\t"
".pushsection __jump_table, \"aw\"\n\t" ".pushsection __jump_table, \"aw\"\n\t"
".align 3\n\t" ".align 3\n\t"
".quad 1b, %l[l_yes], %c0\n\t" ".quad 1b, %l[l_yes], %c0\n\t"
".popsection\n\t" ".popsection\n\t"
: : "i"(key) : : l_yes); : : "i"(&((char *)key)[branch]) : : l_yes);
return false;
l_yes:
return true;
}
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
{
asm goto("1: b %l[l_yes]\n\t"
".pushsection __jump_table, \"aw\"\n\t"
".align 3\n\t"
".quad 1b, %l[l_yes], %c0\n\t"
".popsection\n\t"
: : "i"(&((char *)key)[branch]) : : l_yes);
return false; return false;
l_yes: l_yes:
......
...@@ -26,14 +26,29 @@ ...@@ -26,14 +26,29 @@
#define NOP_INSN "nop" #define NOP_INSN "nop"
#endif #endif
static __always_inline bool arch_static_branch(struct static_key *key) static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
{ {
asm_volatile_goto("1:\t" NOP_INSN "\n\t" asm_volatile_goto("1:\t" NOP_INSN "\n\t"
"nop\n\t" "nop\n\t"
".pushsection __jump_table, \"aw\"\n\t" ".pushsection __jump_table, \"aw\"\n\t"
WORD_INSN " 1b, %l[l_yes], %0\n\t" WORD_INSN " 1b, %l[l_yes], %0\n\t"
".popsection\n\t" ".popsection\n\t"
: : "i" (key) : : l_yes); : : "i" (&((char *)key)[branch]) : : l_yes);
return false;
l_yes:
return true;
}
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
{
asm_volatile_goto("1:\tj %l[l_yes]\n\t"
"nop\n\t"
".pushsection __jump_table, \"aw\"\n\t"
WORD_INSN " 1b, %l[l_yes], %0\n\t"
".popsection\n\t"
: : "i" (&((char *)key)[branch]) : : l_yes);
return false; return false;
l_yes: l_yes:
return true; return true;
......
...@@ -18,14 +18,29 @@ ...@@ -18,14 +18,29 @@
#define JUMP_ENTRY_TYPE stringify_in_c(FTR_ENTRY_LONG) #define JUMP_ENTRY_TYPE stringify_in_c(FTR_ENTRY_LONG)
#define JUMP_LABEL_NOP_SIZE 4 #define JUMP_LABEL_NOP_SIZE 4
static __always_inline bool arch_static_branch(struct static_key *key) static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
{ {
asm_volatile_goto("1:\n\t" asm_volatile_goto("1:\n\t"
"nop\n\t" "nop\n\t"
".pushsection __jump_table, \"aw\"\n\t" ".pushsection __jump_table, \"aw\"\n\t"
JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t" JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t"
".popsection \n\t" ".popsection \n\t"
: : "i" (key) : : l_yes); : : "i" (&((char *)key)[branch]) : : l_yes);
return false;
l_yes:
return true;
}
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
{
asm_volatile_goto("1:\n\t"
"b %l[l_yes]\n\t"
".pushsection __jump_table, \"aw\"\n\t"
JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t"
".popsection \n\t"
: : "i" (&((char *)key)[branch]) : : l_yes);
return false; return false;
l_yes: l_yes:
return true; return true;
......
...@@ -12,14 +12,29 @@ ...@@ -12,14 +12,29 @@
* We use a brcl 0,2 instruction for jump labels at compile time so it * We use a brcl 0,2 instruction for jump labels at compile time so it
* can be easily distinguished from a hotpatch generated instruction. * can be easily distinguished from a hotpatch generated instruction.
*/ */
static __always_inline bool arch_static_branch(struct static_key *key) static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
{ {
asm_volatile_goto("0: brcl 0,"__stringify(JUMP_LABEL_NOP_OFFSET)"\n" asm_volatile_goto("0: brcl 0,"__stringify(JUMP_LABEL_NOP_OFFSET)"\n"
".pushsection __jump_table, \"aw\"\n" ".pushsection __jump_table, \"aw\"\n"
".balign 8\n" ".balign 8\n"
".quad 0b, %l[label], %0\n" ".quad 0b, %l[label], %0\n"
".popsection\n" ".popsection\n"
: : "X" (key) : : label); : : "X" (&((char *)key)[branch]) : : label);
return false;
label:
return true;
}
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
{
asm_volatile_goto("0: brcl 15, %l[label]\n"
".pushsection __jump_table, \"aw\"\n"
".balign 8\n"
".quad 0b, %l[label], %0\n"
".popsection\n"
: : "X" (&((char *)key)[branch]) : : label);
return false; return false;
label: label:
return true; return true;
......
...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
#define JUMP_LABEL_NOP_SIZE 4 #define JUMP_LABEL_NOP_SIZE 4
static __always_inline bool arch_static_branch(struct static_key *key) static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
{ {
asm_volatile_goto("1:\n\t" asm_volatile_goto("1:\n\t"
"nop\n\t" "nop\n\t"
...@@ -16,7 +16,24 @@ static __always_inline bool arch_static_branch(struct static_key *key) ...@@ -16,7 +16,24 @@ static __always_inline bool arch_static_branch(struct static_key *key)
".align 4\n\t" ".align 4\n\t"
".word 1b, %l[l_yes], %c0\n\t" ".word 1b, %l[l_yes], %c0\n\t"
".popsection \n\t" ".popsection \n\t"
: : "i" (key) : : l_yes); : : "i" (&((char *)key)[branch]) : : l_yes);
return false;
l_yes:
return true;
}
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
{
asm_volatile_goto("1:\n\t"
"b %l[l_yes]\n\t"
"nop\n\t"
".pushsection __jump_table, \"aw\"\n\t"
".align 4\n\t"
".word 1b, %l[l_yes], %c0\n\t"
".popsection \n\t"
: : "i" (&((char *)key)[branch]) : : l_yes);
return false; return false;
l_yes: l_yes:
return true; return true;
......
...@@ -16,7 +16,7 @@ ...@@ -16,7 +16,7 @@
# define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC # define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC
#endif #endif
static __always_inline bool arch_static_branch(struct static_key *key) static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
{ {
asm_volatile_goto("1:" asm_volatile_goto("1:"
".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t" ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
...@@ -24,7 +24,24 @@ static __always_inline bool arch_static_branch(struct static_key *key) ...@@ -24,7 +24,24 @@ static __always_inline bool arch_static_branch(struct static_key *key)
_ASM_ALIGN "\n\t" _ASM_ALIGN "\n\t"
_ASM_PTR "1b, %l[l_yes], %c0 \n\t" _ASM_PTR "1b, %l[l_yes], %c0 \n\t"
".popsection \n\t" ".popsection \n\t"
: : "i" (key) : : l_yes); : : "i" (&((char *)key)[branch]) : : l_yes);
return false;
l_yes:
return true;
}
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
{
asm_volatile_goto("1:"
".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t"
"2:\n\t"
".pushsection __jump_table, \"aw\" \n\t"
_ASM_ALIGN "\n\t"
_ASM_PTR "1b, %l[l_yes], %c0 \n\t"
".popsection \n\t"
: : "i" (&((char *)key)[branch]) : : l_yes);
return false; return false;
l_yes: l_yes:
return true; return true;
......
...@@ -107,12 +107,12 @@ static inline int static_key_count(struct static_key *key) ...@@ -107,12 +107,12 @@ static inline int static_key_count(struct static_key *key)
static __always_inline bool static_key_false(struct static_key *key) static __always_inline bool static_key_false(struct static_key *key)
{ {
return arch_static_branch(key); return arch_static_branch(key, false);
} }
static __always_inline bool static_key_true(struct static_key *key) static __always_inline bool static_key_true(struct static_key *key)
{ {
return !static_key_false(key); return !arch_static_branch(key, true);
} }
extern struct jump_entry __start___jump_table[]; extern struct jump_entry __start___jump_table[];
...@@ -130,12 +130,12 @@ extern void static_key_slow_inc(struct static_key *key); ...@@ -130,12 +130,12 @@ extern void static_key_slow_inc(struct static_key *key);
extern void static_key_slow_dec(struct static_key *key); extern void static_key_slow_dec(struct static_key *key);
extern void jump_label_apply_nops(struct module *mod); extern void jump_label_apply_nops(struct module *mod);
#define STATIC_KEY_INIT_TRUE ((struct static_key) \ #define STATIC_KEY_INIT_TRUE \
{ .enabled = ATOMIC_INIT(1), \ { .enabled = ATOMIC_INIT(1), \
.entries = (void *)JUMP_TYPE_TRUE }) .entries = (void *)JUMP_TYPE_TRUE }
#define STATIC_KEY_INIT_FALSE ((struct static_key) \ #define STATIC_KEY_INIT_FALSE \
{ .enabled = ATOMIC_INIT(0), \ { .enabled = ATOMIC_INIT(0), \
.entries = (void *)JUMP_TYPE_FALSE }) .entries = (void *)JUMP_TYPE_FALSE }
#else /* !HAVE_JUMP_LABEL */ #else /* !HAVE_JUMP_LABEL */
...@@ -183,10 +183,8 @@ static inline int jump_label_apply_nops(struct module *mod) ...@@ -183,10 +183,8 @@ static inline int jump_label_apply_nops(struct module *mod)
return 0; return 0;
} }
#define STATIC_KEY_INIT_TRUE ((struct static_key) \ #define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) }
{ .enabled = ATOMIC_INIT(1) }) #define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) }
#define STATIC_KEY_INIT_FALSE ((struct static_key) \
{ .enabled = ATOMIC_INIT(0) })
#endif /* HAVE_JUMP_LABEL */ #endif /* HAVE_JUMP_LABEL */
...@@ -218,6 +216,137 @@ static inline void static_key_disable(struct static_key *key) ...@@ -218,6 +216,137 @@ static inline void static_key_disable(struct static_key *key)
static_key_slow_dec(key); static_key_slow_dec(key);
} }
/* -------------------------------------------------------------------------- */
/*
* Two type wrappers around static_key, such that we can use compile time
* type differentiation to emit the right code.
*
* All the below code is macros in order to play type games.
*/
struct static_key_true {
struct static_key key;
};
struct static_key_false {
struct static_key key;
};
#define STATIC_KEY_TRUE_INIT (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE, }
#define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }
#define DEFINE_STATIC_KEY_TRUE(name) \
struct static_key_true name = STATIC_KEY_TRUE_INIT
#define DEFINE_STATIC_KEY_FALSE(name) \
struct static_key_false name = STATIC_KEY_FALSE_INIT
#ifdef HAVE_JUMP_LABEL
/*
* Combine the right initial value (type) with the right branch order
* to generate the desired result.
*
*
* type\branch| likely (1) | unlikely (0)
* -----------+-----------------------+------------------
* | |
* true (1) | ... | ...
* | NOP | JMP L
* | <br-stmts> | 1: ...
* | L: ... |
* | |
* | | L: <br-stmts>
* | | jmp 1b
* | |
* -----------+-----------------------+------------------
* | |
* false (0) | ... | ...
* | JMP L | NOP
* | <br-stmts> | 1: ...
* | L: ... |
* | |
* | | L: <br-stmts>
* | | jmp 1b
* | |
* -----------+-----------------------+------------------
*
* The initial value is encoded in the LSB of static_key::entries,
* type: 0 = false, 1 = true.
*
* The branch type is encoded in the LSB of jump_entry::key,
* branch: 0 = unlikely, 1 = likely.
*
* This gives the following logic table:
*
* enabled type branch instuction
* -----------------------------+-----------
* 0 0 0 | NOP
* 0 0 1 | JMP
* 0 1 0 | NOP
* 0 1 1 | JMP
*
* 1 0 0 | JMP
* 1 0 1 | NOP
* 1 1 0 | JMP
* 1 1 1 | NOP
*
* Which gives the following functions:
*
* dynamic: instruction = enabled ^ branch
* static: instruction = type ^ branch
*
* See jump_label_type() / jump_label_init_type().
*/
extern bool ____wrong_branch_error(void);
#define static_branch_likely(x) \
({ \
bool branch; \
if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
branch = !arch_static_branch(&(x)->key, true); \
else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
branch = !arch_static_branch_jump(&(x)->key, true); \
else \
branch = ____wrong_branch_error(); \
branch; \
})
#define static_branch_unlikely(x) \
({ \
bool branch; \
if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
branch = arch_static_branch_jump(&(x)->key, false); \
else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
branch = arch_static_branch(&(x)->key, false); \
else \
branch = ____wrong_branch_error(); \
branch; \
})
#else /* !HAVE_JUMP_LABEL */
#define static_branch_likely(x) likely(static_key_enabled(&(x)->key))
#define static_branch_unlikely(x) unlikely(static_key_enabled(&(x)->key))
#endif /* HAVE_JUMP_LABEL */
/*
* Advanced usage; refcount, branch is enabled when: count != 0
*/
#define static_branch_inc(x) static_key_slow_inc(&(x)->key)
#define static_branch_dec(x) static_key_slow_dec(&(x)->key)
/*
* Normal usage; boolean enable/disable.
*/
#define static_branch_enable(x) static_key_enable(&(x)->key)
#define static_branch_disable(x) static_key_disable(&(x)->key)
#endif /* _LINUX_JUMP_LABEL_H */ #endif /* _LINUX_JUMP_LABEL_H */
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
...@@ -165,16 +165,22 @@ static inline bool static_key_type(struct static_key *key) ...@@ -165,16 +165,22 @@ static inline bool static_key_type(struct static_key *key)
static inline struct static_key *jump_entry_key(struct jump_entry *entry) static inline struct static_key *jump_entry_key(struct jump_entry *entry)
{ {
return (struct static_key *)((unsigned long)entry->key); return (struct static_key *)((unsigned long)entry->key & ~1UL);
}
static bool jump_entry_branch(struct jump_entry *entry)
{
return (unsigned long)entry->key & 1UL;
} }
static enum jump_label_type jump_label_type(struct jump_entry *entry) static enum jump_label_type jump_label_type(struct jump_entry *entry)
{ {
struct static_key *key = jump_entry_key(entry); struct static_key *key = jump_entry_key(entry);
bool enabled = static_key_enabled(key); bool enabled = static_key_enabled(key);
bool type = static_key_type(key); bool branch = jump_entry_branch(entry);
return enabled ^ type; /* See the comment in linux/jump_label.h */
return enabled ^ branch;
} }
static void __jump_label_update(struct static_key *key, static void __jump_label_update(struct static_key *key,
...@@ -205,7 +211,10 @@ void __init jump_label_init(void) ...@@ -205,7 +211,10 @@ void __init jump_label_init(void)
for (iter = iter_start; iter < iter_stop; iter++) { for (iter = iter_start; iter < iter_stop; iter++) {
struct static_key *iterk; struct static_key *iterk;
arch_jump_label_transform_static(iter, jump_label_type(iter)); /* rewrite NOPs */
if (jump_label_type(iter) == JUMP_LABEL_NOP)
arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);
iterk = jump_entry_key(iter); iterk = jump_entry_key(iter);
if (iterk == key) if (iterk == key)
continue; continue;
...@@ -225,6 +234,16 @@ void __init jump_label_init(void) ...@@ -225,6 +234,16 @@ void __init jump_label_init(void)
#ifdef CONFIG_MODULES #ifdef CONFIG_MODULES
static enum jump_label_type jump_label_init_type(struct jump_entry *entry)
{
struct static_key *key = jump_entry_key(entry);
bool type = static_key_type(key);
bool branch = jump_entry_branch(entry);
/* See the comment in linux/jump_label.h */
return type ^ branch;
}
struct static_key_mod { struct static_key_mod {
struct static_key_mod *next; struct static_key_mod *next;
struct jump_entry *entries; struct jump_entry *entries;
...@@ -276,8 +295,11 @@ void jump_label_apply_nops(struct module *mod) ...@@ -276,8 +295,11 @@ void jump_label_apply_nops(struct module *mod)
if (iter_start == iter_stop) if (iter_start == iter_stop)
return; return;
for (iter = iter_start; iter < iter_stop; iter++) for (iter = iter_start; iter < iter_stop; iter++) {
/* Only write NOPs for arch_branch_static(). */
if (jump_label_init_type(iter) == JUMP_LABEL_NOP)
arch_jump_label_transform_static(iter, JUMP_LABEL_NOP); arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);
}
} }
static int jump_label_add_module(struct module *mod) static int jump_label_add_module(struct module *mod)
...@@ -318,7 +340,8 @@ static int jump_label_add_module(struct module *mod) ...@@ -318,7 +340,8 @@ static int jump_label_add_module(struct module *mod)
jlm->next = key->next; jlm->next = key->next;
key->next = jlm; key->next = jlm;
if (jump_label_type(iter) == JUMP_LABEL_JMP) /* Only update if we've changed from our initial state */
if (jump_label_type(iter) != jump_label_init_type(iter))
__jump_label_update(key, iter, iter_stop); __jump_label_update(key, iter, iter_stop);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment