Commit 9bb15425 authored by Ingo Molnar's avatar Ingo Molnar

Merge branch 'x86/jumplabel' into perf/core

Upcoming kprobes patches rely on the int3 code-patching machinery introduced by:

   fd4363ff x86: Introduce int3 (breakpoint)-based instruction patching
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parents 5a982132 51b2c07b
......@@ -233,6 +233,7 @@ struct text_poke_param {
};
extern void *text_poke(void *addr, const void *opcode, size_t len);
extern void *text_poke_bp(void *addr, const void *opcode, size_t len, void *handler);
extern void *text_poke_smp(void *addr, const void *opcode, size_t len);
extern void text_poke_smp_batch(struct text_poke_param *params, int n);
......
......@@ -11,6 +11,7 @@
#include <linux/memory.h>
#include <linux/stop_machine.h>
#include <linux/slab.h>
#include <linux/kdebug.h>
#include <asm/alternative.h>
#include <asm/sections.h>
#include <asm/pgtable.h>
......@@ -596,6 +597,111 @@ void *__kprobes text_poke(void *addr, const void *opcode, size_t len)
return addr;
}
static void do_sync_core(void *info)
{
sync_core();
}
static bool bp_patching_in_progress;
static void *bp_int3_handler, *bp_int3_addr;
static int int3_notify(struct notifier_block *self, unsigned long val, void *data)
{
struct die_args *args = data;
/* bp_patching_in_progress */
smp_rmb();
if (likely(!bp_patching_in_progress))
return NOTIFY_DONE;
/* we are not interested in non-int3 faults and ring > 0 faults */
if (val != DIE_INT3 || !args->regs || user_mode_vm(args->regs)
|| args->regs->ip != (unsigned long)bp_int3_addr)
return NOTIFY_DONE;
/* set up the specified breakpoint handler */
args->regs->ip = (unsigned long) bp_int3_handler;
return NOTIFY_STOP;
}
/**
* text_poke_bp() -- update instructions on live kernel on SMP
* @addr: address to patch
* @opcode: opcode of new instruction
* @len: length to copy
* @handler: address to jump to when the temporary breakpoint is hit
*
* Modify multi-byte instruction by using int3 breakpoint on SMP.
* In contrary to text_poke_smp(), we completely avoid stop_machine() here,
* and achieve the synchronization using int3 breakpoint.
*
* The way it is done:
* - add a int3 trap to the address that will be patched
* - sync cores
* - update all but the first byte of the patched range
* - sync cores
* - replace the first byte (int3) by the first byte of
* replacing opcode
* - sync cores
*
* Note: must be called under text_mutex.
*/
void *text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)
{
unsigned char int3 = 0xcc;
bp_int3_handler = handler;
bp_int3_addr = (u8 *)addr + sizeof(int3);
bp_patching_in_progress = true;
/*
* Corresponding read barrier in int3 notifier for
* making sure the in_progress flags is correctly ordered wrt.
* patching
*/
smp_wmb();
text_poke(addr, &int3, sizeof(int3));
on_each_cpu(do_sync_core, NULL, 1);
if (len - sizeof(int3) > 0) {
/* patch all but the first byte */
text_poke((char *)addr + sizeof(int3),
(const char *) opcode + sizeof(int3),
len - sizeof(int3));
/*
* According to Intel, this core syncing is very likely
* not necessary and we'd be safe even without it. But
* better safe than sorry (plus there's not only Intel).
*/
on_each_cpu(do_sync_core, NULL, 1);
}
/* patch the first byte */
text_poke(addr, opcode, sizeof(int3));
on_each_cpu(do_sync_core, NULL, 1);
bp_patching_in_progress = false;
smp_wmb();
return addr;
}
/* this one needs to run before anything else handles it as a
* regular exception */
static struct notifier_block int3_nb = {
.priority = 0x7fffffff,
.notifier_call = int3_notify
};
static int __init int3_init(void)
{
return register_die_notifier(&int3_nb);
}
arch_initcall(int3_init);
/*
* Cross-modifying kernel text with stop_machine().
* This code originally comes from immediate value.
......
......@@ -37,7 +37,19 @@ static void __jump_label_transform(struct jump_entry *entry,
} else
memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);
(*poker)((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
/*
* Make text_poke_bp() a default fallback poker.
*
* At the time the change is being done, just ignore whether we
* are doing nop -> jump or jump -> nop transition, and assume
* always nop being the 'currently valid' instruction
*
*/
if (poker)
(*poker)((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
else
text_poke_bp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE,
(void *)entry->code + JUMP_LABEL_NOP_SIZE);
}
void arch_jump_label_transform(struct jump_entry *entry,
......@@ -45,7 +57,7 @@ void arch_jump_label_transform(struct jump_entry *entry,
{
get_online_cpus();
mutex_lock(&text_mutex);
__jump_label_transform(entry, type, text_poke_smp);
__jump_label_transform(entry, type, NULL);
mutex_unlock(&text_mutex);
put_online_cpus();
}
......
......@@ -1709,7 +1709,7 @@ EXPORT_SYMBOL_GPL(unregister_kprobes);
static struct notifier_block kprobe_exceptions_nb = {
.notifier_call = kprobe_exceptions_notify,
.priority = 0x7fffffff /* we need to be notified first */
.priority = 0x7ffffff0 /* High priority, but not first. */
};
unsigned long __weak arch_deref_entry_point(void *entry)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment