Commit 7971e23a authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'x86-trace-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86/trace changes from Ingo Molnar:
 "This adds page fault tracepoints which have zero runtime cost in the
  disabled case via IDT trickery (no NOPs in the page fault hotpath)"

* 'x86-trace-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86, trace: Change user|kernel_page_fault to page_fault_user|kernel
  x86, trace: Add page fault tracepoints
  x86, trace: Delete __trace_alloc_intr_gate()
  x86, trace: Register exception handler to trace IDT
  x86, trace: Remove __alloc_intr_gate()
parents f0d55cc1 a4f61dec
...@@ -327,10 +327,25 @@ static inline void write_trace_idt_entry(int entry, const gate_desc *gate) ...@@ -327,10 +327,25 @@ static inline void write_trace_idt_entry(int entry, const gate_desc *gate)
{ {
write_idt_entry(trace_idt_table, entry, gate); write_idt_entry(trace_idt_table, entry, gate);
} }
static inline void _trace_set_gate(int gate, unsigned type, void *addr,
unsigned dpl, unsigned ist, unsigned seg)
{
gate_desc s;
pack_gate(&s, type, (unsigned long)addr, dpl, ist, seg);
/*
* does not need to be atomic because it is only done once at
* setup time
*/
write_trace_idt_entry(gate, &s);
}
#else #else
static inline void write_trace_idt_entry(int entry, const gate_desc *gate) static inline void write_trace_idt_entry(int entry, const gate_desc *gate)
{ {
} }
#define _trace_set_gate(gate, type, addr, dpl, ist, seg)
#endif #endif
static inline void _set_gate(int gate, unsigned type, void *addr, static inline void _set_gate(int gate, unsigned type, void *addr,
...@@ -353,11 +368,14 @@ static inline void _set_gate(int gate, unsigned type, void *addr, ...@@ -353,11 +368,14 @@ static inline void _set_gate(int gate, unsigned type, void *addr,
* Pentium F0 0F bugfix can have resulted in the mapped * Pentium F0 0F bugfix can have resulted in the mapped
* IDT being write-protected. * IDT being write-protected.
*/ */
static inline void set_intr_gate(unsigned int n, void *addr) #define set_intr_gate(n, addr) \
{ do { \
BUG_ON((unsigned)n > 0xFF); BUG_ON((unsigned)n > 0xFF); \
_set_gate(n, GATE_INTERRUPT, addr, 0, 0, __KERNEL_CS); _set_gate(n, GATE_INTERRUPT, (void *)addr, 0, 0, \
} __KERNEL_CS); \
_trace_set_gate(n, GATE_INTERRUPT, (void *)trace_##addr,\
0, 0, __KERNEL_CS); \
} while (0)
extern int first_system_vector; extern int first_system_vector;
/* used_vectors is BITMAP for irq is not managed by percpu vector_irq */ /* used_vectors is BITMAP for irq is not managed by percpu vector_irq */
...@@ -374,37 +392,10 @@ static inline void alloc_system_vector(int vector) ...@@ -374,37 +392,10 @@ static inline void alloc_system_vector(int vector)
} }
} }
#ifdef CONFIG_TRACING
static inline void trace_set_intr_gate(unsigned int gate, void *addr)
{
gate_desc s;
pack_gate(&s, GATE_INTERRUPT, (unsigned long)addr, 0, 0, __KERNEL_CS);
write_idt_entry(trace_idt_table, gate, &s);
}
static inline void __trace_alloc_intr_gate(unsigned int n, void *addr)
{
trace_set_intr_gate(n, addr);
}
#else
static inline void trace_set_intr_gate(unsigned int gate, void *addr)
{
}
#define __trace_alloc_intr_gate(n, addr)
#endif
static inline void __alloc_intr_gate(unsigned int n, void *addr)
{
set_intr_gate(n, addr);
}
#define alloc_intr_gate(n, addr) \ #define alloc_intr_gate(n, addr) \
do { \ do { \
alloc_system_vector(n); \ alloc_system_vector(n); \
__alloc_intr_gate(n, addr); \ set_intr_gate(n, addr); \
__trace_alloc_intr_gate(n, trace_##addr); \
} while (0) } while (0)
/* /*
......
...@@ -187,6 +187,9 @@ extern __visible void smp_invalidate_interrupt(struct pt_regs *); ...@@ -187,6 +187,9 @@ extern __visible void smp_invalidate_interrupt(struct pt_regs *);
#endif #endif
extern void (*__initconst interrupt[NR_VECTORS-FIRST_EXTERNAL_VECTOR])(void); extern void (*__initconst interrupt[NR_VECTORS-FIRST_EXTERNAL_VECTOR])(void);
#ifdef CONFIG_TRACING
#define trace_interrupt interrupt
#endif
typedef int vector_irq_t[NR_VECTORS]; typedef int vector_irq_t[NR_VECTORS];
DECLARE_PER_CPU(vector_irq_t, vector_irq); DECLARE_PER_CPU(vector_irq_t, vector_irq);
......
...@@ -214,6 +214,9 @@ ...@@ -214,6 +214,9 @@
#ifdef __KERNEL__ #ifdef __KERNEL__
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
extern const char early_idt_handlers[NUM_EXCEPTION_VECTORS][2+2+5]; extern const char early_idt_handlers[NUM_EXCEPTION_VECTORS][2+2+5];
#ifdef CONFIG_TRACING
#define trace_early_idt_handlers early_idt_handlers
#endif
/* /*
* Load a segment. Fall back on loading the zero * Load a segment. Fall back on loading the zero
......
#undef TRACE_SYSTEM
#define TRACE_SYSTEM exceptions
#if !defined(_TRACE_PAGE_FAULT_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_PAGE_FAULT_H
#include <linux/tracepoint.h>
extern void trace_irq_vector_regfunc(void);
extern void trace_irq_vector_unregfunc(void);
DECLARE_EVENT_CLASS(x86_exceptions,
TP_PROTO(unsigned long address, struct pt_regs *regs,
unsigned long error_code),
TP_ARGS(address, regs, error_code),
TP_STRUCT__entry(
__field( unsigned long, address )
__field( unsigned long, ip )
__field( unsigned long, error_code )
),
TP_fast_assign(
__entry->address = address;
__entry->ip = regs->ip;
__entry->error_code = error_code;
),
TP_printk("address=%pf ip=%pf error_code=0x%lx",
(void *)__entry->address, (void *)__entry->ip,
__entry->error_code) );
#define DEFINE_PAGE_FAULT_EVENT(name) \
DEFINE_EVENT_FN(x86_exceptions, name, \
TP_PROTO(unsigned long address, struct pt_regs *regs, \
unsigned long error_code), \
TP_ARGS(address, regs, error_code), \
trace_irq_vector_regfunc, \
trace_irq_vector_unregfunc);
DEFINE_PAGE_FAULT_EVENT(page_fault_user);
DEFINE_PAGE_FAULT_EVENT(page_fault_kernel);
#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH .
#define TRACE_INCLUDE_FILE exceptions
#endif /* _TRACE_PAGE_FAULT_H */
/* This part must be outside protection */
#include <trace/define_trace.h>
...@@ -37,6 +37,23 @@ asmlinkage void machine_check(void); ...@@ -37,6 +37,23 @@ asmlinkage void machine_check(void);
#endif /* CONFIG_X86_MCE */ #endif /* CONFIG_X86_MCE */
asmlinkage void simd_coprocessor_error(void); asmlinkage void simd_coprocessor_error(void);
#ifdef CONFIG_TRACING
asmlinkage void trace_page_fault(void);
#define trace_divide_error divide_error
#define trace_bounds bounds
#define trace_invalid_op invalid_op
#define trace_device_not_available device_not_available
#define trace_coprocessor_segment_overrun coprocessor_segment_overrun
#define trace_invalid_TSS invalid_TSS
#define trace_segment_not_present segment_not_present
#define trace_general_protection general_protection
#define trace_spurious_interrupt_bug spurious_interrupt_bug
#define trace_coprocessor_error coprocessor_error
#define trace_alignment_check alignment_check
#define trace_simd_coprocessor_error simd_coprocessor_error
#define trace_async_page_fault async_page_fault
#endif
dotraplinkage void do_divide_error(struct pt_regs *, long); dotraplinkage void do_divide_error(struct pt_regs *, long);
dotraplinkage void do_debug(struct pt_regs *, long); dotraplinkage void do_debug(struct pt_regs *, long);
dotraplinkage void do_nmi(struct pt_regs *, long); dotraplinkage void do_nmi(struct pt_regs *, long);
...@@ -55,6 +72,9 @@ asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *); ...@@ -55,6 +72,9 @@ asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *);
#endif #endif
dotraplinkage void do_general_protection(struct pt_regs *, long); dotraplinkage void do_general_protection(struct pt_regs *, long);
dotraplinkage void do_page_fault(struct pt_regs *, unsigned long); dotraplinkage void do_page_fault(struct pt_regs *, unsigned long);
#ifdef CONFIG_TRACING
dotraplinkage void trace_do_page_fault(struct pt_regs *, unsigned long);
#endif
dotraplinkage void do_spurious_interrupt_bug(struct pt_regs *, long); dotraplinkage void do_spurious_interrupt_bug(struct pt_regs *, long);
dotraplinkage void do_coprocessor_error(struct pt_regs *, long); dotraplinkage void do_coprocessor_error(struct pt_regs *, long);
dotraplinkage void do_alignment_check(struct pt_regs *, long); dotraplinkage void do_alignment_check(struct pt_regs *, long);
......
...@@ -1244,6 +1244,16 @@ return_to_handler: ...@@ -1244,6 +1244,16 @@ return_to_handler:
*/ */
.pushsection .kprobes.text, "ax" .pushsection .kprobes.text, "ax"
#ifdef CONFIG_TRACING
ENTRY(trace_page_fault)
RING0_EC_FRAME
ASM_CLAC
pushl_cfi $trace_do_page_fault
jmp error_code
CFI_ENDPROC
END(trace_page_fault)
#endif
ENTRY(page_fault) ENTRY(page_fault)
RING0_EC_FRAME RING0_EC_FRAME
ASM_CLAC ASM_CLAC
......
...@@ -1278,6 +1278,17 @@ ENTRY(\sym) ...@@ -1278,6 +1278,17 @@ ENTRY(\sym)
END(\sym) END(\sym)
.endm .endm
#ifdef CONFIG_TRACING
.macro trace_errorentry sym do_sym
errorentry trace(\sym) trace(\do_sym)
errorentry \sym \do_sym
.endm
#else
.macro trace_errorentry sym do_sym
errorentry \sym \do_sym
.endm
#endif
/* error code is on the stack already */ /* error code is on the stack already */
.macro paranoiderrorentry sym do_sym .macro paranoiderrorentry sym do_sym
ENTRY(\sym) ENTRY(\sym)
...@@ -1480,7 +1491,7 @@ zeroentry xen_int3 do_int3 ...@@ -1480,7 +1491,7 @@ zeroentry xen_int3 do_int3
errorentry xen_stack_segment do_stack_segment errorentry xen_stack_segment do_stack_segment
#endif #endif
errorentry general_protection do_general_protection errorentry general_protection do_general_protection
errorentry page_fault do_page_fault trace_errorentry page_fault do_page_fault
#ifdef CONFIG_KVM_GUEST #ifdef CONFIG_KVM_GUEST
errorentry async_page_fault do_async_page_fault errorentry async_page_fault do_async_page_fault
#endif #endif
......
...@@ -162,7 +162,7 @@ asmlinkage void __init x86_64_start_kernel(char * real_mode_data) ...@@ -162,7 +162,7 @@ asmlinkage void __init x86_64_start_kernel(char * real_mode_data)
clear_bss(); clear_bss();
for (i = 0; i < NUM_EXCEPTION_VECTORS; i++) for (i = 0; i < NUM_EXCEPTION_VECTORS; i++)
set_intr_gate(i, &early_idt_handlers[i]); set_intr_gate(i, early_idt_handlers[i]);
load_idt((const struct desc_ptr *)&idt_descr); load_idt((const struct desc_ptr *)&idt_descr);
copy_bootdata(__va(real_mode_data)); copy_bootdata(__va(real_mode_data));
......
...@@ -464,7 +464,7 @@ static struct notifier_block kvm_cpu_notifier = { ...@@ -464,7 +464,7 @@ static struct notifier_block kvm_cpu_notifier = {
static void __init kvm_apf_trap_init(void) static void __init kvm_apf_trap_init(void)
{ {
set_intr_gate(14, &async_page_fault); set_intr_gate(14, async_page_fault);
} }
void __init kvm_guest_init(void) void __init kvm_guest_init(void)
......
...@@ -713,7 +713,7 @@ void __init early_trap_init(void) ...@@ -713,7 +713,7 @@ void __init early_trap_init(void)
/* int3 can be called from all */ /* int3 can be called from all */
set_system_intr_gate_ist(X86_TRAP_BP, &int3, DEBUG_STACK); set_system_intr_gate_ist(X86_TRAP_BP, &int3, DEBUG_STACK);
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
set_intr_gate(X86_TRAP_PF, &page_fault); set_intr_gate(X86_TRAP_PF, page_fault);
#endif #endif
load_idt(&idt_descr); load_idt(&idt_descr);
} }
...@@ -721,7 +721,7 @@ void __init early_trap_init(void) ...@@ -721,7 +721,7 @@ void __init early_trap_init(void)
void __init early_trap_pf_init(void) void __init early_trap_pf_init(void)
{ {
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
set_intr_gate(X86_TRAP_PF, &page_fault); set_intr_gate(X86_TRAP_PF, page_fault);
#endif #endif
} }
...@@ -737,30 +737,30 @@ void __init trap_init(void) ...@@ -737,30 +737,30 @@ void __init trap_init(void)
early_iounmap(p, 4); early_iounmap(p, 4);
#endif #endif
set_intr_gate(X86_TRAP_DE, &divide_error); set_intr_gate(X86_TRAP_DE, divide_error);
set_intr_gate_ist(X86_TRAP_NMI, &nmi, NMI_STACK); set_intr_gate_ist(X86_TRAP_NMI, &nmi, NMI_STACK);
/* int4 can be called from all */ /* int4 can be called from all */
set_system_intr_gate(X86_TRAP_OF, &overflow); set_system_intr_gate(X86_TRAP_OF, &overflow);
set_intr_gate(X86_TRAP_BR, &bounds); set_intr_gate(X86_TRAP_BR, bounds);
set_intr_gate(X86_TRAP_UD, &invalid_op); set_intr_gate(X86_TRAP_UD, invalid_op);
set_intr_gate(X86_TRAP_NM, &device_not_available); set_intr_gate(X86_TRAP_NM, device_not_available);
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
set_task_gate(X86_TRAP_DF, GDT_ENTRY_DOUBLEFAULT_TSS); set_task_gate(X86_TRAP_DF, GDT_ENTRY_DOUBLEFAULT_TSS);
#else #else
set_intr_gate_ist(X86_TRAP_DF, &double_fault, DOUBLEFAULT_STACK); set_intr_gate_ist(X86_TRAP_DF, &double_fault, DOUBLEFAULT_STACK);
#endif #endif
set_intr_gate(X86_TRAP_OLD_MF, &coprocessor_segment_overrun); set_intr_gate(X86_TRAP_OLD_MF, coprocessor_segment_overrun);
set_intr_gate(X86_TRAP_TS, &invalid_TSS); set_intr_gate(X86_TRAP_TS, invalid_TSS);
set_intr_gate(X86_TRAP_NP, &segment_not_present); set_intr_gate(X86_TRAP_NP, segment_not_present);
set_intr_gate_ist(X86_TRAP_SS, &stack_segment, STACKFAULT_STACK); set_intr_gate_ist(X86_TRAP_SS, &stack_segment, STACKFAULT_STACK);
set_intr_gate(X86_TRAP_GP, &general_protection); set_intr_gate(X86_TRAP_GP, general_protection);
set_intr_gate(X86_TRAP_SPURIOUS, &spurious_interrupt_bug); set_intr_gate(X86_TRAP_SPURIOUS, spurious_interrupt_bug);
set_intr_gate(X86_TRAP_MF, &coprocessor_error); set_intr_gate(X86_TRAP_MF, coprocessor_error);
set_intr_gate(X86_TRAP_AC, &alignment_check); set_intr_gate(X86_TRAP_AC, alignment_check);
#ifdef CONFIG_X86_MCE #ifdef CONFIG_X86_MCE
set_intr_gate_ist(X86_TRAP_MC, &machine_check, MCE_STACK); set_intr_gate_ist(X86_TRAP_MC, &machine_check, MCE_STACK);
#endif #endif
set_intr_gate(X86_TRAP_XF, &simd_coprocessor_error); set_intr_gate(X86_TRAP_XF, simd_coprocessor_error);
/* Reserve all the builtin and the syscall vector: */ /* Reserve all the builtin and the syscall vector: */
for (i = 0; i < FIRST_EXTERNAL_VECTOR; i++) for (i = 0; i < FIRST_EXTERNAL_VECTOR; i++)
......
...@@ -6,6 +6,8 @@ nostackp := $(call cc-option, -fno-stack-protector) ...@@ -6,6 +6,8 @@ nostackp := $(call cc-option, -fno-stack-protector)
CFLAGS_physaddr.o := $(nostackp) CFLAGS_physaddr.o := $(nostackp)
CFLAGS_setup_nx.o := $(nostackp) CFLAGS_setup_nx.o := $(nostackp)
CFLAGS_fault.o := -I$(src)/../include/asm/trace
obj-$(CONFIG_X86_PAT) += pat_rbtree.o obj-$(CONFIG_X86_PAT) += pat_rbtree.o
obj-$(CONFIG_SMP) += tlb.o obj-$(CONFIG_SMP) += tlb.o
......
...@@ -20,6 +20,9 @@ ...@@ -20,6 +20,9 @@
#include <asm/kmemcheck.h> /* kmemcheck_*(), ... */ #include <asm/kmemcheck.h> /* kmemcheck_*(), ... */
#include <asm/fixmap.h> /* VSYSCALL_START */ #include <asm/fixmap.h> /* VSYSCALL_START */
#define CREATE_TRACE_POINTS
#include <asm/trace/exceptions.h>
/* /*
* Page fault error code bits: * Page fault error code bits:
* *
...@@ -1232,3 +1235,23 @@ do_page_fault(struct pt_regs *regs, unsigned long error_code) ...@@ -1232,3 +1235,23 @@ do_page_fault(struct pt_regs *regs, unsigned long error_code)
__do_page_fault(regs, error_code); __do_page_fault(regs, error_code);
exception_exit(prev_state); exception_exit(prev_state);
} }
static void trace_page_fault_entries(struct pt_regs *regs,
unsigned long error_code)
{
if (user_mode(regs))
trace_page_fault_user(read_cr2(), regs, error_code);
else
trace_page_fault_kernel(read_cr2(), regs, error_code);
}
dotraplinkage void __kprobes
trace_do_page_fault(struct pt_regs *regs, unsigned long error_code)
{
enum ctx_state prev_state;
prev_state = exception_enter();
trace_page_fault_entries(regs, error_code);
__do_page_fault(regs, error_code);
exception_exit(prev_state);
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment