Commit f8c2ee22 authored by Harvey Harrison's avatar Harvey Harrison Committed by Ingo Molnar

x86: unify fault_32|64.c with ifdefs

Elimination of these ifdefs can be done in a unified file.
Signed-off-by: default avatarHarvey Harrison <harvey.harrison@gmail.com>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
parent 1156e098
...@@ -48,7 +48,11 @@ static inline int notify_page_fault(struct pt_regs *regs) ...@@ -48,7 +48,11 @@ static inline int notify_page_fault(struct pt_regs *regs)
int ret = 0; int ret = 0;
/* kprobe_running() needs smp_processor_id() */ /* kprobe_running() needs smp_processor_id() */
#ifdef CONFIG_X86_32
if (!user_mode_vm(regs)) { if (!user_mode_vm(regs)) {
#else
if (!user_mode(regs)) {
#endif
preempt_disable(); preempt_disable();
if (kprobe_running() && kprobe_fault_handler(regs, 14)) if (kprobe_running() && kprobe_fault_handler(regs, 14))
ret = 1; ret = 1;
...@@ -430,11 +434,15 @@ static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs, ...@@ -430,11 +434,15 @@ static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
#endif #endif
/* /*
* X86_32
* Handle a fault on the vmalloc or module mapping area * Handle a fault on the vmalloc or module mapping area
* *
* X86_64
* Handle a fault on the vmalloc area
*
* This assumes no large pages in there. * This assumes no large pages in there.
*/ */
static inline int vmalloc_fault(unsigned long address) static int vmalloc_fault(unsigned long address)
{ {
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
unsigned long pgd_paddr; unsigned long pgd_paddr;
...@@ -509,6 +517,9 @@ int show_unhandled_signals = 1; ...@@ -509,6 +517,9 @@ int show_unhandled_signals = 1;
* and the problem, and then passes it off to one of the appropriate * and the problem, and then passes it off to one of the appropriate
* routines. * routines.
*/ */
#ifdef CONFIG_X86_64
asmlinkage
#endif
void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code) void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
{ {
struct task_struct *tsk; struct task_struct *tsk;
...@@ -517,6 +528,9 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code) ...@@ -517,6 +528,9 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
unsigned long address; unsigned long address;
int write, si_code; int write, si_code;
int fault; int fault;
#ifdef CONFIG_X86_64
unsigned long flags;
#endif
/* /*
* We can fault from pretty much anywhere, with unknown IRQ state. * We can fault from pretty much anywhere, with unknown IRQ state.
...@@ -548,6 +562,7 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code) ...@@ -548,6 +562,7 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
* (error_code & 4) == 0, and that the fault was not a * (error_code & 4) == 0, and that the fault was not a
* protection error (error_code & 9) == 0. * protection error (error_code & 9) == 0.
*/ */
#ifdef CONFIG_X86_32
if (unlikely(address >= TASK_SIZE)) { if (unlikely(address >= TASK_SIZE)) {
if (!(error_code & (PF_RSVD|PF_USER|PF_PROT)) && if (!(error_code & (PF_RSVD|PF_USER|PF_PROT)) &&
vmalloc_fault(address) >= 0) vmalloc_fault(address) >= 0)
...@@ -570,7 +585,45 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code) ...@@ -570,7 +585,45 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
*/ */
if (in_atomic() || !mm) if (in_atomic() || !mm)
goto bad_area_nosemaphore; goto bad_area_nosemaphore;
#else /* CONFIG_X86_64 */
if (unlikely(address >= TASK_SIZE64)) {
/*
* Don't check for the module range here: its PML4
* is always initialized because it's shared with the main
* kernel text. Only vmalloc may need PML4 syncups.
*/
if (!(error_code & (PF_RSVD|PF_USER|PF_PROT)) &&
((address >= VMALLOC_START && address < VMALLOC_END))) {
if (vmalloc_fault(address) >= 0)
return;
}
/*
* Don't take the mm semaphore here. If we fixup a prefetch
* fault we could otherwise deadlock.
*/
goto bad_area_nosemaphore;
}
if (likely(regs->flags & X86_EFLAGS_IF))
local_irq_enable();
if (unlikely(error_code & PF_RSVD))
pgtable_bad(address, regs, error_code);
/*
* If we're in an interrupt, have no user context or are running in an
* atomic region then we must not take the fault.
*/
if (unlikely(in_atomic() || !mm))
goto bad_area_nosemaphore;
/*
* User-mode registers count as a user access even for any
* potential system fault or CPU buglet.
*/
if (user_mode_vm(regs))
error_code |= PF_USER;
again:
#endif
/* When running in the kernel we expect faults to occur only to /* When running in the kernel we expect faults to occur only to
* addresses in user space. All other faults represent errors in the * addresses in user space. All other faults represent errors in the
* kernel and should generate an OOPS. Unfortunately, in the case of an * kernel and should generate an OOPS. Unfortunately, in the case of an
...@@ -596,7 +649,11 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code) ...@@ -596,7 +649,11 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
vma = find_vma(mm, address); vma = find_vma(mm, address);
if (!vma) if (!vma)
goto bad_area; goto bad_area;
#ifdef CONFIG_X86_32
if (vma->vm_start <= address) if (vma->vm_start <= address)
#else
if (likely(vma->vm_start <= address))
#endif
goto good_area; goto good_area;
if (!(vma->vm_flags & VM_GROWSDOWN)) if (!(vma->vm_flags & VM_GROWSDOWN))
goto bad_area; goto bad_area;
...@@ -634,7 +691,9 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code) ...@@ -634,7 +691,9 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
goto bad_area; goto bad_area;
} }
survive: #ifdef CONFIG_X86_32
survive:
#endif
/* /*
* If for any reason at all we couldn't handle the fault, * If for any reason at all we couldn't handle the fault,
* make sure we exit gracefully rather than endlessly redo * make sure we exit gracefully rather than endlessly redo
...@@ -705,6 +764,7 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code) ...@@ -705,6 +764,7 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
print_vma_addr(" in ", regs->ip); print_vma_addr(" in ", regs->ip);
printk("\n"); printk("\n");
} }
tsk->thread.cr2 = address; tsk->thread.cr2 = address;
/* Kernel addresses are always protection faults */ /* Kernel addresses are always protection faults */
tsk->thread.error_code = error_code | (address >= TASK_SIZE); tsk->thread.error_code = error_code | (address >= TASK_SIZE);
...@@ -722,9 +782,13 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code) ...@@ -722,9 +782,13 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
return; return;
/* /*
* X86_32
* Valid to do another page fault here, because if this fault * Valid to do another page fault here, because if this fault
* had been triggered by is_prefetch fixup_exception would have * had been triggered by is_prefetch fixup_exception would have
* handled it. * handled it.
*
* X86_64
* Hall of shame of CPU/BIOS bugs.
*/ */
if (is_prefetch(regs, address, error_code)) if (is_prefetch(regs, address, error_code))
return; return;
...@@ -736,7 +800,7 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code) ...@@ -736,7 +800,7 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
* Oops. The kernel tried to access some bad page. We'll have to * Oops. The kernel tried to access some bad page. We'll have to
* terminate things with extreme prejudice. * terminate things with extreme prejudice.
*/ */
#ifdef CONFIG_X86_32
bust_spinlocks(1); bust_spinlocks(1);
show_fault_oops(regs, error_code, address); show_fault_oops(regs, error_code, address);
...@@ -747,6 +811,20 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code) ...@@ -747,6 +811,20 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
die("Oops", regs, error_code); die("Oops", regs, error_code);
bust_spinlocks(0); bust_spinlocks(0);
do_exit(SIGKILL); do_exit(SIGKILL);
#else /* CONFIG_X86_64 */
flags = oops_begin();
show_fault_oops(regs, error_code, address);
tsk->thread.cr2 = address;
tsk->thread.trap_no = 14;
tsk->thread.error_code = error_code;
if (__die("Oops", regs, error_code))
regs = NULL;
/* Executive summary in case the body of the oops scrolled away */
printk(KERN_EMERG "CR2: %016lx\n", address);
oops_end(flags, regs, SIGKILL);
#endif
/* /*
* We ran out of memory, or some other thing happened to us that made * We ran out of memory, or some other thing happened to us that made
...@@ -754,11 +832,18 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code) ...@@ -754,11 +832,18 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
*/ */
out_of_memory: out_of_memory:
up_read(&mm->mmap_sem); up_read(&mm->mmap_sem);
#ifdef CONFIG_X86_32
if (is_global_init(tsk)) { if (is_global_init(tsk)) {
yield(); yield();
down_read(&mm->mmap_sem); down_read(&mm->mmap_sem);
goto survive; goto survive;
} }
#else
if (is_global_init(current)) {
yield();
goto again;
}
#endif
printk("VM: killing process %s\n", tsk->comm); printk("VM: killing process %s\n", tsk->comm);
if (error_code & PF_USER) if (error_code & PF_USER)
do_group_exit(SIGKILL); do_group_exit(SIGKILL);
...@@ -770,17 +855,22 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code) ...@@ -770,17 +855,22 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
/* Kernel mode? Handle exceptions or die */ /* Kernel mode? Handle exceptions or die */
if (!(error_code & PF_USER)) if (!(error_code & PF_USER))
goto no_context; goto no_context;
#ifdef CONFIG_X86_32
/* User space => ok to do another page fault */ /* User space => ok to do another page fault */
if (is_prefetch(regs, address, error_code)) if (is_prefetch(regs, address, error_code))
return; return;
#endif
tsk->thread.cr2 = address; tsk->thread.cr2 = address;
tsk->thread.error_code = error_code; tsk->thread.error_code = error_code;
tsk->thread.trap_no = 14; tsk->thread.trap_no = 14;
force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk); force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk);
} }
#ifdef CONFIG_X86_64
DEFINE_SPINLOCK(pgd_lock);
LIST_HEAD(pgd_list);
#endif
void vmalloc_sync_all(void) void vmalloc_sync_all(void)
{ {
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
......
...@@ -51,7 +51,11 @@ static inline int notify_page_fault(struct pt_regs *regs) ...@@ -51,7 +51,11 @@ static inline int notify_page_fault(struct pt_regs *regs)
int ret = 0; int ret = 0;
/* kprobe_running() needs smp_processor_id() */ /* kprobe_running() needs smp_processor_id() */
#ifdef CONFIG_X86_32
if (!user_mode_vm(regs)) {
#else
if (!user_mode(regs)) { if (!user_mode(regs)) {
#endif
preempt_disable(); preempt_disable();
if (kprobe_running() && kprobe_fault_handler(regs, 14)) if (kprobe_running() && kprobe_fault_handler(regs, 14))
ret = 1; ret = 1;
...@@ -433,6 +437,10 @@ static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs, ...@@ -433,6 +437,10 @@ static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
#endif #endif
/* /*
* X86_32
* Handle a fault on the vmalloc or module mapping area
*
* X86_64
* Handle a fault on the vmalloc area * Handle a fault on the vmalloc area
* *
* This assumes no large pages in there. * This assumes no large pages in there.
...@@ -512,16 +520,20 @@ int show_unhandled_signals = 1; ...@@ -512,16 +520,20 @@ int show_unhandled_signals = 1;
* and the problem, and then passes it off to one of the appropriate * and the problem, and then passes it off to one of the appropriate
* routines. * routines.
*/ */
asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, #ifdef CONFIG_X86_64
unsigned long error_code) asmlinkage
#endif
void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
{ {
struct task_struct *tsk; struct task_struct *tsk;
struct mm_struct *mm; struct mm_struct *mm;
struct vm_area_struct *vma; struct vm_area_struct *vma;
unsigned long address; unsigned long address;
int write, fault; int write, si_code;
int fault;
#ifdef CONFIG_X86_64
unsigned long flags; unsigned long flags;
int si_code; #endif
/* /*
* We can fault from pretty much anywhere, with unknown IRQ state. * We can fault from pretty much anywhere, with unknown IRQ state.
...@@ -553,6 +565,30 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, ...@@ -553,6 +565,30 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
* (error_code & 4) == 0, and that the fault was not a * (error_code & 4) == 0, and that the fault was not a
* protection error (error_code & 9) == 0. * protection error (error_code & 9) == 0.
*/ */
#ifdef CONFIG_X86_32
if (unlikely(address >= TASK_SIZE)) {
if (!(error_code & (PF_RSVD|PF_USER|PF_PROT)) &&
vmalloc_fault(address) >= 0)
return;
/*
* Don't take the mm semaphore here. If we fixup a prefetch
* fault we could otherwise deadlock.
*/
goto bad_area_nosemaphore;
}
/* It's safe to allow irq's after cr2 has been saved and the vmalloc
fault has been handled. */
if (regs->flags & (X86_EFLAGS_IF|VM_MASK))
local_irq_enable();
/*
* If we're in an interrupt, have no user context or are running in an
* atomic region then we must not take the fault.
*/
if (in_atomic() || !mm)
goto bad_area_nosemaphore;
#else /* CONFIG_X86_64 */
if (unlikely(address >= TASK_SIZE64)) { if (unlikely(address >= TASK_SIZE64)) {
/* /*
* Don't check for the module range here: its PML4 * Don't check for the module range here: its PML4
...@@ -570,7 +606,6 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, ...@@ -570,7 +606,6 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
*/ */
goto bad_area_nosemaphore; goto bad_area_nosemaphore;
} }
if (likely(regs->flags & X86_EFLAGS_IF)) if (likely(regs->flags & X86_EFLAGS_IF))
local_irq_enable(); local_irq_enable();
...@@ -590,8 +625,8 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, ...@@ -590,8 +625,8 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
*/ */
if (user_mode_vm(regs)) if (user_mode_vm(regs))
error_code |= PF_USER; error_code |= PF_USER;
again:
again: #endif
/* When running in the kernel we expect faults to occur only to /* When running in the kernel we expect faults to occur only to
* addresses in user space. All other faults represent errors in the * addresses in user space. All other faults represent errors in the
* kernel and should generate an OOPS. Unfortunately, in the case of an * kernel and should generate an OOPS. Unfortunately, in the case of an
...@@ -617,7 +652,11 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, ...@@ -617,7 +652,11 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
vma = find_vma(mm, address); vma = find_vma(mm, address);
if (!vma) if (!vma)
goto bad_area; goto bad_area;
#ifdef CONFIG_X86_32
if (vma->vm_start <= address)
#else
if (likely(vma->vm_start <= address)) if (likely(vma->vm_start <= address))
#endif
goto good_area; goto good_area;
if (!(vma->vm_flags & VM_GROWSDOWN)) if (!(vma->vm_flags & VM_GROWSDOWN))
goto bad_area; goto bad_area;
...@@ -655,6 +694,9 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, ...@@ -655,6 +694,9 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
goto bad_area; goto bad_area;
} }
#ifdef CONFIG_X86_32
survive:
#endif
/* /*
* If for any reason at all we couldn't handle the fault, * If for any reason at all we couldn't handle the fault,
* make sure we exit gracefully rather than endlessly redo * make sure we exit gracefully rather than endlessly redo
...@@ -730,7 +772,6 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, ...@@ -730,7 +772,6 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
/* Kernel addresses are always protection faults */ /* Kernel addresses are always protection faults */
tsk->thread.error_code = error_code | (address >= TASK_SIZE); tsk->thread.error_code = error_code | (address >= TASK_SIZE);
tsk->thread.trap_no = 14; tsk->thread.trap_no = 14;
force_sig_info_fault(SIGSEGV, si_code, address, tsk); force_sig_info_fault(SIGSEGV, si_code, address, tsk);
return; return;
} }
...@@ -744,9 +785,14 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, ...@@ -744,9 +785,14 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
return; return;
/* /*
* X86_32
* Valid to do another page fault here, because if this fault
* had been triggered by is_prefetch fixup_exception would have
* handled it.
*
* X86_64
* Hall of shame of CPU/BIOS bugs. * Hall of shame of CPU/BIOS bugs.
*/ */
if (is_prefetch(regs, address, error_code)) if (is_prefetch(regs, address, error_code))
return; return;
...@@ -757,7 +803,18 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, ...@@ -757,7 +803,18 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
* Oops. The kernel tried to access some bad page. We'll have to * Oops. The kernel tried to access some bad page. We'll have to
* terminate things with extreme prejudice. * terminate things with extreme prejudice.
*/ */
#ifdef CONFIG_X86_32
bust_spinlocks(1);
show_fault_oops(regs, error_code, address);
tsk->thread.cr2 = address;
tsk->thread.trap_no = 14;
tsk->thread.error_code = error_code;
die("Oops", regs, error_code);
bust_spinlocks(0);
do_exit(SIGKILL);
#else /* CONFIG_X86_64 */
flags = oops_begin(); flags = oops_begin();
show_fault_oops(regs, error_code, address); show_fault_oops(regs, error_code, address);
...@@ -770,6 +827,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, ...@@ -770,6 +827,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
/* Executive summary in case the body of the oops scrolled away */ /* Executive summary in case the body of the oops scrolled away */
printk(KERN_EMERG "CR2: %016lx\n", address); printk(KERN_EMERG "CR2: %016lx\n", address);
oops_end(flags, regs, SIGKILL); oops_end(flags, regs, SIGKILL);
#endif
/* /*
* We ran out of memory, or some other thing happened to us that made * We ran out of memory, or some other thing happened to us that made
...@@ -777,10 +835,18 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, ...@@ -777,10 +835,18 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
*/ */
out_of_memory: out_of_memory:
up_read(&mm->mmap_sem); up_read(&mm->mmap_sem);
#ifdef CONFIG_X86_32
if (is_global_init(tsk)) {
yield();
down_read(&mm->mmap_sem);
goto survive;
}
#else
if (is_global_init(current)) { if (is_global_init(current)) {
yield(); yield();
goto again; goto again;
} }
#endif
printk("VM: killing process %s\n", tsk->comm); printk("VM: killing process %s\n", tsk->comm);
if (error_code & PF_USER) if (error_code & PF_USER)
do_group_exit(SIGKILL); do_group_exit(SIGKILL);
...@@ -792,16 +858,21 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, ...@@ -792,16 +858,21 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
/* Kernel mode? Handle exceptions or die */ /* Kernel mode? Handle exceptions or die */
if (!(error_code & PF_USER)) if (!(error_code & PF_USER))
goto no_context; goto no_context;
#ifdef CONFIG_X86_32
/* User space => ok to do another page fault */
if (is_prefetch(regs, address, error_code))
return;
#endif
tsk->thread.cr2 = address; tsk->thread.cr2 = address;
tsk->thread.error_code = error_code; tsk->thread.error_code = error_code;
tsk->thread.trap_no = 14; tsk->thread.trap_no = 14;
force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk); force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk);
return;
} }
#ifdef CONFIG_X86_64
DEFINE_SPINLOCK(pgd_lock); DEFINE_SPINLOCK(pgd_lock);
LIST_HEAD(pgd_list); LIST_HEAD(pgd_list);
#endif
void vmalloc_sync_all(void) void vmalloc_sync_all(void)
{ {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment