Skip to content

L
linux
Commit a49976d1 authored by Ingo Molnar's avatar Ingo Molnar
Browse files
Options

x86/asm/entry/32: Clean up entry_32.S 

Make the 32-bit syscall entry code a bit more readable:

 - use consistent assembly coding style similar to entry_64.S

 - remove old comments that are not true anymore

 - eliminate whitespace noise

 - use consistent vertical spacing

 - fix various comments

No code changed:

  # arch/x86/entry/entry_32.o:

   text    data     bss     dec     hex filename
   6025       0       0    6025    1789 entry_32.o.before
   6025       0       0    6025    1789 entry_32.o.after

md5:
   f3fa16b2b0dca804f052deb6b30ba6cb  entry_32.o.before.asm
   f3fa16b2b0dca804f052deb6b30ba6cb  entry_32.o.after.asm

Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent b2502b41
Hide whitespace changes
Inline Side-by-side
Showing with 570 additions and 571 deletions
arch/x86/entry/entry_32.S
View file @ a49976d1
/* /*
* Copyright (C) 1991,1992 Linus Torvalds
* *
* Copyright (C) 1991, 1992 Linus Torvalds * entry_32.S contains the system-call and low-level fault and trap handling routines.
*/
/*
* entry.S contains the system-call and fault low-level handling routines.
* This also contains the timer-interrupt handler, as well as all interrupts
* and faults that can result in a task-switch.
*
* NOTE: This code handles signal-recognition, which happens every time
* after a timer-interrupt and after each system call.
*
* I changed all the .align's to 4 (16 byte alignment), as that's faster
* on a 486.
* *
* Stack layout in 'syscall_exit': * Stack layout in 'syscall_exit':
* ptrace needs to have all regs on the stack. * ptrace needs to have all registers on the stack.
* if the order here is changed, it needs to be * If the order here is changed, it needs to be
* updated in fork.c:copy_process, signal.c:do_signal, * updated in fork.c:copy_process(), signal.c:do_signal(),
* ptrace.c and ptrace.h * ptrace.c and ptrace.h
* *
* 0(%esp) - %ebx * 0(%esp) - %ebx
...@@ -37,8 +26,6 @@ ...@@ -37,8 +26,6 @@
* 38(%esp) - %eflags * 38(%esp) - %eflags
* 3C(%esp) - %oldesp * 3C(%esp) - %oldesp
* 40(%esp) - %oldss * 40(%esp) - %oldss
*
* "current" is in register %ebx during any slow entries.
*/ */
#include <linux/linkage.h> #include <linux/linkage.h>
...@@ -61,11 +48,11 @@ ...@@ -61,11 +48,11 @@
/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */ /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
#include <linux/elf-em.h> #include <linux/elf-em.h>
#define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE) #define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE)
#define __AUDIT_ARCH_LE 0x40000000 #define __AUDIT_ARCH_LE 0x40000000
#ifndef CONFIG_AUDITSYSCALL #ifndef CONFIG_AUDITSYSCALL
#define sysenter_audit syscall_trace_entry # define sysenter_audit syscall_trace_entry
#define sysexit_audit syscall_exit_work # define sysexit_audit syscall_exit_work
#endif #endif
.section .entry.text, "ax" .section .entry.text, "ax"
...@@ -84,16 +71,16 @@ ...@@ -84,16 +71,16 @@
*/ */
#ifdef CONFIG_PREEMPT #ifdef CONFIG_PREEMPT
#define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF # define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
#else #else
#define preempt_stop(clobbers) # define preempt_stop(clobbers)
#define resume_kernel restore_all # define resume_kernel restore_all
#endif #endif
.macro TRACE_IRQS_IRET .macro TRACE_IRQS_IRET
#ifdef CONFIG_TRACE_IRQFLAGS #ifdef CONFIG_TRACE_IRQFLAGS
testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off? testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off?
jz 1f jz 1f
TRACE_IRQS_ON TRACE_IRQS_ON
1: 1:
#endif #endif
...@@ -112,10 +99,10 @@ ...@@ -112,10 +99,10 @@
/* unfortunately push/pop can't be no-op */ /* unfortunately push/pop can't be no-op */
.macro PUSH_GS .macro PUSH_GS
pushl $0 pushl $0
.endm .endm
.macro POP_GS pop=0 .macro POP_GS pop=0
addl $(4 + \pop), %esp addl $(4 + \pop), %esp
.endm .endm
.macro POP_GS_EX .macro POP_GS_EX
.endm .endm
...@@ -135,119 +122,119 @@ ...@@ -135,119 +122,119 @@
#else /* CONFIG_X86_32_LAZY_GS */ #else /* CONFIG_X86_32_LAZY_GS */
.macro PUSH_GS .macro PUSH_GS
pushl %gs pushl %gs
.endm .endm
.macro POP_GS pop=0 .macro POP_GS pop=0
98: popl %gs 98: popl %gs
.if \pop <> 0 .if \pop <> 0
add $\pop, %esp add $\pop, %esp
.endif .endif
.endm .endm
.macro POP_GS_EX .macro POP_GS_EX
.pushsection .fixup, "ax" .pushsection .fixup, "ax"
99: movl $0, (%esp) 99: movl $0, (%esp)
jmp 98b jmp 98b
.popsection .popsection
_ASM_EXTABLE(98b,99b) _ASM_EXTABLE(98b, 99b)
.endm .endm
.macro PTGS_TO_GS .macro PTGS_TO_GS
98: mov PT_GS(%esp), %gs 98: mov PT_GS(%esp), %gs
.endm .endm
.macro PTGS_TO_GS_EX .macro PTGS_TO_GS_EX
.pushsection .fixup, "ax" .pushsection .fixup, "ax"
99: movl $0, PT_GS(%esp) 99: movl $0, PT_GS(%esp)
jmp 98b jmp 98b
.popsection .popsection
_ASM_EXTABLE(98b,99b) _ASM_EXTABLE(98b, 99b)
.endm .endm
.macro GS_TO_REG reg .macro GS_TO_REG reg
movl %gs, \reg movl %gs, \reg
.endm .endm
.macro REG_TO_PTGS reg .macro REG_TO_PTGS reg
movl \reg, PT_GS(%esp) movl \reg, PT_GS(%esp)
.endm .endm
.macro SET_KERNEL_GS reg .macro SET_KERNEL_GS reg
movl $(__KERNEL_STACK_CANARY), \reg movl $(__KERNEL_STACK_CANARY), \reg
movl \reg, %gs movl \reg, %gs
.endm .endm
#endif /* CONFIG_X86_32_LAZY_GS */ #endif /* CONFIG_X86_32_LAZY_GS */
.macro SAVE_ALL .macro SAVE_ALL
cld cld
PUSH_GS PUSH_GS
pushl %fs pushl %fs
pushl %es pushl %es
pushl %ds pushl %ds
pushl %eax pushl %eax
pushl %ebp pushl %ebp
pushl %edi pushl %edi
pushl %esi pushl %esi
pushl %edx pushl %edx
pushl %ecx pushl %ecx
pushl %ebx pushl %ebx
movl $(__USER_DS), %edx movl $(__USER_DS), %edx
movl %edx, %ds movl %edx, %ds
movl %edx, %es movl %edx, %es
movl $(__KERNEL_PERCPU), %edx movl $(__KERNEL_PERCPU), %edx
movl %edx, %fs movl %edx, %fs
SET_KERNEL_GS %edx SET_KERNEL_GS %edx
.endm .endm
.macro RESTORE_INT_REGS .macro RESTORE_INT_REGS
popl %ebx popl %ebx
popl %ecx popl %ecx
popl %edx popl %edx
popl %esi popl %esi
popl %edi popl %edi
popl %ebp popl %ebp
popl %eax popl %eax
.endm .endm
.macro RESTORE_REGS pop=0 .macro RESTORE_REGS pop=0
RESTORE_INT_REGS RESTORE_INT_REGS
1: popl %ds 1: popl %ds
2: popl %es 2: popl %es
3: popl %fs 3: popl %fs
POP_GS \pop POP_GS \pop
.pushsection .fixup, "ax" .pushsection .fixup, "ax"
4: movl $0, (%esp) 4: movl $0, (%esp)
jmp 1b jmp 1b
5: movl $0, (%esp) 5: movl $0, (%esp)
jmp 2b jmp 2b
6: movl $0, (%esp) 6: movl $0, (%esp)
jmp 3b jmp 3b
.popsection .popsection
_ASM_EXTABLE(1b,4b) _ASM_EXTABLE(1b, 4b)
_ASM_EXTABLE(2b,5b) _ASM_EXTABLE(2b, 5b)
_ASM_EXTABLE(3b,6b) _ASM_EXTABLE(3b, 6b)
POP_GS_EX POP_GS_EX
.endm .endm
ENTRY(ret_from_fork) ENTRY(ret_from_fork)
pushl %eax pushl %eax
call schedule_tail call schedule_tail
GET_THREAD_INFO(%ebp) GET_THREAD_INFO(%ebp)
popl %eax popl %eax
pushl $0x0202 # Reset kernel eflags pushl $0x0202 # Reset kernel eflags
popfl popfl
jmp syscall_exit jmp syscall_exit
END(ret_from_fork) END(ret_from_fork)
ENTRY(ret_from_kernel_thread) ENTRY(ret_from_kernel_thread)
pushl %eax pushl %eax
call schedule_tail call schedule_tail
GET_THREAD_INFO(%ebp) GET_THREAD_INFO(%ebp)
popl %eax popl %eax
pushl $0x0202 # Reset kernel eflags pushl $0x0202 # Reset kernel eflags
popfl popfl
movl PT_EBP(%esp),%eax movl PT_EBP(%esp), %eax
call *PT_EBX(%esp) call *PT_EBX(%esp)
movl $0,PT_EAX(%esp) movl $0, PT_EAX(%esp)
jmp syscall_exit jmp syscall_exit
ENDPROC(ret_from_kernel_thread) ENDPROC(ret_from_kernel_thread)
/* /*
...@@ -264,62 +251,65 @@ ret_from_exception: ...@@ -264,62 +251,65 @@ ret_from_exception:
ret_from_intr: ret_from_intr:
GET_THREAD_INFO(%ebp) GET_THREAD_INFO(%ebp)
#ifdef CONFIG_VM86 #ifdef CONFIG_VM86
movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS
movb PT_CS(%esp), %al movb PT_CS(%esp), %al
andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
#else #else
/* /*
* We can be coming here from child spawned by kernel_thread(). * We can be coming here from child spawned by kernel_thread().
*/ */
movl PT_CS(%esp), %eax movl PT_CS(%esp), %eax
andl $SEGMENT_RPL_MASK, %eax andl $SEGMENT_RPL_MASK, %eax
#endif #endif
cmpl $USER_RPL, %eax cmpl $USER_RPL, %eax
jb resume_kernel # not returning to v8086 or userspace jb resume_kernel # not returning to v8086 or userspace
ENTRY(resume_userspace) ENTRY(resume_userspace)
LOCKDEP_SYS_EXIT LOCKDEP_SYS_EXIT
DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
# setting need_resched or sigpending # setting need_resched or sigpending
# between sampling and the iret # between sampling and the iret
TRACE_IRQS_OFF TRACE_IRQS_OFF
movl TI_flags(%ebp), %ecx movl TI_flags(%ebp), %ecx
andl $_TIF_WORK_MASK, %ecx # is there any work to be done on andl $_TIF_WORK_MASK, %ecx # is there any work to be done on
# int/exception return? # int/exception return?
jne work_pending jne work_pending
jmp restore_all jmp restore_all
END(ret_from_exception) END(ret_from_exception)
#ifdef CONFIG_PREEMPT #ifdef CONFIG_PREEMPT
ENTRY(resume_kernel) ENTRY(resume_kernel)
DISABLE_INTERRUPTS(CLBR_ANY) DISABLE_INTERRUPTS(CLBR_ANY)
need_resched: need_resched:
cmpl $0,PER_CPU_VAR(__preempt_count) cmpl $0, PER_CPU_VAR(__preempt_count)
jnz restore_all jnz restore_all
testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ? testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ?
jz restore_all jz restore_all
call preempt_schedule_irq call preempt_schedule_irq
jmp need_resched jmp need_resched
END(resume_kernel) END(resume_kernel)
#endif #endif
/* SYSENTER_RETURN points to after the "sysenter" instruction in /*
the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */ * SYSENTER_RETURN points to after the SYSENTER instruction
* in the vsyscall page. See vsyscall-sysentry.S, which defines
* the symbol.
*/
# sysenter call handler stub # SYSENTER call handler stub
ENTRY(entry_SYSENTER_32) ENTRY(entry_SYSENTER_32)
movl TSS_sysenter_sp0(%esp),%esp movl TSS_sysenter_sp0(%esp), %esp
sysenter_past_esp: sysenter_past_esp:
/* /*
* Interrupts are disabled here, but we can't trace it until * Interrupts are disabled here, but we can't trace it until
* enough kernel state to call TRACE_IRQS_OFF can be called - but * enough kernel state to call TRACE_IRQS_OFF can be called - but
* we immediately enable interrupts at that point anyway. * we immediately enable interrupts at that point anyway.
*/ */
pushl $__USER_DS pushl $__USER_DS
pushl %ebp pushl %ebp
pushfl pushfl
orl $X86_EFLAGS_IF, (%esp) orl $X86_EFLAGS_IF, (%esp)
pushl $__USER_CS pushl $__USER_CS
/* /*
* Push current_thread_info()->sysenter_return to the stack. * Push current_thread_info()->sysenter_return to the stack.
* A tiny bit of offset fixup is necessary: TI_sysenter_return * A tiny bit of offset fixup is necessary: TI_sysenter_return
...@@ -328,9 +318,9 @@ sysenter_past_esp: ...@@ -328,9 +318,9 @@ sysenter_past_esp:
* TOP_OF_KERNEL_STACK_PADDING takes us to the top of the stack; * TOP_OF_KERNEL_STACK_PADDING takes us to the top of the stack;
* and THREAD_SIZE takes us to the bottom. * and THREAD_SIZE takes us to the bottom.
*/ */
pushl ((TI_sysenter_return) - THREAD_SIZE + TOP_OF_KERNEL_STACK_PADDING + 4*4)(%esp) pushl ((TI_sysenter_return) - THREAD_SIZE + TOP_OF_KERNEL_STACK_PADDING + 4*4)(%esp)
pushl %eax pushl %eax
SAVE_ALL SAVE_ALL
ENABLE_INTERRUPTS(CLBR_NONE) ENABLE_INTERRUPTS(CLBR_NONE)
...@@ -338,132 +328,134 @@ sysenter_past_esp: ...@@ -338,132 +328,134 @@ sysenter_past_esp:
* Load the potential sixth argument from user stack. * Load the potential sixth argument from user stack.
* Careful about security. * Careful about security.
*/ */
cmpl $__PAGE_OFFSET-3,%ebp cmpl $__PAGE_OFFSET-3, %ebp
jae syscall_fault jae syscall_fault
ASM_STAC ASM_STAC
1: movl (%ebp),%ebp 1: movl (%ebp), %ebp
ASM_CLAC ASM_CLAC
movl %ebp,PT_EBP(%esp) movl %ebp, PT_EBP(%esp)
_ASM_EXTABLE(1b,syscall_fault) _ASM_EXTABLE(1b, syscall_fault)
GET_THREAD_INFO(%ebp) GET_THREAD_INFO(%ebp)
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp) testl $_TIF_WORK_SYSCALL_ENTRY, TI_flags(%ebp)
jnz sysenter_audit jnz sysenter_audit
sysenter_do_call: sysenter_do_call:
cmpl $(NR_syscalls), %eax cmpl $(NR_syscalls), %eax
jae sysenter_badsys jae sysenter_badsys
call *sys_call_table(,%eax,4) call *sys_call_table(, %eax, 4)
sysenter_after_call: sysenter_after_call:
movl %eax,PT_EAX(%esp) movl %eax, PT_EAX(%esp)
LOCKDEP_SYS_EXIT LOCKDEP_SYS_EXIT
DISABLE_INTERRUPTS(CLBR_ANY) DISABLE_INTERRUPTS(CLBR_ANY)
TRACE_IRQS_OFF TRACE_IRQS_OFF
movl TI_flags(%ebp), %ecx movl TI_flags(%ebp), %ecx
testl $_TIF_ALLWORK_MASK, %ecx testl $_TIF_ALLWORK_MASK, %ecx
jnz sysexit_audit jnz sysexit_audit
sysenter_exit: sysenter_exit:
/* if something modifies registers it must also disable sysexit */ /* if something modifies registers it must also disable sysexit */
movl PT_EIP(%esp), %edx movl PT_EIP(%esp), %edx
movl PT_OLDESP(%esp), %ecx movl PT_OLDESP(%esp), %ecx
xorl %ebp,%ebp xorl %ebp, %ebp
TRACE_IRQS_ON TRACE_IRQS_ON
1: mov PT_FS(%esp), %fs 1: mov PT_FS(%esp), %fs
PTGS_TO_GS PTGS_TO_GS
ENABLE_INTERRUPTS_SYSEXIT ENABLE_INTERRUPTS_SYSEXIT
#ifdef CONFIG_AUDITSYSCALL #ifdef CONFIG_AUDITSYSCALL
sysenter_audit: sysenter_audit:
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%ebp) testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), TI_flags(%ebp)
jnz syscall_trace_entry jnz syscall_trace_entry
/* movl PT_EAX(%esp), %eax already set, syscall number: 1st arg to audit */ /* movl PT_EAX(%esp), %eax already set, syscall number: 1st arg to audit */
movl PT_EBX(%esp), %edx /* ebx/a0: 2nd arg to audit */ movl PT_EBX(%esp), %edx /* ebx/a0: 2nd arg to audit */
/* movl PT_ECX(%esp), %ecx already set, a1: 3nd arg to audit */ /* movl PT_ECX(%esp), %ecx already set, a1: 3nd arg to audit */
pushl PT_ESI(%esp) /* a3: 5th arg */ pushl PT_ESI(%esp) /* a3: 5th arg */
pushl PT_EDX+4(%esp) /* a2: 4th arg */ pushl PT_EDX+4(%esp) /* a2: 4th arg */
call __audit_syscall_entry call __audit_syscall_entry
popl %ecx /* get that remapped edx off the stack */ popl %ecx /* get that remapped edx off the stack */
popl %ecx /* get that remapped esi off the stack */ popl %ecx /* get that remapped esi off the stack */
movl PT_EAX(%esp),%eax /* reload syscall number */ movl PT_EAX(%esp), %eax /* reload syscall number */
jmp sysenter_do_call jmp sysenter_do_call
sysexit_audit: sysexit_audit:
testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx
jnz syscall_exit_work jnz syscall_exit_work
TRACE_IRQS_ON TRACE_IRQS_ON
ENABLE_INTERRUPTS(CLBR_ANY) ENABLE_INTERRUPTS(CLBR_ANY)
movl %eax,%edx /* second arg, syscall return value */ movl %eax, %edx /* second arg, syscall return value */
cmpl $-MAX_ERRNO,%eax /* is it an error ? */ cmpl $-MAX_ERRNO, %eax /* is it an error ? */
setbe %al /* 1 if so, 0 if not */ setbe %al /* 1 if so, 0 if not */
movzbl %al,%eax /* zero-extend that */ movzbl %al, %eax /* zero-extend that */
call __audit_syscall_exit call __audit_syscall_exit
DISABLE_INTERRUPTS(CLBR_ANY) DISABLE_INTERRUPTS(CLBR_ANY)
TRACE_IRQS_OFF TRACE_IRQS_OFF
movl TI_flags(%ebp), %ecx movl TI_flags(%ebp), %ecx
testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx
jnz syscall_exit_work jnz syscall_exit_work
movl PT_EAX(%esp),%eax /* reload syscall return value */ movl PT_EAX(%esp), %eax /* reload syscall return value */
jmp sysenter_exit jmp sysenter_exit
#endif #endif
.pushsection .fixup,"ax" .pushsection .fixup, "ax"
2: movl $0,PT_FS(%esp) 2: movl $0, PT_FS(%esp)
jmp 1b jmp 1b
.popsection .popsection
_ASM_EXTABLE(1b,2b) _ASM_EXTABLE(1b, 2b)
PTGS_TO_GS_EX PTGS_TO_GS_EX
ENDPROC(entry_SYSENTER_32) ENDPROC(entry_SYSENTER_32)
# system call handler stub # system call handler stub
ENTRY(entry_INT80_32) ENTRY(entry_INT80_32)
ASM_CLAC ASM_CLAC
pushl %eax # save orig_eax pushl %eax # save orig_eax
SAVE_ALL SAVE_ALL
GET_THREAD_INFO(%ebp) GET_THREAD_INFO(%ebp)
# system call tracing in operation / emulation # system call tracing in operation / emulation
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp) testl $_TIF_WORK_SYSCALL_ENTRY, TI_flags(%ebp)
jnz syscall_trace_entry jnz syscall_trace_entry
cmpl $(NR_syscalls), %eax cmpl $(NR_syscalls), %eax
jae syscall_badsys jae syscall_badsys
syscall_call: syscall_call:
call *sys_call_table(,%eax,4) call *sys_call_table(, %eax, 4)
syscall_after_call: syscall_after_call:
movl %eax,PT_EAX(%esp) # store the return value movl %eax, PT_EAX(%esp) # store the return value
syscall_exit: syscall_exit:
LOCKDEP_SYS_EXIT LOCKDEP_SYS_EXIT
DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
# setting need_resched or sigpending # setting need_resched or sigpending
# between sampling and the iret # between sampling and the iret
TRACE_IRQS_OFF TRACE_IRQS_OFF
movl TI_flags(%ebp), %ecx movl TI_flags(%ebp), %ecx
testl $_TIF_ALLWORK_MASK, %ecx # current->work testl $_TIF_ALLWORK_MASK, %ecx # current->work
jnz syscall_exit_work jnz syscall_exit_work
restore_all: restore_all:
TRACE_IRQS_IRET TRACE_IRQS_IRET
restore_all_notrace: restore_all_notrace:
#ifdef CONFIG_X86_ESPFIX32 #ifdef CONFIG_X86_ESPFIX32
movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
# Warning: PT_OLDSS(%esp) contains the wrong/random values if we /*
# are returning to the kernel. * Warning: PT_OLDSS(%esp) contains the wrong/random values if we
# See comments in process.c:copy_thread() for details. * are returning to the kernel.
movb PT_OLDSS(%esp), %ah * See comments in process.c:copy_thread() for details.
movb PT_CS(%esp), %al */
andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax movb PT_OLDSS(%esp), %ah
cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax movb PT_CS(%esp), %al
je ldt_ss # returning to user-space with LDT SS andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
je ldt_ss # returning to user-space with LDT SS
#endif #endif
restore_nocheck: restore_nocheck:
RESTORE_REGS 4 # skip orig_eax/error_code RESTORE_REGS 4 # skip orig_eax/error_code
irq_return: irq_return:
INTERRUPT_RETURN INTERRUPT_RETURN
.section .fixup,"ax" .section .fixup, "ax"
ENTRY(iret_exc) ENTRY(iret_exc )
pushl $0 # no error code pushl $0 # no error code
pushl $do_iret_error pushl $do_iret_error
jmp error_code jmp error_code
.previous .previous
_ASM_EXTABLE(irq_return,iret_exc) _ASM_EXTABLE(irq_return, iret_exc)
#ifdef CONFIG_X86_ESPFIX32 #ifdef CONFIG_X86_ESPFIX32
ldt_ss: ldt_ss:
...@@ -476,8 +468,8 @@ ldt_ss: ...@@ -476,8 +468,8 @@ ldt_ss:
* is still available to implement the setting of the high * is still available to implement the setting of the high
* 16-bits in the INTERRUPT_RETURN paravirt-op. * 16-bits in the INTERRUPT_RETURN paravirt-op.
*/ */
cmpl $0, pv_info+PARAVIRT_enabled cmpl $0, pv_info+PARAVIRT_enabled
jne restore_nocheck jne restore_nocheck
#endif #endif
/* /*
...@@ -492,21 +484,23 @@ ldt_ss: ...@@ -492,21 +484,23 @@ ldt_ss:
* a base address that matches for the difference. * a base address that matches for the difference.
*/ */
#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8) #define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8)
mov %esp, %edx /* load kernel esp */ mov %esp, %edx /* load kernel esp */
mov PT_OLDESP(%esp), %eax /* load userspace esp */ mov PT_OLDESP(%esp), %eax /* load userspace esp */
mov %dx, %ax /* eax: new kernel esp */ mov %dx, %ax /* eax: new kernel esp */
sub %eax, %edx /* offset (low word is 0) */ sub %eax, %edx /* offset (low word is 0) */
shr $16, %edx shr $16, %edx
mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */ mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */
mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */ mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */
pushl $__ESPFIX_SS pushl $__ESPFIX_SS
pushl %eax /* new kernel esp */ pushl %eax /* new kernel esp */
/* Disable interrupts, but do not irqtrace this section: we /*
* Disable interrupts, but do not irqtrace this section: we
* will soon execute iret and the tracer was already set to * will soon execute iret and the tracer was already set to
* the irqstate after the iret */ * the irqstate after the IRET:
*/
DISABLE_INTERRUPTS(CLBR_EAX) DISABLE_INTERRUPTS(CLBR_EAX)
lss (%esp), %esp /* switch to espfix segment */ lss (%esp), %esp /* switch to espfix segment */
jmp restore_nocheck jmp restore_nocheck
#endif #endif
ENDPROC(entry_INT80_32) ENDPROC(entry_INT80_32)
...@@ -514,93 +508,93 @@ ENDPROC(entry_INT80_32) ...@@ -514,93 +508,93 @@ ENDPROC(entry_INT80_32)
ALIGN ALIGN
work_pending: work_pending:
testb $_TIF_NEED_RESCHED, %cl testb $_TIF_NEED_RESCHED, %cl
jz work_notifysig jz work_notifysig
work_resched: work_resched:
call schedule call schedule
LOCKDEP_SYS_EXIT LOCKDEP_SYS_EXIT
DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
# setting need_resched or sigpending # setting need_resched or sigpending
# between sampling and the iret # between sampling and the iret
TRACE_IRQS_OFF TRACE_IRQS_OFF
movl TI_flags(%ebp), %ecx movl TI_flags(%ebp), %ecx
andl $_TIF_WORK_MASK, %ecx # is there any work to be done other andl $_TIF_WORK_MASK, %ecx # is there any work to be done other
# than syscall tracing? # than syscall tracing?
jz restore_all jz restore_all
testb $_TIF_NEED_RESCHED, %cl testb $_TIF_NEED_RESCHED, %cl
jnz work_resched jnz work_resched
work_notifysig: # deal with pending signals and work_notifysig: # deal with pending signals and
# notify-resume requests # notify-resume requests
#ifdef CONFIG_VM86 #ifdef CONFIG_VM86
testl $X86_EFLAGS_VM, PT_EFLAGS(%esp) testl $X86_EFLAGS_VM, PT_EFLAGS(%esp)
movl %esp, %eax movl %esp, %eax
jnz work_notifysig_v86 # returning to kernel-space or jnz work_notifysig_v86 # returning to kernel-space or
# vm86-space # vm86-space
1: 1:
#else #else
movl %esp, %eax movl %esp, %eax
#endif #endif
TRACE_IRQS_ON TRACE_IRQS_ON
ENABLE_INTERRUPTS(CLBR_NONE) ENABLE_INTERRUPTS(CLBR_NONE)
movb PT_CS(%esp), %bl movb PT_CS(%esp), %bl
andb $SEGMENT_RPL_MASK, %bl andb $SEGMENT_RPL_MASK, %bl
cmpb $USER_RPL, %bl cmpb $USER_RPL, %bl
jb resume_kernel jb resume_kernel
xorl %edx, %edx xorl %edx, %edx
call do_notify_resume call do_notify_resume
jmp resume_userspace jmp resume_userspace
#ifdef CONFIG_VM86 #ifdef CONFIG_VM86
ALIGN ALIGN
work_notifysig_v86: work_notifysig_v86:
pushl %ecx # save ti_flags for do_notify_resume pushl %ecx # save ti_flags for do_notify_resume
call save_v86_state # %eax contains pt_regs pointer call save_v86_state # %eax contains pt_regs pointer
popl %ecx popl %ecx
movl %eax, %esp movl %eax, %esp
jmp 1b jmp 1b
#endif #endif
END(work_pending) END(work_pending)
# perform syscall exit tracing # perform syscall exit tracing
ALIGN ALIGN
syscall_trace_entry: syscall_trace_entry:
movl $-ENOSYS,PT_EAX(%esp) movl $-ENOSYS, PT_EAX(%esp)
movl %esp, %eax movl %esp, %eax
call syscall_trace_enter call syscall_trace_enter
/* What it returned is what we'll actually use. */ /* What it returned is what we'll actually use. */
cmpl $(NR_syscalls), %eax cmpl $(NR_syscalls), %eax
jnae syscall_call jnae syscall_call
jmp syscall_exit jmp syscall_exit
END(syscall_trace_entry) END(syscall_trace_entry)
# perform syscall exit tracing # perform syscall exit tracing
ALIGN ALIGN
syscall_exit_work: syscall_exit_work:
testl $_TIF_WORK_SYSCALL_EXIT, %ecx testl $_TIF_WORK_SYSCALL_EXIT, %ecx
jz work_pending jz work_pending
TRACE_IRQS_ON TRACE_IRQS_ON
ENABLE_INTERRUPTS(CLBR_ANY) # could let syscall_trace_leave() call ENABLE_INTERRUPTS(CLBR_ANY) # could let syscall_trace_leave() call
# schedule() instead # schedule() instead
movl %esp, %eax movl %esp, %eax
call syscall_trace_leave call syscall_trace_leave
jmp resume_userspace jmp resume_userspace
END(syscall_exit_work) END(syscall_exit_work)
syscall_fault: syscall_fault:
ASM_CLAC ASM_CLAC
GET_THREAD_INFO(%ebp) GET_THREAD_INFO(%ebp)
movl $-EFAULT,PT_EAX(%esp) movl $-EFAULT, PT_EAX(%esp)
jmp resume_userspace jmp resume_userspace
END(syscall_fault) END(syscall_fault)
syscall_badsys: syscall_badsys:
movl $-ENOSYS,%eax movl $-ENOSYS, %eax
jmp syscall_after_call jmp syscall_after_call
END(syscall_badsys) END(syscall_badsys)
sysenter_badsys: sysenter_badsys:
movl $-ENOSYS,%eax movl $-ENOSYS, %eax
jmp sysenter_after_call jmp sysenter_after_call
END(sysenter_badsys) END(sysenter_badsys)
.macro FIXUP_ESPFIX_STACK .macro FIXUP_ESPFIX_STACK
...@@ -613,24 +607,24 @@ END(sysenter_badsys) ...@@ -613,24 +607,24 @@ END(sysenter_badsys)
*/ */
#ifdef CONFIG_X86_ESPFIX32 #ifdef CONFIG_X86_ESPFIX32
/* fixup the stack */ /* fixup the stack */
mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */ mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */
mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */ mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */
shl $16, %eax shl $16, %eax
addl %esp, %eax /* the adjusted stack pointer */ addl %esp, %eax /* the adjusted stack pointer */
pushl $__KERNEL_DS pushl $__KERNEL_DS
pushl %eax pushl %eax
lss (%esp), %esp /* switch to the normal stack segment */ lss (%esp), %esp /* switch to the normal stack segment */
#endif #endif
.endm .endm
.macro UNWIND_ESPFIX_STACK .macro UNWIND_ESPFIX_STACK
#ifdef CONFIG_X86_ESPFIX32 #ifdef CONFIG_X86_ESPFIX32
movl %ss, %eax movl %ss, %eax
/* see if on espfix stack */ /* see if on espfix stack */
cmpw $__ESPFIX_SS, %ax cmpw $__ESPFIX_SS, %ax
jne 27f jne 27f
movl $__KERNEL_DS, %eax movl $__KERNEL_DS, %eax
movl %eax, %ds movl %eax, %ds
movl %eax, %es movl %eax, %es
/* switch to normal stack */ /* switch to normal stack */
FIXUP_ESPFIX_STACK FIXUP_ESPFIX_STACK
27: 27:
...@@ -645,7 +639,7 @@ END(sysenter_badsys) ...@@ -645,7 +639,7 @@ END(sysenter_badsys)
ENTRY(irq_entries_start) ENTRY(irq_entries_start)
vector=FIRST_EXTERNAL_VECTOR vector=FIRST_EXTERNAL_VECTOR
.rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR) .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
pushl $(~vector+0x80) /* Note: always in signed byte range */ pushl $(~vector+0x80) /* Note: always in signed byte range */
vector=vector+1 vector=vector+1
jmp common_interrupt jmp common_interrupt
.align 8 .align 8
...@@ -659,35 +653,34 @@ END(irq_entries_start) ...@@ -659,35 +653,34 @@ END(irq_entries_start)
.p2align CONFIG_X86_L1_CACHE_SHIFT .p2align CONFIG_X86_L1_CACHE_SHIFT
common_interrupt: common_interrupt:
ASM_CLAC ASM_CLAC
addl $-0x80,(%esp) /* Adjust vector into the [-256,-1] range */ addl $-0x80, (%esp) /* Adjust vector into the [-256, -1] range */
SAVE_ALL SAVE_ALL
TRACE_IRQS_OFF TRACE_IRQS_OFF
movl %esp,%eax movl %esp, %eax
call do_IRQ call do_IRQ
jmp ret_from_intr jmp ret_from_intr
ENDPROC(common_interrupt) ENDPROC(common_interrupt)
#define BUILD_INTERRUPT3(name, nr, fn) \ #define BUILD_INTERRUPT3(name, nr, fn) \
ENTRY(name) \ ENTRY(name) \
ASM_CLAC; \ ASM_CLAC; \
pushl $~(nr); \ pushl $~(nr); \
SAVE_ALL; \ SAVE_ALL; \
TRACE_IRQS_OFF \ TRACE_IRQS_OFF \
movl %esp,%eax; \ movl %esp, %eax; \
call fn; \ call fn; \
jmp ret_from_intr; \ jmp ret_from_intr; \
ENDPROC(name) ENDPROC(name)
#ifdef CONFIG_TRACING #ifdef CONFIG_TRACING
#define TRACE_BUILD_INTERRUPT(name, nr) \ # define TRACE_BUILD_INTERRUPT(name, nr) BUILD_INTERRUPT3(trace_##name, nr, smp_trace_##name)
BUILD_INTERRUPT3(trace_##name, nr, smp_trace_##name)
#else #else
#define TRACE_BUILD_INTERRUPT(name, nr) # define TRACE_BUILD_INTERRUPT(name, nr)
#endif #endif
#define BUILD_INTERRUPT(name, nr) \ #define BUILD_INTERRUPT(name, nr) \
BUILD_INTERRUPT3(name, nr, smp_##name); \ BUILD_INTERRUPT3(name, nr, smp_##name); \
TRACE_BUILD_INTERRUPT(name, nr) TRACE_BUILD_INTERRUPT(name, nr)
/* The include is where all of the SMP etc. interrupts come from */ /* The include is where all of the SMP etc. interrupts come from */
...@@ -695,30 +688,30 @@ ENDPROC(name) ...@@ -695,30 +688,30 @@ ENDPROC(name)
ENTRY(coprocessor_error) ENTRY(coprocessor_error)
ASM_CLAC ASM_CLAC
pushl $0 pushl $0
pushl $do_coprocessor_error pushl $do_coprocessor_error
jmp error_code jmp error_code
END(coprocessor_error) END(coprocessor_error)
ENTRY(simd_coprocessor_error) ENTRY(simd_coprocessor_error)
ASM_CLAC ASM_CLAC
pushl $0 pushl $0
#ifdef CONFIG_X86_INVD_BUG #ifdef CONFIG_X86_INVD_BUG
/* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */ /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */
ALTERNATIVE "pushl $do_general_protection", \ ALTERNATIVE "pushl $do_general_protection", \
"pushl $do_simd_coprocessor_error", \ "pushl $do_simd_coprocessor_error", \
X86_FEATURE_XMM X86_FEATURE_XMM
#else #else
pushl $do_simd_coprocessor_error pushl $do_simd_coprocessor_error
#endif #endif
jmp error_code jmp error_code
END(simd_coprocessor_error) END(simd_coprocessor_error)
ENTRY(device_not_available) ENTRY(device_not_available)
ASM_CLAC ASM_CLAC
pushl $-1 # mark this as an int pushl $-1 # mark this as an int
pushl $do_device_not_available pushl $do_device_not_available
jmp error_code jmp error_code
END(device_not_available) END(device_not_available)
#ifdef CONFIG_PARAVIRT #ifdef CONFIG_PARAVIRT
...@@ -735,165 +728,171 @@ END(native_irq_enable_sysexit) ...@@ -735,165 +728,171 @@ END(native_irq_enable_sysexit)
ENTRY(overflow) ENTRY(overflow)
ASM_CLAC ASM_CLAC
pushl $0 pushl $0
pushl $do_overflow pushl $do_overflow
jmp error_code jmp error_code
END(overflow) END(overflow)
ENTRY(bounds) ENTRY(bounds)
ASM_CLAC ASM_CLAC
pushl $0 pushl $0
pushl $do_bounds pushl $do_bounds
jmp error_code jmp error_code
END(bounds) END(bounds)
ENTRY(invalid_op) ENTRY(invalid_op)
ASM_CLAC ASM_CLAC
pushl $0 pushl $0
pushl $do_invalid_op pushl $do_invalid_op
jmp error_code jmp error_code
END(invalid_op) END(invalid_op)
ENTRY(coprocessor_segment_overrun) ENTRY(coprocessor_segment_overrun)
ASM_CLAC ASM_CLAC
pushl $0 pushl $0
pushl $do_coprocessor_segment_overrun pushl $do_coprocessor_segment_overrun
jmp error_code jmp error_code
END(coprocessor_segment_overrun) END(coprocessor_segment_overrun)
ENTRY(invalid_TSS) ENTRY(invalid_TSS)
ASM_CLAC ASM_CLAC
pushl $do_invalid_TSS pushl $do_invalid_TSS
jmp error_code jmp error_code
END(invalid_TSS) END(invalid_TSS)
ENTRY(segment_not_present) ENTRY(segment_not_present)
ASM_CLAC ASM_CLAC
pushl $do_segment_not_present pushl $do_segment_not_present
jmp error_code jmp error_code
END(segment_not_present) END(segment_not_present)
ENTRY(stack_segment) ENTRY(stack_segment)
ASM_CLAC ASM_CLAC
pushl $do_stack_segment pushl $do_stack_segment
jmp error_code jmp error_code
END(stack_segment) END(stack_segment)
ENTRY(alignment_check) ENTRY(alignment_check)
ASM_CLAC ASM_CLAC
pushl $do_alignment_check pushl $do_alignment_check
jmp error_code jmp error_code
END(alignment_check) END(alignment_check)
ENTRY(divide_error) ENTRY(divide_error)
ASM_CLAC ASM_CLAC
pushl $0 # no error code pushl $0 # no error code
pushl $do_divide_error pushl $do_divide_error
jmp error_code jmp error_code
END(divide_error) END(divide_error)
#ifdef CONFIG_X86_MCE #ifdef CONFIG_X86_MCE
ENTRY(machine_check) ENTRY(machine_check)
ASM_CLAC ASM_CLAC
pushl $0 pushl $0
pushl machine_check_vector pushl machine_check_vector
jmp error_code jmp error_code
END(machine_check) END(machine_check)
#endif #endif
ENTRY(spurious_interrupt_bug) ENTRY(spurious_interrupt_bug)
ASM_CLAC ASM_CLAC
pushl $0 pushl $0
pushl $do_spurious_interrupt_bug pushl $do_spurious_interrupt_bug
jmp error_code jmp error_code
END(spurious_interrupt_bug) END(spurious_interrupt_bug)
#ifdef CONFIG_XEN #ifdef CONFIG_XEN
/* Xen doesn't set %esp to be precisely what the normal sysenter /*
entrypoint expects, so fix it up before using the normal path. */ * Xen doesn't set %esp to be precisely what the normal SYSENTER
* entry point expects, so fix it up before using the normal path.
*/
ENTRY(xen_sysenter_target) ENTRY(xen_sysenter_target)
addl $5*4, %esp /* remove xen-provided frame */ addl $5*4, %esp /* remove xen-provided frame */
jmp sysenter_past_esp jmp sysenter_past_esp
ENTRY(xen_hypervisor_callback) ENTRY(xen_hypervisor_callback)
pushl $-1 /* orig_ax = -1 => not a system call */ pushl $-1 /* orig_ax = -1 => not a system call */
SAVE_ALL SAVE_ALL
TRACE_IRQS_OFF TRACE_IRQS_OFF
/* Check to see if we got the event in the critical /*
region in xen_iret_direct, after we've reenabled * Check to see if we got the event in the critical
events and checked for pending events. This simulates * region in xen_iret_direct, after we've reenabled
iret instruction's behaviour where it delivers a * events and checked for pending events. This simulates
pending interrupt when enabling interrupts. */ * iret instruction's behaviour where it delivers a
movl PT_EIP(%esp),%eax * pending interrupt when enabling interrupts:
cmpl $xen_iret_start_crit,%eax */
jb 1f movl PT_EIP(%esp), %eax
cmpl $xen_iret_end_crit,%eax cmpl $xen_iret_start_crit, %eax
jae 1f jb 1f
cmpl $xen_iret_end_crit, %eax
jae 1f
jmp xen_iret_crit_fixup jmp xen_iret_crit_fixup
ENTRY(xen_do_upcall) ENTRY(xen_do_upcall)
1: mov %esp, %eax 1: mov %esp, %eax
call xen_evtchn_do_upcall call xen_evtchn_do_upcall
#ifndef CONFIG_PREEMPT #ifndef CONFIG_PREEMPT
call xen_maybe_preempt_hcall call xen_maybe_preempt_hcall
#endif #endif
jmp ret_from_intr jmp ret_from_intr
ENDPROC(xen_hypervisor_callback) ENDPROC(xen_hypervisor_callback)
# Hypervisor uses this for application faults while it executes. /*
# We get here for two reasons: * Hypervisor uses this for application faults while it executes.
# 1. Fault while reloading DS, ES, FS or GS * We get here for two reasons:
# 2. Fault while executing IRET * 1. Fault while reloading DS, ES, FS or GS
# Category 1 we fix up by reattempting the load, and zeroing the segment * 2. Fault while executing IRET
# register if the load fails. * Category 1 we fix up by reattempting the load, and zeroing the segment
# Category 2 we fix up by jumping to do_iret_error. We cannot use the * register if the load fails.
# normal Linux return path in this case because if we use the IRET hypercall * Category 2 we fix up by jumping to do_iret_error. We cannot use the
# to pop the stack frame we end up in an infinite loop of failsafe callbacks. * normal Linux return path in this case because if we use the IRET hypercall
# We distinguish between categories by maintaining a status value in EAX. * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
* We distinguish between categories by maintaining a status value in EAX.
*/
ENTRY(xen_failsafe_callback) ENTRY(xen_failsafe_callback)
pushl %eax pushl %eax
movl $1,%eax movl $1, %eax
1: mov 4(%esp),%ds 1: mov 4(%esp), %ds
2: mov 8(%esp),%es 2: mov 8(%esp), %es
3: mov 12(%esp),%fs 3: mov 12(%esp), %fs
4: mov 16(%esp),%gs 4: mov 16(%esp), %gs
/* EAX == 0 => Category 1 (Bad segment) /* EAX == 0 => Category 1 (Bad segment)
EAX != 0 => Category 2 (Bad IRET) */ EAX != 0 => Category 2 (Bad IRET) */
testl %eax,%eax testl %eax, %eax
popl %eax popl %eax
lea 16(%esp),%esp lea 16(%esp), %esp
jz 5f jz 5f
jmp iret_exc jmp iret_exc
5: pushl $-1 /* orig_ax = -1 => not a system call */ 5: pushl $-1 /* orig_ax = -1 => not a system call */
SAVE_ALL SAVE_ALL
jmp ret_from_exception jmp ret_from_exception
.section .fixup,"ax" .section .fixup, "ax"
6: xorl %eax,%eax 6: xorl %eax, %eax
movl %eax,4(%esp) movl %eax, 4(%esp)
jmp 1b jmp 1b
7: xorl %eax,%eax 7: xorl %eax, %eax
movl %eax,8(%esp) movl %eax, 8(%esp)
jmp 2b jmp 2b
8: xorl %eax,%eax 8: xorl %eax, %eax
movl %eax,12(%esp) movl %eax, 12(%esp)
jmp 3b jmp 3b
9: xorl %eax,%eax 9: xorl %eax, %eax
movl %eax,16(%esp) movl %eax, 16(%esp)
jmp 4b jmp 4b
.previous .previous
_ASM_EXTABLE(1b,6b) _ASM_EXTABLE(1b, 6b)
_ASM_EXTABLE(2b,7b) _ASM_EXTABLE(2b, 7b)
_ASM_EXTABLE(3b,8b) _ASM_EXTABLE(3b, 8b)
_ASM_EXTABLE(4b,9b) _ASM_EXTABLE(4b, 9b)
ENDPROC(xen_failsafe_callback) ENDPROC(xen_failsafe_callback)
BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR, BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR,
xen_evtchn_do_upcall) xen_evtchn_do_upcall)
#endif /* CONFIG_XEN */ #endif /* CONFIG_XEN */
#if IS_ENABLED(CONFIG_HYPERV) #if IS_ENABLED(CONFIG_HYPERV)
...@@ -910,28 +909,28 @@ ENTRY(mcount) ...@@ -910,28 +909,28 @@ ENTRY(mcount)
END(mcount) END(mcount)
ENTRY(ftrace_caller) ENTRY(ftrace_caller)
pushl %eax pushl %eax
pushl %ecx pushl %ecx
pushl %edx pushl %edx
pushl $0 /* Pass NULL as regs pointer */ pushl $0 /* Pass NULL as regs pointer */
movl 4*4(%esp), %eax movl 4*4(%esp), %eax
movl 0x4(%ebp), %edx movl 0x4(%ebp), %edx
movl function_trace_op, %ecx movl function_trace_op, %ecx
subl $MCOUNT_INSN_SIZE, %eax subl $MCOUNT_INSN_SIZE, %eax
.globl ftrace_call .globl ftrace_call
ftrace_call: ftrace_call:
call ftrace_stub call ftrace_stub
addl $4,%esp /* skip NULL pointer */ addl $4, %esp /* skip NULL pointer */
popl %edx popl %edx
popl %ecx popl %ecx
popl %eax popl %eax
ftrace_ret: ftrace_ret:
#ifdef CONFIG_FUNCTION_GRAPH_TRACER #ifdef CONFIG_FUNCTION_GRAPH_TRACER
.globl ftrace_graph_call .globl ftrace_graph_call
ftrace_graph_call: ftrace_graph_call:
jmp ftrace_stub jmp ftrace_stub
#endif #endif
.globl ftrace_stub .globl ftrace_stub
...@@ -949,72 +948,72 @@ ENTRY(ftrace_regs_caller) ...@@ -949,72 +948,72 @@ ENTRY(ftrace_regs_caller)
* as the current return ip is. We move the return ip into the * as the current return ip is. We move the return ip into the
* ip location, and move flags into the return ip location. * ip location, and move flags into the return ip location.
*/ */
pushl 4(%esp) /* save return ip into ip slot */ pushl 4(%esp) /* save return ip into ip slot */
pushl $0 /* Load 0 into orig_ax */ pushl $0 /* Load 0 into orig_ax */
pushl %gs pushl %gs
pushl %fs pushl %fs
pushl %es pushl %es
pushl %ds pushl %ds
pushl %eax pushl %eax
pushl %ebp pushl %ebp
pushl %edi pushl %edi
pushl %esi pushl %esi
pushl %edx pushl %edx
pushl %ecx pushl %ecx
pushl %ebx pushl %ebx
movl 13*4(%esp), %eax /* Get the saved flags */ movl 13*4(%esp), %eax /* Get the saved flags */
movl %eax, 14*4(%esp) /* Move saved flags into regs->flags location */ movl %eax, 14*4(%esp) /* Move saved flags into regs->flags location */
/* clobbering return ip */ /* clobbering return ip */
movl $__KERNEL_CS,13*4(%esp) movl $__KERNEL_CS, 13*4(%esp)
movl 12*4(%esp), %eax /* Load ip (1st parameter) */ movl 12*4(%esp), %eax /* Load ip (1st parameter) */
subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */ subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */
movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */ movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */
movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */ movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */
pushl %esp /* Save pt_regs as 4th parameter */ pushl %esp /* Save pt_regs as 4th parameter */
GLOBAL(ftrace_regs_call) GLOBAL(ftrace_regs_call)
call ftrace_stub call ftrace_stub
addl $4, %esp /* Skip pt_regs */ addl $4, %esp /* Skip pt_regs */
movl 14*4(%esp), %eax /* Move flags back into cs */ movl 14*4(%esp), %eax /* Move flags back into cs */
movl %eax, 13*4(%esp) /* Needed to keep addl from modifying flags */ movl %eax, 13*4(%esp) /* Needed to keep addl from modifying flags */
movl 12*4(%esp), %eax /* Get return ip from regs->ip */ movl 12*4(%esp), %eax /* Get return ip from regs->ip */
movl %eax, 14*4(%esp) /* Put return ip back for ret */ movl %eax, 14*4(%esp) /* Put return ip back for ret */
popl %ebx popl %ebx
popl %ecx popl %ecx
popl %edx popl %edx
popl %esi popl %esi
popl %edi popl %edi
popl %ebp popl %ebp
popl %eax popl %eax
popl %ds popl %ds
popl %es popl %es
popl %fs popl %fs
popl %gs popl %gs
addl $8, %esp /* Skip orig_ax and ip */ addl $8, %esp /* Skip orig_ax and ip */
popf /* Pop flags at end (no addl to corrupt flags) */ popf /* Pop flags at end (no addl to corrupt flags) */
jmp ftrace_ret jmp ftrace_ret
popf popf
jmp ftrace_stub jmp ftrace_stub
#else /* ! CONFIG_DYNAMIC_FTRACE */ #else /* ! CONFIG_DYNAMIC_FTRACE */
ENTRY(mcount) ENTRY(mcount)
cmpl $__PAGE_OFFSET, %esp cmpl $__PAGE_OFFSET, %esp
jb ftrace_stub /* Paging not enabled yet? */ jb ftrace_stub /* Paging not enabled yet? */
cmpl $ftrace_stub, ftrace_trace_function cmpl $ftrace_stub, ftrace_trace_function
jnz trace jnz trace
#ifdef CONFIG_FUNCTION_GRAPH_TRACER #ifdef CONFIG_FUNCTION_GRAPH_TRACER
cmpl $ftrace_stub, ftrace_graph_return cmpl $ftrace_stub, ftrace_graph_return
jnz ftrace_graph_caller jnz ftrace_graph_caller
cmpl $ftrace_graph_entry_stub, ftrace_graph_entry cmpl $ftrace_graph_entry_stub, ftrace_graph_entry
jnz ftrace_graph_caller jnz ftrace_graph_caller
#endif #endif
.globl ftrace_stub .globl ftrace_stub
ftrace_stub: ftrace_stub:
...@@ -1022,92 +1021,92 @@ ftrace_stub: ...@@ -1022,92 +1021,92 @@ ftrace_stub:
/* taken from glibc */ /* taken from glibc */
trace: trace:
pushl %eax pushl %eax
pushl %ecx pushl %ecx
pushl %edx pushl %edx
movl 0xc(%esp), %eax movl 0xc(%esp), %eax
movl 0x4(%ebp), %edx movl 0x4(%ebp), %edx
subl $MCOUNT_INSN_SIZE, %eax subl $MCOUNT_INSN_SIZE, %eax
call *ftrace_trace_function call *ftrace_trace_function
popl %edx popl %edx
popl %ecx popl %ecx
popl %eax popl %eax
jmp ftrace_stub jmp ftrace_stub
END(mcount) END(mcount)
#endif /* CONFIG_DYNAMIC_FTRACE */ #endif /* CONFIG_DYNAMIC_FTRACE */
#endif /* CONFIG_FUNCTION_TRACER */ #endif /* CONFIG_FUNCTION_TRACER */
#ifdef CONFIG_FUNCTION_GRAPH_TRACER #ifdef CONFIG_FUNCTION_GRAPH_TRACER
ENTRY(ftrace_graph_caller) ENTRY(ftrace_graph_caller)
pushl %eax pushl %eax
pushl %ecx pushl %ecx
pushl %edx pushl %edx
movl 0xc(%esp), %eax movl 0xc(%esp), %eax
lea 0x4(%ebp), %edx lea 0x4(%ebp), %edx
movl (%ebp), %ecx movl (%ebp), %ecx
subl $MCOUNT_INSN_SIZE, %eax subl $MCOUNT_INSN_SIZE, %eax
call prepare_ftrace_return call prepare_ftrace_return
popl %edx popl %edx
popl %ecx popl %ecx
popl %eax popl %eax
ret ret
END(ftrace_graph_caller) END(ftrace_graph_caller)
.globl return_to_handler .globl return_to_handler
return_to_handler: return_to_handler:
pushl %eax pushl %eax
pushl %edx pushl %edx
movl %ebp, %eax movl %ebp, %eax
call ftrace_return_to_handler call ftrace_return_to_handler
movl %eax, %ecx movl %eax, %ecx
popl %edx popl %edx
popl %eax popl %eax
jmp *%ecx jmp *%ecx
#endif #endif
#ifdef CONFIG_TRACING #ifdef CONFIG_TRACING
ENTRY(trace_page_fault) ENTRY(trace_page_fault)
ASM_CLAC ASM_CLAC
pushl $trace_do_page_fault pushl $trace_do_page_fault
jmp error_code jmp error_code
END(trace_page_fault) END(trace_page_fault)
#endif #endif
ENTRY(page_fault) ENTRY(page_fault)
ASM_CLAC ASM_CLAC
pushl $do_page_fault pushl $do_page_fault
ALIGN ALIGN
error_code: error_code:
/* the function address is in %gs's slot on the stack */ /* the function address is in %gs's slot on the stack */
pushl %fs pushl %fs
pushl %es pushl %es
pushl %ds pushl %ds
pushl %eax pushl %eax
pushl %ebp pushl %ebp
pushl %edi pushl %edi
pushl %esi pushl %esi
pushl %edx pushl %edx
pushl %ecx pushl %ecx
pushl %ebx pushl %ebx
cld cld
movl $(__KERNEL_PERCPU), %ecx movl $(__KERNEL_PERCPU), %ecx
movl %ecx, %fs movl %ecx, %fs
UNWIND_ESPFIX_STACK UNWIND_ESPFIX_STACK
GS_TO_REG %ecx GS_TO_REG %ecx
movl PT_GS(%esp), %edi # get the function address movl PT_GS(%esp), %edi # get the function address
movl PT_ORIG_EAX(%esp), %edx # get the error code movl PT_ORIG_EAX(%esp), %edx # get the error code
movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
REG_TO_PTGS %ecx REG_TO_PTGS %ecx
SET_KERNEL_GS %ecx SET_KERNEL_GS %ecx
movl $(__USER_DS), %ecx movl $(__USER_DS), %ecx
movl %ecx, %ds movl %ecx, %ds
movl %ecx, %es movl %ecx, %es
TRACE_IRQS_OFF TRACE_IRQS_OFF
movl %esp,%eax # pt_regs pointer movl %esp, %eax # pt_regs pointer
call *%edi call *%edi
jmp ret_from_exception jmp ret_from_exception
END(page_fault) END(page_fault)
/* /*
...@@ -1124,28 +1123,28 @@ END(page_fault) ...@@ -1124,28 +1123,28 @@ END(page_fault)
* the instruction that would have done it for sysenter. * the instruction that would have done it for sysenter.
*/ */
.macro FIX_STACK offset ok label .macro FIX_STACK offset ok label
cmpw $__KERNEL_CS, 4(%esp) cmpw $__KERNEL_CS, 4(%esp)
jne \ok jne \ok
\label: \label:
movl TSS_sysenter_sp0 + \offset(%esp), %esp movl TSS_sysenter_sp0 + \offset(%esp), %esp
pushfl pushfl
pushl $__KERNEL_CS pushl $__KERNEL_CS
pushl $sysenter_past_esp pushl $sysenter_past_esp
.endm .endm
ENTRY(debug) ENTRY(debug)
ASM_CLAC ASM_CLAC
cmpl $entry_SYSENTER_32,(%esp) cmpl $entry_SYSENTER_32, (%esp)
jne debug_stack_correct jne debug_stack_correct
FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn
debug_stack_correct: debug_stack_correct:
pushl $-1 # mark this as an int pushl $-1 # mark this as an int
SAVE_ALL SAVE_ALL
TRACE_IRQS_OFF TRACE_IRQS_OFF
xorl %edx,%edx # error code 0 xorl %edx, %edx # error code 0
movl %esp,%eax # pt_regs pointer movl %esp, %eax # pt_regs pointer
call do_debug call do_debug
jmp ret_from_exception jmp ret_from_exception
END(debug) END(debug)
/* /*
...@@ -1159,91 +1158,91 @@ END(debug) ...@@ -1159,91 +1158,91 @@ END(debug)
ENTRY(nmi) ENTRY(nmi)
ASM_CLAC ASM_CLAC
#ifdef CONFIG_X86_ESPFIX32 #ifdef CONFIG_X86_ESPFIX32
pushl %eax pushl %eax
movl %ss, %eax movl %ss, %eax
cmpw $__ESPFIX_SS, %ax cmpw $__ESPFIX_SS, %ax
popl %eax popl %eax
je nmi_espfix_stack je nmi_espfix_stack
#endif #endif
cmpl $entry_SYSENTER_32,(%esp) cmpl $entry_SYSENTER_32, (%esp)
je nmi_stack_fixup je nmi_stack_fixup
pushl %eax pushl %eax
movl %esp,%eax movl %esp, %eax
/* Do not access memory above the end of our stack page, /*
* Do not access memory above the end of our stack page,
* it might not exist. * it might not exist.
*/ */
andl $(THREAD_SIZE-1),%eax andl $(THREAD_SIZE-1), %eax
cmpl $(THREAD_SIZE-20),%eax cmpl $(THREAD_SIZE-20), %eax
popl %eax popl %eax
jae nmi_stack_correct jae nmi_stack_correct
cmpl $entry_SYSENTER_32,12(%esp) cmpl $entry_SYSENTER_32, 12(%esp)
je nmi_debug_stack_check je nmi_debug_stack_check
nmi_stack_correct: nmi_stack_correct:
pushl %eax pushl %eax
SAVE_ALL SAVE_ALL
xorl %edx,%edx # zero error code xorl %edx, %edx # zero error code
movl %esp,%eax # pt_regs pointer movl %esp, %eax # pt_regs pointer
call do_nmi call do_nmi
jmp restore_all_notrace jmp restore_all_notrace
nmi_stack_fixup: nmi_stack_fixup:
FIX_STACK 12, nmi_stack_correct, 1 FIX_STACK 12, nmi_stack_correct, 1
jmp nmi_stack_correct jmp nmi_stack_correct
nmi_debug_stack_check: nmi_debug_stack_check:
cmpw $__KERNEL_CS,16(%esp) cmpw $__KERNEL_CS, 16(%esp)
jne nmi_stack_correct jne nmi_stack_correct
cmpl $debug,(%esp) cmpl $debug, (%esp)
jb nmi_stack_correct jb nmi_stack_correct
cmpl $debug_esp_fix_insn,(%esp) cmpl $debug_esp_fix_insn, (%esp)
ja nmi_stack_correct ja nmi_stack_correct
FIX_STACK 24, nmi_stack_correct, 1 FIX_STACK 24, nmi_stack_correct, 1
jmp nmi_stack_correct jmp nmi_stack_correct
#ifdef CONFIG_X86_ESPFIX32 #ifdef CONFIG_X86_ESPFIX32
nmi_espfix_stack: nmi_espfix_stack:
/* /*
* create the pointer to lss back * create the pointer to lss back
*/ */
pushl %ss pushl %ss
pushl %esp pushl %esp
addl $4, (%esp) addl $4, (%esp)
/* copy the iret frame of 12 bytes */ /* copy the iret frame of 12 bytes */
.rept 3 .rept 3
pushl 16(%esp) pushl 16(%esp)
.endr .endr
pushl %eax pushl %eax
SAVE_ALL SAVE_ALL
FIXUP_ESPFIX_STACK # %eax == %esp FIXUP_ESPFIX_STACK # %eax == %esp
xorl %edx,%edx # zero error code xorl %edx, %edx # zero error code
call do_nmi call do_nmi
RESTORE_REGS RESTORE_REGS
lss 12+4(%esp), %esp # back to espfix stack lss 12+4(%esp), %esp # back to espfix stack
jmp irq_return jmp irq_return
#endif #endif
END(nmi) END(nmi)
ENTRY(int3) ENTRY(int3)
ASM_CLAC ASM_CLAC
pushl $-1 # mark this as an int pushl $-1 # mark this as an int
SAVE_ALL SAVE_ALL
TRACE_IRQS_OFF TRACE_IRQS_OFF
xorl %edx,%edx # zero error code xorl %edx, %edx # zero error code
movl %esp,%eax # pt_regs pointer movl %esp, %eax # pt_regs pointer
call do_int3 call do_int3
jmp ret_from_exception jmp ret_from_exception
END(int3) END(int3)
ENTRY(general_protection) ENTRY(general_protection)
pushl $do_general_protection pushl $do_general_protection
jmp error_code jmp error_code
END(general_protection) END(general_protection)
#ifdef CONFIG_KVM_GUEST #ifdef CONFIG_KVM_GUEST
ENTRY(async_page_fault) ENTRY(async_page_fault)
ASM_CLAC ASM_CLAC
pushl $do_async_page_fault pushl $do_async_page_fault
jmp error_code jmp error_code
END(async_page_fault) END(async_page_fault)
#endif #endif
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7