Skip to content

L
linux
Commit 7effaa88 authored by Jan Beulich's avatar Jan Beulich Committed by Linus Torvalds
Browse files
Options

[PATCH] x86-64: Fix CFI information 

Being the foundation for reliable stack unwinding, this fixes CFI unwind
annotations in many low-level x86_64 routines, plus a config option
(available to all architectures, and also present in the previously sent
patch adding such annotations to i386 code) to enable them separatly
rather than only along with adding full debug information.
Signed-off-by: default avatarJan Beulich <jbeulich@novell.com>
Signed-off-by: default avatarAndi Kleen <ak@suse.de>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent b3ab8382
Show whitespace changes
Inline Side-by-side
Showing with 220 additions and 69 deletions
arch/x86_64/ia32/ia32entry.S
View file @ 7effaa88
...@@ -55,20 +55,34 @@ ...@@ -55,20 +55,34 @@
* with the int 0x80 path. * with the int 0x80 path.
*/ */
ENTRY(ia32_sysenter_target) ENTRY(ia32_sysenter_target)
CFI_STARTPROC CFI_STARTPROC simple
CFI_DEF_CFA rsp,0
CFI_REGISTER rsp,rbp
swapgs swapgs
movq %gs:pda_kernelstack, %rsp movq %gs:pda_kernelstack, %rsp
addq $(PDA_STACKOFFSET),%rsp addq $(PDA_STACKOFFSET),%rsp
sti sti
movl %ebp,%ebp /* zero extension */ movl %ebp,%ebp /* zero extension */
pushq $__USER32_DS pushq $__USER32_DS
CFI_ADJUST_CFA_OFFSET 8
/*CFI_REL_OFFSET ss,0*/
pushq %rbp pushq %rbp
CFI_ADJUST_CFA_OFFSET 8
CFI_REL_OFFSET rsp,0
pushfq pushfq
CFI_ADJUST_CFA_OFFSET 8
/*CFI_REL_OFFSET rflags,0*/
movl $VSYSCALL32_SYSEXIT, %r10d movl $VSYSCALL32_SYSEXIT, %r10d
CFI_REGISTER rip,r10
pushq $__USER32_CS pushq $__USER32_CS
CFI_ADJUST_CFA_OFFSET 8
/*CFI_REL_OFFSET cs,0*/
movl %eax, %eax movl %eax, %eax
pushq %r10 pushq %r10
CFI_ADJUST_CFA_OFFSET 8
CFI_REL_OFFSET rip,0
pushq %rax pushq %rax
CFI_ADJUST_CFA_OFFSET 8
cld cld
SAVE_ARGS 0,0,1 SAVE_ARGS 0,0,1
/* no need to do an access_ok check here because rbp has been /* no need to do an access_ok check here because rbp has been
...@@ -79,6 +93,7 @@ ENTRY(ia32_sysenter_target) ...@@ -79,6 +93,7 @@ ENTRY(ia32_sysenter_target)
.previous .previous
GET_THREAD_INFO(%r10) GET_THREAD_INFO(%r10)
testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10) testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10)
CFI_REMEMBER_STATE
jnz sysenter_tracesys jnz sysenter_tracesys
sysenter_do_call: sysenter_do_call:
cmpl $(IA32_NR_syscalls),%eax cmpl $(IA32_NR_syscalls),%eax
...@@ -94,14 +109,20 @@ sysenter_do_call: ...@@ -94,14 +109,20 @@ sysenter_do_call:
andl $~0x200,EFLAGS-R11(%rsp) andl $~0x200,EFLAGS-R11(%rsp)
RESTORE_ARGS 1,24,1,1,1,1 RESTORE_ARGS 1,24,1,1,1,1
popfq popfq
CFI_ADJUST_CFA_OFFSET -8
/*CFI_RESTORE rflags*/
popq %rcx /* User %esp */ popq %rcx /* User %esp */
CFI_ADJUST_CFA_OFFSET -8
CFI_REGISTER rsp,rcx
movl $VSYSCALL32_SYSEXIT,%edx /* User %eip */ movl $VSYSCALL32_SYSEXIT,%edx /* User %eip */
CFI_REGISTER rip,rdx
swapgs swapgs
sti /* sti only takes effect after the next instruction */ sti /* sti only takes effect after the next instruction */
/* sysexit */ /* sysexit */
.byte 0xf, 0x35 .byte 0xf, 0x35
sysenter_tracesys: sysenter_tracesys:
CFI_RESTORE_STATE
SAVE_REST SAVE_REST
CLEAR_RREGS CLEAR_RREGS
movq $-ENOSYS,RAX(%rsp) /* really needed? */ movq $-ENOSYS,RAX(%rsp) /* really needed? */
...@@ -140,21 +161,28 @@ sysenter_tracesys: ...@@ -140,21 +161,28 @@ sysenter_tracesys:
* with the int 0x80 path. * with the int 0x80 path.
*/ */
ENTRY(ia32_cstar_target) ENTRY(ia32_cstar_target)
CFI_STARTPROC CFI_STARTPROC simple
CFI_DEF_CFA rsp,0
CFI_REGISTER rip,rcx
/*CFI_REGISTER rflags,r11*/
swapgs swapgs
movl %esp,%r8d movl %esp,%r8d
CFI_REGISTER rsp,r8
movq %gs:pda_kernelstack,%rsp movq %gs:pda_kernelstack,%rsp
sti sti
SAVE_ARGS 8,1,1 SAVE_ARGS 8,1,1
movl %eax,%eax /* zero extension */ movl %eax,%eax /* zero extension */
movq %rax,ORIG_RAX-ARGOFFSET(%rsp) movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
movq %rcx,RIP-ARGOFFSET(%rsp) movq %rcx,RIP-ARGOFFSET(%rsp)
CFI_REL_OFFSET rip,RIP-ARGOFFSET
movq %rbp,RCX-ARGOFFSET(%rsp) /* this lies slightly to ptrace */ movq %rbp,RCX-ARGOFFSET(%rsp) /* this lies slightly to ptrace */
movl %ebp,%ecx movl %ebp,%ecx
movq $__USER32_CS,CS-ARGOFFSET(%rsp) movq $__USER32_CS,CS-ARGOFFSET(%rsp)
movq $__USER32_DS,SS-ARGOFFSET(%rsp) movq $__USER32_DS,SS-ARGOFFSET(%rsp)
movq %r11,EFLAGS-ARGOFFSET(%rsp) movq %r11,EFLAGS-ARGOFFSET(%rsp)
/*CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/
movq %r8,RSP-ARGOFFSET(%rsp) movq %r8,RSP-ARGOFFSET(%rsp)
CFI_REL_OFFSET rsp,RSP-ARGOFFSET
/* no need to do an access_ok check here because r8 has been /* no need to do an access_ok check here because r8 has been
32bit zero extended */ 32bit zero extended */
/* hardware stack frame is complete now */ /* hardware stack frame is complete now */
...@@ -164,6 +192,7 @@ ENTRY(ia32_cstar_target) ...@@ -164,6 +192,7 @@ ENTRY(ia32_cstar_target)
.previous .previous
GET_THREAD_INFO(%r10) GET_THREAD_INFO(%r10)
testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10) testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10)
CFI_REMEMBER_STATE
jnz cstar_tracesys jnz cstar_tracesys
cstar_do_call: cstar_do_call:
cmpl $IA32_NR_syscalls,%eax cmpl $IA32_NR_syscalls,%eax
...@@ -177,12 +206,16 @@ cstar_do_call: ...@@ -177,12 +206,16 @@ cstar_do_call:
jnz int_ret_from_sys_call jnz int_ret_from_sys_call
RESTORE_ARGS 1,-ARG_SKIP,1,1,1 RESTORE_ARGS 1,-ARG_SKIP,1,1,1
movl RIP-ARGOFFSET(%rsp),%ecx movl RIP-ARGOFFSET(%rsp),%ecx
CFI_REGISTER rip,rcx
movl EFLAGS-ARGOFFSET(%rsp),%r11d movl EFLAGS-ARGOFFSET(%rsp),%r11d
/*CFI_REGISTER rflags,r11*/
movl RSP-ARGOFFSET(%rsp),%esp movl RSP-ARGOFFSET(%rsp),%esp
CFI_RESTORE rsp
swapgs swapgs
sysretl sysretl
cstar_tracesys: cstar_tracesys:
CFI_RESTORE_STATE
SAVE_REST SAVE_REST
CLEAR_RREGS CLEAR_RREGS
movq $-ENOSYS,RAX(%rsp) /* really needed? */ movq $-ENOSYS,RAX(%rsp) /* really needed? */
...@@ -226,11 +259,18 @@ ia32_badarg: ...@@ -226,11 +259,18 @@ ia32_badarg:
*/ */
ENTRY(ia32_syscall) ENTRY(ia32_syscall)
CFI_STARTPROC CFI_STARTPROC simple
CFI_DEF_CFA rsp,SS+8-RIP
/*CFI_REL_OFFSET ss,SS-RIP*/
CFI_REL_OFFSET rsp,RSP-RIP
/*CFI_REL_OFFSET rflags,EFLAGS-RIP*/
/*CFI_REL_OFFSET cs,CS-RIP*/
CFI_REL_OFFSET rip,RIP-RIP
swapgs swapgs
sti sti
movl %eax,%eax movl %eax,%eax
pushq %rax pushq %rax
CFI_ADJUST_CFA_OFFSET 8
cld cld
/* note the registers are not zero extended to the sf. /* note the registers are not zero extended to the sf.
this could be a problem. */ this could be a problem. */
...@@ -278,6 +318,8 @@ quiet_ni_syscall: ...@@ -278,6 +318,8 @@ quiet_ni_syscall:
jmp ia32_ptregs_common jmp ia32_ptregs_common
.endm .endm
CFI_STARTPROC
PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn, %rdi PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn, %rdi
PTREGSCALL stub32_sigreturn, sys32_sigreturn, %rdi PTREGSCALL stub32_sigreturn, sys32_sigreturn, %rdi
PTREGSCALL stub32_sigaltstack, sys32_sigaltstack, %rdx PTREGSCALL stub32_sigaltstack, sys32_sigaltstack, %rdx
...@@ -290,8 +332,9 @@ quiet_ni_syscall: ...@@ -290,8 +332,9 @@ quiet_ni_syscall:
PTREGSCALL stub32_rt_sigsuspend, sys_rt_sigsuspend, %rdx PTREGSCALL stub32_rt_sigsuspend, sys_rt_sigsuspend, %rdx
ENTRY(ia32_ptregs_common) ENTRY(ia32_ptregs_common)
CFI_STARTPROC
popq %r11 popq %r11
CFI_ADJUST_CFA_OFFSET -8
CFI_REGISTER rip, r11
SAVE_REST SAVE_REST
call *%rax call *%rax
RESTORE_REST RESTORE_REST
......
arch/x86_64/kernel/entry.S
View file @ 7effaa88
...@@ -79,16 +79,19 @@ ...@@ -79,16 +79,19 @@
xorl %eax, %eax xorl %eax, %eax
pushq %rax /* ss */ pushq %rax /* ss */
CFI_ADJUST_CFA_OFFSET 8 CFI_ADJUST_CFA_OFFSET 8
/*CFI_REL_OFFSET ss,0*/
pushq %rax /* rsp */ pushq %rax /* rsp */
CFI_ADJUST_CFA_OFFSET 8 CFI_ADJUST_CFA_OFFSET 8
CFI_OFFSET rip,0 CFI_REL_OFFSET rsp,0
pushq $(1<<9) /* eflags - interrupts on */ pushq $(1<<9) /* eflags - interrupts on */
CFI_ADJUST_CFA_OFFSET 8 CFI_ADJUST_CFA_OFFSET 8
/*CFI_REL_OFFSET rflags,0*/
pushq $__KERNEL_CS /* cs */ pushq $__KERNEL_CS /* cs */
CFI_ADJUST_CFA_OFFSET 8 CFI_ADJUST_CFA_OFFSET 8
/*CFI_REL_OFFSET cs,0*/
pushq \child_rip /* rip */ pushq \child_rip /* rip */
CFI_ADJUST_CFA_OFFSET 8 CFI_ADJUST_CFA_OFFSET 8
CFI_OFFSET rip,0 CFI_REL_OFFSET rip,0
pushq %rax /* orig rax */ pushq %rax /* orig rax */
CFI_ADJUST_CFA_OFFSET 8 CFI_ADJUST_CFA_OFFSET 8
.endm .endm
...@@ -98,32 +101,39 @@ ...@@ -98,32 +101,39 @@
CFI_ADJUST_CFA_OFFSET -(6*8) CFI_ADJUST_CFA_OFFSET -(6*8)
.endm .endm
.macro CFI_DEFAULT_STACK .macro CFI_DEFAULT_STACK start=1
CFI_ADJUST_CFA_OFFSET (SS) .if \start
CFI_OFFSET r15,R15-SS CFI_STARTPROC simple
CFI_OFFSET r14,R14-SS CFI_DEF_CFA rsp,SS+8
CFI_OFFSET r13,R13-SS .else
CFI_OFFSET r12,R12-SS CFI_DEF_CFA_OFFSET SS+8
CFI_OFFSET rbp,RBP-SS .endif
CFI_OFFSET rbx,RBX-SS CFI_REL_OFFSET r15,R15
CFI_OFFSET r11,R11-SS CFI_REL_OFFSET r14,R14
CFI_OFFSET r10,R10-SS CFI_REL_OFFSET r13,R13
CFI_OFFSET r9,R9-SS CFI_REL_OFFSET r12,R12
CFI_OFFSET r8,R8-SS CFI_REL_OFFSET rbp,RBP
CFI_OFFSET rax,RAX-SS CFI_REL_OFFSET rbx,RBX
CFI_OFFSET rcx,RCX-SS CFI_REL_OFFSET r11,R11
CFI_OFFSET rdx,RDX-SS CFI_REL_OFFSET r10,R10
CFI_OFFSET rsi,RSI-SS CFI_REL_OFFSET r9,R9
CFI_OFFSET rdi,RDI-SS CFI_REL_OFFSET r8,R8
CFI_OFFSET rsp,RSP-SS CFI_REL_OFFSET rax,RAX
CFI_OFFSET rip,RIP-SS CFI_REL_OFFSET rcx,RCX
CFI_REL_OFFSET rdx,RDX
CFI_REL_OFFSET rsi,RSI
CFI_REL_OFFSET rdi,RDI
CFI_REL_OFFSET rip,RIP
/*CFI_REL_OFFSET cs,CS*/
/*CFI_REL_OFFSET rflags,EFLAGS*/
CFI_REL_OFFSET rsp,RSP
/*CFI_REL_OFFSET ss,SS*/
.endm .endm
/* /*
* A newly forked process directly context switches into this. * A newly forked process directly context switches into this.
*/ */
/* rdi: prev */ /* rdi: prev */
ENTRY(ret_from_fork) ENTRY(ret_from_fork)
CFI_STARTPROC
CFI_DEFAULT_STACK CFI_DEFAULT_STACK
call schedule_tail call schedule_tail
GET_THREAD_INFO(%rcx) GET_THREAD_INFO(%rcx)
...@@ -172,7 +182,10 @@ rff_trace: ...@@ -172,7 +182,10 @@ rff_trace:
*/ */
ENTRY(system_call) ENTRY(system_call)
CFI_STARTPROC CFI_STARTPROC simple
CFI_DEF_CFA rsp,0
CFI_REGISTER rip,rcx
/*CFI_REGISTER rflags,r11*/
swapgs swapgs
movq %rsp,%gs:pda_oldrsp movq %rsp,%gs:pda_oldrsp
movq %gs:pda_kernelstack,%rsp movq %gs:pda_kernelstack,%rsp
...@@ -180,8 +193,10 @@ ENTRY(system_call) ...@@ -180,8 +193,10 @@ ENTRY(system_call)
SAVE_ARGS 8,1 SAVE_ARGS 8,1
movq %rax,ORIG_RAX-ARGOFFSET(%rsp) movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
movq %rcx,RIP-ARGOFFSET(%rsp) movq %rcx,RIP-ARGOFFSET(%rsp)
CFI_REL_OFFSET rip,RIP-ARGOFFSET
GET_THREAD_INFO(%rcx) GET_THREAD_INFO(%rcx)
testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx) testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
CFI_REMEMBER_STATE
jnz tracesys jnz tracesys
cmpq $__NR_syscall_max,%rax cmpq $__NR_syscall_max,%rax
ja badsys ja badsys
...@@ -201,9 +216,12 @@ sysret_check: ...@@ -201,9 +216,12 @@ sysret_check:
cli cli
movl threadinfo_flags(%rcx),%edx movl threadinfo_flags(%rcx),%edx
andl %edi,%edx andl %edi,%edx
CFI_REMEMBER_STATE
jnz sysret_careful jnz sysret_careful
movq RIP-ARGOFFSET(%rsp),%rcx movq RIP-ARGOFFSET(%rsp),%rcx
CFI_REGISTER rip,rcx
RESTORE_ARGS 0,-ARG_SKIP,1 RESTORE_ARGS 0,-ARG_SKIP,1
/*CFI_REGISTER rflags,r11*/
movq %gs:pda_oldrsp,%rsp movq %gs:pda_oldrsp,%rsp
swapgs swapgs
sysretq sysretq
...@@ -211,12 +229,15 @@ sysret_check: ...@@ -211,12 +229,15 @@ sysret_check:
/* Handle reschedules */ /* Handle reschedules */
/* edx: work, edi: workmask */ /* edx: work, edi: workmask */
sysret_careful: sysret_careful:
CFI_RESTORE_STATE
bt $TIF_NEED_RESCHED,%edx bt $TIF_NEED_RESCHED,%edx
jnc sysret_signal jnc sysret_signal
sti sti
pushq %rdi pushq %rdi
CFI_ADJUST_CFA_OFFSET 8
call schedule call schedule
popq %rdi popq %rdi
CFI_ADJUST_CFA_OFFSET -8
jmp sysret_check jmp sysret_check
/* Handle a signal */ /* Handle a signal */
...@@ -234,8 +255,13 @@ sysret_signal: ...@@ -234,8 +255,13 @@ sysret_signal:
1: movl $_TIF_NEED_RESCHED,%edi 1: movl $_TIF_NEED_RESCHED,%edi
jmp sysret_check jmp sysret_check
badsys:
movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
jmp ret_from_sys_call
/* Do syscall tracing */ /* Do syscall tracing */
tracesys: tracesys:
CFI_RESTORE_STATE
SAVE_REST SAVE_REST
movq $-ENOSYS,RAX(%rsp) movq $-ENOSYS,RAX(%rsp)
FIXUP_TOP_OF_STACK %rdi FIXUP_TOP_OF_STACK %rdi
...@@ -254,16 +280,29 @@ tracesys: ...@@ -254,16 +280,29 @@ tracesys:
RESTORE_TOP_OF_STACK %rbx RESTORE_TOP_OF_STACK %rbx
RESTORE_REST RESTORE_REST
jmp ret_from_sys_call jmp ret_from_sys_call
CFI_ENDPROC
badsys:
movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
jmp ret_from_sys_call
/* /*
* Syscall return path ending with IRET. * Syscall return path ending with IRET.
* Has correct top of stack, but partial stack frame. * Has correct top of stack, but partial stack frame.
*/ */
ENTRY(int_ret_from_sys_call) ENTRY(int_ret_from_sys_call)
CFI_STARTPROC simple
CFI_DEF_CFA rsp,SS+8-ARGOFFSET
/*CFI_REL_OFFSET ss,SS-ARGOFFSET*/
CFI_REL_OFFSET rsp,RSP-ARGOFFSET
/*CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/
/*CFI_REL_OFFSET cs,CS-ARGOFFSET*/
CFI_REL_OFFSET rip,RIP-ARGOFFSET
CFI_REL_OFFSET rdx,RDX-ARGOFFSET
CFI_REL_OFFSET rcx,RCX-ARGOFFSET
CFI_REL_OFFSET rax,RAX-ARGOFFSET
CFI_REL_OFFSET rdi,RDI-ARGOFFSET
CFI_REL_OFFSET rsi,RSI-ARGOFFSET
CFI_REL_OFFSET r8,R8-ARGOFFSET
CFI_REL_OFFSET r9,R9-ARGOFFSET
CFI_REL_OFFSET r10,R10-ARGOFFSET
CFI_REL_OFFSET r11,R11-ARGOFFSET
cli cli
testl $3,CS-ARGOFFSET(%rsp) testl $3,CS-ARGOFFSET(%rsp)
je retint_restore_args je retint_restore_args
...@@ -284,8 +323,10 @@ int_careful: ...@@ -284,8 +323,10 @@ int_careful:
jnc int_very_careful jnc int_very_careful
sti sti
pushq %rdi pushq %rdi
CFI_ADJUST_CFA_OFFSET 8
call schedule call schedule
popq %rdi popq %rdi
CFI_ADJUST_CFA_OFFSET -8
cli cli
jmp int_with_check jmp int_with_check
...@@ -297,9 +338,11 @@ int_very_careful: ...@@ -297,9 +338,11 @@ int_very_careful:
testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
jz int_signal jz int_signal
pushq %rdi pushq %rdi
CFI_ADJUST_CFA_OFFSET 8
leaq 8(%rsp),%rdi # &ptregs -> arg1 leaq 8(%rsp),%rdi # &ptregs -> arg1
call syscall_trace_leave call syscall_trace_leave
popq %rdi popq %rdi
CFI_ADJUST_CFA_OFFSET -8
andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
cli cli
jmp int_restore_rest jmp int_restore_rest
...@@ -329,6 +372,8 @@ int_restore_rest: ...@@ -329,6 +372,8 @@ int_restore_rest:
jmp ptregscall_common jmp ptregscall_common
.endm .endm
CFI_STARTPROC
PTREGSCALL stub_clone, sys_clone, %r8 PTREGSCALL stub_clone, sys_clone, %r8
PTREGSCALL stub_fork, sys_fork, %rdi PTREGSCALL stub_fork, sys_fork, %rdi
PTREGSCALL stub_vfork, sys_vfork, %rdi PTREGSCALL stub_vfork, sys_vfork, %rdi
...@@ -337,18 +382,21 @@ int_restore_rest: ...@@ -337,18 +382,21 @@ int_restore_rest:
PTREGSCALL stub_iopl, sys_iopl, %rsi PTREGSCALL stub_iopl, sys_iopl, %rsi
ENTRY(ptregscall_common) ENTRY(ptregscall_common)
CFI_STARTPROC
popq %r11 popq %r11
CFI_ADJUST_CFA_OFFSET -8 CFI_ADJUST_CFA_OFFSET -8
CFI_REGISTER rip, r11
SAVE_REST SAVE_REST
movq %r11, %r15 movq %r11, %r15
CFI_REGISTER rip, r15
FIXUP_TOP_OF_STACK %r11 FIXUP_TOP_OF_STACK %r11
call *%rax call *%rax
RESTORE_TOP_OF_STACK %r11 RESTORE_TOP_OF_STACK %r11
movq %r15, %r11 movq %r15, %r11
CFI_REGISTER rip, r11
RESTORE_REST RESTORE_REST
pushq %r11 pushq %r11
CFI_ADJUST_CFA_OFFSET 8 CFI_ADJUST_CFA_OFFSET 8
CFI_REL_OFFSET rip, 0
ret ret
CFI_ENDPROC CFI_ENDPROC
...@@ -356,21 +404,27 @@ ENTRY(stub_execve) ...@@ -356,21 +404,27 @@ ENTRY(stub_execve)
CFI_STARTPROC CFI_STARTPROC
popq %r11 popq %r11
CFI_ADJUST_CFA_OFFSET -8 CFI_ADJUST_CFA_OFFSET -8
CFI_REGISTER rip, r11
SAVE_REST SAVE_REST
movq %r11, %r15 movq %r11, %r15
CFI_REGISTER rip, r15
FIXUP_TOP_OF_STACK %r11 FIXUP_TOP_OF_STACK %r11
call sys_execve call sys_execve
GET_THREAD_INFO(%rcx) GET_THREAD_INFO(%rcx)
bt $TIF_IA32,threadinfo_flags(%rcx) bt $TIF_IA32,threadinfo_flags(%rcx)
CFI_REMEMBER_STATE
jc exec_32bit jc exec_32bit
RESTORE_TOP_OF_STACK %r11 RESTORE_TOP_OF_STACK %r11
movq %r15, %r11 movq %r15, %r11
CFI_REGISTER rip, r11
RESTORE_REST RESTORE_REST
push %r11 pushq %r11
CFI_ADJUST_CFA_OFFSET 8
CFI_REL_OFFSET rip, 0
ret ret
exec_32bit: exec_32bit:
CFI_ADJUST_CFA_OFFSET REST_SKIP CFI_RESTORE_STATE
movq %rax,RAX(%rsp) movq %rax,RAX(%rsp)
RESTORE_REST RESTORE_REST
jmp int_ret_from_sys_call jmp int_ret_from_sys_call
...@@ -383,6 +437,7 @@ exec_32bit: ...@@ -383,6 +437,7 @@ exec_32bit:
ENTRY(stub_rt_sigreturn) ENTRY(stub_rt_sigreturn)
CFI_STARTPROC CFI_STARTPROC
addq $8, %rsp addq $8, %rsp
CFI_ADJUST_CFA_OFFSET -8
SAVE_REST SAVE_REST
movq %rsp,%rdi movq %rsp,%rdi
FIXUP_TOP_OF_STACK %r11 FIXUP_TOP_OF_STACK %r11
...@@ -392,6 +447,25 @@ ENTRY(stub_rt_sigreturn) ...@@ -392,6 +447,25 @@ ENTRY(stub_rt_sigreturn)
jmp int_ret_from_sys_call jmp int_ret_from_sys_call
CFI_ENDPROC CFI_ENDPROC
/*
* initial frame state for interrupts and exceptions
*/
.macro _frame ref
CFI_STARTPROC simple
CFI_DEF_CFA rsp,SS+8-\ref
/*CFI_REL_OFFSET ss,SS-\ref*/
CFI_REL_OFFSET rsp,RSP-\ref
/*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
/*CFI_REL_OFFSET cs,CS-\ref*/
CFI_REL_OFFSET rip,RIP-\ref
.endm
/* initial frame state for interrupts (and exceptions without error code) */
#define INTR_FRAME _frame RIP
/* initial frame state for exceptions with error code (and interrupts with
vector already pushed) */
#define XCPT_FRAME _frame ORIG_RAX
/* /*
* Interrupt entry/exit. * Interrupt entry/exit.
* *
...@@ -402,10 +476,6 @@ ENTRY(stub_rt_sigreturn) ...@@ -402,10 +476,6 @@ ENTRY(stub_rt_sigreturn)
/* 0(%rsp): interrupt number */ /* 0(%rsp): interrupt number */
.macro interrupt func .macro interrupt func
CFI_STARTPROC simple
CFI_DEF_CFA rsp,(SS-RDI)
CFI_REL_OFFSET rsp,(RSP-ORIG_RAX)
CFI_REL_OFFSET rip,(RIP-ORIG_RAX)
cld cld
#ifdef CONFIG_DEBUG_INFO #ifdef CONFIG_DEBUG_INFO
SAVE_ALL SAVE_ALL
...@@ -425,22 +495,26 @@ ENTRY(stub_rt_sigreturn) ...@@ -425,22 +495,26 @@ ENTRY(stub_rt_sigreturn)
swapgs swapgs
1: incl %gs:pda_irqcount # RED-PEN should check preempt count 1: incl %gs:pda_irqcount # RED-PEN should check preempt count
movq %gs:pda_irqstackptr,%rax movq %gs:pda_irqstackptr,%rax
cmoveq %rax,%rsp cmoveq %rax,%rsp /*todo This needs CFI annotation! */
pushq %rdi # save old stack pushq %rdi # save old stack
CFI_ADJUST_CFA_OFFSET 8
call \func call \func
.endm .endm
ENTRY(common_interrupt) ENTRY(common_interrupt)
XCPT_FRAME
interrupt do_IRQ interrupt do_IRQ
/* 0(%rsp): oldrsp-ARGOFFSET */ /* 0(%rsp): oldrsp-ARGOFFSET */
ret_from_intr: ret_from_intr:
popq %rdi popq %rdi
CFI_ADJUST_CFA_OFFSET -8
cli cli
decl %gs:pda_irqcount decl %gs:pda_irqcount
#ifdef CONFIG_DEBUG_INFO #ifdef CONFIG_DEBUG_INFO
movq RBP(%rdi),%rbp movq RBP(%rdi),%rbp
CFI_DEF_CFA_REGISTER rsp
#endif #endif
leaq ARGOFFSET(%rdi),%rsp leaq ARGOFFSET(%rdi),%rsp /*todo This needs CFI annotation! */
exit_intr: exit_intr:
GET_THREAD_INFO(%rcx) GET_THREAD_INFO(%rcx)
testl $3,CS-ARGOFFSET(%rsp) testl $3,CS-ARGOFFSET(%rsp)
...@@ -456,6 +530,7 @@ retint_with_reschedule: ...@@ -456,6 +530,7 @@ retint_with_reschedule:
retint_check: retint_check:
movl threadinfo_flags(%rcx),%edx movl threadinfo_flags(%rcx),%edx
andl %edi,%edx andl %edi,%edx
CFI_REMEMBER_STATE
jnz retint_careful jnz retint_careful
retint_swapgs: retint_swapgs:
swapgs swapgs
...@@ -478,12 +553,15 @@ bad_iret: ...@@ -478,12 +553,15 @@ bad_iret:
/* edi: workmask, edx: work */ /* edi: workmask, edx: work */
retint_careful: retint_careful:
CFI_RESTORE_STATE
bt $TIF_NEED_RESCHED,%edx bt $TIF_NEED_RESCHED,%edx
jnc retint_signal jnc retint_signal
sti sti
pushq %rdi pushq %rdi
CFI_ADJUST_CFA_OFFSET 8
call schedule call schedule
popq %rdi popq %rdi
CFI_ADJUST_CFA_OFFSET -8
GET_THREAD_INFO(%rcx) GET_THREAD_INFO(%rcx)
cli cli
jmp retint_check jmp retint_check
...@@ -523,7 +601,9 @@ retint_kernel: ...@@ -523,7 +601,9 @@ retint_kernel:
* APIC interrupts. * APIC interrupts.
*/ */
.macro apicinterrupt num,func .macro apicinterrupt num,func
INTR_FRAME
pushq $\num-256 pushq $\num-256
CFI_ADJUST_CFA_OFFSET 8
interrupt \func interrupt \func
jmp ret_from_intr jmp ret_from_intr
CFI_ENDPROC CFI_ENDPROC
...@@ -569,16 +649,23 @@ ENTRY(spurious_interrupt) ...@@ -569,16 +649,23 @@ ENTRY(spurious_interrupt)
* Exception entry points. * Exception entry points.
*/ */
.macro zeroentry sym .macro zeroentry sym
INTR_FRAME
pushq $0 /* push error code/oldrax */ pushq $0 /* push error code/oldrax */
CFI_ADJUST_CFA_OFFSET 8
pushq %rax /* push real oldrax to the rdi slot */ pushq %rax /* push real oldrax to the rdi slot */
CFI_ADJUST_CFA_OFFSET 8
leaq \sym(%rip),%rax leaq \sym(%rip),%rax
jmp error_entry jmp error_entry
CFI_ENDPROC
.endm .endm
.macro errorentry sym .macro errorentry sym
XCPT_FRAME
pushq %rax pushq %rax
CFI_ADJUST_CFA_OFFSET 8
leaq \sym(%rip),%rax leaq \sym(%rip),%rax
jmp error_entry jmp error_entry
CFI_ENDPROC
.endm .endm
/* error code is on the stack already */ /* error code is on the stack already */
...@@ -605,10 +692,7 @@ ENTRY(spurious_interrupt) ...@@ -605,10 +692,7 @@ ENTRY(spurious_interrupt)
* and the exception handler in %rax. * and the exception handler in %rax.
*/ */
ENTRY(error_entry) ENTRY(error_entry)
CFI_STARTPROC simple _frame RDI
CFI_DEF_CFA rsp,(SS-RDI)
CFI_REL_OFFSET rsp,(RSP-RDI)
CFI_REL_OFFSET rip,(RIP-RDI)
/* rdi slot contains rax, oldrax contains error code */ /* rdi slot contains rax, oldrax contains error code */
cld cld
subq $14*8,%rsp subq $14*8,%rsp
...@@ -690,7 +774,9 @@ error_kernelspace: ...@@ -690,7 +774,9 @@ error_kernelspace:
/* Reload gs selector with exception handling */ /* Reload gs selector with exception handling */
/* edi: new selector */ /* edi: new selector */
ENTRY(load_gs_index) ENTRY(load_gs_index)
CFI_STARTPROC
pushf pushf
CFI_ADJUST_CFA_OFFSET 8
cli cli
swapgs swapgs
gs_change: gs_change:
...@@ -698,7 +784,9 @@ gs_change: ...@@ -698,7 +784,9 @@ gs_change:
2: mfence /* workaround */ 2: mfence /* workaround */
swapgs swapgs
popf popf
CFI_ADJUST_CFA_OFFSET -8
ret ret
CFI_ENDPROC
.section __ex_table,"a" .section __ex_table,"a"
.align 8 .align 8
...@@ -810,7 +898,7 @@ ENTRY(device_not_available) ...@@ -810,7 +898,7 @@ ENTRY(device_not_available)
/* runs on exception stack */ /* runs on exception stack */
KPROBE_ENTRY(debug) KPROBE_ENTRY(debug)
CFI_STARTPROC INTR_FRAME
pushq $0 pushq $0
CFI_ADJUST_CFA_OFFSET 8 CFI_ADJUST_CFA_OFFSET 8
paranoidentry do_debug paranoidentry do_debug
...@@ -820,7 +908,7 @@ KPROBE_ENTRY(debug) ...@@ -820,7 +908,7 @@ KPROBE_ENTRY(debug)
/* runs on exception stack */ /* runs on exception stack */
ENTRY(nmi) ENTRY(nmi)
CFI_STARTPROC INTR_FRAME
pushq $-1 pushq $-1
CFI_ADJUST_CFA_OFFSET 8 CFI_ADJUST_CFA_OFFSET 8
paranoidentry do_nmi paranoidentry do_nmi
...@@ -888,7 +976,7 @@ ENTRY(reserved) ...@@ -888,7 +976,7 @@ ENTRY(reserved)
/* runs on exception stack */ /* runs on exception stack */
ENTRY(double_fault) ENTRY(double_fault)
CFI_STARTPROC XCPT_FRAME
paranoidentry do_double_fault paranoidentry do_double_fault
jmp paranoid_exit jmp paranoid_exit
CFI_ENDPROC CFI_ENDPROC
...@@ -901,7 +989,7 @@ ENTRY(segment_not_present) ...@@ -901,7 +989,7 @@ ENTRY(segment_not_present)
/* runs on exception stack */ /* runs on exception stack */
ENTRY(stack_segment) ENTRY(stack_segment)
CFI_STARTPROC XCPT_FRAME
paranoidentry do_stack_segment paranoidentry do_stack_segment
jmp paranoid_exit jmp paranoid_exit
CFI_ENDPROC CFI_ENDPROC
...@@ -922,7 +1010,7 @@ ENTRY(spurious_interrupt_bug) ...@@ -922,7 +1010,7 @@ ENTRY(spurious_interrupt_bug)
#ifdef CONFIG_X86_MCE #ifdef CONFIG_X86_MCE
/* runs on exception stack */ /* runs on exception stack */
ENTRY(machine_check) ENTRY(machine_check)
CFI_STARTPROC INTR_FRAME
pushq $0 pushq $0
CFI_ADJUST_CFA_OFFSET 8 CFI_ADJUST_CFA_OFFSET 8
paranoidentry do_machine_check paranoidentry do_machine_check
...@@ -934,14 +1022,19 @@ ENTRY(call_debug) ...@@ -934,14 +1022,19 @@ ENTRY(call_debug)
zeroentry do_call_debug zeroentry do_call_debug
ENTRY(call_softirq) ENTRY(call_softirq)
CFI_STARTPROC
movq %gs:pda_irqstackptr,%rax movq %gs:pda_irqstackptr,%rax
pushq %r15 pushq %r15
CFI_ADJUST_CFA_OFFSET 8
movq %rsp,%r15 movq %rsp,%r15
CFI_DEF_CFA_REGISTER r15
incl %gs:pda_irqcount incl %gs:pda_irqcount
cmove %rax,%rsp cmove %rax,%rsp
call __do_softirq call __do_softirq
movq %r15,%rsp movq %r15,%rsp
CFI_DEF_CFA_REGISTER rsp
decl %gs:pda_irqcount decl %gs:pda_irqcount
popq %r15 popq %r15
CFI_ADJUST_CFA_OFFSET -8
ret ret
CFI_ENDPROC
include/asm-x86_64/calling.h
View file @ 7effaa88
...@@ -65,27 +65,36 @@ ...@@ -65,27 +65,36 @@
.if \skipr11 .if \skipr11
.else .else
movq (%rsp),%r11 movq (%rsp),%r11
CFI_RESTORE r11
.endif .endif
.if \skipr8910 .if \skipr8910
.else .else
movq 1*8(%rsp),%r10 movq 1*8(%rsp),%r10
CFI_RESTORE r10
movq 2*8(%rsp),%r9 movq 2*8(%rsp),%r9
CFI_RESTORE r9
movq 3*8(%rsp),%r8 movq 3*8(%rsp),%r8
CFI_RESTORE r8
.endif .endif
.if \skiprax .if \skiprax
.else .else
movq 4*8(%rsp),%rax movq 4*8(%rsp),%rax
CFI_RESTORE rax
.endif .endif
.if \skiprcx .if \skiprcx
.else .else
movq 5*8(%rsp),%rcx movq 5*8(%rsp),%rcx
CFI_RESTORE rcx
.endif .endif
.if \skiprdx .if \skiprdx
.else .else
movq 6*8(%rsp),%rdx movq 6*8(%rsp),%rdx
CFI_RESTORE rdx
.endif .endif
movq 7*8(%rsp),%rsi movq 7*8(%rsp),%rsi
CFI_RESTORE rsi
movq 8*8(%rsp),%rdi movq 8*8(%rsp),%rdi
CFI_RESTORE rdi
.if ARG_SKIP+\addskip > 0 .if ARG_SKIP+\addskip > 0
addq $ARG_SKIP+\addskip,%rsp addq $ARG_SKIP+\addskip,%rsp
CFI_ADJUST_CFA_OFFSET -(ARG_SKIP+\addskip) CFI_ADJUST_CFA_OFFSET -(ARG_SKIP+\addskip)
...@@ -124,11 +133,17 @@ ...@@ -124,11 +133,17 @@
.macro RESTORE_REST .macro RESTORE_REST
movq (%rsp),%r15 movq (%rsp),%r15
CFI_RESTORE r15
movq 1*8(%rsp),%r14 movq 1*8(%rsp),%r14
CFI_RESTORE r14
movq 2*8(%rsp),%r13 movq 2*8(%rsp),%r13
CFI_RESTORE r13
movq 3*8(%rsp),%r12 movq 3*8(%rsp),%r12
CFI_RESTORE r12
movq 4*8(%rsp),%rbp movq 4*8(%rsp),%rbp
CFI_RESTORE rbp
movq 5*8(%rsp),%rbx movq 5*8(%rsp),%rbx
CFI_RESTORE rbx
addq $REST_SKIP,%rsp addq $REST_SKIP,%rsp
CFI_ADJUST_CFA_OFFSET -(REST_SKIP) CFI_ADJUST_CFA_OFFSET -(REST_SKIP)
.endm .endm
...@@ -146,11 +161,3 @@ ...@@ -146,11 +161,3 @@
.macro icebp .macro icebp
.byte 0xf1 .byte 0xf1
.endm .endm
#ifdef CONFIG_FRAME_POINTER
#define ENTER enter
#define LEAVE leave
#else
#define ENTER
#define LEAVE
#endif
include/asm-x86_64/dwarf2.h
View file @ 7effaa88
...@@ -24,6 +24,10 @@ ...@@ -24,6 +24,10 @@
#define CFI_ADJUST_CFA_OFFSET .cfi_adjust_cfa_offset #define CFI_ADJUST_CFA_OFFSET .cfi_adjust_cfa_offset
#define CFI_OFFSET .cfi_offset #define CFI_OFFSET .cfi_offset
#define CFI_REL_OFFSET .cfi_rel_offset #define CFI_REL_OFFSET .cfi_rel_offset
#define CFI_REGISTER .cfi_register
#define CFI_RESTORE .cfi_restore
#define CFI_REMEMBER_STATE .cfi_remember_state
#define CFI_RESTORE_STATE .cfi_restore_state
#else #else
...@@ -36,6 +40,10 @@ ...@@ -36,6 +40,10 @@
#define CFI_ADJUST_CFA_OFFSET # #define CFI_ADJUST_CFA_OFFSET #
#define CFI_OFFSET # #define CFI_OFFSET #
#define CFI_REL_OFFSET # #define CFI_REL_OFFSET #
#define CFI_REGISTER #
#define CFI_RESTORE #
#define CFI_REMEMBER_STATE #
#define CFI_RESTORE_STATE #
#endif #endif
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7