[PATCH] x86-64: Fix CFI information

Being the foundation for reliable stack unwinding, this fixes CFI unwind
annotations in many low-level x86_64 routines, plus a config option
(available to all architectures, and also present in the previously sent
patch adding such annotations to i386 code) to enable them separatly
rather than only along with adding full debug information.

Signed-off-by: Jan Beulich <jbeulich@novell.com>
Signed-off-by: Andi Kleen <ak@suse.de>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

1 files changed, 150 insertions, 57 deletions

diff --git a/arch/x86_64/kernel/entry.S b/arch/x86_64/kernel/entry.S
index 873c39d8f818..7937971d1853 100644
--- a/arch/x86_64/kernel/entry.S
+++ b/arch/x86_64/kernel/entry.S
@@ -79,16 +79,19 @@
 	xorl %eax, %eax
 	pushq %rax /* ss */
 	CFI_ADJUST_CFA_OFFSET	8
+	/*CFI_REL_OFFSET	ss,0*/
 	pushq %rax /* rsp */
 	CFI_ADJUST_CFA_OFFSET	8
-	CFI_OFFSET	rip,0
+	CFI_REL_OFFSET	rsp,0
 	pushq $(1<<9) /* eflags - interrupts on */
 	CFI_ADJUST_CFA_OFFSET	8
+	/*CFI_REL_OFFSET	rflags,0*/
 	pushq $__KERNEL_CS /* cs */
 	CFI_ADJUST_CFA_OFFSET	8
+	/*CFI_REL_OFFSET	cs,0*/
 	pushq \child_rip /* rip */
 	CFI_ADJUST_CFA_OFFSET	8
-	CFI_OFFSET	rip,0
+	CFI_REL_OFFSET	rip,0
 	pushq	%rax /* orig rax */
 	CFI_ADJUST_CFA_OFFSET	8
 	.endm
@@ -98,32 +101,39 @@
 	CFI_ADJUST_CFA_OFFSET	-(6*8)
 	.endm
 
-	.macro	CFI_DEFAULT_STACK
-	CFI_ADJUST_CFA_OFFSET  (SS)
-	CFI_OFFSET	r15,R15-SS
-	CFI_OFFSET	r14,R14-SS
-	CFI_OFFSET	r13,R13-SS
-	CFI_OFFSET	r12,R12-SS
-	CFI_OFFSET	rbp,RBP-SS
-	CFI_OFFSET	rbx,RBX-SS
-	CFI_OFFSET	r11,R11-SS
-	CFI_OFFSET	r10,R10-SS
-	CFI_OFFSET	r9,R9-SS
-	CFI_OFFSET	r8,R8-SS
-	CFI_OFFSET	rax,RAX-SS
-	CFI_OFFSET	rcx,RCX-SS
-	CFI_OFFSET	rdx,RDX-SS
-	CFI_OFFSET	rsi,RSI-SS
-	CFI_OFFSET	rdi,RDI-SS
-	CFI_OFFSET	rsp,RSP-SS
-	CFI_OFFSET	rip,RIP-SS
+	.macro	CFI_DEFAULT_STACK start=1
+	.if \start
+	CFI_STARTPROC	simple
+	CFI_DEF_CFA	rsp,SS+8
+	.else
+	CFI_DEF_CFA_OFFSET SS+8
+	.endif
+	CFI_REL_OFFSET	r15,R15
+	CFI_REL_OFFSET	r14,R14
+	CFI_REL_OFFSET	r13,R13
+	CFI_REL_OFFSET	r12,R12
+	CFI_REL_OFFSET	rbp,RBP
+	CFI_REL_OFFSET	rbx,RBX
+	CFI_REL_OFFSET	r11,R11
+	CFI_REL_OFFSET	r10,R10
+	CFI_REL_OFFSET	r9,R9
+	CFI_REL_OFFSET	r8,R8
+	CFI_REL_OFFSET	rax,RAX
+	CFI_REL_OFFSET	rcx,RCX
+	CFI_REL_OFFSET	rdx,RDX
+	CFI_REL_OFFSET	rsi,RSI
+	CFI_REL_OFFSET	rdi,RDI
+	CFI_REL_OFFSET	rip,RIP
+	/*CFI_REL_OFFSET	cs,CS*/
+	/*CFI_REL_OFFSET	rflags,EFLAGS*/
+	CFI_REL_OFFSET	rsp,RSP
+	/*CFI_REL_OFFSET	ss,SS*/
 	.endm
 /*
  * A newly forked process directly context switches into this.
  */ 	
 /* rdi:	prev */	
 ENTRY(ret_from_fork)
-	CFI_STARTPROC
 	CFI_DEFAULT_STACK
 	call schedule_tail
 	GET_THREAD_INFO(%rcx)
@@ -172,16 +182,21 @@ rff_trace:
  */ 			 		
 
 ENTRY(system_call)
-	CFI_STARTPROC
+	CFI_STARTPROC	simple
+	CFI_DEF_CFA	rsp,0
+	CFI_REGISTER	rip,rcx
+	/*CFI_REGISTER	rflags,r11*/
 	swapgs
 	movq	%rsp,%gs:pda_oldrsp 
 	movq	%gs:pda_kernelstack,%rsp
 	sti					
 	SAVE_ARGS 8,1
 	movq  %rax,ORIG_RAX-ARGOFFSET(%rsp) 
-	movq  %rcx,RIP-ARGOFFSET(%rsp)  
+	movq  %rcx,RIP-ARGOFFSET(%rsp)
+	CFI_REL_OFFSET rip,RIP-ARGOFFSET
 	GET_THREAD_INFO(%rcx)
 	testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
+	CFI_REMEMBER_STATE
 	jnz tracesys
 	cmpq $__NR_syscall_max,%rax
 	ja badsys
@@ -201,9 +216,12 @@ sysret_check:
 	cli
 	movl threadinfo_flags(%rcx),%edx
 	andl %edi,%edx
+	CFI_REMEMBER_STATE
 	jnz  sysret_careful 
 	movq RIP-ARGOFFSET(%rsp),%rcx
+	CFI_REGISTER	rip,rcx
 	RESTORE_ARGS 0,-ARG_SKIP,1
+	/*CFI_REGISTER	rflags,r11*/
 	movq	%gs:pda_oldrsp,%rsp
 	swapgs
 	sysretq
@@ -211,12 +229,15 @@ sysret_check:
 	/* Handle reschedules */
 	/* edx:	work, edi: workmask */	
 sysret_careful:
+	CFI_RESTORE_STATE
 	bt $TIF_NEED_RESCHED,%edx
 	jnc sysret_signal
 	sti
 	pushq %rdi
+	CFI_ADJUST_CFA_OFFSET 8
 	call schedule
 	popq  %rdi
+	CFI_ADJUST_CFA_OFFSET -8
 	jmp sysret_check
 
 	/* Handle a signal */ 
@@ -234,8 +255,13 @@ sysret_signal:
 1:	movl $_TIF_NEED_RESCHED,%edi
 	jmp sysret_check
 	
+badsys:
+	movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
+	jmp ret_from_sys_call
+
 	/* Do syscall tracing */
 tracesys:			 
+	CFI_RESTORE_STATE
 	SAVE_REST
 	movq $-ENOSYS,RAX(%rsp)
 	FIXUP_TOP_OF_STACK %rdi
@@ -254,16 +280,29 @@ tracesys:
 	RESTORE_TOP_OF_STACK %rbx
 	RESTORE_REST
 	jmp ret_from_sys_call
+	CFI_ENDPROC
 		
-badsys:
-	movq $-ENOSYS,RAX-ARGOFFSET(%rsp)	
-	jmp ret_from_sys_call
-
 /* 
  * Syscall return path ending with IRET.
  * Has correct top of stack, but partial stack frame.
  */ 	
-ENTRY(int_ret_from_sys_call)	
+ENTRY(int_ret_from_sys_call)
+	CFI_STARTPROC	simple
+	CFI_DEF_CFA	rsp,SS+8-ARGOFFSET
+	/*CFI_REL_OFFSET	ss,SS-ARGOFFSET*/
+	CFI_REL_OFFSET	rsp,RSP-ARGOFFSET
+	/*CFI_REL_OFFSET	rflags,EFLAGS-ARGOFFSET*/
+	/*CFI_REL_OFFSET	cs,CS-ARGOFFSET*/
+	CFI_REL_OFFSET	rip,RIP-ARGOFFSET
+	CFI_REL_OFFSET	rdx,RDX-ARGOFFSET
+	CFI_REL_OFFSET	rcx,RCX-ARGOFFSET
+	CFI_REL_OFFSET	rax,RAX-ARGOFFSET
+	CFI_REL_OFFSET	rdi,RDI-ARGOFFSET
+	CFI_REL_OFFSET	rsi,RSI-ARGOFFSET
+	CFI_REL_OFFSET	r8,R8-ARGOFFSET
+	CFI_REL_OFFSET	r9,R9-ARGOFFSET
+	CFI_REL_OFFSET	r10,R10-ARGOFFSET
+	CFI_REL_OFFSET	r11,R11-ARGOFFSET
 	cli
 	testl $3,CS-ARGOFFSET(%rsp)
 	je retint_restore_args
@@ -284,8 +323,10 @@ int_careful:
 	jnc  int_very_careful
 	sti
 	pushq %rdi
+	CFI_ADJUST_CFA_OFFSET 8
 	call schedule
 	popq %rdi
+	CFI_ADJUST_CFA_OFFSET -8
 	cli
 	jmp int_with_check
 
@@ -297,9 +338,11 @@ int_very_careful:
 	testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
 	jz int_signal
 	pushq %rdi
+	CFI_ADJUST_CFA_OFFSET 8
 	leaq 8(%rsp),%rdi	# &ptregs -> arg1	
 	call syscall_trace_leave
 	popq %rdi
+	CFI_ADJUST_CFA_OFFSET -8
 	andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
 	cli
 	jmp int_restore_rest
@@ -329,6 +372,8 @@ int_restore_rest:
 	jmp	ptregscall_common
 	.endm
 
+	CFI_STARTPROC
+
 	PTREGSCALL stub_clone, sys_clone, %r8
 	PTREGSCALL stub_fork, sys_fork, %rdi
 	PTREGSCALL stub_vfork, sys_vfork, %rdi
@@ -337,40 +382,49 @@ int_restore_rest:
 	PTREGSCALL stub_iopl, sys_iopl, %rsi
 
 ENTRY(ptregscall_common)
-	CFI_STARTPROC
 	popq %r11
-	CFI_ADJUST_CFA_OFFSET	-8
+	CFI_ADJUST_CFA_OFFSET -8
+	CFI_REGISTER rip, r11
 	SAVE_REST
 	movq %r11, %r15
+	CFI_REGISTER rip, r15
 	FIXUP_TOP_OF_STACK %r11
 	call *%rax
 	RESTORE_TOP_OF_STACK %r11
 	movq %r15, %r11
+	CFI_REGISTER rip, r11
 	RESTORE_REST
 	pushq %r11
-	CFI_ADJUST_CFA_OFFSET	8
+	CFI_ADJUST_CFA_OFFSET 8
+	CFI_REL_OFFSET rip, 0
 	ret
 	CFI_ENDPROC
 	
 ENTRY(stub_execve)
 	CFI_STARTPROC
 	popq %r11
-	CFI_ADJUST_CFA_OFFSET	-8
+	CFI_ADJUST_CFA_OFFSET -8
+	CFI_REGISTER rip, r11
 	SAVE_REST
 	movq %r11, %r15
+	CFI_REGISTER rip, r15
 	FIXUP_TOP_OF_STACK %r11
 	call sys_execve
 	GET_THREAD_INFO(%rcx)
 	bt $TIF_IA32,threadinfo_flags(%rcx)
+	CFI_REMEMBER_STATE
 	jc exec_32bit
 	RESTORE_TOP_OF_STACK %r11
 	movq %r15, %r11
+	CFI_REGISTER rip, r11
 	RESTORE_REST
-	push %r11
+	pushq %r11
+	CFI_ADJUST_CFA_OFFSET 8
+	CFI_REL_OFFSET rip, 0
 	ret
 
 exec_32bit:
-	CFI_ADJUST_CFA_OFFSET	REST_SKIP
+	CFI_RESTORE_STATE
 	movq %rax,RAX(%rsp)
 	RESTORE_REST
 	jmp int_ret_from_sys_call
@@ -382,7 +436,8 @@ exec_32bit:
  */                
 ENTRY(stub_rt_sigreturn)
 	CFI_STARTPROC
-	addq $8, %rsp		
+	addq $8, %rsp
+	CFI_ADJUST_CFA_OFFSET	-8
 	SAVE_REST
 	movq %rsp,%rdi
 	FIXUP_TOP_OF_STACK %r11
@@ -392,6 +447,25 @@ ENTRY(stub_rt_sigreturn)
 	jmp int_ret_from_sys_call
 	CFI_ENDPROC
 
+/*
+ * initial frame state for interrupts and exceptions
+ */
+	.macro _frame ref
+	CFI_STARTPROC simple
+	CFI_DEF_CFA rsp,SS+8-\ref
+	/*CFI_REL_OFFSET ss,SS-\ref*/
+	CFI_REL_OFFSET rsp,RSP-\ref
+	/*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
+	/*CFI_REL_OFFSET cs,CS-\ref*/
+	CFI_REL_OFFSET rip,RIP-\ref
+	.endm
+
+/* initial frame state for interrupts (and exceptions without error code) */
+#define INTR_FRAME _frame RIP
+/* initial frame state for exceptions with error code (and interrupts with
+   vector already pushed) */
+#define XCPT_FRAME _frame ORIG_RAX
+
 /* 
  * Interrupt entry/exit.
  *
@@ -402,10 +476,6 @@ ENTRY(stub_rt_sigreturn)
 
 /* 0(%rsp): interrupt number */ 
 	.macro interrupt func
-	CFI_STARTPROC	simple
-	CFI_DEF_CFA	rsp,(SS-RDI)
-	CFI_REL_OFFSET	rsp,(RSP-ORIG_RAX)
-	CFI_REL_OFFSET	rip,(RIP-ORIG_RAX)
 	cld
 #ifdef CONFIG_DEBUG_INFO
 	SAVE_ALL	
@@ -425,23 +495,27 @@ ENTRY(stub_rt_sigreturn)
 	swapgs	
 1:	incl	%gs:pda_irqcount	# RED-PEN should check preempt count
 	movq %gs:pda_irqstackptr,%rax
-	cmoveq %rax,%rsp							
+	cmoveq %rax,%rsp /*todo This needs CFI annotation! */
 	pushq %rdi			# save old stack	
+	CFI_ADJUST_CFA_OFFSET	8
 	call \func
 	.endm
 
 ENTRY(common_interrupt)
+	XCPT_FRAME
 	interrupt do_IRQ
 	/* 0(%rsp): oldrsp-ARGOFFSET */
-ret_from_intr:		
+ret_from_intr:
 	popq  %rdi
+	CFI_ADJUST_CFA_OFFSET	-8
 	cli	
 	decl %gs:pda_irqcount
 #ifdef CONFIG_DEBUG_INFO
 	movq RBP(%rdi),%rbp
+	CFI_DEF_CFA_REGISTER	rsp
 #endif
-	leaq ARGOFFSET(%rdi),%rsp
-exit_intr:	 	
+	leaq ARGOFFSET(%rdi),%rsp /*todo This needs CFI annotation! */
+exit_intr:
 	GET_THREAD_INFO(%rcx)
 	testl $3,CS-ARGOFFSET(%rsp)
 	je retint_kernel
@@ -453,9 +527,10 @@ exit_intr:
 	 */		
 retint_with_reschedule:
 	movl $_TIF_WORK_MASK,%edi
-retint_check:			
+retint_check:
 	movl threadinfo_flags(%rcx),%edx
 	andl %edi,%edx
+	CFI_REMEMBER_STATE
 	jnz  retint_careful
 retint_swapgs:	 	
 	swapgs 
@@ -476,14 +551,17 @@ bad_iret:
 	jmp do_exit			
 	.previous	
 	
-	/* edi: workmask, edx: work */	
+	/* edi: workmask, edx: work */
 retint_careful:
+	CFI_RESTORE_STATE
 	bt    $TIF_NEED_RESCHED,%edx
 	jnc   retint_signal
 	sti
 	pushq %rdi
+	CFI_ADJUST_CFA_OFFSET	8
 	call  schedule
 	popq %rdi		
+	CFI_ADJUST_CFA_OFFSET	-8
 	GET_THREAD_INFO(%rcx)
 	cli
 	jmp retint_check
@@ -523,7 +601,9 @@ retint_kernel:
  * APIC interrupts.
  */		
 	.macro apicinterrupt num,func
+	INTR_FRAME
 	pushq $\num-256
+	CFI_ADJUST_CFA_OFFSET 8
 	interrupt \func
 	jmp ret_from_intr
 	CFI_ENDPROC
@@ -569,16 +649,23 @@ ENTRY(spurious_interrupt)
  * Exception entry points.
  */ 		
 	.macro zeroentry sym
+	INTR_FRAME
 	pushq $0	/* push error code/oldrax */ 
+	CFI_ADJUST_CFA_OFFSET 8
 	pushq %rax	/* push real oldrax to the rdi slot */ 
+	CFI_ADJUST_CFA_OFFSET 8
 	leaq  \sym(%rip),%rax
 	jmp error_entry
+	CFI_ENDPROC
 	.endm	
 
 	.macro errorentry sym
+	XCPT_FRAME
 	pushq %rax
+	CFI_ADJUST_CFA_OFFSET 8
 	leaq  \sym(%rip),%rax
 	jmp error_entry
+	CFI_ENDPROC
 	.endm
 
 	/* error code is on the stack already */
@@ -605,10 +692,7 @@ ENTRY(spurious_interrupt)
  * and the exception handler in %rax.	
  */ 		  				
 ENTRY(error_entry)
-	CFI_STARTPROC	simple
-	CFI_DEF_CFA	rsp,(SS-RDI)
-	CFI_REL_OFFSET	rsp,(RSP-RDI)
-	CFI_REL_OFFSET	rip,(RIP-RDI)
+	_frame RDI
 	/* rdi slot contains rax, oldrax contains error code */
 	cld	
 	subq  $14*8,%rsp
@@ -690,7 +774,9 @@ error_kernelspace:
        /* Reload gs selector with exception handling */
        /* edi:  new selector */ 
 ENTRY(load_gs_index)
+	CFI_STARTPROC
 	pushf
+	CFI_ADJUST_CFA_OFFSET 8
 	cli
         swapgs
 gs_change:     
@@ -698,7 +784,9 @@ gs_change:
 2:	mfence		/* workaround */
 	swapgs
         popf
+	CFI_ADJUST_CFA_OFFSET -8
         ret
+	CFI_ENDPROC
        
         .section __ex_table,"a"
         .align 8
@@ -810,7 +898,7 @@ ENTRY(device_not_available)
 
 	/* runs on exception stack */
 KPROBE_ENTRY(debug)
-	CFI_STARTPROC
+ 	INTR_FRAME
 	pushq $0
 	CFI_ADJUST_CFA_OFFSET 8		
 	paranoidentry do_debug
@@ -820,9 +908,9 @@ KPROBE_ENTRY(debug)
 
 	/* runs on exception stack */	
 ENTRY(nmi)
-	CFI_STARTPROC
+	INTR_FRAME
 	pushq $-1
-	CFI_ADJUST_CFA_OFFSET 8		
+	CFI_ADJUST_CFA_OFFSET 8
 	paranoidentry do_nmi
 	/*
  	 * "Paranoid" exit path from exception stack.
@@ -888,7 +976,7 @@ ENTRY(reserved)
 
 	/* runs on exception stack */
 ENTRY(double_fault)
-	CFI_STARTPROC
+	XCPT_FRAME
 	paranoidentry do_double_fault
 	jmp paranoid_exit
 	CFI_ENDPROC
@@ -901,7 +989,7 @@ ENTRY(segment_not_present)
 
 	/* runs on exception stack */
 ENTRY(stack_segment)
-	CFI_STARTPROC
+	XCPT_FRAME
 	paranoidentry do_stack_segment
 	jmp paranoid_exit
 	CFI_ENDPROC
@@ -922,7 +1010,7 @@ ENTRY(spurious_interrupt_bug)
 #ifdef CONFIG_X86_MCE
 	/* runs on exception stack */
 ENTRY(machine_check)
-	CFI_STARTPROC
+	INTR_FRAME
 	pushq $0
 	CFI_ADJUST_CFA_OFFSET 8	
 	paranoidentry do_machine_check
@@ -934,14 +1022,19 @@ ENTRY(call_debug)
        zeroentry do_call_debug
 
 ENTRY(call_softirq)
+	CFI_STARTPROC
 	movq %gs:pda_irqstackptr,%rax
 	pushq %r15
+	CFI_ADJUST_CFA_OFFSET 8
 	movq %rsp,%r15
+	CFI_DEF_CFA_REGISTER	r15
 	incl %gs:pda_irqcount
 	cmove %rax,%rsp
 	call __do_softirq
 	movq %r15,%rsp
+	CFI_DEF_CFA_REGISTER	rsp
 	decl %gs:pda_irqcount
 	popq %r15
+	CFI_ADJUST_CFA_OFFSET -8
 	ret
-
+	CFI_ENDPROC