[PATCH] x86_64: Clean up copy_*_user

- Remove optimization for old B stepping Opteron
- Make the fast path for copies with a multiple of eight length faster.
- Minor instruction rearrangement to hopefully avoid a pipeline
stall or two.
- Add comment about errata to consider.

Signed-off-by: Andi Kleen <ak@suse.de>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

1 files changed, 23 insertions, 221 deletions

diff --git a/arch/x86_64/lib/copy_user.S b/arch/x86_64/lib/copy_user.S
index dfa358b05c89..79422b6559c3 100644
--- a/arch/x86_64/lib/copy_user.S
+++ b/arch/x86_64/lib/copy_user.S
@@ -4,12 +4,9 @@
  * Functions to copy from and to user space.		
  */		 
 
-#define FIX_ALIGNMENT 1
-		
 	#include <asm/current.h>
 	#include <asm/asm-offsets.h>
 	#include <asm/thread_info.h>
-	#include <asm/cpufeature.h>
 
 /* Standard copy_to_user with segment limit checking */		
 	.globl copy_to_user
@@ -21,23 +18,7 @@ copy_to_user:
 	jc  bad_to_user
 	cmpq threadinfo_addr_limit(%rax),%rcx
 	jae bad_to_user
-2:	
-	.byte 0xe9	/* 32bit jump */
-	.long .Lcug-1f
-1:
-
-	.section .altinstr_replacement,"ax"
-3:	.byte 0xe9			/* replacement jmp with 8 bit immediate */
-	.long copy_user_generic_c-1b	/* offset */
-	.previous
-	.section .altinstructions,"a"
-	.align 8
-	.quad  2b
-	.quad  3b
-	.byte  X86_FEATURE_K8_C
-	.byte  5
-	.byte  5
-	.previous
+	jmp copy_user_generic
 
 /* Standard copy_from_user with segment limit checking */	
 	.globl copy_from_user
@@ -72,223 +53,44 @@ bad_to_user:
  * rsi source
  * rdx count
  *
+ * Only 4GB of copy is supported. This shouldn't be a problem
+ * because the kernel normally only writes from/to page sized chunks
+ * even if user space passed a longer buffer.
+ * And more would be dangerous because both Intel and AMD have
+ * errata with rep movsq > 4GB. If someone feels the need to fix
+ * this please consider this.
+ *
  * Output:		
  * eax uncopied bytes or 0 if successful.
  */
-	.globl copy_user_generic	
-	.p2align 4
-copy_user_generic:	
-	.byte 0x66,0x66,0x90	/* 5 byte nop for replacement jump */	
-	.byte 0x66,0x90
-1:		
-	.section .altinstr_replacement,"ax"
-2:	.byte 0xe9	             /* near jump with 32bit immediate */
-	.long copy_user_generic_c-1b /* offset */
-	.previous
-	.section .altinstructions,"a"
-	.align 8
-	.quad  copy_user_generic
-	.quad  2b
-	.byte  X86_FEATURE_K8_C
-	.byte  5
-	.byte  5
-	.previous
-.Lcug:	
-	pushq %rbx
-	xorl %eax,%eax		/*zero for the exception handler */
-
-#ifdef FIX_ALIGNMENT
-	/* check for bad alignment of destination */
-	movl %edi,%ecx
-	andl $7,%ecx
-	jnz  .Lbad_alignment
-.Lafter_bad_alignment:
-#endif
 
-	movq %rdx,%rcx
-
-	movl $64,%ebx	
-	shrq $6,%rdx
-	decq %rdx
-	js   .Lhandle_tail
-	
-	.p2align 4
-.Lloop:
-.Ls1:	movq (%rsi),%r11
-.Ls2:	movq 1*8(%rsi),%r8
-.Ls3:	movq 2*8(%rsi),%r9
-.Ls4:	movq 3*8(%rsi),%r10
-.Ld1:	movq %r11,(%rdi)
-.Ld2:	movq %r8,1*8(%rdi)
-.Ld3:	movq %r9,2*8(%rdi)
-.Ld4:	movq %r10,3*8(%rdi)
-		
-.Ls5:	movq 4*8(%rsi),%r11
-.Ls6:	movq 5*8(%rsi),%r8
-.Ls7:	movq 6*8(%rsi),%r9
-.Ls8:	movq 7*8(%rsi),%r10
-.Ld5:	movq %r11,4*8(%rdi)
-.Ld6:	movq %r8,5*8(%rdi)
-.Ld7:	movq %r9,6*8(%rdi)
-.Ld8:	movq %r10,7*8(%rdi)
-	
-	decq %rdx
-
-	leaq 64(%rsi),%rsi
-	leaq 64(%rdi),%rdi
-	
-	jns  .Lloop
-
-	.p2align 4
-.Lhandle_tail:
-	movl %ecx,%edx
-	andl $63,%ecx
-	shrl $3,%ecx
-	jz   .Lhandle_7
-	movl $8,%ebx
-	.p2align 4
-.Lloop_8:
-.Ls9:	movq (%rsi),%r8
-.Ld9:	movq %r8,(%rdi)
-	decl %ecx
-	leaq 8(%rdi),%rdi
-	leaq 8(%rsi),%rsi
-	jnz .Lloop_8
-	
-.Lhandle_7:		
-	movl %edx,%ecx	
-	andl $7,%ecx
-	jz   .Lende
-	.p2align 4
-.Lloop_1:
-.Ls10:	movb (%rsi),%bl
-.Ld10:	movb %bl,(%rdi)
-	incq %rdi
-	incq %rsi
-	decl %ecx
-	jnz .Lloop_1
-			
-.Lende:
-	popq %rbx
-	ret	
-
-#ifdef FIX_ALIGNMENT		  		
-	/* align destination */
-	.p2align 4
-.Lbad_alignment:
-	movl $8,%r9d
-	subl %ecx,%r9d
-	movl %r9d,%ecx
-	cmpq %r9,%rdx
-	jz   .Lhandle_7
-	js   .Lhandle_7
-.Lalign_1:		
-.Ls11:	movb (%rsi),%bl
-.Ld11:	movb %bl,(%rdi)
-	incq %rsi
-	incq %rdi
-	decl %ecx
-	jnz .Lalign_1
-	subq %r9,%rdx
-	jmp .Lafter_bad_alignment
-#endif
-	
-	/* table sorted by exception address */	
-	.section __ex_table,"a"
-	.align 8
-	.quad .Ls1,.Ls1e
-	.quad .Ls2,.Ls2e
-	.quad .Ls3,.Ls3e
-	.quad .Ls4,.Ls4e	
-	.quad .Ld1,.Ls1e
-	.quad .Ld2,.Ls2e
-	.quad .Ld3,.Ls3e
-	.quad .Ld4,.Ls4e
-	.quad .Ls5,.Ls5e
-	.quad .Ls6,.Ls6e
-	.quad .Ls7,.Ls7e
-	.quad .Ls8,.Ls8e	
-	.quad .Ld5,.Ls5e
-	.quad .Ld6,.Ls6e
-	.quad .Ld7,.Ls7e
-	.quad .Ld8,.Ls8e
-	.quad .Ls9,.Le_quad
-	.quad .Ld9,.Le_quad
-	.quad .Ls10,.Le_byte
-	.quad .Ld10,.Le_byte
-#ifdef FIX_ALIGNMENT	
-	.quad .Ls11,.Lzero_rest
-	.quad .Ld11,.Lzero_rest
-#endif
-	.quad .Le5,.Le_zero
-	.previous
-
-	/* compute 64-offset for main loop. 8 bytes accuracy with error on the 
-	   pessimistic side. this is gross. it would be better to fix the 
-	   interface. */	
-	/* eax: zero, ebx: 64 */
-.Ls1e: 	addl $8,%eax
-.Ls2e: 	addl $8,%eax
-.Ls3e: 	addl $8,%eax
-.Ls4e: 	addl $8,%eax
-.Ls5e: 	addl $8,%eax
-.Ls6e: 	addl $8,%eax
-.Ls7e: 	addl $8,%eax
-.Ls8e: 	addl $8,%eax
-	addq %rbx,%rdi	/* +64 */
-	subq %rax,%rdi  /* correct destination with computed offset */
-
-	shlq $6,%rdx	/* loop counter * 64 (stride length) */
-	addq %rax,%rdx	/* add offset to loopcnt */
-	andl $63,%ecx	/* remaining bytes */
-	addq %rcx,%rdx	/* add them */
-	jmp .Lzero_rest
-
-	/* exception on quad word loop in tail handling */
-	/* ecx:	loopcnt/8, %edx: length, rdi: correct */
-.Le_quad:
-	shll $3,%ecx
-	andl $7,%edx
-	addl %ecx,%edx
-	/* edx: bytes to zero, rdi: dest, eax:zero */
-.Lzero_rest:
-	movq %rdx,%rcx
-.Le_byte:
-	xorl %eax,%eax
-.Le5:	rep 
-	stosb
-	/* when there is another exception while zeroing the rest just return */
-.Le_zero:		
-	movq %rdx,%rax
-	jmp .Lende
-
-	/* C stepping K8 run faster using the string copy instructions.
-	   This is also a lot simpler. Use them when possible.
-	   Patch in jmps to this code instead of copying it fully
-	   to avoid unwanted aliasing in the exception tables. */
-		
- /* rdi	destination
-  * rsi source
-  * rdx count
-  *
-  * Output:		
-  * eax uncopied bytes or 0 if successfull.
-  */			
-copy_user_generic_c:
+	.globl copy_user_generic
+copy_user_generic:
 	movl %edx,%ecx
 	shrl $3,%ecx
 	andl $7,%edx	
+	jz   5f
 1:	rep 
 	movsq 
 	movl %edx,%ecx
+	xor %eax,%eax
 2:	rep
 	movsb
-4:	movl %ecx,%eax
 	ret
+	/* align here? */
+5: 	xorl %eax,%eax
+6:	rep movsq
+	ret
+
+	.section .fixup,"ax"
 3:	lea (%rdx,%rcx,8),%rax
 	ret
-	
+4:	movl %ecx,%eax
+	ret
+	.previous
+
 	.section __ex_table,"a"
 	.quad 1b,3b
 	.quad 2b,4b
+	.quad 6b,4b
 	.previous