[SPARC64]: Fix bugs in new U1memcpy code.

- U1copy_from_user needs PREAMBLE since it uses
  explicit ASI_BLK_AIUS references.
- Need to use EX_RETVAL() in U1memcpy.S
- U1memcpy.S can load one 64-bit word too
  many, passing the source buffer boundary
  and thus potentially causing exceptions.
Signed-off-by: David S. Miller <davem@redhat.com>

arch/sparc64/lib/U1copy_from_user.S

@@ -20,4 +20,14 @@
 #define LOAD_BLK(addr,dest)	ldda [addr] ASI_BLK_AIUS, dest
 #define EX_RETVAL(x)		0
 
+	/* Writing to %asi is _expensive_ so we hardcode it.
+	 * Reading %asi to check for KERNEL_DS is comparatively
+	 * cheap.
+	 */
+#define PREAMBLE					\
+	rd		%asi, %g1;			\
+	cmp		%g1, ASI_AIUS;			\
+	bne,pn		%icc, memcpy_user_stub;		\
+	 nop;						\
+
 #include "U1memcpy.S"

arch/sparc64/lib/U1memcpy.S

@@ -193,6 +193,7 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
 	and		%g2, 7, %g2
 	andncc		%g3, 0x7, %g3
 	fmovd		%f0, %f2
+	sub		%g3, 0x8, %g3
 	sub		%o2, %o4, %o2
 
 	add		%g1, %o4, %g1
@@ -444,7 +445,7 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
 2:	membar		#StoreLoad | #StoreStore
 	VISExit
 	retl
-	 mov		%g5, %o0
+	 mov		EX_RETVAL(%g5), %o0
 
 	.align		64
 70:	/* 16 < len <= (5 * 64) */
@@ -539,7 +540,7 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
 	 add		%o1, 4, %o1
 
 85:	retl
-	 mov		%g5, %o0
+	 mov		EX_RETVAL(%g5), %o0
 
 	.align		32
 90:	EX_LD(LOAD(ldub, %o1, %g1))
@@ -548,4 +549,4 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
 	bgu,pt		%XCC, 90b
 	 add		%o1, 1, %o1
 	retl
-	 mov		%g5, %o0
+	 mov		EX_RETVAL(%g5), %o0