[PATCH] inline 1,2 and 4-byte copy_*_user operations

The patch arranges for constant 1, 2 and 4-byte copy_*_user() invokations to
be inlined.

It's hard to tell really, but the AIM9 creat_clo, signal_test and dir_rtns_1
numbers went up by 3%-9%, which is to be expected.

arch/i386/kernel/i386_ksyms.c

@@ -118,10 +118,8 @@ EXPORT_SYMBOL(strncpy_from_user);
 EXPORT_SYMBOL(__strncpy_from_user);
 EXPORT_SYMBOL(clear_user);
 EXPORT_SYMBOL(__clear_user);
-EXPORT_SYMBOL(copy_from_user);
-EXPORT_SYMBOL(__copy_from_user);
-EXPORT_SYMBOL(copy_to_user);
-EXPORT_SYMBOL(__copy_to_user);
+EXPORT_SYMBOL(__copy_from_user_ll);
+EXPORT_SYMBOL(__copy_to_user_ll);
 EXPORT_SYMBOL(strnlen_user);
 
 EXPORT_SYMBOL(dma_alloc_coherent);

arch/i386/lib/usercopy.c

@@ -417,7 +417,7 @@ do {									\
 } while (0)
 
 
-unsigned long __copy_to_user(void *to, const void *from, unsigned long n)
+unsigned long __copy_to_user_ll(void *to, const void *from, unsigned long n)
 {
 	if (movsl_is_ok(to, from, n))
 		__copy_user(to, from, n);
@@ -426,7 +426,7 @@ unsigned long __copy_to_user(void *to, const void *from, unsigned long n)
 	return n;
 }
 
-unsigned long __copy_from_user(void *to, const void *from, unsigned long n)
+unsigned long __copy_from_user_ll(void *to, const void *from, unsigned long n)
 {
 	if (movsl_is_ok(to, from, n))
 		__copy_user_zeroing(to, from, n);
@@ -434,19 +434,3 @@ unsigned long __copy_from_user(void *to, const void *from, unsigned long n)
 		n = __copy_user_zeroing_intel(to, from, n);
 	return n;
 }
-
-unsigned long copy_to_user(void *to, const void *from, unsigned long n)
-{
-	prefetch(from);
-	if (access_ok(VERIFY_WRITE, to, n))
-		n = __copy_to_user(to, from, n);
-	return n;
-}
-
-unsigned long copy_from_user(void *to, const void *from, unsigned long n)
-{
-	prefetchw(to);
-	if (access_ok(VERIFY_READ, from, n))
-		n = __copy_from_user(to, from, n);
-	return n;
-}

include/asm-i386/uaccess.h

@@ -147,7 +147,7 @@ extern void __put_user_bad(void);
 #define __put_user_nocheck(x,ptr,size)				\
 ({								\
 	long __pu_err;						\
-	__put_user_size((x),(ptr),(size),__pu_err);	\
+	__put_user_size((x),(ptr),(size),__pu_err,-EFAULT);	\
 	__pu_err;						\
 })
 
@@ -157,7 +157,7 @@ extern void __put_user_bad(void);
 	long __pu_err = -EFAULT;					\
 	__typeof__(*(ptr)) *__pu_addr = (ptr);				\
 	if (access_ok(VERIFY_WRITE,__pu_addr,size))			\
-		__put_user_size((x),__pu_addr,(size),__pu_err);	\
+		__put_user_size((x),__pu_addr,(size),__pu_err,-EFAULT);	\
 	__pu_err;							\
 })							
 
@@ -178,14 +178,14 @@ extern void __put_user_bad(void);
 		: "=r"(err)					\
 		: "A" (x), "r" (addr), "i"(-EFAULT), "0"(err))
 
-#define __put_user_size(x,ptr,size,retval)				\
+#define __put_user_size(x,ptr,size,retval,errret)			\
 do {									\
 	retval = 0;							\
 	switch (size) {							\
-	  case 1: __put_user_asm(x,ptr,retval,"b","b","iq"); break;	\
-	  case 2: __put_user_asm(x,ptr,retval,"w","w","ir"); break;	\
-	  case 4: __put_user_asm(x,ptr,retval,"l","","ir"); break;	\
-	  case 8: __put_user_u64((__typeof__(*ptr))(x),ptr,retval); break;	\
+	case 1: __put_user_asm(x,ptr,retval,"b","b","iq",errret);break;	\
+	case 2: __put_user_asm(x,ptr,retval,"w","w","ir",errret);break; \
+	case 4: __put_user_asm(x,ptr,retval,"l","","ir",errret); break;	\
+	case 8: __put_user_u64((__typeof__(*ptr))(x),ptr,retval); break;\
 	  default: __put_user_bad();					\
 	}								\
 } while (0)
@@ -198,7 +198,7 @@ struct __large_struct { unsigned long buf[100]; };
  * we do not write to any memory gcc knows about, so there are no
  * aliasing issues.
  */
-#define __put_user_asm(x, addr, err, itype, rtype, ltype)	\
+#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret)	\
 	__asm__ __volatile__(						\
 		"1:	mov"itype" %"rtype"1,%2\n"			\
 		"2:\n"							\
@@ -211,31 +211,31 @@ struct __large_struct { unsigned long buf[100]; };
 		"	.long 1b,3b\n"					\
 		".previous"						\
 		: "=r"(err)						\
-		: ltype (x), "m"(__m(addr)), "i"(-EFAULT), "0"(err))
+		: ltype (x), "m"(__m(addr)), "i"(errret), "0"(err))
 
 
 #define __get_user_nocheck(x,ptr,size)				\
 ({								\
 	long __gu_err, __gu_val;				\
-	__get_user_size(__gu_val,(ptr),(size),__gu_err);	\
+	__get_user_size(__gu_val,(ptr),(size),__gu_err,-EFAULT);\
 	(x) = (__typeof__(*(ptr)))__gu_val;			\
 	__gu_err;						\
 })
 
 extern long __get_user_bad(void);
 
-#define __get_user_size(x,ptr,size,retval)				\
+#define __get_user_size(x,ptr,size,retval,errret)			\
 do {									\
 	retval = 0;							\
 	switch (size) {							\
-	  case 1: __get_user_asm(x,ptr,retval,"b","b","=q"); break;	\
-	  case 2: __get_user_asm(x,ptr,retval,"w","w","=r"); break;	\
-	  case 4: __get_user_asm(x,ptr,retval,"l","","=r"); break;	\
+	case 1: __get_user_asm(x,ptr,retval,"b","b","=q",errret);break;	\
+	case 2: __get_user_asm(x,ptr,retval,"w","w","=r",errret);break;	\
+	case 4: __get_user_asm(x,ptr,retval,"l","","=r",errret);break;	\
 	default: (x) = __get_user_bad();				\
 	}								\
 } while (0)
 
-#define __get_user_asm(x, addr, err, itype, rtype, ltype)	\
+#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret)	\
 	__asm__ __volatile__(						\
 		"1:	mov"itype" %2,%"rtype"1\n"			\
 		"2:\n"							\
@@ -249,13 +249,75 @@ do {									\
 		"	.long 1b,3b\n"					\
 		".previous"						\
 		: "=r"(err), ltype (x)					\
-		: "m"(__m(addr)), "i"(-EFAULT), "0"(err))
+		: "m"(__m(addr)), "i"(errret), "0"(err))
 
 
-unsigned long copy_to_user(void *to, const void *from, unsigned long n);
-unsigned long copy_from_user(void *to, const void *from, unsigned long n);
-unsigned long __copy_to_user(void *to, const void *from, unsigned long n);
-unsigned long __copy_from_user(void *to, const void *from, unsigned long n);
+unsigned long __copy_to_user_ll(void *to, const void *from, unsigned long n);
+unsigned long __copy_from_user_ll(void *to, const void *from, unsigned long n);
+
+/*
+ * Here we special-case 1, 2 and 4-byte copy_*_user invokations.  On a fault
+ * we return the initial request size (1, 2 or 4), as copy_*_user should do.
+ * If a store crosses a page boundary and gets a fault, the x86 will not write
+ * anything, so this is accurate.
+ */
+static inline unsigned long
+__copy_to_user(void *to, const void *from, unsigned long n)
+{
+	if (__builtin_constant_p(n)) {
+		unsigned long ret;
+
+		switch (n) {
+		case 1:
+			__put_user_size(*(u8 *)from, (u8 *)to, 1, ret, 1);
+			return ret;
+		case 2:
+			__put_user_size(*(u16 *)from, (u16 *)to, 2, ret, 2);
+			return ret;
+		case 4:
+			__put_user_size(*(u32 *)from, (u32 *)to, 4, ret, 4);
+			return ret;
+		}
+	}
+	return __copy_to_user_ll(to, from, n);
+}
+
+static inline unsigned long
+__copy_from_user(void *to, const void *from, unsigned long n)
+{
+	if (__builtin_constant_p(n)) {
+		unsigned long ret;
+
+		switch (n) {
+		case 1:
+			__get_user_size(*(u8 *)to, from, 1, ret, 1);
+			return ret;
+		case 2:
+			__get_user_size(*(u16 *)to, from, 2, ret, 2);
+			return ret;
+		case 4:
+			__get_user_size(*(u32 *)to, from, 4, ret, 4);
+			return ret;
+		}
+	}
+	return __copy_from_user_ll(to, from, n);
+}
+
+static inline unsigned long
+copy_to_user(void *to, const void *from, unsigned long n)
+{
+	if (access_ok(VERIFY_WRITE, to, n))
+		n = __copy_to_user(to, from, n);
+	return n;
+}
+
+static inline unsigned long
+copy_from_user(void *to, const void *from, unsigned long n)
+{
+	if (access_ok(VERIFY_READ, from, n))
+		n = __copy_from_user(to, from, n);
+	return n;
+}
 
 long strncpy_from_user(char *dst, const char *src, long count);
 long __strncpy_from_user(char *dst, const char *src, long count);