Commit 865c50e1 authored by Nick Desaulniers's avatar Nick Desaulniers Committed by Linus Torvalds

x86/uaccess: utilize CONFIG_CC_HAS_ASM_GOTO_OUTPUT

Clang-11 shipped support for outputs to asm goto statments along the
fallthrough path.  Double up some of the get_user() and related macros
to be able to take advantage of this extended GNU C extension. This
should help improve the generated code's performance for these accesses.

Cc: Bill Wendling <morbo@google.com>
Suggested-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: default avatarNick Desaulniers <ndesaulniers@google.com>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent d55564cf
...@@ -310,6 +310,55 @@ do { \ ...@@ -310,6 +310,55 @@ do { \
} \ } \
} while (0) } while (0)
#ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
#ifdef CONFIG_X86_32
#define __get_user_asm_u64(x, ptr, label) do { \
unsigned int __gu_low, __gu_high; \
const unsigned int __user *__gu_ptr; \
__gu_ptr = (const void __user *)(ptr); \
__get_user_asm(__gu_low, ptr, "l", "=r", label); \
__get_user_asm(__gu_high, ptr+1, "l", "=r", label); \
(x) = ((unsigned long long)__gu_high << 32) | __gu_low; \
} while (0)
#else
#define __get_user_asm_u64(x, ptr, label) \
__get_user_asm(x, ptr, "q", "=r", label)
#endif
#define __get_user_size(x, ptr, size, label) \
do { \
__chk_user_ptr(ptr); \
switch (size) { \
unsigned char x_u8__; \
case 1: \
__get_user_asm(x_u8__, ptr, "b", "=q", label); \
(x) = x_u8__; \
break; \
case 2: \
__get_user_asm(x, ptr, "w", "=r", label); \
break; \
case 4: \
__get_user_asm(x, ptr, "l", "=r", label); \
break; \
case 8: \
__get_user_asm_u64(x, ptr, label); \
break; \
default: \
(x) = __get_user_bad(); \
} \
} while (0)
#define __get_user_asm(x, addr, itype, ltype, label) \
asm_volatile_goto("\n" \
"1: mov"itype" %[umem],%[output]\n" \
_ASM_EXTABLE_UA(1b, %l2) \
: [output] ltype(x) \
: [umem] "m" (__m(addr)) \
: : label)
#else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
#define __get_user_asm_u64(x, ptr, retval) \ #define __get_user_asm_u64(x, ptr, retval) \
({ \ ({ \
...@@ -378,6 +427,8 @@ do { \ ...@@ -378,6 +427,8 @@ do { \
: [umem] "m" (__m(addr)), \ : [umem] "m" (__m(addr)), \
[efault] "i" (-EFAULT), "0" (err)) [efault] "i" (-EFAULT), "0" (err))
#endif // CONFIG_CC_ASM_GOTO_OUTPUT
/* FIXME: this hack is definitely wrong -AK */ /* FIXME: this hack is definitely wrong -AK */
struct __large_struct { unsigned long buf[100]; }; struct __large_struct { unsigned long buf[100]; };
#define __m(x) (*(struct __large_struct __user *)(x)) #define __m(x) (*(struct __large_struct __user *)(x))
...@@ -452,6 +503,14 @@ static __must_check __always_inline bool user_access_begin(const void __user *pt ...@@ -452,6 +503,14 @@ static __must_check __always_inline bool user_access_begin(const void __user *pt
#define unsafe_put_user(x, ptr, label) \ #define unsafe_put_user(x, ptr, label) \
__put_user_size((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)), label) __put_user_size((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)), label)
#ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
#define unsafe_get_user(x, ptr, err_label) \
do { \
__inttype(*(ptr)) __gu_val; \
__get_user_size(__gu_val, (ptr), sizeof(*(ptr)), err_label); \
(x) = (__force __typeof__(*(ptr)))__gu_val; \
} while (0)
#else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
#define unsafe_get_user(x, ptr, err_label) \ #define unsafe_get_user(x, ptr, err_label) \
do { \ do { \
int __gu_err; \ int __gu_err; \
...@@ -460,6 +519,7 @@ do { \ ...@@ -460,6 +519,7 @@ do { \
(x) = (__force __typeof__(*(ptr)))__gu_val; \ (x) = (__force __typeof__(*(ptr)))__gu_val; \
if (unlikely(__gu_err)) goto err_label; \ if (unlikely(__gu_err)) goto err_label; \
} while (0) } while (0)
#endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
/* /*
* We want the unsafe accessors to always be inlined and use * We want the unsafe accessors to always be inlined and use
...@@ -486,6 +546,11 @@ do { \ ...@@ -486,6 +546,11 @@ do { \
#define HAVE_GET_KERNEL_NOFAULT #define HAVE_GET_KERNEL_NOFAULT
#ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
#define __get_kernel_nofault(dst, src, type, err_label) \
__get_user_size(*((type *)(dst)), (__force type __user *)(src), \
sizeof(type), err_label)
#else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
#define __get_kernel_nofault(dst, src, type, err_label) \ #define __get_kernel_nofault(dst, src, type, err_label) \
do { \ do { \
int __kr_err; \ int __kr_err; \
...@@ -495,6 +560,7 @@ do { \ ...@@ -495,6 +560,7 @@ do { \
if (unlikely(__kr_err)) \ if (unlikely(__kr_err)) \
goto err_label; \ goto err_label; \
} while (0) } while (0)
#endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
#define __put_kernel_nofault(dst, src, type, err_label) \ #define __put_kernel_nofault(dst, src, type, err_label) \
__put_user_size(*((type *)(src)), (__force type __user *)(dst), \ __put_user_size(*((type *)(src)), (__force type __user *)(dst), \
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment