Commit d6cd4715 authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'x86-rwsem-for-linus' of...

Merge branch 'x86-rwsem-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

* 'x86-rwsem-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip:
  x86-64, rwsem: Avoid store forwarding hazard in __downgrade_write
  x86-64, rwsem: 64-bit xadd rwsem implementation
  x86: Fix breakage of UML from the changes in the rwsem system
  x86-64: support native xadd rwsem implementation
  x86: clean up rwsem type system
parents 1eca9acb 0d1622d7
...@@ -8,7 +8,8 @@ obj-y = bug.o bugs.o delay.o fault.o ldt.o mem.o ptrace.o ptrace_user.o \ ...@@ -8,7 +8,8 @@ obj-y = bug.o bugs.o delay.o fault.o ldt.o mem.o ptrace.o ptrace_user.o \
setjmp.o signal.o stub.o stub_segv.o syscalls.o syscall_table.o \ setjmp.o signal.o stub.o stub_segv.o syscalls.o syscall_table.o \
sysrq.o ksyms.o tls.o sysrq.o ksyms.o tls.o
subarch-obj-y = lib/csum-partial_64.o lib/memcpy_64.o lib/thunk_64.o subarch-obj-y = lib/csum-partial_64.o lib/memcpy_64.o lib/thunk_64.o \
lib/rwsem_64.o
subarch-obj-$(CONFIG_MODULES) += kernel/module.o subarch-obj-$(CONFIG_MODULES) += kernel/module.o
ldt-y = ../sys-i386/ldt.o ldt-y = ../sys-i386/ldt.o
......
...@@ -319,7 +319,7 @@ config X86_L1_CACHE_SHIFT ...@@ -319,7 +319,7 @@ config X86_L1_CACHE_SHIFT
config X86_XADD config X86_XADD
def_bool y def_bool y
depends on X86_32 && !M386 depends on X86_64 || !M386
config X86_PPRO_FENCE config X86_PPRO_FENCE
bool "PentiumPro memory ordering errata workaround" bool "PentiumPro memory ordering errata workaround"
......
...@@ -41,6 +41,7 @@ ...@@ -41,6 +41,7 @@
#include <linux/list.h> #include <linux/list.h>
#include <linux/spinlock.h> #include <linux/spinlock.h>
#include <linux/lockdep.h> #include <linux/lockdep.h>
#include <asm/asm.h>
struct rwsem_waiter; struct rwsem_waiter;
...@@ -55,17 +56,28 @@ extern asmregparm struct rw_semaphore * ...@@ -55,17 +56,28 @@ extern asmregparm struct rw_semaphore *
/* /*
* the semaphore definition * the semaphore definition
*
* The bias values and the counter type limits the number of
* potential readers/writers to 32767 for 32 bits and 2147483647
* for 64 bits.
*/ */
#define RWSEM_UNLOCKED_VALUE 0x00000000 #ifdef CONFIG_X86_64
#define RWSEM_ACTIVE_BIAS 0x00000001 # define RWSEM_ACTIVE_MASK 0xffffffffL
#define RWSEM_ACTIVE_MASK 0x0000ffff #else
#define RWSEM_WAITING_BIAS (-0x00010000) # define RWSEM_ACTIVE_MASK 0x0000ffffL
#endif
#define RWSEM_UNLOCKED_VALUE 0x00000000L
#define RWSEM_ACTIVE_BIAS 0x00000001L
#define RWSEM_WAITING_BIAS (-RWSEM_ACTIVE_MASK-1)
#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS #define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS
#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
typedef signed long rwsem_count_t;
struct rw_semaphore { struct rw_semaphore {
signed long count; rwsem_count_t count;
spinlock_t wait_lock; spinlock_t wait_lock;
struct list_head wait_list; struct list_head wait_list;
#ifdef CONFIG_DEBUG_LOCK_ALLOC #ifdef CONFIG_DEBUG_LOCK_ALLOC
...@@ -105,7 +117,7 @@ do { \ ...@@ -105,7 +117,7 @@ do { \
static inline void __down_read(struct rw_semaphore *sem) static inline void __down_read(struct rw_semaphore *sem)
{ {
asm volatile("# beginning down_read\n\t" asm volatile("# beginning down_read\n\t"
LOCK_PREFIX " inc%z0 (%1)\n\t" LOCK_PREFIX _ASM_INC "(%1)\n\t"
/* adds 0x00000001, returns the old value */ /* adds 0x00000001, returns the old value */
" jns 1f\n" " jns 1f\n"
" call call_rwsem_down_read_failed\n" " call call_rwsem_down_read_failed\n"
...@@ -121,7 +133,7 @@ static inline void __down_read(struct rw_semaphore *sem) ...@@ -121,7 +133,7 @@ static inline void __down_read(struct rw_semaphore *sem)
*/ */
static inline int __down_read_trylock(struct rw_semaphore *sem) static inline int __down_read_trylock(struct rw_semaphore *sem)
{ {
__s32 result, tmp; rwsem_count_t result, tmp;
asm volatile("# beginning __down_read_trylock\n\t" asm volatile("# beginning __down_read_trylock\n\t"
" mov %0,%1\n\t" " mov %0,%1\n\t"
"1:\n\t" "1:\n\t"
...@@ -143,7 +155,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem) ...@@ -143,7 +155,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)
*/ */
static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
{ {
int tmp; rwsem_count_t tmp;
tmp = RWSEM_ACTIVE_WRITE_BIAS; tmp = RWSEM_ACTIVE_WRITE_BIAS;
asm volatile("# beginning down_write\n\t" asm volatile("# beginning down_write\n\t"
...@@ -170,9 +182,9 @@ static inline void __down_write(struct rw_semaphore *sem) ...@@ -170,9 +182,9 @@ static inline void __down_write(struct rw_semaphore *sem)
*/ */
static inline int __down_write_trylock(struct rw_semaphore *sem) static inline int __down_write_trylock(struct rw_semaphore *sem)
{ {
signed long ret = cmpxchg(&sem->count, rwsem_count_t ret = cmpxchg(&sem->count,
RWSEM_UNLOCKED_VALUE, RWSEM_UNLOCKED_VALUE,
RWSEM_ACTIVE_WRITE_BIAS); RWSEM_ACTIVE_WRITE_BIAS);
if (ret == RWSEM_UNLOCKED_VALUE) if (ret == RWSEM_UNLOCKED_VALUE)
return 1; return 1;
return 0; return 0;
...@@ -183,7 +195,7 @@ static inline int __down_write_trylock(struct rw_semaphore *sem) ...@@ -183,7 +195,7 @@ static inline int __down_write_trylock(struct rw_semaphore *sem)
*/ */
static inline void __up_read(struct rw_semaphore *sem) static inline void __up_read(struct rw_semaphore *sem)
{ {
__s32 tmp = -RWSEM_ACTIVE_READ_BIAS; rwsem_count_t tmp = -RWSEM_ACTIVE_READ_BIAS;
asm volatile("# beginning __up_read\n\t" asm volatile("# beginning __up_read\n\t"
LOCK_PREFIX " xadd %1,(%2)\n\t" LOCK_PREFIX " xadd %1,(%2)\n\t"
/* subtracts 1, returns the old value */ /* subtracts 1, returns the old value */
...@@ -201,7 +213,7 @@ static inline void __up_read(struct rw_semaphore *sem) ...@@ -201,7 +213,7 @@ static inline void __up_read(struct rw_semaphore *sem)
*/ */
static inline void __up_write(struct rw_semaphore *sem) static inline void __up_write(struct rw_semaphore *sem)
{ {
unsigned long tmp; rwsem_count_t tmp;
asm volatile("# beginning __up_write\n\t" asm volatile("# beginning __up_write\n\t"
LOCK_PREFIX " xadd %1,(%2)\n\t" LOCK_PREFIX " xadd %1,(%2)\n\t"
/* tries to transition /* tries to transition
...@@ -221,33 +233,38 @@ static inline void __up_write(struct rw_semaphore *sem) ...@@ -221,33 +233,38 @@ static inline void __up_write(struct rw_semaphore *sem)
static inline void __downgrade_write(struct rw_semaphore *sem) static inline void __downgrade_write(struct rw_semaphore *sem)
{ {
asm volatile("# beginning __downgrade_write\n\t" asm volatile("# beginning __downgrade_write\n\t"
LOCK_PREFIX " add%z0 %2,(%1)\n\t" LOCK_PREFIX _ASM_ADD "%2,(%1)\n\t"
/* transitions 0xZZZZ0001 -> 0xYYYY0001 */ /*
* transitions 0xZZZZ0001 -> 0xYYYY0001 (i386)
* 0xZZZZZZZZ00000001 -> 0xYYYYYYYY00000001 (x86_64)
*/
" jns 1f\n\t" " jns 1f\n\t"
" call call_rwsem_downgrade_wake\n" " call call_rwsem_downgrade_wake\n"
"1:\n\t" "1:\n\t"
"# ending __downgrade_write\n" "# ending __downgrade_write\n"
: "+m" (sem->count) : "+m" (sem->count)
: "a" (sem), "i" (-RWSEM_WAITING_BIAS) : "a" (sem), "er" (-RWSEM_WAITING_BIAS)
: "memory", "cc"); : "memory", "cc");
} }
/* /*
* implement atomic add functionality * implement atomic add functionality
*/ */
static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem) static inline void rwsem_atomic_add(rwsem_count_t delta,
struct rw_semaphore *sem)
{ {
asm volatile(LOCK_PREFIX "add%z0 %1,%0" asm volatile(LOCK_PREFIX _ASM_ADD "%1,%0"
: "+m" (sem->count) : "+m" (sem->count)
: "ir" (delta)); : "er" (delta));
} }
/* /*
* implement exchange and add functionality * implement exchange and add functionality
*/ */
static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem) static inline rwsem_count_t rwsem_atomic_update(rwsem_count_t delta,
struct rw_semaphore *sem)
{ {
int tmp = delta; rwsem_count_t tmp = delta;
asm volatile(LOCK_PREFIX "xadd %0,%1" asm volatile(LOCK_PREFIX "xadd %0,%1"
: "+r" (tmp), "+m" (sem->count) : "+r" (tmp), "+m" (sem->count)
......
...@@ -39,4 +39,5 @@ else ...@@ -39,4 +39,5 @@ else
lib-y += thunk_64.o clear_page_64.o copy_page_64.o lib-y += thunk_64.o clear_page_64.o copy_page_64.o
lib-y += memmove_64.o memset_64.o lib-y += memmove_64.o memset_64.o
lib-y += copy_user_64.o rwlock_64.o copy_user_nocache_64.o lib-y += copy_user_64.o rwlock_64.o copy_user_nocache_64.o
lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem_64.o
endif endif
/*
* x86-64 rwsem wrappers
*
* This interfaces the inline asm code to the slow-path
* C routines. We need to save the call-clobbered regs
* that the asm does not mark as clobbered, and move the
* argument from %rax to %rdi.
*
* NOTE! We don't need to save %rax, because the functions
* will always return the semaphore pointer in %rax (which
* is also the input argument to these helpers)
*
* The following can clobber %rdx because the asm clobbers it:
* call_rwsem_down_write_failed
* call_rwsem_wake
* but %rdi, %rsi, %rcx, %r8-r11 always need saving.
*/
#include <linux/linkage.h>
#include <asm/rwlock.h>
#include <asm/alternative-asm.h>
#include <asm/frame.h>
#include <asm/dwarf2.h>
#define save_common_regs \
pushq %rdi; \
pushq %rsi; \
pushq %rcx; \
pushq %r8; \
pushq %r9; \
pushq %r10; \
pushq %r11
#define restore_common_regs \
popq %r11; \
popq %r10; \
popq %r9; \
popq %r8; \
popq %rcx; \
popq %rsi; \
popq %rdi
/* Fix up special calling conventions */
ENTRY(call_rwsem_down_read_failed)
save_common_regs
pushq %rdx
movq %rax,%rdi
call rwsem_down_read_failed
popq %rdx
restore_common_regs
ret
ENDPROC(call_rwsem_down_read_failed)
ENTRY(call_rwsem_down_write_failed)
save_common_regs
movq %rax,%rdi
call rwsem_down_write_failed
restore_common_regs
ret
ENDPROC(call_rwsem_down_write_failed)
ENTRY(call_rwsem_wake)
decw %dx /* do nothing if still outstanding active readers */
jnz 1f
save_common_regs
movq %rax,%rdi
call rwsem_wake
restore_common_regs
1: ret
ENDPROC(call_rwsem_wake)
/* Fix up special calling conventions */
ENTRY(call_rwsem_downgrade_wake)
save_common_regs
pushq %rdx
movq %rax,%rdi
call rwsem_downgrade_wake
popq %rdx
restore_common_regs
ret
ENDPROC(call_rwsem_downgrade_wake)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment