Commit cc76ee75 authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull core locking changes from Ingo Molnar:
 "Main changes:

   - jump label asm preparatory work for PowerPC (Anton Blanchard)

   - rwsem optimizations and cleanups (Davidlohr Bueso)

   - mutex optimizations and cleanups (Jason Low)

   - futex fix (Oleg Nesterov)

   - remove broken atomicity checks from {READ,WRITE}_ONCE() (Peter
     Zijlstra)"

* 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  powerpc, jump_label: Include linux/jump_label.h to get HAVE_JUMP_LABEL define
  jump_label: Allow jump labels to be used in assembly
  jump_label: Allow asm/jump_label.h to be included in assembly
  locking/mutex: Further simplify mutex_spin_on_owner()
  locking: Remove atomicy checks from {READ,WRITE}_ONCE
  locking/rtmutex: Rename argument in the rt_mutex_adjust_prio_chain() documentation as well
  locking/rwsem: Fix lock optimistic spinning when owner is not running
  locking: Remove ACCESS_ONCE() usage
  locking/rwsem: Check for active lock before bailing on spinning
  locking/rwsem: Avoid deceiving lock spinners
  locking/rwsem: Set lock ownership ASAP
  locking/rwsem: Document barrier need when waking tasks
  locking/futex: Check PF_KTHREAD rather than !p->mm to filter out kthreads
  locking/mutex: Refactor mutex_spin_on_owner()
  locking/mutex: In mutex_spin_on_owner(), return true when owner changes
parents 9c65e12a 58995a9a
...@@ -779,6 +779,7 @@ KBUILD_ARFLAGS := $(call ar-option,D) ...@@ -779,6 +779,7 @@ KBUILD_ARFLAGS := $(call ar-option,D)
# check for 'asm goto' # check for 'asm goto'
ifeq ($(shell $(CONFIG_SHELL) $(srctree)/scripts/gcc-goto.sh $(CC)), y) ifeq ($(shell $(CONFIG_SHELL) $(srctree)/scripts/gcc-goto.sh $(CC)), y)
KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO
KBUILD_AFLAGS += -DCC_HAVE_ASM_GOTO
endif endif
include $(srctree)/scripts/Makefile.kasan include $(srctree)/scripts/Makefile.kasan
......
#ifndef _ASM_ARM_JUMP_LABEL_H #ifndef _ASM_ARM_JUMP_LABEL_H
#define _ASM_ARM_JUMP_LABEL_H #define _ASM_ARM_JUMP_LABEL_H
#ifdef __KERNEL__ #ifndef __ASSEMBLY__
#include <linux/types.h> #include <linux/types.h>
...@@ -27,8 +27,6 @@ static __always_inline bool arch_static_branch(struct static_key *key) ...@@ -27,8 +27,6 @@ static __always_inline bool arch_static_branch(struct static_key *key)
return true; return true;
} }
#endif /* __KERNEL__ */
typedef u32 jump_label_t; typedef u32 jump_label_t;
struct jump_entry { struct jump_entry {
...@@ -37,4 +35,5 @@ struct jump_entry { ...@@ -37,4 +35,5 @@ struct jump_entry {
jump_label_t key; jump_label_t key;
}; };
#endif /* __ASSEMBLY__ */
#endif #endif
...@@ -18,11 +18,12 @@ ...@@ -18,11 +18,12 @@
*/ */
#ifndef __ASM_JUMP_LABEL_H #ifndef __ASM_JUMP_LABEL_H
#define __ASM_JUMP_LABEL_H #define __ASM_JUMP_LABEL_H
#ifndef __ASSEMBLY__
#include <linux/types.h> #include <linux/types.h>
#include <asm/insn.h> #include <asm/insn.h>
#ifdef __KERNEL__
#define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE
static __always_inline bool arch_static_branch(struct static_key *key) static __always_inline bool arch_static_branch(struct static_key *key)
...@@ -39,8 +40,6 @@ static __always_inline bool arch_static_branch(struct static_key *key) ...@@ -39,8 +40,6 @@ static __always_inline bool arch_static_branch(struct static_key *key)
return true; return true;
} }
#endif /* __KERNEL__ */
typedef u64 jump_label_t; typedef u64 jump_label_t;
struct jump_entry { struct jump_entry {
...@@ -49,4 +48,5 @@ struct jump_entry { ...@@ -49,4 +48,5 @@ struct jump_entry {
jump_label_t key; jump_label_t key;
}; };
#endif /* __ASSEMBLY__ */
#endif /* __ASM_JUMP_LABEL_H */ #endif /* __ASM_JUMP_LABEL_H */
...@@ -8,9 +8,9 @@ ...@@ -8,9 +8,9 @@
#ifndef _ASM_MIPS_JUMP_LABEL_H #ifndef _ASM_MIPS_JUMP_LABEL_H
#define _ASM_MIPS_JUMP_LABEL_H #define _ASM_MIPS_JUMP_LABEL_H
#include <linux/types.h> #ifndef __ASSEMBLY__
#ifdef __KERNEL__ #include <linux/types.h>
#define JUMP_LABEL_NOP_SIZE 4 #define JUMP_LABEL_NOP_SIZE 4
...@@ -39,8 +39,6 @@ static __always_inline bool arch_static_branch(struct static_key *key) ...@@ -39,8 +39,6 @@ static __always_inline bool arch_static_branch(struct static_key *key)
return true; return true;
} }
#endif /* __KERNEL__ */
#ifdef CONFIG_64BIT #ifdef CONFIG_64BIT
typedef u64 jump_label_t; typedef u64 jump_label_t;
#else #else
...@@ -53,4 +51,5 @@ struct jump_entry { ...@@ -53,4 +51,5 @@ struct jump_entry {
jump_label_t key; jump_label_t key;
}; };
#endif /* __ASSEMBLY__ */
#endif /* _ASM_MIPS_JUMP_LABEL_H */ #endif /* _ASM_MIPS_JUMP_LABEL_H */
...@@ -9,11 +9,11 @@ ...@@ -9,11 +9,11 @@
* 2 of the License, or (at your option) any later version. * 2 of the License, or (at your option) any later version.
*/ */
#include <linux/jump_label.h>
#include <asm/ppc_asm.h> #include <asm/ppc_asm.h>
#include <asm/hvcall.h> #include <asm/hvcall.h>
#include <asm/asm-offsets.h> #include <asm/asm-offsets.h>
#include <asm/opal.h> #include <asm/opal.h>
#include <asm/jump_label.h>
.section ".text" .section ".text"
......
...@@ -7,12 +7,12 @@ ...@@ -7,12 +7,12 @@
* as published by the Free Software Foundation; either version * as published by the Free Software Foundation; either version
* 2 of the License, or (at your option) any later version. * 2 of the License, or (at your option) any later version.
*/ */
#include <linux/jump_label.h>
#include <asm/hvcall.h> #include <asm/hvcall.h>
#include <asm/processor.h> #include <asm/processor.h>
#include <asm/ppc_asm.h> #include <asm/ppc_asm.h>
#include <asm/asm-offsets.h> #include <asm/asm-offsets.h>
#include <asm/ptrace.h> #include <asm/ptrace.h>
#include <asm/jump_label.h>
.section ".text" .section ".text"
......
...@@ -26,7 +26,7 @@ ...@@ -26,7 +26,7 @@
#include <linux/dma-mapping.h> #include <linux/dma-mapping.h>
#include <linux/console.h> #include <linux/console.h>
#include <linux/export.h> #include <linux/export.h>
#include <linux/static_key.h> #include <linux/jump_label.h>
#include <asm/processor.h> #include <asm/processor.h>
#include <asm/mmu.h> #include <asm/mmu.h>
#include <asm/page.h> #include <asm/page.h>
......
#ifndef _ASM_S390_JUMP_LABEL_H #ifndef _ASM_S390_JUMP_LABEL_H
#define _ASM_S390_JUMP_LABEL_H #define _ASM_S390_JUMP_LABEL_H
#ifndef __ASSEMBLY__
#include <linux/types.h> #include <linux/types.h>
#define JUMP_LABEL_NOP_SIZE 6 #define JUMP_LABEL_NOP_SIZE 6
...@@ -39,4 +41,5 @@ struct jump_entry { ...@@ -39,4 +41,5 @@ struct jump_entry {
jump_label_t key; jump_label_t key;
}; };
#endif /* __ASSEMBLY__ */
#endif #endif
#ifndef _ASM_SPARC_JUMP_LABEL_H #ifndef _ASM_SPARC_JUMP_LABEL_H
#define _ASM_SPARC_JUMP_LABEL_H #define _ASM_SPARC_JUMP_LABEL_H
#ifdef __KERNEL__ #ifndef __ASSEMBLY__
#include <linux/types.h> #include <linux/types.h>
...@@ -22,8 +22,6 @@ static __always_inline bool arch_static_branch(struct static_key *key) ...@@ -22,8 +22,6 @@ static __always_inline bool arch_static_branch(struct static_key *key)
return true; return true;
} }
#endif /* __KERNEL__ */
typedef u32 jump_label_t; typedef u32 jump_label_t;
struct jump_entry { struct jump_entry {
...@@ -32,4 +30,5 @@ struct jump_entry { ...@@ -32,4 +30,5 @@ struct jump_entry {
jump_label_t key; jump_label_t key;
}; };
#endif /* __ASSEMBLY__ */
#endif #endif
#ifndef _ASM_X86_JUMP_LABEL_H #ifndef _ASM_X86_JUMP_LABEL_H
#define _ASM_X86_JUMP_LABEL_H #define _ASM_X86_JUMP_LABEL_H
#ifdef __KERNEL__ #ifndef __ASSEMBLY__
#include <linux/stringify.h> #include <linux/stringify.h>
#include <linux/types.h> #include <linux/types.h>
...@@ -30,8 +30,6 @@ static __always_inline bool arch_static_branch(struct static_key *key) ...@@ -30,8 +30,6 @@ static __always_inline bool arch_static_branch(struct static_key *key)
return true; return true;
} }
#endif /* __KERNEL__ */
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
typedef u64 jump_label_t; typedef u64 jump_label_t;
#else #else
...@@ -44,4 +42,5 @@ struct jump_entry { ...@@ -44,4 +42,5 @@ struct jump_entry {
jump_label_t key; jump_label_t key;
}; };
#endif /* __ASSEMBLY__ */
#endif #endif
...@@ -192,29 +192,16 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect); ...@@ -192,29 +192,16 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect);
#include <uapi/linux/types.h> #include <uapi/linux/types.h>
static __always_inline void data_access_exceeds_word_size(void)
#ifdef __compiletime_warning
__compiletime_warning("data access exceeds word size and won't be atomic")
#endif
;
static __always_inline void data_access_exceeds_word_size(void)
{
}
static __always_inline void __read_once_size(const volatile void *p, void *res, int size) static __always_inline void __read_once_size(const volatile void *p, void *res, int size)
{ {
switch (size) { switch (size) {
case 1: *(__u8 *)res = *(volatile __u8 *)p; break; case 1: *(__u8 *)res = *(volatile __u8 *)p; break;
case 2: *(__u16 *)res = *(volatile __u16 *)p; break; case 2: *(__u16 *)res = *(volatile __u16 *)p; break;
case 4: *(__u32 *)res = *(volatile __u32 *)p; break; case 4: *(__u32 *)res = *(volatile __u32 *)p; break;
#ifdef CONFIG_64BIT
case 8: *(__u64 *)res = *(volatile __u64 *)p; break; case 8: *(__u64 *)res = *(volatile __u64 *)p; break;
#endif
default: default:
barrier(); barrier();
__builtin_memcpy((void *)res, (const void *)p, size); __builtin_memcpy((void *)res, (const void *)p, size);
data_access_exceeds_word_size();
barrier(); barrier();
} }
} }
...@@ -225,13 +212,10 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s ...@@ -225,13 +212,10 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s
case 1: *(volatile __u8 *)p = *(__u8 *)res; break; case 1: *(volatile __u8 *)p = *(__u8 *)res; break;
case 2: *(volatile __u16 *)p = *(__u16 *)res; break; case 2: *(volatile __u16 *)p = *(__u16 *)res; break;
case 4: *(volatile __u32 *)p = *(__u32 *)res; break; case 4: *(volatile __u32 *)p = *(__u32 *)res; break;
#ifdef CONFIG_64BIT
case 8: *(volatile __u64 *)p = *(__u64 *)res; break; case 8: *(volatile __u64 *)p = *(__u64 *)res; break;
#endif
default: default:
barrier(); barrier();
__builtin_memcpy((void *)p, (const void *)res, size); __builtin_memcpy((void *)p, (const void *)res, size);
data_access_exceeds_word_size();
barrier(); barrier();
} }
} }
......
...@@ -45,6 +45,12 @@ ...@@ -45,6 +45,12 @@
* same as using STATIC_KEY_INIT_FALSE. * same as using STATIC_KEY_INIT_FALSE.
*/ */
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
# define HAVE_JUMP_LABEL
#endif
#ifndef __ASSEMBLY__
#include <linux/types.h> #include <linux/types.h>
#include <linux/compiler.h> #include <linux/compiler.h>
#include <linux/bug.h> #include <linux/bug.h>
...@@ -55,7 +61,7 @@ extern bool static_key_initialized; ...@@ -55,7 +61,7 @@ extern bool static_key_initialized;
"%s used before call to jump_label_init", \ "%s used before call to jump_label_init", \
__func__) __func__)
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) #ifdef HAVE_JUMP_LABEL
struct static_key { struct static_key {
atomic_t enabled; atomic_t enabled;
...@@ -66,13 +72,18 @@ struct static_key { ...@@ -66,13 +72,18 @@ struct static_key {
#endif #endif
}; };
# include <asm/jump_label.h>
# define HAVE_JUMP_LABEL
#else #else
struct static_key { struct static_key {
atomic_t enabled; atomic_t enabled;
}; };
#endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */ #endif /* HAVE_JUMP_LABEL */
#endif /* __ASSEMBLY__ */
#ifdef HAVE_JUMP_LABEL
#include <asm/jump_label.h>
#endif
#ifndef __ASSEMBLY__
enum jump_label_type { enum jump_label_type {
JUMP_LABEL_DISABLE = 0, JUMP_LABEL_DISABLE = 0,
...@@ -203,3 +214,5 @@ static inline bool static_key_enabled(struct static_key *key) ...@@ -203,3 +214,5 @@ static inline bool static_key_enabled(struct static_key *key)
} }
#endif /* _LINUX_JUMP_LABEL_H */ #endif /* _LINUX_JUMP_LABEL_H */
#endif /* __ASSEMBLY__ */
...@@ -108,7 +108,7 @@ static inline unsigned __read_seqcount_begin(const seqcount_t *s) ...@@ -108,7 +108,7 @@ static inline unsigned __read_seqcount_begin(const seqcount_t *s)
unsigned ret; unsigned ret;
repeat: repeat:
ret = ACCESS_ONCE(s->sequence); ret = READ_ONCE(s->sequence);
if (unlikely(ret & 1)) { if (unlikely(ret & 1)) {
cpu_relax(); cpu_relax();
goto repeat; goto repeat;
...@@ -127,7 +127,7 @@ static inline unsigned __read_seqcount_begin(const seqcount_t *s) ...@@ -127,7 +127,7 @@ static inline unsigned __read_seqcount_begin(const seqcount_t *s)
*/ */
static inline unsigned raw_read_seqcount(const seqcount_t *s) static inline unsigned raw_read_seqcount(const seqcount_t *s)
{ {
unsigned ret = ACCESS_ONCE(s->sequence); unsigned ret = READ_ONCE(s->sequence);
smp_rmb(); smp_rmb();
return ret; return ret;
} }
...@@ -179,7 +179,7 @@ static inline unsigned read_seqcount_begin(const seqcount_t *s) ...@@ -179,7 +179,7 @@ static inline unsigned read_seqcount_begin(const seqcount_t *s)
*/ */
static inline unsigned raw_seqcount_begin(const seqcount_t *s) static inline unsigned raw_seqcount_begin(const seqcount_t *s)
{ {
unsigned ret = ACCESS_ONCE(s->sequence); unsigned ret = READ_ONCE(s->sequence);
smp_rmb(); smp_rmb();
return ret & ~1; return ret & ~1;
} }
......
...@@ -900,7 +900,7 @@ static int attach_to_pi_owner(u32 uval, union futex_key *key, ...@@ -900,7 +900,7 @@ static int attach_to_pi_owner(u32 uval, union futex_key *key,
if (!p) if (!p)
return -ESRCH; return -ESRCH;
if (!p->mm) { if (unlikely(p->flags & PF_KTHREAD)) {
put_task_struct(p); put_task_struct(p);
return -EPERM; return -EPERM;
} }
......
...@@ -78,7 +78,7 @@ void mcs_spin_lock(struct mcs_spinlock **lock, struct mcs_spinlock *node) ...@@ -78,7 +78,7 @@ void mcs_spin_lock(struct mcs_spinlock **lock, struct mcs_spinlock *node)
*/ */
return; return;
} }
ACCESS_ONCE(prev->next) = node; WRITE_ONCE(prev->next, node);
/* Wait until the lock holder passes the lock down. */ /* Wait until the lock holder passes the lock down. */
arch_mcs_spin_lock_contended(&node->locked); arch_mcs_spin_lock_contended(&node->locked);
...@@ -91,7 +91,7 @@ void mcs_spin_lock(struct mcs_spinlock **lock, struct mcs_spinlock *node) ...@@ -91,7 +91,7 @@ void mcs_spin_lock(struct mcs_spinlock **lock, struct mcs_spinlock *node)
static inline static inline
void mcs_spin_unlock(struct mcs_spinlock **lock, struct mcs_spinlock *node) void mcs_spin_unlock(struct mcs_spinlock **lock, struct mcs_spinlock *node)
{ {
struct mcs_spinlock *next = ACCESS_ONCE(node->next); struct mcs_spinlock *next = READ_ONCE(node->next);
if (likely(!next)) { if (likely(!next)) {
/* /*
...@@ -100,7 +100,7 @@ void mcs_spin_unlock(struct mcs_spinlock **lock, struct mcs_spinlock *node) ...@@ -100,7 +100,7 @@ void mcs_spin_unlock(struct mcs_spinlock **lock, struct mcs_spinlock *node)
if (likely(cmpxchg(lock, node, NULL) == node)) if (likely(cmpxchg(lock, node, NULL) == node))
return; return;
/* Wait until the next pointer is set */ /* Wait until the next pointer is set */
while (!(next = ACCESS_ONCE(node->next))) while (!(next = READ_ONCE(node->next)))
cpu_relax_lowlatency(); cpu_relax_lowlatency();
} }
......
...@@ -25,7 +25,7 @@ ...@@ -25,7 +25,7 @@
#include <linux/spinlock.h> #include <linux/spinlock.h>
#include <linux/interrupt.h> #include <linux/interrupt.h>
#include <linux/debug_locks.h> #include <linux/debug_locks.h>
#include "mcs_spinlock.h" #include <linux/osq_lock.h>
/* /*
* In the DEBUG case we are using the "NULL fastpath" for mutexes, * In the DEBUG case we are using the "NULL fastpath" for mutexes,
...@@ -217,44 +217,35 @@ ww_mutex_set_context_slowpath(struct ww_mutex *lock, ...@@ -217,44 +217,35 @@ ww_mutex_set_context_slowpath(struct ww_mutex *lock,
} }
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER #ifdef CONFIG_MUTEX_SPIN_ON_OWNER
static inline bool owner_running(struct mutex *lock, struct task_struct *owner)
{
if (lock->owner != owner)
return false;
/*
* Ensure we emit the owner->on_cpu, dereference _after_ checking
* lock->owner still matches owner, if that fails, owner might
* point to free()d memory, if it still matches, the rcu_read_lock()
* ensures the memory stays valid.
*/
barrier();
return owner->on_cpu;
}
/* /*
* Look out! "owner" is an entirely speculative pointer * Look out! "owner" is an entirely speculative pointer
* access and not reliable. * access and not reliable.
*/ */
static noinline static noinline
int mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner) bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner)
{ {
bool ret = true;
rcu_read_lock(); rcu_read_lock();
while (owner_running(lock, owner)) { while (lock->owner == owner) {
if (need_resched()) /*
* Ensure we emit the owner->on_cpu, dereference _after_
* checking lock->owner still matches owner. If that fails,
* owner might point to freed memory. If it still matches,
* the rcu_read_lock() ensures the memory stays valid.
*/
barrier();
if (!owner->on_cpu || need_resched()) {
ret = false;
break; break;
}
cpu_relax_lowlatency(); cpu_relax_lowlatency();
} }
rcu_read_unlock(); rcu_read_unlock();
/* return ret;
* We break out the loop above on need_resched() and when the
* owner changed, which is a sign for heavy contention. Return
* success only when lock->owner is NULL.
*/
return lock->owner == NULL;
} }
/* /*
...@@ -269,7 +260,7 @@ static inline int mutex_can_spin_on_owner(struct mutex *lock) ...@@ -269,7 +260,7 @@ static inline int mutex_can_spin_on_owner(struct mutex *lock)
return 0; return 0;
rcu_read_lock(); rcu_read_lock();
owner = ACCESS_ONCE(lock->owner); owner = READ_ONCE(lock->owner);
if (owner) if (owner)
retval = owner->on_cpu; retval = owner->on_cpu;
rcu_read_unlock(); rcu_read_unlock();
...@@ -343,7 +334,7 @@ static bool mutex_optimistic_spin(struct mutex *lock, ...@@ -343,7 +334,7 @@ static bool mutex_optimistic_spin(struct mutex *lock,
* As such, when deadlock detection needs to be * As such, when deadlock detection needs to be
* performed the optimistic spinning cannot be done. * performed the optimistic spinning cannot be done.
*/ */
if (ACCESS_ONCE(ww->ctx)) if (READ_ONCE(ww->ctx))
break; break;
} }
...@@ -351,7 +342,7 @@ static bool mutex_optimistic_spin(struct mutex *lock, ...@@ -351,7 +342,7 @@ static bool mutex_optimistic_spin(struct mutex *lock,
* If there's an owner, wait for it to either * If there's an owner, wait for it to either
* release the lock or go to sleep. * release the lock or go to sleep.
*/ */
owner = ACCESS_ONCE(lock->owner); owner = READ_ONCE(lock->owner);
if (owner && !mutex_spin_on_owner(lock, owner)) if (owner && !mutex_spin_on_owner(lock, owner))
break; break;
...@@ -490,7 +481,7 @@ static inline int __sched ...@@ -490,7 +481,7 @@ static inline int __sched
__ww_mutex_lock_check_stamp(struct mutex *lock, struct ww_acquire_ctx *ctx) __ww_mutex_lock_check_stamp(struct mutex *lock, struct ww_acquire_ctx *ctx)
{ {
struct ww_mutex *ww = container_of(lock, struct ww_mutex, base); struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
struct ww_acquire_ctx *hold_ctx = ACCESS_ONCE(ww->ctx); struct ww_acquire_ctx *hold_ctx = READ_ONCE(ww->ctx);
if (!hold_ctx) if (!hold_ctx)
return 0; return 0;
......
...@@ -98,7 +98,7 @@ bool osq_lock(struct optimistic_spin_queue *lock) ...@@ -98,7 +98,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)
prev = decode_cpu(old); prev = decode_cpu(old);
node->prev = prev; node->prev = prev;
ACCESS_ONCE(prev->next) = node; WRITE_ONCE(prev->next, node);
/* /*
* Normally @prev is untouchable after the above store; because at that * Normally @prev is untouchable after the above store; because at that
...@@ -109,7 +109,7 @@ bool osq_lock(struct optimistic_spin_queue *lock) ...@@ -109,7 +109,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)
* cmpxchg in an attempt to undo our queueing. * cmpxchg in an attempt to undo our queueing.
*/ */
while (!ACCESS_ONCE(node->locked)) { while (!READ_ONCE(node->locked)) {
/* /*
* If we need to reschedule bail... so we can block. * If we need to reschedule bail... so we can block.
*/ */
...@@ -148,7 +148,7 @@ bool osq_lock(struct optimistic_spin_queue *lock) ...@@ -148,7 +148,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)
* Or we race against a concurrent unqueue()'s step-B, in which * Or we race against a concurrent unqueue()'s step-B, in which
* case its step-C will write us a new @node->prev pointer. * case its step-C will write us a new @node->prev pointer.
*/ */
prev = ACCESS_ONCE(node->prev); prev = READ_ONCE(node->prev);
} }
/* /*
...@@ -170,8 +170,8 @@ bool osq_lock(struct optimistic_spin_queue *lock) ...@@ -170,8 +170,8 @@ bool osq_lock(struct optimistic_spin_queue *lock)
* it will wait in Step-A. * it will wait in Step-A.
*/ */
ACCESS_ONCE(next->prev) = prev; WRITE_ONCE(next->prev, prev);
ACCESS_ONCE(prev->next) = next; WRITE_ONCE(prev->next, next);
return false; return false;
} }
...@@ -193,11 +193,11 @@ void osq_unlock(struct optimistic_spin_queue *lock) ...@@ -193,11 +193,11 @@ void osq_unlock(struct optimistic_spin_queue *lock)
node = this_cpu_ptr(&osq_node); node = this_cpu_ptr(&osq_node);
next = xchg(&node->next, NULL); next = xchg(&node->next, NULL);
if (next) { if (next) {
ACCESS_ONCE(next->locked) = 1; WRITE_ONCE(next->locked, 1);
return; return;
} }
next = osq_wait_next(lock, node, NULL); next = osq_wait_next(lock, node, NULL);
if (next) if (next)
ACCESS_ONCE(next->locked) = 1; WRITE_ONCE(next->locked, 1);
} }
...@@ -349,7 +349,7 @@ static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p) ...@@ -349,7 +349,7 @@ static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p)
* *
* @task: the task owning the mutex (owner) for which a chain walk is * @task: the task owning the mutex (owner) for which a chain walk is
* probably needed * probably needed
* @deadlock_detect: do we have to carry out deadlock detection? * @chwalk: do we have to carry out deadlock detection?
* @orig_lock: the mutex (can be NULL if we are walking the chain to recheck * @orig_lock: the mutex (can be NULL if we are walking the chain to recheck
* things for a task that has just got its priority adjusted, and * things for a task that has just got its priority adjusted, and
* is waiting on a mutex) * is waiting on a mutex)
......
...@@ -85,6 +85,13 @@ __rwsem_do_wake(struct rw_semaphore *sem, int wakewrite) ...@@ -85,6 +85,13 @@ __rwsem_do_wake(struct rw_semaphore *sem, int wakewrite)
list_del(&waiter->list); list_del(&waiter->list);
tsk = waiter->task; tsk = waiter->task;
/*
* Make sure we do not wakeup the next reader before
* setting the nil condition to grant the next reader;
* otherwise we could miss the wakeup on the other
* side and end up sleeping again. See the pairing
* in rwsem_down_read_failed().
*/
smp_mb(); smp_mb();
waiter->task = NULL; waiter->task = NULL;
wake_up_process(tsk); wake_up_process(tsk);
......
...@@ -14,8 +14,9 @@ ...@@ -14,8 +14,9 @@
#include <linux/init.h> #include <linux/init.h>
#include <linux/export.h> #include <linux/export.h>
#include <linux/sched/rt.h> #include <linux/sched/rt.h>
#include <linux/osq_lock.h>
#include "mcs_spinlock.h" #include "rwsem.h"
/* /*
* Guide to the rw_semaphore's count field for common values. * Guide to the rw_semaphore's count field for common values.
...@@ -186,6 +187,13 @@ __rwsem_do_wake(struct rw_semaphore *sem, enum rwsem_wake_type wake_type) ...@@ -186,6 +187,13 @@ __rwsem_do_wake(struct rw_semaphore *sem, enum rwsem_wake_type wake_type)
waiter = list_entry(next, struct rwsem_waiter, list); waiter = list_entry(next, struct rwsem_waiter, list);
next = waiter->list.next; next = waiter->list.next;
tsk = waiter->task; tsk = waiter->task;
/*
* Make sure we do not wakeup the next reader before
* setting the nil condition to grant the next reader;
* otherwise we could miss the wakeup on the other
* side and end up sleeping again. See the pairing
* in rwsem_down_read_failed().
*/
smp_mb(); smp_mb();
waiter->task = NULL; waiter->task = NULL;
wake_up_process(tsk); wake_up_process(tsk);
...@@ -258,6 +266,7 @@ static inline bool rwsem_try_write_lock(long count, struct rw_semaphore *sem) ...@@ -258,6 +266,7 @@ static inline bool rwsem_try_write_lock(long count, struct rw_semaphore *sem)
RWSEM_ACTIVE_WRITE_BIAS) == RWSEM_WAITING_BIAS) { RWSEM_ACTIVE_WRITE_BIAS) == RWSEM_WAITING_BIAS) {
if (!list_is_singular(&sem->wait_list)) if (!list_is_singular(&sem->wait_list))
rwsem_atomic_update(RWSEM_WAITING_BIAS, sem); rwsem_atomic_update(RWSEM_WAITING_BIAS, sem);
rwsem_set_owner(sem);
return true; return true;
} }
...@@ -270,15 +279,17 @@ static inline bool rwsem_try_write_lock(long count, struct rw_semaphore *sem) ...@@ -270,15 +279,17 @@ static inline bool rwsem_try_write_lock(long count, struct rw_semaphore *sem)
*/ */
static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem) static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem)
{ {
long old, count = ACCESS_ONCE(sem->count); long old, count = READ_ONCE(sem->count);
while (true) { while (true) {
if (!(count == 0 || count == RWSEM_WAITING_BIAS)) if (!(count == 0 || count == RWSEM_WAITING_BIAS))
return false; return false;
old = cmpxchg(&sem->count, count, count + RWSEM_ACTIVE_WRITE_BIAS); old = cmpxchg(&sem->count, count, count + RWSEM_ACTIVE_WRITE_BIAS);
if (old == count) if (old == count) {
rwsem_set_owner(sem);
return true; return true;
}
count = old; count = old;
} }
...@@ -287,60 +298,67 @@ static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem) ...@@ -287,60 +298,67 @@ static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem)
static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem) static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem)
{ {
struct task_struct *owner; struct task_struct *owner;
bool on_cpu = false; bool ret = true;
if (need_resched()) if (need_resched())
return false; return false;
rcu_read_lock(); rcu_read_lock();
owner = ACCESS_ONCE(sem->owner); owner = READ_ONCE(sem->owner);
if (owner) if (!owner) {
on_cpu = owner->on_cpu; long count = READ_ONCE(sem->count);
rcu_read_unlock();
/* /*
* If sem->owner is not set, yet we have just recently entered the * If sem->owner is not set, yet we have just recently entered the
* slowpath, then there is a possibility reader(s) may have the lock. * slowpath with the lock being active, then there is a possibility
* To be safe, avoid spinning in these situations. * reader(s) may have the lock. To be safe, bail spinning in these
* situations.
*/ */
return on_cpu; if (count & RWSEM_ACTIVE_MASK)
ret = false;
goto done;
}
ret = owner->on_cpu;
done:
rcu_read_unlock();
return ret;
} }
static inline bool owner_running(struct rw_semaphore *sem, static noinline
struct task_struct *owner) bool rwsem_spin_on_owner(struct rw_semaphore *sem, struct task_struct *owner)
{ {
if (sem->owner != owner) long count;
return false;
rcu_read_lock();
while (sem->owner == owner) {
/* /*
* Ensure we emit the owner->on_cpu, dereference _after_ checking * Ensure we emit the owner->on_cpu, dereference _after_
* sem->owner still matches owner, if that fails, owner might * checking sem->owner still matches owner, if that fails,
* point to free()d memory, if it still matches, the rcu_read_lock() * owner might point to free()d memory, if it still matches,
* ensures the memory stays valid. * the rcu_read_lock() ensures the memory stays valid.
*/ */
barrier(); barrier();
return owner->on_cpu; /* abort spinning when need_resched or owner is not running */
} if (!owner->on_cpu || need_resched()) {
rcu_read_unlock();
static noinline return false;
bool rwsem_spin_on_owner(struct rw_semaphore *sem, struct task_struct *owner) }
{
rcu_read_lock();
while (owner_running(sem, owner)) {
if (need_resched())
break;
cpu_relax_lowlatency(); cpu_relax_lowlatency();
} }
rcu_read_unlock(); rcu_read_unlock();
if (READ_ONCE(sem->owner))
return true; /* new owner, continue spinning */
/* /*
* We break out the loop above on need_resched() or when the * When the owner is not set, the lock could be free or
* owner changed, which is a sign for heavy contention. Return * held by readers. Check the counter to verify the
* success only when sem->owner is NULL. * state.
*/ */
return sem->owner == NULL; count = READ_ONCE(sem->count);
return (count == 0 || count == RWSEM_WAITING_BIAS);
} }
static bool rwsem_optimistic_spin(struct rw_semaphore *sem) static bool rwsem_optimistic_spin(struct rw_semaphore *sem)
...@@ -358,7 +376,7 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem) ...@@ -358,7 +376,7 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem)
goto done; goto done;
while (true) { while (true) {
owner = ACCESS_ONCE(sem->owner); owner = READ_ONCE(sem->owner);
if (owner && !rwsem_spin_on_owner(sem, owner)) if (owner && !rwsem_spin_on_owner(sem, owner))
break; break;
...@@ -432,7 +450,7 @@ struct rw_semaphore __sched *rwsem_down_write_failed(struct rw_semaphore *sem) ...@@ -432,7 +450,7 @@ struct rw_semaphore __sched *rwsem_down_write_failed(struct rw_semaphore *sem)
/* we're now waiting on the lock, but no longer actively locking */ /* we're now waiting on the lock, but no longer actively locking */
if (waiting) { if (waiting) {
count = ACCESS_ONCE(sem->count); count = READ_ONCE(sem->count);
/* /*
* If there were already threads queued before us and there are * If there were already threads queued before us and there are
......
...@@ -9,29 +9,9 @@ ...@@ -9,29 +9,9 @@
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/export.h> #include <linux/export.h>
#include <linux/rwsem.h> #include <linux/rwsem.h>
#include <linux/atomic.h> #include <linux/atomic.h>
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER #include "rwsem.h"
static inline void rwsem_set_owner(struct rw_semaphore *sem)
{
sem->owner = current;
}
static inline void rwsem_clear_owner(struct rw_semaphore *sem)
{
sem->owner = NULL;
}
#else
static inline void rwsem_set_owner(struct rw_semaphore *sem)
{
}
static inline void rwsem_clear_owner(struct rw_semaphore *sem)
{
}
#endif
/* /*
* lock for reading * lock for reading
......
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
static inline void rwsem_set_owner(struct rw_semaphore *sem)
{
sem->owner = current;
}
static inline void rwsem_clear_owner(struct rw_semaphore *sem)
{
sem->owner = NULL;
}
#else
static inline void rwsem_set_owner(struct rw_semaphore *sem)
{
}
static inline void rwsem_clear_owner(struct rw_semaphore *sem)
{
}
#endif
...@@ -18,7 +18,7 @@ ...@@ -18,7 +18,7 @@
#define CMPXCHG_LOOP(CODE, SUCCESS) do { \ #define CMPXCHG_LOOP(CODE, SUCCESS) do { \
struct lockref old; \ struct lockref old; \
BUILD_BUG_ON(sizeof(old) != 8); \ BUILD_BUG_ON(sizeof(old) != 8); \
old.lock_count = ACCESS_ONCE(lockref->lock_count); \ old.lock_count = READ_ONCE(lockref->lock_count); \
while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \ while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \
struct lockref new = old, prev = old; \ struct lockref new = old, prev = old; \
CODE \ CODE \
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment