Commit 732a27a0 authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'locking-urgent-2021-05-09' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull locking fixes from Thomas Gleixner:
 "A set of locking related fixes and updates:

   - Two fixes for the futex syscall related to the timeout handling.

     FUTEX_LOCK_PI does not support the FUTEX_CLOCK_REALTIME bit and
     because it's not set the time namespace adjustment for clock
     MONOTONIC is applied wrongly.

     FUTEX_WAIT cannot support the FUTEX_CLOCK_REALTIME bit because its
     always a relative timeout.

   - Cleanups in the futex syscall entry points which became obvious
     when the two timeout handling bugs were fixed.

   - Cleanup of queued_write_lock_slowpath() as suggested by Linus

   - Fixup of the smp_call_function_single_async() prototype"

* tag 'locking-urgent-2021-05-09' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  futex: Make syscall entry points less convoluted
  futex: Get rid of the val2 conditional dance
  futex: Do not apply time namespace adjustment on FUTEX_LOCK_PI
  Revert 337f1304 ("futex: Allow FUTEX_CLOCK_REALTIME with FUTEX_WAIT op")
  locking/qrwlock: Cleanup queued_write_lock_slowpath()
  smp: Fix smp_call_function_single_async prototype
parents 85bbba1c 51cf94d1
...@@ -53,7 +53,7 @@ int smp_call_function_single(int cpuid, smp_call_func_t func, void *info, ...@@ -53,7 +53,7 @@ int smp_call_function_single(int cpuid, smp_call_func_t func, void *info,
void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func, void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func,
void *info, bool wait, const struct cpumask *mask); void *info, bool wait, const struct cpumask *mask);
int smp_call_function_single_async(int cpu, call_single_data_t *csd); int smp_call_function_single_async(int cpu, struct __call_single_data *csd);
/* /*
* Cpus stopping functions in panic. All have default weak definitions. * Cpus stopping functions in panic. All have default weak definitions.
......
...@@ -3710,8 +3710,7 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, ...@@ -3710,8 +3710,7 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
if (op & FUTEX_CLOCK_REALTIME) { if (op & FUTEX_CLOCK_REALTIME) {
flags |= FLAGS_CLOCKRT; flags |= FLAGS_CLOCKRT;
if (cmd != FUTEX_WAIT && cmd != FUTEX_WAIT_BITSET && \ if (cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI)
cmd != FUTEX_WAIT_REQUEUE_PI)
return -ENOSYS; return -ENOSYS;
} }
...@@ -3758,42 +3757,52 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, ...@@ -3758,42 +3757,52 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
return -ENOSYS; return -ENOSYS;
} }
static __always_inline bool futex_cmd_has_timeout(u32 cmd)
{
switch (cmd) {
case FUTEX_WAIT:
case FUTEX_LOCK_PI:
case FUTEX_WAIT_BITSET:
case FUTEX_WAIT_REQUEUE_PI:
return true;
}
return false;
}
static __always_inline int
futex_init_timeout(u32 cmd, u32 op, struct timespec64 *ts, ktime_t *t)
{
if (!timespec64_valid(ts))
return -EINVAL;
*t = timespec64_to_ktime(*ts);
if (cmd == FUTEX_WAIT)
*t = ktime_add_safe(ktime_get(), *t);
else if (cmd != FUTEX_LOCK_PI && !(op & FUTEX_CLOCK_REALTIME))
*t = timens_ktime_to_host(CLOCK_MONOTONIC, *t);
return 0;
}
SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val, SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,
const struct __kernel_timespec __user *, utime, const struct __kernel_timespec __user *, utime,
u32 __user *, uaddr2, u32, val3) u32 __user *, uaddr2, u32, val3)
{ {
struct timespec64 ts; int ret, cmd = op & FUTEX_CMD_MASK;
ktime_t t, *tp = NULL; ktime_t t, *tp = NULL;
u32 val2 = 0; struct timespec64 ts;
int cmd = op & FUTEX_CMD_MASK;
if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI || if (utime && futex_cmd_has_timeout(cmd)) {
cmd == FUTEX_WAIT_BITSET ||
cmd == FUTEX_WAIT_REQUEUE_PI)) {
if (unlikely(should_fail_futex(!(op & FUTEX_PRIVATE_FLAG)))) if (unlikely(should_fail_futex(!(op & FUTEX_PRIVATE_FLAG))))
return -EFAULT; return -EFAULT;
if (get_timespec64(&ts, utime)) if (get_timespec64(&ts, utime))
return -EFAULT; return -EFAULT;
if (!timespec64_valid(&ts)) ret = futex_init_timeout(cmd, op, &ts, &t);
return -EINVAL; if (ret)
return ret;
t = timespec64_to_ktime(ts);
if (cmd == FUTEX_WAIT)
t = ktime_add_safe(ktime_get(), t);
else if (!(op & FUTEX_CLOCK_REALTIME))
t = timens_ktime_to_host(CLOCK_MONOTONIC, t);
tp = &t; tp = &t;
} }
/*
* requeue parameter in 'utime' if cmd == FUTEX_*_REQUEUE_*.
* number of waiters to wake in 'utime' if cmd == FUTEX_WAKE_OP.
*/
if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE ||
cmd == FUTEX_CMP_REQUEUE_PI || cmd == FUTEX_WAKE_OP)
val2 = (u32) (unsigned long) utime;
return do_futex(uaddr, op, val, tp, uaddr2, val2, val3); return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3);
} }
#ifdef CONFIG_COMPAT #ifdef CONFIG_COMPAT
...@@ -3959,31 +3968,20 @@ SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val, ...@@ -3959,31 +3968,20 @@ SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val,
const struct old_timespec32 __user *, utime, u32 __user *, uaddr2, const struct old_timespec32 __user *, utime, u32 __user *, uaddr2,
u32, val3) u32, val3)
{ {
struct timespec64 ts; int ret, cmd = op & FUTEX_CMD_MASK;
ktime_t t, *tp = NULL; ktime_t t, *tp = NULL;
int val2 = 0; struct timespec64 ts;
int cmd = op & FUTEX_CMD_MASK;
if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI || if (utime && futex_cmd_has_timeout(cmd)) {
cmd == FUTEX_WAIT_BITSET ||
cmd == FUTEX_WAIT_REQUEUE_PI)) {
if (get_old_timespec32(&ts, utime)) if (get_old_timespec32(&ts, utime))
return -EFAULT; return -EFAULT;
if (!timespec64_valid(&ts)) ret = futex_init_timeout(cmd, op, &ts, &t);
return -EINVAL; if (ret)
return ret;
t = timespec64_to_ktime(ts);
if (cmd == FUTEX_WAIT)
t = ktime_add_safe(ktime_get(), t);
else if (!(op & FUTEX_CLOCK_REALTIME))
t = timens_ktime_to_host(CLOCK_MONOTONIC, t);
tp = &t; tp = &t;
} }
if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE ||
cmd == FUTEX_CMP_REQUEUE_PI || cmd == FUTEX_WAKE_OP)
val2 = (int) (unsigned long) utime;
return do_futex(uaddr, op, val, tp, uaddr2, val2, val3); return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3);
} }
#endif /* CONFIG_COMPAT_32BIT_TIME */ #endif /* CONFIG_COMPAT_32BIT_TIME */
......
...@@ -66,12 +66,12 @@ void queued_write_lock_slowpath(struct qrwlock *lock) ...@@ -66,12 +66,12 @@ void queued_write_lock_slowpath(struct qrwlock *lock)
arch_spin_lock(&lock->wait_lock); arch_spin_lock(&lock->wait_lock);
/* Try to acquire the lock directly if no reader is present */ /* Try to acquire the lock directly if no reader is present */
if (!atomic_read(&lock->cnts) && if (!(cnts = atomic_read(&lock->cnts)) &&
(atomic_cmpxchg_acquire(&lock->cnts, 0, _QW_LOCKED) == 0)) atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED))
goto unlock; goto unlock;
/* Set the waiting flag to notify readers that a writer is pending */ /* Set the waiting flag to notify readers that a writer is pending */
atomic_add(_QW_WAITING, &lock->cnts); atomic_or(_QW_WAITING, &lock->cnts);
/* When no more readers or writers, set the locked flag */ /* When no more readers or writers, set the locked flag */
do { do {
......
...@@ -211,7 +211,7 @@ static u64 cfd_seq_inc(unsigned int src, unsigned int dst, unsigned int type) ...@@ -211,7 +211,7 @@ static u64 cfd_seq_inc(unsigned int src, unsigned int dst, unsigned int type)
} while (0) } while (0)
/* Record current CSD work for current CPU, NULL to erase. */ /* Record current CSD work for current CPU, NULL to erase. */
static void __csd_lock_record(call_single_data_t *csd) static void __csd_lock_record(struct __call_single_data *csd)
{ {
if (!csd) { if (!csd) {
smp_mb(); /* NULL cur_csd after unlock. */ smp_mb(); /* NULL cur_csd after unlock. */
...@@ -226,13 +226,13 @@ static void __csd_lock_record(call_single_data_t *csd) ...@@ -226,13 +226,13 @@ static void __csd_lock_record(call_single_data_t *csd)
/* Or before unlock, as the case may be. */ /* Or before unlock, as the case may be. */
} }
static __always_inline void csd_lock_record(call_single_data_t *csd) static __always_inline void csd_lock_record(struct __call_single_data *csd)
{ {
if (static_branch_unlikely(&csdlock_debug_enabled)) if (static_branch_unlikely(&csdlock_debug_enabled))
__csd_lock_record(csd); __csd_lock_record(csd);
} }
static int csd_lock_wait_getcpu(call_single_data_t *csd) static int csd_lock_wait_getcpu(struct __call_single_data *csd)
{ {
unsigned int csd_type; unsigned int csd_type;
...@@ -282,7 +282,7 @@ static const char *csd_lock_get_type(unsigned int type) ...@@ -282,7 +282,7 @@ static const char *csd_lock_get_type(unsigned int type)
return (type >= ARRAY_SIZE(seq_type)) ? "?" : seq_type[type]; return (type >= ARRAY_SIZE(seq_type)) ? "?" : seq_type[type];
} }
static void csd_lock_print_extended(call_single_data_t *csd, int cpu) static void csd_lock_print_extended(struct __call_single_data *csd, int cpu)
{ {
struct cfd_seq_local *seq = &per_cpu(cfd_seq_local, cpu); struct cfd_seq_local *seq = &per_cpu(cfd_seq_local, cpu);
unsigned int srccpu = csd->node.src; unsigned int srccpu = csd->node.src;
...@@ -321,7 +321,7 @@ static void csd_lock_print_extended(call_single_data_t *csd, int cpu) ...@@ -321,7 +321,7 @@ static void csd_lock_print_extended(call_single_data_t *csd, int cpu)
* the CSD_TYPE_SYNC/ASYNC types provide the destination CPU, * the CSD_TYPE_SYNC/ASYNC types provide the destination CPU,
* so waiting on other types gets much less information. * so waiting on other types gets much less information.
*/ */
static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, int *bug_id) static bool csd_lock_wait_toolong(struct __call_single_data *csd, u64 ts0, u64 *ts1, int *bug_id)
{ {
int cpu = -1; int cpu = -1;
int cpux; int cpux;
...@@ -387,7 +387,7 @@ static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, in ...@@ -387,7 +387,7 @@ static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, in
* previous function call. For multi-cpu calls its even more interesting * previous function call. For multi-cpu calls its even more interesting
* as we'll have to ensure no other cpu is observing our csd. * as we'll have to ensure no other cpu is observing our csd.
*/ */
static void __csd_lock_wait(call_single_data_t *csd) static void __csd_lock_wait(struct __call_single_data *csd)
{ {
int bug_id = 0; int bug_id = 0;
u64 ts0, ts1; u64 ts0, ts1;
...@@ -401,7 +401,7 @@ static void __csd_lock_wait(call_single_data_t *csd) ...@@ -401,7 +401,7 @@ static void __csd_lock_wait(call_single_data_t *csd)
smp_acquire__after_ctrl_dep(); smp_acquire__after_ctrl_dep();
} }
static __always_inline void csd_lock_wait(call_single_data_t *csd) static __always_inline void csd_lock_wait(struct __call_single_data *csd)
{ {
if (static_branch_unlikely(&csdlock_debug_enabled)) { if (static_branch_unlikely(&csdlock_debug_enabled)) {
__csd_lock_wait(csd); __csd_lock_wait(csd);
...@@ -431,17 +431,17 @@ static void __smp_call_single_queue_debug(int cpu, struct llist_node *node) ...@@ -431,17 +431,17 @@ static void __smp_call_single_queue_debug(int cpu, struct llist_node *node)
#else #else
#define cfd_seq_store(var, src, dst, type) #define cfd_seq_store(var, src, dst, type)
static void csd_lock_record(call_single_data_t *csd) static void csd_lock_record(struct __call_single_data *csd)
{ {
} }
static __always_inline void csd_lock_wait(call_single_data_t *csd) static __always_inline void csd_lock_wait(struct __call_single_data *csd)
{ {
smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));
} }
#endif #endif
static __always_inline void csd_lock(call_single_data_t *csd) static __always_inline void csd_lock(struct __call_single_data *csd)
{ {
csd_lock_wait(csd); csd_lock_wait(csd);
csd->node.u_flags |= CSD_FLAG_LOCK; csd->node.u_flags |= CSD_FLAG_LOCK;
...@@ -454,7 +454,7 @@ static __always_inline void csd_lock(call_single_data_t *csd) ...@@ -454,7 +454,7 @@ static __always_inline void csd_lock(call_single_data_t *csd)
smp_wmb(); smp_wmb();
} }
static __always_inline void csd_unlock(call_single_data_t *csd) static __always_inline void csd_unlock(struct __call_single_data *csd)
{ {
WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK)); WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK));
...@@ -501,7 +501,7 @@ void __smp_call_single_queue(int cpu, struct llist_node *node) ...@@ -501,7 +501,7 @@ void __smp_call_single_queue(int cpu, struct llist_node *node)
* for execution on the given CPU. data must already have * for execution on the given CPU. data must already have
* ->func, ->info, and ->flags set. * ->func, ->info, and ->flags set.
*/ */
static int generic_exec_single(int cpu, call_single_data_t *csd) static int generic_exec_single(int cpu, struct __call_single_data *csd)
{ {
if (cpu == smp_processor_id()) { if (cpu == smp_processor_id()) {
smp_call_func_t func = csd->func; smp_call_func_t func = csd->func;
...@@ -784,7 +784,7 @@ EXPORT_SYMBOL(smp_call_function_single); ...@@ -784,7 +784,7 @@ EXPORT_SYMBOL(smp_call_function_single);
* NOTE: Be careful, there is unfortunately no current debugging facility to * NOTE: Be careful, there is unfortunately no current debugging facility to
* validate the correctness of this serialization. * validate the correctness of this serialization.
*/ */
int smp_call_function_single_async(int cpu, call_single_data_t *csd) int smp_call_function_single_async(int cpu, struct __call_single_data *csd)
{ {
int err = 0; int err = 0;
......
...@@ -25,7 +25,7 @@ int smp_call_function_single(int cpu, void (*func) (void *info), void *info, ...@@ -25,7 +25,7 @@ int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
} }
EXPORT_SYMBOL(smp_call_function_single); EXPORT_SYMBOL(smp_call_function_single);
int smp_call_function_single_async(int cpu, call_single_data_t *csd) int smp_call_function_single_async(int cpu, struct __call_single_data *csd)
{ {
unsigned long flags; unsigned long flags;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment