six locks: Fix a lost wakeup

There was a lost wakeup between a read unlock in percpu mode and a write
lock. The unlock path unlocks, then executes a barrier, then checks for
waiters; correspondingly, the lock side should set the wait bit and
execute a barrier, then attempt to take the lock.
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

fs/bcachefs/six.c

@@ -198,6 +198,14 @@ static __always_inline bool do_six_trylock_type(struct six_lock *lock,
 			atomic64_add(__SIX_VAL(write_locking, 1),
 				     &lock->state.counter);
 			smp_mb__after_atomic();
+		} else if (!(lock->state.waiters & (1 << SIX_LOCK_write))) {
+			atomic64_add(__SIX_VAL(waiters, 1 << SIX_LOCK_write),
+				     &lock->state.counter);
+			/*
+			 * pairs with barrier after unlock and before checking
+			 * for readers in unlock path
+			 */
+			smp_mb__after_atomic();
 		}
 
 		ret = !pcpu_read_count(lock);
@@ -212,9 +220,6 @@ static __always_inline bool do_six_trylock_type(struct six_lock *lock,
 		if (ret || try)
 			v -= __SIX_VAL(write_locking, 1);
 
-		if (!ret && !try && !(lock->state.waiters & (1 << SIX_LOCK_write)))
-			v += __SIX_VAL(waiters, 1 << SIX_LOCK_write);
-
 		if (try && !ret) {
 			old.v = atomic64_add_return(v, &lock->state.counter);
 			six_lock_wakeup(lock, old, SIX_LOCK_read);