Commit c29ed89c authored by Linus Torvalds's avatar Linus Torvalds

Merge bk://gkernel.bkbits.net/libata-2.5

into ppc970.osdl.org:/home/torvalds/v2.5/linux
parents 345f1947 1e32c988
...@@ -9,31 +9,39 @@ ...@@ -9,31 +9,39 @@
#include <linux/sched.h> #include <linux/sched.h>
/* /*
* Semaphores are implemented using a two-way counter: * This is basically the PPC semaphore scheme ported to use
* * the Alpha ll/sc sequences, so see the PPC code for
* The "count" variable is decremented for each process that tries to sleep, * credits.
* while the "waking" variable is incremented when the "up()" code goes to */
* wake up waiting processes.
* /*
* Notably, the inline "up()" and "down()" functions can efficiently test * Atomically update sem->count.
* if they need to do any extra work (up needs to do something only if count * This does the equivalent of the following:
* was negative before the increment operation.
*
* waking_non_zero() (from asm/semaphore.h) must execute atomically.
*
* When __up() is called, the count was negative before incrementing it,
* and we need to wake up somebody.
*
* This routine adds one to the count of processes that need to wake up and
* exit. ALL waiting processes actually wake up but only the one that gets
* to the "waking" field first will gate through and acquire the semaphore.
* The others will go back to sleep.
* *
* Note that these functions are only called when there is contention on the * old_count = sem->count;
* lock, and as such all this is the "non-critical" part of the whole * tmp = MAX(old_count, 0) + incr;
* semaphore business. The critical part is the inline stuff in * sem->count = tmp;
* <asm/semaphore.h> where we want to avoid any extra jumps and calls. * return old_count;
*/ */
static inline int __sem_update_count(struct semaphore *sem, int incr)
{
long old_count, tmp = 0;
__asm__ __volatile__(
"1: ldl_l %0,%2\n"
" cmovgt %0,%0,%1\n"
" addl %1,%3,%1\n"
" stl_c %1,%2\n"
" beq %1,2f\n"
" mb\n"
".subsection 2\n"
"2: br 1b\n"
".previous"
: "=&r" (old_count), "=&r" (tmp), "=m" (sem->count)
: "Ir" (incr), "1" (tmp), "m" (sem->count));
return old_count;
}
/* /*
* Perform the "down" function. Return zero for semaphore acquired, * Perform the "down" function. Return zero for semaphore acquired,
...@@ -55,134 +63,77 @@ ...@@ -55,134 +63,77 @@
void void
__down_failed(struct semaphore *sem) __down_failed(struct semaphore *sem)
{ {
DECLARE_WAITQUEUE(wait, current); struct task_struct *tsk = current;
DECLARE_WAITQUEUE(wait, tsk);
#ifdef CONFIG_DEBUG_SEMAPHORE #ifdef CONFIG_DEBUG_SEMAPHORE
printk("%s(%d): down failed(%p)\n", printk("%s(%d): down failed(%p)\n",
current->comm, current->pid, sem); tsk->comm, tsk->pid, sem);
#endif #endif
current->state = TASK_UNINTERRUPTIBLE; tsk->state = TASK_UNINTERRUPTIBLE;
wmb(); wmb();
add_wait_queue_exclusive(&sem->wait, &wait); add_wait_queue_exclusive(&sem->wait, &wait);
/* At this point we know that sem->count is negative. In order /*
to avoid racing with __up, we must check for wakeup before * Try to get the semaphore. If the count is > 0, then we've
going to sleep the first time. */ * got the semaphore; we decrement count and exit the loop.
* If the count is 0 or negative, we set it to -1, indicating
while (1) { * that we are asleep, and then sleep.
long ret, tmp; */
while (__sem_update_count(sem, -1) <= 0) {
/* An atomic conditional decrement of sem->waking. */
__asm__ __volatile__(
"1: ldl_l %1,%2\n"
" blt %1,2f\n"
" subl %1,1,%0\n"
" stl_c %0,%2\n"
" beq %0,3f\n"
"2:\n"
".subsection 2\n"
"3: br 1b\n"
".previous"
: "=r"(ret), "=&r"(tmp), "=m"(sem->waking)
: "0"(0));
if (ret)
break;
schedule(); schedule();
set_task_state(current, TASK_UNINTERRUPTIBLE); set_task_state(tsk, TASK_UNINTERRUPTIBLE);
} }
remove_wait_queue(&sem->wait, &wait); remove_wait_queue(&sem->wait, &wait);
current->state = TASK_RUNNING; tsk->state = TASK_RUNNING;
/*
* If there are any more sleepers, wake one of them up so
* that it can either get the semaphore, or set count to -1
* indicating that there are still processes sleeping.
*/
wake_up(&sem->wait);
#ifdef CONFIG_DEBUG_SEMAPHORE #ifdef CONFIG_DEBUG_SEMAPHORE
printk("%s(%d): down acquired(%p)\n", printk("%s(%d): down acquired(%p)\n",
current->comm, current->pid, sem); tsk->comm, tsk->pid, sem);
#endif #endif
} }
int int
__down_failed_interruptible(struct semaphore *sem) __down_failed_interruptible(struct semaphore *sem)
{ {
DECLARE_WAITQUEUE(wait, current); struct task_struct *tsk = current;
long ret; DECLARE_WAITQUEUE(wait, tsk);
long ret = 0;
#ifdef CONFIG_DEBUG_SEMAPHORE #ifdef CONFIG_DEBUG_SEMAPHORE
printk("%s(%d): down failed(%p)\n", printk("%s(%d): down failed(%p)\n",
current->comm, current->pid, sem); tsk->comm, tsk->pid, sem);
#endif #endif
current->state = TASK_INTERRUPTIBLE; tsk->state = TASK_INTERRUPTIBLE;
wmb(); wmb();
add_wait_queue_exclusive(&sem->wait, &wait); add_wait_queue_exclusive(&sem->wait, &wait);
while (1) { while (__sem_update_count(sem, -1) <= 0) {
long tmp, tmp2, tmp3; if (signal_pending(current)) {
/*
/* We must undo the sem->count down_interruptible decrement * A signal is pending - give up trying.
simultaneously and atomically with the sem->waking * Set sem->count to 0 if it is negative,
adjustment, otherwise we can race with __up. This is * since we are no longer sleeping.
accomplished by doing a 64-bit ll/sc on two 32-bit words.
"Equivalent" C. Note that we have to do this all without
(taken) branches in order to be a valid ll/sc sequence.
do {
tmp = ldq_l;
ret = 0;
if (tmp >= 0) { // waking >= 0
tmp += 0xffffffff00000000; // waking -= 1
ret = 1;
}
else if (pending) {
// count += 1, but since -1 + 1 carries into the
// high word, we have to be more careful here.
tmp = (tmp & 0xffffffff00000000)
| ((tmp + 1) & 0x00000000ffffffff);
ret = -EINTR;
}
tmp = stq_c = tmp;
} while (tmp == 0);
*/ */
__sem_update_count(sem, 0);
__asm__ __volatile__( ret = -EINTR;
"1: ldq_l %1,%4\n"
" lda %0,0\n"
" cmovne %5,%6,%0\n"
" addq %1,1,%2\n"
" and %1,%7,%3\n"
" andnot %2,%7,%2\n"
" cmovge %1,1,%0\n"
" or %3,%2,%2\n"
" addq %1,%7,%3\n"
" cmovne %5,%2,%1\n"
" cmovge %2,%3,%1\n"
" stq_c %1,%4\n"
" beq %1,3f\n"
"2:\n"
".subsection 2\n"
"3: br 1b\n"
".previous"
: "=&r"(ret), "=&r"(tmp), "=&r"(tmp2),
"=&r"(tmp3), "=m"(*sem)
: "r"(signal_pending(current)), "r"(-EINTR),
"r"(0xffffffff00000000));
/* At this point we have ret
1 got the lock
0 go to sleep
-EINTR interrupted */
if (ret != 0)
break; break;
}
schedule(); schedule();
set_task_state(current, TASK_INTERRUPTIBLE); set_task_state(tsk, TASK_INTERRUPTIBLE);
} }
remove_wait_queue(&sem->wait, &wait); remove_wait_queue(&sem->wait, &wait);
current->state = TASK_RUNNING; tsk->state = TASK_RUNNING;
wake_up(&sem->wait); wake_up(&sem->wait);
#ifdef CONFIG_DEBUG_SEMAPHORE #ifdef CONFIG_DEBUG_SEMAPHORE
...@@ -190,14 +141,21 @@ __down_failed_interruptible(struct semaphore *sem) ...@@ -190,14 +141,21 @@ __down_failed_interruptible(struct semaphore *sem)
current->comm, current->pid, current->comm, current->pid,
(ret < 0 ? "interrupted" : "acquired"), sem); (ret < 0 ? "interrupted" : "acquired"), sem);
#endif #endif
return ret;
/* Convert "got the lock" to 0==success. */
return (ret < 0 ? ret : 0);
} }
void void
__up_wakeup(struct semaphore *sem) __up_wakeup(struct semaphore *sem)
{ {
/*
* Note that we incremented count in up() before we came here,
* but that was ineffective since the result was <= 0, and
* any negative value of count is equivalent to 0.
* This ends up setting count to 1, unless count is now > 0
* (i.e. because some other cpu has called up() in the meantime),
* in which case we just increment count.
*/
__sem_update_count(sem, 1);
wake_up(&sem->wait); wake_up(&sem->wait);
} }
......
...@@ -16,10 +16,7 @@ ...@@ -16,10 +16,7 @@
#include <linux/rwsem.h> #include <linux/rwsem.h>
struct semaphore { struct semaphore {
/* Careful, inline assembly knows about the position of these two. */ atomic_t count;
atomic_t count __attribute__((aligned(8)));
atomic_t waking; /* biased by -1 */
wait_queue_head_t wait; wait_queue_head_t wait;
#if WAITQUEUE_DEBUG #if WAITQUEUE_DEBUG
long __magic; long __magic;
...@@ -33,7 +30,7 @@ struct semaphore { ...@@ -33,7 +30,7 @@ struct semaphore {
#endif #endif
#define __SEMAPHORE_INITIALIZER(name,count) \ #define __SEMAPHORE_INITIALIZER(name,count) \
{ ATOMIC_INIT(count), ATOMIC_INIT(-1), \ { ATOMIC_INIT(count), \
__WAIT_QUEUE_HEAD_INITIALIZER((name).wait) \ __WAIT_QUEUE_HEAD_INITIALIZER((name).wait) \
__SEM_DEBUG_INIT(name) } __SEM_DEBUG_INIT(name) }
...@@ -55,7 +52,6 @@ static inline void sema_init(struct semaphore *sem, int val) ...@@ -55,7 +52,6 @@ static inline void sema_init(struct semaphore *sem, int val)
*/ */
atomic_set(&sem->count, val); atomic_set(&sem->count, val);
atomic_set(&sem->waking, -1);
init_waitqueue_head(&sem->wait); init_waitqueue_head(&sem->wait);
#if WAITQUEUE_DEBUG #if WAITQUEUE_DEBUG
sem->__magic = (long)&sem->__magic; sem->__magic = (long)&sem->__magic;
...@@ -107,102 +103,42 @@ static inline int __down_interruptible(struct semaphore *sem) ...@@ -107,102 +103,42 @@ static inline int __down_interruptible(struct semaphore *sem)
/* /*
* down_trylock returns 0 on success, 1 if we failed to get the lock. * down_trylock returns 0 on success, 1 if we failed to get the lock.
*
* We must manipulate count and waking simultaneously and atomically.
* Do this by using ll/sc on the pair of 32-bit words.
*/ */
static inline int __down_trylock(struct semaphore * sem) static inline int __down_trylock(struct semaphore *sem)
{ {
long ret, tmp, tmp2, sub; long ret;
/* "Equivalent" C. Note that we have to do this all without /* "Equivalent" C:
(taken) branches in order to be a valid ll/sc sequence.
do { do {
tmp = ldq_l; ret = ldl_l;
sub = 0x0000000100000000; --ret;
ret = ((int)tmp <= 0); // count <= 0 ? if (ret < 0)
// Note that if count=0, the decrement overflows into break;
// waking, so cancel the 1 loaded above. Also cancel ret = stl_c = ret;
// it if the lock was already free. } while (ret == 0);
if ((int)tmp >= 0) sub = 0; // count >= 0 ?
ret &= ((long)tmp < 0); // waking < 0 ?
sub += 1;
if (ret) break;
tmp -= sub;
tmp = stq_c = tmp;
} while (tmp == 0);
*/ */
__asm__ __volatile__( __asm__ __volatile__(
"1: ldq_l %1,%4\n" "1: ldl_l %0,%1\n"
" lda %3,1\n" " subl %0,1,%0\n"
" addl %1,0,%2\n" " blt %0,2f\n"
" sll %3,32,%3\n" " stl_c %0,%1\n"
" cmple %2,0,%0\n" " beq %0,3f\n"
" cmovge %2,0,%3\n" " mb\n"
" cmplt %1,0,%2\n" "2:\n"
" addq %3,1,%3\n"
" and %0,%2,%0\n"
" bne %0,2f\n"
" subq %1,%3,%1\n"
" stq_c %1,%4\n"
" beq %1,3f\n"
"2: mb\n"
".subsection 2\n" ".subsection 2\n"
"3: br 1b\n" "3: br 1b\n"
".previous" ".previous"
: "=&r"(ret), "=&r"(tmp), "=&r"(tmp2), "=&r"(sub) : "=&r" (ret), "=m" (sem->count)
: "m"(*sem) : "m" (sem->count));
: "memory");
return ret; return ret < 0;
} }
static inline void __up(struct semaphore *sem) static inline void __up(struct semaphore *sem)
{ {
long ret, tmp, tmp2, tmp3; if (unlikely(atomic_inc_return(&sem->count) <= 0))
/* We must manipulate count and waking simultaneously and atomically.
Otherwise we have races between up and __down_failed_interruptible
waking up on a signal.
"Equivalent" C. Note that we have to do this all without
(taken) branches in order to be a valid ll/sc sequence.
do {
tmp = ldq_l;
ret = (int)tmp + 1; // count += 1;
tmp2 = tmp & 0xffffffff00000000; // extract waking
if (ret <= 0) // still sleepers?
tmp2 += 0x0000000100000000; // waking += 1;
tmp = ret & 0x00000000ffffffff; // insert count
tmp |= tmp2; // insert waking;
tmp = stq_c = tmp;
} while (tmp == 0);
*/
__asm__ __volatile__(
" mb\n"
"1: ldq_l %1,%4\n"
" addl %1,1,%0\n"
" zapnot %1,0xf0,%2\n"
" addq %2,%5,%3\n"
" cmovle %0,%3,%2\n"
" zapnot %0,0x0f,%1\n"
" bis %1,%2,%1\n"
" stq_c %1,%4\n"
" beq %1,3f\n"
"2:\n"
".subsection 2\n"
"3: br 1b\n"
".previous"
: "=&r"(ret), "=&r"(tmp), "=&r"(tmp2), "=&r"(tmp3)
: "m"(*sem), "r"(0x0000000100000000)
: "memory");
if (unlikely(ret <= 0))
__up_wakeup(sem); __up_wakeup(sem);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment