Commit 4f07118f authored by David S. Miller's avatar David S. Miller

[SPARC64]: More fully work around Spitfire Errata 51.

It appears that a memory barrier soon after a mispredicted
branch, not just in the delay slot, can cause the hang
condition of this cpu errata.

So move them out-of-line, and explicitly put them into
a "branch always, predict taken" delay slot which should
fully kill this problem.
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 442464a5
...@@ -466,7 +466,7 @@ static void pci_strbuf_flush(struct pci_strbuf *strbuf, struct pci_iommu *iommu, ...@@ -466,7 +466,7 @@ static void pci_strbuf_flush(struct pci_strbuf *strbuf, struct pci_iommu *iommu,
if (!limit) if (!limit)
break; break;
udelay(1); udelay(1);
membar("#LoadLoad"); rmb();
} }
if (!limit) if (!limit)
printk(KERN_WARNING "pci_strbuf_flush: flushflag timeout " printk(KERN_WARNING "pci_strbuf_flush: flushflag timeout "
......
...@@ -103,7 +103,7 @@ void cpu_idle(void) ...@@ -103,7 +103,7 @@ void cpu_idle(void)
* other cpus see our increasing idleness for the buddy * other cpus see our increasing idleness for the buddy
* redistribution algorithm. -DaveM * redistribution algorithm. -DaveM
*/ */
membar("#StoreStore | #StoreLoad"); membar_storeload_storestore();
} }
} }
......
...@@ -147,7 +147,7 @@ static void sbus_strbuf_flush(struct sbus_iommu *iommu, u32 base, unsigned long ...@@ -147,7 +147,7 @@ static void sbus_strbuf_flush(struct sbus_iommu *iommu, u32 base, unsigned long
if (!limit) if (!limit)
break; break;
udelay(1); udelay(1);
membar("#LoadLoad"); rmb();
} }
if (!limit) if (!limit)
printk(KERN_WARNING "sbus_strbuf_flush: flushflag timeout " printk(KERN_WARNING "sbus_strbuf_flush: flushflag timeout "
......
...@@ -877,11 +877,12 @@ static void new_setup_frame32(struct k_sigaction *ka, struct pt_regs *regs, ...@@ -877,11 +877,12 @@ static void new_setup_frame32(struct k_sigaction *ka, struct pt_regs *regs,
unsigned long page = (unsigned long) unsigned long page = (unsigned long)
page_address(pte_page(*ptep)); page_address(pte_page(*ptep));
__asm__ __volatile__( wmb();
" membar #StoreStore\n" __asm__ __volatile__("flush %0 + %1"
" flush %0 + %1" : /* no outputs */
: : "r" (page), "r" (address & (PAGE_SIZE - 1)) : "r" (page),
: "memory"); "r" (address & (PAGE_SIZE - 1))
: "memory");
} }
pte_unmap(ptep); pte_unmap(ptep);
preempt_enable(); preempt_enable();
...@@ -1292,11 +1293,12 @@ static void setup_rt_frame32(struct k_sigaction *ka, struct pt_regs *regs, ...@@ -1292,11 +1293,12 @@ static void setup_rt_frame32(struct k_sigaction *ka, struct pt_regs *regs,
unsigned long page = (unsigned long) unsigned long page = (unsigned long)
page_address(pte_page(*ptep)); page_address(pte_page(*ptep));
__asm__ __volatile__( wmb();
" membar #StoreStore\n" __asm__ __volatile__("flush %0 + %1"
" flush %0 + %1" : /* no outputs */
: : "r" (page), "r" (address & (PAGE_SIZE - 1)) : "r" (page),
: "memory"); "r" (address & (PAGE_SIZE - 1))
: "memory");
} }
pte_unmap(ptep); pte_unmap(ptep);
preempt_enable(); preempt_enable();
......
...@@ -144,7 +144,7 @@ void __init smp_callin(void) ...@@ -144,7 +144,7 @@ void __init smp_callin(void)
current->active_mm = &init_mm; current->active_mm = &init_mm;
while (!cpu_isset(cpuid, smp_commenced_mask)) while (!cpu_isset(cpuid, smp_commenced_mask))
membar("#LoadLoad"); rmb();
cpu_set(cpuid, cpu_online_map); cpu_set(cpuid, cpu_online_map);
} }
...@@ -184,11 +184,11 @@ static inline long get_delta (long *rt, long *master) ...@@ -184,11 +184,11 @@ static inline long get_delta (long *rt, long *master)
for (i = 0; i < NUM_ITERS; i++) { for (i = 0; i < NUM_ITERS; i++) {
t0 = tick_ops->get_tick(); t0 = tick_ops->get_tick();
go[MASTER] = 1; go[MASTER] = 1;
membar("#StoreLoad"); membar_storeload();
while (!(tm = go[SLAVE])) while (!(tm = go[SLAVE]))
membar("#LoadLoad"); rmb();
go[SLAVE] = 0; go[SLAVE] = 0;
membar("#StoreStore"); wmb();
t1 = tick_ops->get_tick(); t1 = tick_ops->get_tick();
if (t1 - t0 < best_t1 - best_t0) if (t1 - t0 < best_t1 - best_t0)
...@@ -221,7 +221,7 @@ void smp_synchronize_tick_client(void) ...@@ -221,7 +221,7 @@ void smp_synchronize_tick_client(void)
go[MASTER] = 1; go[MASTER] = 1;
while (go[MASTER]) while (go[MASTER])
membar("#LoadLoad"); rmb();
local_irq_save(flags); local_irq_save(flags);
{ {
...@@ -273,21 +273,21 @@ static void smp_synchronize_one_tick(int cpu) ...@@ -273,21 +273,21 @@ static void smp_synchronize_one_tick(int cpu)
/* wait for client to be ready */ /* wait for client to be ready */
while (!go[MASTER]) while (!go[MASTER])
membar("#LoadLoad"); rmb();
/* now let the client proceed into his loop */ /* now let the client proceed into his loop */
go[MASTER] = 0; go[MASTER] = 0;
membar("#StoreLoad"); membar_storeload();
spin_lock_irqsave(&itc_sync_lock, flags); spin_lock_irqsave(&itc_sync_lock, flags);
{ {
for (i = 0; i < NUM_ROUNDS*NUM_ITERS; i++) { for (i = 0; i < NUM_ROUNDS*NUM_ITERS; i++) {
while (!go[MASTER]) while (!go[MASTER])
membar("#LoadLoad"); rmb();
go[MASTER] = 0; go[MASTER] = 0;
membar("#StoreStore"); wmb();
go[SLAVE] = tick_ops->get_tick(); go[SLAVE] = tick_ops->get_tick();
membar("#StoreLoad"); membar_storeload();
} }
} }
spin_unlock_irqrestore(&itc_sync_lock, flags); spin_unlock_irqrestore(&itc_sync_lock, flags);
...@@ -927,11 +927,11 @@ void smp_capture(void) ...@@ -927,11 +927,11 @@ void smp_capture(void)
smp_processor_id()); smp_processor_id());
#endif #endif
penguins_are_doing_time = 1; penguins_are_doing_time = 1;
membar("#StoreStore | #LoadStore"); membar_storestore_loadstore();
atomic_inc(&smp_capture_registry); atomic_inc(&smp_capture_registry);
smp_cross_call(&xcall_capture, 0, 0, 0); smp_cross_call(&xcall_capture, 0, 0, 0);
while (atomic_read(&smp_capture_registry) != ncpus) while (atomic_read(&smp_capture_registry) != ncpus)
membar("#LoadLoad"); rmb();
#ifdef CAPTURE_DEBUG #ifdef CAPTURE_DEBUG
printk("done\n"); printk("done\n");
#endif #endif
...@@ -947,7 +947,7 @@ void smp_release(void) ...@@ -947,7 +947,7 @@ void smp_release(void)
smp_processor_id()); smp_processor_id());
#endif #endif
penguins_are_doing_time = 0; penguins_are_doing_time = 0;
membar("#StoreStore | #StoreLoad"); membar_storeload_storestore();
atomic_dec(&smp_capture_registry); atomic_dec(&smp_capture_registry);
} }
} }
...@@ -970,9 +970,9 @@ void smp_penguin_jailcell(int irq, struct pt_regs *regs) ...@@ -970,9 +970,9 @@ void smp_penguin_jailcell(int irq, struct pt_regs *regs)
save_alternate_globals(global_save); save_alternate_globals(global_save);
prom_world(1); prom_world(1);
atomic_inc(&smp_capture_registry); atomic_inc(&smp_capture_registry);
membar("#StoreLoad | #StoreStore"); membar_storeload_storestore();
while (penguins_are_doing_time) while (penguins_are_doing_time)
membar("#LoadLoad"); rmb();
restore_alternate_globals(global_save); restore_alternate_globals(global_save);
atomic_dec(&smp_capture_registry); atomic_dec(&smp_capture_registry);
prom_world(0); prom_world(0);
......
...@@ -406,3 +406,12 @@ EXPORT_SYMBOL(xor_vis_4); ...@@ -406,3 +406,12 @@ EXPORT_SYMBOL(xor_vis_4);
EXPORT_SYMBOL(xor_vis_5); EXPORT_SYMBOL(xor_vis_5);
EXPORT_SYMBOL(prom_palette); EXPORT_SYMBOL(prom_palette);
/* memory barriers */
EXPORT_SYMBOL(mb);
EXPORT_SYMBOL(rmb);
EXPORT_SYMBOL(wmb);
EXPORT_SYMBOL(membar_storeload);
EXPORT_SYMBOL(membar_storeload_storestore);
EXPORT_SYMBOL(membar_storeload_loadload);
EXPORT_SYMBOL(membar_storestore_loadstore);
...@@ -12,7 +12,7 @@ lib-y := PeeCeeI.o copy_page.o clear_page.o strlen.o strncmp.o \ ...@@ -12,7 +12,7 @@ lib-y := PeeCeeI.o copy_page.o clear_page.o strlen.o strncmp.o \
U1memcpy.o U1copy_from_user.o U1copy_to_user.o \ U1memcpy.o U1copy_from_user.o U1copy_to_user.o \
U3memcpy.o U3copy_from_user.o U3copy_to_user.o U3patch.o \ U3memcpy.o U3copy_from_user.o U3copy_to_user.o U3patch.o \
copy_in_user.o user_fixup.o memmove.o \ copy_in_user.o user_fixup.o memmove.o \
mcount.o ipcsum.o rwsem.o xor.o find_bit.o delay.o mcount.o ipcsum.o rwsem.o xor.o find_bit.o delay.o mb.o
lib-$(CONFIG_DEBUG_SPINLOCK) += debuglocks.o lib-$(CONFIG_DEBUG_SPINLOCK) += debuglocks.o
lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o
......
...@@ -61,7 +61,7 @@ void _do_spin_lock(spinlock_t *lock, char *str, unsigned long caller) ...@@ -61,7 +61,7 @@ void _do_spin_lock(spinlock_t *lock, char *str, unsigned long caller)
: "=r" (val) : "=r" (val)
: "r" (&(lock->lock)) : "r" (&(lock->lock))
: "memory"); : "memory");
membar("#StoreLoad | #StoreStore"); membar_storeload_storestore();
if (val) { if (val) {
while (lock->lock) { while (lock->lock) {
if (!--stuck) { if (!--stuck) {
...@@ -69,7 +69,7 @@ void _do_spin_lock(spinlock_t *lock, char *str, unsigned long caller) ...@@ -69,7 +69,7 @@ void _do_spin_lock(spinlock_t *lock, char *str, unsigned long caller)
show(str, lock, caller); show(str, lock, caller);
stuck = INIT_STUCK; stuck = INIT_STUCK;
} }
membar("#LoadLoad"); rmb();
} }
goto again; goto again;
} }
...@@ -90,7 +90,7 @@ int _do_spin_trylock(spinlock_t *lock, unsigned long caller) ...@@ -90,7 +90,7 @@ int _do_spin_trylock(spinlock_t *lock, unsigned long caller)
: "=r" (val) : "=r" (val)
: "r" (&(lock->lock)) : "r" (&(lock->lock))
: "memory"); : "memory");
membar("#StoreLoad | #StoreStore"); membar_storeload_storestore();
if (!val) { if (!val) {
lock->owner_pc = ((unsigned int)caller); lock->owner_pc = ((unsigned int)caller);
lock->owner_cpu = cpu; lock->owner_cpu = cpu;
...@@ -107,7 +107,7 @@ void _do_spin_unlock(spinlock_t *lock) ...@@ -107,7 +107,7 @@ void _do_spin_unlock(spinlock_t *lock)
{ {
lock->owner_pc = 0; lock->owner_pc = 0;
lock->owner_cpu = NO_PROC_ID; lock->owner_cpu = NO_PROC_ID;
membar("#StoreStore | #LoadStore"); membar_storestore_loadstore();
lock->lock = 0; lock->lock = 0;
current->thread.smp_lock_count--; current->thread.smp_lock_count--;
} }
...@@ -129,7 +129,7 @@ void _do_read_lock(rwlock_t *rw, char *str, unsigned long caller) ...@@ -129,7 +129,7 @@ void _do_read_lock(rwlock_t *rw, char *str, unsigned long caller)
show_read(str, rw, caller); show_read(str, rw, caller);
stuck = INIT_STUCK; stuck = INIT_STUCK;
} }
membar("#LoadLoad"); rmb();
} }
/* Try once to increment the counter. */ /* Try once to increment the counter. */
__asm__ __volatile__( __asm__ __volatile__(
...@@ -142,7 +142,7 @@ void _do_read_lock(rwlock_t *rw, char *str, unsigned long caller) ...@@ -142,7 +142,7 @@ void _do_read_lock(rwlock_t *rw, char *str, unsigned long caller)
"2:" : "=r" (val) "2:" : "=r" (val)
: "0" (&(rw->lock)) : "0" (&(rw->lock))
: "g1", "g7", "memory"); : "g1", "g7", "memory");
membar("#StoreLoad | #StoreStore"); membar_storeload_storestore();
if (val) if (val)
goto wlock_again; goto wlock_again;
rw->reader_pc[cpu] = ((unsigned int)caller); rw->reader_pc[cpu] = ((unsigned int)caller);
...@@ -201,7 +201,7 @@ void _do_write_lock(rwlock_t *rw, char *str, unsigned long caller) ...@@ -201,7 +201,7 @@ void _do_write_lock(rwlock_t *rw, char *str, unsigned long caller)
show_write(str, rw, caller); show_write(str, rw, caller);
stuck = INIT_STUCK; stuck = INIT_STUCK;
} }
membar("#LoadLoad"); rmb();
} }
/* Try to acuire the write bit. */ /* Try to acuire the write bit. */
...@@ -256,7 +256,7 @@ void _do_write_lock(rwlock_t *rw, char *str, unsigned long caller) ...@@ -256,7 +256,7 @@ void _do_write_lock(rwlock_t *rw, char *str, unsigned long caller)
show_write(str, rw, caller); show_write(str, rw, caller);
stuck = INIT_STUCK; stuck = INIT_STUCK;
} }
membar("#LoadLoad"); rmb();
} }
goto wlock_again; goto wlock_again;
} }
......
/* mb.S: Out of line memory barriers.
*
* Copyright (C) 2005 David S. Miller (davem@davemloft.net)
*/
/* These are here in an effort to more fully work around
* Spitfire Errata #51. Essentially, if a memory barrier
* occurs soon after a mispredicted branch, the chip can stop
* executing instructions until a trap occurs. Therefore, if
* interrupts are disabled, the chip can hang forever.
*
* It used to be believed that the memory barrier had to be
* right in the delay slot, but a case has been traced
* recently wherein the memory barrier was one instruction
* after the branch delay slot and the chip still hung. The
* offending sequence was the following in sym_wakeup_done()
* of the sym53c8xx_2 driver:
*
* call sym_ccb_from_dsa, 0
* movge %icc, 0, %l0
* brz,pn %o0, .LL1303
* mov %o0, %l2
* membar #LoadLoad
*
* The branch has to be mispredicted for the bug to occur.
* Therefore, we put the memory barrier explicitly into a
* "branch always, predicted taken" delay slot to avoid the
* problem case.
*/
.text
99: retl
nop
.globl mb
mb: ba,pt %xcc, 99b
membar #LoadLoad | #LoadStore | #StoreStore | #StoreLoad
.size mb, .-mb
.globl rmb
rmb: ba,pt %xcc, 99b
membar #LoadLoad
.size rmb, .-rmb
.globl wmb
wmb: ba,pt %xcc, 99b
membar #StoreStore
.size wmb, .-wmb
.globl membar_storeload
membar_storeload:
ba,pt %xcc, 99b
membar #StoreLoad
.size membar_storeload, .-membar_storeload
.globl membar_storeload_storestore
membar_storeload_storestore:
ba,pt %xcc, 99b
membar #StoreLoad | #StoreStore
.size membar_storeload_storestore, .-membar_storeload_storestore
.globl membar_storeload_loadload
membar_storeload_loadload:
ba,pt %xcc, 99b
membar #StoreLoad | #LoadLoad
.size membar_storeload_loadload, .-membar_storeload_loadload
.globl membar_storestore_loadstore
membar_storestore_loadstore:
ba,pt %xcc, 99b
membar #StoreStore | #LoadStore
.size membar_storestore_loadstore, .-membar_storestore_loadstore
...@@ -737,7 +737,8 @@ MODULE_LICENSE("GPL"); ...@@ -737,7 +737,8 @@ MODULE_LICENSE("GPL");
extern u32 tl0_solaris[8]; extern u32 tl0_solaris[8];
#define update_ttable(x) \ #define update_ttable(x) \
tl0_solaris[3] = (((long)(x) - (long)tl0_solaris - 3) >> 2) | 0x40000000; \ tl0_solaris[3] = (((long)(x) - (long)tl0_solaris - 3) >> 2) | 0x40000000; \
__asm__ __volatile__ ("membar #StoreStore; flush %0" : : "r" (&tl0_solaris[3])) wmb(); \
__asm__ __volatile__ ("flush %0" : : "r" (&tl0_solaris[3]))
#else #else
#endif #endif
...@@ -761,7 +762,8 @@ int init_module(void) ...@@ -761,7 +762,8 @@ int init_module(void)
entry64_personality_patch |= entry64_personality_patch |=
(offsetof(struct task_struct, personality) + (offsetof(struct task_struct, personality) +
(sizeof(unsigned long) - 1)); (sizeof(unsigned long) - 1));
__asm__ __volatile__("membar #StoreStore; flush %0" wmb();
__asm__ __volatile__("flush %0"
: : "r" (&entry64_personality_patch)); : : "r" (&entry64_personality_patch));
return 0; return 0;
} }
......
...@@ -72,10 +72,10 @@ extern int atomic64_sub_ret(int, atomic64_t *); ...@@ -72,10 +72,10 @@ extern int atomic64_sub_ret(int, atomic64_t *);
/* Atomic operations are already serializing */ /* Atomic operations are already serializing */
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
#define smp_mb__before_atomic_dec() membar("#StoreLoad | #LoadLoad") #define smp_mb__before_atomic_dec() membar_storeload_loadload();
#define smp_mb__after_atomic_dec() membar("#StoreLoad | #StoreStore") #define smp_mb__after_atomic_dec() membar_storeload_storestore();
#define smp_mb__before_atomic_inc() membar("#StoreLoad | #LoadLoad") #define smp_mb__before_atomic_inc() membar_storeload_loadload();
#define smp_mb__after_atomic_inc() membar("#StoreLoad | #StoreStore") #define smp_mb__after_atomic_inc() membar_storeload_storestore();
#else #else
#define smp_mb__before_atomic_dec() barrier() #define smp_mb__before_atomic_dec() barrier()
#define smp_mb__after_atomic_dec() barrier() #define smp_mb__after_atomic_dec() barrier()
......
...@@ -72,8 +72,8 @@ static inline int __test_and_change_bit(int nr, volatile unsigned long *addr) ...@@ -72,8 +72,8 @@ static inline int __test_and_change_bit(int nr, volatile unsigned long *addr)
} }
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
#define smp_mb__before_clear_bit() membar("#StoreLoad | #LoadLoad") #define smp_mb__before_clear_bit() membar_storeload_loadload()
#define smp_mb__after_clear_bit() membar("#StoreLoad | #StoreStore") #define smp_mb__after_clear_bit() membar_storeload_storestore()
#else #else
#define smp_mb__before_clear_bit() barrier() #define smp_mb__before_clear_bit() barrier()
#define smp_mb__after_clear_bit() barrier() #define smp_mb__after_clear_bit() barrier()
......
...@@ -43,7 +43,7 @@ typedef struct { ...@@ -43,7 +43,7 @@ typedef struct {
#define spin_is_locked(lp) ((lp)->lock != 0) #define spin_is_locked(lp) ((lp)->lock != 0)
#define spin_unlock_wait(lp) \ #define spin_unlock_wait(lp) \
do { membar("#LoadLoad"); \ do { rmb(); \
} while((lp)->lock) } while((lp)->lock)
static inline void _raw_spin_lock(spinlock_t *lock) static inline void _raw_spin_lock(spinlock_t *lock)
...@@ -129,7 +129,7 @@ typedef struct { ...@@ -129,7 +129,7 @@ typedef struct {
#define spin_is_locked(__lock) ((__lock)->lock != 0) #define spin_is_locked(__lock) ((__lock)->lock != 0)
#define spin_unlock_wait(__lock) \ #define spin_unlock_wait(__lock) \
do { \ do { \
membar("#LoadLoad"); \ rmb(); \
} while((__lock)->lock) } while((__lock)->lock)
extern void _do_spin_lock(spinlock_t *lock, char *str, unsigned long caller); extern void _do_spin_lock(spinlock_t *lock, char *str, unsigned long caller);
......
...@@ -28,6 +28,14 @@ enum sparc_cpu { ...@@ -28,6 +28,14 @@ enum sparc_cpu {
#define ARCH_SUN4C_SUN4 0 #define ARCH_SUN4C_SUN4 0
#define ARCH_SUN4 0 #define ARCH_SUN4 0
extern void mb(void);
extern void rmb(void);
extern void wmb(void);
extern void membar_storeload(void);
extern void membar_storeload_storestore(void);
extern void membar_storeload_loadload(void);
extern void membar_storestore_loadstore(void);
#endif #endif
#define setipl(__new_ipl) \ #define setipl(__new_ipl) \
...@@ -78,16 +86,11 @@ enum sparc_cpu { ...@@ -78,16 +86,11 @@ enum sparc_cpu {
#define nop() __asm__ __volatile__ ("nop") #define nop() __asm__ __volatile__ ("nop")
#define membar(type) __asm__ __volatile__ ("membar " type : : : "memory")
#define mb() \
membar("#LoadLoad | #LoadStore | #StoreStore | #StoreLoad")
#define rmb() membar("#LoadLoad")
#define wmb() membar("#StoreStore")
#define read_barrier_depends() do { } while(0) #define read_barrier_depends() do { } while(0)
#define set_mb(__var, __value) \ #define set_mb(__var, __value) \
do { __var = __value; membar("#StoreLoad | #StoreStore"); } while(0) do { __var = __value; membar_storeload_storestore(); } while(0)
#define set_wmb(__var, __value) \ #define set_wmb(__var, __value) \
do { __var = __value; membar("#StoreStore"); } while(0) do { __var = __value; wmb(); } while(0)
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
#define smp_mb() mb() #define smp_mb() mb()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment