Commit 8a1e97ee authored by Ralf Baechle's avatar Ralf Baechle

[MIPS] SMTC: Fix recursion in instant IPI replay code.

local_irq_restore -> raw_local_irq_restore -> irq_restore_epilog ->
	smtc_ipi_replay -> smtc_ipi_dq -> spin_unlock_irqrestore ->
	_spin_unlock_irqrestore -> local_irq_restore

The recursion does abort when there is no more IPI queued for a CPU, so
this isn't usually fatal which is why we got away with this for so long
until this was discovered by code inspection.
Signed-off-by: default avatarRalf Baechle <ralf@linux-mips.org>
parent 6c9fde4b
...@@ -999,10 +999,17 @@ static void setup_cross_vpe_interrupts(unsigned int nvpe) ...@@ -999,10 +999,17 @@ static void setup_cross_vpe_interrupts(unsigned int nvpe)
/* /*
* SMTC-specific hacks invoked from elsewhere in the kernel. * SMTC-specific hacks invoked from elsewhere in the kernel.
*
* smtc_ipi_replay is called from raw_local_irq_restore which is only ever
* called with interrupts disabled. We do rely on interrupts being disabled
* here because using spin_lock_irqsave()/spin_unlock_irqrestore() would
* result in a recursive call to raw_local_irq_restore().
*/ */
void smtc_ipi_replay(void) static void __smtc_ipi_replay(void)
{ {
unsigned int cpu = smp_processor_id();
/* /*
* To the extent that we've ever turned interrupts off, * To the extent that we've ever turned interrupts off,
* we may have accumulated deferred IPIs. This is subtle. * we may have accumulated deferred IPIs. This is subtle.
...@@ -1017,17 +1024,30 @@ void smtc_ipi_replay(void) ...@@ -1017,17 +1024,30 @@ void smtc_ipi_replay(void)
* is clear, and we'll handle it as a real pseudo-interrupt * is clear, and we'll handle it as a real pseudo-interrupt
* and not a pseudo-pseudo interrupt. * and not a pseudo-pseudo interrupt.
*/ */
if (IPIQ[smp_processor_id()].depth > 0) { if (IPIQ[cpu].depth > 0) {
struct smtc_ipi *pipi; while (1) {
extern void self_ipi(struct smtc_ipi *); struct smtc_ipi_q *q = &IPIQ[cpu];
struct smtc_ipi *pipi;
extern void self_ipi(struct smtc_ipi *);
spin_lock(&q->lock);
pipi = __smtc_ipi_dq(q);
spin_unlock(&q->lock);
if (!pipi)
break;
while ((pipi = smtc_ipi_dq(&IPIQ[smp_processor_id()]))) {
self_ipi(pipi); self_ipi(pipi);
smtc_cpu_stats[smp_processor_id()].selfipis++; smtc_cpu_stats[cpu].selfipis++;
} }
} }
} }
void smtc_ipi_replay(void)
{
raw_local_irq_disable();
__smtc_ipi_replay();
}
EXPORT_SYMBOL(smtc_ipi_replay); EXPORT_SYMBOL(smtc_ipi_replay);
void smtc_idle_loop_hook(void) void smtc_idle_loop_hook(void)
...@@ -1132,7 +1152,13 @@ void smtc_idle_loop_hook(void) ...@@ -1132,7 +1152,13 @@ void smtc_idle_loop_hook(void)
* is in use, there should never be any. * is in use, there should never be any.
*/ */
#ifndef CONFIG_MIPS_MT_SMTC_INSTANT_REPLAY #ifndef CONFIG_MIPS_MT_SMTC_INSTANT_REPLAY
smtc_ipi_replay(); {
unsigned long flags;
local_irq_save(flags);
__smtc_ipi_replay();
local_irq_restore(flags);
}
#endif /* CONFIG_MIPS_MT_SMTC_INSTANT_REPLAY */ #endif /* CONFIG_MIPS_MT_SMTC_INSTANT_REPLAY */
} }
......
...@@ -13,29 +13,9 @@ ...@@ -13,29 +13,9 @@
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
#include <linux/compiler.h>
#include <asm/hazards.h> #include <asm/hazards.h>
/*
* CONFIG_MIPS_MT_SMTC_INSTANT_REPLAY does prompt replay of deferred IPIs,
* at the cost of branch and call overhead on each local_irq_restore()
*/
#ifdef CONFIG_MIPS_MT_SMTC_INSTANT_REPLAY
extern void smtc_ipi_replay(void);
#define irq_restore_epilog(flags) \
do { \
if (!(flags & 0x0400)) \
smtc_ipi_replay(); \
} while (0)
#else
#define irq_restore_epilog(ignore) do { } while (0)
#endif /* CONFIG_MIPS_MT_SMTC_INSTANT_REPLAY */
__asm__ ( __asm__ (
" .macro raw_local_irq_enable \n" " .macro raw_local_irq_enable \n"
" .set push \n" " .set push \n"
...@@ -205,17 +185,28 @@ __asm__ ( ...@@ -205,17 +185,28 @@ __asm__ (
" .set pop \n" " .set pop \n"
" .endm \n"); " .endm \n");
#define raw_local_irq_restore(flags) \ extern void smtc_ipi_replay(void);
do { \
unsigned long __tmp1; \ static inline void raw_local_irq_restore(unsigned long flags)
\ {
__asm__ __volatile__( \ unsigned long __tmp1;
"raw_local_irq_restore\t%0" \
: "=r" (__tmp1) \ #ifdef CONFIG_MIPS_MT_SMTC_INSTANT_REPLAY
: "0" (flags) \ /*
: "memory"); \ * CONFIG_MIPS_MT_SMTC_INSTANT_REPLAY does prompt replay of deferred
irq_restore_epilog(flags); \ * IPIs, at the cost of branch and call overhead on each
} while(0) * local_irq_restore()
*/
if (unlikely(!(flags & 0x0400)))
smtc_ipi_replay();
#endif
__asm__ __volatile__(
"raw_local_irq_restore\t%0"
: "=r" (__tmp1)
: "0" (flags)
: "memory");
}
static inline int raw_irqs_disabled_flags(unsigned long flags) static inline int raw_irqs_disabled_flags(unsigned long flags)
{ {
......
...@@ -65,12 +65,10 @@ static inline void smtc_ipi_nq(struct smtc_ipi_q *q, struct smtc_ipi *p) ...@@ -65,12 +65,10 @@ static inline void smtc_ipi_nq(struct smtc_ipi_q *q, struct smtc_ipi *p)
spin_unlock_irqrestore(&q->lock, flags); spin_unlock_irqrestore(&q->lock, flags);
} }
static inline struct smtc_ipi *smtc_ipi_dq(struct smtc_ipi_q *q) static inline struct smtc_ipi *__smtc_ipi_dq(struct smtc_ipi_q *q)
{ {
struct smtc_ipi *p; struct smtc_ipi *p;
long flags;
spin_lock_irqsave(&q->lock, flags);
if (q->head == NULL) if (q->head == NULL)
p = NULL; p = NULL;
else { else {
...@@ -81,7 +79,19 @@ static inline struct smtc_ipi *smtc_ipi_dq(struct smtc_ipi_q *q) ...@@ -81,7 +79,19 @@ static inline struct smtc_ipi *smtc_ipi_dq(struct smtc_ipi_q *q)
if (q->head == NULL) if (q->head == NULL)
q->tail = NULL; q->tail = NULL;
} }
return p;
}
static inline struct smtc_ipi *smtc_ipi_dq(struct smtc_ipi_q *q)
{
unsigned long flags;
struct smtc_ipi *p;
spin_lock_irqsave(&q->lock, flags);
p = __smtc_ipi_dq(q);
spin_unlock_irqrestore(&q->lock, flags); spin_unlock_irqrestore(&q->lock, flags);
return p; return p;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment