Commit b866cc21 authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Michael Ellerman

powerpc: Change the doorbell IPI calling convention

Change the doorbell callers to know about their msgsnd addressing,
rather than have them set a per-cpu target data tag at boot that gets
sent to the cause_ipi functions. The data is only used for doorbell IPI
functions, no other IPI types, so it makes sense to keep that detail
local to doorbell.

Have the platform code understand doorbell IPIs, rather than the
interrupt controller code understand them. Platform code can look at
capabilities it has available and decide which to use.
Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent 9b7ff0c6
...@@ -35,8 +35,6 @@ enum ppc_dbell { ...@@ -35,8 +35,6 @@ enum ppc_dbell {
#ifdef CONFIG_PPC_BOOK3S #ifdef CONFIG_PPC_BOOK3S
#define PPC_DBELL_MSGTYPE PPC_DBELL_SERVER #define PPC_DBELL_MSGTYPE PPC_DBELL_SERVER
#define SPRN_DOORBELL_CPUTAG SPRN_TIR
#define PPC_DBELL_TAG_MASK 0x7f
static inline void _ppc_msgsnd(u32 msg) static inline void _ppc_msgsnd(u32 msg)
{ {
...@@ -49,8 +47,6 @@ static inline void _ppc_msgsnd(u32 msg) ...@@ -49,8 +47,6 @@ static inline void _ppc_msgsnd(u32 msg)
#else /* CONFIG_PPC_BOOK3S */ #else /* CONFIG_PPC_BOOK3S */
#define PPC_DBELL_MSGTYPE PPC_DBELL #define PPC_DBELL_MSGTYPE PPC_DBELL
#define SPRN_DOORBELL_CPUTAG SPRN_PIR
#define PPC_DBELL_TAG_MASK 0x3fff
static inline void _ppc_msgsnd(u32 msg) static inline void _ppc_msgsnd(u32 msg)
{ {
...@@ -59,9 +55,10 @@ static inline void _ppc_msgsnd(u32 msg) ...@@ -59,9 +55,10 @@ static inline void _ppc_msgsnd(u32 msg)
#endif /* CONFIG_PPC_BOOK3S */ #endif /* CONFIG_PPC_BOOK3S */
extern void doorbell_cause_ipi(int cpu, unsigned long data); extern void doorbell_global_ipi(int cpu);
extern void doorbell_core_ipi(int cpu);
extern int doorbell_try_core_ipi(int cpu);
extern void doorbell_exception(struct pt_regs *regs); extern void doorbell_exception(struct pt_regs *regs);
extern void doorbell_setup_this_cpu(void);
static inline void ppc_msgsnd(enum ppc_dbell type, u32 flags, u32 tag) static inline void ppc_msgsnd(enum ppc_dbell type, u32 flags, u32 tag)
{ {
......
...@@ -40,7 +40,7 @@ extern int cpu_to_chip_id(int cpu); ...@@ -40,7 +40,7 @@ extern int cpu_to_chip_id(int cpu);
struct smp_ops_t { struct smp_ops_t {
void (*message_pass)(int cpu, int msg); void (*message_pass)(int cpu, int msg);
#ifdef CONFIG_PPC_SMP_MUXED_IPI #ifdef CONFIG_PPC_SMP_MUXED_IPI
void (*cause_ipi)(int cpu, unsigned long data); void (*cause_ipi)(int cpu);
#endif #endif
void (*probe)(void); void (*probe)(void);
int (*kick_cpu)(int nr); int (*kick_cpu)(int nr);
...@@ -125,7 +125,6 @@ extern int smp_request_message_ipi(int virq, int message); ...@@ -125,7 +125,6 @@ extern int smp_request_message_ipi(int virq, int message);
extern const char *smp_ipi_name[]; extern const char *smp_ipi_name[];
/* for irq controllers with only a single ipi */ /* for irq controllers with only a single ipi */
extern void smp_muxed_ipi_set_data(int cpu, unsigned long data);
extern void smp_muxed_ipi_message_pass(int cpu, int msg); extern void smp_muxed_ipi_message_pass(int cpu, int msg);
extern void smp_muxed_ipi_set_message(int cpu, int msg); extern void smp_muxed_ipi_set_message(int cpu, int msg);
extern irqreturn_t smp_ipi_demux(void); extern irqreturn_t smp_ipi_demux(void);
......
...@@ -57,7 +57,7 @@ struct icp_ops { ...@@ -57,7 +57,7 @@ struct icp_ops {
void (*teardown_cpu)(void); void (*teardown_cpu)(void);
void (*flush_ipi)(void); void (*flush_ipi)(void);
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
void (*cause_ipi)(int cpu, unsigned long data); void (*cause_ipi)(int cpu);
irq_handler_t ipi_action; irq_handler_t ipi_action;
#endif #endif
}; };
......
...@@ -20,18 +20,60 @@ ...@@ -20,18 +20,60 @@
#include <asm/kvm_ppc.h> #include <asm/kvm_ppc.h>
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
void doorbell_setup_this_cpu(void)
/*
* Doorbells must only be used if CPU_FTR_DBELL is available.
* msgsnd is used in HV, and msgsndp is used in !HV.
*
* These should be used by platform code that is aware of restrictions.
* Other arch code should use ->cause_ipi.
*
* doorbell_global_ipi() sends a dbell to any target CPU.
* Must be used only by architectures that address msgsnd target
* by PIR/get_hard_smp_processor_id.
*/
void doorbell_global_ipi(int cpu)
{ {
unsigned long tag = mfspr(SPRN_DOORBELL_CPUTAG) & PPC_DBELL_TAG_MASK; u32 tag = get_hard_smp_processor_id(cpu);
smp_muxed_ipi_set_data(smp_processor_id(), tag); kvmppc_set_host_ipi(cpu, 1);
/* Order previous accesses vs. msgsnd, which is treated as a store */
mb();
ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, tag);
} }
void doorbell_cause_ipi(int cpu, unsigned long data) /*
* doorbell_core_ipi() sends a dbell to a target CPU in the same core.
* Must be used only by architectures that address msgsnd target
* by TIR/cpu_thread_in_core.
*/
void doorbell_core_ipi(int cpu)
{ {
u32 tag = cpu_thread_in_core(cpu);
kvmppc_set_host_ipi(cpu, 1);
/* Order previous accesses vs. msgsnd, which is treated as a store */ /* Order previous accesses vs. msgsnd, which is treated as a store */
mb(); mb();
ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, data); ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, tag);
}
/*
* Attempt to cause a core doorbell if destination is on the same core.
* Returns 1 on success, 0 on failure.
*/
int doorbell_try_core_ipi(int cpu)
{
int this_cpu = get_cpu();
int ret = 0;
if (cpumask_test_cpu(cpu, cpu_sibling_mask(this_cpu))) {
doorbell_core_ipi(cpu);
ret = 1;
}
put_cpu();
return ret;
} }
void doorbell_exception(struct pt_regs *regs) void doorbell_exception(struct pt_regs *regs)
......
...@@ -39,6 +39,7 @@ ...@@ -39,6 +39,7 @@
#include <asm/irq.h> #include <asm/irq.h>
#include <asm/hw_irq.h> #include <asm/hw_irq.h>
#include <asm/kvm_ppc.h> #include <asm/kvm_ppc.h>
#include <asm/dbell.h>
#include <asm/page.h> #include <asm/page.h>
#include <asm/pgtable.h> #include <asm/pgtable.h>
#include <asm/prom.h> #include <asm/prom.h>
...@@ -211,17 +212,9 @@ int smp_request_message_ipi(int virq, int msg) ...@@ -211,17 +212,9 @@ int smp_request_message_ipi(int virq, int msg)
#ifdef CONFIG_PPC_SMP_MUXED_IPI #ifdef CONFIG_PPC_SMP_MUXED_IPI
struct cpu_messages { struct cpu_messages {
long messages; /* current messages */ long messages; /* current messages */
unsigned long data; /* data for cause ipi */
}; };
static DEFINE_PER_CPU_SHARED_ALIGNED(struct cpu_messages, ipi_message); static DEFINE_PER_CPU_SHARED_ALIGNED(struct cpu_messages, ipi_message);
void smp_muxed_ipi_set_data(int cpu, unsigned long data)
{
struct cpu_messages *info = &per_cpu(ipi_message, cpu);
info->data = data;
}
void smp_muxed_ipi_set_message(int cpu, int msg) void smp_muxed_ipi_set_message(int cpu, int msg)
{ {
struct cpu_messages *info = &per_cpu(ipi_message, cpu); struct cpu_messages *info = &per_cpu(ipi_message, cpu);
...@@ -236,14 +229,13 @@ void smp_muxed_ipi_set_message(int cpu, int msg) ...@@ -236,14 +229,13 @@ void smp_muxed_ipi_set_message(int cpu, int msg)
void smp_muxed_ipi_message_pass(int cpu, int msg) void smp_muxed_ipi_message_pass(int cpu, int msg)
{ {
struct cpu_messages *info = &per_cpu(ipi_message, cpu);
smp_muxed_ipi_set_message(cpu, msg); smp_muxed_ipi_set_message(cpu, msg);
/* /*
* cause_ipi functions are required to include a full barrier * cause_ipi functions are required to include a full barrier
* before doing whatever causes the IPI. * before doing whatever causes the IPI.
*/ */
smp_ops->cause_ipi(cpu, info->data); smp_ops->cause_ipi(cpu);
} }
#ifdef __BIG_ENDIAN__ #ifdef __BIG_ENDIAN__
...@@ -254,11 +246,12 @@ void smp_muxed_ipi_message_pass(int cpu, int msg) ...@@ -254,11 +246,12 @@ void smp_muxed_ipi_message_pass(int cpu, int msg)
irqreturn_t smp_ipi_demux(void) irqreturn_t smp_ipi_demux(void)
{ {
struct cpu_messages *info = this_cpu_ptr(&ipi_message); struct cpu_messages *info;
unsigned long all; unsigned long all;
mb(); /* order any irq clear */ mb(); /* order any irq clear */
info = this_cpu_ptr(&ipi_message);
do { do {
all = xchg(&info->messages, 0); all = xchg(&info->messages, 0);
#if defined(CONFIG_KVM_XICS) && defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE) #if defined(CONFIG_KVM_XICS) && defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE)
......
...@@ -461,16 +461,9 @@ static void mpc85xx_smp_machine_kexec(struct kimage *image) ...@@ -461,16 +461,9 @@ static void mpc85xx_smp_machine_kexec(struct kimage *image)
} }
#endif /* CONFIG_KEXEC_CORE */ #endif /* CONFIG_KEXEC_CORE */
static void smp_85xx_basic_setup(int cpu_nr)
{
if (cpu_has_feature(CPU_FTR_DBELL))
doorbell_setup_this_cpu();
}
static void smp_85xx_setup_cpu(int cpu_nr) static void smp_85xx_setup_cpu(int cpu_nr)
{ {
mpic_setup_this_cpu(); mpic_setup_this_cpu();
smp_85xx_basic_setup(cpu_nr);
} }
void __init mpc85xx_smp_init(void) void __init mpc85xx_smp_init(void)
...@@ -484,7 +477,7 @@ void __init mpc85xx_smp_init(void) ...@@ -484,7 +477,7 @@ void __init mpc85xx_smp_init(void)
smp_85xx_ops.setup_cpu = smp_85xx_setup_cpu; smp_85xx_ops.setup_cpu = smp_85xx_setup_cpu;
smp_85xx_ops.message_pass = smp_mpic_message_pass; smp_85xx_ops.message_pass = smp_mpic_message_pass;
} else } else
smp_85xx_ops.setup_cpu = smp_85xx_basic_setup; smp_85xx_ops.setup_cpu = NULL;
if (cpu_has_feature(CPU_FTR_DBELL)) { if (cpu_has_feature(CPU_FTR_DBELL)) {
/* /*
...@@ -492,7 +485,7 @@ void __init mpc85xx_smp_init(void) ...@@ -492,7 +485,7 @@ void __init mpc85xx_smp_init(void)
* smp_muxed_ipi_message_pass * smp_muxed_ipi_message_pass
*/ */
smp_85xx_ops.message_pass = NULL; smp_85xx_ops.message_pass = NULL;
smp_85xx_ops.cause_ipi = doorbell_cause_ipi; smp_85xx_ops.cause_ipi = doorbell_global_ipi;
smp_85xx_ops.probe = NULL; smp_85xx_ops.probe = NULL;
} }
......
...@@ -172,7 +172,7 @@ static irqreturn_t psurge_ipi_intr(int irq, void *d) ...@@ -172,7 +172,7 @@ static irqreturn_t psurge_ipi_intr(int irq, void *d)
return IRQ_HANDLED; return IRQ_HANDLED;
} }
static void smp_psurge_cause_ipi(int cpu, unsigned long data) static void smp_psurge_cause_ipi(int cpu)
{ {
psurge_set_ipi(cpu); psurge_set_ipi(cpu);
} }
......
...@@ -53,11 +53,6 @@ static void pnv_smp_setup_cpu(int cpu) ...@@ -53,11 +53,6 @@ static void pnv_smp_setup_cpu(int cpu)
xive_smp_setup_cpu(); xive_smp_setup_cpu();
else if (cpu != boot_cpuid) else if (cpu != boot_cpuid)
xics_setup_cpu(); xics_setup_cpu();
#ifdef CONFIG_PPC_DOORBELL
if (cpu_has_feature(CPU_FTR_DBELL))
doorbell_setup_this_cpu();
#endif
} }
static int pnv_smp_kick_cpu(int nr) static int pnv_smp_kick_cpu(int nr)
...@@ -254,17 +249,31 @@ static int pnv_smp_prepare_cpu(int cpu) ...@@ -254,17 +249,31 @@ static int pnv_smp_prepare_cpu(int cpu)
return 0; return 0;
} }
static void pnv_cause_ipi(int cpu)
{
if (doorbell_try_core_ipi(cpu))
return;
icp_ops->cause_ipi(cpu);
}
static void __init pnv_smp_probe(void) static void __init pnv_smp_probe(void)
{ {
if (xive_enabled()) if (xive_enabled())
xive_smp_probe(); xive_smp_probe();
else else
xics_smp_probe(); xics_smp_probe();
if (cpu_has_feature(CPU_FTR_DBELL) && !cpu_has_feature(CPU_FTR_ARCH_300)) {
smp_ops->cause_ipi = pnv_cause_ipi;
} else {
smp_ops->cause_ipi = icp_ops->cause_ipi;
}
} }
static struct smp_ops_t pnv_smp_ops = { static struct smp_ops_t pnv_smp_ops = {
.message_pass = smp_muxed_ipi_message_pass, .message_pass = NULL, /* Use smp_muxed_ipi_message_pass */
.cause_ipi = NULL, /* Filled at runtime by xi{cs,ve}_smp_probe() */ .cause_ipi = NULL, /* Filled at runtime by pnv_smp_probe() */
.probe = pnv_smp_probe, .probe = pnv_smp_probe,
.prepare_cpu = pnv_smp_prepare_cpu, .prepare_cpu = pnv_smp_prepare_cpu,
.kick_cpu = pnv_smp_kick_cpu, .kick_cpu = pnv_smp_kick_cpu,
......
...@@ -55,11 +55,6 @@ ...@@ -55,11 +55,6 @@
*/ */
static cpumask_var_t of_spin_mask; static cpumask_var_t of_spin_mask;
/*
* If we multiplex IPI mechanisms, store the appropriate XICS IPI mechanism here
*/
static void (*xics_cause_ipi)(int cpu, unsigned long data);
/* Query where a cpu is now. Return codes #defined in plpar_wrappers.h */ /* Query where a cpu is now. Return codes #defined in plpar_wrappers.h */
int smp_query_cpu_stopped(unsigned int pcpu) int smp_query_cpu_stopped(unsigned int pcpu)
{ {
...@@ -143,8 +138,6 @@ static void smp_setup_cpu(int cpu) ...@@ -143,8 +138,6 @@ static void smp_setup_cpu(int cpu)
{ {
if (cpu != boot_cpuid) if (cpu != boot_cpuid)
xics_setup_cpu(); xics_setup_cpu();
if (cpu_has_feature(CPU_FTR_DBELL))
doorbell_setup_this_cpu();
if (firmware_has_feature(FW_FEATURE_SPLPAR)) if (firmware_has_feature(FW_FEATURE_SPLPAR))
vpa_init(cpu); vpa_init(cpu);
...@@ -187,23 +180,23 @@ static int smp_pSeries_kick_cpu(int nr) ...@@ -187,23 +180,23 @@ static int smp_pSeries_kick_cpu(int nr)
return 0; return 0;
} }
/* Only used on systems that support multiple IPI mechanisms */ static void smp_pseries_cause_ipi(int cpu)
static void pSeries_cause_ipi_mux(int cpu, unsigned long data)
{ {
if (cpumask_test_cpu(cpu, cpu_sibling_mask(smp_processor_id()))) /* POWER9 should not use this handler */
doorbell_cause_ipi(cpu, data); if (doorbell_try_core_ipi(cpu))
else return;
xics_cause_ipi(cpu, data);
icp_ops->cause_ipi(cpu);
} }
static __init void pSeries_smp_probe(void) static __init void pSeries_smp_probe(void)
{ {
xics_smp_probe(); xics_smp_probe();
if (cpu_has_feature(CPU_FTR_DBELL)) { if (cpu_has_feature(CPU_FTR_DBELL))
xics_cause_ipi = smp_ops->cause_ipi; smp_ops->cause_ipi = smp_pseries_cause_ipi;
smp_ops->cause_ipi = pSeries_cause_ipi_mux; else
} smp_ops->cause_ipi = icp_ops->cause_ipi;
} }
static struct smp_ops_t pseries_smp_ops = { static struct smp_ops_t pseries_smp_ops = {
......
...@@ -138,7 +138,7 @@ static void icp_hv_set_cpu_priority(unsigned char cppr) ...@@ -138,7 +138,7 @@ static void icp_hv_set_cpu_priority(unsigned char cppr)
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
static void icp_hv_cause_ipi(int cpu, unsigned long data) static void icp_hv_cause_ipi(int cpu)
{ {
icp_hv_set_qirr(cpu, IPI_PRIORITY); icp_hv_set_qirr(cpu, IPI_PRIORITY);
} }
......
...@@ -143,19 +143,9 @@ static unsigned int icp_native_get_irq(void) ...@@ -143,19 +143,9 @@ static unsigned int icp_native_get_irq(void)
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
static void icp_native_cause_ipi(int cpu, unsigned long data) static void icp_native_cause_ipi(int cpu)
{ {
kvmppc_set_host_ipi(cpu, 1); kvmppc_set_host_ipi(cpu, 1);
#ifdef CONFIG_PPC_DOORBELL
if (cpu_has_feature(CPU_FTR_DBELL)) {
if (cpumask_test_cpu(cpu, cpu_sibling_mask(get_cpu()))) {
doorbell_cause_ipi(cpu, data);
put_cpu();
return;
}
put_cpu();
}
#endif
icp_native_set_qirr(cpu, IPI_PRIORITY); icp_native_set_qirr(cpu, IPI_PRIORITY);
} }
......
...@@ -126,7 +126,7 @@ static void icp_opal_eoi(struct irq_data *d) ...@@ -126,7 +126,7 @@ static void icp_opal_eoi(struct irq_data *d)
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
static void icp_opal_cause_ipi(int cpu, unsigned long data) static void icp_opal_cause_ipi(int cpu)
{ {
int hw_cpu = get_hard_smp_processor_id(cpu); int hw_cpu = get_hard_smp_processor_id(cpu);
......
...@@ -143,9 +143,6 @@ static void xics_request_ipi(void) ...@@ -143,9 +143,6 @@ static void xics_request_ipi(void)
void __init xics_smp_probe(void) void __init xics_smp_probe(void)
{ {
/* Setup cause_ipi callback based on which ICP is used */
smp_ops->cause_ipi = icp_ops->cause_ipi;
/* Register all the IPIs */ /* Register all the IPIs */
xics_request_ipi(); xics_request_ipi();
} }
......
...@@ -834,15 +834,15 @@ static void xive_irq_free_data(unsigned int virq) ...@@ -834,15 +834,15 @@ static void xive_irq_free_data(unsigned int virq)
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
static void xive_cause_ipi(int cpu, unsigned long msg) static void xive_cause_ipi(int cpu)
{ {
struct xive_cpu *xc; struct xive_cpu *xc;
struct xive_irq_data *xd; struct xive_irq_data *xd;
xc = per_cpu(xive_cpu, cpu); xc = per_cpu(xive_cpu, cpu);
DBG_VERBOSE("IPI msg#%ld CPU %d -> %d (HW IRQ 0x%x)\n", DBG_VERBOSE("IPI CPU %d -> %d (HW IRQ 0x%x)\n",
msg, smp_processor_id(), cpu, xc->hw_ipi); smp_processor_id(), cpu, xc->hw_ipi);
xd = &xc->ipi_data; xd = &xc->ipi_data;
if (WARN_ON(!xd->trig_mmio)) if (WARN_ON(!xd->trig_mmio))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment