Commit f2266504 authored by Marc Zyngier's avatar Marc Zyngier Committed by Catalin Marinas

arm64: Relax ICC_PMR_EL1 accesses when ICC_CTLR_EL1.PMHE is clear

The GICv3 architecture specification is incredibly misleading when it
comes to PMR and the requirement for a DSB. It turns out that this DSB
is only required if the CPU interface sends an Upstream Control
message to the redistributor in order to update the RD's view of PMR.

This message is only sent when ICC_CTLR_EL1.PMHE is set, which isn't
the case in Linux. It can still be set from EL3, so some special care
is required. But the upshot is that in the (hopefuly large) majority
of the cases, we can drop the DSB altogether.

This relies on a new static key being set if the boot CPU has PMHE
set. The drawback is that this static key has to be exported to
modules.

Cc: Will Deacon <will@kernel.org>
Cc: James Morse <james.morse@arm.com>
Cc: Julien Thierry <julien.thierry.kdev@gmail.com>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Signed-off-by: default avatarMarc Zyngier <maz@kernel.org>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 4f5cafb5
...@@ -29,6 +29,18 @@ ...@@ -29,6 +29,18 @@
SB_BARRIER_INSN"nop\n", \ SB_BARRIER_INSN"nop\n", \
ARM64_HAS_SB)) ARM64_HAS_SB))
#ifdef CONFIG_ARM64_PSEUDO_NMI
#define pmr_sync() \
do { \
extern struct static_key_false gic_pmr_sync; \
\
if (static_branch_unlikely(&gic_pmr_sync)) \
dsb(sy); \
} while(0)
#else
#define pmr_sync() do {} while (0)
#endif
#define mb() dsb(sy) #define mb() dsb(sy)
#define rmb() dsb(ld) #define rmb() dsb(ld)
#define wmb() dsb(st) #define wmb() dsb(st)
......
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
#include <linux/irqflags.h> #include <linux/irqflags.h>
#include <asm/arch_gicv3.h> #include <asm/arch_gicv3.h>
#include <asm/barrier.h>
#include <asm/cpufeature.h> #include <asm/cpufeature.h>
#define DAIF_PROCCTX 0 #define DAIF_PROCCTX 0
...@@ -65,7 +66,7 @@ static inline void local_daif_restore(unsigned long flags) ...@@ -65,7 +66,7 @@ static inline void local_daif_restore(unsigned long flags)
if (system_uses_irq_prio_masking()) { if (system_uses_irq_prio_masking()) {
gic_write_pmr(GIC_PRIO_IRQON); gic_write_pmr(GIC_PRIO_IRQON);
dsb(sy); pmr_sync();
} }
} else if (system_uses_irq_prio_masking()) { } else if (system_uses_irq_prio_masking()) {
u64 pmr; u64 pmr;
......
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
#define __ASM_IRQFLAGS_H #define __ASM_IRQFLAGS_H
#include <asm/alternative.h> #include <asm/alternative.h>
#include <asm/barrier.h>
#include <asm/ptrace.h> #include <asm/ptrace.h>
#include <asm/sysreg.h> #include <asm/sysreg.h>
...@@ -34,14 +35,14 @@ static inline void arch_local_irq_enable(void) ...@@ -34,14 +35,14 @@ static inline void arch_local_irq_enable(void)
} }
asm volatile(ALTERNATIVE( asm volatile(ALTERNATIVE(
"msr daifclr, #2 // arch_local_irq_enable\n" "msr daifclr, #2 // arch_local_irq_enable",
"nop", __msr_s(SYS_ICC_PMR_EL1, "%0"),
__msr_s(SYS_ICC_PMR_EL1, "%0")
"dsb sy",
ARM64_HAS_IRQ_PRIO_MASKING) ARM64_HAS_IRQ_PRIO_MASKING)
: :
: "r" ((unsigned long) GIC_PRIO_IRQON) : "r" ((unsigned long) GIC_PRIO_IRQON)
: "memory"); : "memory");
pmr_sync();
} }
static inline void arch_local_irq_disable(void) static inline void arch_local_irq_disable(void)
...@@ -116,14 +117,14 @@ static inline unsigned long arch_local_irq_save(void) ...@@ -116,14 +117,14 @@ static inline unsigned long arch_local_irq_save(void)
static inline void arch_local_irq_restore(unsigned long flags) static inline void arch_local_irq_restore(unsigned long flags)
{ {
asm volatile(ALTERNATIVE( asm volatile(ALTERNATIVE(
"msr daif, %0\n" "msr daif, %0",
"nop", __msr_s(SYS_ICC_PMR_EL1, "%0"),
__msr_s(SYS_ICC_PMR_EL1, "%0") ARM64_HAS_IRQ_PRIO_MASKING)
"dsb sy",
ARM64_HAS_IRQ_PRIO_MASKING)
: :
: "r" (flags) : "r" (flags)
: "memory"); : "memory");
pmr_sync();
} }
#endif /* __ASM_IRQFLAGS_H */ #endif /* __ASM_IRQFLAGS_H */
...@@ -600,8 +600,7 @@ static inline void kvm_arm_vhe_guest_enter(void) ...@@ -600,8 +600,7 @@ static inline void kvm_arm_vhe_guest_enter(void)
* local_daif_mask() already sets GIC_PRIO_PSR_I_SET, we just need a * local_daif_mask() already sets GIC_PRIO_PSR_I_SET, we just need a
* dsb to ensure the redistributor is forwards EL2 IRQs to the CPU. * dsb to ensure the redistributor is forwards EL2 IRQs to the CPU.
*/ */
if (system_uses_irq_prio_masking()) pmr_sync();
dsb(sy);
} }
static inline void kvm_arm_vhe_guest_exit(void) static inline void kvm_arm_vhe_guest_exit(void)
......
...@@ -269,8 +269,10 @@ alternative_else_nop_endif ...@@ -269,8 +269,10 @@ alternative_else_nop_endif
alternative_if ARM64_HAS_IRQ_PRIO_MASKING alternative_if ARM64_HAS_IRQ_PRIO_MASKING
ldr x20, [sp, #S_PMR_SAVE] ldr x20, [sp, #S_PMR_SAVE]
msr_s SYS_ICC_PMR_EL1, x20 msr_s SYS_ICC_PMR_EL1, x20
/* Ensure priority change is seen by redistributor */ mrs_s x21, SYS_ICC_CTLR_EL1
dsb sy tbz x21, #6, .L__skip_pmr_sync\@ // Check for ICC_CTLR_EL1.PMHE
dsb sy // Ensure priority change is seen by redistributor
.L__skip_pmr_sync\@:
alternative_else_nop_endif alternative_else_nop_endif
ldp x21, x22, [sp, #S_PC] // load ELR, SPSR ldp x21, x22, [sp, #S_PC] // load ELR, SPSR
......
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
#include <kvm/arm_psci.h> #include <kvm/arm_psci.h>
#include <asm/arch_gicv3.h> #include <asm/barrier.h>
#include <asm/cpufeature.h> #include <asm/cpufeature.h>
#include <asm/kprobes.h> #include <asm/kprobes.h>
#include <asm/kvm_asm.h> #include <asm/kvm_asm.h>
...@@ -592,7 +592,7 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu) ...@@ -592,7 +592,7 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
*/ */
if (system_uses_irq_prio_masking()) { if (system_uses_irq_prio_masking()) {
gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
dsb(sy); pmr_sync();
} }
vcpu = kern_hyp_va(vcpu); vcpu = kern_hyp_va(vcpu);
......
...@@ -87,6 +87,15 @@ static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key); ...@@ -87,6 +87,15 @@ static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key);
*/ */
static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis); static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis);
/*
* Global static key controlling whether an update to PMR allowing more
* interrupts requires to be propagated to the redistributor (DSB SY).
* And this needs to be exported for modules to be able to enable
* interrupts...
*/
DEFINE_STATIC_KEY_FALSE(gic_pmr_sync);
EXPORT_SYMBOL(gic_pmr_sync);
/* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */ /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */
static refcount_t *ppi_nmi_refs; static refcount_t *ppi_nmi_refs;
...@@ -1502,6 +1511,17 @@ static void gic_enable_nmi_support(void) ...@@ -1502,6 +1511,17 @@ static void gic_enable_nmi_support(void)
for (i = 0; i < gic_data.ppi_nr; i++) for (i = 0; i < gic_data.ppi_nr; i++)
refcount_set(&ppi_nmi_refs[i], 0); refcount_set(&ppi_nmi_refs[i], 0);
/*
* Linux itself doesn't use 1:N distribution, so has no need to
* set PMHE. The only reason to have it set is if EL3 requires it
* (and we can't change it).
*/
if (gic_read_ctlr() & ICC_CTLR_EL1_PMHE_MASK)
static_branch_enable(&gic_pmr_sync);
pr_info("%s ICC_PMR_EL1 synchronisation\n",
static_branch_unlikely(&gic_pmr_sync) ? "Forcing" : "Relaxing");
static_branch_enable(&supports_pseudo_nmis); static_branch_enable(&supports_pseudo_nmis);
if (static_branch_likely(&supports_deactivate_key)) if (static_branch_likely(&supports_deactivate_key))
......
...@@ -487,6 +487,8 @@ ...@@ -487,6 +487,8 @@
#define ICC_CTLR_EL1_EOImode_MASK (1 << ICC_CTLR_EL1_EOImode_SHIFT) #define ICC_CTLR_EL1_EOImode_MASK (1 << ICC_CTLR_EL1_EOImode_SHIFT)
#define ICC_CTLR_EL1_CBPR_SHIFT 0 #define ICC_CTLR_EL1_CBPR_SHIFT 0
#define ICC_CTLR_EL1_CBPR_MASK (1 << ICC_CTLR_EL1_CBPR_SHIFT) #define ICC_CTLR_EL1_CBPR_MASK (1 << ICC_CTLR_EL1_CBPR_SHIFT)
#define ICC_CTLR_EL1_PMHE_SHIFT 6
#define ICC_CTLR_EL1_PMHE_MASK (1 << ICC_CTLR_EL1_PMHE_SHIFT)
#define ICC_CTLR_EL1_PRI_BITS_SHIFT 8 #define ICC_CTLR_EL1_PRI_BITS_SHIFT 8
#define ICC_CTLR_EL1_PRI_BITS_MASK (0x7 << ICC_CTLR_EL1_PRI_BITS_SHIFT) #define ICC_CTLR_EL1_PRI_BITS_MASK (0x7 << ICC_CTLR_EL1_PRI_BITS_SHIFT)
#define ICC_CTLR_EL1_ID_BITS_SHIFT 11 #define ICC_CTLR_EL1_ID_BITS_SHIFT 11
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment