Commit a5d4b589 authored by Aneesh Kumar K.V's avatar Aneesh Kumar K.V Committed by Michael Ellerman

powerpc/mm: Fixup tlbie vs store ordering issue on POWER9

On POWER9, under some circumstances, a broadcast TLB invalidation
might complete before all previous stores have drained, potentially
allowing stale stores from becoming visible after the invalidation.
This works around it by doubling up those TLB invalidations which was
verified by HW to be sufficient to close the risk window.

This will be documented in a yet-to-be-published errata.

Fixes: 1a472c9d ("powerpc/mm/radix: Add tlbflush routines")
Signed-off-by: default avatarAneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
[mpe: Enable the feature in the DT CPU features code for all Power9,
      rename the feature to CPU_FTR_P9_TLBIE_BUG per benh.]
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent 243fee32
...@@ -203,6 +203,7 @@ static inline void cpu_feature_keys_init(void) { } ...@@ -203,6 +203,7 @@ static inline void cpu_feature_keys_init(void) { }
#define CPU_FTR_DAWR LONG_ASM_CONST(0x0400000000000000) #define CPU_FTR_DAWR LONG_ASM_CONST(0x0400000000000000)
#define CPU_FTR_DABRX LONG_ASM_CONST(0x0800000000000000) #define CPU_FTR_DABRX LONG_ASM_CONST(0x0800000000000000)
#define CPU_FTR_PMAO_BUG LONG_ASM_CONST(0x1000000000000000) #define CPU_FTR_PMAO_BUG LONG_ASM_CONST(0x1000000000000000)
#define CPU_FTR_P9_TLBIE_BUG LONG_ASM_CONST(0x2000000000000000)
#define CPU_FTR_POWER9_DD1 LONG_ASM_CONST(0x4000000000000000) #define CPU_FTR_POWER9_DD1 LONG_ASM_CONST(0x4000000000000000)
#define CPU_FTR_POWER9_DD2_1 LONG_ASM_CONST(0x8000000000000000) #define CPU_FTR_POWER9_DD2_1 LONG_ASM_CONST(0x8000000000000000)
...@@ -465,7 +466,7 @@ static inline void cpu_feature_keys_init(void) { } ...@@ -465,7 +466,7 @@ static inline void cpu_feature_keys_init(void) { }
CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \ CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \
CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_DAWR | \ CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_DAWR | \
CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | \ CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | \
CPU_FTR_PKEY) CPU_FTR_PKEY | CPU_FTR_P9_TLBIE_BUG)
#define CPU_FTRS_POWER9_DD1 ((CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD1) & \ #define CPU_FTRS_POWER9_DD1 ((CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD1) & \
(~CPU_FTR_SAO)) (~CPU_FTR_SAO))
#define CPU_FTRS_POWER9_DD2_0 CPU_FTRS_POWER9 #define CPU_FTRS_POWER9_DD2_0 CPU_FTRS_POWER9
......
...@@ -709,6 +709,9 @@ static __init void cpufeatures_cpu_quirks(void) ...@@ -709,6 +709,9 @@ static __init void cpufeatures_cpu_quirks(void)
cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD1; cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD1;
else if ((version & 0xffffefff) == 0x004e0201) else if ((version & 0xffffefff) == 0x004e0201)
cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1;
if ((version & 0xffff0000) == 0x004e0000)
cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_BUG;
} }
static void __init cpufeatures_setup_finished(void) static void __init cpufeatures_setup_finished(void)
......
...@@ -157,6 +157,9 @@ static void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr, ...@@ -157,6 +157,9 @@ static void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr,
asm volatile("ptesync": : :"memory"); asm volatile("ptesync": : :"memory");
asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1) asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)
: : "r" (addr), "r" (kvm->arch.lpid) : "memory"); : : "r" (addr), "r" (kvm->arch.lpid) : "memory");
if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG))
asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)
: : "r" (addr), "r" (kvm->arch.lpid) : "memory");
asm volatile("ptesync": : :"memory"); asm volatile("ptesync": : :"memory");
} }
......
...@@ -473,6 +473,17 @@ static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues, ...@@ -473,6 +473,17 @@ static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues,
trace_tlbie(kvm->arch.lpid, 0, rbvalues[i], trace_tlbie(kvm->arch.lpid, 0, rbvalues[i],
kvm->arch.lpid, 0, 0, 0); kvm->arch.lpid, 0, 0, 0);
} }
if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
/*
* Need the extra ptesync to make sure we don't
* re-order the tlbie
*/
asm volatile("ptesync": : :"memory");
asm volatile(PPC_TLBIE_5(%0,%1,0,0,0) : :
"r" (rbvalues[0]), "r" (kvm->arch.lpid));
}
asm volatile("eieio; tlbsync; ptesync" : : : "memory"); asm volatile("eieio; tlbsync; ptesync" : : : "memory");
kvm->arch.tlbie_lock = 0; kvm->arch.tlbie_lock = 0;
} else { } else {
......
...@@ -201,6 +201,15 @@ static inline unsigned long ___tlbie(unsigned long vpn, int psize, ...@@ -201,6 +201,15 @@ static inline unsigned long ___tlbie(unsigned long vpn, int psize,
return va; return va;
} }
static inline void fixup_tlbie(unsigned long vpn, int psize, int apsize, int ssize)
{
if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
/* Need the extra ptesync to ensure we don't reorder tlbie*/
asm volatile("ptesync": : :"memory");
___tlbie(vpn, psize, apsize, ssize);
}
}
static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize) static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)
{ {
unsigned long rb; unsigned long rb;
...@@ -278,6 +287,7 @@ static inline void tlbie(unsigned long vpn, int psize, int apsize, ...@@ -278,6 +287,7 @@ static inline void tlbie(unsigned long vpn, int psize, int apsize,
asm volatile("ptesync": : :"memory"); asm volatile("ptesync": : :"memory");
} else { } else {
__tlbie(vpn, psize, apsize, ssize); __tlbie(vpn, psize, apsize, ssize);
fixup_tlbie(vpn, psize, apsize, ssize);
asm volatile("eieio; tlbsync; ptesync": : :"memory"); asm volatile("eieio; tlbsync; ptesync": : :"memory");
} }
if (lock_tlbie && !use_local) if (lock_tlbie && !use_local)
...@@ -771,7 +781,7 @@ static void native_hpte_clear(void) ...@@ -771,7 +781,7 @@ static void native_hpte_clear(void)
*/ */
static void native_flush_hash_range(unsigned long number, int local) static void native_flush_hash_range(unsigned long number, int local)
{ {
unsigned long vpn; unsigned long vpn = 0;
unsigned long hash, index, hidx, shift, slot; unsigned long hash, index, hidx, shift, slot;
struct hash_pte *hptep; struct hash_pte *hptep;
unsigned long hpte_v; unsigned long hpte_v;
...@@ -843,6 +853,10 @@ static void native_flush_hash_range(unsigned long number, int local) ...@@ -843,6 +853,10 @@ static void native_flush_hash_range(unsigned long number, int local)
__tlbie(vpn, psize, psize, ssize); __tlbie(vpn, psize, psize, ssize);
} pte_iterate_hashed_end(); } pte_iterate_hashed_end();
} }
/*
* Just do one more with the last used values.
*/
fixup_tlbie(vpn, psize, psize, ssize);
asm volatile("eieio; tlbsync; ptesync":::"memory"); asm volatile("eieio; tlbsync; ptesync":::"memory");
if (lock_tlbie) if (lock_tlbie)
......
...@@ -481,6 +481,7 @@ void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0, ...@@ -481,6 +481,7 @@ void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
"r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
trace_tlbie(lpid, 0, TLBIEL_INVAL_SET_LPID, lpid, 2, 0, 0); trace_tlbie(lpid, 0, TLBIEL_INVAL_SET_LPID, lpid, 2, 0, 0);
} }
/* do we need fixup here ?*/
asm volatile("eieio; tlbsync; ptesync" : : : "memory"); asm volatile("eieio; tlbsync; ptesync" : : : "memory");
} }
EXPORT_SYMBOL_GPL(mmu_partition_table_set_entry); EXPORT_SYMBOL_GPL(mmu_partition_table_set_entry);
......
...@@ -151,6 +151,17 @@ static inline void __tlbie_va(unsigned long va, unsigned long pid, ...@@ -151,6 +151,17 @@ static inline void __tlbie_va(unsigned long va, unsigned long pid,
trace_tlbie(0, 0, rb, rs, ric, prs, r); trace_tlbie(0, 0, rb, rs, ric, prs, r);
} }
static inline void fixup_tlbie(void)
{
unsigned long pid = 0;
unsigned long va = ((1UL << 52) - 1);
if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
asm volatile("ptesync": : :"memory");
__tlbie_va(va, pid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB);
}
}
/* /*
* We use 128 set in radix mode and 256 set in hpt mode. * We use 128 set in radix mode and 256 set in hpt mode.
*/ */
...@@ -200,6 +211,7 @@ static inline void _tlbie_pid(unsigned long pid, unsigned long ric) ...@@ -200,6 +211,7 @@ static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
default: default:
__tlbie_pid(pid, RIC_FLUSH_ALL); __tlbie_pid(pid, RIC_FLUSH_ALL);
} }
fixup_tlbie();
asm volatile("eieio; tlbsync; ptesync": : :"memory"); asm volatile("eieio; tlbsync; ptesync": : :"memory");
} }
...@@ -253,6 +265,7 @@ static inline void _tlbie_va(unsigned long va, unsigned long pid, ...@@ -253,6 +265,7 @@ static inline void _tlbie_va(unsigned long va, unsigned long pid,
asm volatile("ptesync": : :"memory"); asm volatile("ptesync": : :"memory");
__tlbie_va(va, pid, ap, ric); __tlbie_va(va, pid, ap, ric);
fixup_tlbie();
asm volatile("eieio; tlbsync; ptesync": : :"memory"); asm volatile("eieio; tlbsync; ptesync": : :"memory");
} }
...@@ -264,6 +277,7 @@ static inline void _tlbie_va_range(unsigned long start, unsigned long end, ...@@ -264,6 +277,7 @@ static inline void _tlbie_va_range(unsigned long start, unsigned long end,
if (also_pwc) if (also_pwc)
__tlbie_pid(pid, RIC_FLUSH_PWC); __tlbie_pid(pid, RIC_FLUSH_PWC);
__tlbie_va_range(start, end, pid, page_size, psize); __tlbie_va_range(start, end, pid, page_size, psize);
fixup_tlbie();
asm volatile("eieio; tlbsync; ptesync": : :"memory"); asm volatile("eieio; tlbsync; ptesync": : :"memory");
} }
...@@ -498,6 +512,7 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, ...@@ -498,6 +512,7 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
if (hflush) if (hflush)
__tlbie_va_range(hstart, hend, pid, __tlbie_va_range(hstart, hend, pid,
HPAGE_PMD_SIZE, MMU_PAGE_2M); HPAGE_PMD_SIZE, MMU_PAGE_2M);
fixup_tlbie();
asm volatile("eieio; tlbsync; ptesync": : :"memory"); asm volatile("eieio; tlbsync; ptesync": : :"memory");
} }
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment