Commit 6b243fcf authored by Paul Mackerras's avatar Paul Mackerras Committed by Michael Ellerman

powerpc/64: Simplify adaptation to new ISA v3.00 HPTE format

This changes the way that we support the new ISA v3.00 HPTE format.
Instead of adapting everything that uses HPTE values to handle either
the old format or the new format, depending on which CPU we are on,
we now convert explicitly between old and new formats if necessary
in the low-level routines that actually access HPTEs in memory.
This limits the amount of code that needs to know about the new
format and makes the conversions explicit.  This is OK because the
old format contains all the information that is in the new format.

This also fixes operation under a hypervisor, because the H_ENTER
hypercall (and other hypercalls that deal with HPTEs) will continue
to require the HPTE value to be supplied in the old format.  At
present the kernel will not boot in HPT mode on POWER9 under a
hypervisor.

This fixes and partially reverts commit 50de596d
("powerpc/mm/hash: Add support for Power9 Hash", 2016-04-29).

Fixes: 50de596d ("powerpc/mm/hash: Add support for Power9 Hash")
Signed-off-by: default avatarPaul Mackerras <paulus@ozlabs.org>
Reviewed-by: default avatarAneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent a25f0944
...@@ -70,7 +70,9 @@ ...@@ -70,7 +70,9 @@
#define HPTE_V_SSIZE_SHIFT 62 #define HPTE_V_SSIZE_SHIFT 62
#define HPTE_V_AVPN_SHIFT 7 #define HPTE_V_AVPN_SHIFT 7
#define HPTE_V_COMMON_BITS ASM_CONST(0x000fffffffffffff)
#define HPTE_V_AVPN ASM_CONST(0x3fffffffffffff80) #define HPTE_V_AVPN ASM_CONST(0x3fffffffffffff80)
#define HPTE_V_AVPN_3_0 ASM_CONST(0x000fffffffffff80)
#define HPTE_V_AVPN_VAL(x) (((x) & HPTE_V_AVPN) >> HPTE_V_AVPN_SHIFT) #define HPTE_V_AVPN_VAL(x) (((x) & HPTE_V_AVPN) >> HPTE_V_AVPN_SHIFT)
#define HPTE_V_COMPARE(x,y) (!(((x) ^ (y)) & 0xffffffffffffff80UL)) #define HPTE_V_COMPARE(x,y) (!(((x) ^ (y)) & 0xffffffffffffff80UL))
#define HPTE_V_BOLTED ASM_CONST(0x0000000000000010) #define HPTE_V_BOLTED ASM_CONST(0x0000000000000010)
...@@ -80,14 +82,16 @@ ...@@ -80,14 +82,16 @@
#define HPTE_V_VALID ASM_CONST(0x0000000000000001) #define HPTE_V_VALID ASM_CONST(0x0000000000000001)
/* /*
* ISA 3.0 have a different HPTE format. * ISA 3.0 has a different HPTE format.
*/ */
#define HPTE_R_3_0_SSIZE_SHIFT 58 #define HPTE_R_3_0_SSIZE_SHIFT 58
#define HPTE_R_3_0_SSIZE_MASK (3ull << HPTE_R_3_0_SSIZE_SHIFT)
#define HPTE_R_PP0 ASM_CONST(0x8000000000000000) #define HPTE_R_PP0 ASM_CONST(0x8000000000000000)
#define HPTE_R_TS ASM_CONST(0x4000000000000000) #define HPTE_R_TS ASM_CONST(0x4000000000000000)
#define HPTE_R_KEY_HI ASM_CONST(0x3000000000000000) #define HPTE_R_KEY_HI ASM_CONST(0x3000000000000000)
#define HPTE_R_RPN_SHIFT 12 #define HPTE_R_RPN_SHIFT 12
#define HPTE_R_RPN ASM_CONST(0x0ffffffffffff000) #define HPTE_R_RPN ASM_CONST(0x0ffffffffffff000)
#define HPTE_R_RPN_3_0 ASM_CONST(0x01fffffffffff000)
#define HPTE_R_PP ASM_CONST(0x0000000000000003) #define HPTE_R_PP ASM_CONST(0x0000000000000003)
#define HPTE_R_PPP ASM_CONST(0x8000000000000003) #define HPTE_R_PPP ASM_CONST(0x8000000000000003)
#define HPTE_R_N ASM_CONST(0x0000000000000004) #define HPTE_R_N ASM_CONST(0x0000000000000004)
...@@ -316,11 +320,42 @@ static inline unsigned long hpte_encode_avpn(unsigned long vpn, int psize, ...@@ -316,11 +320,42 @@ static inline unsigned long hpte_encode_avpn(unsigned long vpn, int psize,
*/ */
v = (vpn >> (23 - VPN_SHIFT)) & ~(mmu_psize_defs[psize].avpnm); v = (vpn >> (23 - VPN_SHIFT)) & ~(mmu_psize_defs[psize].avpnm);
v <<= HPTE_V_AVPN_SHIFT; v <<= HPTE_V_AVPN_SHIFT;
if (!cpu_has_feature(CPU_FTR_ARCH_300)) v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT;
v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT;
return v; return v;
} }
/*
* ISA v3.0 defines a new HPTE format, which differs from the old
* format in having smaller AVPN and ARPN fields, and the B field
* in the second dword instead of the first.
*/
static inline unsigned long hpte_old_to_new_v(unsigned long v)
{
/* trim AVPN, drop B */
return v & HPTE_V_COMMON_BITS;
}
static inline unsigned long hpte_old_to_new_r(unsigned long v, unsigned long r)
{
/* move B field from 1st to 2nd dword, trim ARPN */
return (r & ~HPTE_R_3_0_SSIZE_MASK) |
(((v) >> HPTE_V_SSIZE_SHIFT) << HPTE_R_3_0_SSIZE_SHIFT);
}
static inline unsigned long hpte_new_to_old_v(unsigned long v, unsigned long r)
{
/* insert B field */
return (v & HPTE_V_COMMON_BITS) |
((r & HPTE_R_3_0_SSIZE_MASK) <<
(HPTE_V_SSIZE_SHIFT - HPTE_R_3_0_SSIZE_SHIFT));
}
static inline unsigned long hpte_new_to_old_r(unsigned long r)
{
/* clear out B field */
return r & ~HPTE_R_3_0_SSIZE_MASK;
}
/* /*
* This function sets the AVPN and L fields of the HPTE appropriately * This function sets the AVPN and L fields of the HPTE appropriately
* using the base page size and actual page size. * using the base page size and actual page size.
...@@ -341,12 +376,8 @@ static inline unsigned long hpte_encode_v(unsigned long vpn, int base_psize, ...@@ -341,12 +376,8 @@ static inline unsigned long hpte_encode_v(unsigned long vpn, int base_psize,
* aligned for the requested page size * aligned for the requested page size
*/ */
static inline unsigned long hpte_encode_r(unsigned long pa, int base_psize, static inline unsigned long hpte_encode_r(unsigned long pa, int base_psize,
int actual_psize, int ssize) int actual_psize)
{ {
if (cpu_has_feature(CPU_FTR_ARCH_300))
pa |= ((unsigned long) ssize) << HPTE_R_3_0_SSIZE_SHIFT;
/* A 4K page needs no special encoding */ /* A 4K page needs no special encoding */
if (actual_psize == MMU_PAGE_4K) if (actual_psize == MMU_PAGE_4K)
return pa & HPTE_R_RPN; return pa & HPTE_R_RPN;
......
...@@ -221,13 +221,18 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn, ...@@ -221,13 +221,18 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
return -1; return -1;
hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID; hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
hpte_r = hpte_encode_r(pa, psize, apsize, ssize) | rflags; hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
if (!(vflags & HPTE_V_BOLTED)) { if (!(vflags & HPTE_V_BOLTED)) {
DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n", DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n",
i, hpte_v, hpte_r); i, hpte_v, hpte_r);
} }
if (cpu_has_feature(CPU_FTR_ARCH_300)) {
hpte_r = hpte_old_to_new_r(hpte_v, hpte_r);
hpte_v = hpte_old_to_new_v(hpte_v);
}
hptep->r = cpu_to_be64(hpte_r); hptep->r = cpu_to_be64(hpte_r);
/* Guarantee the second dword is visible before the valid bit */ /* Guarantee the second dword is visible before the valid bit */
eieio(); eieio();
...@@ -295,6 +300,8 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, ...@@ -295,6 +300,8 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
vpn, want_v & HPTE_V_AVPN, slot, newpp); vpn, want_v & HPTE_V_AVPN, slot, newpp);
hpte_v = be64_to_cpu(hptep->v); hpte_v = be64_to_cpu(hptep->v);
if (cpu_has_feature(CPU_FTR_ARCH_300))
hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r));
/* /*
* We need to invalidate the TLB always because hpte_remove doesn't do * We need to invalidate the TLB always because hpte_remove doesn't do
* a tlb invalidate. If a hash bucket gets full, we "evict" a more/less * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
...@@ -309,6 +316,8 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, ...@@ -309,6 +316,8 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
native_lock_hpte(hptep); native_lock_hpte(hptep);
/* recheck with locks held */ /* recheck with locks held */
hpte_v = be64_to_cpu(hptep->v); hpte_v = be64_to_cpu(hptep->v);
if (cpu_has_feature(CPU_FTR_ARCH_300))
hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r));
if (unlikely(!HPTE_V_COMPARE(hpte_v, want_v) || if (unlikely(!HPTE_V_COMPARE(hpte_v, want_v) ||
!(hpte_v & HPTE_V_VALID))) { !(hpte_v & HPTE_V_VALID))) {
ret = -1; ret = -1;
...@@ -350,6 +359,8 @@ static long native_hpte_find(unsigned long vpn, int psize, int ssize) ...@@ -350,6 +359,8 @@ static long native_hpte_find(unsigned long vpn, int psize, int ssize)
for (i = 0; i < HPTES_PER_GROUP; i++) { for (i = 0; i < HPTES_PER_GROUP; i++) {
hptep = htab_address + slot; hptep = htab_address + slot;
hpte_v = be64_to_cpu(hptep->v); hpte_v = be64_to_cpu(hptep->v);
if (cpu_has_feature(CPU_FTR_ARCH_300))
hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r));
if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID))
/* HPTE matches */ /* HPTE matches */
...@@ -409,6 +420,8 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long vpn, ...@@ -409,6 +420,8 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
want_v = hpte_encode_avpn(vpn, bpsize, ssize); want_v = hpte_encode_avpn(vpn, bpsize, ssize);
native_lock_hpte(hptep); native_lock_hpte(hptep);
hpte_v = be64_to_cpu(hptep->v); hpte_v = be64_to_cpu(hptep->v);
if (cpu_has_feature(CPU_FTR_ARCH_300))
hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r));
/* /*
* We need to invalidate the TLB always because hpte_remove doesn't do * We need to invalidate the TLB always because hpte_remove doesn't do
...@@ -467,6 +480,8 @@ static void native_hugepage_invalidate(unsigned long vsid, ...@@ -467,6 +480,8 @@ static void native_hugepage_invalidate(unsigned long vsid,
want_v = hpte_encode_avpn(vpn, psize, ssize); want_v = hpte_encode_avpn(vpn, psize, ssize);
native_lock_hpte(hptep); native_lock_hpte(hptep);
hpte_v = be64_to_cpu(hptep->v); hpte_v = be64_to_cpu(hptep->v);
if (cpu_has_feature(CPU_FTR_ARCH_300))
hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r));
/* Even if we miss, we need to invalidate the TLB */ /* Even if we miss, we need to invalidate the TLB */
if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
...@@ -504,6 +519,10 @@ static void hpte_decode(struct hash_pte *hpte, unsigned long slot, ...@@ -504,6 +519,10 @@ static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
/* Look at the 8 bit LP value */ /* Look at the 8 bit LP value */
unsigned int lp = (hpte_r >> LP_SHIFT) & ((1 << LP_BITS) - 1); unsigned int lp = (hpte_r >> LP_SHIFT) & ((1 << LP_BITS) - 1);
if (cpu_has_feature(CPU_FTR_ARCH_300)) {
hpte_v = hpte_new_to_old_v(hpte_v, hpte_r);
hpte_r = hpte_new_to_old_r(hpte_r);
}
if (!(hpte_v & HPTE_V_LARGE)) { if (!(hpte_v & HPTE_V_LARGE)) {
size = MMU_PAGE_4K; size = MMU_PAGE_4K;
a_size = MMU_PAGE_4K; a_size = MMU_PAGE_4K;
...@@ -512,11 +531,7 @@ static void hpte_decode(struct hash_pte *hpte, unsigned long slot, ...@@ -512,11 +531,7 @@ static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
a_size = hpte_page_sizes[lp] >> 4; a_size = hpte_page_sizes[lp] >> 4;
} }
/* This works for all page sizes, and for 256M and 1T segments */ /* This works for all page sizes, and for 256M and 1T segments */
if (cpu_has_feature(CPU_FTR_ARCH_300)) *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT;
*ssize = hpte_r >> HPTE_R_3_0_SSIZE_SHIFT;
else
*ssize = hpte_v >> HPTE_V_SSIZE_SHIFT;
shift = mmu_psize_defs[size].shift; shift = mmu_psize_defs[size].shift;
avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm); avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm);
...@@ -639,6 +654,9 @@ static void native_flush_hash_range(unsigned long number, int local) ...@@ -639,6 +654,9 @@ static void native_flush_hash_range(unsigned long number, int local)
want_v = hpte_encode_avpn(vpn, psize, ssize); want_v = hpte_encode_avpn(vpn, psize, ssize);
native_lock_hpte(hptep); native_lock_hpte(hptep);
hpte_v = be64_to_cpu(hptep->v); hpte_v = be64_to_cpu(hptep->v);
if (cpu_has_feature(CPU_FTR_ARCH_300))
hpte_v = hpte_new_to_old_v(hpte_v,
be64_to_cpu(hptep->r));
if (!HPTE_V_COMPARE(hpte_v, want_v) || if (!HPTE_V_COMPARE(hpte_v, want_v) ||
!(hpte_v & HPTE_V_VALID)) !(hpte_v & HPTE_V_VALID))
native_unlock_hpte(hptep); native_unlock_hpte(hptep);
......
...@@ -63,7 +63,7 @@ static long ps3_hpte_insert(unsigned long hpte_group, unsigned long vpn, ...@@ -63,7 +63,7 @@ static long ps3_hpte_insert(unsigned long hpte_group, unsigned long vpn,
vflags &= ~HPTE_V_SECONDARY; vflags &= ~HPTE_V_SECONDARY;
hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID; hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
hpte_r = hpte_encode_r(ps3_mm_phys_to_lpar(pa), psize, apsize, ssize) | rflags; hpte_r = hpte_encode_r(ps3_mm_phys_to_lpar(pa), psize, apsize) | rflags;
spin_lock_irqsave(&ps3_htab_lock, flags); spin_lock_irqsave(&ps3_htab_lock, flags);
......
...@@ -145,7 +145,7 @@ static long pSeries_lpar_hpte_insert(unsigned long hpte_group, ...@@ -145,7 +145,7 @@ static long pSeries_lpar_hpte_insert(unsigned long hpte_group,
hpte_group, vpn, pa, rflags, vflags, psize); hpte_group, vpn, pa, rflags, vflags, psize);
hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID; hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
hpte_r = hpte_encode_r(pa, psize, apsize, ssize) | rflags; hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
if (!(vflags & HPTE_V_BOLTED)) if (!(vflags & HPTE_V_BOLTED))
pr_devel(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r); pr_devel(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment