Skip to content

L
linux
Commit 2d27fc5e authored by Alexander Graf's avatar Alexander Graf Committed by Avi Kivity
Browse files
Options

KVM: PPC: Add book3s_32 tlbie flush acceleration 

On Book3s_32 the tlbie instruction flushed effective addresses by the mask
0x0ffff000. This is pretty hard to reflect with a hash that hashes ~0xfff, so
to speed up that target we should also keep a special hash around for it.
Signed-off-by: default avatarAlexander Graf <agraf@suse.de>
Signed-off-by: default avatarAvi Kivity <avi@redhat.com>
parent 49451389
Hide whitespace changes
Inline Side-by-side
Showing with 39 additions and 5 deletions
arch/powerpc/include/asm/kvm_host.h
View file @ 2d27fc5e
...@@ -42,9 +42,11 @@ ...@@ -42,9 +42,11 @@
#define HPTEG_CACHE_NUM (1 << 15) #define HPTEG_CACHE_NUM (1 << 15)
#define HPTEG_HASH_BITS_PTE 13 #define HPTEG_HASH_BITS_PTE 13
#define HPTEG_HASH_BITS_PTE_LONG 12
#define HPTEG_HASH_BITS_VPTE 13 #define HPTEG_HASH_BITS_VPTE 13
#define HPTEG_HASH_BITS_VPTE_LONG 5 #define HPTEG_HASH_BITS_VPTE_LONG 5
#define HPTEG_HASH_NUM_PTE (1 << HPTEG_HASH_BITS_PTE) #define HPTEG_HASH_NUM_PTE (1 << HPTEG_HASH_BITS_PTE)
#define HPTEG_HASH_NUM_PTE_LONG (1 << HPTEG_HASH_BITS_PTE_LONG)
#define HPTEG_HASH_NUM_VPTE (1 << HPTEG_HASH_BITS_VPTE) #define HPTEG_HASH_NUM_VPTE (1 << HPTEG_HASH_BITS_VPTE)
#define HPTEG_HASH_NUM_VPTE_LONG (1 << HPTEG_HASH_BITS_VPTE_LONG) #define HPTEG_HASH_NUM_VPTE_LONG (1 << HPTEG_HASH_BITS_VPTE_LONG)
...@@ -163,6 +165,7 @@ struct kvmppc_mmu { ...@@ -163,6 +165,7 @@ struct kvmppc_mmu {
struct hpte_cache { struct hpte_cache {
struct hlist_node list_pte; struct hlist_node list_pte;
struct hlist_node list_pte_long;
struct hlist_node list_vpte; struct hlist_node list_vpte;
struct hlist_node list_vpte_long; struct hlist_node list_vpte_long;
struct rcu_head rcu_head; struct rcu_head rcu_head;
...@@ -293,6 +296,7 @@ struct kvm_vcpu_arch { ...@@ -293,6 +296,7 @@ struct kvm_vcpu_arch {
#ifdef CONFIG_PPC_BOOK3S #ifdef CONFIG_PPC_BOOK3S
struct hlist_head hpte_hash_pte[HPTEG_HASH_NUM_PTE]; struct hlist_head hpte_hash_pte[HPTEG_HASH_NUM_PTE];
struct hlist_head hpte_hash_pte_long[HPTEG_HASH_NUM_PTE_LONG];
struct hlist_head hpte_hash_vpte[HPTEG_HASH_NUM_VPTE]; struct hlist_head hpte_hash_vpte[HPTEG_HASH_NUM_VPTE];
struct hlist_head hpte_hash_vpte_long[HPTEG_HASH_NUM_VPTE_LONG]; struct hlist_head hpte_hash_vpte_long[HPTEG_HASH_NUM_VPTE_LONG];
int hpte_cache_count; int hpte_cache_count;
......
arch/powerpc/kvm/book3s_mmu_hpte.c
View file @ 2d27fc5e
...@@ -45,6 +45,12 @@ static inline u64 kvmppc_mmu_hash_pte(u64 eaddr) ...@@ -45,6 +45,12 @@ static inline u64 kvmppc_mmu_hash_pte(u64 eaddr)
return hash_64(eaddr >> PTE_SIZE, HPTEG_HASH_BITS_PTE); return hash_64(eaddr >> PTE_SIZE, HPTEG_HASH_BITS_PTE);
} }
static inline u64 kvmppc_mmu_hash_pte_long(u64 eaddr)
{
return hash_64((eaddr & 0x0ffff000) >> PTE_SIZE,
HPTEG_HASH_BITS_PTE_LONG);
}
static inline u64 kvmppc_mmu_hash_vpte(u64 vpage) static inline u64 kvmppc_mmu_hash_vpte(u64 vpage)
{ {
return hash_64(vpage & 0xfffffffffULL, HPTEG_HASH_BITS_VPTE); return hash_64(vpage & 0xfffffffffULL, HPTEG_HASH_BITS_VPTE);
...@@ -66,6 +72,11 @@ void kvmppc_mmu_hpte_cache_map(struct kvm_vcpu *vcpu, struct hpte_cache *pte) ...@@ -66,6 +72,11 @@ void kvmppc_mmu_hpte_cache_map(struct kvm_vcpu *vcpu, struct hpte_cache *pte)
index = kvmppc_mmu_hash_pte(pte->pte.eaddr); index = kvmppc_mmu_hash_pte(pte->pte.eaddr);
hlist_add_head_rcu(&pte->list_pte, &vcpu->arch.hpte_hash_pte[index]); hlist_add_head_rcu(&pte->list_pte, &vcpu->arch.hpte_hash_pte[index]);
/* Add to ePTE_long list */
index = kvmppc_mmu_hash_pte_long(pte->pte.eaddr);
hlist_add_head_rcu(&pte->list_pte_long,
&vcpu->arch.hpte_hash_pte_long[index]);
/* Add to vPTE list */ /* Add to vPTE list */
index = kvmppc_mmu_hash_vpte(pte->pte.vpage); index = kvmppc_mmu_hash_vpte(pte->pte.vpage);
hlist_add_head_rcu(&pte->list_vpte, &vcpu->arch.hpte_hash_vpte[index]); hlist_add_head_rcu(&pte->list_vpte, &vcpu->arch.hpte_hash_vpte[index]);
...@@ -99,6 +110,7 @@ static void invalidate_pte(struct kvm_vcpu *vcpu, struct hpte_cache *pte) ...@@ -99,6 +110,7 @@ static void invalidate_pte(struct kvm_vcpu *vcpu, struct hpte_cache *pte)
spin_lock(&vcpu->arch.mmu_lock); spin_lock(&vcpu->arch.mmu_lock);
hlist_del_init_rcu(&pte->list_pte); hlist_del_init_rcu(&pte->list_pte);
hlist_del_init_rcu(&pte->list_pte_long);
hlist_del_init_rcu(&pte->list_vpte); hlist_del_init_rcu(&pte->list_vpte);
hlist_del_init_rcu(&pte->list_vpte_long); hlist_del_init_rcu(&pte->list_vpte_long);
...@@ -150,10 +162,28 @@ static void kvmppc_mmu_pte_flush_page(struct kvm_vcpu *vcpu, ulong guest_ea) ...@@ -150,10 +162,28 @@ static void kvmppc_mmu_pte_flush_page(struct kvm_vcpu *vcpu, ulong guest_ea)
rcu_read_unlock(); rcu_read_unlock();
} }
void kvmppc_mmu_pte_flush(struct kvm_vcpu *vcpu, ulong guest_ea, ulong ea_mask) static void kvmppc_mmu_pte_flush_long(struct kvm_vcpu *vcpu, ulong guest_ea)
{ {
u64 i; struct hlist_head *list;
struct hlist_node *node;
struct hpte_cache *pte;
/* Find the list of entries in the map */
list = &vcpu->arch.hpte_hash_pte_long[
kvmppc_mmu_hash_pte_long(guest_ea)];
rcu_read_lock();
/* Check the list for matching entries and invalidate */
hlist_for_each_entry_rcu(pte, node, list, list_pte_long)
if ((pte->pte.eaddr & 0x0ffff000UL) == guest_ea)
invalidate_pte(vcpu, pte);
rcu_read_unlock();
}
void kvmppc_mmu_pte_flush(struct kvm_vcpu *vcpu, ulong guest_ea, ulong ea_mask)
{
dprintk_mmu("KVM: Flushing %d Shadow PTEs: 0x%lx & 0x%lx\n", dprintk_mmu("KVM: Flushing %d Shadow PTEs: 0x%lx & 0x%lx\n",
vcpu->arch.hpte_cache_count, guest_ea, ea_mask); vcpu->arch.hpte_cache_count, guest_ea, ea_mask);
...@@ -164,9 +194,7 @@ void kvmppc_mmu_pte_flush(struct kvm_vcpu *vcpu, ulong guest_ea, ulong ea_mask) ...@@ -164,9 +194,7 @@ void kvmppc_mmu_pte_flush(struct kvm_vcpu *vcpu, ulong guest_ea, ulong ea_mask)
kvmppc_mmu_pte_flush_page(vcpu, guest_ea); kvmppc_mmu_pte_flush_page(vcpu, guest_ea);
break; break;
case 0x0ffff000: case 0x0ffff000:
/* 32-bit flush w/o segment, go through all possible segments */ kvmppc_mmu_pte_flush_long(vcpu, guest_ea);
for (i = 0; i < 0x100000000ULL; i += 0x10000000ULL)
kvmppc_mmu_pte_flush(vcpu, guest_ea | i, ~0xfffUL);
break; break;
case 0: case 0:
/* Doing a complete flush -> start from scratch */ /* Doing a complete flush -> start from scratch */
...@@ -292,6 +320,8 @@ int kvmppc_mmu_hpte_init(struct kvm_vcpu *vcpu) ...@@ -292,6 +320,8 @@ int kvmppc_mmu_hpte_init(struct kvm_vcpu *vcpu)
/* init hpte lookup hashes */ /* init hpte lookup hashes */
kvmppc_mmu_hpte_init_hash(vcpu->arch.hpte_hash_pte, kvmppc_mmu_hpte_init_hash(vcpu->arch.hpte_hash_pte,
ARRAY_SIZE(vcpu->arch.hpte_hash_pte)); ARRAY_SIZE(vcpu->arch.hpte_hash_pte));
kvmppc_mmu_hpte_init_hash(vcpu->arch.hpte_hash_pte_long,
ARRAY_SIZE(vcpu->arch.hpte_hash_pte_long));
kvmppc_mmu_hpte_init_hash(vcpu->arch.hpte_hash_vpte, kvmppc_mmu_hpte_init_hash(vcpu->arch.hpte_hash_vpte,
ARRAY_SIZE(vcpu->arch.hpte_hash_vpte)); ARRAY_SIZE(vcpu->arch.hpte_hash_vpte));
kvmppc_mmu_hpte_init_hash(vcpu->arch.hpte_hash_vpte_long, kvmppc_mmu_hpte_init_hash(vcpu->arch.hpte_hash_vpte_long,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7