Commit 83aae4a8 authored by Hollis Blanchard's avatar Hollis Blanchard Committed by Avi Kivity

KVM: ppc: Write only modified shadow entries into the TLB on exit

Track which TLB entries need to be written, instead of overwriting everything
below the high water mark. Typically only a single guest TLB entry will be
modified in a single exit.

Guest boot time performance improvement: about 15%.
Signed-off-by: default avatarHollis Blanchard <hollisb@us.ibm.com>
Signed-off-by: default avatarAvi Kivity <avi@qumranet.com>
parent 20754c24
...@@ -82,6 +82,9 @@ struct kvm_vcpu_arch { ...@@ -82,6 +82,9 @@ struct kvm_vcpu_arch {
/* Pages which are referenced in the shadow TLB. */ /* Pages which are referenced in the shadow TLB. */
struct page *shadow_pages[PPC44x_TLB_SIZE]; struct page *shadow_pages[PPC44x_TLB_SIZE];
/* Track which TLB entries we've modified in the current exit. */
u8 shadow_tlb_mod[PPC44x_TLB_SIZE];
u32 host_stack; u32 host_stack;
u32 host_pid; u32 host_pid;
u32 host_dbcr0; u32 host_dbcr0;
......
...@@ -65,6 +65,9 @@ extern void kvmppc_mmu_invalidate(struct kvm_vcpu *vcpu, gva_t eaddr, ...@@ -65,6 +65,9 @@ extern void kvmppc_mmu_invalidate(struct kvm_vcpu *vcpu, gva_t eaddr,
gva_t eend, u32 asid); gva_t eend, u32 asid);
extern void kvmppc_mmu_priv_switch(struct kvm_vcpu *vcpu, int usermode); extern void kvmppc_mmu_priv_switch(struct kvm_vcpu *vcpu, int usermode);
/* XXX Book E specific */
extern void kvmppc_tlbe_set_modified(struct kvm_vcpu *vcpu, unsigned int i);
extern void kvmppc_check_and_deliver_interrupts(struct kvm_vcpu *vcpu); extern void kvmppc_check_and_deliver_interrupts(struct kvm_vcpu *vcpu);
static inline void kvmppc_queue_exception(struct kvm_vcpu *vcpu, int exception) static inline void kvmppc_queue_exception(struct kvm_vcpu *vcpu, int exception)
......
...@@ -357,6 +357,7 @@ int main(void) ...@@ -357,6 +357,7 @@ int main(void)
DEFINE(VCPU_HOST_STACK, offsetof(struct kvm_vcpu, arch.host_stack)); DEFINE(VCPU_HOST_STACK, offsetof(struct kvm_vcpu, arch.host_stack));
DEFINE(VCPU_HOST_PID, offsetof(struct kvm_vcpu, arch.host_pid)); DEFINE(VCPU_HOST_PID, offsetof(struct kvm_vcpu, arch.host_pid));
DEFINE(VCPU_SHADOW_TLB, offsetof(struct kvm_vcpu, arch.shadow_tlb)); DEFINE(VCPU_SHADOW_TLB, offsetof(struct kvm_vcpu, arch.shadow_tlb));
DEFINE(VCPU_SHADOW_MOD, offsetof(struct kvm_vcpu, arch.shadow_tlb_mod));
DEFINE(VCPU_GPRS, offsetof(struct kvm_vcpu, arch.gpr)); DEFINE(VCPU_GPRS, offsetof(struct kvm_vcpu, arch.gpr));
DEFINE(VCPU_LR, offsetof(struct kvm_vcpu, arch.lr)); DEFINE(VCPU_LR, offsetof(struct kvm_vcpu, arch.lr));
DEFINE(VCPU_CR, offsetof(struct kvm_vcpu, arch.cr)); DEFINE(VCPU_CR, offsetof(struct kvm_vcpu, arch.cr));
......
...@@ -125,6 +125,11 @@ static void kvmppc_44x_shadow_release(struct kvm_vcpu *vcpu, ...@@ -125,6 +125,11 @@ static void kvmppc_44x_shadow_release(struct kvm_vcpu *vcpu,
} }
} }
void kvmppc_tlbe_set_modified(struct kvm_vcpu *vcpu, unsigned int i)
{
vcpu->arch.shadow_tlb_mod[i] = 1;
}
/* Caller must ensure that the specified guest TLB entry is safe to insert into /* Caller must ensure that the specified guest TLB entry is safe to insert into
* the shadow TLB. */ * the shadow TLB. */
void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 gvaddr, gfn_t gfn, u64 asid, void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 gvaddr, gfn_t gfn, u64 asid,
...@@ -172,10 +177,10 @@ void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 gvaddr, gfn_t gfn, u64 asid, ...@@ -172,10 +177,10 @@ void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 gvaddr, gfn_t gfn, u64 asid,
* use host large pages in the future. */ * use host large pages in the future. */
stlbe->word0 = (gvaddr & PAGE_MASK) | PPC44x_TLB_VALID | PPC44x_TLB_TS stlbe->word0 = (gvaddr & PAGE_MASK) | PPC44x_TLB_VALID | PPC44x_TLB_TS
| PPC44x_TLB_4K; | PPC44x_TLB_4K;
stlbe->word1 = (hpaddr & 0xfffffc00) | ((hpaddr >> 32) & 0xf); stlbe->word1 = (hpaddr & 0xfffffc00) | ((hpaddr >> 32) & 0xf);
stlbe->word2 = kvmppc_44x_tlb_shadow_attrib(flags, stlbe->word2 = kvmppc_44x_tlb_shadow_attrib(flags,
vcpu->arch.msr & MSR_PR); vcpu->arch.msr & MSR_PR);
kvmppc_tlbe_set_modified(vcpu, victim);
KVMTRACE_5D(STLB_WRITE, vcpu, victim, KVMTRACE_5D(STLB_WRITE, vcpu, victim,
stlbe->tid, stlbe->word0, stlbe->word1, stlbe->word2, stlbe->tid, stlbe->word0, stlbe->word1, stlbe->word2,
...@@ -209,6 +214,7 @@ void kvmppc_mmu_invalidate(struct kvm_vcpu *vcpu, gva_t eaddr, ...@@ -209,6 +214,7 @@ void kvmppc_mmu_invalidate(struct kvm_vcpu *vcpu, gva_t eaddr,
kvmppc_44x_shadow_release(vcpu, i); kvmppc_44x_shadow_release(vcpu, i);
stlbe->word0 = 0; stlbe->word0 = 0;
kvmppc_tlbe_set_modified(vcpu, i);
KVMTRACE_5D(STLB_INVAL, vcpu, i, KVMTRACE_5D(STLB_INVAL, vcpu, i,
stlbe->tid, stlbe->word0, stlbe->word1, stlbe->tid, stlbe->word0, stlbe->word1,
stlbe->word2, handler); stlbe->word2, handler);
...@@ -229,6 +235,7 @@ void kvmppc_mmu_priv_switch(struct kvm_vcpu *vcpu, int usermode) ...@@ -229,6 +235,7 @@ void kvmppc_mmu_priv_switch(struct kvm_vcpu *vcpu, int usermode)
kvmppc_44x_shadow_release(vcpu, i); kvmppc_44x_shadow_release(vcpu, i);
stlbe->word0 = 0; stlbe->word0 = 0;
kvmppc_tlbe_set_modified(vcpu, i);
KVMTRACE_5D(STLB_INVAL, vcpu, i, KVMTRACE_5D(STLB_INVAL, vcpu, i,
stlbe->tid, stlbe->word0, stlbe->word1, stlbe->tid, stlbe->word0, stlbe->word1,
stlbe->word2, handler); stlbe->word2, handler);
......
...@@ -335,7 +335,7 @@ lightweight_exit: ...@@ -335,7 +335,7 @@ lightweight_exit:
lwz r3, VCPU_PID(r4) lwz r3, VCPU_PID(r4)
mtspr SPRN_PID, r3 mtspr SPRN_PID, r3
/* Prevent all TLB updates. */ /* Prevent all asynchronous TLB updates. */
mfmsr r5 mfmsr r5
lis r6, (MSR_EE|MSR_CE|MSR_ME|MSR_DE)@h lis r6, (MSR_EE|MSR_CE|MSR_ME|MSR_DE)@h
ori r6, r6, (MSR_EE|MSR_CE|MSR_ME|MSR_DE)@l ori r6, r6, (MSR_EE|MSR_CE|MSR_ME|MSR_DE)@l
...@@ -344,28 +344,45 @@ lightweight_exit: ...@@ -344,28 +344,45 @@ lightweight_exit:
/* Load the guest mappings, leaving the host's "pinned" kernel mappings /* Load the guest mappings, leaving the host's "pinned" kernel mappings
* in place. */ * in place. */
/* XXX optimization: load only modified guest entries. */
mfspr r10, SPRN_MMUCR /* Save host MMUCR. */ mfspr r10, SPRN_MMUCR /* Save host MMUCR. */
lis r8, tlb_44x_hwater@ha li r5, PPC44x_TLB_SIZE
lwz r8, tlb_44x_hwater@l(r8) lis r5, tlb_44x_hwater@ha
addi r9, r4, VCPU_SHADOW_TLB - 4 lwz r5, tlb_44x_hwater@l(r5)
li r6, 0 mtctr r5
addi r9, r4, VCPU_SHADOW_TLB
addi r5, r4, VCPU_SHADOW_MOD
li r3, 0
1: 1:
lbzx r7, r3, r5
cmpwi r7, 0
beq 3f
/* Load guest entry. */ /* Load guest entry. */
lwzu r7, 4(r9) mulli r11, r3, TLBE_BYTES
add r11, r11, r9
lwz r7, 0(r11)
mtspr SPRN_MMUCR, r7 mtspr SPRN_MMUCR, r7
lwzu r7, 4(r9) lwz r7, 4(r11)
tlbwe r7, r6, PPC44x_TLB_PAGEID tlbwe r7, r3, PPC44x_TLB_PAGEID
lwzu r7, 4(r9) lwz r7, 8(r11)
tlbwe r7, r6, PPC44x_TLB_XLAT tlbwe r7, r3, PPC44x_TLB_XLAT
lwzu r7, 4(r9) lwz r7, 12(r11)
tlbwe r7, r6, PPC44x_TLB_ATTRIB tlbwe r7, r3, PPC44x_TLB_ATTRIB
/* Increment index. */ 3:
addi r6, r6, 1 addi r3, r3, 1 /* Increment index. */
cmpw r6, r8 bdnz 1b
blt 1b
mtspr SPRN_MMUCR, r10 /* Restore host MMUCR. */ mtspr SPRN_MMUCR, r10 /* Restore host MMUCR. */
/* Clear bitmap of modified TLB entries */
li r5, PPC44x_TLB_SIZE>>2
mtctr r5
addi r5, r4, VCPU_SHADOW_MOD - 4
li r6, 0
1:
stwu r6, 4(r5)
bdnz 1b
iccci 0, 0 /* XXX hack */ iccci 0, 0 /* XXX hack */
/* Load some guest volatiles. */ /* Load some guest volatiles. */
......
...@@ -27,6 +27,7 @@ ...@@ -27,6 +27,7 @@
#include <asm/cputable.h> #include <asm/cputable.h>
#include <asm/uaccess.h> #include <asm/uaccess.h>
#include <asm/kvm_ppc.h> #include <asm/kvm_ppc.h>
#include <asm/tlbflush.h>
gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
...@@ -307,14 +308,28 @@ static void kvmppc_load_guest_debug_registers(struct kvm_vcpu *vcpu) ...@@ -307,14 +308,28 @@ static void kvmppc_load_guest_debug_registers(struct kvm_vcpu *vcpu)
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{ {
int i;
if (vcpu->guest_debug.enabled) if (vcpu->guest_debug.enabled)
kvmppc_load_guest_debug_registers(vcpu); kvmppc_load_guest_debug_registers(vcpu);
/* Mark every guest entry in the shadow TLB entry modified, so that they
* will all be reloaded on the next vcpu run (instead of being
* demand-faulted). */
for (i = 0; i <= tlb_44x_hwater; i++)
kvmppc_tlbe_set_modified(vcpu, i);
} }
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
{ {
if (vcpu->guest_debug.enabled) if (vcpu->guest_debug.enabled)
kvmppc_restore_host_debug_state(vcpu); kvmppc_restore_host_debug_state(vcpu);
/* Don't leave guest TLB entries resident when being de-scheduled. */
/* XXX It would be nice to differentiate between heavyweight exit and
* sched_out here, since we could avoid the TLB flush for heavyweight
* exits. */
_tlbia();
} }
int kvm_arch_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu, int kvm_arch_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment