Commit 5efac074 authored by Paolo Bonzini's avatar Paolo Bonzini

KVM: x86: introduce kvm_mmu_invalidate_gva

Wrap the combination of mmu->invlpg and kvm_x86_ops->tlb_flush_gva
into a new function.  This function also lets us specify the host PGD to
invalidate and also the MMU, both of which will be useful in fixing and
simplifying kvm_inject_emulated_page_fault.

A nested guest's MMU however has g_context->invlpg == NULL.  Instead of
setting it to nonpaging_invlpg, make kvm_mmu_invalidate_gva the only
entry point to mmu->invlpg and make a NULL invlpg pointer equivalent
to nonpaging_invlpg, saving a retpoline.
Signed-off-by: default avatarPaolo Bonzini <pbonzini@redhat.com>
parent 53b3d8e9
...@@ -1511,6 +1511,8 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); ...@@ -1511,6 +1511,8 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu);
int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code, int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
void *insn, int insn_len); void *insn, int insn_len);
void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva); void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
gva_t gva, hpa_t root_hpa);
void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid); void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid);
void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, bool skip_tlb_flush); void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, bool skip_tlb_flush);
......
...@@ -2153,10 +2153,6 @@ static int nonpaging_sync_page(struct kvm_vcpu *vcpu, ...@@ -2153,10 +2153,6 @@ static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
return 0; return 0;
} }
static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root)
{
}
static void nonpaging_update_pte(struct kvm_vcpu *vcpu, static void nonpaging_update_pte(struct kvm_vcpu *vcpu,
struct kvm_mmu_page *sp, u64 *spte, struct kvm_mmu_page *sp, u64 *spte,
const void *pte) const void *pte)
...@@ -4237,7 +4233,7 @@ static void nonpaging_init_context(struct kvm_vcpu *vcpu, ...@@ -4237,7 +4233,7 @@ static void nonpaging_init_context(struct kvm_vcpu *vcpu,
context->page_fault = nonpaging_page_fault; context->page_fault = nonpaging_page_fault;
context->gva_to_gpa = nonpaging_gva_to_gpa; context->gva_to_gpa = nonpaging_gva_to_gpa;
context->sync_page = nonpaging_sync_page; context->sync_page = nonpaging_sync_page;
context->invlpg = nonpaging_invlpg; context->invlpg = NULL;
context->update_pte = nonpaging_update_pte; context->update_pte = nonpaging_update_pte;
context->root_level = 0; context->root_level = 0;
context->shadow_root_level = PT32E_ROOT_LEVEL; context->shadow_root_level = PT32E_ROOT_LEVEL;
...@@ -4928,7 +4924,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) ...@@ -4928,7 +4924,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
context->mmu_role.as_u64 = new_role.as_u64; context->mmu_role.as_u64 = new_role.as_u64;
context->page_fault = kvm_tdp_page_fault; context->page_fault = kvm_tdp_page_fault;
context->sync_page = nonpaging_sync_page; context->sync_page = nonpaging_sync_page;
context->invlpg = nonpaging_invlpg; context->invlpg = NULL;
context->update_pte = nonpaging_update_pte; context->update_pte = nonpaging_update_pte;
context->shadow_root_level = kvm_x86_ops.get_tdp_level(vcpu); context->shadow_root_level = kvm_x86_ops.get_tdp_level(vcpu);
context->direct_map = true; context->direct_map = true;
...@@ -5096,6 +5092,12 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu) ...@@ -5096,6 +5092,12 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
g_context->get_pdptr = kvm_pdptr_read; g_context->get_pdptr = kvm_pdptr_read;
g_context->inject_page_fault = kvm_inject_page_fault; g_context->inject_page_fault = kvm_inject_page_fault;
/*
* L2 page tables are never shadowed, so there is no need to sync
* SPTEs.
*/
g_context->invlpg = NULL;
/* /*
* Note that arch.mmu->gva_to_gpa translates l2_gpa to l1_gpa using * Note that arch.mmu->gva_to_gpa translates l2_gpa to l1_gpa using
* L1's nested page tables (e.g. EPT12). The nested translation * L1's nested page tables (e.g. EPT12). The nested translation
...@@ -5497,37 +5499,54 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code, ...@@ -5497,37 +5499,54 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
} }
EXPORT_SYMBOL_GPL(kvm_mmu_page_fault); EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva) void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
gva_t gva, hpa_t root_hpa)
{ {
struct kvm_mmu *mmu = vcpu->arch.mmu;
int i; int i;
/* INVLPG on a * non-canonical address is a NOP according to the SDM. */ /* It's actually a GPA for vcpu->arch.guest_mmu. */
if (is_noncanonical_address(gva, vcpu)) if (mmu != &vcpu->arch.guest_mmu) {
/* INVLPG on a non-canonical address is a NOP according to the SDM. */
if (is_noncanonical_address(gva, vcpu))
return;
kvm_x86_ops.tlb_flush_gva(vcpu, gva);
}
if (!mmu->invlpg)
return; return;
mmu->invlpg(vcpu, gva, mmu->root_hpa); if (root_hpa == INVALID_PAGE) {
mmu->invlpg(vcpu, gva, mmu->root_hpa);
/* /*
* INVLPG is required to invalidate any global mappings for the VA, * INVLPG is required to invalidate any global mappings for the VA,
* irrespective of PCID. Since it would take us roughly similar amount * irrespective of PCID. Since it would take us roughly similar amount
* of work to determine whether any of the prev_root mappings of the VA * of work to determine whether any of the prev_root mappings of the VA
* is marked global, or to just sync it blindly, so we might as well * is marked global, or to just sync it blindly, so we might as well
* just always sync it. * just always sync it.
* *
* Mappings not reachable via the current cr3 or the prev_roots will be * Mappings not reachable via the current cr3 or the prev_roots will be
* synced when switching to that cr3, so nothing needs to be done here * synced when switching to that cr3, so nothing needs to be done here
* for them. * for them.
*/ */
for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
if (VALID_PAGE(mmu->prev_roots[i].hpa)) if (VALID_PAGE(mmu->prev_roots[i].hpa))
mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa); mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
} else {
mmu->invlpg(vcpu, gva, root_hpa);
}
}
EXPORT_SYMBOL_GPL(kvm_mmu_invalidate_gva);
kvm_x86_ops.tlb_flush_gva(vcpu, gva); void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
{
kvm_mmu_invalidate_gva(vcpu, vcpu->arch.mmu, gva, INVALID_PAGE);
++vcpu->stat.invlpg; ++vcpu->stat.invlpg;
} }
EXPORT_SYMBOL_GPL(kvm_mmu_invlpg); EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid) void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
{ {
struct kvm_mmu *mmu = vcpu->arch.mmu; struct kvm_mmu *mmu = vcpu->arch.mmu;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment