Commit 689f3bf2 authored by Paolo Bonzini's avatar Paolo Bonzini

KVM: x86: unify callbacks to load paging root

Similar to what kvm-intel.ko is doing, provide a single callback that
merges svm_set_cr3, set_tdp_cr3 and nested_svm_set_tdp_cr3.

This lets us unify the set_cr3 and set_tdp_cr3 entries in kvm_x86_ops.
I'm doing that in this same patch because splitting it adds quite a bit
of churn due to the need for forward declarations.  For the same reason
the assignment to vcpu->arch.mmu->set_cr3 is moved to kvm_init_shadow_mmu
from init_kvm_softmmu and nested_svm_init_mmu_context.
Signed-off-by: default avatarPaolo Bonzini <pbonzini@redhat.com>
parent f91af517
...@@ -387,7 +387,6 @@ struct kvm_mmu_root_info { ...@@ -387,7 +387,6 @@ struct kvm_mmu_root_info {
* current mmu mode. * current mmu mode.
*/ */
struct kvm_mmu { struct kvm_mmu {
void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long root);
unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu); unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu);
u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index); u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
int (*page_fault)(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u32 err, int (*page_fault)(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u32 err,
...@@ -1156,8 +1155,6 @@ struct kvm_x86_ops { ...@@ -1156,8 +1155,6 @@ struct kvm_x86_ops {
int (*get_tdp_level)(struct kvm_vcpu *vcpu); int (*get_tdp_level)(struct kvm_vcpu *vcpu);
u64 (*get_mt_mask)(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio); u64 (*get_mt_mask)(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio);
void (*set_tdp_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3);
bool (*has_wbinvd_exit)(void); bool (*has_wbinvd_exit)(void);
u64 (*read_l1_tsc_offset)(struct kvm_vcpu *vcpu); u64 (*read_l1_tsc_offset)(struct kvm_vcpu *vcpu);
......
...@@ -95,11 +95,11 @@ static inline unsigned long kvm_get_active_pcid(struct kvm_vcpu *vcpu) ...@@ -95,11 +95,11 @@ static inline unsigned long kvm_get_active_pcid(struct kvm_vcpu *vcpu)
return kvm_get_pcid(vcpu, kvm_read_cr3(vcpu)); return kvm_get_pcid(vcpu, kvm_read_cr3(vcpu));
} }
static inline void kvm_mmu_load_cr3(struct kvm_vcpu *vcpu) static inline void kvm_mmu_load_pgd(struct kvm_vcpu *vcpu)
{ {
if (VALID_PAGE(vcpu->arch.mmu->root_hpa)) if (VALID_PAGE(vcpu->arch.mmu->root_hpa))
vcpu->arch.mmu->set_cr3(vcpu, vcpu->arch.mmu->root_hpa | kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu->root_hpa |
kvm_get_active_pcid(vcpu)); kvm_get_active_pcid(vcpu));
} }
int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code,
......
...@@ -4932,7 +4932,6 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) ...@@ -4932,7 +4932,6 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
context->update_pte = nonpaging_update_pte; context->update_pte = nonpaging_update_pte;
context->shadow_root_level = kvm_x86_ops->get_tdp_level(vcpu); context->shadow_root_level = kvm_x86_ops->get_tdp_level(vcpu);
context->direct_map = true; context->direct_map = true;
context->set_cr3 = kvm_x86_ops->set_tdp_cr3;
context->get_guest_pgd = get_cr3; context->get_guest_pgd = get_cr3;
context->get_pdptr = kvm_pdptr_read; context->get_pdptr = kvm_pdptr_read;
context->inject_page_fault = kvm_inject_page_fault; context->inject_page_fault = kvm_inject_page_fault;
...@@ -5079,7 +5078,6 @@ static void init_kvm_softmmu(struct kvm_vcpu *vcpu) ...@@ -5079,7 +5078,6 @@ static void init_kvm_softmmu(struct kvm_vcpu *vcpu)
struct kvm_mmu *context = vcpu->arch.mmu; struct kvm_mmu *context = vcpu->arch.mmu;
kvm_init_shadow_mmu(vcpu); kvm_init_shadow_mmu(vcpu);
context->set_cr3 = kvm_x86_ops->set_cr3;
context->get_guest_pgd = get_cr3; context->get_guest_pgd = get_cr3;
context->get_pdptr = kvm_pdptr_read; context->get_pdptr = kvm_pdptr_read;
context->inject_page_fault = kvm_inject_page_fault; context->inject_page_fault = kvm_inject_page_fault;
......
...@@ -2989,15 +2989,6 @@ static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) ...@@ -2989,15 +2989,6 @@ static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
return pdpte; return pdpte;
} }
static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
unsigned long root)
{
struct vcpu_svm *svm = to_svm(vcpu);
svm->vmcb->control.nested_cr3 = __sme_set(root);
mark_dirty(svm->vmcb, VMCB_NPT);
}
static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
struct x86_exception *fault) struct x86_exception *fault)
{ {
...@@ -3033,7 +3024,6 @@ static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu) ...@@ -3033,7 +3024,6 @@ static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
vcpu->arch.mmu = &vcpu->arch.guest_mmu; vcpu->arch.mmu = &vcpu->arch.guest_mmu;
kvm_init_shadow_mmu(vcpu); kvm_init_shadow_mmu(vcpu);
vcpu->arch.mmu->set_cr3 = nested_svm_set_tdp_cr3;
vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3; vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3;
vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr; vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr;
vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit; vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit;
...@@ -5955,21 +5945,27 @@ STACK_FRAME_NON_STANDARD(svm_vcpu_run); ...@@ -5955,21 +5945,27 @@ STACK_FRAME_NON_STANDARD(svm_vcpu_run);
static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root) static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
{ {
struct vcpu_svm *svm = to_svm(vcpu); struct vcpu_svm *svm = to_svm(vcpu);
bool update_guest_cr3 = true;
unsigned long cr3;
svm->vmcb->save.cr3 = __sme_set(root); cr3 = __sme_set(root);
mark_dirty(svm->vmcb, VMCB_CR); if (npt_enabled) {
} svm->vmcb->control.nested_cr3 = cr3;
mark_dirty(svm->vmcb, VMCB_NPT);
static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
{
struct vcpu_svm *svm = to_svm(vcpu);
svm->vmcb->control.nested_cr3 = __sme_set(root); /* Loading L2's CR3 is handled by enter_svm_guest_mode. */
mark_dirty(svm->vmcb, VMCB_NPT); if (is_guest_mode(vcpu))
update_guest_cr3 = false;
else if (test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
cr3 = vcpu->arch.cr3;
else /* CR3 is already up-to-date. */
update_guest_cr3 = false;
}
/* Also sync guest cr3 here in case we live migrate */ if (update_guest_cr3) {
svm->vmcb->save.cr3 = kvm_read_cr3(vcpu); svm->vmcb->save.cr3 = cr3;
mark_dirty(svm->vmcb, VMCB_CR); mark_dirty(svm->vmcb, VMCB_CR);
}
} }
static int is_disabled(void) static int is_disabled(void)
...@@ -7418,8 +7414,6 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = { ...@@ -7418,8 +7414,6 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
.read_l1_tsc_offset = svm_read_l1_tsc_offset, .read_l1_tsc_offset = svm_read_l1_tsc_offset,
.write_l1_tsc_offset = svm_write_l1_tsc_offset, .write_l1_tsc_offset = svm_write_l1_tsc_offset,
.set_tdp_cr3 = set_tdp_cr3,
.check_intercept = svm_check_intercept, .check_intercept = svm_check_intercept,
.handle_exit_irqoff = svm_handle_exit_irqoff, .handle_exit_irqoff = svm_handle_exit_irqoff,
......
...@@ -354,7 +354,6 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu) ...@@ -354,7 +354,6 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
VMX_EPT_EXECUTE_ONLY_BIT, VMX_EPT_EXECUTE_ONLY_BIT,
nested_ept_ad_enabled(vcpu), nested_ept_ad_enabled(vcpu),
nested_ept_get_eptp(vcpu)); nested_ept_get_eptp(vcpu));
vcpu->arch.mmu->set_cr3 = vmx_set_cr3;
vcpu->arch.mmu->get_guest_pgd = nested_ept_get_eptp; vcpu->arch.mmu->get_guest_pgd = nested_ept_get_eptp;
vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault; vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault;
vcpu->arch.mmu->get_pdptr = kvm_pdptr_read; vcpu->arch.mmu->get_pdptr = kvm_pdptr_read;
......
...@@ -7922,8 +7922,6 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = { ...@@ -7922,8 +7922,6 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = {
.read_l1_tsc_offset = vmx_read_l1_tsc_offset, .read_l1_tsc_offset = vmx_read_l1_tsc_offset,
.write_l1_tsc_offset = vmx_write_l1_tsc_offset, .write_l1_tsc_offset = vmx_write_l1_tsc_offset,
.set_tdp_cr3 = vmx_set_cr3,
.check_intercept = vmx_check_intercept, .check_intercept = vmx_check_intercept,
.handle_exit_irqoff = vmx_handle_exit_irqoff, .handle_exit_irqoff = vmx_handle_exit_irqoff,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment