vmx.c 309 KB
Newer Older
1 2 3 4 5 6 7
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
8
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
9 10 11 12 13 14 15 16 17 18
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

19
#include "irq.h"
20
#include "mmu.h"
21
#include "cpuid.h"
Avi Kivity's avatar
Avi Kivity committed
22

23
#include <linux/kvm_host.h>
24
#include <linux/module.h>
25
#include <linux/kernel.h>
26 27
#include <linux/mm.h>
#include <linux/highmem.h>
Alexey Dobriyan's avatar
Alexey Dobriyan committed
28
#include <linux/sched.h>
29
#include <linux/moduleparam.h>
30
#include <linux/mod_devicetable.h>
31
#include <linux/trace_events.h>
32
#include <linux/slab.h>
33
#include <linux/tboot.h>
34
#include <linux/hrtimer.h>
35
#include "kvm_cache_regs.h"
36
#include "x86.h"
Avi Kivity's avatar
Avi Kivity committed
37

38
#include <asm/cpu.h>
39
#include <asm/io.h>
40
#include <asm/desc.h>
41
#include <asm/vmx.h>
42
#include <asm/virtext.h>
43
#include <asm/mce.h>
44
#include <asm/fpu/internal.h>
45
#include <asm/perf_event.h>
46
#include <asm/debugreg.h>
47
#include <asm/kexec.h>
48
#include <asm/apic.h>
49
#include <asm/irq_remapping.h>
50

51
#include "trace.h"
52
#include "pmu.h"
53

54
#define __ex(x) __kvm_handle_fault_on_reboot(x)
55 56
#define __ex_clear(x, reg) \
	____kvm_handle_fault_on_reboot(x, "xor " reg " , " reg)
57

58 59 60
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

61 62 63 64 65 66
static const struct x86_cpu_id vmx_cpu_id[] = {
	X86_FEATURE_MATCH(X86_FEATURE_VMX),
	{}
};
MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id);

67
static bool __read_mostly enable_vpid = 1;
68
module_param_named(vpid, enable_vpid, bool, 0444);
69

70
static bool __read_mostly flexpriority_enabled = 1;
71
module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
72

73
static bool __read_mostly enable_ept = 1;
74
module_param_named(ept, enable_ept, bool, S_IRUGO);
Sheng Yang's avatar
Sheng Yang committed
75

76
static bool __read_mostly enable_unrestricted_guest = 1;
77 78 79
module_param_named(unrestricted_guest,
			enable_unrestricted_guest, bool, S_IRUGO);

80 81 82
static bool __read_mostly enable_ept_ad_bits = 1;
module_param_named(eptad, enable_ept_ad_bits, bool, S_IRUGO);

83
static bool __read_mostly emulate_invalid_guest_state = true;
84
module_param(emulate_invalid_guest_state, bool, S_IRUGO);
85

86
static bool __read_mostly vmm_exclusive = 1;
87 88
module_param(vmm_exclusive, bool, S_IRUGO);

89
static bool __read_mostly fasteoi = 1;
90 91
module_param(fasteoi, bool, S_IRUGO);

92
static bool __read_mostly enable_apicv = 1;
93
module_param(enable_apicv, bool, S_IRUGO);
94

95 96
static bool __read_mostly enable_shadow_vmcs = 1;
module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO);
97 98 99 100 101
/*
 * If nested=1, nested virtualization is supported, i.e., guests may use
 * VMX and be a hypervisor for its own guests. If nested=0, guests may not
 * use VMX instructions.
 */
102
static bool __read_mostly nested = 0;
103 104
module_param(nested, bool, S_IRUGO);

105 106
static u64 __read_mostly host_xss;

Kai Huang's avatar
Kai Huang committed
107 108 109
static bool __read_mostly enable_pml = 1;
module_param_named(pml, enable_pml, bool, S_IRUGO);

110 111
#define KVM_VMX_TSC_MULTIPLIER_MAX     0xffffffffffffffffULL

112 113
#define KVM_GUEST_CR0_MASK (X86_CR0_NW | X86_CR0_CD)
#define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST (X86_CR0_WP | X86_CR0_NE)
114 115
#define KVM_VM_CR0_ALWAYS_ON						\
	(KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
116 117
#define KVM_CR4_GUEST_OWNED_BITS				      \
	(X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR      \
118
	 | X86_CR4_OSXMMEXCPT | X86_CR4_TSD)
119

120 121 122
#define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
#define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)

123 124
#define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM))

125 126
#define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5

127 128 129 130
/*
 * These 2 parameters are used to config the controls for Pause-Loop Exiting:
 * ple_gap:    upper bound on the amount of time between two successive
 *             executions of PAUSE in a loop. Also indicate if ple enabled.
131
 *             According to test, this time is usually smaller than 128 cycles.
132 133 134 135 136 137
 * ple_window: upper bound on the amount of time a guest is allowed to execute
 *             in a PAUSE loop. Tests indicate that most spinlocks are held for
 *             less than 2^12 cycles
 * Time is measured based on a counter that runs at the same rate as the TSC,
 * refer SDM volume 3b section 21.6.13 & 22.1.3.
 */
138 139 140 141 142 143 144
#define KVM_VMX_DEFAULT_PLE_GAP           128
#define KVM_VMX_DEFAULT_PLE_WINDOW        4096
#define KVM_VMX_DEFAULT_PLE_WINDOW_GROW   2
#define KVM_VMX_DEFAULT_PLE_WINDOW_SHRINK 0
#define KVM_VMX_DEFAULT_PLE_WINDOW_MAX    \
		INT_MAX / KVM_VMX_DEFAULT_PLE_WINDOW_GROW

145 146 147 148 149 150
static int ple_gap = KVM_VMX_DEFAULT_PLE_GAP;
module_param(ple_gap, int, S_IRUGO);

static int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
module_param(ple_window, int, S_IRUGO);

151 152 153 154 155 156 157 158 159 160 161 162 163
/* Default doubles per-vcpu window every exit. */
static int ple_window_grow = KVM_VMX_DEFAULT_PLE_WINDOW_GROW;
module_param(ple_window_grow, int, S_IRUGO);

/* Default resets per-vcpu window every exit to ple_window. */
static int ple_window_shrink = KVM_VMX_DEFAULT_PLE_WINDOW_SHRINK;
module_param(ple_window_shrink, int, S_IRUGO);

/* Default is to compute the maximum so we can never overflow. */
static int ple_window_actual_max = KVM_VMX_DEFAULT_PLE_WINDOW_MAX;
static int ple_window_max        = KVM_VMX_DEFAULT_PLE_WINDOW_MAX;
module_param(ple_window_max, int, S_IRUGO);

Avi Kivity's avatar
Avi Kivity committed
164 165
extern const ulong vmx_return;

166
#define NR_AUTOLOAD_MSRS 8
167
#define VMCS02_POOL_SIZE 1
168

169 170 171 172 173 174
struct vmcs {
	u32 revision_id;
	u32 abort;
	char data[0];
};

175 176 177 178 179 180 181 182 183 184 185 186
/*
 * Track a VMCS that may be loaded on a certain CPU. If it is (cpu!=-1), also
 * remember whether it was VMLAUNCHed, and maintain a linked list of all VMCSs
 * loaded on this CPU (so we can clear them if the CPU goes down).
 */
struct loaded_vmcs {
	struct vmcs *vmcs;
	int cpu;
	int launched;
	struct list_head loaded_vmcss_on_cpu_link;
};

187 188 189
struct shared_msr_entry {
	unsigned index;
	u64 data;
190
	u64 mask;
191 192
};

193 194 195 196 197 198 199 200 201 202 203 204 205
/*
 * struct vmcs12 describes the state that our guest hypervisor (L1) keeps for a
 * single nested guest (L2), hence the name vmcs12. Any VMX implementation has
 * a VMCS structure, and vmcs12 is our emulated VMX's VMCS. This structure is
 * stored in guest memory specified by VMPTRLD, but is opaque to the guest,
 * which must access it using VMREAD/VMWRITE/VMCLEAR instructions.
 * More than one of these structures may exist, if L1 runs multiple L2 guests.
 * nested_vmx_run() will use the data here to build a vmcs02: a VMCS for the
 * underlying hardware which will be used to run L2.
 * This structure is packed to ensure that its layout is identical across
 * machines (necessary for live migration).
 * If there are changes in this struct, VMCS12_REVISION must be changed.
 */
206
typedef u64 natural_width;
207 208 209 210 211 212
struct __packed vmcs12 {
	/* According to the Intel spec, a VMCS region must start with the
	 * following two fields. Then follow implementation-specific data.
	 */
	u32 revision_id;
	u32 abort;
213

Nadav Har'El's avatar
Nadav Har'El committed
214 215 216
	u32 launch_state; /* set to 0 by VMCLEAR, to 1 by VMLAUNCH */
	u32 padding[7]; /* room for future expansion */

217 218 219 220 221 222 223 224 225
	u64 io_bitmap_a;
	u64 io_bitmap_b;
	u64 msr_bitmap;
	u64 vm_exit_msr_store_addr;
	u64 vm_exit_msr_load_addr;
	u64 vm_entry_msr_load_addr;
	u64 tsc_offset;
	u64 virtual_apic_page_addr;
	u64 apic_access_addr;
226
	u64 posted_intr_desc_addr;
227
	u64 ept_pointer;
228 229 230 231
	u64 eoi_exit_bitmap0;
	u64 eoi_exit_bitmap1;
	u64 eoi_exit_bitmap2;
	u64 eoi_exit_bitmap3;
232
	u64 xss_exit_bitmap;
233 234 235 236 237 238 239 240 241 242
	u64 guest_physical_address;
	u64 vmcs_link_pointer;
	u64 guest_ia32_debugctl;
	u64 guest_ia32_pat;
	u64 guest_ia32_efer;
	u64 guest_ia32_perf_global_ctrl;
	u64 guest_pdptr0;
	u64 guest_pdptr1;
	u64 guest_pdptr2;
	u64 guest_pdptr3;
243
	u64 guest_bndcfgs;
244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342
	u64 host_ia32_pat;
	u64 host_ia32_efer;
	u64 host_ia32_perf_global_ctrl;
	u64 padding64[8]; /* room for future expansion */
	/*
	 * To allow migration of L1 (complete with its L2 guests) between
	 * machines of different natural widths (32 or 64 bit), we cannot have
	 * unsigned long fields with no explict size. We use u64 (aliased
	 * natural_width) instead. Luckily, x86 is little-endian.
	 */
	natural_width cr0_guest_host_mask;
	natural_width cr4_guest_host_mask;
	natural_width cr0_read_shadow;
	natural_width cr4_read_shadow;
	natural_width cr3_target_value0;
	natural_width cr3_target_value1;
	natural_width cr3_target_value2;
	natural_width cr3_target_value3;
	natural_width exit_qualification;
	natural_width guest_linear_address;
	natural_width guest_cr0;
	natural_width guest_cr3;
	natural_width guest_cr4;
	natural_width guest_es_base;
	natural_width guest_cs_base;
	natural_width guest_ss_base;
	natural_width guest_ds_base;
	natural_width guest_fs_base;
	natural_width guest_gs_base;
	natural_width guest_ldtr_base;
	natural_width guest_tr_base;
	natural_width guest_gdtr_base;
	natural_width guest_idtr_base;
	natural_width guest_dr7;
	natural_width guest_rsp;
	natural_width guest_rip;
	natural_width guest_rflags;
	natural_width guest_pending_dbg_exceptions;
	natural_width guest_sysenter_esp;
	natural_width guest_sysenter_eip;
	natural_width host_cr0;
	natural_width host_cr3;
	natural_width host_cr4;
	natural_width host_fs_base;
	natural_width host_gs_base;
	natural_width host_tr_base;
	natural_width host_gdtr_base;
	natural_width host_idtr_base;
	natural_width host_ia32_sysenter_esp;
	natural_width host_ia32_sysenter_eip;
	natural_width host_rsp;
	natural_width host_rip;
	natural_width paddingl[8]; /* room for future expansion */
	u32 pin_based_vm_exec_control;
	u32 cpu_based_vm_exec_control;
	u32 exception_bitmap;
	u32 page_fault_error_code_mask;
	u32 page_fault_error_code_match;
	u32 cr3_target_count;
	u32 vm_exit_controls;
	u32 vm_exit_msr_store_count;
	u32 vm_exit_msr_load_count;
	u32 vm_entry_controls;
	u32 vm_entry_msr_load_count;
	u32 vm_entry_intr_info_field;
	u32 vm_entry_exception_error_code;
	u32 vm_entry_instruction_len;
	u32 tpr_threshold;
	u32 secondary_vm_exec_control;
	u32 vm_instruction_error;
	u32 vm_exit_reason;
	u32 vm_exit_intr_info;
	u32 vm_exit_intr_error_code;
	u32 idt_vectoring_info_field;
	u32 idt_vectoring_error_code;
	u32 vm_exit_instruction_len;
	u32 vmx_instruction_info;
	u32 guest_es_limit;
	u32 guest_cs_limit;
	u32 guest_ss_limit;
	u32 guest_ds_limit;
	u32 guest_fs_limit;
	u32 guest_gs_limit;
	u32 guest_ldtr_limit;
	u32 guest_tr_limit;
	u32 guest_gdtr_limit;
	u32 guest_idtr_limit;
	u32 guest_es_ar_bytes;
	u32 guest_cs_ar_bytes;
	u32 guest_ss_ar_bytes;
	u32 guest_ds_ar_bytes;
	u32 guest_fs_ar_bytes;
	u32 guest_gs_ar_bytes;
	u32 guest_ldtr_ar_bytes;
	u32 guest_tr_ar_bytes;
	u32 guest_interruptibility_info;
	u32 guest_activity_state;
	u32 guest_sysenter_cs;
	u32 host_ia32_sysenter_cs;
343 344
	u32 vmx_preemption_timer_value;
	u32 padding32[7]; /* room for future expansion */
345
	u16 virtual_processor_id;
346
	u16 posted_intr_nv;
347 348 349 350 351 352 353 354
	u16 guest_es_selector;
	u16 guest_cs_selector;
	u16 guest_ss_selector;
	u16 guest_ds_selector;
	u16 guest_fs_selector;
	u16 guest_gs_selector;
	u16 guest_ldtr_selector;
	u16 guest_tr_selector;
355
	u16 guest_intr_status;
356 357 358 359 360 361 362
	u16 host_es_selector;
	u16 host_cs_selector;
	u16 host_ss_selector;
	u16 host_ds_selector;
	u16 host_fs_selector;
	u16 host_gs_selector;
	u16 host_tr_selector;
363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378
};

/*
 * VMCS12_REVISION is an arbitrary id that should be changed if the content or
 * layout of struct vmcs12 is changed. MSR_IA32_VMX_BASIC returns this id, and
 * VMPTRLD verifies that the VMCS region that L1 is loading contains this id.
 */
#define VMCS12_REVISION 0x11e57ed0

/*
 * VMCS12_SIZE is the number of bytes L1 should allocate for the VMXON region
 * and any VMCS region. Although only sizeof(struct vmcs12) are used by the
 * current implementation, 4K are reserved to avoid future complications.
 */
#define VMCS12_SIZE 0x1000

379 380 381 382 383 384 385
/* Used to remember the last vmcs02 used for some recently used vmcs12s */
struct vmcs02_list {
	struct list_head list;
	gpa_t vmptr;
	struct loaded_vmcs vmcs02;
};

386 387 388 389 390 391 392
/*
 * The nested_vmx structure is part of vcpu_vmx, and holds information we need
 * for correct emulation of VMX (i.e., nested VMX) on this vcpu.
 */
struct nested_vmx {
	/* Has the level1 guest done vmxon? */
	bool vmxon;
393
	gpa_t vmxon_ptr;
394 395 396 397 398 399

	/* The guest-physical address of the current VMCS L1 keeps for L2 */
	gpa_t current_vmptr;
	/* The host-usable pointer to the above */
	struct page *current_vmcs12_page;
	struct vmcs12 *current_vmcs12;
400
	struct vmcs *current_shadow_vmcs;
401 402 403 404 405
	/*
	 * Indicates if the shadow vmcs must be updated with the
	 * data hold by vmcs12
	 */
	bool sync_shadow_vmcs;
406 407 408 409

	/* vmcs02_list cache of VMCSs recently used to run L2 guests */
	struct list_head vmcs02_pool;
	int vmcs02_num;
410
	u64 vmcs01_tsc_offset;
411
	bool change_vmcs01_virtual_x2apic_mode;
412 413
	/* L2 must run next, and mustn't decide to exit to L1. */
	bool nested_run_pending;
414 415 416 417 418
	/*
	 * Guest pages referred to in vmcs02 with host-physical pointers, so
	 * we must keep them pinned while L2 runs.
	 */
	struct page *apic_access_page;
419
	struct page *virtual_apic_page;
420 421 422 423
	struct page *pi_desc_page;
	struct pi_desc *pi_desc;
	bool pi_pending;
	u16 posted_intr_nv;
424
	u64 msr_ia32_feature_control;
425 426 427

	struct hrtimer preemption_timer;
	bool preemption_timer_expired;
428 429 430

	/* to migrate it to L2 if VM_ENTRY_LOAD_DEBUG_CONTROLS is off */
	u64 vmcs01_debugctl;
431

432 433 434
	u16 vpid02;
	u16 last_vpid;

435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450
	u32 nested_vmx_procbased_ctls_low;
	u32 nested_vmx_procbased_ctls_high;
	u32 nested_vmx_true_procbased_ctls_low;
	u32 nested_vmx_secondary_ctls_low;
	u32 nested_vmx_secondary_ctls_high;
	u32 nested_vmx_pinbased_ctls_low;
	u32 nested_vmx_pinbased_ctls_high;
	u32 nested_vmx_exit_ctls_low;
	u32 nested_vmx_exit_ctls_high;
	u32 nested_vmx_true_exit_ctls_low;
	u32 nested_vmx_entry_ctls_low;
	u32 nested_vmx_entry_ctls_high;
	u32 nested_vmx_true_entry_ctls_low;
	u32 nested_vmx_misc_low;
	u32 nested_vmx_misc_high;
	u32 nested_vmx_ept_caps;
451
	u32 nested_vmx_vpid_caps;
452 453
};

454
#define POSTED_INTR_ON  0
455 456
#define POSTED_INTR_SN  1

457 458 459
/* Posted-Interrupt Descriptor */
struct pi_desc {
	u32 pir[8];     /* Posted interrupt requested */
460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477
	union {
		struct {
				/* bit 256 - Outstanding Notification */
			u16	on	: 1,
				/* bit 257 - Suppress Notification */
				sn	: 1,
				/* bit 271:258 - Reserved */
				rsvd_1	: 14;
				/* bit 279:272 - Notification Vector */
			u8	nv;
				/* bit 287:280 - Reserved */
			u8	rsvd_2;
				/* bit 319:288 - Notification Destination */
			u32	ndst;
		};
		u64 control;
	};
	u32 rsvd[6];
478 479
} __aligned(64);

480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496
static bool pi_test_and_set_on(struct pi_desc *pi_desc)
{
	return test_and_set_bit(POSTED_INTR_ON,
			(unsigned long *)&pi_desc->control);
}

static bool pi_test_and_clear_on(struct pi_desc *pi_desc)
{
	return test_and_clear_bit(POSTED_INTR_ON,
			(unsigned long *)&pi_desc->control);
}

static int pi_test_and_set_pir(int vector, struct pi_desc *pi_desc)
{
	return test_and_set_bit(vector, (unsigned long *)pi_desc->pir);
}

497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520
static inline void pi_clear_sn(struct pi_desc *pi_desc)
{
	return clear_bit(POSTED_INTR_SN,
			(unsigned long *)&pi_desc->control);
}

static inline void pi_set_sn(struct pi_desc *pi_desc)
{
	return set_bit(POSTED_INTR_SN,
			(unsigned long *)&pi_desc->control);
}

static inline int pi_test_on(struct pi_desc *pi_desc)
{
	return test_bit(POSTED_INTR_ON,
			(unsigned long *)&pi_desc->control);
}

static inline int pi_test_sn(struct pi_desc *pi_desc)
{
	return test_bit(POSTED_INTR_SN,
			(unsigned long *)&pi_desc->control);
}

521
struct vcpu_vmx {
522
	struct kvm_vcpu       vcpu;
523
	unsigned long         host_rsp;
524
	u8                    fail;
525
	bool                  nmi_known_unmasked;
526
	u32                   exit_intr_info;
527
	u32                   idt_vectoring_info;
528
	ulong                 rflags;
529
	struct shared_msr_entry *guest_msrs;
530 531
	int                   nmsrs;
	int                   save_nmsrs;
532
	unsigned long	      host_idt_base;
533
#ifdef CONFIG_X86_64
534 535
	u64 		      msr_host_kernel_gs_base;
	u64 		      msr_guest_kernel_gs_base;
536
#endif
537 538
	u32 vm_entry_controls_shadow;
	u32 vm_exit_controls_shadow;
539 540 541 542 543 544 545 546
	/*
	 * loaded_vmcs points to the VMCS currently used in this vcpu. For a
	 * non-nested (L1) guest, it always points to vmcs01. For a nested
	 * guest (L2), it points to a different VMCS.
	 */
	struct loaded_vmcs    vmcs01;
	struct loaded_vmcs   *loaded_vmcs;
	bool                  __launched; /* temporary, used in vmx_vcpu_run */
547 548 549 550 551
	struct msr_autoload {
		unsigned nr;
		struct vmx_msr_entry guest[NR_AUTOLOAD_MSRS];
		struct vmx_msr_entry host[NR_AUTOLOAD_MSRS];
	} msr_autoload;
552 553 554
	struct {
		int           loaded;
		u16           fs_sel, gs_sel, ldt_sel;
555 556 557
#ifdef CONFIG_X86_64
		u16           ds_sel, es_sel;
#endif
558 559
		int           gs_ldt_reload_needed;
		int           fs_reload_needed;
560
		u64           msr_host_bndcfgs;
561
		unsigned long vmcs_host_cr4;	/* May not match real cr4 */
Mike Day's avatar
Mike Day committed
562
	} host_state;
563
	struct {
564
		int vm86_active;
565
		ulong save_rflags;
566 567 568 569
		struct kvm_segment segs[8];
	} rmode;
	struct {
		u32 bitmask; /* 4 bits per segment (1 bit per field) */
570 571 572 573 574
		struct kvm_save_segment {
			u16 selector;
			unsigned long base;
			u32 limit;
			u32 ar;
575
		} seg[8];
576
	} segment_cache;
577
	int vpid;
578
	bool emulation_required;
579 580 581 582 583

	/* Support for vnmi-less CPUs */
	int soft_vnmi_blocked;
	ktime_t entry_time;
	s64 vnmi_blocked_time;
584
	u32 exit_reason;
585

586 587 588
	/* Posted interrupt descriptor */
	struct pi_desc pi_desc;

589 590
	/* Support for a guest hypervisor (nested VMX) */
	struct nested_vmx nested;
591 592 593 594

	/* Dynamic PLE window. */
	int ple_window;
	bool ple_window_dirty;
Kai Huang's avatar
Kai Huang committed
595 596 597 598

	/* Support for PML */
#define PML_ENTITY_NUM		512
	struct page *pml_pg;
599 600

	u64 current_tsc_ratio;
601 602
};

603 604 605 606 607 608 609 610 611
enum segment_cache_field {
	SEG_FIELD_SEL = 0,
	SEG_FIELD_BASE = 1,
	SEG_FIELD_LIMIT = 2,
	SEG_FIELD_AR = 3,

	SEG_FIELD_NR = 4
};

612 613
static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
{
614
	return container_of(vcpu, struct vcpu_vmx, vcpu);
615 616
}

617 618 619 620 621
static struct pi_desc *vcpu_to_pi_desc(struct kvm_vcpu *vcpu)
{
	return &(to_vmx(vcpu)->pi_desc);
}

622 623 624 625 626
#define VMCS12_OFFSET(x) offsetof(struct vmcs12, x)
#define FIELD(number, name)	[number] = VMCS12_OFFSET(name)
#define FIELD64(number, name)	[number] = VMCS12_OFFSET(name), \
				[number##_HIGH] = VMCS12_OFFSET(name)+4

627

628
static unsigned long shadow_read_only_fields[] = {
629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650
	/*
	 * We do NOT shadow fields that are modified when L0
	 * traps and emulates any vmx instruction (e.g. VMPTRLD,
	 * VMXON...) executed by L1.
	 * For example, VM_INSTRUCTION_ERROR is read
	 * by L1 if a vmx instruction fails (part of the error path).
	 * Note the code assumes this logic. If for some reason
	 * we start shadowing these fields then we need to
	 * force a shadow sync when L0 emulates vmx instructions
	 * (e.g. force a sync if VM_INSTRUCTION_ERROR is modified
	 * by nested_vmx_failValid)
	 */
	VM_EXIT_REASON,
	VM_EXIT_INTR_INFO,
	VM_EXIT_INSTRUCTION_LEN,
	IDT_VECTORING_INFO_FIELD,
	IDT_VECTORING_ERROR_CODE,
	VM_EXIT_INTR_ERROR_CODE,
	EXIT_QUALIFICATION,
	GUEST_LINEAR_ADDRESS,
	GUEST_PHYSICAL_ADDRESS
};
651
static int max_shadow_read_only_fields =
652 653
	ARRAY_SIZE(shadow_read_only_fields);

654
static unsigned long shadow_read_write_fields[] = {
655
	TPR_THRESHOLD,
656 657 658 659 660 661 662 663 664 665 666 667
	GUEST_RIP,
	GUEST_RSP,
	GUEST_CR0,
	GUEST_CR3,
	GUEST_CR4,
	GUEST_INTERRUPTIBILITY_INFO,
	GUEST_RFLAGS,
	GUEST_CS_SELECTOR,
	GUEST_CS_AR_BYTES,
	GUEST_CS_LIMIT,
	GUEST_CS_BASE,
	GUEST_ES_BASE,
668
	GUEST_BNDCFGS,
669 670 671 672 673 674 675 676 677 678 679 680 681 682 683
	CR0_GUEST_HOST_MASK,
	CR0_READ_SHADOW,
	CR4_READ_SHADOW,
	TSC_OFFSET,
	EXCEPTION_BITMAP,
	CPU_BASED_VM_EXEC_CONTROL,
	VM_ENTRY_EXCEPTION_ERROR_CODE,
	VM_ENTRY_INTR_INFO_FIELD,
	VM_ENTRY_INSTRUCTION_LEN,
	VM_ENTRY_EXCEPTION_ERROR_CODE,
	HOST_FS_BASE,
	HOST_GS_BASE,
	HOST_FS_SELECTOR,
	HOST_GS_SELECTOR
};
684
static int max_shadow_read_write_fields =
685 686
	ARRAY_SIZE(shadow_read_write_fields);

687
static const unsigned short vmcs_field_to_offset_table[] = {
688
	FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id),
689
	FIELD(POSTED_INTR_NV, posted_intr_nv),
690 691 692 693 694 695 696 697
	FIELD(GUEST_ES_SELECTOR, guest_es_selector),
	FIELD(GUEST_CS_SELECTOR, guest_cs_selector),
	FIELD(GUEST_SS_SELECTOR, guest_ss_selector),
	FIELD(GUEST_DS_SELECTOR, guest_ds_selector),
	FIELD(GUEST_FS_SELECTOR, guest_fs_selector),
	FIELD(GUEST_GS_SELECTOR, guest_gs_selector),
	FIELD(GUEST_LDTR_SELECTOR, guest_ldtr_selector),
	FIELD(GUEST_TR_SELECTOR, guest_tr_selector),
698
	FIELD(GUEST_INTR_STATUS, guest_intr_status),
699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714
	FIELD(HOST_ES_SELECTOR, host_es_selector),
	FIELD(HOST_CS_SELECTOR, host_cs_selector),
	FIELD(HOST_SS_SELECTOR, host_ss_selector),
	FIELD(HOST_DS_SELECTOR, host_ds_selector),
	FIELD(HOST_FS_SELECTOR, host_fs_selector),
	FIELD(HOST_GS_SELECTOR, host_gs_selector),
	FIELD(HOST_TR_SELECTOR, host_tr_selector),
	FIELD64(IO_BITMAP_A, io_bitmap_a),
	FIELD64(IO_BITMAP_B, io_bitmap_b),
	FIELD64(MSR_BITMAP, msr_bitmap),
	FIELD64(VM_EXIT_MSR_STORE_ADDR, vm_exit_msr_store_addr),
	FIELD64(VM_EXIT_MSR_LOAD_ADDR, vm_exit_msr_load_addr),
	FIELD64(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr),
	FIELD64(TSC_OFFSET, tsc_offset),
	FIELD64(VIRTUAL_APIC_PAGE_ADDR, virtual_apic_page_addr),
	FIELD64(APIC_ACCESS_ADDR, apic_access_addr),
715
	FIELD64(POSTED_INTR_DESC_ADDR, posted_intr_desc_addr),
716
	FIELD64(EPT_POINTER, ept_pointer),
717 718 719 720
	FIELD64(EOI_EXIT_BITMAP0, eoi_exit_bitmap0),
	FIELD64(EOI_EXIT_BITMAP1, eoi_exit_bitmap1),
	FIELD64(EOI_EXIT_BITMAP2, eoi_exit_bitmap2),
	FIELD64(EOI_EXIT_BITMAP3, eoi_exit_bitmap3),
721
	FIELD64(XSS_EXIT_BITMAP, xss_exit_bitmap),
722 723 724 725 726 727 728 729 730 731
	FIELD64(GUEST_PHYSICAL_ADDRESS, guest_physical_address),
	FIELD64(VMCS_LINK_POINTER, vmcs_link_pointer),
	FIELD64(GUEST_IA32_DEBUGCTL, guest_ia32_debugctl),
	FIELD64(GUEST_IA32_PAT, guest_ia32_pat),
	FIELD64(GUEST_IA32_EFER, guest_ia32_efer),
	FIELD64(GUEST_IA32_PERF_GLOBAL_CTRL, guest_ia32_perf_global_ctrl),
	FIELD64(GUEST_PDPTR0, guest_pdptr0),
	FIELD64(GUEST_PDPTR1, guest_pdptr1),
	FIELD64(GUEST_PDPTR2, guest_pdptr2),
	FIELD64(GUEST_PDPTR3, guest_pdptr3),
732
	FIELD64(GUEST_BNDCFGS, guest_bndcfgs),
733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781
	FIELD64(HOST_IA32_PAT, host_ia32_pat),
	FIELD64(HOST_IA32_EFER, host_ia32_efer),
	FIELD64(HOST_IA32_PERF_GLOBAL_CTRL, host_ia32_perf_global_ctrl),
	FIELD(PIN_BASED_VM_EXEC_CONTROL, pin_based_vm_exec_control),
	FIELD(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control),
	FIELD(EXCEPTION_BITMAP, exception_bitmap),
	FIELD(PAGE_FAULT_ERROR_CODE_MASK, page_fault_error_code_mask),
	FIELD(PAGE_FAULT_ERROR_CODE_MATCH, page_fault_error_code_match),
	FIELD(CR3_TARGET_COUNT, cr3_target_count),
	FIELD(VM_EXIT_CONTROLS, vm_exit_controls),
	FIELD(VM_EXIT_MSR_STORE_COUNT, vm_exit_msr_store_count),
	FIELD(VM_EXIT_MSR_LOAD_COUNT, vm_exit_msr_load_count),
	FIELD(VM_ENTRY_CONTROLS, vm_entry_controls),
	FIELD(VM_ENTRY_MSR_LOAD_COUNT, vm_entry_msr_load_count),
	FIELD(VM_ENTRY_INTR_INFO_FIELD, vm_entry_intr_info_field),
	FIELD(VM_ENTRY_EXCEPTION_ERROR_CODE, vm_entry_exception_error_code),
	FIELD(VM_ENTRY_INSTRUCTION_LEN, vm_entry_instruction_len),
	FIELD(TPR_THRESHOLD, tpr_threshold),
	FIELD(SECONDARY_VM_EXEC_CONTROL, secondary_vm_exec_control),
	FIELD(VM_INSTRUCTION_ERROR, vm_instruction_error),
	FIELD(VM_EXIT_REASON, vm_exit_reason),
	FIELD(VM_EXIT_INTR_INFO, vm_exit_intr_info),
	FIELD(VM_EXIT_INTR_ERROR_CODE, vm_exit_intr_error_code),
	FIELD(IDT_VECTORING_INFO_FIELD, idt_vectoring_info_field),
	FIELD(IDT_VECTORING_ERROR_CODE, idt_vectoring_error_code),
	FIELD(VM_EXIT_INSTRUCTION_LEN, vm_exit_instruction_len),
	FIELD(VMX_INSTRUCTION_INFO, vmx_instruction_info),
	FIELD(GUEST_ES_LIMIT, guest_es_limit),
	FIELD(GUEST_CS_LIMIT, guest_cs_limit),
	FIELD(GUEST_SS_LIMIT, guest_ss_limit),
	FIELD(GUEST_DS_LIMIT, guest_ds_limit),
	FIELD(GUEST_FS_LIMIT, guest_fs_limit),
	FIELD(GUEST_GS_LIMIT, guest_gs_limit),
	FIELD(GUEST_LDTR_LIMIT, guest_ldtr_limit),
	FIELD(GUEST_TR_LIMIT, guest_tr_limit),
	FIELD(GUEST_GDTR_LIMIT, guest_gdtr_limit),
	FIELD(GUEST_IDTR_LIMIT, guest_idtr_limit),
	FIELD(GUEST_ES_AR_BYTES, guest_es_ar_bytes),
	FIELD(GUEST_CS_AR_BYTES, guest_cs_ar_bytes),
	FIELD(GUEST_SS_AR_BYTES, guest_ss_ar_bytes),
	FIELD(GUEST_DS_AR_BYTES, guest_ds_ar_bytes),
	FIELD(GUEST_FS_AR_BYTES, guest_fs_ar_bytes),
	FIELD(GUEST_GS_AR_BYTES, guest_gs_ar_bytes),
	FIELD(GUEST_LDTR_AR_BYTES, guest_ldtr_ar_bytes),
	FIELD(GUEST_TR_AR_BYTES, guest_tr_ar_bytes),
	FIELD(GUEST_INTERRUPTIBILITY_INFO, guest_interruptibility_info),
	FIELD(GUEST_ACTIVITY_STATE, guest_activity_state),
	FIELD(GUEST_SYSENTER_CS, guest_sysenter_cs),
	FIELD(HOST_IA32_SYSENTER_CS, host_ia32_sysenter_cs),
782
	FIELD(VMX_PREEMPTION_TIMER_VALUE, vmx_preemption_timer_value),
783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828
	FIELD(CR0_GUEST_HOST_MASK, cr0_guest_host_mask),
	FIELD(CR4_GUEST_HOST_MASK, cr4_guest_host_mask),
	FIELD(CR0_READ_SHADOW, cr0_read_shadow),
	FIELD(CR4_READ_SHADOW, cr4_read_shadow),
	FIELD(CR3_TARGET_VALUE0, cr3_target_value0),
	FIELD(CR3_TARGET_VALUE1, cr3_target_value1),
	FIELD(CR3_TARGET_VALUE2, cr3_target_value2),
	FIELD(CR3_TARGET_VALUE3, cr3_target_value3),
	FIELD(EXIT_QUALIFICATION, exit_qualification),
	FIELD(GUEST_LINEAR_ADDRESS, guest_linear_address),
	FIELD(GUEST_CR0, guest_cr0),
	FIELD(GUEST_CR3, guest_cr3),
	FIELD(GUEST_CR4, guest_cr4),
	FIELD(GUEST_ES_BASE, guest_es_base),
	FIELD(GUEST_CS_BASE, guest_cs_base),
	FIELD(GUEST_SS_BASE, guest_ss_base),
	FIELD(GUEST_DS_BASE, guest_ds_base),
	FIELD(GUEST_FS_BASE, guest_fs_base),
	FIELD(GUEST_GS_BASE, guest_gs_base),
	FIELD(GUEST_LDTR_BASE, guest_ldtr_base),
	FIELD(GUEST_TR_BASE, guest_tr_base),
	FIELD(GUEST_GDTR_BASE, guest_gdtr_base),
	FIELD(GUEST_IDTR_BASE, guest_idtr_base),
	FIELD(GUEST_DR7, guest_dr7),
	FIELD(GUEST_RSP, guest_rsp),
	FIELD(GUEST_RIP, guest_rip),
	FIELD(GUEST_RFLAGS, guest_rflags),
	FIELD(GUEST_PENDING_DBG_EXCEPTIONS, guest_pending_dbg_exceptions),
	FIELD(GUEST_SYSENTER_ESP, guest_sysenter_esp),
	FIELD(GUEST_SYSENTER_EIP, guest_sysenter_eip),
	FIELD(HOST_CR0, host_cr0),
	FIELD(HOST_CR3, host_cr3),
	FIELD(HOST_CR4, host_cr4),
	FIELD(HOST_FS_BASE, host_fs_base),
	FIELD(HOST_GS_BASE, host_gs_base),
	FIELD(HOST_TR_BASE, host_tr_base),
	FIELD(HOST_GDTR_BASE, host_gdtr_base),
	FIELD(HOST_IDTR_BASE, host_idtr_base),
	FIELD(HOST_IA32_SYSENTER_ESP, host_ia32_sysenter_esp),
	FIELD(HOST_IA32_SYSENTER_EIP, host_ia32_sysenter_eip),
	FIELD(HOST_RSP, host_rsp),
	FIELD(HOST_RIP, host_rip),
};

static inline short vmcs_field_to_offset(unsigned long field)
{
829 830 831 832 833 834
	BUILD_BUG_ON(ARRAY_SIZE(vmcs_field_to_offset_table) > SHRT_MAX);

	if (field >= ARRAY_SIZE(vmcs_field_to_offset_table) ||
	    vmcs_field_to_offset_table[field] == 0)
		return -ENOENT;

835 836 837
	return vmcs_field_to_offset_table[field];
}

838 839 840 841 842 843 844
static inline struct vmcs12 *get_vmcs12(struct kvm_vcpu *vcpu)
{
	return to_vmx(vcpu)->nested.current_vmcs12;
}

static struct page *nested_get_page(struct kvm_vcpu *vcpu, gpa_t addr)
{
845
	struct page *page = kvm_vcpu_gfn_to_page(vcpu, addr >> PAGE_SHIFT);
846
	if (is_error_page(page))
847
		return NULL;
848

849 850 851 852 853 854 855 856 857 858 859 860 861
	return page;
}

static void nested_release_page(struct page *page)
{
	kvm_release_page_dirty(page);
}

static void nested_release_page_clean(struct page *page)
{
	kvm_release_page_clean(page);
}

Nadav Har'El's avatar
Nadav Har'El committed
862
static unsigned long nested_ept_get_cr3(struct kvm_vcpu *vcpu);
863
static u64 construct_eptp(unsigned long root_hpa);
864 865
static void kvm_cpu_vmxon(u64 addr);
static void kvm_cpu_vmxoff(void);
866
static bool vmx_xsaves_supported(void);
867
static int vmx_cpu_uses_apicv(struct kvm_vcpu *vcpu);
868
static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr);
869 870 871 872
static void vmx_set_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg);
static void vmx_get_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg);
873 874
static bool guest_state_valid(struct kvm_vcpu *vcpu);
static u32 vmx_segment_access_rights(struct kvm_segment *var);
875
static void vmx_sync_pir_to_irr_dummy(struct kvm_vcpu *vcpu);
876
static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx);
877
static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx);
878
static int alloc_identity_pagetable(struct kvm *kvm);
879

880 881
static DEFINE_PER_CPU(struct vmcs *, vmxarea);
static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
882 883 884 885 886
/*
 * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed
 * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it.
 */
static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu);
887
static DEFINE_PER_CPU(struct desc_ptr, host_gdt);
888

889 890 891 892 893 894 895
/*
 * We maintian a per-CPU linked-list of vCPU, so in wakeup_handler() we
 * can find which vCPU should be waken up.
 */
static DEFINE_PER_CPU(struct list_head, blocked_vcpu_on_cpu);
static DEFINE_PER_CPU(spinlock_t, blocked_vcpu_on_cpu_lock);

896 897
static unsigned long *vmx_io_bitmap_a;
static unsigned long *vmx_io_bitmap_b;
898 899
static unsigned long *vmx_msr_bitmap_legacy;
static unsigned long *vmx_msr_bitmap_longmode;
900 901
static unsigned long *vmx_msr_bitmap_legacy_x2apic;
static unsigned long *vmx_msr_bitmap_longmode_x2apic;
902
static unsigned long *vmx_msr_bitmap_nested;
903 904
static unsigned long *vmx_vmread_bitmap;
static unsigned long *vmx_vmwrite_bitmap;
905

906
static bool cpu_has_load_ia32_efer;
907
static bool cpu_has_load_perf_global_ctrl;
908

909 910 911
static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
static DEFINE_SPINLOCK(vmx_vpid_lock);

912
static struct vmcs_config {
913 914 915
	int size;
	int order;
	u32 revision_id;
916 917
	u32 pin_based_exec_ctrl;
	u32 cpu_based_exec_ctrl;
918
	u32 cpu_based_2nd_exec_ctrl;
919 920 921
	u32 vmexit_ctrl;
	u32 vmentry_ctrl;
} vmcs_config;
922

Hannes Eder's avatar
Hannes Eder committed
923
static struct vmx_capability {
Sheng Yang's avatar
Sheng Yang committed
924 925 926 927
	u32 ept;
	u32 vpid;
} vmx_capability;

928 929 930 931 932 933 934 935
#define VMX_SEGMENT_FIELD(seg)					\
	[VCPU_SREG_##seg] = {                                   \
		.selector = GUEST_##seg##_SELECTOR,		\
		.base = GUEST_##seg##_BASE,		   	\
		.limit = GUEST_##seg##_LIMIT,		   	\
		.ar_bytes = GUEST_##seg##_AR_BYTES,	   	\
	}

936
static const struct kvm_vmx_segment_field {
937 938 939 940 941 942 943 944 945 946 947 948 949 950 951
	unsigned selector;
	unsigned base;
	unsigned limit;
	unsigned ar_bytes;
} kvm_vmx_segment_fields[] = {
	VMX_SEGMENT_FIELD(CS),
	VMX_SEGMENT_FIELD(DS),
	VMX_SEGMENT_FIELD(ES),
	VMX_SEGMENT_FIELD(FS),
	VMX_SEGMENT_FIELD(GS),
	VMX_SEGMENT_FIELD(SS),
	VMX_SEGMENT_FIELD(TR),
	VMX_SEGMENT_FIELD(LDTR),
};

952 953
static u64 host_efer;

Avi Kivity's avatar
Avi Kivity committed
954 955
static void ept_save_pdptrs(struct kvm_vcpu *vcpu);

956
/*
Brian Gerst's avatar
Brian Gerst committed
957
 * Keep MSR_STAR at the end, as setup_msrs() will try to optimize it
958 959
 * away by decrementing the array size.
 */
960
static const u32 vmx_msr_index[] = {
961
#ifdef CONFIG_X86_64
962
	MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
963
#endif
Brian Gerst's avatar
Brian Gerst committed
964
	MSR_EFER, MSR_TSC_AUX, MSR_STAR,
965 966
};

967
static inline bool is_page_fault(u32 intr_info)
968 969 970
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
971
		(INTR_TYPE_HARD_EXCEPTION | PF_VECTOR | INTR_INFO_VALID_MASK);
972 973
}

974
static inline bool is_no_device(u32 intr_info)
975 976 977
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
978
		(INTR_TYPE_HARD_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK);
979 980
}

981
static inline bool is_invalid_opcode(u32 intr_info)
982 983 984
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
985
		(INTR_TYPE_HARD_EXCEPTION | UD_VECTOR | INTR_INFO_VALID_MASK);
986 987
}

988
static inline bool is_external_interrupt(u32 intr_info)
989 990 991 992 993
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
		== (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
}

994
static inline bool is_machine_check(u32 intr_info)
995 996 997 998 999 1000
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
		(INTR_TYPE_HARD_EXCEPTION | MC_VECTOR | INTR_INFO_VALID_MASK);
}

1001
static inline bool cpu_has_vmx_msr_bitmap(void)
1002
{
1003
	return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_USE_MSR_BITMAPS;
1004 1005
}

1006
static inline bool cpu_has_vmx_tpr_shadow(void)
1007
{
1008
	return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW;
1009 1010
}

1011
static inline bool cpu_need_tpr_shadow(struct kvm_vcpu *vcpu)
1012
{
1013
	return cpu_has_vmx_tpr_shadow() && lapic_in_kernel(vcpu);
1014 1015
}

1016
static inline bool cpu_has_secondary_exec_ctrls(void)
1017
{
1018 1019
	return vmcs_config.cpu_based_exec_ctrl &
		CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
1020 1021
}

1022
static inline bool cpu_has_vmx_virtualize_apic_accesses(void)
1023
{
1024 1025 1026 1027
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
}

1028 1029 1030 1031 1032 1033
static inline bool cpu_has_vmx_virtualize_x2apic_mode(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
}

1034 1035 1036 1037 1038 1039
static inline bool cpu_has_vmx_apic_register_virt(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_APIC_REGISTER_VIRT;
}

1040 1041 1042 1043 1044 1045
static inline bool cpu_has_vmx_virtual_intr_delivery(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY;
}

1046 1047
static inline bool cpu_has_vmx_posted_intr(void)
{
1048 1049
	return IS_ENABLED(CONFIG_X86_LOCAL_APIC) &&
		vmcs_config.pin_based_exec_ctrl & PIN_BASED_POSTED_INTR;
1050 1051 1052 1053 1054 1055 1056 1057 1058
}

static inline bool cpu_has_vmx_apicv(void)
{
	return cpu_has_vmx_apic_register_virt() &&
		cpu_has_vmx_virtual_intr_delivery() &&
		cpu_has_vmx_posted_intr();
}

1059 1060 1061 1062
static inline bool cpu_has_vmx_flexpriority(void)
{
	return cpu_has_vmx_tpr_shadow() &&
		cpu_has_vmx_virtualize_apic_accesses();
1063 1064
}

1065 1066
static inline bool cpu_has_vmx_ept_execute_only(void)
{
1067
	return vmx_capability.ept & VMX_EPT_EXECUTE_ONLY_BIT;
1068 1069 1070 1071
}

static inline bool cpu_has_vmx_ept_2m_page(void)
{
1072
	return vmx_capability.ept & VMX_EPT_2MB_PAGE_BIT;
1073 1074
}

1075 1076
static inline bool cpu_has_vmx_ept_1g_page(void)
{
1077
	return vmx_capability.ept & VMX_EPT_1GB_PAGE_BIT;
1078 1079
}

1080 1081 1082 1083 1084
static inline bool cpu_has_vmx_ept_4levels(void)
{
	return vmx_capability.ept & VMX_EPT_PAGE_WALK_4_BIT;
}

1085 1086 1087 1088 1089
static inline bool cpu_has_vmx_ept_ad_bits(void)
{
	return vmx_capability.ept & VMX_EPT_AD_BIT;
}

1090
static inline bool cpu_has_vmx_invept_context(void)
Sheng Yang's avatar
Sheng Yang committed
1091
{
1092
	return vmx_capability.ept & VMX_EPT_EXTENT_CONTEXT_BIT;
Sheng Yang's avatar
Sheng Yang committed
1093 1094
}

1095
static inline bool cpu_has_vmx_invept_global(void)
Sheng Yang's avatar
Sheng Yang committed
1096
{
1097
	return vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT;
Sheng Yang's avatar
Sheng Yang committed
1098 1099
}

1100 1101 1102 1103 1104
static inline bool cpu_has_vmx_invvpid_single(void)
{
	return vmx_capability.vpid & VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT;
}

1105 1106 1107 1108 1109
static inline bool cpu_has_vmx_invvpid_global(void)
{
	return vmx_capability.vpid & VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT;
}

1110
static inline bool cpu_has_vmx_ept(void)
Sheng Yang's avatar
Sheng Yang committed
1111
{
1112 1113
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_ENABLE_EPT;
Sheng Yang's avatar
Sheng Yang committed
1114 1115
}

1116
static inline bool cpu_has_vmx_unrestricted_guest(void)
1117 1118 1119 1120 1121
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_UNRESTRICTED_GUEST;
}

1122
static inline bool cpu_has_vmx_ple(void)
1123 1124 1125 1126 1127
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_PAUSE_LOOP_EXITING;
}

1128
static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu)
1129
{
1130
	return flexpriority_enabled && lapic_in_kernel(vcpu);
1131 1132
}

1133
static inline bool cpu_has_vmx_vpid(void)
1134
{
1135 1136
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_ENABLE_VPID;
1137 1138
}

1139
static inline bool cpu_has_vmx_rdtscp(void)
1140 1141 1142 1143 1144
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_RDTSCP;
}

1145 1146 1147 1148 1149 1150
static inline bool cpu_has_vmx_invpcid(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_ENABLE_INVPCID;
}

1151
static inline bool cpu_has_virtual_nmis(void)
1152 1153 1154 1155
{
	return vmcs_config.pin_based_exec_ctrl & PIN_BASED_VIRTUAL_NMIS;
}

1156 1157 1158 1159 1160 1161
static inline bool cpu_has_vmx_wbinvd_exit(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_WBINVD_EXITING;
}

1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173
static inline bool cpu_has_vmx_shadow_vmcs(void)
{
	u64 vmx_msr;
	rdmsrl(MSR_IA32_VMX_MISC, vmx_msr);
	/* check if the cpu supports writing r/o exit information fields */
	if (!(vmx_msr & MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS))
		return false;

	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_SHADOW_VMCS;
}

Kai Huang's avatar
Kai Huang committed
1174 1175 1176 1177 1178
static inline bool cpu_has_vmx_pml(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl & SECONDARY_EXEC_ENABLE_PML;
}

1179 1180 1181 1182 1183 1184
static inline bool cpu_has_vmx_tsc_scaling(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_TSC_SCALING;
}

1185 1186 1187 1188 1189
static inline bool report_flexpriority(void)
{
	return flexpriority_enabled;
}

1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201
static inline bool nested_cpu_has(struct vmcs12 *vmcs12, u32 bit)
{
	return vmcs12->cpu_based_vm_exec_control & bit;
}

static inline bool nested_cpu_has2(struct vmcs12 *vmcs12, u32 bit)
{
	return (vmcs12->cpu_based_vm_exec_control &
			CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
		(vmcs12->secondary_vm_exec_control & bit);
}

Nadav Har'El's avatar
Nadav Har'El committed
1202
static inline bool nested_cpu_has_virtual_nmis(struct vmcs12 *vmcs12)
1203 1204 1205 1206
{
	return vmcs12->pin_based_vm_exec_control & PIN_BASED_VIRTUAL_NMIS;
}

1207 1208 1209 1210 1211 1212
static inline bool nested_cpu_has_preemption_timer(struct vmcs12 *vmcs12)
{
	return vmcs12->pin_based_vm_exec_control &
		PIN_BASED_VMX_PREEMPTION_TIMER;
}

1213 1214 1215 1216 1217
static inline int nested_cpu_has_ept(struct vmcs12 *vmcs12)
{
	return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_EPT);
}

1218 1219 1220 1221 1222 1223
static inline bool nested_cpu_has_xsaves(struct vmcs12 *vmcs12)
{
	return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES) &&
		vmx_xsaves_supported();
}

1224 1225 1226 1227 1228
static inline bool nested_cpu_has_virt_x2apic_mode(struct vmcs12 *vmcs12)
{
	return nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE);
}

1229 1230 1231 1232 1233
static inline bool nested_cpu_has_vpid(struct vmcs12 *vmcs12)
{
	return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_VPID);
}

1234 1235 1236 1237 1238
static inline bool nested_cpu_has_apic_reg_virt(struct vmcs12 *vmcs12)
{
	return nested_cpu_has2(vmcs12, SECONDARY_EXEC_APIC_REGISTER_VIRT);
}

1239 1240 1241 1242 1243
static inline bool nested_cpu_has_vid(struct vmcs12 *vmcs12)
{
	return nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
}

1244 1245 1246 1247 1248
static inline bool nested_cpu_has_posted_intr(struct vmcs12 *vmcs12)
{
	return vmcs12->pin_based_vm_exec_control & PIN_BASED_POSTED_INTR;
}

1249
static inline bool is_nmi(u32 intr_info)
1250 1251
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
1252
		== (INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK);
1253 1254
}

1255 1256 1257
static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
			      u32 exit_intr_info,
			      unsigned long exit_qualification);
1258 1259 1260 1261
static void nested_vmx_entry_failure(struct kvm_vcpu *vcpu,
			struct vmcs12 *vmcs12,
			u32 reason, unsigned long qualification);

1262
static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
1263 1264 1265
{
	int i;

1266
	for (i = 0; i < vmx->nmsrs; ++i)
1267
		if (vmx_msr_index[vmx->guest_msrs[i].index] == msr)
1268 1269 1270 1271
			return i;
	return -1;
}

1272 1273 1274 1275 1276 1277 1278 1279
static inline void __invvpid(int ext, u16 vpid, gva_t gva)
{
    struct {
	u64 vpid : 16;
	u64 rsvd : 48;
	u64 gva;
    } operand = { vpid, 0, gva };

1280
    asm volatile (__ex(ASM_VMX_INVVPID)
1281 1282 1283 1284 1285
		  /* CF==1 or ZF==1 --> rc = -1 */
		  "; ja 1f ; ud2 ; 1:"
		  : : "a"(&operand), "c"(ext) : "cc", "memory");
}

1286 1287 1288 1289 1290 1291
static inline void __invept(int ext, u64 eptp, gpa_t gpa)
{
	struct {
		u64 eptp, gpa;
	} operand = {eptp, gpa};

1292
	asm volatile (__ex(ASM_VMX_INVEPT)
1293 1294 1295 1296 1297
			/* CF==1 or ZF==1 --> rc = -1 */
			"; ja 1f ; ud2 ; 1:\n"
			: : "a" (&operand), "c" (ext) : "cc", "memory");
}

1298
static struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
1299 1300 1301
{
	int i;

1302
	i = __find_msr_index(vmx, msr);
1303
	if (i >= 0)
1304
		return &vmx->guest_msrs[i];
Al Viro's avatar
Al Viro committed
1305
	return NULL;
1306 1307
}

1308 1309 1310 1311 1312
static void vmcs_clear(struct vmcs *vmcs)
{
	u64 phys_addr = __pa(vmcs);
	u8 error;

1313
	asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) "; setna %0"
1314
		      : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
1315 1316 1317 1318 1319 1320
		      : "cc", "memory");
	if (error)
		printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
		       vmcs, phys_addr);
}

1321 1322 1323 1324 1325 1326 1327
static inline void loaded_vmcs_init(struct loaded_vmcs *loaded_vmcs)
{
	vmcs_clear(loaded_vmcs->vmcs);
	loaded_vmcs->cpu = -1;
	loaded_vmcs->launched = 0;
}

1328 1329 1330 1331 1332 1333
static void vmcs_load(struct vmcs *vmcs)
{
	u64 phys_addr = __pa(vmcs);
	u8 error;

	asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) "; setna %0"
1334
			: "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
1335 1336
			: "cc", "memory");
	if (error)
1337
		printk(KERN_ERR "kvm: vmptrld %p/%llx failed\n",
1338 1339 1340
		       vmcs, phys_addr);
}

1341
#ifdef CONFIG_KEXEC_CORE
1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378
/*
 * This bitmap is used to indicate whether the vmclear
 * operation is enabled on all cpus. All disabled by
 * default.
 */
static cpumask_t crash_vmclear_enabled_bitmap = CPU_MASK_NONE;

static inline void crash_enable_local_vmclear(int cpu)
{
	cpumask_set_cpu(cpu, &crash_vmclear_enabled_bitmap);
}

static inline void crash_disable_local_vmclear(int cpu)
{
	cpumask_clear_cpu(cpu, &crash_vmclear_enabled_bitmap);
}

static inline int crash_local_vmclear_enabled(int cpu)
{
	return cpumask_test_cpu(cpu, &crash_vmclear_enabled_bitmap);
}

static void crash_vmclear_local_loaded_vmcss(void)
{
	int cpu = raw_smp_processor_id();
	struct loaded_vmcs *v;

	if (!crash_local_vmclear_enabled(cpu))
		return;

	list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu),
			    loaded_vmcss_on_cpu_link)
		vmcs_clear(v->vmcs);
}
#else
static inline void crash_enable_local_vmclear(int cpu) { }
static inline void crash_disable_local_vmclear(int cpu) { }
1379
#endif /* CONFIG_KEXEC_CORE */
1380

1381
static void __loaded_vmcs_clear(void *arg)
1382
{
1383
	struct loaded_vmcs *loaded_vmcs = arg;
1384
	int cpu = raw_smp_processor_id();
1385

1386 1387 1388
	if (loaded_vmcs->cpu != cpu)
		return; /* vcpu migration can race with cpu offline */
	if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs)
1389
		per_cpu(current_vmcs, cpu) = NULL;
1390
	crash_disable_local_vmclear(cpu);
1391
	list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link);
1392 1393 1394 1395 1396 1397 1398 1399 1400

	/*
	 * we should ensure updating loaded_vmcs->loaded_vmcss_on_cpu_link
	 * is before setting loaded_vmcs->vcpu to -1 which is done in
	 * loaded_vmcs_init. Otherwise, other cpu can see vcpu = -1 fist
	 * then adds the vmcs into percpu list before it is deleted.
	 */
	smp_wmb();

1401
	loaded_vmcs_init(loaded_vmcs);
1402
	crash_enable_local_vmclear(cpu);
1403 1404
}

1405
static void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs)
1406
{
1407 1408 1409 1410 1411
	int cpu = loaded_vmcs->cpu;

	if (cpu != -1)
		smp_call_function_single(cpu,
			 __loaded_vmcs_clear, loaded_vmcs, 1);
1412 1413
}

1414
static inline void vpid_sync_vcpu_single(int vpid)
1415
{
1416
	if (vpid == 0)
1417 1418
		return;

1419
	if (cpu_has_vmx_invvpid_single())
1420
		__invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vpid, 0);
1421 1422
}

1423 1424 1425 1426 1427 1428
static inline void vpid_sync_vcpu_global(void)
{
	if (cpu_has_vmx_invvpid_global())
		__invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0);
}

1429
static inline void vpid_sync_context(int vpid)
1430 1431
{
	if (cpu_has_vmx_invvpid_single())
1432
		vpid_sync_vcpu_single(vpid);
1433 1434 1435 1436
	else
		vpid_sync_vcpu_global();
}

1437 1438 1439 1440 1441 1442 1443 1444
static inline void ept_sync_global(void)
{
	if (cpu_has_vmx_invept_global())
		__invept(VMX_EPT_EXTENT_GLOBAL, 0, 0);
}

static inline void ept_sync_context(u64 eptp)
{
1445
	if (enable_ept) {
1446 1447 1448 1449 1450 1451 1452
		if (cpu_has_vmx_invept_context())
			__invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
		else
			ept_sync_global();
	}
}

Avi Kivity's avatar
Avi Kivity committed
1453
static __always_inline unsigned long vmcs_readl(unsigned long field)
1454
{
1455
	unsigned long value;
1456

1457 1458
	asm volatile (__ex_clear(ASM_VMX_VMREAD_RDX_RAX, "%0")
		      : "=a"(value) : "d"(field) : "cc");
1459 1460 1461
	return value;
}

Avi Kivity's avatar
Avi Kivity committed
1462
static __always_inline u16 vmcs_read16(unsigned long field)
1463 1464 1465 1466
{
	return vmcs_readl(field);
}

Avi Kivity's avatar
Avi Kivity committed
1467
static __always_inline u32 vmcs_read32(unsigned long field)
1468 1469 1470 1471
{
	return vmcs_readl(field);
}

Avi Kivity's avatar
Avi Kivity committed
1472
static __always_inline u64 vmcs_read64(unsigned long field)
1473
{
1474
#ifdef CONFIG_X86_64
1475 1476 1477 1478 1479 1480
	return vmcs_readl(field);
#else
	return vmcs_readl(field) | ((u64)vmcs_readl(field+1) << 32);
#endif
}

1481 1482 1483 1484 1485 1486 1487
static noinline void vmwrite_error(unsigned long field, unsigned long value)
{
	printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
	       field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
	dump_stack();
}

1488 1489 1490 1491
static void vmcs_writel(unsigned long field, unsigned long value)
{
	u8 error;

1492
	asm volatile (__ex(ASM_VMX_VMWRITE_RAX_RDX) "; setna %0"
Mike Day's avatar
Mike Day committed
1493
		       : "=q"(error) : "a"(value), "d"(field) : "cc");
1494 1495
	if (unlikely(error))
		vmwrite_error(field, value);
1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510
}

static void vmcs_write16(unsigned long field, u16 value)
{
	vmcs_writel(field, value);
}

static void vmcs_write32(unsigned long field, u32 value)
{
	vmcs_writel(field, value);
}

static void vmcs_write64(unsigned long field, u64 value)
{
	vmcs_writel(field, value);
1511
#ifndef CONFIG_X86_64
1512 1513 1514 1515 1516
	asm volatile ("");
	vmcs_writel(field+1, value >> 32);
#endif
}

1517 1518 1519 1520 1521 1522 1523 1524 1525 1526
static void vmcs_clear_bits(unsigned long field, u32 mask)
{
	vmcs_writel(field, vmcs_readl(field) & ~mask);
}

static void vmcs_set_bits(unsigned long field, u32 mask)
{
	vmcs_writel(field, vmcs_readl(field) | mask);
}

1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582
static inline void vm_entry_controls_init(struct vcpu_vmx *vmx, u32 val)
{
	vmcs_write32(VM_ENTRY_CONTROLS, val);
	vmx->vm_entry_controls_shadow = val;
}

static inline void vm_entry_controls_set(struct vcpu_vmx *vmx, u32 val)
{
	if (vmx->vm_entry_controls_shadow != val)
		vm_entry_controls_init(vmx, val);
}

static inline u32 vm_entry_controls_get(struct vcpu_vmx *vmx)
{
	return vmx->vm_entry_controls_shadow;
}


static inline void vm_entry_controls_setbit(struct vcpu_vmx *vmx, u32 val)
{
	vm_entry_controls_set(vmx, vm_entry_controls_get(vmx) | val);
}

static inline void vm_entry_controls_clearbit(struct vcpu_vmx *vmx, u32 val)
{
	vm_entry_controls_set(vmx, vm_entry_controls_get(vmx) & ~val);
}

static inline void vm_exit_controls_init(struct vcpu_vmx *vmx, u32 val)
{
	vmcs_write32(VM_EXIT_CONTROLS, val);
	vmx->vm_exit_controls_shadow = val;
}

static inline void vm_exit_controls_set(struct vcpu_vmx *vmx, u32 val)
{
	if (vmx->vm_exit_controls_shadow != val)
		vm_exit_controls_init(vmx, val);
}

static inline u32 vm_exit_controls_get(struct vcpu_vmx *vmx)
{
	return vmx->vm_exit_controls_shadow;
}


static inline void vm_exit_controls_setbit(struct vcpu_vmx *vmx, u32 val)
{
	vm_exit_controls_set(vmx, vm_exit_controls_get(vmx) | val);
}

static inline void vm_exit_controls_clearbit(struct vcpu_vmx *vmx, u32 val)
{
	vm_exit_controls_set(vmx, vm_exit_controls_get(vmx) & ~val);
}

1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638
static void vmx_segment_cache_clear(struct vcpu_vmx *vmx)
{
	vmx->segment_cache.bitmask = 0;
}

static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg,
				       unsigned field)
{
	bool ret;
	u32 mask = 1 << (seg * SEG_FIELD_NR + field);

	if (!(vmx->vcpu.arch.regs_avail & (1 << VCPU_EXREG_SEGMENTS))) {
		vmx->vcpu.arch.regs_avail |= (1 << VCPU_EXREG_SEGMENTS);
		vmx->segment_cache.bitmask = 0;
	}
	ret = vmx->segment_cache.bitmask & mask;
	vmx->segment_cache.bitmask |= mask;
	return ret;
}

static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg)
{
	u16 *p = &vmx->segment_cache.seg[seg].selector;

	if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL))
		*p = vmcs_read16(kvm_vmx_segment_fields[seg].selector);
	return *p;
}

static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg)
{
	ulong *p = &vmx->segment_cache.seg[seg].base;

	if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE))
		*p = vmcs_readl(kvm_vmx_segment_fields[seg].base);
	return *p;
}

static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg)
{
	u32 *p = &vmx->segment_cache.seg[seg].limit;

	if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT))
		*p = vmcs_read32(kvm_vmx_segment_fields[seg].limit);
	return *p;
}

static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg)
{
	u32 *p = &vmx->segment_cache.seg[seg].ar;

	if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR))
		*p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes);
	return *p;
}

1639 1640 1641 1642
static void update_exception_bitmap(struct kvm_vcpu *vcpu)
{
	u32 eb;

1643
	eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) |
1644
	     (1u << NM_VECTOR) | (1u << DB_VECTOR) | (1u << AC_VECTOR);
1645 1646 1647 1648
	if ((vcpu->guest_debug &
	     (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) ==
	    (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP))
		eb |= 1u << BP_VECTOR;
1649
	if (to_vmx(vcpu)->rmode.vm86_active)
1650
		eb = ~0;
1651
	if (enable_ept)
1652
		eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */
1653 1654
	if (vcpu->fpu_active)
		eb &= ~(1u << NM_VECTOR);
1655 1656 1657 1658 1659 1660 1661 1662 1663

	/* When we are running a nested L2 guest and L1 specified for it a
	 * certain exception bitmap, we must trap the same exceptions and pass
	 * them to L1. When running L2, we will only handle the exceptions
	 * specified above if L1 did not want them.
	 */
	if (is_guest_mode(vcpu))
		eb |= get_vmcs12(vcpu)->exception_bitmap;

1664 1665 1666
	vmcs_write32(EXCEPTION_BITMAP, eb);
}

1667 1668
static void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx,
		unsigned long entry, unsigned long exit)
1669
{
1670 1671
	vm_entry_controls_clearbit(vmx, entry);
	vm_exit_controls_clearbit(vmx, exit);
1672 1673
}

1674 1675 1676 1677 1678
static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
{
	unsigned i;
	struct msr_autoload *m = &vmx->msr_autoload;

1679 1680 1681
	switch (msr) {
	case MSR_EFER:
		if (cpu_has_load_ia32_efer) {
1682 1683
			clear_atomic_switch_msr_special(vmx,
					VM_ENTRY_LOAD_IA32_EFER,
1684 1685 1686 1687 1688 1689
					VM_EXIT_LOAD_IA32_EFER);
			return;
		}
		break;
	case MSR_CORE_PERF_GLOBAL_CTRL:
		if (cpu_has_load_perf_global_ctrl) {
1690
			clear_atomic_switch_msr_special(vmx,
1691 1692 1693 1694 1695
					VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
					VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
			return;
		}
		break;
1696 1697
	}

1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710
	for (i = 0; i < m->nr; ++i)
		if (m->guest[i].index == msr)
			break;

	if (i == m->nr)
		return;
	--m->nr;
	m->guest[i] = m->guest[m->nr];
	m->host[i] = m->host[m->nr];
	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
}

1711 1712 1713 1714
static void add_atomic_switch_msr_special(struct vcpu_vmx *vmx,
		unsigned long entry, unsigned long exit,
		unsigned long guest_val_vmcs, unsigned long host_val_vmcs,
		u64 guest_val, u64 host_val)
1715 1716 1717
{
	vmcs_write64(guest_val_vmcs, guest_val);
	vmcs_write64(host_val_vmcs, host_val);
1718 1719
	vm_entry_controls_setbit(vmx, entry);
	vm_exit_controls_setbit(vmx, exit);
1720 1721
}

1722 1723 1724 1725 1726 1727
static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
				  u64 guest_val, u64 host_val)
{
	unsigned i;
	struct msr_autoload *m = &vmx->msr_autoload;

1728 1729 1730
	switch (msr) {
	case MSR_EFER:
		if (cpu_has_load_ia32_efer) {
1731 1732
			add_atomic_switch_msr_special(vmx,
					VM_ENTRY_LOAD_IA32_EFER,
1733 1734 1735 1736 1737 1738 1739 1740 1741
					VM_EXIT_LOAD_IA32_EFER,
					GUEST_IA32_EFER,
					HOST_IA32_EFER,
					guest_val, host_val);
			return;
		}
		break;
	case MSR_CORE_PERF_GLOBAL_CTRL:
		if (cpu_has_load_perf_global_ctrl) {
1742
			add_atomic_switch_msr_special(vmx,
1743 1744 1745 1746 1747 1748 1749 1750
					VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
					VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL,
					GUEST_IA32_PERF_GLOBAL_CTRL,
					HOST_IA32_PERF_GLOBAL_CTRL,
					guest_val, host_val);
			return;
		}
		break;
1751 1752 1753 1754 1755 1756 1757
	case MSR_IA32_PEBS_ENABLE:
		/* PEBS needs a quiescent period after being disabled (to write
		 * a record).  Disabling PEBS through VMX MSR swapping doesn't
		 * provide that period, so a CPU could write host's record into
		 * guest's memory.
		 */
		wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
1758 1759
	}

1760 1761 1762 1763
	for (i = 0; i < m->nr; ++i)
		if (m->guest[i].index == msr)
			break;

1764
	if (i == NR_AUTOLOAD_MSRS) {
1765
		printk_once(KERN_WARNING "Not enough msr switch entries. "
1766 1767 1768
				"Can't add msr %x\n", msr);
		return;
	} else if (i == m->nr) {
1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779
		++m->nr;
		vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
		vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
	}

	m->guest[i].index = msr;
	m->guest[i].value = guest_val;
	m->host[i].index = msr;
	m->host[i].value = host_val;
}

1780 1781 1782 1783 1784
static void reload_tss(void)
{
	/*
	 * VT restores TR but not its size.  Useless.
	 */
1785
	struct desc_ptr *gdt = this_cpu_ptr(&host_gdt);
1786
	struct desc_struct *descs;
1787

1788
	descs = (void *)gdt->address;
1789 1790 1791 1792
	descs[GDT_ENTRY_TSS].type = 9; /* available TSS */
	load_TR_desc();
}

1793
static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
1794
{
1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808
	u64 guest_efer = vmx->vcpu.arch.efer;
	u64 ignore_bits = 0;

	if (!enable_ept) {
		/*
		 * NX is needed to handle CR0.WP=1, CR4.SMEP=1.  Testing
		 * host CPUID is more efficient than testing guest CPUID
		 * or CR4.  Host SMEP is anyway a requirement for guest SMEP.
		 */
		if (boot_cpu_has(X86_FEATURE_SMEP))
			guest_efer |= EFER_NX;
		else if (!(guest_efer & EFER_NX))
			ignore_bits |= EFER_NX;
	}
1809

1810
	/*
1811
	 * LMA and LME handled by hardware; SCE meaningless outside long mode.
1812
	 */
1813
	ignore_bits |= EFER_SCE;
1814 1815 1816 1817 1818 1819
#ifdef CONFIG_X86_64
	ignore_bits |= EFER_LMA | EFER_LME;
	/* SCE is meaningful only in long mode on Intel */
	if (guest_efer & EFER_LMA)
		ignore_bits &= ~(u64)EFER_SCE;
#endif
1820 1821

	clear_atomic_switch_msr(vmx, MSR_EFER);
1822 1823 1824 1825 1826 1827 1828 1829

	/*
	 * On EPT, we can't emulate NX, so we must switch EFER atomically.
	 * On CPUs that support "load IA32_EFER", always switch EFER
	 * atomically, since it's faster than switching it manually.
	 */
	if (cpu_has_load_ia32_efer ||
	    (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX))) {
1830 1831
		if (!(guest_efer & EFER_LMA))
			guest_efer &= ~EFER_LME;
1832 1833 1834
		if (guest_efer != host_efer)
			add_atomic_switch_msr(vmx, MSR_EFER,
					      guest_efer, host_efer);
1835
		return false;
1836 1837 1838 1839 1840 1841
	} else {
		guest_efer &= ~ignore_bits;
		guest_efer |= host_efer & ignore_bits;

		vmx->guest_msrs[efer_offset].data = guest_efer;
		vmx->guest_msrs[efer_offset].mask = ~ignore_bits;
1842

1843 1844
		return true;
	}
1845 1846
}

1847 1848
static unsigned long segment_base(u16 selector)
{
1849
	struct desc_ptr *gdt = this_cpu_ptr(&host_gdt);
1850 1851 1852 1853 1854 1855 1856
	struct desc_struct *d;
	unsigned long table_base;
	unsigned long v;

	if (!(selector & ~3))
		return 0;

1857
	table_base = gdt->address;
1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882

	if (selector & 4) {           /* from ldt */
		u16 ldt_selector = kvm_read_ldt();

		if (!(ldt_selector & ~3))
			return 0;

		table_base = segment_base(ldt_selector);
	}
	d = (struct desc_struct *)(table_base + (selector & ~7));
	v = get_desc_base(d);
#ifdef CONFIG_X86_64
       if (d->s == 0 && (d->type == 2 || d->type == 9 || d->type == 11))
               v |= ((unsigned long)((struct ldttss_desc64 *)d)->base3) << 32;
#endif
	return v;
}

static inline unsigned long kvm_read_tr_base(void)
{
	u16 tr;
	asm("str %0" : "=g"(tr));
	return segment_base(tr);
}

1883
static void vmx_save_host_state(struct kvm_vcpu *vcpu)
1884
{
1885
	struct vcpu_vmx *vmx = to_vmx(vcpu);
1886
	int i;
1887

1888
	if (vmx->host_state.loaded)
1889 1890
		return;

1891
	vmx->host_state.loaded = 1;
1892 1893 1894 1895
	/*
	 * Set host fs and gs selectors.  Unfortunately, 22.2.3 does not
	 * allow segment selectors with cpl > 0 or ti == 1.
	 */
1896
	vmx->host_state.ldt_sel = kvm_read_ldt();
1897
	vmx->host_state.gs_ldt_reload_needed = vmx->host_state.ldt_sel;
1898
	savesegment(fs, vmx->host_state.fs_sel);
1899
	if (!(vmx->host_state.fs_sel & 7)) {
1900
		vmcs_write16(HOST_FS_SELECTOR, vmx->host_state.fs_sel);
1901 1902
		vmx->host_state.fs_reload_needed = 0;
	} else {
1903
		vmcs_write16(HOST_FS_SELECTOR, 0);
1904
		vmx->host_state.fs_reload_needed = 1;
1905
	}
1906
	savesegment(gs, vmx->host_state.gs_sel);
1907 1908
	if (!(vmx->host_state.gs_sel & 7))
		vmcs_write16(HOST_GS_SELECTOR, vmx->host_state.gs_sel);
1909 1910
	else {
		vmcs_write16(HOST_GS_SELECTOR, 0);
1911
		vmx->host_state.gs_ldt_reload_needed = 1;
1912 1913
	}

1914 1915 1916 1917 1918
#ifdef CONFIG_X86_64
	savesegment(ds, vmx->host_state.ds_sel);
	savesegment(es, vmx->host_state.es_sel);
#endif

1919 1920 1921 1922
#ifdef CONFIG_X86_64
	vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE));
	vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE));
#else
1923 1924
	vmcs_writel(HOST_FS_BASE, segment_base(vmx->host_state.fs_sel));
	vmcs_writel(HOST_GS_BASE, segment_base(vmx->host_state.gs_sel));
1925
#endif
1926 1927

#ifdef CONFIG_X86_64
1928 1929
	rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
	if (is_long_mode(&vmx->vcpu))
1930
		wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
1931
#endif
1932 1933
	if (boot_cpu_has(X86_FEATURE_MPX))
		rdmsrl(MSR_IA32_BNDCFGS, vmx->host_state.msr_host_bndcfgs);
1934 1935
	for (i = 0; i < vmx->save_nmsrs; ++i)
		kvm_set_shared_msr(vmx->guest_msrs[i].index,
1936 1937
				   vmx->guest_msrs[i].data,
				   vmx->guest_msrs[i].mask);
1938 1939
}

1940
static void __vmx_load_host_state(struct vcpu_vmx *vmx)
1941
{
1942
	if (!vmx->host_state.loaded)
1943 1944
		return;

1945
	++vmx->vcpu.stat.host_state_reload;
1946
	vmx->host_state.loaded = 0;
1947 1948 1949 1950
#ifdef CONFIG_X86_64
	if (is_long_mode(&vmx->vcpu))
		rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
#endif
1951
	if (vmx->host_state.gs_ldt_reload_needed) {
1952
		kvm_load_ldt(vmx->host_state.ldt_sel);
1953
#ifdef CONFIG_X86_64
1954 1955 1956
		load_gs_index(vmx->host_state.gs_sel);
#else
		loadsegment(gs, vmx->host_state.gs_sel);
1957 1958
#endif
	}
1959 1960
	if (vmx->host_state.fs_reload_needed)
		loadsegment(fs, vmx->host_state.fs_sel);
1961 1962 1963 1964 1965 1966
#ifdef CONFIG_X86_64
	if (unlikely(vmx->host_state.ds_sel | vmx->host_state.es_sel)) {
		loadsegment(ds, vmx->host_state.ds_sel);
		loadsegment(es, vmx->host_state.es_sel);
	}
#endif
1967
	reload_tss();
1968
#ifdef CONFIG_X86_64
1969
	wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
1970
#endif
1971 1972
	if (vmx->host_state.msr_host_bndcfgs)
		wrmsrl(MSR_IA32_BNDCFGS, vmx->host_state.msr_host_bndcfgs);
1973 1974 1975 1976
	/*
	 * If the FPU is not active (through the host task or
	 * the guest vcpu), then restore the cr0.TS bit.
	 */
1977
	if (!fpregs_active() && !vmx->vcpu.guest_fpu_loaded)
1978
		stts();
1979
	load_gdt(this_cpu_ptr(&host_gdt));
1980 1981
}

1982 1983 1984 1985 1986 1987 1988
static void vmx_load_host_state(struct vcpu_vmx *vmx)
{
	preempt_disable();
	__vmx_load_host_state(vmx);
	preempt_enable();
}

1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034
static void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
{
	struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
	struct pi_desc old, new;
	unsigned int dest;

	if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
		!irq_remapping_cap(IRQ_POSTING_CAP))
		return;

	do {
		old.control = new.control = pi_desc->control;

		/*
		 * If 'nv' field is POSTED_INTR_WAKEUP_VECTOR, there
		 * are two possible cases:
		 * 1. After running 'pre_block', context switch
		 *    happened. For this case, 'sn' was set in
		 *    vmx_vcpu_put(), so we need to clear it here.
		 * 2. After running 'pre_block', we were blocked,
		 *    and woken up by some other guy. For this case,
		 *    we don't need to do anything, 'pi_post_block'
		 *    will do everything for us. However, we cannot
		 *    check whether it is case #1 or case #2 here
		 *    (maybe, not needed), so we also clear sn here,
		 *    I think it is not a big deal.
		 */
		if (pi_desc->nv != POSTED_INTR_WAKEUP_VECTOR) {
			if (vcpu->cpu != cpu) {
				dest = cpu_physical_id(cpu);

				if (x2apic_enabled())
					new.ndst = dest;
				else
					new.ndst = (dest << 8) & 0xFF00;
			}

			/* set 'NV' to 'notification vector' */
			new.nv = POSTED_INTR_VECTOR;
		}

		/* Allow posting non-urgent interrupts */
		new.sn = 0;
	} while (cmpxchg(&pi_desc->control, old.control,
			new.control) != old.control);
}
2035 2036 2037 2038
/*
 * Switches to specified vcpu, until a matching vcpu_put(), but assumes
 * vcpu mutex is already taken.
 */
2039
static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2040
{
2041
	struct vcpu_vmx *vmx = to_vmx(vcpu);
2042
	u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
2043

2044 2045
	if (!vmm_exclusive)
		kvm_cpu_vmxon(phys_addr);
2046 2047
	else if (vmx->loaded_vmcs->cpu != cpu)
		loaded_vmcs_clear(vmx->loaded_vmcs);
2048

2049 2050 2051
	if (per_cpu(current_vmcs, cpu) != vmx->loaded_vmcs->vmcs) {
		per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
		vmcs_load(vmx->loaded_vmcs->vmcs);
2052 2053
	}

2054
	if (vmx->loaded_vmcs->cpu != cpu) {
2055
		struct desc_ptr *gdt = this_cpu_ptr(&host_gdt);
2056 2057
		unsigned long sysenter_esp;

2058
		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2059
		local_irq_disable();
2060
		crash_disable_local_vmclear(cpu);
2061 2062 2063 2064 2065 2066 2067 2068

		/*
		 * Read loaded_vmcs->cpu should be before fetching
		 * loaded_vmcs->loaded_vmcss_on_cpu_link.
		 * See the comments in __loaded_vmcs_clear().
		 */
		smp_rmb();

2069 2070
		list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link,
			 &per_cpu(loaded_vmcss_on_cpu, cpu));
2071
		crash_enable_local_vmclear(cpu);
2072 2073
		local_irq_enable();

2074 2075 2076 2077
		/*
		 * Linux uses per-cpu TSS and GDT, so set these when switching
		 * processors.
		 */
2078
		vmcs_writel(HOST_TR_BASE, kvm_read_tr_base()); /* 22.2.4 */
2079
		vmcs_writel(HOST_GDTR_BASE, gdt->address);   /* 22.2.4 */
2080 2081 2082

		rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
		vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
2083

2084
		vmx->loaded_vmcs->cpu = cpu;
2085
	}
2086

2087 2088 2089 2090 2091 2092 2093
	/* Setup TSC multiplier */
	if (kvm_has_tsc_control &&
	    vmx->current_tsc_ratio != vcpu->arch.tsc_scaling_ratio) {
		vmx->current_tsc_ratio = vcpu->arch.tsc_scaling_ratio;
		vmcs_write64(TSC_MULTIPLIER, vmx->current_tsc_ratio);
	}

2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107
	vmx_vcpu_pi_load(vcpu, cpu);
}

static void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu)
{
	struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);

	if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
		!irq_remapping_cap(IRQ_POSTING_CAP))
		return;

	/* Set SN when the vCPU is preempted */
	if (vcpu->preempted)
		pi_set_sn(pi_desc);
2108 2109 2110 2111
}

static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
{
2112 2113
	vmx_vcpu_pi_put(vcpu);

2114
	__vmx_load_host_state(to_vmx(vcpu));
2115
	if (!vmm_exclusive) {
2116 2117
		__loaded_vmcs_clear(to_vmx(vcpu)->loaded_vmcs);
		vcpu->cpu = -1;
2118 2119
		kvm_cpu_vmxoff();
	}
2120 2121
}

2122 2123
static void vmx_fpu_activate(struct kvm_vcpu *vcpu)
{
2124 2125
	ulong cr0;

2126 2127 2128
	if (vcpu->fpu_active)
		return;
	vcpu->fpu_active = 1;
2129 2130 2131 2132
	cr0 = vmcs_readl(GUEST_CR0);
	cr0 &= ~(X86_CR0_TS | X86_CR0_MP);
	cr0 |= kvm_read_cr0_bits(vcpu, X86_CR0_TS | X86_CR0_MP);
	vmcs_writel(GUEST_CR0, cr0);
2133
	update_exception_bitmap(vcpu);
2134
	vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
2135 2136 2137
	if (is_guest_mode(vcpu))
		vcpu->arch.cr0_guest_owned_bits &=
			~get_vmcs12(vcpu)->cr0_guest_host_mask;
2138
	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
2139 2140
}

2141 2142
static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu);

2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158
/*
 * Return the cr0 value that a nested guest would read. This is a combination
 * of the real cr0 used to run the guest (guest_cr0), and the bits shadowed by
 * its hypervisor (cr0_read_shadow).
 */
static inline unsigned long nested_read_cr0(struct vmcs12 *fields)
{
	return (fields->guest_cr0 & ~fields->cr0_guest_host_mask) |
		(fields->cr0_read_shadow & fields->cr0_guest_host_mask);
}
static inline unsigned long nested_read_cr4(struct vmcs12 *fields)
{
	return (fields->guest_cr4 & ~fields->cr4_guest_host_mask) |
		(fields->cr4_read_shadow & fields->cr4_guest_host_mask);
}

2159 2160
static void vmx_fpu_deactivate(struct kvm_vcpu *vcpu)
{
2161 2162 2163
	/* Note that there is no vcpu->fpu_active = 0 here. The caller must
	 * set this *before* calling this function.
	 */
2164
	vmx_decache_cr0_guest_bits(vcpu);
2165
	vmcs_set_bits(GUEST_CR0, X86_CR0_TS | X86_CR0_MP);
2166
	update_exception_bitmap(vcpu);
2167 2168
	vcpu->arch.cr0_guest_owned_bits = 0;
	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183
	if (is_guest_mode(vcpu)) {
		/*
		 * L1's specified read shadow might not contain the TS bit,
		 * so now that we turned on shadowing of this bit, we need to
		 * set this bit of the shadow. Like in nested_vmx_run we need
		 * nested_read_cr0(vmcs12), but vmcs12->guest_cr0 is not yet
		 * up-to-date here because we just decached cr0.TS (and we'll
		 * only update vmcs12->guest_cr0 on nested exit).
		 */
		struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
		vmcs12->guest_cr0 = (vmcs12->guest_cr0 & ~X86_CR0_TS) |
			(vcpu->arch.cr0 & X86_CR0_TS);
		vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12));
	} else
		vmcs_writel(CR0_READ_SHADOW, vcpu->arch.cr0);
2184 2185
}

2186 2187
static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
{
2188
	unsigned long rflags, save_rflags;
2189

2190 2191 2192 2193 2194 2195 2196 2197 2198
	if (!test_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail)) {
		__set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
		rflags = vmcs_readl(GUEST_RFLAGS);
		if (to_vmx(vcpu)->rmode.vm86_active) {
			rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
			save_rflags = to_vmx(vcpu)->rmode.save_rflags;
			rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
		}
		to_vmx(vcpu)->rflags = rflags;
2199
	}
2200
	return to_vmx(vcpu)->rflags;
2201 2202 2203 2204
}

static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
2205 2206
	__set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
	to_vmx(vcpu)->rflags = rflags;
2207 2208
	if (to_vmx(vcpu)->rmode.vm86_active) {
		to_vmx(vcpu)->rmode.save_rflags = rflags;
2209
		rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
2210
	}
2211 2212 2213
	vmcs_writel(GUEST_RFLAGS, rflags);
}

2214
static u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu)
2215 2216 2217 2218 2219
{
	u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
	int ret = 0;

	if (interruptibility & GUEST_INTR_STATE_STI)
2220
		ret |= KVM_X86_SHADOW_INT_STI;
2221
	if (interruptibility & GUEST_INTR_STATE_MOV_SS)
2222
		ret |= KVM_X86_SHADOW_INT_MOV_SS;
2223

2224
	return ret;
2225 2226 2227 2228 2229 2230 2231 2232 2233
}

static void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
{
	u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
	u32 interruptibility = interruptibility_old;

	interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);

2234
	if (mask & KVM_X86_SHADOW_INT_MOV_SS)
2235
		interruptibility |= GUEST_INTR_STATE_MOV_SS;
2236
	else if (mask & KVM_X86_SHADOW_INT_STI)
2237 2238 2239 2240 2241 2242
		interruptibility |= GUEST_INTR_STATE_STI;

	if ((interruptibility != interruptibility_old))
		vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
}

2243 2244 2245 2246
static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
	unsigned long rip;

2247
	rip = kvm_rip_read(vcpu);
2248
	rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
2249
	kvm_rip_write(vcpu, rip);
2250

2251 2252
	/* skipping an emulated instruction also counts */
	vmx_set_interrupt_shadow(vcpu, 0);
2253 2254
}

2255 2256 2257 2258
/*
 * KVM wants to inject page-faults which it got to the guest. This function
 * checks whether in a nested guest, we need to inject them to L1 or L2.
 */
2259
static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned nr)
2260 2261 2262
{
	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);

2263
	if (!(vmcs12->exception_bitmap & (1u << nr)))
2264 2265
		return 0;

2266
	nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
2267 2268
			  vmcs_read32(VM_EXIT_INTR_INFO),
			  vmcs_readl(EXIT_QUALIFICATION));
2269 2270 2271
	return 1;
}

2272
static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
2273 2274
				bool has_error_code, u32 error_code,
				bool reinject)
2275
{
2276
	struct vcpu_vmx *vmx = to_vmx(vcpu);
2277
	u32 intr_info = nr | INTR_INFO_VALID_MASK;
2278

2279 2280
	if (!reinject && is_guest_mode(vcpu) &&
	    nested_vmx_check_exception(vcpu, nr))
2281 2282
		return;

2283
	if (has_error_code) {
2284
		vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
2285 2286
		intr_info |= INTR_INFO_DELIVER_CODE_MASK;
	}
2287

2288
	if (vmx->rmode.vm86_active) {
2289 2290 2291 2292
		int inc_eip = 0;
		if (kvm_exception_is_soft(nr))
			inc_eip = vcpu->arch.event_exit_inst_len;
		if (kvm_inject_realmode_interrupt(vcpu, nr, inc_eip) != EMULATE_DONE)
2293
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2294 2295 2296
		return;
	}

2297 2298 2299
	if (kvm_exception_is_soft(nr)) {
		vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
			     vmx->vcpu.arch.event_exit_inst_len);
2300 2301 2302 2303 2304
		intr_info |= INTR_TYPE_SOFT_EXCEPTION;
	} else
		intr_info |= INTR_TYPE_HARD_EXCEPTION;

	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
2305 2306
}

2307 2308 2309 2310 2311
static bool vmx_rdtscp_supported(void)
{
	return cpu_has_vmx_rdtscp();
}

2312 2313 2314 2315 2316
static bool vmx_invpcid_supported(void)
{
	return cpu_has_vmx_invpcid() && enable_ept;
}

2317 2318 2319
/*
 * Swap MSR entry in host/guest MSR entry array.
 */
2320
static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
2321
{
2322
	struct shared_msr_entry tmp;
2323 2324 2325 2326

	tmp = vmx->guest_msrs[to];
	vmx->guest_msrs[to] = vmx->guest_msrs[from];
	vmx->guest_msrs[from] = tmp;
2327 2328
}

2329 2330 2331 2332
static void vmx_set_msr_bitmap(struct kvm_vcpu *vcpu)
{
	unsigned long *msr_bitmap;

2333 2334
	if (is_guest_mode(vcpu))
		msr_bitmap = vmx_msr_bitmap_nested;
2335
	else if (vcpu->arch.apic_base & X2APIC_ENABLE) {
2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349
		if (is_long_mode(vcpu))
			msr_bitmap = vmx_msr_bitmap_longmode_x2apic;
		else
			msr_bitmap = vmx_msr_bitmap_legacy_x2apic;
	} else {
		if (is_long_mode(vcpu))
			msr_bitmap = vmx_msr_bitmap_longmode;
		else
			msr_bitmap = vmx_msr_bitmap_legacy;
	}

	vmcs_write64(MSR_BITMAP, __pa(msr_bitmap));
}

2350 2351 2352 2353 2354
/*
 * Set up the vmcs to automatically save and restore system
 * msrs.  Don't touch the 64-bit msrs if the guest is in legacy
 * mode, as fiddling with msrs is very expensive.
 */
2355
static void setup_msrs(struct vcpu_vmx *vmx)
2356
{
2357
	int save_nmsrs, index;
2358

2359 2360
	save_nmsrs = 0;
#ifdef CONFIG_X86_64
2361 2362
	if (is_long_mode(&vmx->vcpu)) {
		index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
2363
		if (index >= 0)
2364 2365
			move_msr_up(vmx, index, save_nmsrs++);
		index = __find_msr_index(vmx, MSR_LSTAR);
2366
		if (index >= 0)
2367 2368
			move_msr_up(vmx, index, save_nmsrs++);
		index = __find_msr_index(vmx, MSR_CSTAR);
2369
		if (index >= 0)
2370
			move_msr_up(vmx, index, save_nmsrs++);
2371
		index = __find_msr_index(vmx, MSR_TSC_AUX);
2372
		if (index >= 0 && guest_cpuid_has_rdtscp(&vmx->vcpu))
2373
			move_msr_up(vmx, index, save_nmsrs++);
2374
		/*
Brian Gerst's avatar
Brian Gerst committed
2375
		 * MSR_STAR is only needed on long mode guests, and only
2376 2377
		 * if efer.sce is enabled.
		 */
Brian Gerst's avatar
Brian Gerst committed
2378
		index = __find_msr_index(vmx, MSR_STAR);
2379
		if ((index >= 0) && (vmx->vcpu.arch.efer & EFER_SCE))
2380
			move_msr_up(vmx, index, save_nmsrs++);
2381 2382
	}
#endif
2383 2384
	index = __find_msr_index(vmx, MSR_EFER);
	if (index >= 0 && update_transition_efer(vmx, index))
2385
		move_msr_up(vmx, index, save_nmsrs++);
2386

2387
	vmx->save_nmsrs = save_nmsrs;
2388

2389 2390
	if (cpu_has_vmx_msr_bitmap())
		vmx_set_msr_bitmap(&vmx->vcpu);
2391 2392
}

2393 2394
/*
 * reads and returns guest's timestamp counter "register"
2395 2396
 * guest_tsc = (host_tsc * tsc multiplier) >> 48 + tsc_offset
 * -- Intel TSC Scaling for Virtualization White Paper, sec 1.3
2397
 */
2398
static u64 guest_read_tsc(struct kvm_vcpu *vcpu)
2399 2400 2401
{
	u64 host_tsc, tsc_offset;

2402
	host_tsc = rdtsc();
2403
	tsc_offset = vmcs_read64(TSC_OFFSET);
2404
	return kvm_scale_tsc(vcpu, host_tsc) + tsc_offset;
2405 2406
}

Nadav Har'El's avatar
Nadav Har'El committed
2407 2408 2409 2410
/*
 * Like guest_read_tsc, but always returns L1's notion of the timestamp
 * counter, even if a nested guest (L2) is currently running.
 */
2411
static u64 vmx_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
Nadav Har'El's avatar
Nadav Har'El committed
2412
{
2413
	u64 tsc_offset;
Nadav Har'El's avatar
Nadav Har'El committed
2414 2415 2416 2417 2418 2419 2420

	tsc_offset = is_guest_mode(vcpu) ?
		to_vmx(vcpu)->nested.vmcs01_tsc_offset :
		vmcs_read64(TSC_OFFSET);
	return host_tsc + tsc_offset;
}

2421 2422 2423 2424 2425
static u64 vmx_read_tsc_offset(struct kvm_vcpu *vcpu)
{
	return vmcs_read64(TSC_OFFSET);
}

2426
/*
2427
 * writes 'offset' into guest's timestamp counter offset register
2428
 */
2429
static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
2430
{
2431
	if (is_guest_mode(vcpu)) {
2432
		/*
2433 2434 2435 2436
		 * We're here if L1 chose not to trap WRMSR to TSC. According
		 * to the spec, this should set L1's TSC; The offset that L1
		 * set for L2 remains unchanged, and still needs to be added
		 * to the newly set TSC to get L2's TSC.
2437
		 */
2438 2439 2440 2441 2442 2443 2444 2445
		struct vmcs12 *vmcs12;
		to_vmx(vcpu)->nested.vmcs01_tsc_offset = offset;
		/* recalculate vmcs02.TSC_OFFSET: */
		vmcs12 = get_vmcs12(vcpu);
		vmcs_write64(TSC_OFFSET, offset +
			(nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETING) ?
			 vmcs12->tsc_offset : 0));
	} else {
2446 2447
		trace_kvm_write_tsc_offset(vcpu->vcpu_id,
					   vmcs_read64(TSC_OFFSET), offset);
2448 2449
		vmcs_write64(TSC_OFFSET, offset);
	}
2450 2451
}

2452
static void vmx_adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, s64 adjustment)
Zachary Amsden's avatar
Zachary Amsden committed
2453 2454
{
	u64 offset = vmcs_read64(TSC_OFFSET);
2455

Zachary Amsden's avatar
Zachary Amsden committed
2456
	vmcs_write64(TSC_OFFSET, offset + adjustment);
2457 2458 2459
	if (is_guest_mode(vcpu)) {
		/* Even when running L2, the adjustment needs to apply to L1 */
		to_vmx(vcpu)->nested.vmcs01_tsc_offset += adjustment;
2460 2461 2462
	} else
		trace_kvm_write_tsc_offset(vcpu->vcpu_id, offset,
					   offset + adjustment);
Zachary Amsden's avatar
Zachary Amsden committed
2463 2464
}

2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481
static bool guest_cpuid_has_vmx(struct kvm_vcpu *vcpu)
{
	struct kvm_cpuid_entry2 *best = kvm_find_cpuid_entry(vcpu, 1, 0);
	return best && (best->ecx & (1 << (X86_FEATURE_VMX & 31)));
}

/*
 * nested_vmx_allowed() checks whether a guest should be allowed to use VMX
 * instructions and MSRs (i.e., nested VMX). Nested VMX is disabled for
 * all guests if the "nested" module option is off, and can also be disabled
 * for a single guest by disabling its VMX cpuid bit.
 */
static inline bool nested_vmx_allowed(struct kvm_vcpu *vcpu)
{
	return nested && guest_cpuid_has_vmx(vcpu);
}

2482 2483 2484 2485 2486 2487 2488 2489 2490 2491
/*
 * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be
 * returned for the various VMX controls MSRs when nested VMX is enabled.
 * The same values should also be used to verify that vmcs12 control fields are
 * valid during nested entry from L1 to L2.
 * Each of these control msrs has a low and high 32-bit half: A low bit is on
 * if the corresponding bit in the (32-bit) control field *must* be on, and a
 * bit in the high half is on if the corresponding bit in the control field
 * may be on. See also vmx_control_verify().
 */
2492
static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx)
2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509
{
	/*
	 * Note that as a general rule, the high half of the MSRs (bits in
	 * the control fields which may be 1) should be initialized by the
	 * intersection of the underlying hardware's MSR (i.e., features which
	 * can be supported) and the list of features we want to expose -
	 * because they are known to be properly supported in our code.
	 * Also, usually, the low half of the MSRs (bits which must be 1) can
	 * be set to 0, meaning that L1 may turn off any of these bits. The
	 * reason is that if one of these bits is necessary, it will appear
	 * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control
	 * fields of vmcs01 and vmcs02, will turn these bits off - and
	 * nested_vmx_exit_handled() will not pass related exits to L1.
	 * These rules have exceptions below.
	 */

	/* pin-based controls */
2510
	rdmsr(MSR_IA32_VMX_PINBASED_CTLS,
2511 2512 2513 2514 2515 2516 2517 2518 2519 2520
		vmx->nested.nested_vmx_pinbased_ctls_low,
		vmx->nested.nested_vmx_pinbased_ctls_high);
	vmx->nested.nested_vmx_pinbased_ctls_low |=
		PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
	vmx->nested.nested_vmx_pinbased_ctls_high &=
		PIN_BASED_EXT_INTR_MASK |
		PIN_BASED_NMI_EXITING |
		PIN_BASED_VIRTUAL_NMIS;
	vmx->nested.nested_vmx_pinbased_ctls_high |=
		PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
2521
		PIN_BASED_VMX_PREEMPTION_TIMER;
2522
	if (vmx_cpu_uses_apicv(&vmx->vcpu))
2523 2524
		vmx->nested.nested_vmx_pinbased_ctls_high |=
			PIN_BASED_POSTED_INTR;
2525

2526
	/* exit controls */
2527
	rdmsr(MSR_IA32_VMX_EXIT_CTLS,
2528 2529 2530 2531
		vmx->nested.nested_vmx_exit_ctls_low,
		vmx->nested.nested_vmx_exit_ctls_high);
	vmx->nested.nested_vmx_exit_ctls_low =
		VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
2532

2533
	vmx->nested.nested_vmx_exit_ctls_high &=
2534
#ifdef CONFIG_X86_64
2535
		VM_EXIT_HOST_ADDR_SPACE_SIZE |
2536
#endif
2537
		VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT;
2538 2539
	vmx->nested.nested_vmx_exit_ctls_high |=
		VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |
2540
		VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER |
2541 2542
		VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT;

2543
	if (kvm_mpx_supported())
2544
		vmx->nested.nested_vmx_exit_ctls_high |= VM_EXIT_CLEAR_BNDCFGS;
2545

2546
	/* We support free control of debug control saving. */
2547 2548
	vmx->nested.nested_vmx_true_exit_ctls_low =
		vmx->nested.nested_vmx_exit_ctls_low &
2549 2550
		~VM_EXIT_SAVE_DEBUG_CONTROLS;

2551 2552
	/* entry controls */
	rdmsr(MSR_IA32_VMX_ENTRY_CTLS,
2553 2554 2555 2556 2557
		vmx->nested.nested_vmx_entry_ctls_low,
		vmx->nested.nested_vmx_entry_ctls_high);
	vmx->nested.nested_vmx_entry_ctls_low =
		VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
	vmx->nested.nested_vmx_entry_ctls_high &=
2558 2559 2560 2561
#ifdef CONFIG_X86_64
		VM_ENTRY_IA32E_MODE |
#endif
		VM_ENTRY_LOAD_IA32_PAT;
2562 2563
	vmx->nested.nested_vmx_entry_ctls_high |=
		(VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER);
2564
	if (kvm_mpx_supported())
2565
		vmx->nested.nested_vmx_entry_ctls_high |= VM_ENTRY_LOAD_BNDCFGS;
2566

2567
	/* We support free control of debug control loading. */
2568 2569
	vmx->nested.nested_vmx_true_entry_ctls_low =
		vmx->nested.nested_vmx_entry_ctls_low &
2570 2571
		~VM_ENTRY_LOAD_DEBUG_CONTROLS;

2572 2573
	/* cpu-based controls */
	rdmsr(MSR_IA32_VMX_PROCBASED_CTLS,
2574 2575 2576 2577 2578
		vmx->nested.nested_vmx_procbased_ctls_low,
		vmx->nested.nested_vmx_procbased_ctls_high);
	vmx->nested.nested_vmx_procbased_ctls_low =
		CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
	vmx->nested.nested_vmx_procbased_ctls_high &=
2579 2580
		CPU_BASED_VIRTUAL_INTR_PENDING |
		CPU_BASED_VIRTUAL_NMI_PENDING | CPU_BASED_USE_TSC_OFFSETING |
2581 2582 2583 2584 2585 2586 2587
		CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING |
		CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING |
		CPU_BASED_CR3_STORE_EXITING |
#ifdef CONFIG_X86_64
		CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING |
#endif
		CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING |
2588 2589 2590 2591
		CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG |
		CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING |
		CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING |
		CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
2592 2593 2594 2595 2596 2597
	/*
	 * We can allow some features even when not supported by the
	 * hardware. For example, L1 can specify an MSR bitmap - and we
	 * can use it to avoid exits to L1 - even when L0 runs L2
	 * without MSR bitmaps.
	 */
2598 2599
	vmx->nested.nested_vmx_procbased_ctls_high |=
		CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
2600
		CPU_BASED_USE_MSR_BITMAPS;
2601

2602
	/* We support free control of CR3 access interception. */
2603 2604
	vmx->nested.nested_vmx_true_procbased_ctls_low =
		vmx->nested.nested_vmx_procbased_ctls_low &
2605 2606
		~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING);

2607 2608
	/* secondary cpu-based controls */
	rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2,
2609 2610 2611 2612
		vmx->nested.nested_vmx_secondary_ctls_low,
		vmx->nested.nested_vmx_secondary_ctls_high);
	vmx->nested.nested_vmx_secondary_ctls_low = 0;
	vmx->nested.nested_vmx_secondary_ctls_high &=
2613
		SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
2614
		SECONDARY_EXEC_RDTSCP |
2615
		SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2616
		SECONDARY_EXEC_ENABLE_VPID |
2617
		SECONDARY_EXEC_APIC_REGISTER_VIRT |
2618
		SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
2619
		SECONDARY_EXEC_WBINVD_EXITING |
2620 2621
		SECONDARY_EXEC_XSAVES |
		SECONDARY_EXEC_PCOMMIT;
2622

2623 2624
	if (enable_ept) {
		/* nested EPT: emulate EPT also to L1 */
2625
		vmx->nested.nested_vmx_secondary_ctls_high |=
2626
			SECONDARY_EXEC_ENABLE_EPT;
2627
		vmx->nested.nested_vmx_ept_caps = VMX_EPT_PAGE_WALK_4_BIT |
2628 2629
			 VMX_EPTP_WB_BIT | VMX_EPT_2MB_PAGE_BIT |
			 VMX_EPT_INVEPT_BIT;
2630
		vmx->nested.nested_vmx_ept_caps &= vmx_capability.ept;
2631
		/*
2632 2633 2634
		 * For nested guests, we don't do anything specific
		 * for single context invalidation. Hence, only advertise
		 * support for global context invalidation.
2635
		 */
2636
		vmx->nested.nested_vmx_ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT;
2637
	} else
2638
		vmx->nested.nested_vmx_ept_caps = 0;
2639

2640 2641 2642 2643 2644 2645
	/*
	 * Old versions of KVM use the single-context version without
	 * checking for support, so declare that it is supported even
	 * though it is treated as global context.  The alternative is
	 * not failing the single-context invvpid, and it is worse.
	 */
2646 2647
	if (enable_vpid)
		vmx->nested.nested_vmx_vpid_caps = VMX_VPID_INVVPID_BIT |
2648
				VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT |
2649 2650 2651
				VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT;
	else
		vmx->nested.nested_vmx_vpid_caps = 0;
2652

2653 2654 2655 2656
	if (enable_unrestricted_guest)
		vmx->nested.nested_vmx_secondary_ctls_high |=
			SECONDARY_EXEC_UNRESTRICTED_GUEST;

2657
	/* miscellaneous data */
2658 2659 2660 2661 2662 2663
	rdmsr(MSR_IA32_VMX_MISC,
		vmx->nested.nested_vmx_misc_low,
		vmx->nested.nested_vmx_misc_high);
	vmx->nested.nested_vmx_misc_low &= VMX_MISC_SAVE_EFER_LMA;
	vmx->nested.nested_vmx_misc_low |=
		VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE |
2664
		VMX_MISC_ACTIVITY_HLT;
2665
	vmx->nested.nested_vmx_misc_high = 0;
2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680
}

static inline bool vmx_control_verify(u32 control, u32 low, u32 high)
{
	/*
	 * Bits 0 in high must be 0, and bits 1 in low must be 1.
	 */
	return ((control & high) | low) == control;
}

static inline u64 vmx_control_msr(u32 low, u32 high)
{
	return low | ((u64)high << 32);
}

2681
/* Returns 0 on success, non-0 otherwise. */
2682 2683
static int vmx_get_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
{
2684 2685
	struct vcpu_vmx *vmx = to_vmx(vcpu);

2686 2687 2688 2689 2690 2691 2692 2693
	switch (msr_index) {
	case MSR_IA32_VMX_BASIC:
		/*
		 * This MSR reports some information about VMX support. We
		 * should return information about the VMX we emulate for the
		 * guest, and the VMCS structure we give it - not about the
		 * VMX support of the underlying hardware.
		 */
2694
		*pdata = VMCS12_REVISION | VMX_BASIC_TRUE_CTLS |
2695 2696 2697 2698 2699
			   ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) |
			   (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT);
		break;
	case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
	case MSR_IA32_VMX_PINBASED_CTLS:
2700 2701 2702
		*pdata = vmx_control_msr(
			vmx->nested.nested_vmx_pinbased_ctls_low,
			vmx->nested.nested_vmx_pinbased_ctls_high);
2703 2704
		break;
	case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
2705 2706 2707
		*pdata = vmx_control_msr(
			vmx->nested.nested_vmx_true_procbased_ctls_low,
			vmx->nested.nested_vmx_procbased_ctls_high);
2708
		break;
2709
	case MSR_IA32_VMX_PROCBASED_CTLS:
2710 2711 2712
		*pdata = vmx_control_msr(
			vmx->nested.nested_vmx_procbased_ctls_low,
			vmx->nested.nested_vmx_procbased_ctls_high);
2713 2714
		break;
	case MSR_IA32_VMX_TRUE_EXIT_CTLS:
2715 2716 2717
		*pdata = vmx_control_msr(
			vmx->nested.nested_vmx_true_exit_ctls_low,
			vmx->nested.nested_vmx_exit_ctls_high);
2718
		break;
2719
	case MSR_IA32_VMX_EXIT_CTLS:
2720 2721 2722
		*pdata = vmx_control_msr(
			vmx->nested.nested_vmx_exit_ctls_low,
			vmx->nested.nested_vmx_exit_ctls_high);
2723 2724
		break;
	case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
2725 2726 2727
		*pdata = vmx_control_msr(
			vmx->nested.nested_vmx_true_entry_ctls_low,
			vmx->nested.nested_vmx_entry_ctls_high);
2728
		break;
2729
	case MSR_IA32_VMX_ENTRY_CTLS:
2730 2731 2732
		*pdata = vmx_control_msr(
			vmx->nested.nested_vmx_entry_ctls_low,
			vmx->nested.nested_vmx_entry_ctls_high);
2733 2734
		break;
	case MSR_IA32_VMX_MISC:
2735 2736 2737
		*pdata = vmx_control_msr(
			vmx->nested.nested_vmx_misc_low,
			vmx->nested.nested_vmx_misc_high);
2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758
		break;
	/*
	 * These MSRs specify bits which the guest must keep fixed (on or off)
	 * while L1 is in VMXON mode (in L1's root mode, or running an L2).
	 * We picked the standard core2 setting.
	 */
#define VMXON_CR0_ALWAYSON	(X86_CR0_PE | X86_CR0_PG | X86_CR0_NE)
#define VMXON_CR4_ALWAYSON	X86_CR4_VMXE
	case MSR_IA32_VMX_CR0_FIXED0:
		*pdata = VMXON_CR0_ALWAYSON;
		break;
	case MSR_IA32_VMX_CR0_FIXED1:
		*pdata = -1ULL;
		break;
	case MSR_IA32_VMX_CR4_FIXED0:
		*pdata = VMXON_CR4_ALWAYSON;
		break;
	case MSR_IA32_VMX_CR4_FIXED1:
		*pdata = -1ULL;
		break;
	case MSR_IA32_VMX_VMCS_ENUM:
2759
		*pdata = 0x2e; /* highest index: VMX_PREEMPTION_TIMER_VALUE */
2760 2761
		break;
	case MSR_IA32_VMX_PROCBASED_CTLS2:
2762 2763 2764
		*pdata = vmx_control_msr(
			vmx->nested.nested_vmx_secondary_ctls_low,
			vmx->nested.nested_vmx_secondary_ctls_high);
2765 2766
		break;
	case MSR_IA32_VMX_EPT_VPID_CAP:
2767
		/* Currently, no nested vpid support */
2768 2769
		*pdata = vmx->nested.nested_vmx_ept_caps |
			((u64)vmx->nested.nested_vmx_vpid_caps << 32);
2770 2771 2772
		break;
	default:
		return 1;
2773 2774
	}

2775 2776 2777
	return 0;
}

2778 2779 2780 2781 2782
/*
 * Reads an msr value (of 'msr_index') into 'pdata'.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
2783
static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2784
{
2785
	struct shared_msr_entry *msr;
2786

2787
	switch (msr_info->index) {
2788
#ifdef CONFIG_X86_64
2789
	case MSR_FS_BASE:
2790
		msr_info->data = vmcs_readl(GUEST_FS_BASE);
2791 2792
		break;
	case MSR_GS_BASE:
2793
		msr_info->data = vmcs_readl(GUEST_GS_BASE);
2794
		break;
2795 2796
	case MSR_KERNEL_GS_BASE:
		vmx_load_host_state(to_vmx(vcpu));
2797
		msr_info->data = to_vmx(vcpu)->msr_guest_kernel_gs_base;
2798
		break;
2799
#endif
2800
	case MSR_EFER:
2801
		return kvm_get_msr_common(vcpu, msr_info);
2802
	case MSR_IA32_TSC:
2803
		msr_info->data = guest_read_tsc(vcpu);
2804 2805
		break;
	case MSR_IA32_SYSENTER_CS:
2806
		msr_info->data = vmcs_read32(GUEST_SYSENTER_CS);
2807 2808
		break;
	case MSR_IA32_SYSENTER_EIP:
2809
		msr_info->data = vmcs_readl(GUEST_SYSENTER_EIP);
2810 2811
		break;
	case MSR_IA32_SYSENTER_ESP:
2812
		msr_info->data = vmcs_readl(GUEST_SYSENTER_ESP);
2813
		break;
2814
	case MSR_IA32_BNDCFGS:
2815 2816
		if (!kvm_mpx_supported() ||
		    (!msr_info->host_initiated && !guest_cpuid_has_mpx(vcpu)))
2817
			return 1;
2818
		msr_info->data = vmcs_read64(GUEST_BNDCFGS);
2819
		break;
2820 2821 2822
	case MSR_IA32_FEATURE_CONTROL:
		if (!nested_vmx_allowed(vcpu))
			return 1;
2823
		msr_info->data = to_vmx(vcpu)->nested.msr_ia32_feature_control;
2824 2825 2826 2827
		break;
	case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
		if (!nested_vmx_allowed(vcpu))
			return 1;
2828
		return vmx_get_vmx_msr(vcpu, msr_info->index, &msr_info->data);
2829 2830 2831
	case MSR_IA32_XSS:
		if (!vmx_xsaves_supported())
			return 1;
2832
		msr_info->data = vcpu->arch.ia32_xss;
2833
		break;
2834
	case MSR_TSC_AUX:
2835
		if (!guest_cpuid_has_rdtscp(vcpu) && !msr_info->host_initiated)
2836 2837
			return 1;
		/* Otherwise falls through */
2838
	default:
2839
		msr = find_msr_entry(to_vmx(vcpu), msr_info->index);
2840
		if (msr) {
2841
			msr_info->data = msr->data;
2842
			break;
2843
		}
2844
		return kvm_get_msr_common(vcpu, msr_info);
2845 2846 2847 2848 2849
	}

	return 0;
}

2850 2851
static void vmx_leave_nested(struct kvm_vcpu *vcpu);

2852 2853 2854 2855 2856
/*
 * Writes msr value into into the appropriate "register".
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
2857
static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2858
{
2859
	struct vcpu_vmx *vmx = to_vmx(vcpu);
2860
	struct shared_msr_entry *msr;
2861
	int ret = 0;
2862 2863
	u32 msr_index = msr_info->index;
	u64 data = msr_info->data;
2864

2865
	switch (msr_index) {
2866
	case MSR_EFER:
2867
		ret = kvm_set_msr_common(vcpu, msr_info);
2868
		break;
2869
#ifdef CONFIG_X86_64
2870
	case MSR_FS_BASE:
2871
		vmx_segment_cache_clear(vmx);
2872 2873 2874
		vmcs_writel(GUEST_FS_BASE, data);
		break;
	case MSR_GS_BASE:
2875
		vmx_segment_cache_clear(vmx);
2876 2877
		vmcs_writel(GUEST_GS_BASE, data);
		break;
2878 2879 2880 2881
	case MSR_KERNEL_GS_BASE:
		vmx_load_host_state(vmx);
		vmx->msr_guest_kernel_gs_base = data;
		break;
2882 2883 2884 2885 2886
#endif
	case MSR_IA32_SYSENTER_CS:
		vmcs_write32(GUEST_SYSENTER_CS, data);
		break;
	case MSR_IA32_SYSENTER_EIP:
Avi Kivity's avatar
Avi Kivity committed
2887
		vmcs_writel(GUEST_SYSENTER_EIP, data);
2888 2889
		break;
	case MSR_IA32_SYSENTER_ESP:
Avi Kivity's avatar
Avi Kivity committed
2890
		vmcs_writel(GUEST_SYSENTER_ESP, data);
2891
		break;
2892
	case MSR_IA32_BNDCFGS:
2893 2894
		if (!kvm_mpx_supported() ||
		    (!msr_info->host_initiated && !guest_cpuid_has_mpx(vcpu)))
2895
			return 1;
2896 2897 2898
		if (is_noncanonical_address(data & PAGE_MASK) ||
		    (data & MSR_IA32_BNDCFGS_RSVD))
			return 1;
2899 2900
		vmcs_write64(GUEST_BNDCFGS, data);
		break;
2901
	case MSR_IA32_TSC:
2902
		kvm_write_tsc(vcpu, msr_info);
2903
		break;
2904 2905
	case MSR_IA32_CR_PAT:
		if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
2906 2907
			if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
				return 1;
2908 2909 2910 2911
			vmcs_write64(GUEST_IA32_PAT, data);
			vcpu->arch.pat = data;
			break;
		}
2912
		ret = kvm_set_msr_common(vcpu, msr_info);
2913
		break;
2914 2915
	case MSR_IA32_TSC_ADJUST:
		ret = kvm_set_msr_common(vcpu, msr_info);
2916
		break;
2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927
	case MSR_IA32_FEATURE_CONTROL:
		if (!nested_vmx_allowed(vcpu) ||
		    (to_vmx(vcpu)->nested.msr_ia32_feature_control &
		     FEATURE_CONTROL_LOCKED && !msr_info->host_initiated))
			return 1;
		vmx->nested.msr_ia32_feature_control = data;
		if (msr_info->host_initiated && data == 0)
			vmx_leave_nested(vcpu);
		break;
	case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
		return 1; /* they are read-only */
2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943
	case MSR_IA32_XSS:
		if (!vmx_xsaves_supported())
			return 1;
		/*
		 * The only supported bit as of Skylake is bit 8, but
		 * it is not supported on KVM.
		 */
		if (data != 0)
			return 1;
		vcpu->arch.ia32_xss = data;
		if (vcpu->arch.ia32_xss != host_xss)
			add_atomic_switch_msr(vmx, MSR_IA32_XSS,
				vcpu->arch.ia32_xss, host_xss);
		else
			clear_atomic_switch_msr(vmx, MSR_IA32_XSS);
		break;
2944
	case MSR_TSC_AUX:
2945
		if (!guest_cpuid_has_rdtscp(vcpu) && !msr_info->host_initiated)
2946 2947 2948 2949 2950
			return 1;
		/* Check reserved bit, higher 32 bits should be zero */
		if ((data >> 32) != 0)
			return 1;
		/* Otherwise falls through */
2951
	default:
2952
		msr = find_msr_entry(vmx, msr_index);
2953
		if (msr) {
2954
			u64 old_msr_data = msr->data;
2955
			msr->data = data;
2956 2957
			if (msr - vmx->guest_msrs < vmx->save_nmsrs) {
				preempt_disable();
2958 2959
				ret = kvm_set_shared_msr(msr->index, msr->data,
							 msr->mask);
2960
				preempt_enable();
2961 2962
				if (ret)
					msr->data = old_msr_data;
2963
			}
2964
			break;
2965
		}
2966
		ret = kvm_set_msr_common(vcpu, msr_info);
2967 2968
	}

2969
	return ret;
2970 2971
}

2972
static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
2973
{
2974 2975 2976 2977 2978 2979 2980 2981
	__set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
	switch (reg) {
	case VCPU_REGS_RSP:
		vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
		break;
	case VCPU_REGS_RIP:
		vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
		break;
Avi Kivity's avatar
Avi Kivity committed
2982 2983 2984 2985
	case VCPU_EXREG_PDPTR:
		if (enable_ept)
			ept_save_pdptrs(vcpu);
		break;
2986 2987 2988
	default:
		break;
	}
2989 2990 2991 2992
}

static __init int cpu_has_kvm_support(void)
{
2993
	return cpu_has_vmx();
2994 2995 2996 2997 2998 2999 3000
}

static __init int vmx_disabled_by_bios(void)
{
	u64 msr;

	rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
3001
	if (msr & FEATURE_CONTROL_LOCKED) {
3002
		/* launched w/ TXT and VMX disabled */
3003 3004 3005
		if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
			&& tboot_enabled())
			return 1;
3006
		/* launched w/o TXT and VMX only enabled w/ TXT */
3007
		if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
3008
			&& (msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
3009 3010
			&& !tboot_enabled()) {
			printk(KERN_WARNING "kvm: disable TXT in the BIOS or "
3011
				"activate TXT before enabling KVM\n");
3012
			return 1;
3013
		}
3014 3015 3016 3017
		/* launched w/o TXT and VMX disabled */
		if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
			&& !tboot_enabled())
			return 1;
3018 3019 3020
	}

	return 0;
3021 3022
}

3023 3024 3025 3026 3027 3028 3029
static void kvm_cpu_vmxon(u64 addr)
{
	asm volatile (ASM_VMX_VMXON_RAX
			: : "a"(&addr), "m"(addr)
			: "memory", "cc");
}

3030
static int hardware_enable(void)
3031 3032 3033
{
	int cpu = raw_smp_processor_id();
	u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
3034
	u64 old, test_bits;
3035

3036
	if (cr4_read_shadow() & X86_CR4_VMXE)
3037 3038
		return -EBUSY;

3039
	INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu));
3040 3041
	INIT_LIST_HEAD(&per_cpu(blocked_vcpu_on_cpu, cpu));
	spin_lock_init(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053

	/*
	 * Now we can enable the vmclear operation in kdump
	 * since the loaded_vmcss_on_cpu list on this cpu
	 * has been initialized.
	 *
	 * Though the cpu is not in VMX operation now, there
	 * is no problem to enable the vmclear operation
	 * for the loaded_vmcss_on_cpu list is empty!
	 */
	crash_enable_local_vmclear(cpu);

3054
	rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
3055 3056 3057 3058 3059 3060 3061

	test_bits = FEATURE_CONTROL_LOCKED;
	test_bits |= FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
	if (tboot_enabled())
		test_bits |= FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX;

	if ((old & test_bits) != test_bits) {
3062
		/* enable and lock */
3063 3064
		wrmsrl(MSR_IA32_FEATURE_CONTROL, old | test_bits);
	}
3065
	cr4_set_bits(X86_CR4_VMXE);
3066

3067 3068 3069 3070
	if (vmm_exclusive) {
		kvm_cpu_vmxon(phys_addr);
		ept_sync_global();
	}
3071

3072
	native_store_gdt(this_cpu_ptr(&host_gdt));
3073

3074
	return 0;
3075 3076
}

3077
static void vmclear_local_loaded_vmcss(void)
3078 3079
{
	int cpu = raw_smp_processor_id();
3080
	struct loaded_vmcs *v, *n;
3081

3082 3083 3084
	list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu),
				 loaded_vmcss_on_cpu_link)
		__loaded_vmcs_clear(v);
3085 3086
}

3087 3088 3089 3090 3091

/* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
 * tricks.
 */
static void kvm_cpu_vmxoff(void)
3092
{
3093
	asm volatile (__ex(ASM_VMX_VMXOFF) : : : "cc");
3094 3095
}

3096
static void hardware_disable(void)
3097
{
3098
	if (vmm_exclusive) {
3099
		vmclear_local_loaded_vmcss();
3100 3101
		kvm_cpu_vmxoff();
	}
3102
	cr4_clear_bits(X86_CR4_VMXE);
3103 3104
}

3105
static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
Mike Day's avatar
Mike Day committed
3106
				      u32 msr, u32 *result)
3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117
{
	u32 vmx_msr_low, vmx_msr_high;
	u32 ctl = ctl_min | ctl_opt;

	rdmsr(msr, vmx_msr_low, vmx_msr_high);

	ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
	ctl |= vmx_msr_low;  /* bit == 1 in low word  ==> must be one  */

	/* Ensure minimum (required) set of control bits are supported. */
	if (ctl_min & ~ctl)
3118
		return -EIO;
3119 3120 3121 3122 3123

	*result = ctl;
	return 0;
}

3124 3125 3126 3127 3128 3129 3130 3131
static __init bool allow_1_setting(u32 msr, u32 ctl)
{
	u32 vmx_msr_low, vmx_msr_high;

	rdmsr(msr, vmx_msr_low, vmx_msr_high);
	return vmx_msr_high & ctl;
}

3132
static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
3133 3134
{
	u32 vmx_msr_low, vmx_msr_high;
Sheng Yang's avatar
Sheng Yang committed
3135
	u32 min, opt, min2, opt2;
3136 3137
	u32 _pin_based_exec_control = 0;
	u32 _cpu_based_exec_control = 0;
3138
	u32 _cpu_based_2nd_exec_control = 0;
3139 3140 3141
	u32 _vmexit_control = 0;
	u32 _vmentry_control = 0;

3142
	min = CPU_BASED_HLT_EXITING |
3143 3144 3145 3146
#ifdef CONFIG_X86_64
	      CPU_BASED_CR8_LOAD_EXITING |
	      CPU_BASED_CR8_STORE_EXITING |
#endif
Sheng Yang's avatar
Sheng Yang committed
3147 3148
	      CPU_BASED_CR3_LOAD_EXITING |
	      CPU_BASED_CR3_STORE_EXITING |
3149 3150
	      CPU_BASED_USE_IO_BITMAPS |
	      CPU_BASED_MOV_DR_EXITING |
Marcelo Tosatti's avatar
Marcelo Tosatti committed
3151
	      CPU_BASED_USE_TSC_OFFSETING |
3152 3153
	      CPU_BASED_MWAIT_EXITING |
	      CPU_BASED_MONITOR_EXITING |
Avi Kivity's avatar
Avi Kivity committed
3154 3155
	      CPU_BASED_INVLPG_EXITING |
	      CPU_BASED_RDPMC_EXITING;
3156

3157
	opt = CPU_BASED_TPR_SHADOW |
3158
	      CPU_BASED_USE_MSR_BITMAPS |
3159
	      CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
3160 3161
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
				&_cpu_based_exec_control) < 0)
3162
		return -EIO;
3163 3164 3165 3166 3167
#ifdef CONFIG_X86_64
	if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
		_cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
					   ~CPU_BASED_CR8_STORE_EXITING;
#endif
3168
	if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
Sheng Yang's avatar
Sheng Yang committed
3169 3170
		min2 = 0;
		opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
3171
			SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
3172
			SECONDARY_EXEC_WBINVD_EXITING |
Sheng Yang's avatar
Sheng Yang committed
3173
			SECONDARY_EXEC_ENABLE_VPID |
3174
			SECONDARY_EXEC_ENABLE_EPT |
3175
			SECONDARY_EXEC_UNRESTRICTED_GUEST |
3176
			SECONDARY_EXEC_PAUSE_LOOP_EXITING |
3177
			SECONDARY_EXEC_RDTSCP |
3178
			SECONDARY_EXEC_ENABLE_INVPCID |
3179
			SECONDARY_EXEC_APIC_REGISTER_VIRT |
3180
			SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
3181
			SECONDARY_EXEC_SHADOW_VMCS |
Kai Huang's avatar
Kai Huang committed
3182
			SECONDARY_EXEC_XSAVES |
3183
			SECONDARY_EXEC_ENABLE_PML |
3184 3185
			SECONDARY_EXEC_PCOMMIT |
			SECONDARY_EXEC_TSC_SCALING;
Sheng Yang's avatar
Sheng Yang committed
3186 3187
		if (adjust_vmx_controls(min2, opt2,
					MSR_IA32_VMX_PROCBASED_CTLS2,
3188 3189 3190 3191 3192 3193 3194 3195
					&_cpu_based_2nd_exec_control) < 0)
			return -EIO;
	}
#ifndef CONFIG_X86_64
	if (!(_cpu_based_2nd_exec_control &
				SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
		_cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
#endif
3196 3197 3198

	if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
		_cpu_based_2nd_exec_control &= ~(
3199
				SECONDARY_EXEC_APIC_REGISTER_VIRT |
3200 3201
				SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
				SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
3202

Sheng Yang's avatar
Sheng Yang committed
3203
	if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
Marcelo Tosatti's avatar
Marcelo Tosatti committed
3204 3205
		/* CR3 accesses and invlpg don't need to cause VM Exits when EPT
		   enabled */
3206 3207 3208
		_cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
					     CPU_BASED_CR3_STORE_EXITING |
					     CPU_BASED_INVLPG_EXITING);
Sheng Yang's avatar
Sheng Yang committed
3209 3210 3211
		rdmsr(MSR_IA32_VMX_EPT_VPID_CAP,
		      vmx_capability.ept, vmx_capability.vpid);
	}
3212

3213
	min = VM_EXIT_SAVE_DEBUG_CONTROLS;
3214 3215 3216
#ifdef CONFIG_X86_64
	min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
#endif
3217
	opt = VM_EXIT_SAVE_IA32_PAT | VM_EXIT_LOAD_IA32_PAT |
3218
		VM_EXIT_ACK_INTR_ON_EXIT | VM_EXIT_CLEAR_BNDCFGS;
3219 3220
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
				&_vmexit_control) < 0)
3221
		return -EIO;
3222

3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233
	min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
	opt = PIN_BASED_VIRTUAL_NMIS | PIN_BASED_POSTED_INTR;
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
				&_pin_based_exec_control) < 0)
		return -EIO;

	if (!(_cpu_based_2nd_exec_control &
		SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) ||
		!(_vmexit_control & VM_EXIT_ACK_INTR_ON_EXIT))
		_pin_based_exec_control &= ~PIN_BASED_POSTED_INTR;

3234
	min = VM_ENTRY_LOAD_DEBUG_CONTROLS;
3235
	opt = VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_BNDCFGS;
3236 3237
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
				&_vmentry_control) < 0)
3238
		return -EIO;
3239

3240
	rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
3241 3242 3243

	/* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
	if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
3244
		return -EIO;
3245 3246 3247 3248

#ifdef CONFIG_X86_64
	/* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
	if (vmx_msr_high & (1u<<16))
3249
		return -EIO;
3250 3251 3252 3253
#endif

	/* Require Write-Back (WB) memory type for VMCS accesses. */
	if (((vmx_msr_high >> 18) & 15) != 6)
3254
		return -EIO;
3255

3256 3257 3258
	vmcs_conf->size = vmx_msr_high & 0x1fff;
	vmcs_conf->order = get_order(vmcs_config.size);
	vmcs_conf->revision_id = vmx_msr_low;
3259

3260 3261
	vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
	vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
3262
	vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
3263 3264
	vmcs_conf->vmexit_ctrl         = _vmexit_control;
	vmcs_conf->vmentry_ctrl        = _vmentry_control;
3265

3266 3267 3268 3269 3270 3271
	cpu_has_load_ia32_efer =
		allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
				VM_ENTRY_LOAD_IA32_EFER)
		&& allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
				   VM_EXIT_LOAD_IA32_EFER);

3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307
	cpu_has_load_perf_global_ctrl =
		allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
				VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
		&& allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
				   VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);

	/*
	 * Some cpus support VM_ENTRY_(LOAD|SAVE)_IA32_PERF_GLOBAL_CTRL
	 * but due to arrata below it can't be used. Workaround is to use
	 * msr load mechanism to switch IA32_PERF_GLOBAL_CTRL.
	 *
	 * VM Exit May Incorrectly Clear IA32_PERF_GLOBAL_CTRL [34:32]
	 *
	 * AAK155             (model 26)
	 * AAP115             (model 30)
	 * AAT100             (model 37)
	 * BC86,AAY89,BD102   (model 44)
	 * BA97               (model 46)
	 *
	 */
	if (cpu_has_load_perf_global_ctrl && boot_cpu_data.x86 == 0x6) {
		switch (boot_cpu_data.x86_model) {
		case 26:
		case 30:
		case 37:
		case 44:
		case 46:
			cpu_has_load_perf_global_ctrl = false;
			printk_once(KERN_WARNING"kvm: VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL "
					"does not work properly. Using workaround\n");
			break;
		default:
			break;
		}
	}

3308 3309 3310
	if (cpu_has_xsaves)
		rdmsrl(MSR_IA32_XSS, host_xss);

3311
	return 0;
3312
}
3313 3314 3315 3316 3317 3318 3319

static struct vmcs *alloc_vmcs_cpu(int cpu)
{
	int node = cpu_to_node(cpu);
	struct page *pages;
	struct vmcs *vmcs;

3320
	pages = __alloc_pages_node(node, GFP_KERNEL, vmcs_config.order);
3321 3322 3323
	if (!pages)
		return NULL;
	vmcs = page_address(pages);
3324 3325
	memset(vmcs, 0, vmcs_config.size);
	vmcs->revision_id = vmcs_config.revision_id; /* vmcs revision id */
3326 3327 3328 3329 3330
	return vmcs;
}

static struct vmcs *alloc_vmcs(void)
{
3331
	return alloc_vmcs_cpu(raw_smp_processor_id());
3332 3333 3334 3335
}

static void free_vmcs(struct vmcs *vmcs)
{
3336
	free_pages((unsigned long)vmcs, vmcs_config.order);
3337 3338
}

3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350
/*
 * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded
 */
static void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
{
	if (!loaded_vmcs->vmcs)
		return;
	loaded_vmcs_clear(loaded_vmcs);
	free_vmcs(loaded_vmcs->vmcs);
	loaded_vmcs->vmcs = NULL;
}

3351
static void free_kvm_area(void)
3352 3353 3354
{
	int cpu;

Zachary Amsden's avatar
Zachary Amsden committed
3355
	for_each_possible_cpu(cpu) {
3356
		free_vmcs(per_cpu(vmxarea, cpu));
Zachary Amsden's avatar
Zachary Amsden committed
3357 3358
		per_cpu(vmxarea, cpu) = NULL;
	}
3359 3360
}

3361 3362 3363 3364 3365 3366 3367 3368 3369
static void init_vmcs_shadow_fields(void)
{
	int i, j;

	/* No checks for read only fields yet */

	for (i = j = 0; i < max_shadow_read_write_fields; i++) {
		switch (shadow_read_write_fields[i]) {
		case GUEST_BNDCFGS:
3370
			if (!kvm_mpx_supported())
3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395
				continue;
			break;
		default:
			break;
		}

		if (j < i)
			shadow_read_write_fields[j] =
				shadow_read_write_fields[i];
		j++;
	}
	max_shadow_read_write_fields = j;

	/* shadowed fields guest access without vmexit */
	for (i = 0; i < max_shadow_read_write_fields; i++) {
		clear_bit(shadow_read_write_fields[i],
			  vmx_vmwrite_bitmap);
		clear_bit(shadow_read_write_fields[i],
			  vmx_vmread_bitmap);
	}
	for (i = 0; i < max_shadow_read_only_fields; i++)
		clear_bit(shadow_read_only_fields[i],
			  vmx_vmread_bitmap);
}

3396 3397 3398 3399
static __init int alloc_kvm_area(void)
{
	int cpu;

Zachary Amsden's avatar
Zachary Amsden committed
3400
	for_each_possible_cpu(cpu) {
3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413
		struct vmcs *vmcs;

		vmcs = alloc_vmcs_cpu(cpu);
		if (!vmcs) {
			free_kvm_area();
			return -ENOMEM;
		}

		per_cpu(vmxarea, cpu) = vmcs;
	}
	return 0;
}

3414 3415 3416 3417 3418
static bool emulation_required(struct kvm_vcpu *vcpu)
{
	return emulate_invalid_guest_state && !guest_state_valid(vcpu);
}

3419
static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg,
3420
		struct kvm_segment *save)
3421
{
3422 3423 3424 3425 3426 3427 3428 3429 3430
	if (!emulate_invalid_guest_state) {
		/*
		 * CS and SS RPL should be equal during guest entry according
		 * to VMX spec, but in reality it is not always so. Since vcpu
		 * is in the middle of the transition from real mode to
		 * protected mode it is safe to assume that RPL 0 is a good
		 * default value.
		 */
		if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS)
3431 3432
			save->selector &= ~SEGMENT_RPL_MASK;
		save->dpl = save->selector & SEGMENT_RPL_MASK;
3433
		save->s = 1;
3434
	}
3435
	vmx_set_segment(vcpu, save, seg);
3436 3437 3438 3439 3440
}

static void enter_pmode(struct kvm_vcpu *vcpu)
{
	unsigned long flags;
3441
	struct vcpu_vmx *vmx = to_vmx(vcpu);
3442

3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453
	/*
	 * Update real mode segment cache. It may be not up-to-date if sement
	 * register was written while vcpu was in a guest mode.
	 */
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);

3454
	vmx->rmode.vm86_active = 0;
3455

3456 3457
	vmx_segment_cache_clear(vmx);

3458
	vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
3459 3460

	flags = vmcs_readl(GUEST_RFLAGS);
3461 3462
	flags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
	flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
3463 3464
	vmcs_writel(GUEST_RFLAGS, flags);

3465 3466
	vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
			(vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
3467 3468 3469

	update_exception_bitmap(vcpu);

3470 3471 3472 3473 3474 3475
	fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
	fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
	fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
	fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
	fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
	fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
3476 3477
}

3478
static void fix_rmode_seg(int seg, struct kvm_segment *save)
3479
{
3480
	const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503
	struct kvm_segment var = *save;

	var.dpl = 0x3;
	if (seg == VCPU_SREG_CS)
		var.type = 0x3;

	if (!emulate_invalid_guest_state) {
		var.selector = var.base >> 4;
		var.base = var.base & 0xffff0;
		var.limit = 0xffff;
		var.g = 0;
		var.db = 0;
		var.present = 1;
		var.s = 1;
		var.l = 0;
		var.unusable = 0;
		var.type = 0x3;
		var.avl = 0;
		if (save->base & 0xf)
			printk_once(KERN_WARNING "kvm: segment base is not "
					"paragraph aligned when entering "
					"protected mode (seg=%d)", seg);
	}
3504

3505
	vmcs_write16(sf->selector, var.selector);
3506
	vmcs_writel(sf->base, var.base);
3507 3508
	vmcs_write32(sf->limit, var.limit);
	vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var));
3509 3510 3511 3512 3513
}

static void enter_rmode(struct kvm_vcpu *vcpu)
{
	unsigned long flags;
3514
	struct vcpu_vmx *vmx = to_vmx(vcpu);
3515

3516 3517 3518 3519 3520
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
3521 3522
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
3523

3524
	vmx->rmode.vm86_active = 1;
3525

3526 3527
	/*
	 * Very old userspace does not call KVM_SET_TSS_ADDR before entering
3528
	 * vcpu. Warn the user that an update is overdue.
3529
	 */
3530
	if (!vcpu->kvm->arch.tss_addr)
3531 3532 3533
		printk_once(KERN_WARNING "kvm: KVM_SET_TSS_ADDR need to be "
			     "called before entering vcpu\n");

3534 3535
	vmx_segment_cache_clear(vmx);

3536
	vmcs_writel(GUEST_TR_BASE, vcpu->kvm->arch.tss_addr);
3537 3538 3539 3540
	vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
	vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);

	flags = vmcs_readl(GUEST_RFLAGS);
3541
	vmx->rmode.save_rflags = flags;
3542

3543
	flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
3544 3545

	vmcs_writel(GUEST_RFLAGS, flags);
3546
	vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
3547 3548
	update_exception_bitmap(vcpu);

3549 3550 3551 3552 3553 3554
	fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
	fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
	fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
	fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
	fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
	fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
3555

3556
	kvm_mmu_reset_context(vcpu);
3557 3558
}

3559 3560 3561
static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
3562 3563 3564 3565
	struct shared_msr_entry *msr = find_msr_entry(vmx, MSR_EFER);

	if (!msr)
		return;
3566

3567 3568 3569 3570 3571
	/*
	 * Force kernel_gs_base reloading before EFER changes, as control
	 * of this msr depends on is_long_mode().
	 */
	vmx_load_host_state(to_vmx(vcpu));
3572
	vcpu->arch.efer = efer;
3573
	if (efer & EFER_LMA) {
3574
		vm_entry_controls_setbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
3575 3576
		msr->data = efer;
	} else {
3577
		vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
3578 3579 3580 3581 3582 3583

		msr->data = efer & ~EFER_LME;
	}
	setup_msrs(vmx);
}

3584
#ifdef CONFIG_X86_64
3585 3586 3587 3588 3589

static void enter_lmode(struct kvm_vcpu *vcpu)
{
	u32 guest_tr_ar;

3590 3591
	vmx_segment_cache_clear(to_vmx(vcpu));

3592
	guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
3593
	if ((guest_tr_ar & VMX_AR_TYPE_MASK) != VMX_AR_TYPE_BUSY_64_TSS) {
3594 3595
		pr_debug_ratelimited("%s: tss fixup for long mode. \n",
				     __func__);
3596
		vmcs_write32(GUEST_TR_AR_BYTES,
3597 3598
			     (guest_tr_ar & ~VMX_AR_TYPE_MASK)
			     | VMX_AR_TYPE_BUSY_64_TSS);
3599
	}
3600
	vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA);
3601 3602 3603 3604
}

static void exit_lmode(struct kvm_vcpu *vcpu)
{
3605
	vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
3606
	vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA);
3607 3608 3609 3610
}

#endif

3611
static inline void __vmx_flush_tlb(struct kvm_vcpu *vcpu, int vpid)
3612
{
3613
	vpid_sync_context(vpid);
3614 3615 3616
	if (enable_ept) {
		if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
			return;
3617
		ept_sync_context(construct_eptp(vcpu->arch.mmu.root_hpa));
3618
	}
3619 3620
}

3621 3622 3623 3624 3625
static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
{
	__vmx_flush_tlb(vcpu, to_vmx(vcpu)->vpid);
}

3626 3627 3628 3629 3630 3631 3632 3633
static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
{
	ulong cr0_guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;

	vcpu->arch.cr0 &= ~cr0_guest_owned_bits;
	vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits;
}

3634 3635 3636 3637 3638 3639 3640
static void vmx_decache_cr3(struct kvm_vcpu *vcpu)
{
	if (enable_ept && is_paging(vcpu))
		vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
	__set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
}

3641
static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
3642
{
3643 3644 3645 3646
	ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;

	vcpu->arch.cr4 &= ~cr4_guest_owned_bits;
	vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & cr4_guest_owned_bits;
3647 3648
}

3649 3650
static void ept_load_pdptrs(struct kvm_vcpu *vcpu)
{
Gleb Natapov's avatar
Gleb Natapov committed
3651 3652
	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;

Avi Kivity's avatar
Avi Kivity committed
3653 3654 3655 3656
	if (!test_bit(VCPU_EXREG_PDPTR,
		      (unsigned long *)&vcpu->arch.regs_dirty))
		return;

3657
	if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
Gleb Natapov's avatar
Gleb Natapov committed
3658 3659 3660 3661
		vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]);
		vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]);
		vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]);
		vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]);
3662 3663 3664
	}
}

3665 3666
static void ept_save_pdptrs(struct kvm_vcpu *vcpu)
{
Gleb Natapov's avatar
Gleb Natapov committed
3667 3668
	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;

3669
	if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
Gleb Natapov's avatar
Gleb Natapov committed
3670 3671 3672 3673
		mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
		mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
		mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
		mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
3674
	}
Avi Kivity's avatar
Avi Kivity committed
3675 3676 3677 3678 3679

	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_avail);
	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_dirty);
3680 3681
}

3682
static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
3683 3684 3685 3686 3687

static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
					unsigned long cr0,
					struct kvm_vcpu *vcpu)
{
3688 3689
	if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
		vmx_decache_cr3(vcpu);
3690 3691 3692
	if (!(cr0 & X86_CR0_PG)) {
		/* From paging/starting to nonpaging */
		vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
3693
			     vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) |
3694 3695 3696
			     (CPU_BASED_CR3_LOAD_EXITING |
			      CPU_BASED_CR3_STORE_EXITING));
		vcpu->arch.cr0 = cr0;
3697
		vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
3698 3699 3700
	} else if (!is_paging(vcpu)) {
		/* From nonpaging to paging */
		vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
3701
			     vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) &
3702 3703 3704
			     ~(CPU_BASED_CR3_LOAD_EXITING |
			       CPU_BASED_CR3_STORE_EXITING));
		vcpu->arch.cr0 = cr0;
3705
		vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
3706
	}
3707 3708 3709

	if (!(cr0 & X86_CR0_WP))
		*hw_cr0 &= ~X86_CR0_WP;
3710 3711
}

3712 3713
static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
{
3714
	struct vcpu_vmx *vmx = to_vmx(vcpu);
3715 3716
	unsigned long hw_cr0;

3717
	hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK);
3718
	if (enable_unrestricted_guest)
3719
		hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
3720
	else {
3721
		hw_cr0 |= KVM_VM_CR0_ALWAYS_ON;
3722

3723 3724
		if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
			enter_pmode(vcpu);
3725

3726 3727 3728
		if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
			enter_rmode(vcpu);
	}
3729

3730
#ifdef CONFIG_X86_64
3731
	if (vcpu->arch.efer & EFER_LME) {
3732
		if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
3733
			enter_lmode(vcpu);
3734
		if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
3735 3736 3737 3738
			exit_lmode(vcpu);
	}
#endif

3739
	if (enable_ept)
3740 3741
		ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);

3742
	if (!vcpu->fpu_active)
3743
		hw_cr0 |= X86_CR0_TS | X86_CR0_MP;
3744

3745
	vmcs_writel(CR0_READ_SHADOW, cr0);
3746
	vmcs_writel(GUEST_CR0, hw_cr0);
3747
	vcpu->arch.cr0 = cr0;
3748 3749 3750

	/* depends on vcpu->arch.cr0 to be set to a new value */
	vmx->emulation_required = emulation_required(vcpu);
3751 3752
}

3753 3754 3755 3756 3757 3758 3759
static u64 construct_eptp(unsigned long root_hpa)
{
	u64 eptp;

	/* TODO write the value reading from MSR */
	eptp = VMX_EPT_DEFAULT_MT |
		VMX_EPT_DEFAULT_GAW << VMX_EPT_GAW_EPTP_SHIFT;
3760 3761
	if (enable_ept_ad_bits)
		eptp |= VMX_EPT_AD_ENABLE_BIT;
3762 3763 3764 3765 3766
	eptp |= (root_hpa & PAGE_MASK);

	return eptp;
}

3767 3768
static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
{
3769 3770 3771 3772
	unsigned long guest_cr3;
	u64 eptp;

	guest_cr3 = cr3;
3773
	if (enable_ept) {
3774 3775
		eptp = construct_eptp(cr3);
		vmcs_write64(EPT_POINTER, eptp);
3776 3777 3778 3779
		if (is_paging(vcpu) || is_guest_mode(vcpu))
			guest_cr3 = kvm_read_cr3(vcpu);
		else
			guest_cr3 = vcpu->kvm->arch.ept_identity_map_addr;
3780
		ept_load_pdptrs(vcpu);
3781 3782
	}

3783
	vmx_flush_tlb(vcpu);
3784
	vmcs_writel(GUEST_CR3, guest_cr3);
3785 3786
}

3787
static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
3788
{
3789 3790 3791 3792 3793 3794 3795 3796 3797 3798
	/*
	 * Pass through host's Machine Check Enable value to hw_cr4, which
	 * is in force while we are in guest mode.  Do not let guests control
	 * this bit, even if host CR4.MCE == 0.
	 */
	unsigned long hw_cr4 =
		(cr4_read_shadow() & X86_CR4_MCE) |
		(cr4 & ~X86_CR4_MCE) |
		(to_vmx(vcpu)->rmode.vm86_active ?
		 KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON);
3799

3800 3801 3802 3803 3804 3805 3806 3807 3808
	if (cr4 & X86_CR4_VMXE) {
		/*
		 * To use VMXON (and later other VMX instructions), a guest
		 * must first be able to turn on cr4.VMXE (see handle_vmon()).
		 * So basically the check on whether to allow nested VMX
		 * is here.
		 */
		if (!nested_vmx_allowed(vcpu))
			return 1;
3809 3810 3811
	}
	if (to_vmx(vcpu)->nested.vmxon &&
	    ((cr4 & VMXON_CR4_ALWAYSON) != VMXON_CR4_ALWAYSON))
3812 3813
		return 1;

3814
	vcpu->arch.cr4 = cr4;
3815 3816 3817 3818 3819 3820 3821 3822
	if (enable_ept) {
		if (!is_paging(vcpu)) {
			hw_cr4 &= ~X86_CR4_PAE;
			hw_cr4 |= X86_CR4_PSE;
		} else if (!(cr4 & X86_CR4_PAE)) {
			hw_cr4 &= ~X86_CR4_PAE;
		}
	}
3823

3824 3825 3826 3827 3828 3829 3830 3831 3832 3833
	if (!enable_unrestricted_guest && !is_paging(vcpu))
		/*
		 * SMEP/SMAP is disabled if CPU is in non-paging mode in
		 * hardware.  However KVM always uses paging mode without
		 * unrestricted guest.
		 * To emulate this behavior, SMEP/SMAP needs to be manually
		 * disabled when guest switches to non-paging mode.
		 */
		hw_cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP);

3834 3835
	vmcs_writel(CR4_READ_SHADOW, cr4);
	vmcs_writel(GUEST_CR4, hw_cr4);
3836
	return 0;
3837 3838 3839 3840 3841
}

static void vmx_get_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
3842
	struct vcpu_vmx *vmx = to_vmx(vcpu);
3843 3844
	u32 ar;

3845
	if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
3846
		*var = vmx->rmode.segs[seg];
3847
		if (seg == VCPU_SREG_TR
3848
		    || var->selector == vmx_read_guest_seg_selector(vmx, seg))
3849
			return;
3850 3851 3852
		var->base = vmx_read_guest_seg_base(vmx, seg);
		var->selector = vmx_read_guest_seg_selector(vmx, seg);
		return;
3853
	}
3854 3855 3856 3857
	var->base = vmx_read_guest_seg_base(vmx, seg);
	var->limit = vmx_read_guest_seg_limit(vmx, seg);
	var->selector = vmx_read_guest_seg_selector(vmx, seg);
	ar = vmx_read_guest_seg_ar(vmx, seg);
3858
	var->unusable = (ar >> 16) & 1;
3859 3860 3861
	var->type = ar & 15;
	var->s = (ar >> 4) & 1;
	var->dpl = (ar >> 5) & 3;
3862 3863 3864 3865 3866 3867 3868 3869
	/*
	 * Some userspaces do not preserve unusable property. Since usable
	 * segment has to be present according to VMX spec we can use present
	 * property to amend userspace bug by making unusable segment always
	 * nonpresent. vmx_segment_access_rights() already marks nonpresent
	 * segment as unusable.
	 */
	var->present = !var->unusable;
3870 3871 3872 3873 3874 3875
	var->avl = (ar >> 12) & 1;
	var->l = (ar >> 13) & 1;
	var->db = (ar >> 14) & 1;
	var->g = (ar >> 15) & 1;
}

3876 3877 3878 3879 3880 3881 3882 3883
static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
	struct kvm_segment s;

	if (to_vmx(vcpu)->rmode.vm86_active) {
		vmx_get_segment(vcpu, &s, seg);
		return s.base;
	}
3884
	return vmx_read_guest_seg_base(to_vmx(vcpu), seg);
3885 3886
}

3887
static int vmx_get_cpl(struct kvm_vcpu *vcpu)
3888
{
3889 3890
	struct vcpu_vmx *vmx = to_vmx(vcpu);

3891
	if (unlikely(vmx->rmode.vm86_active))
3892
		return 0;
3893 3894
	else {
		int ar = vmx_read_guest_seg_ar(vmx, VCPU_SREG_SS);
3895
		return VMX_AR_DPL(ar);
Avi Kivity's avatar
Avi Kivity committed
3896 3897 3898
	}
}

3899
static u32 vmx_segment_access_rights(struct kvm_segment *var)
3900 3901 3902
{
	u32 ar;

3903
	if (var->unusable || !var->present)
3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914
		ar = 1 << 16;
	else {
		ar = var->type & 15;
		ar |= (var->s & 1) << 4;
		ar |= (var->dpl & 3) << 5;
		ar |= (var->present & 1) << 7;
		ar |= (var->avl & 1) << 12;
		ar |= (var->l & 1) << 13;
		ar |= (var->db & 1) << 14;
		ar |= (var->g & 1) << 15;
	}
3915 3916 3917 3918 3919 3920 3921

	return ar;
}

static void vmx_set_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
3922
	struct vcpu_vmx *vmx = to_vmx(vcpu);
3923
	const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
3924

3925 3926
	vmx_segment_cache_clear(vmx);

3927 3928 3929 3930 3931 3932
	if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
		vmx->rmode.segs[seg] = *var;
		if (seg == VCPU_SREG_TR)
			vmcs_write16(sf->selector, var->selector);
		else if (var->s)
			fix_rmode_seg(seg, &vmx->rmode.segs[seg]);
3933
		goto out;
3934
	}
3935

3936 3937 3938
	vmcs_writel(sf->base, var->base);
	vmcs_write32(sf->limit, var->limit);
	vmcs_write16(sf->selector, var->selector);
3939 3940 3941 3942 3943 3944

	/*
	 *   Fix the "Accessed" bit in AR field of segment registers for older
	 * qemu binaries.
	 *   IA32 arch specifies that at the time of processor reset the
	 * "Accessed" bit in the AR field of segment registers is 1. And qemu
Guo Chao's avatar
Guo Chao committed
3945
	 * is setting it to 0 in the userland code. This causes invalid guest
3946 3947 3948 3949 3950 3951
	 * state vmexit when "unrestricted guest" mode is turned on.
	 *    Fix for this setup issue in cpu_reset is being pushed in the qemu
	 * tree. Newer qemu binaries with that qemu fix would not need this
	 * kvm hack.
	 */
	if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
3952
		var->type |= 0x1; /* Accessed */
3953

3954
	vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var));
3955 3956

out:
3957
	vmx->emulation_required = emulation_required(vcpu);
3958 3959 3960 3961
}

static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
{
3962
	u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS);
3963 3964 3965 3966 3967

	*db = (ar >> 14) & 1;
	*l = (ar >> 13) & 1;
}

3968
static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3969
{
3970 3971
	dt->size = vmcs_read32(GUEST_IDTR_LIMIT);
	dt->address = vmcs_readl(GUEST_IDTR_BASE);
3972 3973
}

3974
static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3975
{
3976 3977
	vmcs_write32(GUEST_IDTR_LIMIT, dt->size);
	vmcs_writel(GUEST_IDTR_BASE, dt->address);
3978 3979
}

3980
static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3981
{
3982 3983
	dt->size = vmcs_read32(GUEST_GDTR_LIMIT);
	dt->address = vmcs_readl(GUEST_GDTR_BASE);
3984 3985
}

3986
static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3987
{
3988 3989
	vmcs_write32(GUEST_GDTR_LIMIT, dt->size);
	vmcs_writel(GUEST_GDTR_BASE, dt->address);
3990 3991
}

3992 3993 3994 3995 3996 3997
static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
{
	struct kvm_segment var;
	u32 ar;

	vmx_get_segment(vcpu, &var, seg);
3998
	var.dpl = 0x3;
3999 4000
	if (seg == VCPU_SREG_CS)
		var.type = 0x3;
4001 4002 4003 4004
	ar = vmx_segment_access_rights(&var);

	if (var.base != (var.selector << 4))
		return false;
4005
	if (var.limit != 0xffff)
4006
		return false;
4007
	if (ar != 0xf3)
4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018
		return false;

	return true;
}

static bool code_segment_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment cs;
	unsigned int cs_rpl;

	vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
4019
	cs_rpl = cs.selector & SEGMENT_RPL_MASK;
4020

4021 4022
	if (cs.unusable)
		return false;
4023
	if (~cs.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_ACCESSES_MASK))
4024 4025 4026
		return false;
	if (!cs.s)
		return false;
4027
	if (cs.type & VMX_AR_TYPE_WRITEABLE_MASK) {
4028 4029
		if (cs.dpl > cs_rpl)
			return false;
4030
	} else {
4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046
		if (cs.dpl != cs_rpl)
			return false;
	}
	if (!cs.present)
		return false;

	/* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
	return true;
}

static bool stack_segment_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment ss;
	unsigned int ss_rpl;

	vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
4047
	ss_rpl = ss.selector & SEGMENT_RPL_MASK;
4048

4049 4050 4051
	if (ss.unusable)
		return true;
	if (ss.type != 3 && ss.type != 7)
4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068
		return false;
	if (!ss.s)
		return false;
	if (ss.dpl != ss_rpl) /* DPL != RPL */
		return false;
	if (!ss.present)
		return false;

	return true;
}

static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
{
	struct kvm_segment var;
	unsigned int rpl;

	vmx_get_segment(vcpu, &var, seg);
4069
	rpl = var.selector & SEGMENT_RPL_MASK;
4070

4071 4072
	if (var.unusable)
		return true;
4073 4074 4075 4076
	if (!var.s)
		return false;
	if (!var.present)
		return false;
4077
	if (~var.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_WRITEABLE_MASK)) {
4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093
		if (var.dpl < rpl) /* DPL < RPL */
			return false;
	}

	/* TODO: Add other members to kvm_segment_field to allow checking for other access
	 * rights flags
	 */
	return true;
}

static bool tr_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment tr;

	vmx_get_segment(vcpu, &tr, VCPU_SREG_TR);

4094 4095
	if (tr.unusable)
		return false;
4096
	if (tr.selector & SEGMENT_TI_MASK)	/* TI = 1 */
4097
		return false;
4098
	if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111
		return false;
	if (!tr.present)
		return false;

	return true;
}

static bool ldtr_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment ldtr;

	vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR);

4112 4113
	if (ldtr.unusable)
		return true;
4114
	if (ldtr.selector & SEGMENT_TI_MASK)	/* TI = 1 */
4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130
		return false;
	if (ldtr.type != 2)
		return false;
	if (!ldtr.present)
		return false;

	return true;
}

static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
{
	struct kvm_segment cs, ss;

	vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
	vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);

4131 4132
	return ((cs.selector & SEGMENT_RPL_MASK) ==
		 (ss.selector & SEGMENT_RPL_MASK));
4133 4134 4135 4136 4137 4138 4139 4140 4141
}

/*
 * Check if guest state is valid. Returns true if valid, false if
 * not.
 * We assume that registers are always usable
 */
static bool guest_state_valid(struct kvm_vcpu *vcpu)
{
4142 4143 4144
	if (enable_unrestricted_guest)
		return true;

4145
	/* real mode guest state checks */
4146
	if (!is_protmode(vcpu) || (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) {
4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187
		if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_SS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_DS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_ES))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_FS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_GS))
			return false;
	} else {
	/* protected mode guest state checks */
		if (!cs_ss_rpl_check(vcpu))
			return false;
		if (!code_segment_valid(vcpu))
			return false;
		if (!stack_segment_valid(vcpu))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_DS))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_ES))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_FS))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_GS))
			return false;
		if (!tr_valid(vcpu))
			return false;
		if (!ldtr_valid(vcpu))
			return false;
	}
	/* TODO:
	 * - Add checks on RIP
	 * - Add checks on RFLAGS
	 */

	return true;
}

Mike Day's avatar
Mike Day committed
4188
static int init_rmode_tss(struct kvm *kvm)
4189
{
4190
	gfn_t fn;
4191
	u16 data = 0;
4192
	int idx, r;
4193

4194
	idx = srcu_read_lock(&kvm->srcu);
4195
	fn = kvm->arch.tss_addr >> PAGE_SHIFT;
4196 4197
	r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
	if (r < 0)
4198
		goto out;
4199
	data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
4200 4201
	r = kvm_write_guest_page(kvm, fn++, &data,
			TSS_IOPB_BASE_OFFSET, sizeof(u16));
4202
	if (r < 0)
4203
		goto out;
4204 4205
	r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
	if (r < 0)
4206
		goto out;
4207 4208
	r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
	if (r < 0)
4209
		goto out;
4210
	data = ~0;
4211 4212 4213 4214
	r = kvm_write_guest_page(kvm, fn, &data,
				 RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
				 sizeof(u8));
out:
4215
	srcu_read_unlock(&kvm->srcu, idx);
4216
	return r;
4217 4218
}

4219 4220
static int init_rmode_identity_map(struct kvm *kvm)
{
4221
	int i, idx, r = 0;
4222 4223 4224
	pfn_t identity_map_pfn;
	u32 tmp;

4225
	if (!enable_ept)
4226
		return 0;
4227 4228 4229 4230

	/* Protect kvm->arch.ept_identity_pagetable_done. */
	mutex_lock(&kvm->slots_lock);

4231
	if (likely(kvm->arch.ept_identity_pagetable_done))
4232 4233
		goto out2;

4234
	identity_map_pfn = kvm->arch.ept_identity_map_addr >> PAGE_SHIFT;
4235 4236

	r = alloc_identity_pagetable(kvm);
4237
	if (r < 0)
4238 4239
		goto out2;

4240
	idx = srcu_read_lock(&kvm->srcu);
4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253
	r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
	if (r < 0)
		goto out;
	/* Set up identity-mapping pagetable for EPT in real mode */
	for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
		tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
			_PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
		r = kvm_write_guest_page(kvm, identity_map_pfn,
				&tmp, i * sizeof(tmp), sizeof(tmp));
		if (r < 0)
			goto out;
	}
	kvm->arch.ept_identity_pagetable_done = true;
4254

4255
out:
4256
	srcu_read_unlock(&kvm->srcu, idx);
4257 4258 4259

out2:
	mutex_unlock(&kvm->slots_lock);
4260
	return r;
4261 4262
}

4263 4264
static void seg_setup(int seg)
{
4265
	const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
4266
	unsigned int ar;
4267 4268 4269 4270

	vmcs_write16(sf->selector, 0);
	vmcs_writel(sf->base, 0);
	vmcs_write32(sf->limit, 0xffff);
4271 4272 4273
	ar = 0x93;
	if (seg == VCPU_SREG_CS)
		ar |= 0x08; /* code segment */
4274 4275

	vmcs_write32(sf->ar_bytes, ar);
4276 4277
}

4278 4279
static int alloc_apic_access_page(struct kvm *kvm)
{
4280
	struct page *page;
4281 4282
	int r = 0;

4283
	mutex_lock(&kvm->slots_lock);
4284
	if (kvm->arch.apic_access_page_done)
4285
		goto out;
4286 4287
	r = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
				    APIC_DEFAULT_PHYS_BASE, PAGE_SIZE);
4288 4289
	if (r)
		goto out;
4290

4291
	page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
4292 4293 4294 4295 4296
	if (is_error_page(page)) {
		r = -EFAULT;
		goto out;
	}

4297 4298 4299 4300 4301 4302
	/*
	 * Do not pin the page in memory, so that memory hot-unplug
	 * is able to migrate it.
	 */
	put_page(page);
	kvm->arch.apic_access_page_done = true;
4303
out:
4304
	mutex_unlock(&kvm->slots_lock);
4305 4306 4307
	return r;
}

4308 4309
static int alloc_identity_pagetable(struct kvm *kvm)
{
4310 4311
	/* Called with kvm->slots_lock held. */

4312 4313
	int r = 0;

4314 4315
	BUG_ON(kvm->arch.ept_identity_pagetable_done);

4316 4317
	r = __x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
				    kvm->arch.ept_identity_map_addr, PAGE_SIZE);
4318 4319 4320 4321

	return r;
}

4322
static int allocate_vpid(void)
4323 4324 4325
{
	int vpid;

4326
	if (!enable_vpid)
4327
		return 0;
4328 4329
	spin_lock(&vmx_vpid_lock);
	vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
4330
	if (vpid < VMX_NR_VPIDS)
4331
		__set_bit(vpid, vmx_vpid_bitmap);
4332 4333
	else
		vpid = 0;
4334
	spin_unlock(&vmx_vpid_lock);
4335
	return vpid;
4336 4337
}

4338
static void free_vpid(int vpid)
4339
{
4340
	if (!enable_vpid || vpid == 0)
4341 4342
		return;
	spin_lock(&vmx_vpid_lock);
4343
	__clear_bit(vpid, vmx_vpid_bitmap);
4344 4345 4346
	spin_unlock(&vmx_vpid_lock);
}

4347 4348 4349 4350
#define MSR_TYPE_R	1
#define MSR_TYPE_W	2
static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
						u32 msr, int type)
4351
{
4352
	int f = sizeof(unsigned long);
4353 4354 4355 4356 4357 4358 4359 4360 4361 4362

	if (!cpu_has_vmx_msr_bitmap())
		return;

	/*
	 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
	 * have the write-low and read-high bitmap offsets the wrong way round.
	 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
	 */
	if (msr <= 0x1fff) {
4363 4364 4365 4366 4367 4368 4369 4370
		if (type & MSR_TYPE_R)
			/* read-low */
			__clear_bit(msr, msr_bitmap + 0x000 / f);

		if (type & MSR_TYPE_W)
			/* write-low */
			__clear_bit(msr, msr_bitmap + 0x800 / f);

4371 4372
	} else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
		msr &= 0x1fff;
4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415
		if (type & MSR_TYPE_R)
			/* read-high */
			__clear_bit(msr, msr_bitmap + 0x400 / f);

		if (type & MSR_TYPE_W)
			/* write-high */
			__clear_bit(msr, msr_bitmap + 0xc00 / f);

	}
}

static void __vmx_enable_intercept_for_msr(unsigned long *msr_bitmap,
						u32 msr, int type)
{
	int f = sizeof(unsigned long);

	if (!cpu_has_vmx_msr_bitmap())
		return;

	/*
	 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
	 * have the write-low and read-high bitmap offsets the wrong way round.
	 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
	 */
	if (msr <= 0x1fff) {
		if (type & MSR_TYPE_R)
			/* read-low */
			__set_bit(msr, msr_bitmap + 0x000 / f);

		if (type & MSR_TYPE_W)
			/* write-low */
			__set_bit(msr, msr_bitmap + 0x800 / f);

	} else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
		msr &= 0x1fff;
		if (type & MSR_TYPE_R)
			/* read-high */
			__set_bit(msr, msr_bitmap + 0x400 / f);

		if (type & MSR_TYPE_W)
			/* write-high */
			__set_bit(msr, msr_bitmap + 0xc00 / f);

4416 4417 4418
	}
}

4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464
/*
 * If a msr is allowed by L0, we should check whether it is allowed by L1.
 * The corresponding bit will be cleared unless both of L0 and L1 allow it.
 */
static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1,
					       unsigned long *msr_bitmap_nested,
					       u32 msr, int type)
{
	int f = sizeof(unsigned long);

	if (!cpu_has_vmx_msr_bitmap()) {
		WARN_ON(1);
		return;
	}

	/*
	 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
	 * have the write-low and read-high bitmap offsets the wrong way round.
	 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
	 */
	if (msr <= 0x1fff) {
		if (type & MSR_TYPE_R &&
		   !test_bit(msr, msr_bitmap_l1 + 0x000 / f))
			/* read-low */
			__clear_bit(msr, msr_bitmap_nested + 0x000 / f);

		if (type & MSR_TYPE_W &&
		   !test_bit(msr, msr_bitmap_l1 + 0x800 / f))
			/* write-low */
			__clear_bit(msr, msr_bitmap_nested + 0x800 / f);

	} else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
		msr &= 0x1fff;
		if (type & MSR_TYPE_R &&
		   !test_bit(msr, msr_bitmap_l1 + 0x400 / f))
			/* read-high */
			__clear_bit(msr, msr_bitmap_nested + 0x400 / f);

		if (type & MSR_TYPE_W &&
		   !test_bit(msr, msr_bitmap_l1 + 0xc00 / f))
			/* write-high */
			__clear_bit(msr, msr_bitmap_nested + 0xc00 / f);

	}
}

4465 4466 4467
static void vmx_disable_intercept_for_msr(u32 msr, bool longmode_only)
{
	if (!longmode_only)
4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495
		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy,
						msr, MSR_TYPE_R | MSR_TYPE_W);
	__vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode,
						msr, MSR_TYPE_R | MSR_TYPE_W);
}

static void vmx_enable_intercept_msr_read_x2apic(u32 msr)
{
	__vmx_enable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic,
			msr, MSR_TYPE_R);
	__vmx_enable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic,
			msr, MSR_TYPE_R);
}

static void vmx_disable_intercept_msr_read_x2apic(u32 msr)
{
	__vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic,
			msr, MSR_TYPE_R);
	__vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic,
			msr, MSR_TYPE_R);
}

static void vmx_disable_intercept_msr_write_x2apic(u32 msr)
{
	__vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic,
			msr, MSR_TYPE_W);
	__vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic,
			msr, MSR_TYPE_W);
4496 4497
}

4498 4499
static int vmx_cpu_uses_apicv(struct kvm_vcpu *vcpu)
{
4500
	return enable_apicv && lapic_in_kernel(vcpu);
4501 4502
}

4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539
static int vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	int max_irr;
	void *vapic_page;
	u16 status;

	if (vmx->nested.pi_desc &&
	    vmx->nested.pi_pending) {
		vmx->nested.pi_pending = false;
		if (!pi_test_and_clear_on(vmx->nested.pi_desc))
			return 0;

		max_irr = find_last_bit(
			(unsigned long *)vmx->nested.pi_desc->pir, 256);

		if (max_irr == 256)
			return 0;

		vapic_page = kmap(vmx->nested.virtual_apic_page);
		if (!vapic_page) {
			WARN_ON(1);
			return -ENOMEM;
		}
		__kvm_apic_update_irr(vmx->nested.pi_desc->pir, vapic_page);
		kunmap(vmx->nested.virtual_apic_page);

		status = vmcs_read16(GUEST_INTR_STATUS);
		if ((u8)max_irr > ((u8)status & 0xff)) {
			status &= ~0xff;
			status |= (u8)max_irr;
			vmcs_write16(GUEST_INTR_STATUS, status);
		}
	}
	return 0;
}

4540 4541 4542 4543
static inline bool kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu)
{
#ifdef CONFIG_SMP
	if (vcpu->mode == IN_GUEST_MODE) {
4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559
		struct vcpu_vmx *vmx = to_vmx(vcpu);

		/*
		 * Currently, we don't support urgent interrupt,
		 * all interrupts are recognized as non-urgent
		 * interrupt, so we cannot post interrupts when
		 * 'SN' is set.
		 *
		 * If the vcpu is in guest mode, it means it is
		 * running instead of being scheduled out and
		 * waiting in the run queue, and that's the only
		 * case when 'SN' is set currently, warning if
		 * 'SN' is set.
		 */
		WARN_ON_ONCE(pi_test_sn(&vmx->pi_desc));

4560 4561 4562 4563 4564 4565 4566 4567
		apic->send_IPI_mask(get_cpu_mask(vcpu->cpu),
				POSTED_INTR_VECTOR);
		return true;
	}
#endif
	return false;
}

4568 4569 4570 4571 4572 4573 4574 4575
static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu,
						int vector)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	if (is_guest_mode(vcpu) &&
	    vector == vmx->nested.posted_intr_nv) {
		/* the PIR and ON have been set by L1. */
4576
		kvm_vcpu_trigger_posted_interrupt(vcpu);
4577 4578 4579 4580 4581 4582 4583 4584 4585 4586
		/*
		 * If a posted intr is not recognized by hardware,
		 * we will accomplish it in the next vmentry.
		 */
		vmx->nested.pi_pending = true;
		kvm_make_request(KVM_REQ_EVENT, vcpu);
		return 0;
	}
	return -1;
}
4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598
/*
 * Send interrupt to vcpu via posted interrupt way.
 * 1. If target vcpu is running(non-root mode), send posted interrupt
 * notification to vcpu and hardware will sync PIR to vIRR atomically.
 * 2. If target vcpu isn't running(root mode), kick it to pick up the
 * interrupt from PIR in next vmentry.
 */
static void vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	int r;

4599 4600 4601 4602
	r = vmx_deliver_nested_posted_interrupt(vcpu, vector);
	if (!r)
		return;

4603 4604 4605 4606 4607
	if (pi_test_and_set_pir(vector, &vmx->pi_desc))
		return;

	r = pi_test_and_set_on(&vmx->pi_desc);
	kvm_make_request(KVM_REQ_EVENT, vcpu);
4608
	if (r || !kvm_vcpu_trigger_posted_interrupt(vcpu))
4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626
		kvm_vcpu_kick(vcpu);
}

static void vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	if (!pi_test_and_clear_on(&vmx->pi_desc))
		return;

	kvm_apic_update_irr(vcpu, vmx->pi_desc.pir);
}

static void vmx_sync_pir_to_irr_dummy(struct kvm_vcpu *vcpu)
{
	return;
}

4627 4628 4629 4630 4631 4632
/*
 * Set up the vmcs's constant host-state fields, i.e., host-state fields that
 * will not change in the lifetime of the guest.
 * Note that host-state that does change is set elsewhere. E.g., host-state
 * that is set differently for each CPU is set in vmx_vcpu_load(), not here.
 */
4633
static void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
4634 4635 4636 4637
{
	u32 low32, high32;
	unsigned long tmpl;
	struct desc_ptr dt;
4638
	unsigned long cr4;
4639

4640
	vmcs_writel(HOST_CR0, read_cr0() & ~X86_CR0_TS);  /* 22.2.3 */
4641 4642
	vmcs_writel(HOST_CR3, read_cr3());  /* 22.2.3  FIXME: shadow tables */

4643
	/* Save the most likely value for this task's CR4 in the VMCS. */
4644
	cr4 = cr4_read_shadow();
4645 4646 4647
	vmcs_writel(HOST_CR4, cr4);			/* 22.2.3, 22.2.5 */
	vmx->host_state.vmcs_host_cr4 = cr4;

4648
	vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS);  /* 22.2.4 */
4649 4650 4651 4652 4653 4654 4655 4656 4657
#ifdef CONFIG_X86_64
	/*
	 * Load null selectors, so we can avoid reloading them in
	 * __vmx_load_host_state(), in case userspace uses the null selectors
	 * too (the expected case).
	 */
	vmcs_write16(HOST_DS_SELECTOR, 0);
	vmcs_write16(HOST_ES_SELECTOR, 0);
#else
4658 4659
	vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
	vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
4660
#endif
4661 4662 4663 4664 4665
	vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
	vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8);  /* 22.2.4 */

	native_store_idt(&dt);
	vmcs_writel(HOST_IDTR_BASE, dt.address);   /* 22.2.4 */
4666
	vmx->host_idt_base = dt.address;
4667

Avi Kivity's avatar
Avi Kivity committed
4668
	vmcs_writel(HOST_RIP, vmx_return); /* 22.2.5 */
4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680

	rdmsr(MSR_IA32_SYSENTER_CS, low32, high32);
	vmcs_write32(HOST_IA32_SYSENTER_CS, low32);
	rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl);
	vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl);   /* 22.2.3 */

	if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
		rdmsr(MSR_IA32_CR_PAT, low32, high32);
		vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32));
	}
}

4681 4682 4683 4684 4685
static void set_cr4_guest_host_mask(struct vcpu_vmx *vmx)
{
	vmx->vcpu.arch.cr4_guest_owned_bits = KVM_CR4_GUEST_OWNED_BITS;
	if (enable_ept)
		vmx->vcpu.arch.cr4_guest_owned_bits |= X86_CR4_PGE;
4686 4687 4688
	if (is_guest_mode(&vmx->vcpu))
		vmx->vcpu.arch.cr4_guest_owned_bits &=
			~get_vmcs12(&vmx->vcpu)->cr4_guest_host_mask;
4689 4690 4691
	vmcs_writel(CR4_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr4_guest_owned_bits);
}

4692 4693 4694 4695
static u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx)
{
	u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl;

4696
	if (!vmx_cpu_uses_apicv(&vmx->vcpu))
4697 4698 4699 4700
		pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR;
	return pin_based_exec_ctrl;
}

4701 4702 4703
static u32 vmx_exec_control(struct vcpu_vmx *vmx)
{
	u32 exec_control = vmcs_config.cpu_based_exec_ctrl;
4704 4705 4706 4707

	if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)
		exec_control &= ~CPU_BASED_MOV_DR_EXITING;

4708
	if (!cpu_need_tpr_shadow(&vmx->vcpu)) {
4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724
		exec_control &= ~CPU_BASED_TPR_SHADOW;
#ifdef CONFIG_X86_64
		exec_control |= CPU_BASED_CR8_STORE_EXITING |
				CPU_BASED_CR8_LOAD_EXITING;
#endif
	}
	if (!enable_ept)
		exec_control |= CPU_BASED_CR3_STORE_EXITING |
				CPU_BASED_CR3_LOAD_EXITING  |
				CPU_BASED_INVLPG_EXITING;
	return exec_control;
}

static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx)
{
	u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
4725
	if (!cpu_need_virtualize_apic_accesses(&vmx->vcpu))
4726 4727 4728 4729 4730 4731
		exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
	if (vmx->vpid == 0)
		exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
	if (!enable_ept) {
		exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
		enable_unrestricted_guest = 0;
4732 4733
		/* Enable INVPCID for non-ept guests may cause performance regression. */
		exec_control &= ~SECONDARY_EXEC_ENABLE_INVPCID;
4734 4735 4736 4737 4738
	}
	if (!enable_unrestricted_guest)
		exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
	if (!ple_gap)
		exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
4739
	if (!vmx_cpu_uses_apicv(&vmx->vcpu))
4740 4741
		exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT |
				  SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
4742
	exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
4743 4744 4745 4746 4747 4748
	/* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD
	   (handle_vmptrld).
	   We can NOT enable shadow_vmcs here because we don't have yet
	   a current VMCS12
	*/
	exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
4749 4750 4751

	if (!enable_pml)
		exec_control &= ~SECONDARY_EXEC_ENABLE_PML;
Kai Huang's avatar
Kai Huang committed
4752

4753 4754 4755
	/* Currently, we allow L1 guest to directly run pcommit instruction. */
	exec_control &= ~SECONDARY_EXEC_PCOMMIT;

4756 4757 4758
	return exec_control;
}

4759 4760 4761 4762 4763
static void ept_set_mmio_spte_mask(void)
{
	/*
	 * EPT Misconfigurations can be generated if the value of bits 2:0
	 * of an EPT paging-structure entry is 110b (write/execute).
4764
	 * Also, magic bits (0x3ull << 62) is set to quickly identify mmio
4765 4766
	 * spte.
	 */
4767
	kvm_mmu_set_mmio_spte_mask((0x3ull << 62) | 0x6ull);
4768 4769
}

4770
#define VMX_XSS_EXIT_BITMAP 0
4771 4772 4773
/*
 * Sets up the vmcs for emulated real mode.
 */
4774
static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
4775
{
4776
#ifdef CONFIG_X86_64
4777
	unsigned long a;
4778
#endif
4779 4780 4781
	int i;

	/* I/O */
4782 4783
	vmcs_write64(IO_BITMAP_A, __pa(vmx_io_bitmap_a));
	vmcs_write64(IO_BITMAP_B, __pa(vmx_io_bitmap_b));
4784

4785 4786 4787 4788
	if (enable_shadow_vmcs) {
		vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap));
		vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap));
	}
4789
	if (cpu_has_vmx_msr_bitmap())
4790
		vmcs_write64(MSR_BITMAP, __pa(vmx_msr_bitmap_legacy));
4791

4792 4793 4794
	vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */

	/* Control */
4795
	vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, vmx_pin_based_exec_ctrl(vmx));
4796

4797
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, vmx_exec_control(vmx));
4798

4799
	if (cpu_has_secondary_exec_ctrls())
4800 4801
		vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
				vmx_secondary_exec_control(vmx));
4802

4803
	if (vmx_cpu_uses_apicv(&vmx->vcpu)) {
4804 4805 4806 4807 4808 4809
		vmcs_write64(EOI_EXIT_BITMAP0, 0);
		vmcs_write64(EOI_EXIT_BITMAP1, 0);
		vmcs_write64(EOI_EXIT_BITMAP2, 0);
		vmcs_write64(EOI_EXIT_BITMAP3, 0);

		vmcs_write16(GUEST_INTR_STATUS, 0);
4810 4811 4812

		vmcs_write64(POSTED_INTR_NV, POSTED_INTR_VECTOR);
		vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc)));
4813 4814
	}

4815 4816
	if (ple_gap) {
		vmcs_write32(PLE_GAP, ple_gap);
4817 4818
		vmx->ple_window = ple_window;
		vmx->ple_window_dirty = true;
4819 4820
	}

4821 4822
	vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
	vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
4823 4824
	vmcs_write32(CR3_TARGET_COUNT, 0);           /* 22.2.1 */

4825 4826
	vmcs_write16(HOST_FS_SELECTOR, 0);            /* 22.2.4 */
	vmcs_write16(HOST_GS_SELECTOR, 0);            /* 22.2.4 */
4827
	vmx_set_constant_host_state(vmx);
4828
#ifdef CONFIG_X86_64
4829 4830 4831 4832 4833 4834 4835 4836 4837
	rdmsrl(MSR_FS_BASE, a);
	vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */
	rdmsrl(MSR_GS_BASE, a);
	vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */
#else
	vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
	vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
#endif

4838 4839
	vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
4840
	vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host));
4841
	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
4842
	vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest));
4843

4844 4845
	if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT)
		vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
4846

4847
	for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i) {
4848 4849
		u32 index = vmx_msr_index[i];
		u32 data_low, data_high;
4850
		int j = vmx->nmsrs;
4851 4852 4853

		if (rdmsr_safe(index, &data_low, &data_high) < 0)
			continue;
4854 4855
		if (wrmsr_safe(index, data_low, data_high) < 0)
			continue;
4856 4857
		vmx->guest_msrs[j].index = i;
		vmx->guest_msrs[j].data = 0;
4858
		vmx->guest_msrs[j].mask = -1ull;
4859
		++vmx->nmsrs;
4860 4861
	}

4862 4863

	vm_exit_controls_init(vmx, vmcs_config.vmexit_ctrl);
4864 4865

	/* 22.2.1, 20.8.1 */
4866
	vm_entry_controls_init(vmx, vmcs_config.vmentry_ctrl);
4867

4868
	vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL);
4869
	set_cr4_guest_host_mask(vmx);
4870

4871 4872 4873
	if (vmx_xsaves_supported())
		vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP);

4874 4875 4876 4877 4878 4879
	if (enable_pml) {
		ASSERT(vmx->pml_pg);
		vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
		vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
	}

4880 4881 4882
	return 0;
}

4883
static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
4884 4885
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
4886
	struct msr_data apic_base_msr;
4887
	u64 cr0;
4888

4889
	vmx->rmode.vm86_active = 0;
4890

4891 4892
	vmx->soft_vnmi_blocked = 0;

4893
	vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
4894 4895 4896 4897 4898 4899 4900 4901 4902 4903
	kvm_set_cr8(vcpu, 0);

	if (!init_event) {
		apic_base_msr.data = APIC_DEFAULT_PHYS_BASE |
				     MSR_IA32_APICBASE_ENABLE;
		if (kvm_vcpu_is_reset_bsp(vcpu))
			apic_base_msr.data |= MSR_IA32_APICBASE_BSP;
		apic_base_msr.host_initiated = true;
		kvm_set_apic_base(vcpu, &apic_base_msr);
	}
4904

4905 4906
	vmx_segment_cache_clear(vmx);

4907
	seg_setup(VCPU_SREG_CS);
4908
	vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
4909
	vmcs_write32(GUEST_CS_BASE, 0xffff0000);
4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926

	seg_setup(VCPU_SREG_DS);
	seg_setup(VCPU_SREG_ES);
	seg_setup(VCPU_SREG_FS);
	seg_setup(VCPU_SREG_GS);
	seg_setup(VCPU_SREG_SS);

	vmcs_write16(GUEST_TR_SELECTOR, 0);
	vmcs_writel(GUEST_TR_BASE, 0);
	vmcs_write32(GUEST_TR_LIMIT, 0xffff);
	vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);

	vmcs_write16(GUEST_LDTR_SELECTOR, 0);
	vmcs_writel(GUEST_LDTR_BASE, 0);
	vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
	vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);

4927 4928 4929 4930 4931 4932
	if (!init_event) {
		vmcs_write32(GUEST_SYSENTER_CS, 0);
		vmcs_writel(GUEST_SYSENTER_ESP, 0);
		vmcs_writel(GUEST_SYSENTER_EIP, 0);
		vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
	}
4933 4934

	vmcs_writel(GUEST_RFLAGS, 0x02);
4935
	kvm_rip_write(vcpu, 0xfff0);
4936 4937 4938 4939 4940 4941 4942

	vmcs_writel(GUEST_GDTR_BASE, 0);
	vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);

	vmcs_writel(GUEST_IDTR_BASE, 0);
	vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);

4943
	vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
4944 4945 4946 4947 4948
	vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
	vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0);

	setup_msrs(vmx);

4949 4950
	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);  /* 22.2.1 */

4951
	if (cpu_has_vmx_tpr_shadow() && !init_event) {
4952
		vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
4953
		if (cpu_need_tpr_shadow(vcpu))
4954
			vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
4955
				     __pa(vcpu->arch.apic->regs));
4956 4957 4958
		vmcs_write32(TPR_THRESHOLD, 0);
	}

4959
	kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
4960

4961
	if (vmx_cpu_uses_apicv(vcpu))
4962 4963
		memset(&vmx->pi_desc, 0, sizeof(struct pi_desc));

4964 4965 4966
	if (vmx->vpid != 0)
		vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);

4967 4968
	cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
	vmx->vcpu.arch.cr0 = cr0;
4969
	vmx_set_cr0(vcpu, cr0); /* enter rmode */
4970
	vmx_set_cr4(vcpu, 0);
4971
	vmx_set_efer(vcpu, 0);
4972 4973
	vmx_fpu_activate(vcpu);
	update_exception_bitmap(vcpu);
4974

4975
	vpid_sync_context(vmx->vpid);
4976 4977
}

4978 4979 4980 4981 4982 4983 4984 4985 4986 4987
/*
 * In nested virtualization, check if L1 asked to exit on external interrupts.
 * For most existing hypervisors, this will always return true.
 */
static bool nested_exit_on_intr(struct kvm_vcpu *vcpu)
{
	return get_vmcs12(vcpu)->pin_based_vm_exec_control &
		PIN_BASED_EXT_INTR_MASK;
}

4988 4989 4990 4991 4992 4993 4994 4995 4996 4997
/*
 * In nested virtualization, check if L1 has set
 * VM_EXIT_ACK_INTR_ON_EXIT
 */
static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu)
{
	return get_vmcs12(vcpu)->vm_exit_controls &
		VM_EXIT_ACK_INTR_ON_EXIT;
}

4998 4999 5000 5001 5002 5003
static bool nested_exit_on_nmi(struct kvm_vcpu *vcpu)
{
	return get_vmcs12(vcpu)->pin_based_vm_exec_control &
		PIN_BASED_NMI_EXITING;
}

5004
static void enable_irq_window(struct kvm_vcpu *vcpu)
5005 5006
{
	u32 cpu_based_vm_exec_control;
5007

5008 5009 5010 5011 5012
	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
}

5013
static void enable_nmi_window(struct kvm_vcpu *vcpu)
5014 5015 5016
{
	u32 cpu_based_vm_exec_control;

5017 5018 5019 5020 5021
	if (!cpu_has_virtual_nmis() ||
	    vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
		enable_irq_window(vcpu);
		return;
	}
5022 5023 5024 5025 5026 5027

	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_NMI_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
}

5028
static void vmx_inject_irq(struct kvm_vcpu *vcpu)
5029
{
5030
	struct vcpu_vmx *vmx = to_vmx(vcpu);
5031 5032
	uint32_t intr;
	int irq = vcpu->arch.interrupt.nr;
5033

5034
	trace_kvm_inj_virq(irq);
Feng (Eric) Liu's avatar
Feng (Eric) Liu committed
5035

5036
	++vcpu->stat.irq_injections;
5037
	if (vmx->rmode.vm86_active) {
5038 5039 5040 5041
		int inc_eip = 0;
		if (vcpu->arch.interrupt.soft)
			inc_eip = vcpu->arch.event_exit_inst_len;
		if (kvm_inject_realmode_interrupt(vcpu, irq, inc_eip) != EMULATE_DONE)
5042
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5043 5044
		return;
	}
5045 5046 5047 5048 5049 5050 5051 5052
	intr = irq | INTR_INFO_VALID_MASK;
	if (vcpu->arch.interrupt.soft) {
		intr |= INTR_TYPE_SOFT_INTR;
		vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
			     vmx->vcpu.arch.event_exit_inst_len);
	} else
		intr |= INTR_TYPE_EXT_INTR;
	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
5053 5054
}

5055 5056
static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
{
5057 5058
	struct vcpu_vmx *vmx = to_vmx(vcpu);

5059 5060 5061
	if (is_guest_mode(vcpu))
		return;

5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074
	if (!cpu_has_virtual_nmis()) {
		/*
		 * Tracking the NMI-blocked state in software is built upon
		 * finding the next open IRQ window. This, in turn, depends on
		 * well-behaving guests: They have to keep IRQs disabled at
		 * least as long as the NMI handler runs. Otherwise we may
		 * cause NMI nesting, maybe breaking the guest. But as this is
		 * highly unlikely, we can live with the residual risk.
		 */
		vmx->soft_vnmi_blocked = 1;
		vmx->vnmi_blocked_time = 0;
	}

5075
	++vcpu->stat.nmi_injections;
5076
	vmx->nmi_known_unmasked = false;
5077
	if (vmx->rmode.vm86_active) {
5078
		if (kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0) != EMULATE_DONE)
5079
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5080 5081
		return;
	}
5082 5083 5084 5085
	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
			INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
}

5086 5087 5088 5089
static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
{
	if (!cpu_has_virtual_nmis())
		return to_vmx(vcpu)->soft_vnmi_blocked;
5090 5091
	if (to_vmx(vcpu)->nmi_known_unmasked)
		return false;
5092
	return vmcs_read32(GUEST_INTERRUPTIBILITY_INFO)	& GUEST_INTR_STATE_NMI;
5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104
}

static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	if (!cpu_has_virtual_nmis()) {
		if (vmx->soft_vnmi_blocked != masked) {
			vmx->soft_vnmi_blocked = masked;
			vmx->vnmi_blocked_time = 0;
		}
	} else {
5105
		vmx->nmi_known_unmasked = !masked;
5106 5107 5108 5109 5110 5111 5112 5113 5114
		if (masked)
			vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
				      GUEST_INTR_STATE_NMI);
		else
			vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
					GUEST_INTR_STATE_NMI);
	}
}

5115 5116
static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
{
5117 5118
	if (to_vmx(vcpu)->nested.nested_run_pending)
		return 0;
5119

5120 5121 5122 5123 5124 5125 5126 5127
	if (!cpu_has_virtual_nmis() && to_vmx(vcpu)->soft_vnmi_blocked)
		return 0;

	return	!(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
		  (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI
		   | GUEST_INTR_STATE_NMI));
}

5128 5129
static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu)
{
5130 5131
	return (!to_vmx(vcpu)->nested.nested_run_pending &&
		vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
5132 5133
		!(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
			(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
5134 5135
}

5136 5137 5138 5139
static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
{
	int ret;

5140 5141
	ret = x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, addr,
				    PAGE_SIZE * 3);
5142 5143
	if (ret)
		return ret;
5144
	kvm->arch.tss_addr = addr;
5145
	return init_rmode_tss(kvm);
5146 5147
}

5148
static bool rmode_exception(struct kvm_vcpu *vcpu, int vec)
5149
{
5150 5151
	switch (vec) {
	case BP_VECTOR:
5152 5153 5154 5155 5156 5157
		/*
		 * Update instruction length as we may reinject the exception
		 * from user space while in guest debugging mode.
		 */
		to_vmx(vcpu)->vcpu.arch.event_exit_inst_len =
			vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
Jan Kiszka's avatar
Jan Kiszka committed
5158
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
5159 5160 5161 5162 5163 5164
			return false;
		/* fall through */
	case DB_VECTOR:
		if (vcpu->guest_debug &
			(KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
			return false;
Jan Kiszka's avatar
Jan Kiszka committed
5165 5166
		/* fall through */
	case DE_VECTOR:
5167 5168 5169 5170 5171 5172 5173
	case OF_VECTOR:
	case BR_VECTOR:
	case UD_VECTOR:
	case DF_VECTOR:
	case SS_VECTOR:
	case GP_VECTOR:
	case MF_VECTOR:
5174 5175
		return true;
	break;
5176
	}
5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190
	return false;
}

static int handle_rmode_exception(struct kvm_vcpu *vcpu,
				  int vec, u32 err_code)
{
	/*
	 * Instruction with address size override prefix opcode 0x67
	 * Cause the #SS fault with 0 error code in VM86 mode.
	 */
	if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) {
		if (emulate_instruction(vcpu, 0) == EMULATE_DONE) {
			if (vcpu->arch.halt_request) {
				vcpu->arch.halt_request = 0;
5191
				return kvm_vcpu_halt(vcpu);
5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204
			}
			return 1;
		}
		return 0;
	}

	/*
	 * Forward all other exceptions that are valid in real mode.
	 * FIXME: Breaks guest debugging in real mode, needs to be fixed with
	 *        the required debugging infrastructure rework.
	 */
	kvm_queue_exception(vcpu, vec);
	return 1;
5205 5206
}

5207 5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220 5221 5222 5223 5224 5225
/*
 * Trigger machine check on the host. We assume all the MSRs are already set up
 * by the CPU and that we still run on the same CPU as the MCE occurred on.
 * We pass a fake environment to the machine check handler because we want
 * the guest to be always treated like user space, no matter what context
 * it used internally.
 */
static void kvm_machine_check(void)
{
#if defined(CONFIG_X86_MCE) && defined(CONFIG_X86_64)
	struct pt_regs regs = {
		.cs = 3, /* Fake ring 3 no matter what the guest ran on */
		.flags = X86_EFLAGS_IF,
	};

	do_machine_check(&regs, 0);
#endif
}

5226
static int handle_machine_check(struct kvm_vcpu *vcpu)
5227 5228 5229 5230 5231
{
	/* already handled by vcpu_run */
	return 1;
}

5232
static int handle_exception(struct kvm_vcpu *vcpu)
5233
{
5234
	struct vcpu_vmx *vmx = to_vmx(vcpu);
5235
	struct kvm_run *kvm_run = vcpu->run;
Jan Kiszka's avatar
Jan Kiszka committed
5236
	u32 intr_info, ex_no, error_code;
5237
	unsigned long cr2, rip, dr6;
5238 5239 5240
	u32 vect_info;
	enum emulation_result er;

5241
	vect_info = vmx->idt_vectoring_info;
5242
	intr_info = vmx->exit_intr_info;
5243

5244
	if (is_machine_check(intr_info))
5245
		return handle_machine_check(vcpu);
5246

5247
	if (is_nmi(intr_info))
5248
		return 1;  /* already handled by vmx_vcpu_run() */
5249 5250

	if (is_no_device(intr_info)) {
5251
		vmx_fpu_activate(vcpu);
5252 5253 5254
		return 1;
	}

5255
	if (is_invalid_opcode(intr_info)) {
5256 5257 5258 5259
		if (is_guest_mode(vcpu)) {
			kvm_queue_exception(vcpu, UD_VECTOR);
			return 1;
		}
5260
		er = emulate_instruction(vcpu, EMULTYPE_TRAP_UD);
5261
		if (er != EMULATE_DONE)
5262
			kvm_queue_exception(vcpu, UD_VECTOR);
5263 5264 5265
		return 1;
	}

5266
	error_code = 0;
5267
	if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
5268
		error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
5269 5270 5271 5272 5273 5274 5275 5276 5277 5278

	/*
	 * The #PF with PFEC.RSVD = 1 indicates the guest is accessing
	 * MMIO, it is better to report an internal error.
	 * See the comments in vmx_handle_exit.
	 */
	if ((vect_info & VECTORING_INFO_VALID_MASK) &&
	    !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
5279
		vcpu->run->internal.ndata = 3;
5280 5281
		vcpu->run->internal.data[0] = vect_info;
		vcpu->run->internal.data[1] = intr_info;
5282
		vcpu->run->internal.data[2] = error_code;
5283 5284 5285
		return 0;
	}

5286
	if (is_page_fault(intr_info)) {
5287
		/* EPT won't cause page fault directly */
Julia Lawall's avatar
Julia Lawall committed
5288
		BUG_ON(enable_ept);
5289
		cr2 = vmcs_readl(EXIT_QUALIFICATION);
5290 5291
		trace_kvm_page_fault(cr2, error_code);

5292
		if (kvm_event_needs_reinjection(vcpu))
5293
			kvm_mmu_unprotect_page_virt(vcpu, cr2);
5294
		return kvm_mmu_page_fault(vcpu, cr2, error_code, NULL, 0);
5295 5296
	}

Jan Kiszka's avatar
Jan Kiszka committed
5297
	ex_no = intr_info & INTR_INFO_VECTOR_MASK;
5298 5299 5300 5301

	if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no))
		return handle_rmode_exception(vcpu, ex_no, error_code);

5302
	switch (ex_no) {
5303 5304 5305
	case AC_VECTOR:
		kvm_queue_exception_e(vcpu, AC_VECTOR, error_code);
		return 1;
5306 5307 5308 5309
	case DB_VECTOR:
		dr6 = vmcs_readl(EXIT_QUALIFICATION);
		if (!(vcpu->guest_debug &
		      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
5310
			vcpu->arch.dr6 &= ~15;
5311
			vcpu->arch.dr6 |= dr6 | DR6_RTM;
5312 5313 5314
			if (!(dr6 & ~DR6_RESERVED)) /* icebp */
				skip_emulated_instruction(vcpu);

5315 5316 5317 5318 5319 5320 5321
			kvm_queue_exception(vcpu, DB_VECTOR);
			return 1;
		}
		kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1;
		kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
		/* fall through */
	case BP_VECTOR:
5322 5323 5324 5325 5326 5327 5328
		/*
		 * Update instruction length as we may reinject #BP from
		 * user space while in guest debugging mode. Reading it for
		 * #DB as well causes no harm, it is not used in that case.
		 */
		vmx->vcpu.arch.event_exit_inst_len =
			vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
5329
		kvm_run->exit_reason = KVM_EXIT_DEBUG;
5330
		rip = kvm_rip_read(vcpu);
Jan Kiszka's avatar
Jan Kiszka committed
5331 5332
		kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip;
		kvm_run->debug.arch.exception = ex_no;
5333 5334
		break;
	default:
Jan Kiszka's avatar
Jan Kiszka committed
5335 5336 5337
		kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
		kvm_run->ex.exception = ex_no;
		kvm_run->ex.error_code = error_code;
5338
		break;
5339 5340 5341 5342
	}
	return 0;
}

5343
static int handle_external_interrupt(struct kvm_vcpu *vcpu)
5344
{
Avi Kivity's avatar
Avi Kivity committed
5345
	++vcpu->stat.irq_exits;
5346 5347 5348
	return 1;
}

5349
static int handle_triple_fault(struct kvm_vcpu *vcpu)
5350
{
5351
	vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
5352 5353
	return 0;
}
5354

5355
static int handle_io(struct kvm_vcpu *vcpu)
5356
{
5357
	unsigned long exit_qualification;
5358
	int size, in, string;
5359
	unsigned port;
5360

5361
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5362
	string = (exit_qualification & 16) != 0;
5363
	in = (exit_qualification & 8) != 0;
5364

5365
	++vcpu->stat.io_exits;
5366

5367
	if (string || in)
5368
		return emulate_instruction(vcpu, 0) == EMULATE_DONE;
5369

5370 5371
	port = exit_qualification >> 16;
	size = (exit_qualification & 7) + 1;
5372
	skip_emulated_instruction(vcpu);
5373 5374

	return kvm_fast_pio_out(vcpu, size, port);
5375 5376
}

5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387
static void
vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
{
	/*
	 * Patch in the VMCALL instruction:
	 */
	hypercall[0] = 0x0f;
	hypercall[1] = 0x01;
	hypercall[2] = 0xc1;
}

5388
static bool nested_cr0_valid(struct kvm_vcpu *vcpu, unsigned long val)
5389 5390
{
	unsigned long always_on = VMXON_CR0_ALWAYSON;
5391
	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
5392

5393
	if (to_vmx(vcpu)->nested.nested_vmx_secondary_ctls_high &
5394 5395 5396 5397 5398 5399
		SECONDARY_EXEC_UNRESTRICTED_GUEST &&
	    nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST))
		always_on &= ~(X86_CR0_PE | X86_CR0_PG);
	return (val & always_on) == always_on;
}

Guo Chao's avatar
Guo Chao committed
5400
/* called to set cr0 as appropriate for a mov-to-cr0 exit. */
5401 5402 5403
static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val)
{
	if (is_guest_mode(vcpu)) {
5404 5405 5406
		struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
		unsigned long orig_val = val;

5407 5408 5409
		/*
		 * We get here when L2 changed cr0 in a way that did not change
		 * any of L1's shadowed bits (see nested_vmx_exit_handled_cr),
5410 5411 5412 5413
		 * but did change L0 shadowed bits. So we first calculate the
		 * effective cr0 value that L1 would like to write into the
		 * hardware. It consists of the L2-owned bits from the new
		 * value combined with the L1-owned bits from L1's guest_cr0.
5414
		 */
5415 5416 5417
		val = (val & ~vmcs12->cr0_guest_host_mask) |
			(vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask);

5418
		if (!nested_cr0_valid(vcpu, val))
5419
			return 1;
5420 5421 5422 5423

		if (kvm_set_cr0(vcpu, val))
			return 1;
		vmcs_writel(CR0_READ_SHADOW, orig_val);
5424
		return 0;
5425 5426 5427 5428
	} else {
		if (to_vmx(vcpu)->nested.vmxon &&
		    ((val & VMXON_CR0_ALWAYSON) != VMXON_CR0_ALWAYSON))
			return 1;
5429
		return kvm_set_cr0(vcpu, val);
5430
	}
5431 5432 5433 5434 5435
}

static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val)
{
	if (is_guest_mode(vcpu)) {
5436 5437 5438 5439 5440 5441 5442
		struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
		unsigned long orig_val = val;

		/* analogously to handle_set_cr0 */
		val = (val & ~vmcs12->cr4_guest_host_mask) |
			(vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask);
		if (kvm_set_cr4(vcpu, val))
5443
			return 1;
5444
		vmcs_writel(CR4_READ_SHADOW, orig_val);
5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465
		return 0;
	} else
		return kvm_set_cr4(vcpu, val);
}

/* called to set cr0 as approriate for clts instruction exit. */
static void handle_clts(struct kvm_vcpu *vcpu)
{
	if (is_guest_mode(vcpu)) {
		/*
		 * We get here when L2 did CLTS, and L1 didn't shadow CR0.TS
		 * but we did (!fpu_active). We need to keep GUEST_CR0.TS on,
		 * just pretend it's off (also in arch.cr0 for fpu_activate).
		 */
		vmcs_writel(CR0_READ_SHADOW,
			vmcs_readl(CR0_READ_SHADOW) & ~X86_CR0_TS);
		vcpu->arch.cr0 &= ~X86_CR0_TS;
	} else
		vmx_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
}

5466
static int handle_cr(struct kvm_vcpu *vcpu)
5467
{
5468
	unsigned long exit_qualification, val;
5469 5470
	int cr;
	int reg;
5471
	int err;
5472

5473
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5474 5475 5476 5477
	cr = exit_qualification & 15;
	reg = (exit_qualification >> 8) & 15;
	switch ((exit_qualification >> 4) & 3) {
	case 0: /* mov to cr */
5478
		val = kvm_register_readl(vcpu, reg);
5479
		trace_kvm_cr_write(cr, val);
5480 5481
		switch (cr) {
		case 0:
5482
			err = handle_set_cr0(vcpu, val);
5483
			kvm_complete_insn_gp(vcpu, err);
5484 5485
			return 1;
		case 3:
5486
			err = kvm_set_cr3(vcpu, val);
5487
			kvm_complete_insn_gp(vcpu, err);
5488 5489
			return 1;
		case 4:
5490
			err = handle_set_cr4(vcpu, val);
5491
			kvm_complete_insn_gp(vcpu, err);
5492
			return 1;
5493 5494
		case 8: {
				u8 cr8_prev = kvm_get_cr8(vcpu);
5495
				u8 cr8 = (u8)val;
Andre Przywara's avatar
Andre Przywara committed
5496
				err = kvm_set_cr8(vcpu, cr8);
5497
				kvm_complete_insn_gp(vcpu, err);
5498
				if (lapic_in_kernel(vcpu))
5499 5500 5501
					return 1;
				if (cr8_prev <= cr8)
					return 1;
5502
				vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
5503 5504
				return 0;
			}
5505
		}
5506
		break;
5507
	case 2: /* clts */
5508
		handle_clts(vcpu);
5509
		trace_kvm_cr_write(0, kvm_read_cr0(vcpu));
5510
		skip_emulated_instruction(vcpu);
Avi Kivity's avatar
Avi Kivity committed
5511
		vmx_fpu_activate(vcpu);
5512
		return 1;
5513 5514 5515
	case 1: /*mov from cr*/
		switch (cr) {
		case 3:
5516 5517 5518
			val = kvm_read_cr3(vcpu);
			kvm_register_write(vcpu, reg, val);
			trace_kvm_cr_read(cr, val);
5519 5520 5521
			skip_emulated_instruction(vcpu);
			return 1;
		case 8:
5522 5523 5524
			val = kvm_get_cr8(vcpu);
			kvm_register_write(vcpu, reg, val);
			trace_kvm_cr_read(cr, val);
5525 5526 5527 5528 5529
			skip_emulated_instruction(vcpu);
			return 1;
		}
		break;
	case 3: /* lmsw */
5530
		val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
5531
		trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val);
5532
		kvm_lmsw(vcpu, val);
5533 5534 5535 5536 5537 5538

		skip_emulated_instruction(vcpu);
		return 1;
	default:
		break;
	}
5539
	vcpu->run->exit_reason = 0;
5540
	vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
5541 5542 5543 5544
	       (int)(exit_qualification >> 4) & 3, cr);
	return 0;
}

5545
static int handle_dr(struct kvm_vcpu *vcpu)
5546
{
5547
	unsigned long exit_qualification;
5548 5549 5550 5551 5552 5553 5554 5555
	int dr, dr7, reg;

	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	dr = exit_qualification & DEBUG_REG_ACCESS_NUM;

	/* First, if DR does not exist, trigger UD */
	if (!kvm_require_dr(vcpu, dr))
		return 1;
5556

5557
	/* Do not handle if the CPL > 0, will trigger GP on re-entry */
5558 5559
	if (!kvm_require_cpl(vcpu, 0))
		return 1;
5560 5561
	dr7 = vmcs_readl(GUEST_DR7);
	if (dr7 & DR7_GD) {
5562 5563 5564 5565 5566 5567
		/*
		 * As the vm-exit takes precedence over the debug trap, we
		 * need to emulate the latter, either for the host or the
		 * guest debugging itself.
		 */
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
5568
			vcpu->run->debug.arch.dr6 = vcpu->arch.dr6;
5569
			vcpu->run->debug.arch.dr7 = dr7;
5570
			vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu);
5571 5572
			vcpu->run->debug.arch.exception = DB_VECTOR;
			vcpu->run->exit_reason = KVM_EXIT_DEBUG;
5573 5574
			return 0;
		} else {
5575
			vcpu->arch.dr6 &= ~15;
5576
			vcpu->arch.dr6 |= DR6_BD | DR6_RTM;
5577 5578 5579 5580 5581
			kvm_queue_exception(vcpu, DB_VECTOR);
			return 1;
		}
	}

5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597
	if (vcpu->guest_debug == 0) {
		u32 cpu_based_vm_exec_control;

		cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
		cpu_based_vm_exec_control &= ~CPU_BASED_MOV_DR_EXITING;
		vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);

		/*
		 * No more DR vmexits; force a reload of the debug registers
		 * and reenter on this instruction.  The next vmexit will
		 * retrieve the full state of the debug registers.
		 */
		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
		return 1;
	}

5598 5599
	reg = DEBUG_REG_ACCESS_REG(exit_qualification);
	if (exit_qualification & TYPE_MOV_FROM_DR) {
5600
		unsigned long val;
5601 5602 5603 5604

		if (kvm_get_dr(vcpu, dr, &val))
			return 1;
		kvm_register_write(vcpu, reg, val);
5605
	} else
5606
		if (kvm_set_dr(vcpu, dr, kvm_register_readl(vcpu, reg)))
5607 5608
			return 1;

5609 5610 5611 5612
	skip_emulated_instruction(vcpu);
	return 1;
}

Jan Kiszka's avatar
Jan Kiszka committed
5613 5614 5615 5616 5617 5618 5619 5620 5621
static u64 vmx_get_dr6(struct kvm_vcpu *vcpu)
{
	return vcpu->arch.dr6;
}

static void vmx_set_dr6(struct kvm_vcpu *vcpu, unsigned long val)
{
}

5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639
static void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
{
	u32 cpu_based_vm_exec_control;

	get_debugreg(vcpu->arch.db[0], 0);
	get_debugreg(vcpu->arch.db[1], 1);
	get_debugreg(vcpu->arch.db[2], 2);
	get_debugreg(vcpu->arch.db[3], 3);
	get_debugreg(vcpu->arch.dr6, 6);
	vcpu->arch.dr7 = vmcs_readl(GUEST_DR7);

	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;

	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control |= CPU_BASED_MOV_DR_EXITING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
}

5640 5641 5642 5643 5644
static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
{
	vmcs_writel(GUEST_DR7, val);
}

5645
static int handle_cpuid(struct kvm_vcpu *vcpu)
5646
{
5647 5648
	kvm_emulate_cpuid(vcpu);
	return 1;
5649 5650
}

5651
static int handle_rdmsr(struct kvm_vcpu *vcpu)
5652
{
5653
	u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
5654
	struct msr_data msr_info;
5655

5656 5657 5658
	msr_info.index = ecx;
	msr_info.host_initiated = false;
	if (vmx_get_msr(vcpu, &msr_info)) {
5659
		trace_kvm_msr_read_ex(ecx);
5660
		kvm_inject_gp(vcpu, 0);
5661 5662 5663
		return 1;
	}

5664
	trace_kvm_msr_read(ecx, msr_info.data);
Feng (Eric) Liu's avatar
Feng (Eric) Liu committed
5665

5666
	/* FIXME: handling of bits 32:63 of rax, rdx */
5667 5668
	vcpu->arch.regs[VCPU_REGS_RAX] = msr_info.data & -1u;
	vcpu->arch.regs[VCPU_REGS_RDX] = (msr_info.data >> 32) & -1u;
5669 5670 5671 5672
	skip_emulated_instruction(vcpu);
	return 1;
}

5673
static int handle_wrmsr(struct kvm_vcpu *vcpu)
5674
{
5675
	struct msr_data msr;
5676 5677 5678
	u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
	u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u)
		| ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32);
5679

5680 5681 5682
	msr.data = data;
	msr.index = ecx;
	msr.host_initiated = false;
5683
	if (kvm_set_msr(vcpu, &msr) != 0) {
5684
		trace_kvm_msr_write_ex(ecx, data);
5685
		kvm_inject_gp(vcpu, 0);
5686 5687 5688
		return 1;
	}

5689
	trace_kvm_msr_write(ecx, data);
5690 5691 5692 5693
	skip_emulated_instruction(vcpu);
	return 1;
}

5694
static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
5695
{
5696
	kvm_make_request(KVM_REQ_EVENT, vcpu);
5697 5698 5699
	return 1;
}

5700
static int handle_interrupt_window(struct kvm_vcpu *vcpu)
5701
{
5702 5703 5704 5705 5706 5707
	u32 cpu_based_vm_exec_control;

	/* clear pending irq */
	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
Feng (Eric) Liu's avatar
Feng (Eric) Liu committed
5708

5709 5710
	kvm_make_request(KVM_REQ_EVENT, vcpu);

5711
	++vcpu->stat.irq_window_exits;
5712 5713 5714
	return 1;
}

5715
static int handle_halt(struct kvm_vcpu *vcpu)
5716
{
5717
	return kvm_emulate_halt(vcpu);
5718 5719
}

5720
static int handle_vmcall(struct kvm_vcpu *vcpu)
5721
{
5722 5723
	kvm_emulate_hypercall(vcpu);
	return 1;
5724 5725
}

5726 5727
static int handle_invd(struct kvm_vcpu *vcpu)
{
5728
	return emulate_instruction(vcpu, 0) == EMULATE_DONE;
5729 5730
}

5731
static int handle_invlpg(struct kvm_vcpu *vcpu)
Marcelo Tosatti's avatar
Marcelo Tosatti committed
5732
{
5733
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
Marcelo Tosatti's avatar
Marcelo Tosatti committed
5734 5735 5736 5737 5738 5739

	kvm_mmu_invlpg(vcpu, exit_qualification);
	skip_emulated_instruction(vcpu);
	return 1;
}

Avi Kivity's avatar
Avi Kivity committed
5740 5741 5742 5743 5744 5745 5746 5747 5748 5749
static int handle_rdpmc(struct kvm_vcpu *vcpu)
{
	int err;

	err = kvm_rdpmc(vcpu);
	kvm_complete_insn_gp(vcpu, err);

	return 1;
}

5750
static int handle_wbinvd(struct kvm_vcpu *vcpu)
Eddie Dong's avatar
Eddie Dong committed
5751
{
5752
	kvm_emulate_wbinvd(vcpu);
Eddie Dong's avatar
Eddie Dong committed
5753 5754 5755
	return 1;
}

5756 5757 5758 5759 5760 5761 5762 5763 5764 5765
static int handle_xsetbv(struct kvm_vcpu *vcpu)
{
	u64 new_bv = kvm_read_edx_eax(vcpu);
	u32 index = kvm_register_read(vcpu, VCPU_REGS_RCX);

	if (kvm_set_xcr(vcpu, index, new_bv) == 0)
		skip_emulated_instruction(vcpu);
	return 1;
}

5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779
static int handle_xsaves(struct kvm_vcpu *vcpu)
{
	skip_emulated_instruction(vcpu);
	WARN(1, "this should never happen\n");
	return 1;
}

static int handle_xrstors(struct kvm_vcpu *vcpu)
{
	skip_emulated_instruction(vcpu);
	WARN(1, "this should never happen\n");
	return 1;
}

5780
static int handle_apic_access(struct kvm_vcpu *vcpu)
5781
{
5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799
	if (likely(fasteoi)) {
		unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
		int access_type, offset;

		access_type = exit_qualification & APIC_ACCESS_TYPE;
		offset = exit_qualification & APIC_ACCESS_OFFSET;
		/*
		 * Sane guest uses MOV to write EOI, with written value
		 * not cared. So make a short-circuit here by avoiding
		 * heavy instruction emulation.
		 */
		if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) &&
		    (offset == APIC_EOI)) {
			kvm_lapic_set_eoi(vcpu);
			skip_emulated_instruction(vcpu);
			return 1;
		}
	}
5800
	return emulate_instruction(vcpu, 0) == EMULATE_DONE;
5801 5802
}

5803 5804 5805 5806 5807 5808 5809 5810 5811 5812
static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu)
{
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	int vector = exit_qualification & 0xff;

	/* EOI-induced VM exit is trap-like and thus no need to adjust IP */
	kvm_apic_set_eoi_accelerated(vcpu, vector);
	return 1;
}

5813 5814 5815 5816 5817 5818 5819 5820 5821 5822
static int handle_apic_write(struct kvm_vcpu *vcpu)
{
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	u32 offset = exit_qualification & 0xfff;

	/* APIC-write VM exit is trap-like and thus no need to adjust IP */
	kvm_apic_write_nodecode(vcpu, offset);
	return 1;
}

5823
static int handle_task_switch(struct kvm_vcpu *vcpu)
5824
{
5825
	struct vcpu_vmx *vmx = to_vmx(vcpu);
5826
	unsigned long exit_qualification;
5827 5828
	bool has_error_code = false;
	u32 error_code = 0;
5829
	u16 tss_selector;
5830
	int reason, type, idt_v, idt_index;
5831 5832

	idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
5833
	idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK);
5834
	type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
5835 5836 5837 5838

	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);

	reason = (u32)exit_qualification >> 30;
5839 5840 5841 5842
	if (reason == TASK_SWITCH_GATE && idt_v) {
		switch (type) {
		case INTR_TYPE_NMI_INTR:
			vcpu->arch.nmi_injected = false;
5843
			vmx_set_nmi_mask(vcpu, true);
5844 5845
			break;
		case INTR_TYPE_EXT_INTR:
5846
		case INTR_TYPE_SOFT_INTR:
5847 5848 5849
			kvm_clear_interrupt_queue(vcpu);
			break;
		case INTR_TYPE_HARD_EXCEPTION:
5850 5851 5852 5853 5854 5855 5856
			if (vmx->idt_vectoring_info &
			    VECTORING_INFO_DELIVER_CODE_MASK) {
				has_error_code = true;
				error_code =
					vmcs_read32(IDT_VECTORING_ERROR_CODE);
			}
			/* fall through */
5857 5858 5859 5860 5861 5862
		case INTR_TYPE_SOFT_EXCEPTION:
			kvm_clear_exception_queue(vcpu);
			break;
		default:
			break;
		}
5863
	}
5864 5865
	tss_selector = exit_qualification;

5866 5867 5868 5869 5870
	if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
		       type != INTR_TYPE_EXT_INTR &&
		       type != INTR_TYPE_NMI_INTR))
		skip_emulated_instruction(vcpu);

5871 5872 5873
	if (kvm_task_switch(vcpu, tss_selector,
			    type == INTR_TYPE_SOFT_INTR ? idt_index : -1, reason,
			    has_error_code, error_code) == EMULATE_FAIL) {
5874 5875 5876
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
5877
		return 0;
5878
	}
5879 5880 5881 5882 5883 5884 5885

	/*
	 * TODO: What about debug traps on tss switch?
	 *       Are we supposed to inject them and update dr6?
	 */

	return 1;
5886 5887
}

5888
static int handle_ept_violation(struct kvm_vcpu *vcpu)
5889
{
5890
	unsigned long exit_qualification;
5891
	gpa_t gpa;
5892
	u32 error_code;
5893 5894
	int gla_validity;

5895
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5896 5897 5898 5899 5900 5901

	gla_validity = (exit_qualification >> 7) & 0x3;
	if (gla_validity != 0x3 && gla_validity != 0x1 && gla_validity != 0) {
		printk(KERN_ERR "EPT: Handling EPT violation failed!\n");
		printk(KERN_ERR "EPT: GPA: 0x%lx, GVA: 0x%lx\n",
			(long unsigned int)vmcs_read64(GUEST_PHYSICAL_ADDRESS),
5902
			vmcs_readl(GUEST_LINEAR_ADDRESS));
5903 5904
		printk(KERN_ERR "EPT: Exit qualification is 0x%lx\n",
			(long unsigned int)exit_qualification);
5905 5906
		vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
		vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_VIOLATION;
5907
		return 0;
5908 5909
	}

5910 5911 5912 5913 5914 5915
	/*
	 * EPT violation happened while executing iret from NMI,
	 * "blocked by NMI" bit has to be set before next VM entry.
	 * There are errata that may cause this bit to not be set:
	 * AAK134, BY25.
	 */
5916 5917 5918
	if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
			cpu_has_virtual_nmis() &&
			(exit_qualification & INTR_INFO_UNBLOCK_NMI))
5919 5920
		vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI);

5921
	gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
5922
	trace_kvm_page_fault(gpa, exit_qualification);
5923 5924

	/* It is a write fault? */
5925
	error_code = exit_qualification & PFERR_WRITE_MASK;
5926
	/* It is a fetch fault? */
5927
	error_code |= (exit_qualification << 2) & PFERR_FETCH_MASK;
5928
	/* ept page table is present? */
5929
	error_code |= (exit_qualification >> 3) & PFERR_PRESENT_MASK;
5930

5931 5932
	vcpu->arch.exit_qualification = exit_qualification;

5933
	return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0);
5934 5935
}

5936
static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
5937
{
5938
	int ret;
5939 5940 5941
	gpa_t gpa;

	gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
5942
	if (!kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) {
5943
		skip_emulated_instruction(vcpu);
5944
		trace_kvm_fast_mmio(gpa);
5945 5946
		return 1;
	}
5947

5948
	ret = handle_mmio_page_fault(vcpu, gpa, true);
5949
	if (likely(ret == RET_MMIO_PF_EMULATE))
5950 5951
		return x86_emulate_instruction(vcpu, gpa, 0, NULL, 0) ==
					      EMULATE_DONE;
5952 5953 5954 5955

	if (unlikely(ret == RET_MMIO_PF_INVALID))
		return kvm_mmu_page_fault(vcpu, gpa, 0, NULL, 0);

5956
	if (unlikely(ret == RET_MMIO_PF_RETRY))
5957 5958 5959
		return 1;

	/* It is the real ept misconfig */
5960
	WARN_ON(1);
5961

5962 5963
	vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
	vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_MISCONFIG;
5964 5965 5966 5967

	return 0;
}

5968
static int handle_nmi_window(struct kvm_vcpu *vcpu)
5969 5970 5971 5972 5973 5974 5975 5976
{
	u32 cpu_based_vm_exec_control;

	/* clear pending NMI */
	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
	++vcpu->stat.nmi_window_exits;
5977
	kvm_make_request(KVM_REQ_EVENT, vcpu);
5978 5979 5980 5981

	return 1;
}

5982
static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
5983
{
5984 5985
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	enum emulation_result err = EMULATE_DONE;
5986
	int ret = 1;
5987 5988
	u32 cpu_exec_ctrl;
	bool intr_window_requested;
5989
	unsigned count = 130;
5990 5991 5992

	cpu_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	intr_window_requested = cpu_exec_ctrl & CPU_BASED_VIRTUAL_INTR_PENDING;
5993

5994
	while (vmx->emulation_required && count-- != 0) {
5995
		if (intr_window_requested && vmx_interrupt_allowed(vcpu))
5996 5997
			return handle_interrupt_window(&vmx->vcpu);

5998 5999 6000
		if (test_bit(KVM_REQ_EVENT, &vcpu->requests))
			return 1;

6001
		err = emulate_instruction(vcpu, EMULTYPE_NO_REEXECUTE);
6002

6003
		if (err == EMULATE_USER_EXIT) {
6004
			++vcpu->stat.mmio_exits;
6005 6006 6007
			ret = 0;
			goto out;
		}
6008

6009 6010 6011 6012
		if (err != EMULATE_DONE) {
			vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
			vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
			vcpu->run->internal.ndata = 0;
6013
			return 0;
6014
		}
6015

6016 6017
		if (vcpu->arch.halt_request) {
			vcpu->arch.halt_request = 0;
6018
			ret = kvm_vcpu_halt(vcpu);
6019 6020 6021
			goto out;
		}

6022
		if (signal_pending(current))
6023
			goto out;
6024 6025 6026 6027
		if (need_resched())
			schedule();
	}

6028 6029
out:
	return ret;
6030 6031
}

6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064 6065 6066 6067 6068
static int __grow_ple_window(int val)
{
	if (ple_window_grow < 1)
		return ple_window;

	val = min(val, ple_window_actual_max);

	if (ple_window_grow < ple_window)
		val *= ple_window_grow;
	else
		val += ple_window_grow;

	return val;
}

static int __shrink_ple_window(int val, int modifier, int minimum)
{
	if (modifier < 1)
		return ple_window;

	if (modifier < ple_window)
		val /= modifier;
	else
		val -= modifier;

	return max(val, minimum);
}

static void grow_ple_window(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	int old = vmx->ple_window;

	vmx->ple_window = __grow_ple_window(old);

	if (vmx->ple_window != old)
		vmx->ple_window_dirty = true;
6069 6070

	trace_kvm_ple_window_grow(vcpu->vcpu_id, vmx->ple_window, old);
6071 6072 6073 6074 6075 6076 6077 6078 6079 6080 6081 6082
}

static void shrink_ple_window(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	int old = vmx->ple_window;

	vmx->ple_window = __shrink_ple_window(old,
	                                      ple_window_shrink, ple_window);

	if (vmx->ple_window != old)
		vmx->ple_window_dirty = true;
6083 6084

	trace_kvm_ple_window_shrink(vcpu->vcpu_id, vmx->ple_window, old);
6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 6100 6101
}

/*
 * ple_window_actual_max is computed to be one grow_ple_window() below
 * ple_window_max. (See __grow_ple_window for the reason.)
 * This prevents overflows, because ple_window_max is int.
 * ple_window_max effectively rounded down to a multiple of ple_window_grow in
 * this process.
 * ple_window_max is also prevented from setting vmx->ple_window < ple_window.
 */
static void update_ple_window_actual_max(void)
{
	ple_window_actual_max =
			__shrink_ple_window(max(ple_window_max, ple_window),
			                    ple_window_grow, INT_MIN);
}

6102 6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120
/*
 * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
 */
static void wakeup_handler(void)
{
	struct kvm_vcpu *vcpu;
	int cpu = smp_processor_id();

	spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
	list_for_each_entry(vcpu, &per_cpu(blocked_vcpu_on_cpu, cpu),
			blocked_vcpu_list) {
		struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);

		if (pi_test_on(pi_desc) == 1)
			kvm_vcpu_kick(vcpu);
	}
	spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
}

6121 6122
static __init int hardware_setup(void)
{
6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154
	int r = -ENOMEM, i, msr;

	rdmsrl_safe(MSR_EFER, &host_efer);

	for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i)
		kvm_define_shared_msr(i, vmx_msr_index[i]);

	vmx_io_bitmap_a = (unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_io_bitmap_a)
		return r;

	vmx_io_bitmap_b = (unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_io_bitmap_b)
		goto out;

	vmx_msr_bitmap_legacy = (unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_msr_bitmap_legacy)
		goto out1;

	vmx_msr_bitmap_legacy_x2apic =
				(unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_msr_bitmap_legacy_x2apic)
		goto out2;

	vmx_msr_bitmap_longmode = (unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_msr_bitmap_longmode)
		goto out3;

	vmx_msr_bitmap_longmode_x2apic =
				(unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_msr_bitmap_longmode_x2apic)
		goto out4;
6155 6156 6157 6158 6159 6160 6161 6162

	if (nested) {
		vmx_msr_bitmap_nested =
			(unsigned long *)__get_free_page(GFP_KERNEL);
		if (!vmx_msr_bitmap_nested)
			goto out5;
	}

6163 6164
	vmx_vmread_bitmap = (unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_vmread_bitmap)
6165
		goto out6;
6166 6167 6168

	vmx_vmwrite_bitmap = (unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_vmwrite_bitmap)
6169
		goto out7;
6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184

	memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE);
	memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE);

	/*
	 * Allow direct access to the PC debug port (it is often used for I/O
	 * delays, but the vmexits simply slow things down).
	 */
	memset(vmx_io_bitmap_a, 0xff, PAGE_SIZE);
	clear_bit(0x80, vmx_io_bitmap_a);

	memset(vmx_io_bitmap_b, 0xff, PAGE_SIZE);

	memset(vmx_msr_bitmap_legacy, 0xff, PAGE_SIZE);
	memset(vmx_msr_bitmap_longmode, 0xff, PAGE_SIZE);
6185 6186
	if (nested)
		memset(vmx_msr_bitmap_nested, 0xff, PAGE_SIZE);
6187 6188 6189

	if (setup_vmcs_config(&vmcs_config) < 0) {
		r = -EIO;
6190
		goto out8;
6191
	}
6192 6193 6194 6195 6196 6197 6198 6199 6200 6201 6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215

	if (boot_cpu_has(X86_FEATURE_NX))
		kvm_enable_efer_bits(EFER_NX);

	if (!cpu_has_vmx_vpid())
		enable_vpid = 0;
	if (!cpu_has_vmx_shadow_vmcs())
		enable_shadow_vmcs = 0;
	if (enable_shadow_vmcs)
		init_vmcs_shadow_fields();

	if (!cpu_has_vmx_ept() ||
	    !cpu_has_vmx_ept_4levels()) {
		enable_ept = 0;
		enable_unrestricted_guest = 0;
		enable_ept_ad_bits = 0;
	}

	if (!cpu_has_vmx_ept_ad_bits())
		enable_ept_ad_bits = 0;

	if (!cpu_has_vmx_unrestricted_guest())
		enable_unrestricted_guest = 0;

6216
	if (!cpu_has_vmx_flexpriority())
6217 6218
		flexpriority_enabled = 0;

6219 6220 6221 6222 6223 6224
	/*
	 * set_apic_access_page_addr() is used to reload apic access
	 * page upon invalidation.  No need to do anything if not
	 * using the APIC_ACCESS_ADDR VMCS field.
	 */
	if (!flexpriority_enabled)
6225 6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238
		kvm_x86_ops->set_apic_access_page_addr = NULL;

	if (!cpu_has_vmx_tpr_shadow())
		kvm_x86_ops->update_cr8_intercept = NULL;

	if (enable_ept && !cpu_has_vmx_ept_2m_page())
		kvm_disable_largepages();

	if (!cpu_has_vmx_ple())
		ple_gap = 0;

	if (!cpu_has_vmx_apicv())
		enable_apicv = 0;

6239 6240 6241 6242 6243 6244
	if (cpu_has_vmx_tsc_scaling()) {
		kvm_has_tsc_control = true;
		kvm_max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX;
		kvm_tsc_scaling_ratio_frac_bits = 48;
	}

6245 6246 6247 6248
	if (enable_apicv)
		kvm_x86_ops->update_cr8_intercept = NULL;
	else {
		kvm_x86_ops->hwapic_irr_update = NULL;
6249
		kvm_x86_ops->hwapic_isr_update = NULL;
6250 6251 6252 6253
		kvm_x86_ops->deliver_posted_interrupt = NULL;
		kvm_x86_ops->sync_pir_to_irr = vmx_sync_pir_to_irr_dummy;
	}

6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265
	vmx_disable_intercept_for_msr(MSR_FS_BASE, false);
	vmx_disable_intercept_for_msr(MSR_GS_BASE, false);
	vmx_disable_intercept_for_msr(MSR_KERNEL_GS_BASE, true);
	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_CS, false);
	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_ESP, false);
	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_EIP, false);

	memcpy(vmx_msr_bitmap_legacy_x2apic,
			vmx_msr_bitmap_legacy, PAGE_SIZE);
	memcpy(vmx_msr_bitmap_longmode_x2apic,
			vmx_msr_bitmap_longmode, PAGE_SIZE);

6266 6267
	set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */

6268 6269 6270 6271 6272 6273 6274 6275 6276 6277 6278 6279 6280 6281 6282 6283 6284 6285 6286 6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297
	if (enable_apicv) {
		for (msr = 0x800; msr <= 0x8ff; msr++)
			vmx_disable_intercept_msr_read_x2apic(msr);

		/* According SDM, in x2apic mode, the whole id reg is used.
		 * But in KVM, it only use the highest eight bits. Need to
		 * intercept it */
		vmx_enable_intercept_msr_read_x2apic(0x802);
		/* TMCCT */
		vmx_enable_intercept_msr_read_x2apic(0x839);
		/* TPR */
		vmx_disable_intercept_msr_write_x2apic(0x808);
		/* EOI */
		vmx_disable_intercept_msr_write_x2apic(0x80b);
		/* SELF-IPI */
		vmx_disable_intercept_msr_write_x2apic(0x83f);
	}

	if (enable_ept) {
		kvm_mmu_set_mask_ptes(0ull,
			(enable_ept_ad_bits) ? VMX_EPT_ACCESS_BIT : 0ull,
			(enable_ept_ad_bits) ? VMX_EPT_DIRTY_BIT : 0ull,
			0ull, VMX_EPT_EXECUTABLE_MASK);
		ept_set_mmio_spte_mask();
		kvm_enable_tdp();
	} else
		kvm_disable_tdp();

	update_ple_window_actual_max();

Kai Huang's avatar
Kai Huang committed
6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310 6311
	/*
	 * Only enable PML when hardware supports PML feature, and both EPT
	 * and EPT A/D bit features are enabled -- PML depends on them to work.
	 */
	if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml())
		enable_pml = 0;

	if (!enable_pml) {
		kvm_x86_ops->slot_enable_log_dirty = NULL;
		kvm_x86_ops->slot_disable_log_dirty = NULL;
		kvm_x86_ops->flush_log_dirty = NULL;
		kvm_x86_ops->enable_log_dirty_pt_masked = NULL;
	}

6312 6313
	kvm_set_posted_intr_wakeup_handler(wakeup_handler);

6314
	return alloc_kvm_area();
6315

6316
out8:
6317
	free_page((unsigned long)vmx_vmwrite_bitmap);
6318
out7:
6319
	free_page((unsigned long)vmx_vmread_bitmap);
6320 6321 6322
out6:
	if (nested)
		free_page((unsigned long)vmx_msr_bitmap_nested);
6323 6324 6325 6326 6327 6328 6329 6330 6331 6332 6333 6334 6335 6336
out5:
	free_page((unsigned long)vmx_msr_bitmap_longmode_x2apic);
out4:
	free_page((unsigned long)vmx_msr_bitmap_longmode);
out3:
	free_page((unsigned long)vmx_msr_bitmap_legacy_x2apic);
out2:
	free_page((unsigned long)vmx_msr_bitmap_legacy);
out1:
	free_page((unsigned long)vmx_io_bitmap_b);
out:
	free_page((unsigned long)vmx_io_bitmap_a);

    return r;
6337 6338 6339 6340
}

static __exit void hardware_unsetup(void)
{
6341 6342 6343 6344 6345 6346 6347 6348
	free_page((unsigned long)vmx_msr_bitmap_legacy_x2apic);
	free_page((unsigned long)vmx_msr_bitmap_longmode_x2apic);
	free_page((unsigned long)vmx_msr_bitmap_legacy);
	free_page((unsigned long)vmx_msr_bitmap_longmode);
	free_page((unsigned long)vmx_io_bitmap_b);
	free_page((unsigned long)vmx_io_bitmap_a);
	free_page((unsigned long)vmx_vmwrite_bitmap);
	free_page((unsigned long)vmx_vmread_bitmap);
6349 6350
	if (nested)
		free_page((unsigned long)vmx_msr_bitmap_nested);
6351

6352 6353 6354
	free_kvm_area();
}

6355 6356 6357 6358
/*
 * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE
 * exiting, so only get here on cpu with PAUSE-Loop-Exiting.
 */
6359
static int handle_pause(struct kvm_vcpu *vcpu)
6360
{
6361 6362 6363
	if (ple_gap)
		grow_ple_window(vcpu);

6364 6365 6366 6367 6368 6369
	skip_emulated_instruction(vcpu);
	kvm_vcpu_on_spin(vcpu);

	return 1;
}

6370
static int handle_nop(struct kvm_vcpu *vcpu)
6371
{
6372
	skip_emulated_instruction(vcpu);
6373 6374 6375
	return 1;
}

6376 6377 6378 6379 6380 6381
static int handle_mwait(struct kvm_vcpu *vcpu)
{
	printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
	return handle_nop(vcpu);
}

6382 6383 6384 6385 6386
static int handle_monitor_trap(struct kvm_vcpu *vcpu)
{
	return 1;
}

6387 6388 6389 6390 6391 6392
static int handle_monitor(struct kvm_vcpu *vcpu)
{
	printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
	return handle_nop(vcpu);
}

6393 6394 6395 6396 6397 6398 6399 6400 6401 6402 6403 6404 6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415 6416 6417 6418 6419 6420 6421 6422 6423 6424 6425
/*
 * To run an L2 guest, we need a vmcs02 based on the L1-specified vmcs12.
 * We could reuse a single VMCS for all the L2 guests, but we also want the
 * option to allocate a separate vmcs02 for each separate loaded vmcs12 - this
 * allows keeping them loaded on the processor, and in the future will allow
 * optimizations where prepare_vmcs02 doesn't need to set all the fields on
 * every entry if they never change.
 * So we keep, in vmx->nested.vmcs02_pool, a cache of size VMCS02_POOL_SIZE
 * (>=0) with a vmcs02 for each recently loaded vmcs12s, most recent first.
 *
 * The following functions allocate and free a vmcs02 in this pool.
 */

/* Get a VMCS from the pool to use as vmcs02 for the current vmcs12. */
static struct loaded_vmcs *nested_get_current_vmcs02(struct vcpu_vmx *vmx)
{
	struct vmcs02_list *item;
	list_for_each_entry(item, &vmx->nested.vmcs02_pool, list)
		if (item->vmptr == vmx->nested.current_vmptr) {
			list_move(&item->list, &vmx->nested.vmcs02_pool);
			return &item->vmcs02;
		}

	if (vmx->nested.vmcs02_num >= max(VMCS02_POOL_SIZE, 1)) {
		/* Recycle the least recently used VMCS. */
		item = list_entry(vmx->nested.vmcs02_pool.prev,
			struct vmcs02_list, list);
		item->vmptr = vmx->nested.current_vmptr;
		list_move(&item->list, &vmx->nested.vmcs02_pool);
		return &item->vmcs02;
	}

	/* Create a new VMCS */
6426
	item = kmalloc(sizeof(struct vmcs02_list), GFP_KERNEL);
6427 6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456
	if (!item)
		return NULL;
	item->vmcs02.vmcs = alloc_vmcs();
	if (!item->vmcs02.vmcs) {
		kfree(item);
		return NULL;
	}
	loaded_vmcs_init(&item->vmcs02);
	item->vmptr = vmx->nested.current_vmptr;
	list_add(&(item->list), &(vmx->nested.vmcs02_pool));
	vmx->nested.vmcs02_num++;
	return &item->vmcs02;
}

/* Free and remove from pool a vmcs02 saved for a vmcs12 (if there is one) */
static void nested_free_vmcs02(struct vcpu_vmx *vmx, gpa_t vmptr)
{
	struct vmcs02_list *item;
	list_for_each_entry(item, &vmx->nested.vmcs02_pool, list)
		if (item->vmptr == vmptr) {
			free_loaded_vmcs(&item->vmcs02);
			list_del(&item->list);
			kfree(item);
			vmx->nested.vmcs02_num--;
			return;
		}
}

/*
 * Free all VMCSs saved for this vcpu, except the one pointed by
6457 6458
 * vmx->loaded_vmcs. We must be running L1, so vmx->loaded_vmcs
 * must be &vmx->vmcs01.
6459 6460 6461 6462
 */
static void nested_free_all_saved_vmcss(struct vcpu_vmx *vmx)
{
	struct vmcs02_list *item, *n;
6463 6464

	WARN_ON(vmx->loaded_vmcs != &vmx->vmcs01);
6465
	list_for_each_entry_safe(item, n, &vmx->nested.vmcs02_pool, list) {
6466 6467 6468 6469 6470 6471 6472 6473
		/*
		 * Something will leak if the above WARN triggers.  Better than
		 * a use-after-free.
		 */
		if (vmx->loaded_vmcs == &item->vmcs02)
			continue;

		free_loaded_vmcs(&item->vmcs02);
6474 6475
		list_del(&item->list);
		kfree(item);
6476
		vmx->nested.vmcs02_num--;
6477 6478 6479
	}
}

6480 6481 6482 6483 6484 6485 6486 6487 6488 6489 6490 6491 6492 6493 6494 6495 6496 6497 6498 6499
/*
 * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(),
 * set the success or error code of an emulated VMX instruction, as specified
 * by Vol 2B, VMX Instruction Reference, "Conventions".
 */
static void nested_vmx_succeed(struct kvm_vcpu *vcpu)
{
	vmx_set_rflags(vcpu, vmx_get_rflags(vcpu)
			& ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
			    X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF));
}

static void nested_vmx_failInvalid(struct kvm_vcpu *vcpu)
{
	vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
			& ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
			    X86_EFLAGS_SF | X86_EFLAGS_OF))
			| X86_EFLAGS_CF);
}

Abel Gordon's avatar
Abel Gordon committed
6500
static void nested_vmx_failValid(struct kvm_vcpu *vcpu,
6501 6502 6503 6504 6505 6506 6507 6508 6509 6510 6511 6512 6513 6514 6515 6516 6517 6518 6519 6520
					u32 vm_instruction_error)
{
	if (to_vmx(vcpu)->nested.current_vmptr == -1ull) {
		/*
		 * failValid writes the error number to the current VMCS, which
		 * can't be done there isn't a current VMCS.
		 */
		nested_vmx_failInvalid(vcpu);
		return;
	}
	vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
			& ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
			    X86_EFLAGS_SF | X86_EFLAGS_OF))
			| X86_EFLAGS_ZF);
	get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error;
	/*
	 * We don't need to force a shadow sync because
	 * VM_INSTRUCTION_ERROR is not shadowed
	 */
}
Abel Gordon's avatar
Abel Gordon committed
6521

6522 6523 6524 6525 6526 6527 6528
static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator)
{
	/* TODO: not to reset guest simply here. */
	kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
	pr_warn("kvm: nested vmx abort, indicator %d\n", indicator);
}

6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539 6540
static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer)
{
	struct vcpu_vmx *vmx =
		container_of(timer, struct vcpu_vmx, nested.preemption_timer);

	vmx->nested.preemption_timer_expired = true;
	kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu);
	kvm_vcpu_kick(&vmx->vcpu);

	return HRTIMER_NORESTART;
}

6541 6542 6543 6544 6545 6546 6547 6548
/*
 * Decode the memory-address operand of a vmx instruction, as recorded on an
 * exit caused by such an instruction (run by a guest hypervisor).
 * On success, returns 0. When the operand is invalid, returns 1 and throws
 * #UD or #GP.
 */
static int get_vmx_mem_address(struct kvm_vcpu *vcpu,
				 unsigned long exit_qualification,
6549
				 u32 vmx_instruction_info, bool wr, gva_t *ret)
6550
{
6551 6552 6553 6554
	gva_t off;
	bool exn;
	struct kvm_segment s;

6555 6556 6557 6558 6559 6560 6561 6562 6563 6564 6565 6566 6567 6568 6569 6570 6571 6572 6573 6574 6575 6576 6577 6578
	/*
	 * According to Vol. 3B, "Information for VM Exits Due to Instruction
	 * Execution", on an exit, vmx_instruction_info holds most of the
	 * addressing components of the operand. Only the displacement part
	 * is put in exit_qualification (see 3B, "Basic VM-Exit Information").
	 * For how an actual address is calculated from all these components,
	 * refer to Vol. 1, "Operand Addressing".
	 */
	int  scaling = vmx_instruction_info & 3;
	int  addr_size = (vmx_instruction_info >> 7) & 7;
	bool is_reg = vmx_instruction_info & (1u << 10);
	int  seg_reg = (vmx_instruction_info >> 15) & 7;
	int  index_reg = (vmx_instruction_info >> 18) & 0xf;
	bool index_is_valid = !(vmx_instruction_info & (1u << 22));
	int  base_reg       = (vmx_instruction_info >> 23) & 0xf;
	bool base_is_valid  = !(vmx_instruction_info & (1u << 27));

	if (is_reg) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
	}

	/* Addr = segment_base + offset */
	/* offset = base + [index * scale] + displacement */
6579
	off = exit_qualification; /* holds the displacement */
6580
	if (base_is_valid)
6581
		off += kvm_register_read(vcpu, base_reg);
6582
	if (index_is_valid)
6583 6584 6585
		off += kvm_register_read(vcpu, index_reg)<<scaling;
	vmx_get_segment(vcpu, &s, seg_reg);
	*ret = s.base + off;
6586 6587 6588 6589

	if (addr_size == 1) /* 32 bit */
		*ret &= 0xffffffff;

6590 6591
	/* Checks for #GP/#SS exceptions. */
	exn = false;
6592 6593 6594 6595 6596 6597 6598
	if (is_long_mode(vcpu)) {
		/* Long mode: #GP(0)/#SS(0) if the memory address is in a
		 * non-canonical form. This is the only check on the memory
		 * destination for long mode!
		 */
		exn = is_noncanonical_address(*ret);
	} else if (is_protmode(vcpu)) {
6599 6600 6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614
		/* Protected mode: apply checks for segment validity in the
		 * following order:
		 * - segment type check (#GP(0) may be thrown)
		 * - usability check (#GP(0)/#SS(0))
		 * - limit check (#GP(0)/#SS(0))
		 */
		if (wr)
			/* #GP(0) if the destination operand is located in a
			 * read-only data segment or any code segment.
			 */
			exn = ((s.type & 0xa) == 0 || (s.type & 8));
		else
			/* #GP(0) if the source operand is located in an
			 * execute-only code segment
			 */
			exn = ((s.type & 0xa) == 8);
6615 6616 6617 6618
		if (exn) {
			kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
			return 1;
		}
6619 6620 6621 6622 6623 6624 6625 6626 6627 6628 6629 6630 6631 6632 6633 6634
		/* Protected mode: #GP(0)/#SS(0) if the segment is unusable.
		 */
		exn = (s.unusable != 0);
		/* Protected mode: #GP(0)/#SS(0) if the memory
		 * operand is outside the segment limit.
		 */
		exn = exn || (off + sizeof(u64) > s.limit);
	}
	if (exn) {
		kvm_queue_exception_e(vcpu,
				      seg_reg == VCPU_SREG_SS ?
						SS_VECTOR : GP_VECTOR,
				      0);
		return 1;
	}

6635 6636 6637
	return 0;
}

6638 6639 6640 6641 6642
/*
 * This function performs the various checks including
 * - if it's 4KB aligned
 * - No bits beyond the physical address width are set
 * - Returns 0 on success or else 1
6643
 * (Intel SDM Section 30.3)
6644
 */
6645 6646
static int nested_vmx_check_vmptr(struct kvm_vcpu *vcpu, int exit_reason,
				  gpa_t *vmpointer)
6647 6648 6649 6650 6651 6652 6653 6654 6655
{
	gva_t gva;
	gpa_t vmptr;
	struct x86_exception e;
	struct page *page;
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	int maxphyaddr = cpuid_maxphyaddr(vcpu);

	if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
6656
			vmcs_read32(VMX_INSTRUCTION_INFO), false, &gva))
6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676
		return 1;

	if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva, &vmptr,
				sizeof(vmptr), &e)) {
		kvm_inject_page_fault(vcpu, &e);
		return 1;
	}

	switch (exit_reason) {
	case EXIT_REASON_VMON:
		/*
		 * SDM 3: 24.11.5
		 * The first 4 bytes of VMXON region contain the supported
		 * VMCS revision identifier
		 *
		 * Note - IA32_VMX_BASIC[48] will never be 1
		 * for the nested case;
		 * which replaces physical address width with 32
		 *
		 */
6677
		if (!PAGE_ALIGNED(vmptr) || (vmptr >> maxphyaddr)) {
6678 6679 6680 6681 6682 6683
			nested_vmx_failInvalid(vcpu);
			skip_emulated_instruction(vcpu);
			return 1;
		}

		page = nested_get_page(vcpu, vmptr);
6684
		if (page == NULL) {
6685
			nested_vmx_failInvalid(vcpu);
6686 6687 6688 6689
			skip_emulated_instruction(vcpu);
			return 1;
		}
		if (*(u32 *)kmap(page) != VMCS12_REVISION) {
6690
			kunmap(page);
6691 6692
			nested_release_page_clean(page);
			nested_vmx_failInvalid(vcpu);
6693 6694 6695 6696
			skip_emulated_instruction(vcpu);
			return 1;
		}
		kunmap(page);
6697
		nested_release_page_clean(page);
6698 6699
		vmx->nested.vmxon_ptr = vmptr;
		break;
6700
	case EXIT_REASON_VMCLEAR:
6701
		if (!PAGE_ALIGNED(vmptr) || (vmptr >> maxphyaddr)) {
6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714 6715
			nested_vmx_failValid(vcpu,
					     VMXERR_VMCLEAR_INVALID_ADDRESS);
			skip_emulated_instruction(vcpu);
			return 1;
		}

		if (vmptr == vmx->nested.vmxon_ptr) {
			nested_vmx_failValid(vcpu,
					     VMXERR_VMCLEAR_VMXON_POINTER);
			skip_emulated_instruction(vcpu);
			return 1;
		}
		break;
	case EXIT_REASON_VMPTRLD:
6716
		if (!PAGE_ALIGNED(vmptr) || (vmptr >> maxphyaddr)) {
6717 6718 6719 6720 6721
			nested_vmx_failValid(vcpu,
					     VMXERR_VMPTRLD_INVALID_ADDRESS);
			skip_emulated_instruction(vcpu);
			return 1;
		}
6722

6723 6724 6725 6726 6727 6728 6729
		if (vmptr == vmx->nested.vmxon_ptr) {
			nested_vmx_failValid(vcpu,
					     VMXERR_VMCLEAR_VMXON_POINTER);
			skip_emulated_instruction(vcpu);
			return 1;
		}
		break;
6730 6731 6732 6733
	default:
		return 1; /* shouldn't happen */
	}

6734 6735
	if (vmpointer)
		*vmpointer = vmptr;
6736 6737 6738
	return 0;
}

6739 6740 6741 6742 6743 6744 6745 6746 6747 6748 6749 6750
/*
 * Emulate the VMXON instruction.
 * Currently, we just remember that VMX is active, and do not save or even
 * inspect the argument to VMXON (the so-called "VMXON pointer") because we
 * do not currently need to store anything in that guest-allocated memory
 * region. Consequently, VMCLEAR and VMPTRLD also do not verify that the their
 * argument is different from the VMXON pointer (which the spec says they do).
 */
static int handle_vmon(struct kvm_vcpu *vcpu)
{
	struct kvm_segment cs;
	struct vcpu_vmx *vmx = to_vmx(vcpu);
6751
	struct vmcs *shadow_vmcs;
6752 6753
	const u64 VMXON_NEEDED_FEATURES = FEATURE_CONTROL_LOCKED
		| FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
6754 6755 6756 6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773 6774 6775 6776

	/* The Intel VMX Instruction Reference lists a bunch of bits that
	 * are prerequisite to running VMXON, most notably cr4.VMXE must be
	 * set to 1 (see vmx_set_cr4() for when we allow the guest to set this).
	 * Otherwise, we should fail with #UD. We test these now:
	 */
	if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE) ||
	    !kvm_read_cr0_bits(vcpu, X86_CR0_PE) ||
	    (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
	}

	vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
	if (is_long_mode(vcpu) && !cs.l) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
	}

	if (vmx_get_cpl(vcpu)) {
		kvm_inject_gp(vcpu, 0);
		return 1;
	}
6777

6778
	if (nested_vmx_check_vmptr(vcpu, EXIT_REASON_VMON, NULL))
6779 6780
		return 1;

Abel Gordon's avatar
Abel Gordon committed
6781 6782 6783 6784 6785
	if (vmx->nested.vmxon) {
		nested_vmx_failValid(vcpu, VMXERR_VMXON_IN_VMX_ROOT_OPERATION);
		skip_emulated_instruction(vcpu);
		return 1;
	}
6786 6787 6788 6789 6790 6791 6792

	if ((vmx->nested.msr_ia32_feature_control & VMXON_NEEDED_FEATURES)
			!= VMXON_NEEDED_FEATURES) {
		kvm_inject_gp(vcpu, 0);
		return 1;
	}

6793 6794 6795 6796 6797 6798 6799 6800 6801 6802
	if (enable_shadow_vmcs) {
		shadow_vmcs = alloc_vmcs();
		if (!shadow_vmcs)
			return -ENOMEM;
		/* mark vmcs as shadow */
		shadow_vmcs->revision_id |= (1u << 31);
		/* init shadow vmcs */
		vmcs_clear(shadow_vmcs);
		vmx->nested.current_shadow_vmcs = shadow_vmcs;
	}
6803

6804 6805 6806
	INIT_LIST_HEAD(&(vmx->nested.vmcs02_pool));
	vmx->nested.vmcs02_num = 0;

6807 6808 6809 6810
	hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC,
		     HRTIMER_MODE_REL);
	vmx->nested.preemption_timer.function = vmx_preemption_timer_fn;

6811 6812 6813
	vmx->nested.vmxon = true;

	skip_emulated_instruction(vcpu);
6814
	nested_vmx_succeed(vcpu);
6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844 6845 6846 6847
	return 1;
}

/*
 * Intel's VMX Instruction Reference specifies a common set of prerequisites
 * for running VMX instructions (except VMXON, whose prerequisites are
 * slightly different). It also specifies what exception to inject otherwise.
 */
static int nested_vmx_check_permission(struct kvm_vcpu *vcpu)
{
	struct kvm_segment cs;
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	if (!vmx->nested.vmxon) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 0;
	}

	vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
	if ((vmx_get_rflags(vcpu) & X86_EFLAGS_VM) ||
	    (is_long_mode(vcpu) && !cs.l)) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 0;
	}

	if (vmx_get_cpl(vcpu)) {
		kvm_inject_gp(vcpu, 0);
		return 0;
	}

	return 1;
}

Abel Gordon's avatar
Abel Gordon committed
6848 6849
static inline void nested_release_vmcs12(struct vcpu_vmx *vmx)
{
6850 6851 6852 6853 6854 6855 6856
	if (vmx->nested.current_vmptr == -1ull)
		return;

	/* current_vmptr and current_vmcs12 are always set/reset together */
	if (WARN_ON(vmx->nested.current_vmcs12 == NULL))
		return;

6857
	if (enable_shadow_vmcs) {
6858 6859 6860 6861
		/* copy to memory all shadowed fields in case
		   they were modified */
		copy_shadow_to_vmcs12(vmx);
		vmx->nested.sync_shadow_vmcs = false;
6862 6863
		vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL,
				SECONDARY_EXEC_SHADOW_VMCS);
6864
		vmcs_write64(VMCS_LINK_POINTER, -1ull);
6865
	}
6866
	vmx->nested.posted_intr_nv = -1;
Abel Gordon's avatar
Abel Gordon committed
6867 6868
	kunmap(vmx->nested.current_vmcs12_page);
	nested_release_page(vmx->nested.current_vmcs12_page);
6869 6870
	vmx->nested.current_vmptr = -1ull;
	vmx->nested.current_vmcs12 = NULL;
Abel Gordon's avatar
Abel Gordon committed
6871 6872
}

6873 6874 6875 6876 6877 6878 6879 6880
/*
 * Free whatever needs to be freed from vmx->nested when L1 goes down, or
 * just stops using VMX.
 */
static void free_nested(struct vcpu_vmx *vmx)
{
	if (!vmx->nested.vmxon)
		return;
6881

6882
	vmx->nested.vmxon = false;
6883
	free_vpid(vmx->nested.vpid02);
6884
	nested_release_vmcs12(vmx);
Abel Gordon's avatar
Abel Gordon committed
6885 6886
	if (enable_shadow_vmcs)
		free_vmcs(vmx->nested.current_shadow_vmcs);
6887 6888 6889
	/* Unpin physical memory we referred to in current vmcs02 */
	if (vmx->nested.apic_access_page) {
		nested_release_page(vmx->nested.apic_access_page);
6890
		vmx->nested.apic_access_page = NULL;
6891
	}
6892 6893
	if (vmx->nested.virtual_apic_page) {
		nested_release_page(vmx->nested.virtual_apic_page);
6894
		vmx->nested.virtual_apic_page = NULL;
6895
	}
6896 6897 6898 6899 6900 6901
	if (vmx->nested.pi_desc_page) {
		kunmap(vmx->nested.pi_desc_page);
		nested_release_page(vmx->nested.pi_desc_page);
		vmx->nested.pi_desc_page = NULL;
		vmx->nested.pi_desc = NULL;
	}
6902 6903

	nested_free_all_saved_vmcss(vmx);
6904 6905 6906 6907 6908 6909 6910 6911 6912
}

/* Emulate the VMXOFF instruction */
static int handle_vmoff(struct kvm_vcpu *vcpu)
{
	if (!nested_vmx_check_permission(vcpu))
		return 1;
	free_nested(to_vmx(vcpu));
	skip_emulated_instruction(vcpu);
6913
	nested_vmx_succeed(vcpu);
6914 6915 6916
	return 1;
}

Nadav Har'El's avatar
Nadav Har'El committed
6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927
/* Emulate the VMCLEAR instruction */
static int handle_vmclear(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	gpa_t vmptr;
	struct vmcs12 *vmcs12;
	struct page *page;

	if (!nested_vmx_check_permission(vcpu))
		return 1;

6928
	if (nested_vmx_check_vmptr(vcpu, EXIT_REASON_VMCLEAR, &vmptr))
Nadav Har'El's avatar
Nadav Har'El committed
6929 6930
		return 1;

6931
	if (vmptr == vmx->nested.current_vmptr)
Abel Gordon's avatar
Abel Gordon committed
6932
		nested_release_vmcs12(vmx);
Nadav Har'El's avatar
Nadav Har'El committed
6933 6934 6935 6936 6937 6938 6939 6940 6941 6942 6943 6944 6945 6946 6947 6948 6949 6950 6951 6952 6953 6954 6955 6956 6957

	page = nested_get_page(vcpu, vmptr);
	if (page == NULL) {
		/*
		 * For accurate processor emulation, VMCLEAR beyond available
		 * physical memory should do nothing at all. However, it is
		 * possible that a nested vmx bug, not a guest hypervisor bug,
		 * resulted in this case, so let's shut down before doing any
		 * more damage:
		 */
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
		return 1;
	}
	vmcs12 = kmap(page);
	vmcs12->launch_state = 0;
	kunmap(page);
	nested_release_page(page);

	nested_free_vmcs02(vmx, vmptr);

	skip_emulated_instruction(vcpu);
	nested_vmx_succeed(vcpu);
	return 1;
}

6958 6959 6960 6961 6962 6963 6964 6965 6966 6967 6968 6969 6970 6971 6972
static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch);

/* Emulate the VMLAUNCH instruction */
static int handle_vmlaunch(struct kvm_vcpu *vcpu)
{
	return nested_vmx_run(vcpu, true);
}

/* Emulate the VMRESUME instruction */
static int handle_vmresume(struct kvm_vcpu *vcpu)
{

	return nested_vmx_run(vcpu, false);
}

6973 6974 6975 6976 6977 6978 6979 6980 6981 6982 6983 6984 6985 6986 6987 6988 6989 6990 6991 6992 6993 6994 6995 6996 6997 6998
enum vmcs_field_type {
	VMCS_FIELD_TYPE_U16 = 0,
	VMCS_FIELD_TYPE_U64 = 1,
	VMCS_FIELD_TYPE_U32 = 2,
	VMCS_FIELD_TYPE_NATURAL_WIDTH = 3
};

static inline int vmcs_field_type(unsigned long field)
{
	if (0x1 & field)	/* the *_HIGH fields are all 32 bit */
		return VMCS_FIELD_TYPE_U32;
	return (field >> 13) & 0x3 ;
}

static inline int vmcs_field_readonly(unsigned long field)
{
	return (((field >> 10) & 0x3) == 1);
}

/*
 * Read a vmcs12 field. Since these can have varying lengths and we return
 * one type, we chose the biggest type (u64) and zero-extend the return value
 * to that size. Note that the caller, handle_vmread, might need to use only
 * some of the bits we return here (e.g., on 32-bit guests, only 32 bits of
 * 64-bit fields are to be returned).
 */
6999 7000
static inline int vmcs12_read_any(struct kvm_vcpu *vcpu,
				  unsigned long field, u64 *ret)
7001 7002 7003 7004 7005
{
	short offset = vmcs_field_to_offset(field);
	char *p;

	if (offset < 0)
7006
		return offset;
7007 7008 7009 7010 7011 7012

	p = ((char *)(get_vmcs12(vcpu))) + offset;

	switch (vmcs_field_type(field)) {
	case VMCS_FIELD_TYPE_NATURAL_WIDTH:
		*ret = *((natural_width *)p);
7013
		return 0;
7014 7015
	case VMCS_FIELD_TYPE_U16:
		*ret = *((u16 *)p);
7016
		return 0;
7017 7018
	case VMCS_FIELD_TYPE_U32:
		*ret = *((u32 *)p);
7019
		return 0;
7020 7021
	case VMCS_FIELD_TYPE_U64:
		*ret = *((u64 *)p);
7022
		return 0;
7023
	default:
7024 7025
		WARN_ON(1);
		return -ENOENT;
7026 7027 7028
	}
}

7029

7030 7031
static inline int vmcs12_write_any(struct kvm_vcpu *vcpu,
				   unsigned long field, u64 field_value){
7032 7033 7034
	short offset = vmcs_field_to_offset(field);
	char *p = ((char *) get_vmcs12(vcpu)) + offset;
	if (offset < 0)
7035
		return offset;
7036 7037 7038 7039

	switch (vmcs_field_type(field)) {
	case VMCS_FIELD_TYPE_U16:
		*(u16 *)p = field_value;
7040
		return 0;
7041 7042
	case VMCS_FIELD_TYPE_U32:
		*(u32 *)p = field_value;
7043
		return 0;
7044 7045
	case VMCS_FIELD_TYPE_U64:
		*(u64 *)p = field_value;
7046
		return 0;
7047 7048
	case VMCS_FIELD_TYPE_NATURAL_WIDTH:
		*(natural_width *)p = field_value;
7049
		return 0;
7050
	default:
7051 7052
		WARN_ON(1);
		return -ENOENT;
7053 7054 7055 7056
	}

}

7057 7058 7059 7060 7061 7062
static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx)
{
	int i;
	unsigned long field;
	u64 field_value;
	struct vmcs *shadow_vmcs = vmx->nested.current_shadow_vmcs;
7063 7064
	const unsigned long *fields = shadow_read_write_fields;
	const int num_fields = max_shadow_read_write_fields;
7065

7066 7067
	preempt_disable();

7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084
	vmcs_load(shadow_vmcs);

	for (i = 0; i < num_fields; i++) {
		field = fields[i];
		switch (vmcs_field_type(field)) {
		case VMCS_FIELD_TYPE_U16:
			field_value = vmcs_read16(field);
			break;
		case VMCS_FIELD_TYPE_U32:
			field_value = vmcs_read32(field);
			break;
		case VMCS_FIELD_TYPE_U64:
			field_value = vmcs_read64(field);
			break;
		case VMCS_FIELD_TYPE_NATURAL_WIDTH:
			field_value = vmcs_readl(field);
			break;
7085 7086 7087
		default:
			WARN_ON(1);
			continue;
7088 7089 7090 7091 7092 7093
		}
		vmcs12_write_any(&vmx->vcpu, field, field_value);
	}

	vmcs_clear(shadow_vmcs);
	vmcs_load(vmx->loaded_vmcs->vmcs);
7094 7095

	preempt_enable();
7096 7097
}

7098 7099
static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx)
{
7100 7101 7102
	const unsigned long *fields[] = {
		shadow_read_write_fields,
		shadow_read_only_fields
7103
	};
7104
	const int max_fields[] = {
7105 7106 7107 7108 7109 7110 7111 7112 7113 7114
		max_shadow_read_write_fields,
		max_shadow_read_only_fields
	};
	int i, q;
	unsigned long field;
	u64 field_value = 0;
	struct vmcs *shadow_vmcs = vmx->nested.current_shadow_vmcs;

	vmcs_load(shadow_vmcs);

7115
	for (q = 0; q < ARRAY_SIZE(fields); q++) {
7116 7117 7118 7119 7120 7121 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132
		for (i = 0; i < max_fields[q]; i++) {
			field = fields[q][i];
			vmcs12_read_any(&vmx->vcpu, field, &field_value);

			switch (vmcs_field_type(field)) {
			case VMCS_FIELD_TYPE_U16:
				vmcs_write16(field, (u16)field_value);
				break;
			case VMCS_FIELD_TYPE_U32:
				vmcs_write32(field, (u32)field_value);
				break;
			case VMCS_FIELD_TYPE_U64:
				vmcs_write64(field, (u64)field_value);
				break;
			case VMCS_FIELD_TYPE_NATURAL_WIDTH:
				vmcs_writel(field, (long)field_value);
				break;
7133 7134 7135
			default:
				WARN_ON(1);
				break;
7136 7137 7138 7139 7140 7141 7142 7143
			}
		}
	}

	vmcs_clear(shadow_vmcs);
	vmcs_load(vmx->loaded_vmcs->vmcs);
}

7144 7145 7146 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7160 7161 7162 7163 7164 7165 7166 7167 7168 7169 7170 7171
/*
 * VMX instructions which assume a current vmcs12 (i.e., that VMPTRLD was
 * used before) all generate the same failure when it is missing.
 */
static int nested_vmx_check_vmcs12(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	if (vmx->nested.current_vmptr == -1ull) {
		nested_vmx_failInvalid(vcpu);
		skip_emulated_instruction(vcpu);
		return 0;
	}
	return 1;
}

static int handle_vmread(struct kvm_vcpu *vcpu)
{
	unsigned long field;
	u64 field_value;
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
	gva_t gva = 0;

	if (!nested_vmx_check_permission(vcpu) ||
	    !nested_vmx_check_vmcs12(vcpu))
		return 1;

	/* Decode instruction info and find the field to read */
7172
	field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
7173
	/* Read the field, zero-extended to a u64 field_value */
7174
	if (vmcs12_read_any(vcpu, field, &field_value) < 0) {
7175 7176 7177 7178 7179 7180 7181 7182 7183 7184
		nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
		skip_emulated_instruction(vcpu);
		return 1;
	}
	/*
	 * Now copy part of this value to register or memory, as requested.
	 * Note that the number of bits actually copied is 32 or 64 depending
	 * on the guest's mode (32 or 64 bit), not on the given field's length.
	 */
	if (vmx_instruction_info & (1u << 10)) {
7185
		kvm_register_writel(vcpu, (((vmx_instruction_info) >> 3) & 0xf),
7186 7187 7188
			field_value);
	} else {
		if (get_vmx_mem_address(vcpu, exit_qualification,
7189
				vmx_instruction_info, true, &gva))
7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202 7203 7204 7205 7206 7207 7208 7209 7210 7211 7212 7213 7214 7215 7216 7217 7218 7219 7220 7221
			return 1;
		/* _system ok, as nested_vmx_check_permission verified cpl=0 */
		kvm_write_guest_virt_system(&vcpu->arch.emulate_ctxt, gva,
			     &field_value, (is_long_mode(vcpu) ? 8 : 4), NULL);
	}

	nested_vmx_succeed(vcpu);
	skip_emulated_instruction(vcpu);
	return 1;
}


static int handle_vmwrite(struct kvm_vcpu *vcpu)
{
	unsigned long field;
	gva_t gva;
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
	/* The value to write might be 32 or 64 bits, depending on L1's long
	 * mode, and eventually we need to write that into a field of several
	 * possible lengths. The code below first zero-extends the value to 64
	 * bit (field_value), and then copies only the approriate number of
	 * bits into the vmcs12 field.
	 */
	u64 field_value = 0;
	struct x86_exception e;

	if (!nested_vmx_check_permission(vcpu) ||
	    !nested_vmx_check_vmcs12(vcpu))
		return 1;

	if (vmx_instruction_info & (1u << 10))
7222
		field_value = kvm_register_readl(vcpu,
7223 7224 7225
			(((vmx_instruction_info) >> 3) & 0xf));
	else {
		if (get_vmx_mem_address(vcpu, exit_qualification,
7226
				vmx_instruction_info, false, &gva))
7227 7228
			return 1;
		if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva,
7229
			   &field_value, (is_64_bit_mode(vcpu) ? 8 : 4), &e)) {
7230 7231 7232 7233 7234 7235
			kvm_inject_page_fault(vcpu, &e);
			return 1;
		}
	}


7236
	field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
7237 7238 7239 7240 7241 7242 7243
	if (vmcs_field_readonly(field)) {
		nested_vmx_failValid(vcpu,
			VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT);
		skip_emulated_instruction(vcpu);
		return 1;
	}

7244
	if (vmcs12_write_any(vcpu, field, field_value) < 0) {
7245 7246 7247 7248 7249 7250 7251 7252 7253 7254
		nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
		skip_emulated_instruction(vcpu);
		return 1;
	}

	nested_vmx_succeed(vcpu);
	skip_emulated_instruction(vcpu);
	return 1;
}

Nadav Har'El's avatar
Nadav Har'El committed
7255 7256 7257 7258 7259 7260 7261 7262 7263
/* Emulate the VMPTRLD instruction */
static int handle_vmptrld(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	gpa_t vmptr;

	if (!nested_vmx_check_permission(vcpu))
		return 1;

7264
	if (nested_vmx_check_vmptr(vcpu, EXIT_REASON_VMPTRLD, &vmptr))
Nadav Har'El's avatar
Nadav Har'El committed
7265 7266 7267 7268 7269 7270 7271 7272 7273 7274 7275 7276 7277 7278 7279 7280 7281 7282 7283 7284 7285
		return 1;

	if (vmx->nested.current_vmptr != vmptr) {
		struct vmcs12 *new_vmcs12;
		struct page *page;
		page = nested_get_page(vcpu, vmptr);
		if (page == NULL) {
			nested_vmx_failInvalid(vcpu);
			skip_emulated_instruction(vcpu);
			return 1;
		}
		new_vmcs12 = kmap(page);
		if (new_vmcs12->revision_id != VMCS12_REVISION) {
			kunmap(page);
			nested_release_page_clean(page);
			nested_vmx_failValid(vcpu,
				VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
			skip_emulated_instruction(vcpu);
			return 1;
		}

7286
		nested_release_vmcs12(vmx);
Nadav Har'El's avatar
Nadav Har'El committed
7287 7288 7289
		vmx->nested.current_vmptr = vmptr;
		vmx->nested.current_vmcs12 = new_vmcs12;
		vmx->nested.current_vmcs12_page = page;
7290
		if (enable_shadow_vmcs) {
7291 7292
			vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,
				      SECONDARY_EXEC_SHADOW_VMCS);
7293 7294
			vmcs_write64(VMCS_LINK_POINTER,
				     __pa(vmx->nested.current_shadow_vmcs));
7295 7296
			vmx->nested.sync_shadow_vmcs = true;
		}
Nadav Har'El's avatar
Nadav Har'El committed
7297 7298 7299 7300 7301 7302 7303
	}

	nested_vmx_succeed(vcpu);
	skip_emulated_instruction(vcpu);
	return 1;
}

Nadav Har'El's avatar
Nadav Har'El committed
7304 7305 7306 7307 7308 7309 7310 7311 7312 7313 7314 7315
/* Emulate the VMPTRST instruction */
static int handle_vmptrst(struct kvm_vcpu *vcpu)
{
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
	gva_t vmcs_gva;
	struct x86_exception e;

	if (!nested_vmx_check_permission(vcpu))
		return 1;

	if (get_vmx_mem_address(vcpu, exit_qualification,
7316
			vmx_instruction_info, true, &vmcs_gva))
Nadav Har'El's avatar
Nadav Har'El committed
7317 7318 7319 7320 7321 7322 7323 7324 7325 7326 7327 7328 7329
		return 1;
	/* ok to use *_system, as nested_vmx_check_permission verified cpl=0 */
	if (kvm_write_guest_virt_system(&vcpu->arch.emulate_ctxt, vmcs_gva,
				 (void *)&to_vmx(vcpu)->nested.current_vmptr,
				 sizeof(u64), &e)) {
		kvm_inject_page_fault(vcpu, &e);
		return 1;
	}
	nested_vmx_succeed(vcpu);
	skip_emulated_instruction(vcpu);
	return 1;
}

Nadav Har'El's avatar
Nadav Har'El committed
7330 7331 7332
/* Emulate the INVEPT instruction */
static int handle_invept(struct kvm_vcpu *vcpu)
{
7333
	struct vcpu_vmx *vmx = to_vmx(vcpu);
Nadav Har'El's avatar
Nadav Har'El committed
7334 7335 7336 7337 7338 7339 7340 7341
	u32 vmx_instruction_info, types;
	unsigned long type;
	gva_t gva;
	struct x86_exception e;
	struct {
		u64 eptp, gpa;
	} operand;

7342 7343 7344
	if (!(vmx->nested.nested_vmx_secondary_ctls_high &
	      SECONDARY_EXEC_ENABLE_EPT) ||
	    !(vmx->nested.nested_vmx_ept_caps & VMX_EPT_INVEPT_BIT)) {
Nadav Har'El's avatar
Nadav Har'El committed
7345 7346 7347 7348 7349 7350 7351 7352 7353 7354 7355 7356 7357
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
	}

	if (!nested_vmx_check_permission(vcpu))
		return 1;

	if (!kvm_read_cr0_bits(vcpu, X86_CR0_PE)) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
	}

	vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
7358
	type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
Nadav Har'El's avatar
Nadav Har'El committed
7359

7360
	types = (vmx->nested.nested_vmx_ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6;
Nadav Har'El's avatar
Nadav Har'El committed
7361 7362 7363 7364

	if (!(types & (1UL << type))) {
		nested_vmx_failValid(vcpu,
				VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
7365
		skip_emulated_instruction(vcpu);
Nadav Har'El's avatar
Nadav Har'El committed
7366 7367 7368 7369 7370 7371 7372
		return 1;
	}

	/* According to the Intel VMX instruction reference, the memory
	 * operand is read even if it isn't needed (e.g., for type==global)
	 */
	if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
7373
			vmx_instruction_info, false, &gva))
Nadav Har'El's avatar
Nadav Har'El committed
7374 7375 7376 7377 7378 7379 7380 7381 7382 7383
		return 1;
	if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva, &operand,
				sizeof(operand), &e)) {
		kvm_inject_page_fault(vcpu, &e);
		return 1;
	}

	switch (type) {
	case VMX_EPT_EXTENT_GLOBAL:
		kvm_mmu_sync_roots(vcpu);
7384
		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
Nadav Har'El's avatar
Nadav Har'El committed
7385 7386 7387
		nested_vmx_succeed(vcpu);
		break;
	default:
7388
		/* Trap single context invalidation invept calls */
Nadav Har'El's avatar
Nadav Har'El committed
7389 7390 7391 7392 7393 7394 7395 7396
		BUG_ON(1);
		break;
	}

	skip_emulated_instruction(vcpu);
	return 1;
}

7397 7398
static int handle_invvpid(struct kvm_vcpu *vcpu)
{
7399 7400 7401 7402 7403 7404 7405 7406 7407 7408 7409 7410 7411 7412 7413 7414 7415 7416 7417 7418 7419 7420 7421 7422 7423
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	u32 vmx_instruction_info;
	unsigned long type, types;
	gva_t gva;
	struct x86_exception e;
	int vpid;

	if (!(vmx->nested.nested_vmx_secondary_ctls_high &
	      SECONDARY_EXEC_ENABLE_VPID) ||
			!(vmx->nested.nested_vmx_vpid_caps & VMX_VPID_INVVPID_BIT)) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
	}

	if (!nested_vmx_check_permission(vcpu))
		return 1;

	vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
	type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);

	types = (vmx->nested.nested_vmx_vpid_caps >> 8) & 0x7;

	if (!(types & (1UL << type))) {
		nested_vmx_failValid(vcpu,
			VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
7424
		skip_emulated_instruction(vcpu);
7425 7426 7427 7428 7429 7430 7431 7432 7433 7434 7435 7436 7437 7438 7439 7440
		return 1;
	}

	/* according to the intel vmx instruction reference, the memory
	 * operand is read even if it isn't needed (e.g., for type==global)
	 */
	if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
			vmx_instruction_info, false, &gva))
		return 1;
	if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva, &vpid,
				sizeof(u32), &e)) {
		kvm_inject_page_fault(vcpu, &e);
		return 1;
	}

	switch (type) {
7441 7442 7443 7444 7445
	case VMX_VPID_EXTENT_SINGLE_CONTEXT:
		/*
		 * Old versions of KVM use the single-context version so we
		 * have to support it; just treat it the same as all-context.
		 */
7446
	case VMX_VPID_EXTENT_ALL_CONTEXT:
7447
		__vmx_flush_tlb(vcpu, to_vmx(vcpu)->nested.vpid02);
7448 7449 7450
		nested_vmx_succeed(vcpu);
		break;
	default:
7451
		/* Trap individual address invalidation invvpid calls */
7452 7453 7454 7455 7456
		BUG_ON(1);
		break;
	}

	skip_emulated_instruction(vcpu);
7457 7458 7459
	return 1;
}

Kai Huang's avatar
Kai Huang committed
7460 7461 7462 7463 7464 7465 7466 7467 7468 7469 7470 7471 7472 7473 7474 7475 7476 7477 7478 7479 7480 7481 7482 7483 7484
static int handle_pml_full(struct kvm_vcpu *vcpu)
{
	unsigned long exit_qualification;

	trace_kvm_pml_full(vcpu->vcpu_id);

	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);

	/*
	 * PML buffer FULL happened while executing iret from NMI,
	 * "blocked by NMI" bit has to be set before next VM entry.
	 */
	if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
			cpu_has_virtual_nmis() &&
			(exit_qualification & INTR_INFO_UNBLOCK_NMI))
		vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
				GUEST_INTR_STATE_NMI);

	/*
	 * PML buffer already flushed at beginning of VMEXIT. Nothing to do
	 * here.., and there's no userspace involvement needed for PML.
	 */
	return 1;
}

7485 7486 7487 7488 7489 7490 7491
static int handle_pcommit(struct kvm_vcpu *vcpu)
{
	/* we never catch pcommit instruct for L1 guest. */
	WARN_ON(1);
	return 1;
}

7492 7493 7494 7495 7496
/*
 * The exit handlers return 1 if the exit was handled fully and guest execution
 * may resume.  Otherwise they set the kvm_run parameter to indicate what needs
 * to be done to userspace and return 0.
 */
7497
static int (*const kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
7498 7499
	[EXIT_REASON_EXCEPTION_NMI]           = handle_exception,
	[EXIT_REASON_EXTERNAL_INTERRUPT]      = handle_external_interrupt,
7500
	[EXIT_REASON_TRIPLE_FAULT]            = handle_triple_fault,
7501
	[EXIT_REASON_NMI_WINDOW]	      = handle_nmi_window,
7502 7503 7504 7505 7506 7507 7508 7509
	[EXIT_REASON_IO_INSTRUCTION]          = handle_io,
	[EXIT_REASON_CR_ACCESS]               = handle_cr,
	[EXIT_REASON_DR_ACCESS]               = handle_dr,
	[EXIT_REASON_CPUID]                   = handle_cpuid,
	[EXIT_REASON_MSR_READ]                = handle_rdmsr,
	[EXIT_REASON_MSR_WRITE]               = handle_wrmsr,
	[EXIT_REASON_PENDING_INTERRUPT]       = handle_interrupt_window,
	[EXIT_REASON_HLT]                     = handle_halt,
7510
	[EXIT_REASON_INVD]		      = handle_invd,
Marcelo Tosatti's avatar
Marcelo Tosatti committed
7511
	[EXIT_REASON_INVLPG]		      = handle_invlpg,
Avi Kivity's avatar
Avi Kivity committed
7512
	[EXIT_REASON_RDPMC]                   = handle_rdpmc,
7513
	[EXIT_REASON_VMCALL]                  = handle_vmcall,
Nadav Har'El's avatar
Nadav Har'El committed
7514
	[EXIT_REASON_VMCLEAR]	              = handle_vmclear,
7515
	[EXIT_REASON_VMLAUNCH]                = handle_vmlaunch,
Nadav Har'El's avatar
Nadav Har'El committed
7516
	[EXIT_REASON_VMPTRLD]                 = handle_vmptrld,
Nadav Har'El's avatar
Nadav Har'El committed
7517
	[EXIT_REASON_VMPTRST]                 = handle_vmptrst,
7518
	[EXIT_REASON_VMREAD]                  = handle_vmread,
7519
	[EXIT_REASON_VMRESUME]                = handle_vmresume,
7520
	[EXIT_REASON_VMWRITE]                 = handle_vmwrite,
7521 7522
	[EXIT_REASON_VMOFF]                   = handle_vmoff,
	[EXIT_REASON_VMON]                    = handle_vmon,
7523 7524
	[EXIT_REASON_TPR_BELOW_THRESHOLD]     = handle_tpr_below_threshold,
	[EXIT_REASON_APIC_ACCESS]             = handle_apic_access,
7525
	[EXIT_REASON_APIC_WRITE]              = handle_apic_write,
7526
	[EXIT_REASON_EOI_INDUCED]             = handle_apic_eoi_induced,
Eddie Dong's avatar
Eddie Dong committed
7527
	[EXIT_REASON_WBINVD]                  = handle_wbinvd,
7528
	[EXIT_REASON_XSETBV]                  = handle_xsetbv,
7529
	[EXIT_REASON_TASK_SWITCH]             = handle_task_switch,
7530
	[EXIT_REASON_MCE_DURING_VMENTRY]      = handle_machine_check,
7531 7532
	[EXIT_REASON_EPT_VIOLATION]	      = handle_ept_violation,
	[EXIT_REASON_EPT_MISCONFIG]           = handle_ept_misconfig,
7533
	[EXIT_REASON_PAUSE_INSTRUCTION]       = handle_pause,
7534
	[EXIT_REASON_MWAIT_INSTRUCTION]	      = handle_mwait,
7535
	[EXIT_REASON_MONITOR_TRAP_FLAG]       = handle_monitor_trap,
7536
	[EXIT_REASON_MONITOR_INSTRUCTION]     = handle_monitor,
Nadav Har'El's avatar
Nadav Har'El committed
7537
	[EXIT_REASON_INVEPT]                  = handle_invept,
7538
	[EXIT_REASON_INVVPID]                 = handle_invvpid,
7539 7540
	[EXIT_REASON_XSAVES]                  = handle_xsaves,
	[EXIT_REASON_XRSTORS]                 = handle_xrstors,
Kai Huang's avatar
Kai Huang committed
7541
	[EXIT_REASON_PML_FULL]		      = handle_pml_full,
7542
	[EXIT_REASON_PCOMMIT]                 = handle_pcommit,
7543 7544 7545
};

static const int kvm_vmx_max_exit_handlers =
7546
	ARRAY_SIZE(kvm_vmx_exit_handlers);
7547

7548 7549 7550 7551 7552 7553 7554 7555 7556 7557
static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu,
				       struct vmcs12 *vmcs12)
{
	unsigned long exit_qualification;
	gpa_t bitmap, last_bitmap;
	unsigned int port;
	int size;
	u8 b;

	if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
7558
		return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING);
7559 7560 7561 7562 7563 7564 7565 7566 7567 7568 7569 7570 7571 7572 7573

	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);

	port = exit_qualification >> 16;
	size = (exit_qualification & 7) + 1;

	last_bitmap = (gpa_t)-1;
	b = -1;

	while (size > 0) {
		if (port < 0x8000)
			bitmap = vmcs12->io_bitmap_a;
		else if (port < 0x10000)
			bitmap = vmcs12->io_bitmap_b;
		else
7574
			return true;
7575 7576 7577
		bitmap += (port & 0x7fff) / 8;

		if (last_bitmap != bitmap)
7578
			if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1))
7579
				return true;
7580
		if (b & (1 << (port & 7)))
7581
			return true;
7582 7583 7584 7585 7586 7587

		port++;
		size--;
		last_bitmap = bitmap;
	}

7588
	return false;
7589 7590
}

7591 7592 7593 7594 7595 7596 7597 7598 7599 7600 7601 7602
/*
 * Return 1 if we should exit from L2 to L1 to handle an MSR access access,
 * rather than handle it ourselves in L0. I.e., check whether L1 expressed
 * disinterest in the current event (read or write a specific MSR) by using an
 * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps.
 */
static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
	struct vmcs12 *vmcs12, u32 exit_reason)
{
	u32 msr_index = vcpu->arch.regs[VCPU_REGS_RCX];
	gpa_t bitmap;

7603
	if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
7604
		return true;
7605 7606 7607 7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618 7619 7620 7621

	/*
	 * The MSR_BITMAP page is divided into four 1024-byte bitmaps,
	 * for the four combinations of read/write and low/high MSR numbers.
	 * First we need to figure out which of the four to use:
	 */
	bitmap = vmcs12->msr_bitmap;
	if (exit_reason == EXIT_REASON_MSR_WRITE)
		bitmap += 2048;
	if (msr_index >= 0xc0000000) {
		msr_index -= 0xc0000000;
		bitmap += 1024;
	}

	/* Then read the msr_index'th bit from this bitmap: */
	if (msr_index < 1024*8) {
		unsigned char b;
7622
		if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1))
7623
			return true;
7624 7625
		return 1 & (b >> (msr_index & 7));
	} else
7626
		return true; /* let L1 handle the wrong parameter */
7627 7628 7629 7630 7631 7632 7633 7634 7635 7636 7637 7638 7639
}

/*
 * Return 1 if we should exit from L2 to L1 to handle a CR access exit,
 * rather than handle it ourselves in L0. I.e., check if L1 wanted to
 * intercept (via guest_host_mask etc.) the current event.
 */
static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
	struct vmcs12 *vmcs12)
{
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	int cr = exit_qualification & 15;
	int reg = (exit_qualification >> 8) & 15;
7640
	unsigned long val = kvm_register_readl(vcpu, reg);
7641 7642 7643 7644 7645 7646 7647

	switch ((exit_qualification >> 4) & 3) {
	case 0: /* mov to cr */
		switch (cr) {
		case 0:
			if (vmcs12->cr0_guest_host_mask &
			    (val ^ vmcs12->cr0_read_shadow))
7648
				return true;
7649 7650 7651 7652 7653 7654 7655 7656 7657 7658
			break;
		case 3:
			if ((vmcs12->cr3_target_count >= 1 &&
					vmcs12->cr3_target_value0 == val) ||
				(vmcs12->cr3_target_count >= 2 &&
					vmcs12->cr3_target_value1 == val) ||
				(vmcs12->cr3_target_count >= 3 &&
					vmcs12->cr3_target_value2 == val) ||
				(vmcs12->cr3_target_count >= 4 &&
					vmcs12->cr3_target_value3 == val))
7659
				return false;
7660
			if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING))
7661
				return true;
7662 7663 7664 7665
			break;
		case 4:
			if (vmcs12->cr4_guest_host_mask &
			    (vmcs12->cr4_read_shadow ^ val))
7666
				return true;
7667 7668 7669
			break;
		case 8:
			if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING))
7670
				return true;
7671 7672 7673 7674 7675 7676
			break;
		}
		break;
	case 2: /* clts */
		if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) &&
		    (vmcs12->cr0_read_shadow & X86_CR0_TS))
7677
			return true;
7678 7679 7680 7681 7682 7683
		break;
	case 1: /* mov from cr */
		switch (cr) {
		case 3:
			if (vmcs12->cpu_based_vm_exec_control &
			    CPU_BASED_CR3_STORE_EXITING)
7684
				return true;
7685 7686 7687 7688
			break;
		case 8:
			if (vmcs12->cpu_based_vm_exec_control &
			    CPU_BASED_CR8_STORE_EXITING)
7689
				return true;
7690 7691 7692 7693 7694 7695 7696 7697 7698 7699
			break;
		}
		break;
	case 3: /* lmsw */
		/*
		 * lmsw can change bits 1..3 of cr0, and only set bit 0 of
		 * cr0. Other attempted changes are ignored, with no exit.
		 */
		if (vmcs12->cr0_guest_host_mask & 0xe &
		    (val ^ vmcs12->cr0_read_shadow))
7700
			return true;
7701 7702 7703
		if ((vmcs12->cr0_guest_host_mask & 0x1) &&
		    !(vmcs12->cr0_read_shadow & 0x1) &&
		    (val & 0x1))
7704
			return true;
7705 7706
		break;
	}
7707
	return false;
7708 7709 7710 7711 7712 7713 7714 7715 7716 7717 7718 7719
}

/*
 * Return 1 if we should exit from L2 to L1 to handle an exit, or 0 if we
 * should handle it ourselves in L0 (and then continue L2). Only call this
 * when in is_guest_mode (L2).
 */
static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
{
	u32 intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
7720
	u32 exit_reason = vmx->exit_reason;
7721

7722 7723 7724 7725 7726 7727 7728
	trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason,
				vmcs_readl(EXIT_QUALIFICATION),
				vmx->idt_vectoring_info,
				intr_info,
				vmcs_read32(VM_EXIT_INTR_ERROR_CODE),
				KVM_ISA_VMX);

7729
	if (vmx->nested.nested_run_pending)
7730
		return false;
7731 7732

	if (unlikely(vmx->fail)) {
7733 7734
		pr_info_ratelimited("%s failed vm entry %x\n", __func__,
				    vmcs_read32(VM_INSTRUCTION_ERROR));
7735
		return true;
7736 7737 7738 7739
	}

	switch (exit_reason) {
	case EXIT_REASON_EXCEPTION_NMI:
7740
		if (is_nmi(intr_info))
7741
			return false;
7742 7743
		else if (is_page_fault(intr_info))
			return enable_ept;
7744
		else if (is_no_device(intr_info) &&
7745
			 !(vmcs12->guest_cr0 & X86_CR0_TS))
7746
			return false;
7747 7748 7749
		return vmcs12->exception_bitmap &
				(1u << (intr_info & INTR_INFO_VECTOR_MASK));
	case EXIT_REASON_EXTERNAL_INTERRUPT:
7750
		return false;
7751
	case EXIT_REASON_TRIPLE_FAULT:
7752
		return true;
7753
	case EXIT_REASON_PENDING_INTERRUPT:
7754
		return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_INTR_PENDING);
7755
	case EXIT_REASON_NMI_WINDOW:
7756
		return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING);
7757
	case EXIT_REASON_TASK_SWITCH:
7758
		return true;
7759
	case EXIT_REASON_CPUID:
7760
		return true;
7761 7762 7763
	case EXIT_REASON_HLT:
		return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING);
	case EXIT_REASON_INVD:
7764
		return true;
7765 7766 7767 7768
	case EXIT_REASON_INVLPG:
		return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
	case EXIT_REASON_RDPMC:
		return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING);
7769
	case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP:
7770 7771 7772 7773 7774 7775
		return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING);
	case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR:
	case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD:
	case EXIT_REASON_VMPTRST: case EXIT_REASON_VMREAD:
	case EXIT_REASON_VMRESUME: case EXIT_REASON_VMWRITE:
	case EXIT_REASON_VMOFF: case EXIT_REASON_VMON:
7776
	case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID:
7777 7778 7779 7780
		/*
		 * VMX instructions trap unconditionally. This allows L1 to
		 * emulate them for its L2 guest, i.e., allows 3-level nesting!
		 */
7781
		return true;
7782 7783 7784 7785 7786
	case EXIT_REASON_CR_ACCESS:
		return nested_vmx_exit_handled_cr(vcpu, vmcs12);
	case EXIT_REASON_DR_ACCESS:
		return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING);
	case EXIT_REASON_IO_INSTRUCTION:
7787
		return nested_vmx_exit_handled_io(vcpu, vmcs12);
7788 7789 7790 7791
	case EXIT_REASON_MSR_READ:
	case EXIT_REASON_MSR_WRITE:
		return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason);
	case EXIT_REASON_INVALID_STATE:
7792
		return true;
7793 7794
	case EXIT_REASON_MWAIT_INSTRUCTION:
		return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING);
7795 7796
	case EXIT_REASON_MONITOR_TRAP_FLAG:
		return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_TRAP_FLAG);
7797 7798 7799 7800 7801 7802 7803
	case EXIT_REASON_MONITOR_INSTRUCTION:
		return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING);
	case EXIT_REASON_PAUSE_INSTRUCTION:
		return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) ||
			nested_cpu_has2(vmcs12,
				SECONDARY_EXEC_PAUSE_LOOP_EXITING);
	case EXIT_REASON_MCE_DURING_VMENTRY:
7804
		return false;
7805
	case EXIT_REASON_TPR_BELOW_THRESHOLD:
7806
		return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW);
7807 7808 7809
	case EXIT_REASON_APIC_ACCESS:
		return nested_cpu_has2(vmcs12,
			SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
7810
	case EXIT_REASON_APIC_WRITE:
7811 7812
	case EXIT_REASON_EOI_INDUCED:
		/* apic_write and eoi_induced should exit unconditionally. */
7813
		return true;
7814
	case EXIT_REASON_EPT_VIOLATION:
7815 7816 7817 7818 7819 7820
		/*
		 * L0 always deals with the EPT violation. If nested EPT is
		 * used, and the nested mmu code discovers that the address is
		 * missing in the guest EPT table (EPT12), the EPT violation
		 * will be injected with nested_ept_inject_page_fault()
		 */
7821
		return false;
7822
	case EXIT_REASON_EPT_MISCONFIG:
7823 7824 7825 7826 7827 7828
		/*
		 * L2 never uses directly L1's EPT, but rather L0's own EPT
		 * table (shadow on EPT) or a merged EPT table that L0 built
		 * (EPT on EPT). So any problems with the structure of the
		 * table is L0's fault.
		 */
7829
		return false;
7830 7831 7832
	case EXIT_REASON_WBINVD:
		return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING);
	case EXIT_REASON_XSETBV:
7833
		return true;
7834 7835 7836 7837 7838 7839 7840 7841
	case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS:
		/*
		 * This should never happen, since it is not possible to
		 * set XSS to a non-zero value---neither in L1 nor in L2.
		 * If if it were, XSS would have to be checked against
		 * the XSS exit bitmap in vmcs12.
		 */
		return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES);
7842 7843
	case EXIT_REASON_PCOMMIT:
		return nested_cpu_has2(vmcs12, SECONDARY_EXEC_PCOMMIT);
7844 7845 7846
	case EXIT_REASON_PML_FULL:
		/* We don't expose PML support to L1. */
		return false;
7847
	default:
7848
		return true;
7849 7850 7851
	}
}

7852 7853 7854 7855 7856 7857
static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
{
	*info1 = vmcs_readl(EXIT_QUALIFICATION);
	*info2 = vmcs_read32(VM_EXIT_INTR_INFO);
}

7858
static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx)
Kai Huang's avatar
Kai Huang committed
7859
{
7860 7861 7862 7863
	if (vmx->pml_pg) {
		__free_page(vmx->pml_pg);
		vmx->pml_pg = NULL;
	}
Kai Huang's avatar
Kai Huang committed
7864 7865
}

7866
static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu)
Kai Huang's avatar
Kai Huang committed
7867
{
7868
	struct vcpu_vmx *vmx = to_vmx(vcpu);
Kai Huang's avatar
Kai Huang committed
7869 7870 7871 7872 7873 7874 7875 7876 7877 7878 7879 7880 7881 7882 7883 7884 7885 7886 7887 7888 7889
	u64 *pml_buf;
	u16 pml_idx;

	pml_idx = vmcs_read16(GUEST_PML_INDEX);

	/* Do nothing if PML buffer is empty */
	if (pml_idx == (PML_ENTITY_NUM - 1))
		return;

	/* PML index always points to next available PML buffer entity */
	if (pml_idx >= PML_ENTITY_NUM)
		pml_idx = 0;
	else
		pml_idx++;

	pml_buf = page_address(vmx->pml_pg);
	for (; pml_idx < PML_ENTITY_NUM; pml_idx++) {
		u64 gpa;

		gpa = pml_buf[pml_idx];
		WARN_ON(gpa & (PAGE_SIZE - 1));
7890
		kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT);
Kai Huang's avatar
Kai Huang committed
7891 7892 7893 7894 7895 7896 7897 7898 7899 7900 7901 7902 7903 7904 7905 7906 7907 7908 7909 7910 7911 7912 7913 7914
	}

	/* reset PML index */
	vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
}

/*
 * Flush all vcpus' PML buffer and update logged GPAs to dirty_bitmap.
 * Called before reporting dirty_bitmap to userspace.
 */
static void kvm_flush_pml_buffers(struct kvm *kvm)
{
	int i;
	struct kvm_vcpu *vcpu;
	/*
	 * We only need to kick vcpu out of guest mode here, as PML buffer
	 * is flushed at beginning of all VMEXITs, and it's obvious that only
	 * vcpus running in guest are possible to have unflushed GPAs in PML
	 * buffer.
	 */
	kvm_for_each_vcpu(i, vcpu, kvm)
		kvm_vcpu_kick(vcpu);
}

7915 7916 7917
static void vmx_dump_sel(char *name, uint32_t sel)
{
	pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n",
7918
	       name, vmcs_read16(sel),
7919 7920 7921 7922 7923 7924 7925 7926 7927 7928 7929 7930 7931 7932 7933 7934 7935 7936 7937 7938 7939 7940 7941 7942 7943 7944 7945 7946 7947 7948 7949 7950 7951 7952 7953 7954 7955 7956 7957 7958 7959 7960 7961 7962 7963 7964 7965 7966 7967 7968 7969 7970 7971 7972 7973 7974 7975 7976 7977 7978 7979 7980 7981 7982 7983 7984 7985 7986 7987 7988 7989 7990 7991 7992 7993 7994 7995 7996 7997 7998 7999 8000 8001 8002 8003 8004 8005 8006 8007 8008 8009 8010 8011 8012 8013 8014 8015 8016 8017 8018 8019 8020 8021 8022 8023 8024 8025 8026 8027 8028 8029 8030 8031 8032 8033 8034 8035 8036 8037 8038 8039 8040 8041 8042 8043 8044
	       vmcs_read32(sel + GUEST_ES_AR_BYTES - GUEST_ES_SELECTOR),
	       vmcs_read32(sel + GUEST_ES_LIMIT - GUEST_ES_SELECTOR),
	       vmcs_readl(sel + GUEST_ES_BASE - GUEST_ES_SELECTOR));
}

static void vmx_dump_dtsel(char *name, uint32_t limit)
{
	pr_err("%s                           limit=0x%08x, base=0x%016lx\n",
	       name, vmcs_read32(limit),
	       vmcs_readl(limit + GUEST_GDTR_BASE - GUEST_GDTR_LIMIT));
}

static void dump_vmcs(void)
{
	u32 vmentry_ctl = vmcs_read32(VM_ENTRY_CONTROLS);
	u32 vmexit_ctl = vmcs_read32(VM_EXIT_CONTROLS);
	u32 cpu_based_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	u32 pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL);
	u32 secondary_exec_control = 0;
	unsigned long cr4 = vmcs_readl(GUEST_CR4);
	u64 efer = vmcs_readl(GUEST_IA32_EFER);
	int i, n;

	if (cpu_has_secondary_exec_ctrls())
		secondary_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);

	pr_err("*** Guest State ***\n");
	pr_err("CR0: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n",
	       vmcs_readl(GUEST_CR0), vmcs_readl(CR0_READ_SHADOW),
	       vmcs_readl(CR0_GUEST_HOST_MASK));
	pr_err("CR4: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n",
	       cr4, vmcs_readl(CR4_READ_SHADOW), vmcs_readl(CR4_GUEST_HOST_MASK));
	pr_err("CR3 = 0x%016lx\n", vmcs_readl(GUEST_CR3));
	if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT) &&
	    (cr4 & X86_CR4_PAE) && !(efer & EFER_LMA))
	{
		pr_err("PDPTR0 = 0x%016lx  PDPTR1 = 0x%016lx\n",
		       vmcs_readl(GUEST_PDPTR0), vmcs_readl(GUEST_PDPTR1));
		pr_err("PDPTR2 = 0x%016lx  PDPTR3 = 0x%016lx\n",
		       vmcs_readl(GUEST_PDPTR2), vmcs_readl(GUEST_PDPTR3));
	}
	pr_err("RSP = 0x%016lx  RIP = 0x%016lx\n",
	       vmcs_readl(GUEST_RSP), vmcs_readl(GUEST_RIP));
	pr_err("RFLAGS=0x%08lx         DR7 = 0x%016lx\n",
	       vmcs_readl(GUEST_RFLAGS), vmcs_readl(GUEST_DR7));
	pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n",
	       vmcs_readl(GUEST_SYSENTER_ESP),
	       vmcs_read32(GUEST_SYSENTER_CS), vmcs_readl(GUEST_SYSENTER_EIP));
	vmx_dump_sel("CS:  ", GUEST_CS_SELECTOR);
	vmx_dump_sel("DS:  ", GUEST_DS_SELECTOR);
	vmx_dump_sel("SS:  ", GUEST_SS_SELECTOR);
	vmx_dump_sel("ES:  ", GUEST_ES_SELECTOR);
	vmx_dump_sel("FS:  ", GUEST_FS_SELECTOR);
	vmx_dump_sel("GS:  ", GUEST_GS_SELECTOR);
	vmx_dump_dtsel("GDTR:", GUEST_GDTR_LIMIT);
	vmx_dump_sel("LDTR:", GUEST_LDTR_SELECTOR);
	vmx_dump_dtsel("IDTR:", GUEST_IDTR_LIMIT);
	vmx_dump_sel("TR:  ", GUEST_TR_SELECTOR);
	if ((vmexit_ctl & (VM_EXIT_SAVE_IA32_PAT | VM_EXIT_SAVE_IA32_EFER)) ||
	    (vmentry_ctl & (VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_IA32_EFER)))
		pr_err("EFER =     0x%016llx  PAT = 0x%016lx\n",
		       efer, vmcs_readl(GUEST_IA32_PAT));
	pr_err("DebugCtl = 0x%016lx  DebugExceptions = 0x%016lx\n",
	       vmcs_readl(GUEST_IA32_DEBUGCTL),
	       vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS));
	if (vmentry_ctl & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
		pr_err("PerfGlobCtl = 0x%016lx\n",
		       vmcs_readl(GUEST_IA32_PERF_GLOBAL_CTRL));
	if (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS)
		pr_err("BndCfgS = 0x%016lx\n", vmcs_readl(GUEST_BNDCFGS));
	pr_err("Interruptibility = %08x  ActivityState = %08x\n",
	       vmcs_read32(GUEST_INTERRUPTIBILITY_INFO),
	       vmcs_read32(GUEST_ACTIVITY_STATE));
	if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)
		pr_err("InterruptStatus = %04x\n",
		       vmcs_read16(GUEST_INTR_STATUS));

	pr_err("*** Host State ***\n");
	pr_err("RIP = 0x%016lx  RSP = 0x%016lx\n",
	       vmcs_readl(HOST_RIP), vmcs_readl(HOST_RSP));
	pr_err("CS=%04x SS=%04x DS=%04x ES=%04x FS=%04x GS=%04x TR=%04x\n",
	       vmcs_read16(HOST_CS_SELECTOR), vmcs_read16(HOST_SS_SELECTOR),
	       vmcs_read16(HOST_DS_SELECTOR), vmcs_read16(HOST_ES_SELECTOR),
	       vmcs_read16(HOST_FS_SELECTOR), vmcs_read16(HOST_GS_SELECTOR),
	       vmcs_read16(HOST_TR_SELECTOR));
	pr_err("FSBase=%016lx GSBase=%016lx TRBase=%016lx\n",
	       vmcs_readl(HOST_FS_BASE), vmcs_readl(HOST_GS_BASE),
	       vmcs_readl(HOST_TR_BASE));
	pr_err("GDTBase=%016lx IDTBase=%016lx\n",
	       vmcs_readl(HOST_GDTR_BASE), vmcs_readl(HOST_IDTR_BASE));
	pr_err("CR0=%016lx CR3=%016lx CR4=%016lx\n",
	       vmcs_readl(HOST_CR0), vmcs_readl(HOST_CR3),
	       vmcs_readl(HOST_CR4));
	pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n",
	       vmcs_readl(HOST_IA32_SYSENTER_ESP),
	       vmcs_read32(HOST_IA32_SYSENTER_CS),
	       vmcs_readl(HOST_IA32_SYSENTER_EIP));
	if (vmexit_ctl & (VM_EXIT_LOAD_IA32_PAT | VM_EXIT_LOAD_IA32_EFER))
		pr_err("EFER = 0x%016lx  PAT = 0x%016lx\n",
		       vmcs_readl(HOST_IA32_EFER), vmcs_readl(HOST_IA32_PAT));
	if (vmexit_ctl & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
		pr_err("PerfGlobCtl = 0x%016lx\n",
		       vmcs_readl(HOST_IA32_PERF_GLOBAL_CTRL));

	pr_err("*** Control State ***\n");
	pr_err("PinBased=%08x CPUBased=%08x SecondaryExec=%08x\n",
	       pin_based_exec_ctrl, cpu_based_exec_ctrl, secondary_exec_control);
	pr_err("EntryControls=%08x ExitControls=%08x\n", vmentry_ctl, vmexit_ctl);
	pr_err("ExceptionBitmap=%08x PFECmask=%08x PFECmatch=%08x\n",
	       vmcs_read32(EXCEPTION_BITMAP),
	       vmcs_read32(PAGE_FAULT_ERROR_CODE_MASK),
	       vmcs_read32(PAGE_FAULT_ERROR_CODE_MATCH));
	pr_err("VMEntry: intr_info=%08x errcode=%08x ilen=%08x\n",
	       vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
	       vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE),
	       vmcs_read32(VM_ENTRY_INSTRUCTION_LEN));
	pr_err("VMExit: intr_info=%08x errcode=%08x ilen=%08x\n",
	       vmcs_read32(VM_EXIT_INTR_INFO),
	       vmcs_read32(VM_EXIT_INTR_ERROR_CODE),
	       vmcs_read32(VM_EXIT_INSTRUCTION_LEN));
	pr_err("        reason=%08x qualification=%016lx\n",
	       vmcs_read32(VM_EXIT_REASON), vmcs_readl(EXIT_QUALIFICATION));
	pr_err("IDTVectoring: info=%08x errcode=%08x\n",
	       vmcs_read32(IDT_VECTORING_INFO_FIELD),
	       vmcs_read32(IDT_VECTORING_ERROR_CODE));
	pr_err("TSC Offset = 0x%016lx\n", vmcs_readl(TSC_OFFSET));
8045 8046 8047
	if (secondary_exec_control & SECONDARY_EXEC_TSC_SCALING)
		pr_err("TSC Multiplier = 0x%016lx\n",
		       vmcs_readl(TSC_MULTIPLIER));
8048 8049 8050 8051 8052 8053 8054 8055 8056 8057 8058 8059 8060 8061 8062 8063 8064 8065 8066 8067 8068 8069
	if (cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW)
		pr_err("TPR Threshold = 0x%02x\n", vmcs_read32(TPR_THRESHOLD));
	if (pin_based_exec_ctrl & PIN_BASED_POSTED_INTR)
		pr_err("PostedIntrVec = 0x%02x\n", vmcs_read16(POSTED_INTR_NV));
	if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT))
		pr_err("EPT pointer = 0x%016lx\n", vmcs_readl(EPT_POINTER));
	n = vmcs_read32(CR3_TARGET_COUNT);
	for (i = 0; i + 1 < n; i += 4)
		pr_err("CR3 target%u=%016lx target%u=%016lx\n",
		       i, vmcs_readl(CR3_TARGET_VALUE0 + i * 2),
		       i + 1, vmcs_readl(CR3_TARGET_VALUE0 + i * 2 + 2));
	if (i < n)
		pr_err("CR3 target%u=%016lx\n",
		       i, vmcs_readl(CR3_TARGET_VALUE0 + i * 2));
	if (secondary_exec_control & SECONDARY_EXEC_PAUSE_LOOP_EXITING)
		pr_err("PLE Gap=%08x Window=%08x\n",
		       vmcs_read32(PLE_GAP), vmcs_read32(PLE_WINDOW));
	if (secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID)
		pr_err("Virtual processor ID = 0x%04x\n",
		       vmcs_read16(VIRTUAL_PROCESSOR_ID));
}

8070 8071 8072 8073
/*
 * The guest has exited.  See if we can fix it or if we need userspace
 * assistance.
 */
8074
static int vmx_handle_exit(struct kvm_vcpu *vcpu)
8075
{
8076
	struct vcpu_vmx *vmx = to_vmx(vcpu);
8077
	u32 exit_reason = vmx->exit_reason;
8078
	u32 vectoring_info = vmx->idt_vectoring_info;
8079

8080 8081
	trace_kvm_exit(exit_reason, vcpu, KVM_ISA_VMX);

Kai Huang's avatar
Kai Huang committed
8082 8083 8084 8085 8086 8087 8088 8089
	/*
	 * Flush logged GPAs PML buffer, this will make dirty_bitmap more
	 * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before
	 * querying dirty_bitmap, we only need to kick all vcpus out of guest
	 * mode as if vcpus is in root mode, the PML buffer must has been
	 * flushed already.
	 */
	if (enable_pml)
8090
		vmx_flush_pml_buffer(vcpu);
Kai Huang's avatar
Kai Huang committed
8091

8092
	/* If guest state is invalid, start emulating */
8093
	if (vmx->emulation_required)
8094
		return handle_invalid_guest_state(vcpu);
8095

8096
	if (is_guest_mode(vcpu) && nested_vmx_exit_handled(vcpu)) {
8097 8098 8099
		nested_vmx_vmexit(vcpu, exit_reason,
				  vmcs_read32(VM_EXIT_INTR_INFO),
				  vmcs_readl(EXIT_QUALIFICATION));
8100 8101 8102
		return 1;
	}

8103
	if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) {
8104
		dump_vmcs();
8105 8106 8107 8108 8109 8110
		vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
		vcpu->run->fail_entry.hardware_entry_failure_reason
			= exit_reason;
		return 0;
	}

8111
	if (unlikely(vmx->fail)) {
8112 8113
		vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
		vcpu->run->fail_entry.hardware_entry_failure_reason
8114 8115 8116
			= vmcs_read32(VM_INSTRUCTION_ERROR);
		return 0;
	}
8117

8118 8119 8120 8121 8122 8123 8124
	/*
	 * Note:
	 * Do not try to fix EXIT_REASON_EPT_MISCONFIG if it caused by
	 * delivery event since it indicates guest is accessing MMIO.
	 * The vm-exit can be triggered again after return to guest that
	 * will cause infinite loop.
	 */
Mike Day's avatar
Mike Day committed
8125
	if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
8126
			(exit_reason != EXIT_REASON_EXCEPTION_NMI &&
8127
			exit_reason != EXIT_REASON_EPT_VIOLATION &&
8128
			exit_reason != EXIT_REASON_PML_FULL &&
8129 8130 8131 8132 8133 8134 8135 8136
			exit_reason != EXIT_REASON_TASK_SWITCH)) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV;
		vcpu->run->internal.ndata = 2;
		vcpu->run->internal.data[0] = vectoring_info;
		vcpu->run->internal.data[1] = exit_reason;
		return 0;
	}
8137

8138 8139
	if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked &&
	    !(is_guest_mode(vcpu) && nested_cpu_has_virtual_nmis(
Nadav Har'El's avatar
Nadav Har'El committed
8140
					get_vmcs12(vcpu))))) {
8141
		if (vmx_interrupt_allowed(vcpu)) {
8142 8143
			vmx->soft_vnmi_blocked = 0;
		} else if (vmx->vnmi_blocked_time > 1000000000LL &&
8144
			   vcpu->arch.nmi_pending) {
8145 8146 8147 8148 8149 8150 8151 8152 8153 8154 8155 8156 8157
			/*
			 * This CPU don't support us in finding the end of an
			 * NMI-blocked window if the guest runs with IRQs
			 * disabled. So we pull the trigger after 1 s of
			 * futile waiting, but inform the user about this.
			 */
			printk(KERN_WARNING "%s: Breaking out of NMI-blocked "
			       "state on VCPU %d after 1 s timeout\n",
			       __func__, vcpu->vcpu_id);
			vmx->soft_vnmi_blocked = 0;
		}
	}

8158 8159
	if (exit_reason < kvm_vmx_max_exit_handlers
	    && kvm_vmx_exit_handlers[exit_reason])
8160
		return kvm_vmx_exit_handlers[exit_reason](vcpu);
8161
	else {
8162 8163 8164
		WARN_ONCE(1, "vmx: unexpected exit reason 0x%x\n", exit_reason);
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
8165 8166 8167
	}
}

8168
static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
8169
{
8170 8171 8172 8173 8174 8175
	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);

	if (is_guest_mode(vcpu) &&
		nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
		return;

8176
	if (irr == -1 || tpr < irr) {
8177 8178 8179 8180
		vmcs_write32(TPR_THRESHOLD, 0);
		return;
	}

8181
	vmcs_write32(TPR_THRESHOLD, irr);
8182 8183
}

8184 8185 8186 8187
static void vmx_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
{
	u32 sec_exec_control;

8188 8189 8190 8191 8192 8193
	/* Postpone execution until vmcs01 is the current VMCS. */
	if (is_guest_mode(vcpu)) {
		to_vmx(vcpu)->nested.change_vmcs01_virtual_x2apic_mode = true;
		return;
	}

8194 8195 8196 8197
	/*
	 * There is not point to enable virtualize x2apic without enable
	 * apicv
	 */
8198
	if (!cpu_has_vmx_virtualize_x2apic_mode() ||
8199
				!vmx_cpu_uses_apicv(vcpu))
8200 8201
		return;

8202
	if (!cpu_need_tpr_shadow(vcpu))
8203 8204 8205 8206 8207 8208 8209 8210 8211 8212 8213 8214 8215 8216 8217 8218
		return;

	sec_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);

	if (set) {
		sec_exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
		sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
	} else {
		sec_exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
		sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
	}
	vmcs_write32(SECONDARY_VM_EXEC_CONTROL, sec_exec_control);

	vmx_set_msr_bitmap(vcpu);
}

8219 8220 8221 8222 8223 8224 8225 8226 8227 8228 8229 8230 8231 8232 8233 8234 8235 8236 8237 8238 8239 8240 8241
static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu, hpa_t hpa)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	/*
	 * Currently we do not handle the nested case where L2 has an
	 * APIC access page of its own; that page is still pinned.
	 * Hence, we skip the case where the VCPU is in guest mode _and_
	 * L1 prepared an APIC access page for L2.
	 *
	 * For the case where L1 and L2 share the same APIC access page
	 * (flexpriority=Y but SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES clear
	 * in the vmcs12), this function will only update either the vmcs01
	 * or the vmcs02.  If the former, the vmcs02 will be updated by
	 * prepare_vmcs02.  If the latter, the vmcs01 will be updated in
	 * the next L2->L1 exit.
	 */
	if (!is_guest_mode(vcpu) ||
	    !nested_cpu_has2(vmx->nested.current_vmcs12,
			     SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
		vmcs_write64(APIC_ACCESS_ADDR, hpa);
}

8242 8243 8244 8245 8246 8247 8248 8249 8250 8251 8252 8253 8254 8255 8256 8257 8258 8259 8260 8261 8262 8263
static void vmx_hwapic_isr_update(struct kvm *kvm, int isr)
{
	u16 status;
	u8 old;

	if (isr == -1)
		isr = 0;

	status = vmcs_read16(GUEST_INTR_STATUS);
	old = status >> 8;
	if (isr != old) {
		status &= 0xff;
		status |= isr << 8;
		vmcs_write16(GUEST_INTR_STATUS, status);
	}
}

static void vmx_set_rvi(int vector)
{
	u16 status;
	u8 old;

8264 8265 8266
	if (vector == -1)
		vector = 0;

8267 8268 8269 8270 8271 8272 8273 8274 8275 8276 8277
	status = vmcs_read16(GUEST_INTR_STATUS);
	old = (u8)status & 0xff;
	if ((u8)vector != old) {
		status &= ~0xff;
		status |= (u8)vector;
		vmcs_write16(GUEST_INTR_STATUS, status);
	}
}

static void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
{
8278 8279 8280 8281 8282
	if (!is_guest_mode(vcpu)) {
		vmx_set_rvi(max_irr);
		return;
	}

8283 8284 8285
	if (max_irr == -1)
		return;

8286
	/*
8287 8288
	 * In guest mode.  If a vmexit is needed, vmx_check_nested_events
	 * handles it.
8289
	 */
8290
	if (nested_exit_on_intr(vcpu))
8291 8292 8293
		return;

	/*
8294
	 * Else, fall back to pre-APICv interrupt injection since L2
8295 8296 8297 8298 8299 8300 8301
	 * is run without virtual interrupt delivery.
	 */
	if (!kvm_event_needs_reinjection(vcpu) &&
	    vmx_interrupt_allowed(vcpu)) {
		kvm_queue_interrupt(vcpu, max_irr, false);
		vmx_inject_irq(vcpu);
	}
8302 8303
}

8304
static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu)
8305
{
8306
	u64 *eoi_exit_bitmap = vcpu->arch.eoi_exit_bitmap;
8307
	if (!vmx_cpu_uses_apicv(vcpu))
8308 8309
		return;

8310 8311 8312 8313 8314 8315
	vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]);
	vmcs_write64(EOI_EXIT_BITMAP1, eoi_exit_bitmap[1]);
	vmcs_write64(EOI_EXIT_BITMAP2, eoi_exit_bitmap[2]);
	vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
}

8316
static void vmx_complete_atomic_exit(struct vcpu_vmx *vmx)
8317
{
8318 8319 8320 8321 8322 8323
	u32 exit_intr_info;

	if (!(vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY
	      || vmx->exit_reason == EXIT_REASON_EXCEPTION_NMI))
		return;

8324
	vmx->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
8325
	exit_intr_info = vmx->exit_intr_info;
8326 8327

	/* Handle machine checks before interrupts are enabled */
8328
	if (is_machine_check(exit_intr_info))
8329 8330
		kvm_machine_check();

8331
	/* We need to handle NMIs before interrupts are enabled */
8332
	if (is_nmi(exit_intr_info)) {
8333
		kvm_before_handle_nmi(&vmx->vcpu);
8334
		asm("int $2");
8335 8336
		kvm_after_handle_nmi(&vmx->vcpu);
	}
8337
}
8338

8339 8340 8341 8342 8343 8344 8345 8346 8347 8348 8349 8350 8351 8352 8353 8354 8355 8356 8357 8358 8359 8360 8361 8362 8363 8364 8365 8366 8367 8368 8369 8370 8371 8372 8373 8374 8375 8376 8377 8378 8379 8380 8381 8382 8383 8384
static void vmx_handle_external_intr(struct kvm_vcpu *vcpu)
{
	u32 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);

	/*
	 * If external interrupt exists, IF bit is set in rflags/eflags on the
	 * interrupt stack frame, and interrupt will be enabled on a return
	 * from interrupt handler.
	 */
	if ((exit_intr_info & (INTR_INFO_VALID_MASK | INTR_INFO_INTR_TYPE_MASK))
			== (INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR)) {
		unsigned int vector;
		unsigned long entry;
		gate_desc *desc;
		struct vcpu_vmx *vmx = to_vmx(vcpu);
#ifdef CONFIG_X86_64
		unsigned long tmp;
#endif

		vector =  exit_intr_info & INTR_INFO_VECTOR_MASK;
		desc = (gate_desc *)vmx->host_idt_base + vector;
		entry = gate_offset(*desc);
		asm volatile(
#ifdef CONFIG_X86_64
			"mov %%" _ASM_SP ", %[sp]\n\t"
			"and $0xfffffffffffffff0, %%" _ASM_SP "\n\t"
			"push $%c[ss]\n\t"
			"push %[sp]\n\t"
#endif
			"pushf\n\t"
			"orl $0x200, (%%" _ASM_SP ")\n\t"
			__ASM_SIZE(push) " $%c[cs]\n\t"
			"call *%[entry]\n\t"
			:
#ifdef CONFIG_X86_64
			[sp]"=&r"(tmp)
#endif
			:
			[entry]"r"(entry),
			[ss]"i"(__KERNEL_DS),
			[cs]"i"(__KERNEL_CS)
			);
	} else
		local_irq_enable();
}

8385 8386 8387 8388 8389
static bool vmx_has_high_real_mode_segbase(void)
{
	return enable_unrestricted_guest || emulate_invalid_guest_state;
}

8390 8391 8392 8393 8394 8395
static bool vmx_mpx_supported(void)
{
	return (vmcs_config.vmexit_ctrl & VM_EXIT_CLEAR_BNDCFGS) &&
		(vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_BNDCFGS);
}

8396 8397 8398 8399 8400 8401
static bool vmx_xsaves_supported(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_XSAVES;
}

8402 8403
static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx)
{
8404
	u32 exit_intr_info;
8405 8406 8407 8408 8409
	bool unblock_nmi;
	u8 vector;
	bool idtv_info_valid;

	idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK;
8410

8411
	if (cpu_has_virtual_nmis()) {
8412 8413
		if (vmx->nmi_known_unmasked)
			return;
8414 8415 8416 8417 8418
		/*
		 * Can't use vmx->exit_intr_info since we're not sure what
		 * the exit reason is.
		 */
		exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
8419 8420 8421
		unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
		vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
		/*
8422
		 * SDM 3: 27.7.1.2 (September 2008)
8423 8424
		 * Re-set bit "block by NMI" before VM entry if vmexit caused by
		 * a guest IRET fault.
8425 8426 8427 8428 8429
		 * SDM 3: 23.2.2 (September 2008)
		 * Bit 12 is undefined in any of the following cases:
		 *  If the VM exit sets the valid bit in the IDT-vectoring
		 *   information field.
		 *  If the VM exit is due to a double fault.
8430
		 */
8431 8432
		if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi &&
		    vector != DF_VECTOR && !idtv_info_valid)
8433 8434
			vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
				      GUEST_INTR_STATE_NMI);
8435 8436 8437 8438
		else
			vmx->nmi_known_unmasked =
				!(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO)
				  & GUEST_INTR_STATE_NMI);
8439 8440 8441
	} else if (unlikely(vmx->soft_vnmi_blocked))
		vmx->vnmi_blocked_time +=
			ktime_to_ns(ktime_sub(ktime_get(), vmx->entry_time));
8442 8443
}

8444
static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu,
8445 8446 8447
				      u32 idt_vectoring_info,
				      int instr_len_field,
				      int error_code_field)
8448 8449 8450 8451 8452 8453
{
	u8 vector;
	int type;
	bool idtv_info_valid;

	idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
8454

8455 8456 8457
	vcpu->arch.nmi_injected = false;
	kvm_clear_exception_queue(vcpu);
	kvm_clear_interrupt_queue(vcpu);
8458 8459 8460 8461

	if (!idtv_info_valid)
		return;

8462
	kvm_make_request(KVM_REQ_EVENT, vcpu);
8463

8464 8465
	vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK;
	type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK;
8466

8467
	switch (type) {
8468
	case INTR_TYPE_NMI_INTR:
8469
		vcpu->arch.nmi_injected = true;
8470
		/*
8471
		 * SDM 3: 27.7.1.2 (September 2008)
8472 8473
		 * Clear bit "block by NMI" before VM entry if a NMI
		 * delivery faulted.
8474
		 */
8475
		vmx_set_nmi_mask(vcpu, false);
8476 8477
		break;
	case INTR_TYPE_SOFT_EXCEPTION:
8478
		vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
8479 8480
		/* fall through */
	case INTR_TYPE_HARD_EXCEPTION:
8481
		if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
8482
			u32 err = vmcs_read32(error_code_field);
8483
			kvm_requeue_exception_e(vcpu, vector, err);
8484
		} else
8485
			kvm_requeue_exception(vcpu, vector);
8486
		break;
8487
	case INTR_TYPE_SOFT_INTR:
8488
		vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
8489
		/* fall through */
8490
	case INTR_TYPE_EXT_INTR:
8491
		kvm_queue_interrupt(vcpu, vector, type == INTR_TYPE_SOFT_INTR);
8492 8493 8494
		break;
	default:
		break;
8495
	}
8496 8497
}

8498 8499
static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
{
8500
	__vmx_complete_interrupts(&vmx->vcpu, vmx->idt_vectoring_info,
8501 8502 8503 8504
				  VM_EXIT_INSTRUCTION_LEN,
				  IDT_VECTORING_ERROR_CODE);
}

8505 8506
static void vmx_cancel_injection(struct kvm_vcpu *vcpu)
{
8507
	__vmx_complete_interrupts(vcpu,
8508 8509 8510 8511 8512 8513 8514
				  vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
				  VM_ENTRY_INSTRUCTION_LEN,
				  VM_ENTRY_EXCEPTION_ERROR_CODE);

	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
}

8515 8516 8517 8518 8519 8520 8521 8522 8523 8524 8525 8526 8527 8528 8529 8530 8531 8532
static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx)
{
	int i, nr_msrs;
	struct perf_guest_switch_msr *msrs;

	msrs = perf_guest_get_msrs(&nr_msrs);

	if (!msrs)
		return;

	for (i = 0; i < nr_msrs; i++)
		if (msrs[i].host == msrs[i].guest)
			clear_atomic_switch_msr(vmx, msrs[i].msr);
		else
			add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest,
					msrs[i].host);
}

8533
static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
8534
{
8535
	struct vcpu_vmx *vmx = to_vmx(vcpu);
8536
	unsigned long debugctlmsr, cr4;
8537 8538 8539 8540 8541 8542 8543

	/* Record the guest's net vcpu time for enforced NMI injections. */
	if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked))
		vmx->entry_time = ktime_get();

	/* Don't enter VMX if guest state is invalid, let the exit handler
	   start emulation until we arrive back to a valid state */
8544
	if (vmx->emulation_required)
8545 8546
		return;

8547 8548 8549 8550 8551
	if (vmx->ple_window_dirty) {
		vmx->ple_window_dirty = false;
		vmcs_write32(PLE_WINDOW, vmx->ple_window);
	}

8552 8553 8554 8555 8556
	if (vmx->nested.sync_shadow_vmcs) {
		copy_vmcs12_to_shadow(vmx);
		vmx->nested.sync_shadow_vmcs = false;
	}

8557 8558 8559 8560 8561
	if (test_bit(VCPU_REGS_RSP, (unsigned long *)&vcpu->arch.regs_dirty))
		vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);
	if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
		vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);

8562
	cr4 = cr4_read_shadow();
8563 8564 8565 8566 8567
	if (unlikely(cr4 != vmx->host_state.vmcs_host_cr4)) {
		vmcs_writel(HOST_CR4, cr4);
		vmx->host_state.vmcs_host_cr4 = cr4;
	}

8568 8569 8570 8571 8572 8573 8574 8575
	/* When single-stepping over STI and MOV SS, we must clear the
	 * corresponding interruptibility bits in the guest state. Otherwise
	 * vmentry fails as it then expects bit 14 (BS) in pending debug
	 * exceptions being set, but that's not correct for the guest debugging
	 * case. */
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
		vmx_set_interrupt_shadow(vcpu, 0);

8576
	atomic_switch_perf_msrs(vmx);
8577
	debugctlmsr = get_debugctlmsr();
8578

8579
	vmx->__launched = vmx->loaded_vmcs->launched;
8580
	asm(
8581
		/* Store host registers */
Avi Kivity's avatar
Avi Kivity committed
8582 8583 8584 8585
		"push %%" _ASM_DX "; push %%" _ASM_BP ";"
		"push %%" _ASM_CX " \n\t" /* placeholder for guest rcx */
		"push %%" _ASM_CX " \n\t"
		"cmp %%" _ASM_SP ", %c[host_rsp](%0) \n\t"
8586
		"je 1f \n\t"
Avi Kivity's avatar
Avi Kivity committed
8587
		"mov %%" _ASM_SP ", %c[host_rsp](%0) \n\t"
8588
		__ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t"
8589
		"1: \n\t"
8590
		/* Reload cr2 if changed */
Avi Kivity's avatar
Avi Kivity committed
8591 8592 8593
		"mov %c[cr2](%0), %%" _ASM_AX " \n\t"
		"mov %%cr2, %%" _ASM_DX " \n\t"
		"cmp %%" _ASM_AX ", %%" _ASM_DX " \n\t"
8594
		"je 2f \n\t"
Avi Kivity's avatar
Avi Kivity committed
8595
		"mov %%" _ASM_AX", %%cr2 \n\t"
8596
		"2: \n\t"
8597
		/* Check if vmlaunch of vmresume is needed */
8598
		"cmpl $0, %c[launched](%0) \n\t"
8599
		/* Load guest registers.  Don't clobber flags. */
Avi Kivity's avatar
Avi Kivity committed
8600 8601 8602 8603 8604 8605
		"mov %c[rax](%0), %%" _ASM_AX " \n\t"
		"mov %c[rbx](%0), %%" _ASM_BX " \n\t"
		"mov %c[rdx](%0), %%" _ASM_DX " \n\t"
		"mov %c[rsi](%0), %%" _ASM_SI " \n\t"
		"mov %c[rdi](%0), %%" _ASM_DI " \n\t"
		"mov %c[rbp](%0), %%" _ASM_BP " \n\t"
8606
#ifdef CONFIG_X86_64
8607 8608 8609 8610 8611 8612 8613 8614
		"mov %c[r8](%0),  %%r8  \n\t"
		"mov %c[r9](%0),  %%r9  \n\t"
		"mov %c[r10](%0), %%r10 \n\t"
		"mov %c[r11](%0), %%r11 \n\t"
		"mov %c[r12](%0), %%r12 \n\t"
		"mov %c[r13](%0), %%r13 \n\t"
		"mov %c[r14](%0), %%r14 \n\t"
		"mov %c[r15](%0), %%r15 \n\t"
8615
#endif
Avi Kivity's avatar
Avi Kivity committed
8616
		"mov %c[rcx](%0), %%" _ASM_CX " \n\t" /* kills %0 (ecx) */
8617

8618
		/* Enter guest mode */
Avi Kivity's avatar
Avi Kivity committed
8619
		"jne 1f \n\t"
8620
		__ex(ASM_VMX_VMLAUNCH) "\n\t"
Avi Kivity's avatar
Avi Kivity committed
8621 8622 8623
		"jmp 2f \n\t"
		"1: " __ex(ASM_VMX_VMRESUME) "\n\t"
		"2: "
8624
		/* Save guest registers, load host registers, keep flags */
Avi Kivity's avatar
Avi Kivity committed
8625
		"mov %0, %c[wordsize](%%" _ASM_SP ") \n\t"
8626
		"pop %0 \n\t"
Avi Kivity's avatar
Avi Kivity committed
8627 8628 8629 8630 8631 8632 8633
		"mov %%" _ASM_AX ", %c[rax](%0) \n\t"
		"mov %%" _ASM_BX ", %c[rbx](%0) \n\t"
		__ASM_SIZE(pop) " %c[rcx](%0) \n\t"
		"mov %%" _ASM_DX ", %c[rdx](%0) \n\t"
		"mov %%" _ASM_SI ", %c[rsi](%0) \n\t"
		"mov %%" _ASM_DI ", %c[rdi](%0) \n\t"
		"mov %%" _ASM_BP ", %c[rbp](%0) \n\t"
8634
#ifdef CONFIG_X86_64
8635 8636 8637 8638 8639 8640 8641 8642
		"mov %%r8,  %c[r8](%0) \n\t"
		"mov %%r9,  %c[r9](%0) \n\t"
		"mov %%r10, %c[r10](%0) \n\t"
		"mov %%r11, %c[r11](%0) \n\t"
		"mov %%r12, %c[r12](%0) \n\t"
		"mov %%r13, %c[r13](%0) \n\t"
		"mov %%r14, %c[r14](%0) \n\t"
		"mov %%r15, %c[r15](%0) \n\t"
8643
#endif
Avi Kivity's avatar
Avi Kivity committed
8644 8645
		"mov %%cr2, %%" _ASM_AX "   \n\t"
		"mov %%" _ASM_AX ", %c[cr2](%0) \n\t"
8646

Avi Kivity's avatar
Avi Kivity committed
8647
		"pop  %%" _ASM_BP "; pop  %%" _ASM_DX " \n\t"
8648
		"setbe %c[fail](%0) \n\t"
Avi Kivity's avatar
Avi Kivity committed
8649 8650 8651 8652
		".pushsection .rodata \n\t"
		".global vmx_return \n\t"
		"vmx_return: " _ASM_PTR " 2b \n\t"
		".popsection"
8653
	      : : "c"(vmx), "d"((unsigned long)HOST_RSP),
8654
		[launched]"i"(offsetof(struct vcpu_vmx, __launched)),
8655
		[fail]"i"(offsetof(struct vcpu_vmx, fail)),
8656
		[host_rsp]"i"(offsetof(struct vcpu_vmx, host_rsp)),
8657 8658 8659 8660 8661 8662 8663
		[rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])),
		[rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])),
		[rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])),
		[rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])),
		[rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])),
		[rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])),
		[rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])),
8664
#ifdef CONFIG_X86_64
8665 8666 8667 8668 8669 8670 8671 8672
		[r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])),
		[r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])),
		[r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])),
		[r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])),
		[r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])),
		[r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])),
		[r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])),
		[r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])),
8673
#endif
8674 8675
		[cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)),
		[wordsize]"i"(sizeof(ulong))
8676 8677
	      : "cc", "memory"
#ifdef CONFIG_X86_64
Avi Kivity's avatar
Avi Kivity committed
8678
		, "rax", "rbx", "rdi", "rsi"
8679
		, "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
Avi Kivity's avatar
Avi Kivity committed
8680 8681
#else
		, "eax", "ebx", "edi", "esi"
8682 8683
#endif
	      );
8684

8685 8686 8687 8688
	/* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
	if (debugctlmsr)
		update_debugctlmsr(debugctlmsr);

8689 8690 8691 8692 8693 8694 8695 8696 8697 8698 8699 8700 8701
#ifndef CONFIG_X86_64
	/*
	 * The sysexit path does not restore ds/es, so we must set them to
	 * a reasonable value ourselves.
	 *
	 * We can't defer this to vmx_load_host_state() since that function
	 * may be executed in interrupt context, which saves and restore segments
	 * around it, nullifying its effect.
	 */
	loadsegment(ds, __USER_DS);
	loadsegment(es, __USER_DS);
#endif

Avi Kivity's avatar
Avi Kivity committed
8702
	vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)
8703
				  | (1 << VCPU_EXREG_RFLAGS)
8704
				  | (1 << VCPU_EXREG_PDPTR)
8705
				  | (1 << VCPU_EXREG_SEGMENTS)
8706
				  | (1 << VCPU_EXREG_CR3));
8707 8708
	vcpu->arch.regs_dirty = 0;

8709 8710
	vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);

8711
	vmx->loaded_vmcs->launched = 1;
8712

8713 8714
	vmx->exit_reason = vmcs_read32(VM_EXIT_REASON);

8715 8716 8717 8718 8719 8720 8721 8722 8723 8724
	/*
	 * the KVM_REQ_EVENT optimization bit is only on for one entry, and if
	 * we did not inject a still-pending event to L1 now because of
	 * nested_run_pending, we need to re-enable this bit.
	 */
	if (vmx->nested.nested_run_pending)
		kvm_make_request(KVM_REQ_EVENT, vcpu);

	vmx->nested.nested_run_pending = 0;

8725 8726
	vmx_complete_atomic_exit(vmx);
	vmx_recover_nmi_blocking(vmx);
8727
	vmx_complete_interrupts(vmx);
8728 8729
}

8730 8731 8732 8733 8734 8735 8736 8737 8738 8739 8740 8741 8742 8743 8744 8745
static void vmx_load_vmcs01(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	int cpu;

	if (vmx->loaded_vmcs == &vmx->vmcs01)
		return;

	cpu = get_cpu();
	vmx->loaded_vmcs = &vmx->vmcs01;
	vmx_vcpu_put(vcpu);
	vmx_vcpu_load(vcpu, cpu);
	vcpu->cpu = cpu;
	put_cpu();
}

8746 8747 8748 8749 8750 8751 8752 8753 8754 8755 8756 8757 8758 8759 8760 8761
/*
 * Ensure that the current vmcs of the logical processor is the
 * vmcs01 of the vcpu before calling free_nested().
 */
static void vmx_free_vcpu_nested(struct kvm_vcpu *vcpu)
{
       struct vcpu_vmx *vmx = to_vmx(vcpu);
       int r;

       r = vcpu_load(vcpu);
       BUG_ON(r);
       vmx_load_vmcs01(vcpu);
       free_nested(vmx);
       vcpu_put(vcpu);
}

8762 8763
static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
{
8764 8765
	struct vcpu_vmx *vmx = to_vmx(vcpu);

Kai Huang's avatar
Kai Huang committed
8766
	if (enable_pml)
8767
		vmx_destroy_pml_buffer(vmx);
8768
	free_vpid(vmx->vpid);
8769
	leave_guest_mode(vcpu);
8770
	vmx_free_vcpu_nested(vcpu);
8771
	free_loaded_vmcs(vmx->loaded_vmcs);
8772 8773
	kfree(vmx->guest_msrs);
	kvm_vcpu_uninit(vcpu);
8774
	kmem_cache_free(kvm_vcpu_cache, vmx);
8775 8776
}

8777
static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
8778
{
8779
	int err;
8780
	struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
8781
	int cpu;
8782

8783
	if (!vmx)
8784 8785
		return ERR_PTR(-ENOMEM);

8786
	vmx->vpid = allocate_vpid();
8787

8788 8789 8790
	err = kvm_vcpu_init(&vmx->vcpu, kvm, id);
	if (err)
		goto free_vcpu;
8791

8792 8793 8794 8795 8796 8797 8798 8799 8800 8801 8802 8803 8804 8805
	err = -ENOMEM;

	/*
	 * If PML is turned on, failure on enabling PML just results in failure
	 * of creating the vcpu, therefore we can simplify PML logic (by
	 * avoiding dealing with cases, such as enabling PML partially on vcpus
	 * for the guest, etc.
	 */
	if (enable_pml) {
		vmx->pml_pg = alloc_page(GFP_KERNEL | __GFP_ZERO);
		if (!vmx->pml_pg)
			goto uninit_vcpu;
	}

8806
	vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
8807 8808
	BUILD_BUG_ON(ARRAY_SIZE(vmx_msr_index) * sizeof(vmx->guest_msrs[0])
		     > PAGE_SIZE);
8809

8810 8811
	if (!vmx->guest_msrs)
		goto free_pml;
8812

8813 8814 8815
	vmx->loaded_vmcs = &vmx->vmcs01;
	vmx->loaded_vmcs->vmcs = alloc_vmcs();
	if (!vmx->loaded_vmcs->vmcs)
8816
		goto free_msrs;
8817 8818 8819 8820 8821
	if (!vmm_exclusive)
		kvm_cpu_vmxon(__pa(per_cpu(vmxarea, raw_smp_processor_id())));
	loaded_vmcs_init(vmx->loaded_vmcs);
	if (!vmm_exclusive)
		kvm_cpu_vmxoff();
8822

8823 8824
	cpu = get_cpu();
	vmx_vcpu_load(&vmx->vcpu, cpu);
Zachary Amsden's avatar
Zachary Amsden committed
8825
	vmx->vcpu.cpu = cpu;
8826
	err = vmx_vcpu_setup(vmx);
8827
	vmx_vcpu_put(&vmx->vcpu);
8828
	put_cpu();
8829 8830
	if (err)
		goto free_vmcs;
8831
	if (cpu_need_virtualize_apic_accesses(&vmx->vcpu)) {
8832 8833
		err = alloc_apic_access_page(kvm);
		if (err)
8834
			goto free_vmcs;
8835
	}
8836

8837 8838 8839 8840
	if (enable_ept) {
		if (!kvm->arch.ept_identity_map_addr)
			kvm->arch.ept_identity_map_addr =
				VMX_EPT_IDENTITY_PAGETABLE_ADDR;
8841 8842
		err = init_rmode_identity_map(kvm);
		if (err)
8843
			goto free_vmcs;
8844
	}
8845

8846
	if (nested) {
8847
		nested_vmx_setup_ctls_msrs(vmx);
8848 8849
		vmx->nested.vpid02 = allocate_vpid();
	}
8850

8851
	vmx->nested.posted_intr_nv = -1;
8852 8853 8854
	vmx->nested.current_vmptr = -1ull;
	vmx->nested.current_vmcs12 = NULL;

8855 8856 8857
	return &vmx->vcpu;

free_vmcs:
8858
	free_vpid(vmx->nested.vpid02);
8859
	free_loaded_vmcs(vmx->loaded_vmcs);
8860 8861
free_msrs:
	kfree(vmx->guest_msrs);
8862 8863
free_pml:
	vmx_destroy_pml_buffer(vmx);
8864 8865 8866
uninit_vcpu:
	kvm_vcpu_uninit(&vmx->vcpu);
free_vcpu:
8867
	free_vpid(vmx->vpid);
8868
	kmem_cache_free(kvm_vcpu_cache, vmx);
8869
	return ERR_PTR(err);
8870 8871
}

8872 8873 8874 8875 8876 8877 8878 8879 8880 8881 8882 8883 8884 8885
static void __init vmx_check_processor_compat(void *rtn)
{
	struct vmcs_config vmcs_conf;

	*(int *)rtn = 0;
	if (setup_vmcs_config(&vmcs_conf) < 0)
		*(int *)rtn = -EIO;
	if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) {
		printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n",
				smp_processor_id());
		*(int *)rtn = -EIO;
	}
}

8886 8887 8888 8889 8890
static int get_ept_level(void)
{
	return VMX_EPT_DEFAULT_GAW + 1;
}

8891
static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
Sheng Yang's avatar
Sheng Yang committed
8892
{
8893 8894
	u8 cache;
	u64 ipat = 0;
8895

8896
	/* For VT-d and EPT combination
8897
	 * 1. MMIO: always map as UC
8898 8899
	 * 2. EPT with VT-d:
	 *   a. VT-d without snooping control feature: can't guarantee the
8900
	 *	result, try to trust guest.
8901 8902 8903
	 *   b. VT-d with snooping control feature: snooping control feature of
	 *	VT-d engine can guarantee the cache correctness. Just set it
	 *	to WB to keep consistent with host. So the same as item 3.
8904
	 * 3. EPT without VT-d: always map as WB and set IPAT=1 to keep
8905 8906
	 *    consistent with host MTRR
	 */
8907 8908 8909 8910 8911 8912
	if (is_mmio) {
		cache = MTRR_TYPE_UNCACHABLE;
		goto exit;
	}

	if (!kvm_arch_has_noncoherent_dma(vcpu->kvm)) {
8913 8914 8915 8916 8917 8918 8919
		ipat = VMX_EPT_IPAT_BIT;
		cache = MTRR_TYPE_WRBACK;
		goto exit;
	}

	if (kvm_read_cr0(vcpu) & X86_CR0_CD) {
		ipat = VMX_EPT_IPAT_BIT;
8920
		if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
8921 8922 8923
			cache = MTRR_TYPE_WRBACK;
		else
			cache = MTRR_TYPE_UNCACHABLE;
8924 8925 8926
		goto exit;
	}

8927
	cache = kvm_mtrr_get_guest_memory_type(vcpu, gfn);
8928 8929 8930

exit:
	return (cache << VMX_EPT_MT_EPTE_SHIFT) | ipat;
Sheng Yang's avatar
Sheng Yang committed
8931 8932
}

8933
static int vmx_get_lpage_level(void)
8934
{
8935 8936 8937 8938 8939
	if (enable_ept && !cpu_has_vmx_ept_1g_page())
		return PT_DIRECTORY_LEVEL;
	else
		/* For shadow and EPT supported 1GB page */
		return PT_PDPE_LEVEL;
8940 8941
}

8942 8943 8944 8945 8946 8947 8948 8949 8950 8951 8952 8953 8954 8955 8956 8957 8958 8959 8960
static void vmcs_set_secondary_exec_control(u32 new_ctl)
{
	/*
	 * These bits in the secondary execution controls field
	 * are dynamic, the others are mostly based on the hypervisor
	 * architecture and the guest's CPUID.  Do not touch the
	 * dynamic bits.
	 */
	u32 mask =
		SECONDARY_EXEC_SHADOW_VMCS |
		SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
		SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;

	u32 cur_ctl = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);

	vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
		     (new_ctl & ~mask) | (cur_ctl & mask));
}

8961 8962
static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
{
8963 8964
	struct kvm_cpuid_entry2 *best;
	struct vcpu_vmx *vmx = to_vmx(vcpu);
8965
	u32 secondary_exec_ctl = vmx_secondary_exec_control(vmx);
8966 8967

	if (vmx_rdtscp_supported()) {
8968 8969
		bool rdtscp_enabled = guest_cpuid_has_rdtscp(vcpu);
		if (!rdtscp_enabled)
8970
			secondary_exec_ctl &= ~SECONDARY_EXEC_RDTSCP;
8971

8972
		if (nested) {
8973
			if (rdtscp_enabled)
8974 8975 8976 8977 8978 8979
				vmx->nested.nested_vmx_secondary_ctls_high |=
					SECONDARY_EXEC_RDTSCP;
			else
				vmx->nested.nested_vmx_secondary_ctls_high &=
					~SECONDARY_EXEC_RDTSCP;
		}
8980
	}
8981 8982 8983 8984

	/* Exposing INVPCID only when PCID is exposed */
	best = kvm_find_cpuid_entry(vcpu, 0x7, 0);
	if (vmx_invpcid_supported() &&
8985 8986
	    (!best || !(best->ebx & bit(X86_FEATURE_INVPCID)) ||
	    !guest_cpuid_has_pcid(vcpu))) {
8987
		secondary_exec_ctl &= ~SECONDARY_EXEC_ENABLE_INVPCID;
8988

8989
		if (best)
8990
			best->ebx &= ~bit(X86_FEATURE_INVPCID);
8991
	}
8992

8993 8994
	if (cpu_has_secondary_exec_ctrls())
		vmcs_set_secondary_exec_control(secondary_exec_ctl);
8995

8996 8997 8998 8999 9000 9001 9002 9003
	if (static_cpu_has(X86_FEATURE_PCOMMIT) && nested) {
		if (guest_cpuid_has_pcommit(vcpu))
			vmx->nested.nested_vmx_secondary_ctls_high |=
				SECONDARY_EXEC_PCOMMIT;
		else
			vmx->nested.nested_vmx_secondary_ctls_high &=
				~SECONDARY_EXEC_PCOMMIT;
	}
9004 9005
}

9006 9007
static void vmx_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
{
9008 9009
	if (func == 1 && nested)
		entry->ecx |= bit(X86_FEATURE_VMX);
9010 9011
}

9012 9013 9014
static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
		struct x86_exception *fault)
{
9015 9016
	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
	u32 exit_reason;
9017 9018

	if (fault->error_code & PFERR_RSVD_MASK)
9019
		exit_reason = EXIT_REASON_EPT_MISCONFIG;
9020
	else
9021 9022
		exit_reason = EXIT_REASON_EPT_VIOLATION;
	nested_vmx_vmexit(vcpu, exit_reason, 0, vcpu->arch.exit_qualification);
9023 9024 9025
	vmcs12->guest_physical_address = fault->address;
}

9026 9027 9028 9029 9030 9031 9032 9033
/* Callbacks for nested_ept_init_mmu_context: */

static unsigned long nested_ept_get_cr3(struct kvm_vcpu *vcpu)
{
	/* return the page table to be shadowed - in our case, EPT12 */
	return get_vmcs12(vcpu)->ept_pointer;
}

9034
static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
9035
{
9036 9037
	WARN_ON(mmu_is_nested(vcpu));
	kvm_init_shadow_ept_mmu(vcpu,
9038 9039
			to_vmx(vcpu)->nested.nested_vmx_ept_caps &
			VMX_EPT_EXECUTE_ONLY_BIT);
9040 9041 9042 9043 9044 9045 9046 9047 9048 9049 9050 9051
	vcpu->arch.mmu.set_cr3           = vmx_set_cr3;
	vcpu->arch.mmu.get_cr3           = nested_ept_get_cr3;
	vcpu->arch.mmu.inject_page_fault = nested_ept_inject_page_fault;

	vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
}

static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
{
	vcpu->arch.walk_mmu = &vcpu->arch.mmu;
}

9052 9053 9054 9055 9056 9057 9058 9059 9060 9061 9062 9063
static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
					    u16 error_code)
{
	bool inequality, bit;

	bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0;
	inequality =
		(error_code & vmcs12->page_fault_error_code_mask) !=
		 vmcs12->page_fault_error_code_match;
	return inequality ^ bit;
}

9064 9065 9066 9067 9068 9069 9070
static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu,
		struct x86_exception *fault)
{
	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);

	WARN_ON(!is_guest_mode(vcpu));

9071
	if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code))
9072 9073 9074
		nested_vmx_vmexit(vcpu, to_vmx(vcpu)->exit_reason,
				  vmcs_read32(VM_EXIT_INTR_INFO),
				  vmcs_readl(EXIT_QUALIFICATION));
9075 9076 9077 9078
	else
		kvm_inject_page_fault(vcpu, fault);
}

9079 9080 9081 9082
static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu,
					struct vmcs12 *vmcs12)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
9083
	int maxphyaddr = cpuid_maxphyaddr(vcpu);
9084 9085

	if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
9086 9087
		if (!PAGE_ALIGNED(vmcs12->apic_access_addr) ||
		    vmcs12->apic_access_addr >> maxphyaddr)
9088 9089 9090 9091 9092 9093 9094 9095 9096 9097 9098 9099 9100
			return false;

		/*
		 * Translate L1 physical address to host physical
		 * address for vmcs02. Keep the page pinned, so this
		 * physical address remains valid. We keep a reference
		 * to it so we can release it later.
		 */
		if (vmx->nested.apic_access_page) /* shouldn't happen */
			nested_release_page(vmx->nested.apic_access_page);
		vmx->nested.apic_access_page =
			nested_get_page(vcpu, vmcs12->apic_access_addr);
	}
9101 9102

	if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
9103 9104
		if (!PAGE_ALIGNED(vmcs12->virtual_apic_page_addr) ||
		    vmcs12->virtual_apic_page_addr >> maxphyaddr)
9105 9106 9107 9108 9109 9110 9111 9112 9113 9114 9115 9116 9117 9118 9119 9120 9121 9122 9123 9124 9125
			return false;

		if (vmx->nested.virtual_apic_page) /* shouldn't happen */
			nested_release_page(vmx->nested.virtual_apic_page);
		vmx->nested.virtual_apic_page =
			nested_get_page(vcpu, vmcs12->virtual_apic_page_addr);

		/*
		 * Failing the vm entry is _not_ what the processor does
		 * but it's basically the only possibility we have.
		 * We could still enter the guest if CR8 load exits are
		 * enabled, CR8 store exits are enabled, and virtualize APIC
		 * access is disabled; in this case the processor would never
		 * use the TPR shadow and we could simply clear the bit from
		 * the execution control.  But such a configuration is useless,
		 * so let's keep the code simple.
		 */
		if (!vmx->nested.virtual_apic_page)
			return false;
	}

9126
	if (nested_cpu_has_posted_intr(vmcs12)) {
9127 9128
		if (!IS_ALIGNED(vmcs12->posted_intr_desc_addr, 64) ||
		    vmcs12->posted_intr_desc_addr >> maxphyaddr)
9129 9130 9131 9132 9133 9134 9135 9136 9137 9138 9139 9140 9141 9142 9143 9144 9145 9146 9147 9148 9149 9150 9151
			return false;

		if (vmx->nested.pi_desc_page) { /* shouldn't happen */
			kunmap(vmx->nested.pi_desc_page);
			nested_release_page(vmx->nested.pi_desc_page);
		}
		vmx->nested.pi_desc_page =
			nested_get_page(vcpu, vmcs12->posted_intr_desc_addr);
		if (!vmx->nested.pi_desc_page)
			return false;

		vmx->nested.pi_desc =
			(struct pi_desc *)kmap(vmx->nested.pi_desc_page);
		if (!vmx->nested.pi_desc) {
			nested_release_page_clean(vmx->nested.pi_desc_page);
			return false;
		}
		vmx->nested.pi_desc =
			(struct pi_desc *)((void *)vmx->nested.pi_desc +
			(unsigned long)(vmcs12->posted_intr_desc_addr &
			(PAGE_SIZE - 1)));
	}

9152 9153 9154
	return true;
}

9155 9156 9157 9158 9159 9160 9161 9162 9163 9164 9165 9166 9167 9168 9169 9170 9171 9172 9173 9174 9175 9176
static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu)
{
	u64 preemption_timeout = get_vmcs12(vcpu)->vmx_preemption_timer_value;
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	if (vcpu->arch.virtual_tsc_khz == 0)
		return;

	/* Make sure short timeouts reliably trigger an immediate vmexit.
	 * hrtimer_start does not guarantee this. */
	if (preemption_timeout <= 1) {
		vmx_preemption_timer_fn(&vmx->nested.preemption_timer);
		return;
	}

	preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
	preemption_timeout *= 1000000;
	do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz);
	hrtimer_start(&vmx->nested.preemption_timer,
		      ns_to_ktime(preemption_timeout), HRTIMER_MODE_REL);
}

9177 9178 9179 9180 9181 9182 9183 9184 9185 9186 9187 9188 9189 9190 9191 9192 9193 9194 9195 9196 9197 9198 9199 9200 9201 9202 9203 9204 9205
static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu,
						struct vmcs12 *vmcs12)
{
	int maxphyaddr;
	u64 addr;

	if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
		return 0;

	if (vmcs12_read_any(vcpu, MSR_BITMAP, &addr)) {
		WARN_ON(1);
		return -EINVAL;
	}
	maxphyaddr = cpuid_maxphyaddr(vcpu);

	if (!PAGE_ALIGNED(vmcs12->msr_bitmap) ||
	   ((addr + PAGE_SIZE) >> maxphyaddr))
		return -EINVAL;

	return 0;
}

/*
 * Merge L0's and L1's MSR bitmap, return false to indicate that
 * we do not use the hardware.
 */
static inline bool nested_vmx_merge_msr_bitmap(struct kvm_vcpu *vcpu,
					       struct vmcs12 *vmcs12)
{
9206
	int msr;
9207 9208 9209 9210 9211 9212 9213 9214 9215 9216 9217 9218 9219 9220 9221 9222 9223 9224 9225
	struct page *page;
	unsigned long *msr_bitmap;

	if (!nested_cpu_has_virt_x2apic_mode(vmcs12))
		return false;

	page = nested_get_page(vcpu, vmcs12->msr_bitmap);
	if (!page) {
		WARN_ON(1);
		return false;
	}
	msr_bitmap = (unsigned long *)kmap(page);
	if (!msr_bitmap) {
		nested_release_page_clean(page);
		WARN_ON(1);
		return false;
	}

	if (nested_cpu_has_virt_x2apic_mode(vmcs12)) {
9226 9227 9228 9229 9230 9231
		if (nested_cpu_has_apic_reg_virt(vmcs12))
			for (msr = 0x800; msr <= 0x8ff; msr++)
				nested_vmx_disable_intercept_for_msr(
					msr_bitmap,
					vmx_msr_bitmap_nested,
					msr, MSR_TYPE_R);
9232 9233 9234 9235 9236
		/* TPR is allowed */
		nested_vmx_disable_intercept_for_msr(msr_bitmap,
				vmx_msr_bitmap_nested,
				APIC_BASE_MSR + (APIC_TASKPRI >> 4),
				MSR_TYPE_R | MSR_TYPE_W);
9237 9238 9239 9240 9241 9242 9243 9244 9245 9246 9247 9248 9249
		if (nested_cpu_has_vid(vmcs12)) {
			/* EOI and self-IPI are allowed */
			nested_vmx_disable_intercept_for_msr(
				msr_bitmap,
				vmx_msr_bitmap_nested,
				APIC_BASE_MSR + (APIC_EOI >> 4),
				MSR_TYPE_W);
			nested_vmx_disable_intercept_for_msr(
				msr_bitmap,
				vmx_msr_bitmap_nested,
				APIC_BASE_MSR + (APIC_SELF_IPI >> 4),
				MSR_TYPE_W);
		}
9250 9251 9252 9253 9254 9255 9256 9257 9258 9259 9260 9261 9262
	} else {
		/*
		 * Enable reading intercept of all the x2apic
		 * MSRs. We should not rely on vmcs12 to do any
		 * optimizations here, it may have been modified
		 * by L1.
		 */
		for (msr = 0x800; msr <= 0x8ff; msr++)
			__vmx_enable_intercept_for_msr(
				vmx_msr_bitmap_nested,
				msr,
				MSR_TYPE_R);

9263 9264 9265
		__vmx_enable_intercept_for_msr(
				vmx_msr_bitmap_nested,
				APIC_BASE_MSR + (APIC_TASKPRI >> 4),
9266
				MSR_TYPE_W);
9267 9268 9269 9270 9271 9272 9273 9274
		__vmx_enable_intercept_for_msr(
				vmx_msr_bitmap_nested,
				APIC_BASE_MSR + (APIC_EOI >> 4),
				MSR_TYPE_W);
		__vmx_enable_intercept_for_msr(
				vmx_msr_bitmap_nested,
				APIC_BASE_MSR + (APIC_SELF_IPI >> 4),
				MSR_TYPE_W);
9275
	}
9276 9277 9278 9279 9280 9281 9282 9283 9284
	kunmap(page);
	nested_release_page_clean(page);

	return true;
}

static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu,
					   struct vmcs12 *vmcs12)
{
9285
	if (!nested_cpu_has_virt_x2apic_mode(vmcs12) &&
9286
	    !nested_cpu_has_apic_reg_virt(vmcs12) &&
9287 9288
	    !nested_cpu_has_vid(vmcs12) &&
	    !nested_cpu_has_posted_intr(vmcs12))
9289 9290 9291 9292 9293 9294
		return 0;

	/*
	 * If virtualize x2apic mode is enabled,
	 * virtualize apic access must be disabled.
	 */
9295 9296
	if (nested_cpu_has_virt_x2apic_mode(vmcs12) &&
	    nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
9297 9298
		return -EINVAL;

9299 9300 9301 9302 9303 9304 9305 9306
	/*
	 * If virtual interrupt delivery is enabled,
	 * we must exit on external interrupts.
	 */
	if (nested_cpu_has_vid(vmcs12) &&
	   !nested_exit_on_intr(vcpu))
		return -EINVAL;

9307 9308 9309 9310 9311 9312 9313 9314 9315 9316 9317
	/*
	 * bits 15:8 should be zero in posted_intr_nv,
	 * the descriptor address has been already checked
	 * in nested_get_vmcs12_pages.
	 */
	if (nested_cpu_has_posted_intr(vmcs12) &&
	   (!nested_cpu_has_vid(vmcs12) ||
	    !nested_exit_intr_ack_set(vcpu) ||
	    vmcs12->posted_intr_nv & 0xff00))
		return -EINVAL;

9318 9319 9320 9321 9322
	/* tpr shadow is needed by all apicv features. */
	if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
		return -EINVAL;

	return 0;
9323 9324
}

9325 9326
static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
				       unsigned long count_field,
9327
				       unsigned long addr_field)
9328
{
9329
	int maxphyaddr;
9330 9331 9332 9333 9334 9335 9336 9337 9338
	u64 count, addr;

	if (vmcs12_read_any(vcpu, count_field, &count) ||
	    vmcs12_read_any(vcpu, addr_field, &addr)) {
		WARN_ON(1);
		return -EINVAL;
	}
	if (count == 0)
		return 0;
9339
	maxphyaddr = cpuid_maxphyaddr(vcpu);
9340 9341 9342 9343 9344 9345 9346 9347 9348 9349 9350 9351 9352 9353 9354 9355 9356 9357
	if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr ||
	    (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr) {
		pr_warn_ratelimited(
			"nVMX: invalid MSR switch (0x%lx, %d, %llu, 0x%08llx)",
			addr_field, maxphyaddr, count, addr);
		return -EINVAL;
	}
	return 0;
}

static int nested_vmx_check_msr_switch_controls(struct kvm_vcpu *vcpu,
						struct vmcs12 *vmcs12)
{
	if (vmcs12->vm_exit_msr_load_count == 0 &&
	    vmcs12->vm_exit_msr_store_count == 0 &&
	    vmcs12->vm_entry_msr_load_count == 0)
		return 0; /* Fast path */
	if (nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_LOAD_COUNT,
9358
					VM_EXIT_MSR_LOAD_ADDR) ||
9359
	    nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_STORE_COUNT,
9360
					VM_EXIT_MSR_STORE_ADDR) ||
9361
	    nested_vmx_check_msr_switch(vcpu, VM_ENTRY_MSR_LOAD_COUNT,
9362
					VM_ENTRY_MSR_LOAD_ADDR))
9363 9364 9365 9366 9367 9368 9369 9370
		return -EINVAL;
	return 0;
}

static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu,
				       struct vmx_msr_entry *e)
{
	/* x2APIC MSR accesses are not allowed */
9371
	if (vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8)
9372 9373 9374 9375 9376
		return -EINVAL;
	if (e->index == MSR_IA32_UCODE_WRITE || /* SDM Table 35-2 */
	    e->index == MSR_IA32_UCODE_REV)
		return -EINVAL;
	if (e->reserved != 0)
9377 9378 9379 9380
		return -EINVAL;
	return 0;
}

9381 9382
static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu,
				     struct vmx_msr_entry *e)
9383 9384 9385
{
	if (e->index == MSR_FS_BASE ||
	    e->index == MSR_GS_BASE ||
9386 9387 9388 9389 9390 9391 9392 9393 9394 9395 9396
	    e->index == MSR_IA32_SMM_MONITOR_CTL || /* SMM is not supported */
	    nested_vmx_msr_check_common(vcpu, e))
		return -EINVAL;
	return 0;
}

static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu,
				      struct vmx_msr_entry *e)
{
	if (e->index == MSR_IA32_SMBASE || /* SMM is not supported */
	    nested_vmx_msr_check_common(vcpu, e))
9397 9398 9399 9400 9401 9402 9403 9404 9405 9406 9407 9408 9409 9410 9411 9412
		return -EINVAL;
	return 0;
}

/*
 * Load guest's/host's msr at nested entry/exit.
 * return 0 for success, entry index for failure.
 */
static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
{
	u32 i;
	struct vmx_msr_entry e;
	struct msr_data msr;

	msr.host_initiated = false;
	for (i = 0; i < count; i++) {
9413 9414
		if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e),
					&e, sizeof(e))) {
9415 9416 9417
			pr_warn_ratelimited(
				"%s cannot read MSR entry (%u, 0x%08llx)\n",
				__func__, i, gpa + i * sizeof(e));
9418
			goto fail;
9419 9420 9421 9422 9423 9424 9425
		}
		if (nested_vmx_load_msr_check(vcpu, &e)) {
			pr_warn_ratelimited(
				"%s check failed (%u, 0x%x, 0x%x)\n",
				__func__, i, e.index, e.reserved);
			goto fail;
		}
9426 9427
		msr.index = e.index;
		msr.data = e.value;
9428 9429 9430 9431
		if (kvm_set_msr(vcpu, &msr)) {
			pr_warn_ratelimited(
				"%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
				__func__, i, e.index, e.value);
9432
			goto fail;
9433
		}
9434 9435 9436 9437 9438 9439 9440 9441 9442 9443 9444 9445
	}
	return 0;
fail:
	return i + 1;
}

static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
{
	u32 i;
	struct vmx_msr_entry e;

	for (i = 0; i < count; i++) {
9446
		struct msr_data msr_info;
9447 9448 9449
		if (kvm_vcpu_read_guest(vcpu,
					gpa + i * sizeof(e),
					&e, 2 * sizeof(u32))) {
9450 9451 9452
			pr_warn_ratelimited(
				"%s cannot read MSR entry (%u, 0x%08llx)\n",
				__func__, i, gpa + i * sizeof(e));
9453
			return -EINVAL;
9454 9455 9456 9457 9458
		}
		if (nested_vmx_store_msr_check(vcpu, &e)) {
			pr_warn_ratelimited(
				"%s check failed (%u, 0x%x, 0x%x)\n",
				__func__, i, e.index, e.reserved);
9459
			return -EINVAL;
9460
		}
9461 9462 9463
		msr_info.host_initiated = false;
		msr_info.index = e.index;
		if (kvm_get_msr(vcpu, &msr_info)) {
9464 9465 9466 9467 9468
			pr_warn_ratelimited(
				"%s cannot read MSR (%u, 0x%x)\n",
				__func__, i, e.index);
			return -EINVAL;
		}
9469 9470 9471 9472
		if (kvm_vcpu_write_guest(vcpu,
					 gpa + i * sizeof(e) +
					     offsetof(struct vmx_msr_entry, value),
					 &msr_info.data, sizeof(msr_info.data))) {
9473 9474
			pr_warn_ratelimited(
				"%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
9475
				__func__, i, e.index, msr_info.data);
9476 9477
			return -EINVAL;
		}
9478 9479 9480 9481
	}
	return 0;
}

9482 9483 9484
/*
 * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested
 * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it
9485
 * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2
9486 9487 9488 9489 9490 9491 9492 9493 9494 9495 9496 9497 9498 9499 9500 9501 9502 9503 9504 9505 9506 9507 9508 9509 9510 9511 9512 9513 9514 9515 9516 9517 9518 9519 9520 9521 9522 9523 9524 9525 9526 9527 9528 9529 9530 9531 9532
 * guest in a way that will both be appropriate to L1's requests, and our
 * needs. In addition to modifying the active vmcs (which is vmcs02), this
 * function also has additional necessary side-effects, like setting various
 * vcpu->arch fields.
 */
static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	u32 exec_control;

	vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector);
	vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector);
	vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector);
	vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector);
	vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector);
	vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector);
	vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector);
	vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector);
	vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit);
	vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit);
	vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit);
	vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit);
	vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit);
	vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit);
	vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit);
	vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit);
	vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit);
	vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit);
	vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes);
	vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes);
	vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes);
	vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes);
	vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes);
	vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes);
	vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes);
	vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes);
	vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base);
	vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base);
	vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base);
	vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base);
	vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base);
	vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base);
	vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base);
	vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base);
	vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base);
	vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base);

9533 9534 9535 9536 9537 9538 9539
	if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) {
		kvm_set_dr(vcpu, 7, vmcs12->guest_dr7);
		vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl);
	} else {
		kvm_set_dr(vcpu, 7, vcpu->arch.dr7);
		vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl);
	}
9540 9541 9542 9543 9544 9545 9546 9547 9548
	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
		vmcs12->vm_entry_intr_info_field);
	vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE,
		vmcs12->vm_entry_exception_error_code);
	vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
		vmcs12->vm_entry_instruction_len);
	vmcs_write32(GUEST_INTERRUPTIBILITY_INFO,
		vmcs12->guest_interruptibility_info);
	vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs);
9549
	vmx_set_rflags(vcpu, vmcs12->guest_rflags);
9550 9551 9552 9553 9554
	vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
		vmcs12->guest_pending_dbg_exceptions);
	vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp);
	vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip);

9555 9556
	if (nested_cpu_has_xsaves(vmcs12))
		vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap);
9557 9558
	vmcs_write64(VMCS_LINK_POINTER, -1ull);

9559 9560
	exec_control = vmcs12->pin_based_vm_exec_control;
	exec_control |= vmcs_config.pin_based_exec_ctrl;
9561 9562 9563 9564 9565 9566 9567 9568 9569 9570 9571 9572 9573 9574 9575 9576 9577
	exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;

	if (nested_cpu_has_posted_intr(vmcs12)) {
		/*
		 * Note that we use L0's vector here and in
		 * vmx_deliver_nested_posted_interrupt.
		 */
		vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv;
		vmx->nested.pi_pending = false;
		vmcs_write64(POSTED_INTR_NV, POSTED_INTR_VECTOR);
		vmcs_write64(POSTED_INTR_DESC_ADDR,
			page_to_phys(vmx->nested.pi_desc_page) +
			(unsigned long)(vmcs12->posted_intr_desc_addr &
			(PAGE_SIZE - 1)));
	} else
		exec_control &= ~PIN_BASED_POSTED_INTR;

9578
	vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, exec_control);
9579

9580 9581 9582
	vmx->nested.preemption_timer_expired = false;
	if (nested_cpu_has_preemption_timer(vmcs12))
		vmx_start_preemption_timer(vcpu);
9583

9584 9585 9586 9587 9588 9589 9590 9591 9592 9593 9594 9595 9596 9597 9598 9599 9600 9601 9602 9603 9604 9605 9606 9607 9608 9609
	/*
	 * Whether page-faults are trapped is determined by a combination of
	 * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF.
	 * If enable_ept, L0 doesn't care about page faults and we should
	 * set all of these to L1's desires. However, if !enable_ept, L0 does
	 * care about (at least some) page faults, and because it is not easy
	 * (if at all possible?) to merge L0 and L1's desires, we simply ask
	 * to exit on each and every L2 page fault. This is done by setting
	 * MASK=MATCH=0 and (see below) EB.PF=1.
	 * Note that below we don't need special code to set EB.PF beyond the
	 * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept,
	 * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when
	 * !enable_ept, EB.PF is 1, so the "or" will always be 1.
	 *
	 * A problem with this approach (when !enable_ept) is that L1 may be
	 * injected with more page faults than it asked for. This could have
	 * caused problems, but in practice existing hypervisors don't care.
	 * To fix this, we will need to emulate the PFEC checking (on the L1
	 * page tables), using walk_addr(), when injecting PFs to L1.
	 */
	vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK,
		enable_ept ? vmcs12->page_fault_error_code_mask : 0);
	vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH,
		enable_ept ? vmcs12->page_fault_error_code_match : 0);

	if (cpu_has_secondary_exec_ctrls()) {
9610
		exec_control = vmx_secondary_exec_control(vmx);
9611

9612
		/* Take the following fields only from vmcs12 */
9613
		exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
9614
				  SECONDARY_EXEC_RDTSCP |
9615
				  SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
9616 9617
				  SECONDARY_EXEC_APIC_REGISTER_VIRT |
				  SECONDARY_EXEC_PCOMMIT);
9618 9619 9620 9621 9622 9623 9624 9625 9626 9627 9628 9629 9630 9631 9632 9633 9634
		if (nested_cpu_has(vmcs12,
				CPU_BASED_ACTIVATE_SECONDARY_CONTROLS))
			exec_control |= vmcs12->secondary_vm_exec_control;

		if (exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) {
			/*
			 * If translation failed, no matter: This feature asks
			 * to exit when accessing the given address, and if it
			 * can never be accessed, this feature won't do
			 * anything anyway.
			 */
			if (!vmx->nested.apic_access_page)
				exec_control &=
				  ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
			else
				vmcs_write64(APIC_ACCESS_ADDR,
				  page_to_phys(vmx->nested.apic_access_page));
9635
		} else if (!(nested_cpu_has_virt_x2apic_mode(vmcs12)) &&
9636
			    cpu_need_virtualize_apic_accesses(&vmx->vcpu)) {
9637 9638
			exec_control |=
				SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
9639
			kvm_vcpu_reload_apic_access_page(vcpu);
9640 9641
		}

9642 9643 9644 9645 9646 9647 9648 9649 9650 9651 9652 9653 9654
		if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) {
			vmcs_write64(EOI_EXIT_BITMAP0,
				vmcs12->eoi_exit_bitmap0);
			vmcs_write64(EOI_EXIT_BITMAP1,
				vmcs12->eoi_exit_bitmap1);
			vmcs_write64(EOI_EXIT_BITMAP2,
				vmcs12->eoi_exit_bitmap2);
			vmcs_write64(EOI_EXIT_BITMAP3,
				vmcs12->eoi_exit_bitmap3);
			vmcs_write16(GUEST_INTR_STATUS,
				vmcs12->guest_intr_status);
		}

9655 9656 9657 9658 9659 9660 9661 9662 9663 9664
		vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
	}


	/*
	 * Set host-state according to L0's settings (vmcs12 is irrelevant here)
	 * Some constant fields are set here by vmx_set_constant_host_state().
	 * Other fields are different per CPU, and will be set later when
	 * vmx_vcpu_load() is called, and when vmx_save_host_state() is called.
	 */
9665
	vmx_set_constant_host_state(vmx);
9666 9667 9668 9669 9670 9671 9672 9673 9674 9675 9676 9677 9678 9679 9680

	/*
	 * HOST_RSP is normally set correctly in vmx_vcpu_run() just before
	 * entry, but only if the current (host) sp changed from the value
	 * we wrote last (vmx->host_rsp). This cache is no longer relevant
	 * if we switch vmcs, and rather than hold a separate cache per vmcs,
	 * here we just force the write to happen on entry.
	 */
	vmx->host_rsp = 0;

	exec_control = vmx_exec_control(vmx); /* L0's desires */
	exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
	exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
	exec_control &= ~CPU_BASED_TPR_SHADOW;
	exec_control |= vmcs12->cpu_based_vm_exec_control;
9681 9682 9683 9684 9685

	if (exec_control & CPU_BASED_TPR_SHADOW) {
		vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
				page_to_phys(vmx->nested.virtual_apic_page));
		vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold);
9686 9687 9688 9689 9690
	} else {
#ifdef CONFIG_X86_64
		exec_control |= CPU_BASED_CR8_LOAD_EXITING |
				CPU_BASED_CR8_STORE_EXITING;
#endif
9691 9692
	}

9693
	if (cpu_has_vmx_msr_bitmap() &&
9694 9695 9696
	    exec_control & CPU_BASED_USE_MSR_BITMAPS) {
		nested_vmx_merge_msr_bitmap(vcpu, vmcs12);
		/* MSR_BITMAP will be set by following vmx_set_efer. */
9697 9698 9699
	} else
		exec_control &= ~CPU_BASED_USE_MSR_BITMAPS;

9700
	/*
9701
	 * Merging of IO bitmap not currently supported.
9702 9703 9704 9705 9706 9707 9708 9709 9710 9711 9712 9713 9714 9715 9716
	 * Rather, exit every time.
	 */
	exec_control &= ~CPU_BASED_USE_IO_BITMAPS;
	exec_control |= CPU_BASED_UNCOND_IO_EXITING;

	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control);

	/* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the
	 * bitwise-or of what L1 wants to trap for L2, and what we want to
	 * trap. Note that CR0.TS also needs updating - we do this later.
	 */
	update_exception_bitmap(vcpu);
	vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask;
	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);

9717 9718 9719 9720
	/* L2->L1 exit controls are emulated - the hardware exit is to L0 so
	 * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER
	 * bits are further modified by vmx_set_efer() below.
	 */
9721
	vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
9722 9723 9724 9725

	/* vmcs12's VM_ENTRY_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE are
	 * emulated by vmx_set_efer(), below.
	 */
9726
	vm_entry_controls_init(vmx, 
9727 9728
		(vmcs12->vm_entry_controls & ~VM_ENTRY_LOAD_IA32_EFER &
			~VM_ENTRY_IA32E_MODE) |
9729 9730
		(vmcs_config.vmentry_ctrl & ~VM_ENTRY_IA32E_MODE));

9731
	if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) {
9732
		vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat);
9733 9734
		vcpu->arch.pat = vmcs12->guest_ia32_pat;
	} else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT)
9735 9736 9737 9738 9739
		vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);


	set_cr4_guest_host_mask(vmx);

9740 9741 9742
	if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)
		vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs);

9743 9744 9745 9746 9747
	if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING)
		vmcs_write64(TSC_OFFSET,
			vmx->nested.vmcs01_tsc_offset + vmcs12->tsc_offset);
	else
		vmcs_write64(TSC_OFFSET, vmx->nested.vmcs01_tsc_offset);
9748 9749 9750

	if (enable_vpid) {
		/*
9751 9752 9753 9754 9755 9756
		 * There is no direct mapping between vpid02 and vpid12, the
		 * vpid02 is per-vCPU for L0 and reused while the value of
		 * vpid12 is changed w/ one invvpid during nested vmentry.
		 * The vpid12 is allocated by L1 for L2, so it will not
		 * influence global bitmap(for vpid01 and vpid02 allocation)
		 * even if spawn a lot of nested vCPUs.
9757
		 */
9758 9759 9760 9761 9762 9763 9764 9765 9766 9767 9768
		if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) {
			vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02);
			if (vmcs12->virtual_processor_id != vmx->nested.last_vpid) {
				vmx->nested.last_vpid = vmcs12->virtual_processor_id;
				__vmx_flush_tlb(vcpu, to_vmx(vcpu)->nested.vpid02);
			}
		} else {
			vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
			vmx_flush_tlb(vcpu);
		}

9769 9770
	}

9771 9772 9773 9774 9775 9776 9777 9778 9779 9780 9781 9782
	if (enable_pml) {
		/*
		 * Conceptually we want to copy the PML address and index from
		 * vmcs01 here, and then back to vmcs01 on nested vmexit. But,
		 * since we always flush the log on each vmexit, this happens
		 * to be equivalent to simply resetting the fields in vmcs02.
		 */
		ASSERT(vmx->pml_pg);
		vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
		vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
	}

9783 9784 9785 9786 9787
	if (nested_cpu_has_ept(vmcs12)) {
		kvm_mmu_unload(vcpu);
		nested_ept_init_mmu_context(vcpu);
	}

9788 9789
	if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)
		vcpu->arch.efer = vmcs12->guest_ia32_efer;
9790
	else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE)
9791 9792 9793 9794 9795 9796 9797 9798 9799 9800 9801 9802 9803 9804 9805 9806 9807 9808 9809 9810 9811 9812 9813 9814
		vcpu->arch.efer |= (EFER_LMA | EFER_LME);
	else
		vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
	/* Note: modifies VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */
	vmx_set_efer(vcpu, vcpu->arch.efer);

	/*
	 * This sets GUEST_CR0 to vmcs12->guest_cr0, with possibly a modified
	 * TS bit (for lazy fpu) and bits which we consider mandatory enabled.
	 * The CR0_READ_SHADOW is what L2 should have expected to read given
	 * the specifications by L1; It's not enough to take
	 * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we
	 * have more bits than L1 expected.
	 */
	vmx_set_cr0(vcpu, vmcs12->guest_cr0);
	vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12));

	vmx_set_cr4(vcpu, vmcs12->guest_cr4);
	vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12));

	/* shadow page tables on either EPT or shadow page tables */
	kvm_set_cr3(vcpu, vmcs12->guest_cr3);
	kvm_mmu_reset_context(vcpu);

9815 9816 9817
	if (!enable_ept)
		vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested;

9818 9819 9820 9821 9822 9823 9824 9825 9826 9827
	/*
	 * L1 may access the L2's PDPTR, so save them to construct vmcs12
	 */
	if (enable_ept) {
		vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0);
		vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
		vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
		vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
	}

9828 9829 9830 9831
	kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->guest_rsp);
	kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->guest_rip);
}

9832 9833 9834 9835 9836 9837 9838 9839 9840 9841
/*
 * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1
 * for running an L2 nested guest.
 */
static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
{
	struct vmcs12 *vmcs12;
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	int cpu;
	struct loaded_vmcs *vmcs02;
9842
	bool ia32e;
9843
	u32 msr_entry_idx;
9844 9845 9846 9847 9848 9849 9850 9851

	if (!nested_vmx_check_permission(vcpu) ||
	    !nested_vmx_check_vmcs12(vcpu))
		return 1;

	skip_emulated_instruction(vcpu);
	vmcs12 = get_vmcs12(vcpu);

9852 9853 9854
	if (enable_shadow_vmcs)
		copy_shadow_to_vmcs12(vmx);

9855 9856 9857 9858 9859 9860 9861 9862 9863 9864 9865 9866 9867 9868 9869 9870 9871
	/*
	 * The nested entry process starts with enforcing various prerequisites
	 * on vmcs12 as required by the Intel SDM, and act appropriately when
	 * they fail: As the SDM explains, some conditions should cause the
	 * instruction to fail, while others will cause the instruction to seem
	 * to succeed, but return an EXIT_REASON_INVALID_STATE.
	 * To speed up the normal (success) code path, we should avoid checking
	 * for misconfigurations which will anyway be caught by the processor
	 * when using the merged vmcs02.
	 */
	if (vmcs12->launch_state == launch) {
		nested_vmx_failValid(vcpu,
			launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS
			       : VMXERR_VMRESUME_NONLAUNCHED_VMCS);
		return 1;
	}

9872 9873
	if (vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE &&
	    vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT) {
9874 9875 9876 9877
		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
		return 1;
	}

9878
	if (!nested_get_vmcs12_pages(vcpu, vmcs12)) {
9879 9880 9881 9882
		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
		return 1;
	}

9883
	if (nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12)) {
9884 9885 9886 9887
		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
		return 1;
	}

9888 9889 9890 9891 9892
	if (nested_vmx_check_apicv_controls(vcpu, vmcs12)) {
		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
		return 1;
	}

9893 9894 9895 9896 9897
	if (nested_vmx_check_msr_switch_controls(vcpu, vmcs12)) {
		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
		return 1;
	}

9898
	if (!vmx_control_verify(vmcs12->cpu_based_vm_exec_control,
9899 9900
				vmx->nested.nested_vmx_true_procbased_ctls_low,
				vmx->nested.nested_vmx_procbased_ctls_high) ||
9901
	    !vmx_control_verify(vmcs12->secondary_vm_exec_control,
9902 9903
				vmx->nested.nested_vmx_secondary_ctls_low,
				vmx->nested.nested_vmx_secondary_ctls_high) ||
9904
	    !vmx_control_verify(vmcs12->pin_based_vm_exec_control,
9905 9906
				vmx->nested.nested_vmx_pinbased_ctls_low,
				vmx->nested.nested_vmx_pinbased_ctls_high) ||
9907
	    !vmx_control_verify(vmcs12->vm_exit_controls,
9908 9909
				vmx->nested.nested_vmx_true_exit_ctls_low,
				vmx->nested.nested_vmx_exit_ctls_high) ||
9910
	    !vmx_control_verify(vmcs12->vm_entry_controls,
9911 9912
				vmx->nested.nested_vmx_true_entry_ctls_low,
				vmx->nested.nested_vmx_entry_ctls_high))
9913 9914 9915 9916 9917 9918 9919 9920 9921 9922 9923 9924
	{
		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
		return 1;
	}

	if (((vmcs12->host_cr0 & VMXON_CR0_ALWAYSON) != VMXON_CR0_ALWAYSON) ||
	    ((vmcs12->host_cr4 & VMXON_CR4_ALWAYSON) != VMXON_CR4_ALWAYSON)) {
		nested_vmx_failValid(vcpu,
			VMXERR_ENTRY_INVALID_HOST_STATE_FIELD);
		return 1;
	}

9925
	if (!nested_cr0_valid(vcpu, vmcs12->guest_cr0) ||
9926 9927 9928 9929 9930 9931 9932 9933 9934 9935 9936
	    ((vmcs12->guest_cr4 & VMXON_CR4_ALWAYSON) != VMXON_CR4_ALWAYSON)) {
		nested_vmx_entry_failure(vcpu, vmcs12,
			EXIT_REASON_INVALID_STATE, ENTRY_FAIL_DEFAULT);
		return 1;
	}
	if (vmcs12->vmcs_link_pointer != -1ull) {
		nested_vmx_entry_failure(vcpu, vmcs12,
			EXIT_REASON_INVALID_STATE, ENTRY_FAIL_VMCS_LINK_PTR);
		return 1;
	}

9937
	/*
9938
	 * If the load IA32_EFER VM-entry control is 1, the following checks
9939 9940 9941 9942 9943 9944 9945 9946 9947 9948 9949 9950 9951 9952 9953 9954 9955 9956 9957 9958 9959 9960 9961 9962 9963 9964 9965 9966 9967 9968 9969 9970 9971 9972 9973 9974 9975
	 * are performed on the field for the IA32_EFER MSR:
	 * - Bits reserved in the IA32_EFER MSR must be 0.
	 * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of
	 *   the IA-32e mode guest VM-exit control. It must also be identical
	 *   to bit 8 (LME) if bit 31 in the CR0 field (corresponding to
	 *   CR0.PG) is 1.
	 */
	if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER) {
		ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0;
		if (!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer) ||
		    ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA) ||
		    ((vmcs12->guest_cr0 & X86_CR0_PG) &&
		     ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME))) {
			nested_vmx_entry_failure(vcpu, vmcs12,
				EXIT_REASON_INVALID_STATE, ENTRY_FAIL_DEFAULT);
			return 1;
		}
	}

	/*
	 * If the load IA32_EFER VM-exit control is 1, bits reserved in the
	 * IA32_EFER MSR must be 0 in the field for that register. In addition,
	 * the values of the LMA and LME bits in the field must each be that of
	 * the host address-space size VM-exit control.
	 */
	if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) {
		ia32e = (vmcs12->vm_exit_controls &
			 VM_EXIT_HOST_ADDR_SPACE_SIZE) != 0;
		if (!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer) ||
		    ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA) ||
		    ia32e != !!(vmcs12->host_ia32_efer & EFER_LME)) {
			nested_vmx_entry_failure(vcpu, vmcs12,
				EXIT_REASON_INVALID_STATE, ENTRY_FAIL_DEFAULT);
			return 1;
		}
	}

9976 9977 9978 9979 9980
	/*
	 * We're finally done with prerequisite checking, and can start with
	 * the nested entry.
	 */

9981 9982 9983 9984 9985 9986 9987 9988
	vmcs02 = nested_get_current_vmcs02(vmx);
	if (!vmcs02)
		return -ENOMEM;

	enter_guest_mode(vcpu);

	vmx->nested.vmcs01_tsc_offset = vmcs_read64(TSC_OFFSET);

9989 9990 9991
	if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS))
		vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);

9992 9993 9994 9995 9996 9997 9998
	cpu = get_cpu();
	vmx->loaded_vmcs = vmcs02;
	vmx_vcpu_put(vcpu);
	vmx_vcpu_load(vcpu, cpu);
	vcpu->cpu = cpu;
	put_cpu();

9999 10000
	vmx_segment_cache_clear(vmx);

10001 10002
	prepare_vmcs02(vcpu, vmcs12);

10003 10004 10005 10006 10007 10008 10009 10010 10011 10012 10013 10014 10015
	msr_entry_idx = nested_vmx_load_msr(vcpu,
					    vmcs12->vm_entry_msr_load_addr,
					    vmcs12->vm_entry_msr_load_count);
	if (msr_entry_idx) {
		leave_guest_mode(vcpu);
		vmx_load_vmcs01(vcpu);
		nested_vmx_entry_failure(vcpu, vmcs12,
				EXIT_REASON_MSR_LOAD_FAIL, msr_entry_idx);
		return 1;
	}

	vmcs12->launch_state = 1;

10016
	if (vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT)
10017
		return kvm_vcpu_halt(vcpu);
10018

10019 10020
	vmx->nested.nested_run_pending = 1;

10021 10022 10023 10024 10025 10026 10027 10028 10029
	/*
	 * Note no nested_vmx_succeed or nested_vmx_fail here. At this point
	 * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet
	 * returned as far as L1 is concerned. It will only return (and set
	 * the success flag) when L2 exits (see nested_vmx_vmexit()).
	 */
	return 1;
}

10030 10031 10032 10033 10034 10035 10036 10037 10038 10039 10040 10041 10042 10043 10044 10045 10046 10047 10048 10049 10050 10051 10052 10053 10054 10055 10056 10057 10058 10059 10060 10061 10062 10063 10064 10065 10066
/*
 * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date
 * because L2 may have changed some cr0 bits directly (CRO_GUEST_HOST_MASK).
 * This function returns the new value we should put in vmcs12.guest_cr0.
 * It's not enough to just return the vmcs02 GUEST_CR0. Rather,
 *  1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now
 *     available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0
 *     didn't trap the bit, because if L1 did, so would L0).
 *  2. Bits that L1 asked to trap (and therefore L0 also did) could not have
 *     been modified by L2, and L1 knows it. So just leave the old value of
 *     the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0
 *     isn't relevant, because if L0 traps this bit it can set it to anything.
 *  3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have
 *     changed these bits, and therefore they need to be updated, but L0
 *     didn't necessarily allow them to be changed in GUEST_CR0 - and rather
 *     put them in vmcs02 CR0_READ_SHADOW. So take these bits from there.
 */
static inline unsigned long
vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
{
	return
	/*1*/	(vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) |
	/*2*/	(vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) |
	/*3*/	(vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask |
			vcpu->arch.cr0_guest_owned_bits));
}

static inline unsigned long
vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
{
	return
	/*1*/	(vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) |
	/*2*/	(vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) |
	/*3*/	(vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask |
			vcpu->arch.cr4_guest_owned_bits));
}

10067 10068 10069 10070 10071 10072
static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu,
				       struct vmcs12 *vmcs12)
{
	u32 idt_vectoring;
	unsigned int nr;

10073
	if (vcpu->arch.exception.pending && vcpu->arch.exception.reinject) {
10074 10075 10076 10077 10078 10079 10080 10081 10082 10083 10084 10085 10086 10087 10088 10089 10090
		nr = vcpu->arch.exception.nr;
		idt_vectoring = nr | VECTORING_INFO_VALID_MASK;

		if (kvm_exception_is_soft(nr)) {
			vmcs12->vm_exit_instruction_len =
				vcpu->arch.event_exit_inst_len;
			idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION;
		} else
			idt_vectoring |= INTR_TYPE_HARD_EXCEPTION;

		if (vcpu->arch.exception.has_error_code) {
			idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK;
			vmcs12->idt_vectoring_error_code =
				vcpu->arch.exception.error_code;
		}

		vmcs12->idt_vectoring_info_field = idt_vectoring;
10091
	} else if (vcpu->arch.nmi_injected) {
10092 10093 10094 10095 10096 10097 10098 10099 10100 10101 10102 10103 10104 10105 10106 10107 10108
		vmcs12->idt_vectoring_info_field =
			INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR;
	} else if (vcpu->arch.interrupt.pending) {
		nr = vcpu->arch.interrupt.nr;
		idt_vectoring = nr | VECTORING_INFO_VALID_MASK;

		if (vcpu->arch.interrupt.soft) {
			idt_vectoring |= INTR_TYPE_SOFT_INTR;
			vmcs12->vm_entry_instruction_len =
				vcpu->arch.event_exit_inst_len;
		} else
			idt_vectoring |= INTR_TYPE_EXT_INTR;

		vmcs12->idt_vectoring_info_field = idt_vectoring;
	}
}

10109 10110 10111 10112
static int vmx_check_nested_events(struct kvm_vcpu *vcpu, bool external_intr)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);

10113 10114 10115 10116 10117 10118 10119 10120
	if (nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) &&
	    vmx->nested.preemption_timer_expired) {
		if (vmx->nested.nested_run_pending)
			return -EBUSY;
		nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0);
		return 0;
	}

10121
	if (vcpu->arch.nmi_pending && nested_exit_on_nmi(vcpu)) {
10122 10123
		if (vmx->nested.nested_run_pending ||
		    vcpu->arch.interrupt.pending)
10124 10125 10126 10127 10128 10129 10130 10131 10132 10133 10134 10135 10136 10137 10138 10139 10140 10141
			return -EBUSY;
		nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
				  NMI_VECTOR | INTR_TYPE_NMI_INTR |
				  INTR_INFO_VALID_MASK, 0);
		/*
		 * The NMI-triggered VM exit counts as injection:
		 * clear this one and block further NMIs.
		 */
		vcpu->arch.nmi_pending = 0;
		vmx_set_nmi_mask(vcpu, true);
		return 0;
	}

	if ((kvm_cpu_has_interrupt(vcpu) || external_intr) &&
	    nested_exit_on_intr(vcpu)) {
		if (vmx->nested.nested_run_pending)
			return -EBUSY;
		nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0);
10142
		return 0;
10143 10144
	}

10145
	return vmx_complete_nested_posted_interrupt(vcpu);
10146 10147
}

10148 10149 10150 10151 10152 10153 10154 10155 10156 10157 10158 10159 10160 10161
static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu)
{
	ktime_t remaining =
		hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer);
	u64 value;

	if (ktime_to_ns(remaining) <= 0)
		return 0;

	value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz;
	do_div(value, 1000000);
	return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
}

10162 10163 10164 10165 10166 10167 10168 10169 10170 10171 10172
/*
 * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits
 * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12),
 * and this function updates it to reflect the changes to the guest state while
 * L2 was running (and perhaps made some exits which were handled directly by L0
 * without going back to L1), and to reflect the exit reason.
 * Note that we do not have to copy here all VMCS fields, just those that
 * could have changed by the L2 guest or the exit - i.e., the guest-state and
 * exit-information fields only. Other fields are modified by L1 with VMWRITE,
 * which already writes to vmcs12 directly.
 */
10173 10174 10175
static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
			   u32 exit_reason, u32 exit_intr_info,
			   unsigned long exit_qualification)
10176 10177 10178 10179 10180 10181 10182 10183 10184 10185 10186 10187 10188 10189 10190 10191 10192 10193 10194 10195 10196 10197 10198 10199 10200 10201 10202 10203 10204 10205 10206 10207 10208 10209 10210 10211 10212 10213 10214 10215 10216 10217 10218 10219 10220 10221 10222 10223 10224 10225
{
	/* update guest state fields: */
	vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12);
	vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12);

	vmcs12->guest_rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
	vmcs12->guest_rip = kvm_register_read(vcpu, VCPU_REGS_RIP);
	vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS);

	vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR);
	vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR);
	vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR);
	vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR);
	vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR);
	vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR);
	vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR);
	vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR);
	vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT);
	vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT);
	vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT);
	vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT);
	vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT);
	vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT);
	vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT);
	vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT);
	vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT);
	vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT);
	vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES);
	vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES);
	vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES);
	vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES);
	vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES);
	vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES);
	vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES);
	vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES);
	vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE);
	vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE);
	vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE);
	vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE);
	vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE);
	vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE);
	vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE);
	vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE);
	vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE);
	vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE);

	vmcs12->guest_interruptibility_info =
		vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
	vmcs12->guest_pending_dbg_exceptions =
		vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);
10226 10227 10228 10229
	if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
		vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT;
	else
		vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE;
10230

10231 10232 10233 10234 10235 10236 10237
	if (nested_cpu_has_preemption_timer(vmcs12)) {
		if (vmcs12->vm_exit_controls &
		    VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)
			vmcs12->vmx_preemption_timer_value =
				vmx_get_preemption_timer_value(vcpu);
		hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer);
	}
10238

10239 10240 10241 10242 10243 10244 10245 10246 10247 10248 10249 10250 10251 10252 10253 10254
	/*
	 * In some cases (usually, nested EPT), L2 is allowed to change its
	 * own CR3 without exiting. If it has changed it, we must keep it.
	 * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined
	 * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12.
	 *
	 * Additionally, restore L2's PDPTR to vmcs12.
	 */
	if (enable_ept) {
		vmcs12->guest_cr3 = vmcs_read64(GUEST_CR3);
		vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0);
		vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1);
		vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2);
		vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3);
	}

10255 10256 10257
	if (nested_cpu_has_vid(vmcs12))
		vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS);

10258 10259
	vmcs12->vm_entry_controls =
		(vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) |
10260
		(vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE);
10261

10262 10263 10264 10265 10266
	if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) {
		kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7);
		vmcs12->guest_ia32_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);
	}

10267 10268
	/* TODO: These cannot have changed unless we have MSR bitmaps and
	 * the relevant bit asks not to trap the change */
10269
	if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT)
10270
		vmcs12->guest_ia32_pat = vmcs_read64(GUEST_IA32_PAT);
10271 10272
	if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER)
		vmcs12->guest_ia32_efer = vcpu->arch.efer;
10273 10274 10275
	vmcs12->guest_sysenter_cs = vmcs_read32(GUEST_SYSENTER_CS);
	vmcs12->guest_sysenter_esp = vmcs_readl(GUEST_SYSENTER_ESP);
	vmcs12->guest_sysenter_eip = vmcs_readl(GUEST_SYSENTER_EIP);
10276
	if (kvm_mpx_supported())
10277
		vmcs12->guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS);
10278 10279
	if (nested_cpu_has_xsaves(vmcs12))
		vmcs12->xss_exit_bitmap = vmcs_read64(XSS_EXIT_BITMAP);
10280 10281 10282

	/* update exit information fields: */

10283 10284
	vmcs12->vm_exit_reason = exit_reason;
	vmcs12->exit_qualification = exit_qualification;
10285

10286
	vmcs12->vm_exit_intr_info = exit_intr_info;
10287 10288 10289 10290 10291
	if ((vmcs12->vm_exit_intr_info &
	     (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) ==
	    (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK))
		vmcs12->vm_exit_intr_error_code =
			vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
10292
	vmcs12->idt_vectoring_info_field = 0;
10293 10294 10295
	vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
	vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);

10296 10297 10298
	if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) {
		/* vm_entry_intr_info_field is cleared on exit. Emulate this
		 * instead of reading the real value. */
10299
		vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK;
10300 10301 10302 10303 10304 10305 10306 10307 10308 10309 10310 10311 10312 10313 10314

		/*
		 * Transfer the event that L0 or L1 may wanted to inject into
		 * L2 to IDT_VECTORING_INFO_FIELD.
		 */
		vmcs12_save_pending_event(vcpu, vmcs12);
	}

	/*
	 * Drop what we picked up for L2 via vmx_complete_interrupts. It is
	 * preserved above and would only end up incorrectly in L1.
	 */
	vcpu->arch.nmi_injected = false;
	kvm_clear_exception_queue(vcpu);
	kvm_clear_interrupt_queue(vcpu);
10315 10316 10317 10318 10319 10320 10321 10322 10323 10324 10325
}

/*
 * A part of what we need to when the nested L2 guest exits and we want to
 * run its L1 parent, is to reset L1's guest state to the host state specified
 * in vmcs12.
 * This function is to be called not only on normal nested exit, but also on
 * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry
 * Failures During or After Loading Guest State").
 * This function should be called when the active VMCS is L1's (vmcs01).
 */
10326 10327
static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
				   struct vmcs12 *vmcs12)
10328
{
10329 10330
	struct kvm_segment seg;

10331 10332
	if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER)
		vcpu->arch.efer = vmcs12->host_ia32_efer;
10333
	else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
10334 10335 10336 10337 10338 10339 10340
		vcpu->arch.efer |= (EFER_LMA | EFER_LME);
	else
		vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
	vmx_set_efer(vcpu, vcpu->arch.efer);

	kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->host_rsp);
	kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->host_rip);
10341
	vmx_set_rflags(vcpu, X86_EFLAGS_FIXED);
10342 10343 10344 10345 10346 10347
	/*
	 * Note that calling vmx_set_cr0 is important, even if cr0 hasn't
	 * actually changed, because it depends on the current state of
	 * fpu_active (which may have changed).
	 * Note that vmx_set_cr0 refers to efer set above.
	 */
10348
	vmx_set_cr0(vcpu, vmcs12->host_cr0);
10349 10350 10351 10352 10353 10354 10355 10356 10357 10358 10359 10360 10361 10362 10363 10364
	/*
	 * If we did fpu_activate()/fpu_deactivate() during L2's run, we need
	 * to apply the same changes to L1's vmcs. We just set cr0 correctly,
	 * but we also need to update cr0_guest_host_mask and exception_bitmap.
	 */
	update_exception_bitmap(vcpu);
	vcpu->arch.cr0_guest_owned_bits = (vcpu->fpu_active ? X86_CR0_TS : 0);
	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);

	/*
	 * Note that CR4_GUEST_HOST_MASK is already set in the original vmcs01
	 * (KVM doesn't change it)- no reason to call set_cr4_guest_host_mask();
	 */
	vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK);
	kvm_set_cr4(vcpu, vmcs12->host_cr4);

10365
	nested_ept_uninit_mmu_context(vcpu);
10366

10367 10368 10369
	kvm_set_cr3(vcpu, vmcs12->host_cr3);
	kvm_mmu_reset_context(vcpu);

10370 10371 10372
	if (!enable_ept)
		vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;

10373 10374 10375 10376 10377 10378 10379 10380 10381 10382 10383 10384 10385 10386 10387 10388
	if (enable_vpid) {
		/*
		 * Trivially support vpid by letting L2s share their parent
		 * L1's vpid. TODO: move to a more elaborate solution, giving
		 * each L2 its own vpid and exposing the vpid feature to L1.
		 */
		vmx_flush_tlb(vcpu);
	}


	vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs);
	vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp);
	vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip);
	vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base);
	vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base);

10389 10390 10391 10392
	/* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1.  */
	if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS)
		vmcs_write64(GUEST_BNDCFGS, 0);

10393
	if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) {
10394
		vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat);
10395 10396
		vcpu->arch.pat = vmcs12->host_ia32_pat;
	}
10397 10398 10399
	if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
		vmcs_write64(GUEST_IA32_PERF_GLOBAL_CTRL,
			vmcs12->host_ia32_perf_global_ctrl);
10400

10401 10402 10403 10404 10405 10406 10407 10408 10409 10410 10411 10412 10413 10414 10415 10416 10417 10418 10419 10420 10421 10422 10423 10424 10425 10426 10427 10428 10429 10430 10431 10432 10433 10434 10435 10436 10437 10438
	/* Set L1 segment info according to Intel SDM
	    27.5.2 Loading Host Segment and Descriptor-Table Registers */
	seg = (struct kvm_segment) {
		.base = 0,
		.limit = 0xFFFFFFFF,
		.selector = vmcs12->host_cs_selector,
		.type = 11,
		.present = 1,
		.s = 1,
		.g = 1
	};
	if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
		seg.l = 1;
	else
		seg.db = 1;
	vmx_set_segment(vcpu, &seg, VCPU_SREG_CS);
	seg = (struct kvm_segment) {
		.base = 0,
		.limit = 0xFFFFFFFF,
		.type = 3,
		.present = 1,
		.s = 1,
		.db = 1,
		.g = 1
	};
	seg.selector = vmcs12->host_ds_selector;
	vmx_set_segment(vcpu, &seg, VCPU_SREG_DS);
	seg.selector = vmcs12->host_es_selector;
	vmx_set_segment(vcpu, &seg, VCPU_SREG_ES);
	seg.selector = vmcs12->host_ss_selector;
	vmx_set_segment(vcpu, &seg, VCPU_SREG_SS);
	seg.selector = vmcs12->host_fs_selector;
	seg.base = vmcs12->host_fs_base;
	vmx_set_segment(vcpu, &seg, VCPU_SREG_FS);
	seg.selector = vmcs12->host_gs_selector;
	seg.base = vmcs12->host_gs_base;
	vmx_set_segment(vcpu, &seg, VCPU_SREG_GS);
	seg = (struct kvm_segment) {
10439
		.base = vmcs12->host_tr_base,
10440 10441 10442 10443 10444 10445 10446
		.limit = 0x67,
		.selector = vmcs12->host_tr_selector,
		.type = 11,
		.present = 1
	};
	vmx_set_segment(vcpu, &seg, VCPU_SREG_TR);

10447 10448
	kvm_set_dr(vcpu, 7, 0x400);
	vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
10449

10450 10451 10452
	if (cpu_has_vmx_msr_bitmap())
		vmx_set_msr_bitmap(vcpu);

10453 10454 10455
	if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr,
				vmcs12->vm_exit_msr_load_count))
		nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL);
10456 10457 10458 10459 10460 10461 10462
}

/*
 * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1
 * and modify vmcs12 to make it see what it would expect to see there if
 * L2 was its real guest. Must only be called when in L2 (is_guest_mode())
 */
10463 10464 10465
static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
			      u32 exit_intr_info,
			      unsigned long exit_qualification)
10466 10467 10468 10469
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);

10470 10471 10472
	/* trying to cancel vmlaunch/vmresume is a bug */
	WARN_ON_ONCE(vmx->nested.nested_run_pending);

10473
	leave_guest_mode(vcpu);
10474 10475
	prepare_vmcs12(vcpu, vmcs12, exit_reason, exit_intr_info,
		       exit_qualification);
10476

10477 10478 10479 10480
	if (nested_vmx_store_msr(vcpu, vmcs12->vm_exit_msr_store_addr,
				 vmcs12->vm_exit_msr_store_count))
		nested_vmx_abort(vcpu, VMX_ABORT_SAVE_GUEST_MSR_FAIL);

10481 10482
	vmx_load_vmcs01(vcpu);

10483 10484 10485 10486 10487 10488 10489 10490
	if ((exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT)
	    && nested_exit_intr_ack_set(vcpu)) {
		int irq = kvm_cpu_get_interrupt(vcpu);
		WARN_ON(irq < 0);
		vmcs12->vm_exit_intr_info = irq |
			INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR;
	}

10491 10492 10493 10494 10495 10496
	trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason,
				       vmcs12->exit_qualification,
				       vmcs12->idt_vectoring_info_field,
				       vmcs12->vm_exit_intr_info,
				       vmcs12->vm_exit_intr_error_code,
				       KVM_ISA_VMX);
10497

10498 10499
	vm_entry_controls_init(vmx, vmcs_read32(VM_ENTRY_CONTROLS));
	vm_exit_controls_init(vmx, vmcs_read32(VM_EXIT_CONTROLS));
10500 10501
	vmx_segment_cache_clear(vmx);

10502 10503 10504 10505 10506 10507
	/* if no vmcs02 cache requested, remove the one we used */
	if (VMCS02_POOL_SIZE == 0)
		nested_free_vmcs02(vmx, vmx->nested.current_vmptr);

	load_vmcs12_host_state(vcpu, vmcs12);

10508
	/* Update TSC_OFFSET if TSC was changed while L2 ran */
10509 10510
	vmcs_write64(TSC_OFFSET, vmx->nested.vmcs01_tsc_offset);

10511 10512 10513 10514 10515 10516
	if (vmx->nested.change_vmcs01_virtual_x2apic_mode) {
		vmx->nested.change_vmcs01_virtual_x2apic_mode = false;
		vmx_set_virtual_x2apic_mode(vcpu,
				vcpu->arch.apic_base & X2APIC_ENABLE);
	}

10517 10518 10519 10520 10521 10522
	/* This is needed for same reason as it was needed in prepare_vmcs02 */
	vmx->host_rsp = 0;

	/* Unpin physical memory we referred to in vmcs02 */
	if (vmx->nested.apic_access_page) {
		nested_release_page(vmx->nested.apic_access_page);
10523
		vmx->nested.apic_access_page = NULL;
10524
	}
10525 10526
	if (vmx->nested.virtual_apic_page) {
		nested_release_page(vmx->nested.virtual_apic_page);
10527
		vmx->nested.virtual_apic_page = NULL;
10528
	}
10529 10530 10531 10532 10533 10534
	if (vmx->nested.pi_desc_page) {
		kunmap(vmx->nested.pi_desc_page);
		nested_release_page(vmx->nested.pi_desc_page);
		vmx->nested.pi_desc_page = NULL;
		vmx->nested.pi_desc = NULL;
	}
10535

10536 10537 10538 10539 10540 10541
	/*
	 * We are now running in L2, mmu_notifier will force to reload the
	 * page's hpa for L2 vmcs. Need to reload it for L1 before entering L1.
	 */
	kvm_vcpu_reload_apic_access_page(vcpu);

10542 10543 10544 10545 10546 10547 10548 10549 10550 10551
	/*
	 * Exiting from L2 to L1, we're now back to L1 which thinks it just
	 * finished a VMLAUNCH or VMRESUME instruction, so we need to set the
	 * success or failure flag accordingly.
	 */
	if (unlikely(vmx->fail)) {
		vmx->fail = 0;
		nested_vmx_failValid(vcpu, vmcs_read32(VM_INSTRUCTION_ERROR));
	} else
		nested_vmx_succeed(vcpu);
10552 10553
	if (enable_shadow_vmcs)
		vmx->nested.sync_shadow_vmcs = true;
10554 10555 10556

	/* in case we halted in L2 */
	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
10557 10558
}

10559 10560 10561 10562 10563 10564
/*
 * Forcibly leave nested mode in order to be able to reset the VCPU later on.
 */
static void vmx_leave_nested(struct kvm_vcpu *vcpu)
{
	if (is_guest_mode(vcpu))
10565
		nested_vmx_vmexit(vcpu, -1, 0, 0);
10566 10567 10568
	free_nested(to_vmx(vcpu));
}

10569 10570 10571 10572 10573 10574 10575 10576 10577 10578 10579 10580 10581 10582 10583
/*
 * L1's failure to enter L2 is a subset of a normal exit, as explained in
 * 23.7 "VM-entry failures during or after loading guest state" (this also
 * lists the acceptable exit-reason and exit-qualification parameters).
 * It should only be called before L2 actually succeeded to run, and when
 * vmcs01 is current (it doesn't leave_guest_mode() or switch vmcss).
 */
static void nested_vmx_entry_failure(struct kvm_vcpu *vcpu,
			struct vmcs12 *vmcs12,
			u32 reason, unsigned long qualification)
{
	load_vmcs12_host_state(vcpu, vmcs12);
	vmcs12->vm_exit_reason = reason | VMX_EXIT_REASONS_FAILED_VMENTRY;
	vmcs12->exit_qualification = qualification;
	nested_vmx_succeed(vcpu);
10584 10585
	if (enable_shadow_vmcs)
		to_vmx(vcpu)->nested.sync_shadow_vmcs = true;
10586 10587
}

10588 10589 10590 10591 10592 10593 10594
static int vmx_check_intercept(struct kvm_vcpu *vcpu,
			       struct x86_instruction_info *info,
			       enum x86_intercept_stage stage)
{
	return X86EMUL_CONTINUE;
}

10595
static void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu)
10596
{
10597 10598
	if (ple_gap)
		shrink_ple_window(vcpu);
10599 10600
}

Kai Huang's avatar
Kai Huang committed
10601 10602 10603 10604 10605 10606 10607 10608 10609 10610 10611 10612 10613 10614 10615 10616 10617 10618 10619 10620 10621 10622 10623 10624 10625
static void vmx_slot_enable_log_dirty(struct kvm *kvm,
				     struct kvm_memory_slot *slot)
{
	kvm_mmu_slot_leaf_clear_dirty(kvm, slot);
	kvm_mmu_slot_largepage_remove_write_access(kvm, slot);
}

static void vmx_slot_disable_log_dirty(struct kvm *kvm,
				       struct kvm_memory_slot *slot)
{
	kvm_mmu_slot_set_dirty(kvm, slot);
}

static void vmx_flush_log_dirty(struct kvm *kvm)
{
	kvm_flush_pml_buffers(kvm);
}

static void vmx_enable_log_dirty_pt_masked(struct kvm *kvm,
					   struct kvm_memory_slot *memslot,
					   gfn_t offset, unsigned long mask)
{
	kvm_mmu_clear_dirty_pt_masked(kvm, memslot, offset, mask);
}

10626 10627 10628 10629 10630 10631 10632 10633 10634 10635 10636 10637 10638 10639 10640 10641 10642 10643 10644 10645 10646 10647 10648 10649 10650 10651 10652 10653 10654 10655 10656 10657 10658 10659 10660 10661 10662 10663 10664 10665 10666 10667 10668 10669 10670 10671 10672 10673 10674 10675 10676 10677 10678 10679 10680 10681 10682 10683 10684 10685 10686 10687 10688 10689 10690 10691 10692 10693 10694 10695 10696 10697 10698 10699 10700 10701 10702 10703 10704 10705 10706 10707 10708 10709 10710 10711 10712 10713 10714 10715 10716 10717 10718 10719 10720 10721 10722 10723 10724 10725 10726 10727 10728 10729 10730 10731 10732 10733 10734 10735 10736 10737 10738 10739 10740 10741 10742 10743 10744 10745
/*
 * This routine does the following things for vCPU which is going
 * to be blocked if VT-d PI is enabled.
 * - Store the vCPU to the wakeup list, so when interrupts happen
 *   we can find the right vCPU to wake up.
 * - Change the Posted-interrupt descriptor as below:
 *      'NDST' <-- vcpu->pre_pcpu
 *      'NV' <-- POSTED_INTR_WAKEUP_VECTOR
 * - If 'ON' is set during this process, which means at least one
 *   interrupt is posted for this vCPU, we cannot block it, in
 *   this case, return 1, otherwise, return 0.
 *
 */
static int vmx_pre_block(struct kvm_vcpu *vcpu)
{
	unsigned long flags;
	unsigned int dest;
	struct pi_desc old, new;
	struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);

	if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
		!irq_remapping_cap(IRQ_POSTING_CAP))
		return 0;

	vcpu->pre_pcpu = vcpu->cpu;
	spin_lock_irqsave(&per_cpu(blocked_vcpu_on_cpu_lock,
			  vcpu->pre_pcpu), flags);
	list_add_tail(&vcpu->blocked_vcpu_list,
		      &per_cpu(blocked_vcpu_on_cpu,
		      vcpu->pre_pcpu));
	spin_unlock_irqrestore(&per_cpu(blocked_vcpu_on_cpu_lock,
			       vcpu->pre_pcpu), flags);

	do {
		old.control = new.control = pi_desc->control;

		/*
		 * We should not block the vCPU if
		 * an interrupt is posted for it.
		 */
		if (pi_test_on(pi_desc) == 1) {
			spin_lock_irqsave(&per_cpu(blocked_vcpu_on_cpu_lock,
					  vcpu->pre_pcpu), flags);
			list_del(&vcpu->blocked_vcpu_list);
			spin_unlock_irqrestore(
					&per_cpu(blocked_vcpu_on_cpu_lock,
					vcpu->pre_pcpu), flags);
			vcpu->pre_pcpu = -1;

			return 1;
		}

		WARN((pi_desc->sn == 1),
		     "Warning: SN field of posted-interrupts "
		     "is set before blocking\n");

		/*
		 * Since vCPU can be preempted during this process,
		 * vcpu->cpu could be different with pre_pcpu, we
		 * need to set pre_pcpu as the destination of wakeup
		 * notification event, then we can find the right vCPU
		 * to wakeup in wakeup handler if interrupts happen
		 * when the vCPU is in blocked state.
		 */
		dest = cpu_physical_id(vcpu->pre_pcpu);

		if (x2apic_enabled())
			new.ndst = dest;
		else
			new.ndst = (dest << 8) & 0xFF00;

		/* set 'NV' to 'wakeup vector' */
		new.nv = POSTED_INTR_WAKEUP_VECTOR;
	} while (cmpxchg(&pi_desc->control, old.control,
			new.control) != old.control);

	return 0;
}

static void vmx_post_block(struct kvm_vcpu *vcpu)
{
	struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
	struct pi_desc old, new;
	unsigned int dest;
	unsigned long flags;

	if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
		!irq_remapping_cap(IRQ_POSTING_CAP))
		return;

	do {
		old.control = new.control = pi_desc->control;

		dest = cpu_physical_id(vcpu->cpu);

		if (x2apic_enabled())
			new.ndst = dest;
		else
			new.ndst = (dest << 8) & 0xFF00;

		/* Allow posting non-urgent interrupts */
		new.sn = 0;

		/* set 'NV' to 'notification vector' */
		new.nv = POSTED_INTR_VECTOR;
	} while (cmpxchg(&pi_desc->control, old.control,
			new.control) != old.control);

	if(vcpu->pre_pcpu != -1) {
		spin_lock_irqsave(
			&per_cpu(blocked_vcpu_on_cpu_lock,
			vcpu->pre_pcpu), flags);
		list_del(&vcpu->blocked_vcpu_list);
		spin_unlock_irqrestore(
			&per_cpu(blocked_vcpu_on_cpu_lock,
			vcpu->pre_pcpu), flags);
		vcpu->pre_pcpu = -1;
	}
}

10746 10747 10748 10749 10750 10751 10752 10753 10754 10755 10756 10757 10758 10759 10760 10761 10762
/*
 * vmx_update_pi_irte - set IRTE for Posted-Interrupts
 *
 * @kvm: kvm
 * @host_irq: host irq of the interrupt
 * @guest_irq: gsi of the interrupt
 * @set: set or unset PI
 * returns 0 on success, < 0 on failure
 */
static int vmx_update_pi_irte(struct kvm *kvm, unsigned int host_irq,
			      uint32_t guest_irq, bool set)
{
	struct kvm_kernel_irq_routing_entry *e;
	struct kvm_irq_routing_table *irq_rt;
	struct kvm_lapic_irq irq;
	struct kvm_vcpu *vcpu;
	struct vcpu_data vcpu_info;
10763
	int idx, ret = 0;
10764 10765 10766 10767 10768 10769 10770

	if (!kvm_arch_has_assigned_device(kvm) ||
		!irq_remapping_cap(IRQ_POSTING_CAP))
		return 0;

	idx = srcu_read_lock(&kvm->irq_srcu);
	irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
10771 10772 10773 10774 10775 10776
	if (guest_irq >= irq_rt->nr_rt_entries ||
	    hlist_empty(&irq_rt->map[guest_irq])) {
		pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n",
			     guest_irq, irq_rt->nr_rt_entries);
		goto out;
	}
10777 10778 10779 10780 10781 10782 10783 10784 10785 10786 10787 10788 10789 10790 10791 10792 10793 10794 10795 10796 10797 10798 10799 10800 10801 10802 10803 10804 10805

	hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
		if (e->type != KVM_IRQ_ROUTING_MSI)
			continue;
		/*
		 * VT-d PI cannot support posting multicast/broadcast
		 * interrupts to a vCPU, we still use interrupt remapping
		 * for these kind of interrupts.
		 *
		 * For lowest-priority interrupts, we only support
		 * those with single CPU as the destination, e.g. user
		 * configures the interrupts via /proc/irq or uses
		 * irqbalance to make the interrupts single-CPU.
		 *
		 * We will support full lowest-priority interrupt later.
		 */

		kvm_set_msi_irq(e, &irq);
		if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu))
			continue;

		vcpu_info.pi_desc_addr = __pa(vcpu_to_pi_desc(vcpu));
		vcpu_info.vector = irq.vector;

		trace_kvm_pi_irte_update(vcpu->vcpu_id, e->gsi,
				vcpu_info.vector, vcpu_info.pi_desc_addr, set);

		if (set)
			ret = irq_set_vcpu_affinity(host_irq, &vcpu_info);
10806
		else
10807 10808 10809 10810 10811 10812 10813 10814 10815 10816 10817 10818 10819 10820 10821
			ret = irq_set_vcpu_affinity(host_irq, NULL);

		if (ret < 0) {
			printk(KERN_INFO "%s: failed to update PI IRTE\n",
					__func__);
			goto out;
		}
	}

	ret = 0;
out:
	srcu_read_unlock(&kvm->irq_srcu, idx);
	return ret;
}

10822
static struct kvm_x86_ops vmx_x86_ops = {
10823 10824 10825 10826
	.cpu_has_kvm_support = cpu_has_kvm_support,
	.disabled_by_bios = vmx_disabled_by_bios,
	.hardware_setup = hardware_setup,
	.hardware_unsetup = hardware_unsetup,
10827
	.check_processor_compatibility = vmx_check_processor_compat,
10828 10829
	.hardware_enable = hardware_enable,
	.hardware_disable = hardware_disable,
10830
	.cpu_has_accelerated_tpr = report_flexpriority,
10831
	.cpu_has_high_real_mode_segbase = vmx_has_high_real_mode_segbase,
10832 10833 10834

	.vcpu_create = vmx_create_vcpu,
	.vcpu_free = vmx_free_vcpu,
10835
	.vcpu_reset = vmx_vcpu_reset,
10836

10837
	.prepare_guest_switch = vmx_save_host_state,
10838 10839 10840
	.vcpu_load = vmx_vcpu_load,
	.vcpu_put = vmx_vcpu_put,

10841
	.update_bp_intercept = update_exception_bitmap,
10842 10843 10844 10845 10846
	.get_msr = vmx_get_msr,
	.set_msr = vmx_set_msr,
	.get_segment_base = vmx_get_segment_base,
	.get_segment = vmx_get_segment,
	.set_segment = vmx_set_segment,
10847
	.get_cpl = vmx_get_cpl,
10848
	.get_cs_db_l_bits = vmx_get_cs_db_l_bits,
10849
	.decache_cr0_guest_bits = vmx_decache_cr0_guest_bits,
10850
	.decache_cr3 = vmx_decache_cr3,
10851
	.decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
10852 10853 10854 10855 10856 10857 10858 10859
	.set_cr0 = vmx_set_cr0,
	.set_cr3 = vmx_set_cr3,
	.set_cr4 = vmx_set_cr4,
	.set_efer = vmx_set_efer,
	.get_idt = vmx_get_idt,
	.set_idt = vmx_set_idt,
	.get_gdt = vmx_get_gdt,
	.set_gdt = vmx_set_gdt,
Jan Kiszka's avatar
Jan Kiszka committed
10860 10861
	.get_dr6 = vmx_get_dr6,
	.set_dr6 = vmx_set_dr6,
10862
	.set_dr7 = vmx_set_dr7,
10863
	.sync_dirty_debug_regs = vmx_sync_dirty_debug_regs,
10864
	.cache_reg = vmx_cache_reg,
10865 10866
	.get_rflags = vmx_get_rflags,
	.set_rflags = vmx_set_rflags,
10867
	.fpu_activate = vmx_fpu_activate,
10868
	.fpu_deactivate = vmx_fpu_deactivate,
10869 10870 10871 10872

	.tlb_flush = vmx_flush_tlb,

	.run = vmx_vcpu_run,
10873
	.handle_exit = vmx_handle_exit,
10874
	.skip_emulated_instruction = skip_emulated_instruction,
10875 10876
	.set_interrupt_shadow = vmx_set_interrupt_shadow,
	.get_interrupt_shadow = vmx_get_interrupt_shadow,
10877
	.patch_hypercall = vmx_patch_hypercall,
Eddie Dong's avatar
Eddie Dong committed
10878
	.set_irq = vmx_inject_irq,
10879
	.set_nmi = vmx_inject_nmi,
10880
	.queue_exception = vmx_queue_exception,
10881
	.cancel_injection = vmx_cancel_injection,
10882
	.interrupt_allowed = vmx_interrupt_allowed,
10883
	.nmi_allowed = vmx_nmi_allowed,
10884 10885
	.get_nmi_mask = vmx_get_nmi_mask,
	.set_nmi_mask = vmx_set_nmi_mask,
10886 10887 10888
	.enable_nmi_window = enable_nmi_window,
	.enable_irq_window = enable_irq_window,
	.update_cr8_intercept = update_cr8_intercept,
10889
	.set_virtual_x2apic_mode = vmx_set_virtual_x2apic_mode,
10890
	.set_apic_access_page_addr = vmx_set_apic_access_page_addr,
10891
	.cpu_uses_apicv = vmx_cpu_uses_apicv,
10892 10893 10894
	.load_eoi_exitmap = vmx_load_eoi_exitmap,
	.hwapic_irr_update = vmx_hwapic_irr_update,
	.hwapic_isr_update = vmx_hwapic_isr_update,
10895 10896
	.sync_pir_to_irr = vmx_sync_pir_to_irr,
	.deliver_posted_interrupt = vmx_deliver_posted_interrupt,
10897

10898
	.set_tss_addr = vmx_set_tss_addr,
10899
	.get_tdp_level = get_ept_level,
10900
	.get_mt_mask = vmx_get_mt_mask,
10901

10902 10903
	.get_exit_info = vmx_get_exit_info,

10904
	.get_lpage_level = vmx_get_lpage_level,
10905 10906

	.cpuid_update = vmx_cpuid_update,
10907 10908

	.rdtscp_supported = vmx_rdtscp_supported,
10909
	.invpcid_supported = vmx_invpcid_supported,
10910 10911

	.set_supported_cpuid = vmx_set_supported_cpuid,
10912 10913

	.has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
10914

10915
	.read_tsc_offset = vmx_read_tsc_offset,
10916
	.write_tsc_offset = vmx_write_tsc_offset,
10917
	.adjust_tsc_offset_guest = vmx_adjust_tsc_offset_guest,
Nadav Har'El's avatar
Nadav Har'El committed
10918
	.read_l1_tsc = vmx_read_l1_tsc,
10919 10920

	.set_tdp_cr3 = vmx_set_cr3,
10921 10922

	.check_intercept = vmx_check_intercept,
10923
	.handle_external_intr = vmx_handle_external_intr,
10924
	.mpx_supported = vmx_mpx_supported,
10925
	.xsaves_supported = vmx_xsaves_supported,
10926 10927

	.check_nested_events = vmx_check_nested_events,
10928 10929

	.sched_in = vmx_sched_in,
Kai Huang's avatar
Kai Huang committed
10930 10931 10932 10933 10934

	.slot_enable_log_dirty = vmx_slot_enable_log_dirty,
	.slot_disable_log_dirty = vmx_slot_disable_log_dirty,
	.flush_log_dirty = vmx_flush_log_dirty,
	.enable_log_dirty_pt_masked = vmx_enable_log_dirty_pt_masked,
10935

10936 10937 10938
	.pre_block = vmx_pre_block,
	.post_block = vmx_post_block,

10939
	.pmu_ops = &intel_pmu_ops,
10940 10941

	.update_pi_irte = vmx_update_pi_irte,
10942 10943 10944 10945
};

static int __init vmx_init(void)
{
10946 10947
	int r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx),
                     __alignof__(struct vcpu_vmx), THIS_MODULE);
10948
	if (r)
10949
		return r;
10950

10951
#ifdef CONFIG_KEXEC_CORE
10952 10953 10954 10955
	rcu_assign_pointer(crash_vmclear_loaded_vmcss,
			   crash_vmclear_local_loaded_vmcss);
#endif

10956
	return 0;
10957 10958 10959 10960
}

static void __exit vmx_exit(void)
{
10961
#ifdef CONFIG_KEXEC_CORE
10962
	RCU_INIT_POINTER(crash_vmclear_loaded_vmcss, NULL);
10963 10964 10965
	synchronize_rcu();
#endif

10966
	kvm_exit();
10967 10968 10969 10970
}

module_init(vmx_init)
module_exit(vmx_exit)