Commit db6f2040 authored by Linus Torvalds's avatar Linus Torvalds

Merge git://git.kernel.org/pub/scm/linux/kernel/git/rusty/linux-2.6-lguest-and-virtio

* git://git.kernel.org/pub/scm/linux/kernel/git/rusty/linux-2.6-lguest-and-virtio:
  lguest: barrier me harder
  lguest: use bool instead of int
  lguest: use KVM hypercalls
  lguest: wire up pte_update/pte_update_defer
  lguest: fix spurious BUG_ON() on invalid guest stack.
  virtio: more neatening of virtio_ring macros.
  virtio: fix BAD_RING, START_US and END_USE macros
parents 3c6fae67 d1881d31
...@@ -1630,6 +1630,13 @@ static bool service_io(struct device *dev) ...@@ -1630,6 +1630,13 @@ static bool service_io(struct device *dev)
} }
} }
/* OK, so we noted that it was pretty poor to use an fdatasync as a
* barrier. But Christoph Hellwig points out that we need a sync
* *afterwards* as well: "Barriers specify no reordering to the front
* or the back." And Jens Axboe confirmed it, so here we are: */
if (out->type & VIRTIO_BLK_T_BARRIER)
fdatasync(vblk->fd);
/* We can't trigger an IRQ, because we're not the Launcher. It does /* We can't trigger an IRQ, because we're not the Launcher. It does
* that when we tell it we're done. */ * that when we tell it we're done. */
add_used(dev->vq, head, wlen); add_used(dev->vq, head, wlen);
......
...@@ -26,36 +26,20 @@ ...@@ -26,36 +26,20 @@
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
#include <asm/hw_irq.h> #include <asm/hw_irq.h>
#include <asm/kvm_para.h>
/*G:031 But first, how does our Guest contact the Host to ask for privileged /*G:031 But first, how does our Guest contact the Host to ask for privileged
* operations? There are two ways: the direct way is to make a "hypercall", * operations? There are two ways: the direct way is to make a "hypercall",
* to make requests of the Host Itself. * to make requests of the Host Itself.
* *
* Our hypercall mechanism uses the highest unused trap code (traps 32 and * We use the KVM hypercall mechanism. Eighteen hypercalls are
* above are used by real hardware interrupts). Fifteen hypercalls are
* available: the hypercall number is put in the %eax register, and the * available: the hypercall number is put in the %eax register, and the
* arguments (when required) are placed in %edx, %ebx and %ecx. If a return * arguments (when required) are placed in %ebx, %ecx and %edx. If a return
* value makes sense, it's returned in %eax. * value makes sense, it's returned in %eax.
* *
* Grossly invalid calls result in Sudden Death at the hands of the vengeful * Grossly invalid calls result in Sudden Death at the hands of the vengeful
* Host, rather than returning failure. This reflects Winston Churchill's * Host, rather than returning failure. This reflects Winston Churchill's
* definition of a gentleman: "someone who is only rude intentionally". */ * definition of a gentleman: "someone who is only rude intentionally". */
static inline unsigned long
hcall(unsigned long call,
unsigned long arg1, unsigned long arg2, unsigned long arg3)
{
/* "int" is the Intel instruction to trigger a trap. */
asm volatile("int $" __stringify(LGUEST_TRAP_ENTRY)
/* The call in %eax (aka "a") might be overwritten */
: "=a"(call)
/* The arguments are in %eax, %edx, %ebx & %ecx */
: "a"(call), "d"(arg1), "b"(arg2), "c"(arg3)
/* "memory" means this might write somewhere in memory.
* This isn't true for all calls, but it's safe to tell
* gcc that it might happen so it doesn't get clever. */
: "memory");
return call;
}
/*:*/ /*:*/
/* Can't use our min() macro here: needs to be a constant */ /* Can't use our min() macro here: needs to be a constant */
...@@ -64,7 +48,7 @@ hcall(unsigned long call, ...@@ -64,7 +48,7 @@ hcall(unsigned long call,
#define LHCALL_RING_SIZE 64 #define LHCALL_RING_SIZE 64
struct hcall_args { struct hcall_args {
/* These map directly onto eax, ebx, ecx, edx in struct lguest_regs */ /* These map directly onto eax, ebx, ecx, edx in struct lguest_regs */
unsigned long arg0, arg2, arg3, arg1; unsigned long arg0, arg1, arg2, arg3;
}; };
#endif /* !__ASSEMBLY__ */ #endif /* !__ASSEMBLY__ */
......
...@@ -107,7 +107,7 @@ static void async_hcall(unsigned long call, unsigned long arg1, ...@@ -107,7 +107,7 @@ static void async_hcall(unsigned long call, unsigned long arg1,
local_irq_save(flags); local_irq_save(flags);
if (lguest_data.hcall_status[next_call] != 0xFF) { if (lguest_data.hcall_status[next_call] != 0xFF) {
/* Table full, so do normal hcall which will flush table. */ /* Table full, so do normal hcall which will flush table. */
hcall(call, arg1, arg2, arg3); kvm_hypercall3(call, arg1, arg2, arg3);
} else { } else {
lguest_data.hcalls[next_call].arg0 = call; lguest_data.hcalls[next_call].arg0 = call;
lguest_data.hcalls[next_call].arg1 = arg1; lguest_data.hcalls[next_call].arg1 = arg1;
...@@ -134,13 +134,32 @@ static void async_hcall(unsigned long call, unsigned long arg1, ...@@ -134,13 +134,32 @@ static void async_hcall(unsigned long call, unsigned long arg1,
* *
* So, when we're in lazy mode, we call async_hcall() to store the call for * So, when we're in lazy mode, we call async_hcall() to store the call for
* future processing: */ * future processing: */
static void lazy_hcall(unsigned long call, static void lazy_hcall1(unsigned long call,
unsigned long arg1)
{
if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE)
kvm_hypercall1(call, arg1);
else
async_hcall(call, arg1, 0, 0);
}
static void lazy_hcall2(unsigned long call,
unsigned long arg1,
unsigned long arg2)
{
if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE)
kvm_hypercall2(call, arg1, arg2);
else
async_hcall(call, arg1, arg2, 0);
}
static void lazy_hcall3(unsigned long call,
unsigned long arg1, unsigned long arg1,
unsigned long arg2, unsigned long arg2,
unsigned long arg3) unsigned long arg3)
{ {
if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE)
hcall(call, arg1, arg2, arg3); kvm_hypercall3(call, arg1, arg2, arg3);
else else
async_hcall(call, arg1, arg2, arg3); async_hcall(call, arg1, arg2, arg3);
} }
...@@ -150,7 +169,7 @@ static void lazy_hcall(unsigned long call, ...@@ -150,7 +169,7 @@ static void lazy_hcall(unsigned long call,
static void lguest_leave_lazy_mode(void) static void lguest_leave_lazy_mode(void)
{ {
paravirt_leave_lazy(paravirt_get_lazy_mode()); paravirt_leave_lazy(paravirt_get_lazy_mode());
hcall(LHCALL_FLUSH_ASYNC, 0, 0, 0); kvm_hypercall0(LHCALL_FLUSH_ASYNC);
} }
/*G:033 /*G:033
...@@ -229,7 +248,7 @@ static void lguest_write_idt_entry(gate_desc *dt, ...@@ -229,7 +248,7 @@ static void lguest_write_idt_entry(gate_desc *dt,
/* Keep the local copy up to date. */ /* Keep the local copy up to date. */
native_write_idt_entry(dt, entrynum, g); native_write_idt_entry(dt, entrynum, g);
/* Tell Host about this new entry. */ /* Tell Host about this new entry. */
hcall(LHCALL_LOAD_IDT_ENTRY, entrynum, desc[0], desc[1]); kvm_hypercall3(LHCALL_LOAD_IDT_ENTRY, entrynum, desc[0], desc[1]);
} }
/* Changing to a different IDT is very rare: we keep the IDT up-to-date every /* Changing to a different IDT is very rare: we keep the IDT up-to-date every
...@@ -241,7 +260,7 @@ static void lguest_load_idt(const struct desc_ptr *desc) ...@@ -241,7 +260,7 @@ static void lguest_load_idt(const struct desc_ptr *desc)
struct desc_struct *idt = (void *)desc->address; struct desc_struct *idt = (void *)desc->address;
for (i = 0; i < (desc->size+1)/8; i++) for (i = 0; i < (desc->size+1)/8; i++)
hcall(LHCALL_LOAD_IDT_ENTRY, i, idt[i].a, idt[i].b); kvm_hypercall3(LHCALL_LOAD_IDT_ENTRY, i, idt[i].a, idt[i].b);
} }
/* /*
...@@ -261,8 +280,8 @@ static void lguest_load_idt(const struct desc_ptr *desc) ...@@ -261,8 +280,8 @@ static void lguest_load_idt(const struct desc_ptr *desc)
*/ */
static void lguest_load_gdt(const struct desc_ptr *desc) static void lguest_load_gdt(const struct desc_ptr *desc)
{ {
BUG_ON((desc->size+1)/8 != GDT_ENTRIES); BUG_ON((desc->size + 1) / 8 != GDT_ENTRIES);
hcall(LHCALL_LOAD_GDT, __pa(desc->address), GDT_ENTRIES, 0); kvm_hypercall2(LHCALL_LOAD_GDT, __pa(desc->address), GDT_ENTRIES);
} }
/* For a single GDT entry which changes, we do the lazy thing: alter our GDT, /* For a single GDT entry which changes, we do the lazy thing: alter our GDT,
...@@ -272,7 +291,7 @@ static void lguest_write_gdt_entry(struct desc_struct *dt, int entrynum, ...@@ -272,7 +291,7 @@ static void lguest_write_gdt_entry(struct desc_struct *dt, int entrynum,
const void *desc, int type) const void *desc, int type)
{ {
native_write_gdt_entry(dt, entrynum, desc, type); native_write_gdt_entry(dt, entrynum, desc, type);
hcall(LHCALL_LOAD_GDT, __pa(dt), GDT_ENTRIES, 0); kvm_hypercall2(LHCALL_LOAD_GDT, __pa(dt), GDT_ENTRIES);
} }
/* OK, I lied. There are three "thread local storage" GDT entries which change /* OK, I lied. There are three "thread local storage" GDT entries which change
...@@ -284,7 +303,7 @@ static void lguest_load_tls(struct thread_struct *t, unsigned int cpu) ...@@ -284,7 +303,7 @@ static void lguest_load_tls(struct thread_struct *t, unsigned int cpu)
* can't handle us removing entries we're currently using. So we clear * can't handle us removing entries we're currently using. So we clear
* the GS register here: if it's needed it'll be reloaded anyway. */ * the GS register here: if it's needed it'll be reloaded anyway. */
lazy_load_gs(0); lazy_load_gs(0);
lazy_hcall(LHCALL_LOAD_TLS, __pa(&t->tls_array), cpu, 0); lazy_hcall2(LHCALL_LOAD_TLS, __pa(&t->tls_array), cpu);
} }
/*G:038 That's enough excitement for now, back to ploughing through each of /*G:038 That's enough excitement for now, back to ploughing through each of
...@@ -382,7 +401,7 @@ static void lguest_cpuid(unsigned int *ax, unsigned int *bx, ...@@ -382,7 +401,7 @@ static void lguest_cpuid(unsigned int *ax, unsigned int *bx,
static unsigned long current_cr0; static unsigned long current_cr0;
static void lguest_write_cr0(unsigned long val) static void lguest_write_cr0(unsigned long val)
{ {
lazy_hcall(LHCALL_TS, val & X86_CR0_TS, 0, 0); lazy_hcall1(LHCALL_TS, val & X86_CR0_TS);
current_cr0 = val; current_cr0 = val;
} }
...@@ -396,7 +415,7 @@ static unsigned long lguest_read_cr0(void) ...@@ -396,7 +415,7 @@ static unsigned long lguest_read_cr0(void)
* the vowels have been optimized out. */ * the vowels have been optimized out. */
static void lguest_clts(void) static void lguest_clts(void)
{ {
lazy_hcall(LHCALL_TS, 0, 0, 0); lazy_hcall1(LHCALL_TS, 0);
current_cr0 &= ~X86_CR0_TS; current_cr0 &= ~X86_CR0_TS;
} }
...@@ -418,7 +437,7 @@ static bool cr3_changed = false; ...@@ -418,7 +437,7 @@ static bool cr3_changed = false;
static void lguest_write_cr3(unsigned long cr3) static void lguest_write_cr3(unsigned long cr3)
{ {
lguest_data.pgdir = cr3; lguest_data.pgdir = cr3;
lazy_hcall(LHCALL_NEW_PGTABLE, cr3, 0, 0); lazy_hcall1(LHCALL_NEW_PGTABLE, cr3);
cr3_changed = true; cr3_changed = true;
} }
...@@ -490,11 +509,17 @@ static void lguest_write_cr4(unsigned long val) ...@@ -490,11 +509,17 @@ static void lguest_write_cr4(unsigned long val)
* into a process' address space. We set the entry then tell the Host the * into a process' address space. We set the entry then tell the Host the
* toplevel and address this corresponds to. The Guest uses one pagetable per * toplevel and address this corresponds to. The Guest uses one pagetable per
* process, so we need to tell the Host which one we're changing (mm->pgd). */ * process, so we need to tell the Host which one we're changing (mm->pgd). */
static void lguest_pte_update(struct mm_struct *mm, unsigned long addr,
pte_t *ptep)
{
lazy_hcall3(LHCALL_SET_PTE, __pa(mm->pgd), addr, ptep->pte_low);
}
static void lguest_set_pte_at(struct mm_struct *mm, unsigned long addr, static void lguest_set_pte_at(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t pteval) pte_t *ptep, pte_t pteval)
{ {
*ptep = pteval; *ptep = pteval;
lazy_hcall(LHCALL_SET_PTE, __pa(mm->pgd), addr, pteval.pte_low); lguest_pte_update(mm, addr, ptep);
} }
/* The Guest calls this to set a top-level entry. Again, we set the entry then /* The Guest calls this to set a top-level entry. Again, we set the entry then
...@@ -503,8 +528,8 @@ static void lguest_set_pte_at(struct mm_struct *mm, unsigned long addr, ...@@ -503,8 +528,8 @@ static void lguest_set_pte_at(struct mm_struct *mm, unsigned long addr,
static void lguest_set_pmd(pmd_t *pmdp, pmd_t pmdval) static void lguest_set_pmd(pmd_t *pmdp, pmd_t pmdval)
{ {
*pmdp = pmdval; *pmdp = pmdval;
lazy_hcall(LHCALL_SET_PMD, __pa(pmdp)&PAGE_MASK, lazy_hcall2(LHCALL_SET_PMD, __pa(pmdp) & PAGE_MASK,
(__pa(pmdp)&(PAGE_SIZE-1))/4, 0); (__pa(pmdp) & (PAGE_SIZE - 1)) / 4);
} }
/* There are a couple of legacy places where the kernel sets a PTE, but we /* There are a couple of legacy places where the kernel sets a PTE, but we
...@@ -520,7 +545,7 @@ static void lguest_set_pte(pte_t *ptep, pte_t pteval) ...@@ -520,7 +545,7 @@ static void lguest_set_pte(pte_t *ptep, pte_t pteval)
{ {
*ptep = pteval; *ptep = pteval;
if (cr3_changed) if (cr3_changed)
lazy_hcall(LHCALL_FLUSH_TLB, 1, 0, 0); lazy_hcall1(LHCALL_FLUSH_TLB, 1);
} }
/* Unfortunately for Lguest, the pv_mmu_ops for page tables were based on /* Unfortunately for Lguest, the pv_mmu_ops for page tables were based on
...@@ -536,7 +561,7 @@ static void lguest_set_pte(pte_t *ptep, pte_t pteval) ...@@ -536,7 +561,7 @@ static void lguest_set_pte(pte_t *ptep, pte_t pteval)
static void lguest_flush_tlb_single(unsigned long addr) static void lguest_flush_tlb_single(unsigned long addr)
{ {
/* Simply set it to zero: if it was not, it will fault back in. */ /* Simply set it to zero: if it was not, it will fault back in. */
lazy_hcall(LHCALL_SET_PTE, lguest_data.pgdir, addr, 0); lazy_hcall3(LHCALL_SET_PTE, lguest_data.pgdir, addr, 0);
} }
/* This is what happens after the Guest has removed a large number of entries. /* This is what happens after the Guest has removed a large number of entries.
...@@ -544,7 +569,7 @@ static void lguest_flush_tlb_single(unsigned long addr) ...@@ -544,7 +569,7 @@ static void lguest_flush_tlb_single(unsigned long addr)
* have changed, ie. virtual addresses below PAGE_OFFSET. */ * have changed, ie. virtual addresses below PAGE_OFFSET. */
static void lguest_flush_tlb_user(void) static void lguest_flush_tlb_user(void)
{ {
lazy_hcall(LHCALL_FLUSH_TLB, 0, 0, 0); lazy_hcall1(LHCALL_FLUSH_TLB, 0);
} }
/* This is called when the kernel page tables have changed. That's not very /* This is called when the kernel page tables have changed. That's not very
...@@ -552,7 +577,7 @@ static void lguest_flush_tlb_user(void) ...@@ -552,7 +577,7 @@ static void lguest_flush_tlb_user(void)
* slow), so it's worth separating this from the user flushing above. */ * slow), so it's worth separating this from the user flushing above. */
static void lguest_flush_tlb_kernel(void) static void lguest_flush_tlb_kernel(void)
{ {
lazy_hcall(LHCALL_FLUSH_TLB, 1, 0, 0); lazy_hcall1(LHCALL_FLUSH_TLB, 1);
} }
/* /*
...@@ -689,7 +714,7 @@ static int lguest_clockevent_set_next_event(unsigned long delta, ...@@ -689,7 +714,7 @@ static int lguest_clockevent_set_next_event(unsigned long delta,
} }
/* Please wake us this far in the future. */ /* Please wake us this far in the future. */
hcall(LHCALL_SET_CLOCKEVENT, delta, 0, 0); kvm_hypercall1(LHCALL_SET_CLOCKEVENT, delta);
return 0; return 0;
} }
...@@ -700,7 +725,7 @@ static void lguest_clockevent_set_mode(enum clock_event_mode mode, ...@@ -700,7 +725,7 @@ static void lguest_clockevent_set_mode(enum clock_event_mode mode,
case CLOCK_EVT_MODE_UNUSED: case CLOCK_EVT_MODE_UNUSED:
case CLOCK_EVT_MODE_SHUTDOWN: case CLOCK_EVT_MODE_SHUTDOWN:
/* A 0 argument shuts the clock down. */ /* A 0 argument shuts the clock down. */
hcall(LHCALL_SET_CLOCKEVENT, 0, 0, 0); kvm_hypercall0(LHCALL_SET_CLOCKEVENT);
break; break;
case CLOCK_EVT_MODE_ONESHOT: case CLOCK_EVT_MODE_ONESHOT:
/* This is what we expect. */ /* This is what we expect. */
...@@ -775,8 +800,8 @@ static void lguest_time_init(void) ...@@ -775,8 +800,8 @@ static void lguest_time_init(void)
static void lguest_load_sp0(struct tss_struct *tss, static void lguest_load_sp0(struct tss_struct *tss,
struct thread_struct *thread) struct thread_struct *thread)
{ {
lazy_hcall(LHCALL_SET_STACK, __KERNEL_DS|0x1, thread->sp0, lazy_hcall3(LHCALL_SET_STACK, __KERNEL_DS | 0x1, thread->sp0,
THREAD_SIZE/PAGE_SIZE); THREAD_SIZE / PAGE_SIZE);
} }
/* Let's just say, I wouldn't do debugging under a Guest. */ /* Let's just say, I wouldn't do debugging under a Guest. */
...@@ -849,7 +874,7 @@ static void set_lguest_basic_apic_ops(void) ...@@ -849,7 +874,7 @@ static void set_lguest_basic_apic_ops(void)
/* STOP! Until an interrupt comes in. */ /* STOP! Until an interrupt comes in. */
static void lguest_safe_halt(void) static void lguest_safe_halt(void)
{ {
hcall(LHCALL_HALT, 0, 0, 0); kvm_hypercall0(LHCALL_HALT);
} }
/* The SHUTDOWN hypercall takes a string to describe what's happening, and /* The SHUTDOWN hypercall takes a string to describe what's happening, and
...@@ -859,7 +884,8 @@ static void lguest_safe_halt(void) ...@@ -859,7 +884,8 @@ static void lguest_safe_halt(void)
* rather than virtual addresses, so we use __pa() here. */ * rather than virtual addresses, so we use __pa() here. */
static void lguest_power_off(void) static void lguest_power_off(void)
{ {
hcall(LHCALL_SHUTDOWN, __pa("Power down"), LGUEST_SHUTDOWN_POWEROFF, 0); kvm_hypercall2(LHCALL_SHUTDOWN, __pa("Power down"),
LGUEST_SHUTDOWN_POWEROFF);
} }
/* /*
...@@ -869,7 +895,7 @@ static void lguest_power_off(void) ...@@ -869,7 +895,7 @@ static void lguest_power_off(void)
*/ */
static int lguest_panic(struct notifier_block *nb, unsigned long l, void *p) static int lguest_panic(struct notifier_block *nb, unsigned long l, void *p)
{ {
hcall(LHCALL_SHUTDOWN, __pa(p), LGUEST_SHUTDOWN_POWEROFF, 0); kvm_hypercall2(LHCALL_SHUTDOWN, __pa(p), LGUEST_SHUTDOWN_POWEROFF);
/* The hcall won't return, but to keep gcc happy, we're "done". */ /* The hcall won't return, but to keep gcc happy, we're "done". */
return NOTIFY_DONE; return NOTIFY_DONE;
} }
...@@ -910,7 +936,7 @@ static __init int early_put_chars(u32 vtermno, const char *buf, int count) ...@@ -910,7 +936,7 @@ static __init int early_put_chars(u32 vtermno, const char *buf, int count)
len = sizeof(scratch) - 1; len = sizeof(scratch) - 1;
scratch[len] = '\0'; scratch[len] = '\0';
memcpy(scratch, buf, len); memcpy(scratch, buf, len);
hcall(LHCALL_NOTIFY, __pa(scratch), 0, 0); kvm_hypercall1(LHCALL_NOTIFY, __pa(scratch));
/* This routine returns the number of bytes actually written. */ /* This routine returns the number of bytes actually written. */
return len; return len;
...@@ -920,7 +946,7 @@ static __init int early_put_chars(u32 vtermno, const char *buf, int count) ...@@ -920,7 +946,7 @@ static __init int early_put_chars(u32 vtermno, const char *buf, int count)
* Launcher to reboot us. */ * Launcher to reboot us. */
static void lguest_restart(char *reason) static void lguest_restart(char *reason)
{ {
hcall(LHCALL_SHUTDOWN, __pa(reason), LGUEST_SHUTDOWN_RESTART, 0); kvm_hypercall2(LHCALL_SHUTDOWN, __pa(reason), LGUEST_SHUTDOWN_RESTART);
} }
/*G:050 /*G:050
...@@ -1040,6 +1066,8 @@ __init void lguest_init(void) ...@@ -1040,6 +1066,8 @@ __init void lguest_init(void)
pv_mmu_ops.read_cr3 = lguest_read_cr3; pv_mmu_ops.read_cr3 = lguest_read_cr3;
pv_mmu_ops.lazy_mode.enter = paravirt_enter_lazy_mmu; pv_mmu_ops.lazy_mode.enter = paravirt_enter_lazy_mmu;
pv_mmu_ops.lazy_mode.leave = lguest_leave_lazy_mode; pv_mmu_ops.lazy_mode.leave = lguest_leave_lazy_mode;
pv_mmu_ops.pte_update = lguest_pte_update;
pv_mmu_ops.pte_update_defer = lguest_pte_update;
#ifdef CONFIG_X86_LOCAL_APIC #ifdef CONFIG_X86_LOCAL_APIC
/* apic read/write intercepts */ /* apic read/write intercepts */
......
...@@ -27,8 +27,8 @@ ENTRY(lguest_entry) ...@@ -27,8 +27,8 @@ ENTRY(lguest_entry)
/* We make the "initialization" hypercall now to tell the Host about /* We make the "initialization" hypercall now to tell the Host about
* us, and also find out where it put our page tables. */ * us, and also find out where it put our page tables. */
movl $LHCALL_LGUEST_INIT, %eax movl $LHCALL_LGUEST_INIT, %eax
movl $lguest_data - __PAGE_OFFSET, %edx movl $lguest_data - __PAGE_OFFSET, %ebx
int $LGUEST_TRAP_ENTRY .byte 0x0f,0x01,0xc1 /* KVM_HYPERCALL */
/* Set up the initial stack so we can run C code. */ /* Set up the initial stack so we can run C code. */
movl $(init_thread_union+THREAD_SIZE),%esp movl $(init_thread_union+THREAD_SIZE),%esp
......
...@@ -152,7 +152,7 @@ static void unmap_switcher(void) ...@@ -152,7 +152,7 @@ static void unmap_switcher(void)
* code. We have to check that the range is below the pfn_limit the Launcher * code. We have to check that the range is below the pfn_limit the Launcher
* gave us. We have to make sure that addr + len doesn't give us a false * gave us. We have to make sure that addr + len doesn't give us a false
* positive by overflowing, too. */ * positive by overflowing, too. */
int lguest_address_ok(const struct lguest *lg, bool lguest_address_ok(const struct lguest *lg,
unsigned long addr, unsigned long len) unsigned long addr, unsigned long len)
{ {
return (addr+len) / PAGE_SIZE < lg->pfn_limit && (addr+len >= addr); return (addr+len) / PAGE_SIZE < lg->pfn_limit && (addr+len >= addr);
......
...@@ -34,7 +34,7 @@ static int idt_type(u32 lo, u32 hi) ...@@ -34,7 +34,7 @@ static int idt_type(u32 lo, u32 hi)
} }
/* An IDT entry can't be used unless the "present" bit is set. */ /* An IDT entry can't be used unless the "present" bit is set. */
static int idt_present(u32 lo, u32 hi) static bool idt_present(u32 lo, u32 hi)
{ {
return (hi & 0x8000); return (hi & 0x8000);
} }
...@@ -60,7 +60,8 @@ static void push_guest_stack(struct lg_cpu *cpu, unsigned long *gstack, u32 val) ...@@ -60,7 +60,8 @@ static void push_guest_stack(struct lg_cpu *cpu, unsigned long *gstack, u32 val)
* We set up the stack just like the CPU does for a real interrupt, so it's * We set up the stack just like the CPU does for a real interrupt, so it's
* identical for the Guest (and the standard "iret" instruction will undo * identical for the Guest (and the standard "iret" instruction will undo
* it). */ * it). */
static void set_guest_interrupt(struct lg_cpu *cpu, u32 lo, u32 hi, int has_err) static void set_guest_interrupt(struct lg_cpu *cpu, u32 lo, u32 hi,
bool has_err)
{ {
unsigned long gstack, origstack; unsigned long gstack, origstack;
u32 eflags, ss, irq_enable; u32 eflags, ss, irq_enable;
...@@ -184,7 +185,7 @@ void maybe_do_interrupt(struct lg_cpu *cpu) ...@@ -184,7 +185,7 @@ void maybe_do_interrupt(struct lg_cpu *cpu)
/* set_guest_interrupt() takes the interrupt descriptor and a /* set_guest_interrupt() takes the interrupt descriptor and a
* flag to say whether this interrupt pushes an error code onto * flag to say whether this interrupt pushes an error code onto
* the stack as well: virtual interrupts never do. */ * the stack as well: virtual interrupts never do. */
set_guest_interrupt(cpu, idt->a, idt->b, 0); set_guest_interrupt(cpu, idt->a, idt->b, false);
} }
/* Every time we deliver an interrupt, we update the timestamp in the /* Every time we deliver an interrupt, we update the timestamp in the
...@@ -244,26 +245,26 @@ void free_interrupts(void) ...@@ -244,26 +245,26 @@ void free_interrupts(void)
/*H:220 Now we've got the routines to deliver interrupts, delivering traps like /*H:220 Now we've got the routines to deliver interrupts, delivering traps like
* page fault is easy. The only trick is that Intel decided that some traps * page fault is easy. The only trick is that Intel decided that some traps
* should have error codes: */ * should have error codes: */
static int has_err(unsigned int trap) static bool has_err(unsigned int trap)
{ {
return (trap == 8 || (trap >= 10 && trap <= 14) || trap == 17); return (trap == 8 || (trap >= 10 && trap <= 14) || trap == 17);
} }
/* deliver_trap() returns true if it could deliver the trap. */ /* deliver_trap() returns true if it could deliver the trap. */
int deliver_trap(struct lg_cpu *cpu, unsigned int num) bool deliver_trap(struct lg_cpu *cpu, unsigned int num)
{ {
/* Trap numbers are always 8 bit, but we set an impossible trap number /* Trap numbers are always 8 bit, but we set an impossible trap number
* for traps inside the Switcher, so check that here. */ * for traps inside the Switcher, so check that here. */
if (num >= ARRAY_SIZE(cpu->arch.idt)) if (num >= ARRAY_SIZE(cpu->arch.idt))
return 0; return false;
/* Early on the Guest hasn't set the IDT entries (or maybe it put a /* Early on the Guest hasn't set the IDT entries (or maybe it put a
* bogus one in): if we fail here, the Guest will be killed. */ * bogus one in): if we fail here, the Guest will be killed. */
if (!idt_present(cpu->arch.idt[num].a, cpu->arch.idt[num].b)) if (!idt_present(cpu->arch.idt[num].a, cpu->arch.idt[num].b))
return 0; return false;
set_guest_interrupt(cpu, cpu->arch.idt[num].a, set_guest_interrupt(cpu, cpu->arch.idt[num].a,
cpu->arch.idt[num].b, has_err(num)); cpu->arch.idt[num].b, has_err(num));
return 1; return true;
} }
/*H:250 Here's the hard part: returning to the Host every time a trap happens /*H:250 Here's the hard part: returning to the Host every time a trap happens
...@@ -279,18 +280,19 @@ int deliver_trap(struct lg_cpu *cpu, unsigned int num) ...@@ -279,18 +280,19 @@ int deliver_trap(struct lg_cpu *cpu, unsigned int num)
* *
* This routine indicates if a particular trap number could be delivered * This routine indicates if a particular trap number could be delivered
* directly. */ * directly. */
static int direct_trap(unsigned int num) static bool direct_trap(unsigned int num)
{ {
/* Hardware interrupts don't go to the Guest at all (except system /* Hardware interrupts don't go to the Guest at all (except system
* call). */ * call). */
if (num >= FIRST_EXTERNAL_VECTOR && !could_be_syscall(num)) if (num >= FIRST_EXTERNAL_VECTOR && !could_be_syscall(num))
return 0; return false;
/* The Host needs to see page faults (for shadow paging and to save the /* The Host needs to see page faults (for shadow paging and to save the
* fault address), general protection faults (in/out emulation) and * fault address), general protection faults (in/out emulation) and
* device not available (TS handling), and of course, the hypercall * device not available (TS handling), invalid opcode fault (kvm hcall),
* trap. */ * and of course, the hypercall trap. */
return num != 14 && num != 13 && num != 7 && num != LGUEST_TRAP_ENTRY; return num != 14 && num != 13 && num != 7 &&
num != 6 && num != LGUEST_TRAP_ENTRY;
} }
/*:*/ /*:*/
......
...@@ -109,7 +109,7 @@ struct lguest ...@@ -109,7 +109,7 @@ struct lguest
extern struct mutex lguest_lock; extern struct mutex lguest_lock;
/* core.c: */ /* core.c: */
int lguest_address_ok(const struct lguest *lg, bool lguest_address_ok(const struct lguest *lg,
unsigned long addr, unsigned long len); unsigned long addr, unsigned long len);
void __lgread(struct lg_cpu *, void *, unsigned long, unsigned); void __lgread(struct lg_cpu *, void *, unsigned long, unsigned);
void __lgwrite(struct lg_cpu *, unsigned long, const void *, unsigned); void __lgwrite(struct lg_cpu *, unsigned long, const void *, unsigned);
...@@ -140,7 +140,7 @@ int run_guest(struct lg_cpu *cpu, unsigned long __user *user); ...@@ -140,7 +140,7 @@ int run_guest(struct lg_cpu *cpu, unsigned long __user *user);
/* interrupts_and_traps.c: */ /* interrupts_and_traps.c: */
void maybe_do_interrupt(struct lg_cpu *cpu); void maybe_do_interrupt(struct lg_cpu *cpu);
int deliver_trap(struct lg_cpu *cpu, unsigned int num); bool deliver_trap(struct lg_cpu *cpu, unsigned int num);
void load_guest_idt_entry(struct lg_cpu *cpu, unsigned int i, void load_guest_idt_entry(struct lg_cpu *cpu, unsigned int i,
u32 low, u32 hi); u32 low, u32 hi);
void guest_set_stack(struct lg_cpu *cpu, u32 seg, u32 esp, unsigned int pages); void guest_set_stack(struct lg_cpu *cpu, u32 seg, u32 esp, unsigned int pages);
...@@ -173,7 +173,7 @@ void guest_pagetable_flush_user(struct lg_cpu *cpu); ...@@ -173,7 +173,7 @@ void guest_pagetable_flush_user(struct lg_cpu *cpu);
void guest_set_pte(struct lg_cpu *cpu, unsigned long gpgdir, void guest_set_pte(struct lg_cpu *cpu, unsigned long gpgdir,
unsigned long vaddr, pte_t val); unsigned long vaddr, pte_t val);
void map_switcher_in_guest(struct lg_cpu *cpu, struct lguest_pages *pages); void map_switcher_in_guest(struct lg_cpu *cpu, struct lguest_pages *pages);
int demand_page(struct lg_cpu *cpu, unsigned long cr2, int errcode); bool demand_page(struct lg_cpu *cpu, unsigned long cr2, int errcode);
void pin_page(struct lg_cpu *cpu, unsigned long vaddr); void pin_page(struct lg_cpu *cpu, unsigned long vaddr);
unsigned long guest_pa(struct lg_cpu *cpu, unsigned long vaddr); unsigned long guest_pa(struct lg_cpu *cpu, unsigned long vaddr);
void page_table_guest_data_init(struct lg_cpu *cpu); void page_table_guest_data_init(struct lg_cpu *cpu);
......
...@@ -161,7 +161,7 @@ static void set_status(struct virtio_device *vdev, u8 status) ...@@ -161,7 +161,7 @@ static void set_status(struct virtio_device *vdev, u8 status)
/* We set the status. */ /* We set the status. */
to_lgdev(vdev)->desc->status = status; to_lgdev(vdev)->desc->status = status;
hcall(LHCALL_NOTIFY, (max_pfn<<PAGE_SHIFT) + offset, 0, 0); kvm_hypercall1(LHCALL_NOTIFY, (max_pfn << PAGE_SHIFT) + offset);
} }
static void lg_set_status(struct virtio_device *vdev, u8 status) static void lg_set_status(struct virtio_device *vdev, u8 status)
...@@ -209,7 +209,7 @@ static void lg_notify(struct virtqueue *vq) ...@@ -209,7 +209,7 @@ static void lg_notify(struct virtqueue *vq)
* virtqueue structure. */ * virtqueue structure. */
struct lguest_vq_info *lvq = vq->priv; struct lguest_vq_info *lvq = vq->priv;
hcall(LHCALL_NOTIFY, lvq->config.pfn << PAGE_SHIFT, 0, 0); kvm_hypercall1(LHCALL_NOTIFY, lvq->config.pfn << PAGE_SHIFT);
} }
/* An extern declaration inside a C file is bad form. Don't do it. */ /* An extern declaration inside a C file is bad form. Don't do it. */
......
...@@ -199,7 +199,7 @@ static void check_gpgd(struct lg_cpu *cpu, pgd_t gpgd) ...@@ -199,7 +199,7 @@ static void check_gpgd(struct lg_cpu *cpu, pgd_t gpgd)
* *
* If we fixed up the fault (ie. we mapped the address), this routine returns * If we fixed up the fault (ie. we mapped the address), this routine returns
* true. Otherwise, it was a real fault and we need to tell the Guest. */ * true. Otherwise, it was a real fault and we need to tell the Guest. */
int demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode) bool demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode)
{ {
pgd_t gpgd; pgd_t gpgd;
pgd_t *spgd; pgd_t *spgd;
...@@ -211,7 +211,7 @@ int demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode) ...@@ -211,7 +211,7 @@ int demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode)
gpgd = lgread(cpu, gpgd_addr(cpu, vaddr), pgd_t); gpgd = lgread(cpu, gpgd_addr(cpu, vaddr), pgd_t);
/* Toplevel not present? We can't map it in. */ /* Toplevel not present? We can't map it in. */
if (!(pgd_flags(gpgd) & _PAGE_PRESENT)) if (!(pgd_flags(gpgd) & _PAGE_PRESENT))
return 0; return false;
/* Now look at the matching shadow entry. */ /* Now look at the matching shadow entry. */
spgd = spgd_addr(cpu, cpu->cpu_pgd, vaddr); spgd = spgd_addr(cpu, cpu->cpu_pgd, vaddr);
...@@ -222,7 +222,7 @@ int demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode) ...@@ -222,7 +222,7 @@ int demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode)
* simple for this corner case. */ * simple for this corner case. */
if (!ptepage) { if (!ptepage) {
kill_guest(cpu, "out of memory allocating pte page"); kill_guest(cpu, "out of memory allocating pte page");
return 0; return false;
} }
/* We check that the Guest pgd is OK. */ /* We check that the Guest pgd is OK. */
check_gpgd(cpu, gpgd); check_gpgd(cpu, gpgd);
...@@ -238,16 +238,16 @@ int demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode) ...@@ -238,16 +238,16 @@ int demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode)
/* If this page isn't in the Guest page tables, we can't page it in. */ /* If this page isn't in the Guest page tables, we can't page it in. */
if (!(pte_flags(gpte) & _PAGE_PRESENT)) if (!(pte_flags(gpte) & _PAGE_PRESENT))
return 0; return false;
/* Check they're not trying to write to a page the Guest wants /* Check they're not trying to write to a page the Guest wants
* read-only (bit 2 of errcode == write). */ * read-only (bit 2 of errcode == write). */
if ((errcode & 2) && !(pte_flags(gpte) & _PAGE_RW)) if ((errcode & 2) && !(pte_flags(gpte) & _PAGE_RW))
return 0; return false;
/* User access to a kernel-only page? (bit 3 == user access) */ /* User access to a kernel-only page? (bit 3 == user access) */
if ((errcode & 4) && !(pte_flags(gpte) & _PAGE_USER)) if ((errcode & 4) && !(pte_flags(gpte) & _PAGE_USER))
return 0; return false;
/* Check that the Guest PTE flags are OK, and the page number is below /* Check that the Guest PTE flags are OK, and the page number is below
* the pfn_limit (ie. not mapping the Launcher binary). */ * the pfn_limit (ie. not mapping the Launcher binary). */
...@@ -283,7 +283,7 @@ int demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode) ...@@ -283,7 +283,7 @@ int demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode)
* manipulated, the result returned and the code complete. A small * manipulated, the result returned and the code complete. A small
* delay and a trace of alliteration are the only indications the Guest * delay and a trace of alliteration are the only indications the Guest
* has that a page fault occurred at all. */ * has that a page fault occurred at all. */
return 1; return true;
} }
/*H:360 /*H:360
...@@ -296,7 +296,7 @@ int demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode) ...@@ -296,7 +296,7 @@ int demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode)
* *
* This is a quick version which answers the question: is this virtual address * This is a quick version which answers the question: is this virtual address
* mapped by the shadow page tables, and is it writable? */ * mapped by the shadow page tables, and is it writable? */
static int page_writable(struct lg_cpu *cpu, unsigned long vaddr) static bool page_writable(struct lg_cpu *cpu, unsigned long vaddr)
{ {
pgd_t *spgd; pgd_t *spgd;
unsigned long flags; unsigned long flags;
...@@ -304,7 +304,7 @@ static int page_writable(struct lg_cpu *cpu, unsigned long vaddr) ...@@ -304,7 +304,7 @@ static int page_writable(struct lg_cpu *cpu, unsigned long vaddr)
/* Look at the current top level entry: is it present? */ /* Look at the current top level entry: is it present? */
spgd = spgd_addr(cpu, cpu->cpu_pgd, vaddr); spgd = spgd_addr(cpu, cpu->cpu_pgd, vaddr);
if (!(pgd_flags(*spgd) & _PAGE_PRESENT)) if (!(pgd_flags(*spgd) & _PAGE_PRESENT))
return 0; return false;
/* Check the flags on the pte entry itself: it must be present and /* Check the flags on the pte entry itself: it must be present and
* writable. */ * writable. */
...@@ -373,8 +373,10 @@ unsigned long guest_pa(struct lg_cpu *cpu, unsigned long vaddr) ...@@ -373,8 +373,10 @@ unsigned long guest_pa(struct lg_cpu *cpu, unsigned long vaddr)
/* First step: get the top-level Guest page table entry. */ /* First step: get the top-level Guest page table entry. */
gpgd = lgread(cpu, gpgd_addr(cpu, vaddr), pgd_t); gpgd = lgread(cpu, gpgd_addr(cpu, vaddr), pgd_t);
/* Toplevel not present? We can't map it in. */ /* Toplevel not present? We can't map it in. */
if (!(pgd_flags(gpgd) & _PAGE_PRESENT)) if (!(pgd_flags(gpgd) & _PAGE_PRESENT)) {
kill_guest(cpu, "Bad address %#lx", vaddr); kill_guest(cpu, "Bad address %#lx", vaddr);
return -1UL;
}
gpte = lgread(cpu, gpte_addr(gpgd, vaddr), pte_t); gpte = lgread(cpu, gpte_addr(gpgd, vaddr), pte_t);
if (!(pte_flags(gpte) & _PAGE_PRESENT)) if (!(pte_flags(gpte) & _PAGE_PRESENT))
......
...@@ -45,7 +45,7 @@ ...@@ -45,7 +45,7 @@
* "Task State Segment" which controls all kinds of delicate things. The * "Task State Segment" which controls all kinds of delicate things. The
* LGUEST_CS and LGUEST_DS entries are reserved for the Switcher, and the * LGUEST_CS and LGUEST_DS entries are reserved for the Switcher, and the
* the Guest can't be trusted to deal with double faults. */ * the Guest can't be trusted to deal with double faults. */
static int ignored_gdt(unsigned int num) static bool ignored_gdt(unsigned int num)
{ {
return (num == GDT_ENTRY_TSS return (num == GDT_ENTRY_TSS
|| num == GDT_ENTRY_LGUEST_CS || num == GDT_ENTRY_LGUEST_CS
......
...@@ -290,6 +290,57 @@ static int emulate_insn(struct lg_cpu *cpu) ...@@ -290,6 +290,57 @@ static int emulate_insn(struct lg_cpu *cpu)
return 1; return 1;
} }
/* Our hypercalls mechanism used to be based on direct software interrupts.
* After Anthony's "Refactor hypercall infrastructure" kvm patch, we decided to
* change over to using kvm hypercalls.
*
* KVM_HYPERCALL is actually a "vmcall" instruction, which generates an invalid
* opcode fault (fault 6) on non-VT cpus, so the easiest solution seemed to be
* an *emulation approach*: if the fault was really produced by an hypercall
* (is_hypercall() does exactly this check), we can just call the corresponding
* hypercall host implementation function.
*
* But these invalid opcode faults are notably slower than software interrupts.
* So we implemented the *patching (or rewriting) approach*: every time we hit
* the KVM_HYPERCALL opcode in Guest code, we patch it to the old "int 0x1f"
* opcode, so next time the Guest calls this hypercall it will use the
* faster trap mechanism.
*
* Matias even benchmarked it to convince you: this shows the average cycle
* cost of a hypercall. For each alternative solution mentioned above we've
* made 5 runs of the benchmark:
*
* 1) direct software interrupt: 2915, 2789, 2764, 2721, 2898
* 2) emulation technique: 3410, 3681, 3466, 3392, 3780
* 3) patching (rewrite) technique: 2977, 2975, 2891, 2637, 2884
*
* One two-line function is worth a 20% hypercall speed boost!
*/
static void rewrite_hypercall(struct lg_cpu *cpu)
{
/* This are the opcodes we use to patch the Guest. The opcode for "int
* $0x1f" is "0xcd 0x1f" but vmcall instruction is 3 bytes long, so we
* complete the sequence with a NOP (0x90). */
u8 insn[3] = {0xcd, 0x1f, 0x90};
__lgwrite(cpu, guest_pa(cpu, cpu->regs->eip), insn, sizeof(insn));
}
static bool is_hypercall(struct lg_cpu *cpu)
{
u8 insn[3];
/* This must be the Guest kernel trying to do something.
* The bottom two bits of the CS segment register are the privilege
* level. */
if ((cpu->regs->cs & 3) != GUEST_PL)
return false;
/* Is it a vmcall? */
__lgread(cpu, insn, guest_pa(cpu, cpu->regs->eip), sizeof(insn));
return insn[0] == 0x0f && insn[1] == 0x01 && insn[2] == 0xc1;
}
/*H:050 Once we've re-enabled interrupts, we look at why the Guest exited. */ /*H:050 Once we've re-enabled interrupts, we look at why the Guest exited. */
void lguest_arch_handle_trap(struct lg_cpu *cpu) void lguest_arch_handle_trap(struct lg_cpu *cpu)
{ {
...@@ -347,6 +398,15 @@ void lguest_arch_handle_trap(struct lg_cpu *cpu) ...@@ -347,6 +398,15 @@ void lguest_arch_handle_trap(struct lg_cpu *cpu)
* up the pointer now to indicate a hypercall is pending. */ * up the pointer now to indicate a hypercall is pending. */
cpu->hcall = (struct hcall_args *)cpu->regs; cpu->hcall = (struct hcall_args *)cpu->regs;
return; return;
case 6:
/* kvm hypercalls trigger an invalid opcode fault (6).
* We need to check if ring == GUEST_PL and
* faulting instruction == vmcall. */
if (is_hypercall(cpu)) {
rewrite_hypercall(cpu);
return;
}
break;
} }
/* We didn't handle the trap, so it needs to go to the Guest. */ /* We didn't handle the trap, so it needs to go to the Guest. */
......
...@@ -23,15 +23,21 @@ ...@@ -23,15 +23,21 @@
#ifdef DEBUG #ifdef DEBUG
/* For development, we want to crash whenever the ring is screwed. */ /* For development, we want to crash whenever the ring is screwed. */
#define BAD_RING(vq, fmt...) \ #define BAD_RING(_vq, fmt...) \
do { dev_err(&vq->vq.vdev->dev, fmt); BUG(); } while(0) do { dev_err(&(_vq)->vq.vdev->dev, fmt); BUG(); } while(0)
#define START_USE(vq) \ /* Caller is supposed to guarantee no reentry. */
do { if ((vq)->in_use) panic("in_use = %i\n", (vq)->in_use); (vq)->in_use = __LINE__; mb(); } while(0) #define START_USE(_vq) \
#define END_USE(vq) \ do { \
do { BUG_ON(!(vq)->in_use); (vq)->in_use = 0; mb(); } while(0) if ((_vq)->in_use) \
panic("in_use = %i\n", (_vq)->in_use); \
(_vq)->in_use = __LINE__; \
mb(); \
} while(0)
#define END_USE(_vq) \
do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; mb(); } while(0)
#else #else
#define BAD_RING(vq, fmt...) \ #define BAD_RING(_vq, fmt...) \
do { dev_err(&vq->vq.vdev->dev, fmt); (vq)->broken = true; } while(0) do { dev_err(&_vq->vq.vdev->dev, fmt); (_vq)->broken = true; } while(0)
#define START_USE(vq) #define START_USE(vq)
#define END_USE(vq) #define END_USE(vq)
#endif #endif
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment