KVM: Use memdup_user instead of kmalloc/copy_from_user
[linux-3.10.git] / arch / x86 / kvm / x86.c
index 274652a..0a646e2 100644 (file)
@@ -26,6 +26,7 @@
 #include "tss.h"
 #include "kvm_cache_regs.h"
 #include "x86.h"
+#include "cpuid.h"
 
 #include <linux/clocksource.h>
 #include <linux/interrupt.h>
@@ -44,6 +45,7 @@
 #include <linux/perf_event.h>
 #include <linux/uaccess.h>
 #include <linux/hash.h>
+#include <linux/pci.h>
 #include <trace/events/kvm.h>
 
 #define CREATE_TRACE_POINTS
@@ -81,8 +83,7 @@ static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
 
 static void update_cr8_intercept(struct kvm_vcpu *vcpu);
-static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
-                                   struct kvm_cpuid_entry2 __user *entries);
+static void process_nmi(struct kvm_vcpu *vcpu);
 
 struct kvm_x86_ops *kvm_x86_ops;
 EXPORT_SYMBOL_GPL(kvm_x86_ops);
@@ -152,6 +153,8 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
 
 u64 __read_mostly host_xcr0;
 
+int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
+
 static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
 {
        int i;
@@ -345,6 +348,7 @@ void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
        vcpu->arch.cr2 = fault->address;
        kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
 }
+EXPORT_SYMBOL_GPL(kvm_inject_page_fault);
 
 void kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
 {
@@ -356,8 +360,8 @@ void kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
 
 void kvm_inject_nmi(struct kvm_vcpu *vcpu)
 {
-       kvm_make_request(KVM_REQ_EVENT, vcpu);
-       vcpu->arch.nmi_pending = 1;
+       atomic_inc(&vcpu->arch.nmi_queued);
+       kvm_make_request(KVM_REQ_NMI, vcpu);
 }
 EXPORT_SYMBOL_GPL(kvm_inject_nmi);
 
@@ -569,41 +573,23 @@ int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
 }
 EXPORT_SYMBOL_GPL(kvm_set_xcr);
 
-static bool guest_cpuid_has_xsave(struct kvm_vcpu *vcpu)
-{
-       struct kvm_cpuid_entry2 *best;
-
-       best = kvm_find_cpuid_entry(vcpu, 1, 0);
-       return best && (best->ecx & bit(X86_FEATURE_XSAVE));
-}
-
-static void update_cpuid(struct kvm_vcpu *vcpu)
-{
-       struct kvm_cpuid_entry2 *best;
-
-       best = kvm_find_cpuid_entry(vcpu, 1, 0);
-       if (!best)
-               return;
-
-       /* Update OSXSAVE bit */
-       if (cpu_has_xsave && best->function == 0x1) {
-               best->ecx &= ~(bit(X86_FEATURE_OSXSAVE));
-               if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE))
-                       best->ecx |= bit(X86_FEATURE_OSXSAVE);
-       }
-}
-
 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 {
        unsigned long old_cr4 = kvm_read_cr4(vcpu);
-       unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE;
-
+       unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE |
+                                  X86_CR4_PAE | X86_CR4_SMEP;
        if (cr4 & CR4_RESERVED_BITS)
                return 1;
 
        if (!guest_cpuid_has_xsave(vcpu) && (cr4 & X86_CR4_OSXSAVE))
                return 1;
 
+       if (!guest_cpuid_has_smep(vcpu) && (cr4 & X86_CR4_SMEP))
+               return 1;
+
+       if (!guest_cpuid_has_fsgsbase(vcpu) && (cr4 & X86_CR4_RDWRGSFS))
+               return 1;
+
        if (is_long_mode(vcpu)) {
                if (!(cr4 & X86_CR4_PAE))
                        return 1;
@@ -613,16 +599,14 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
                                   kvm_read_cr3(vcpu)))
                return 1;
 
-       if (cr4 & X86_CR4_VMXE)
+       if (kvm_x86_ops->set_cr4(vcpu, cr4))
                return 1;
 
-       kvm_x86_ops->set_cr4(vcpu, cr4);
-
        if ((cr4 ^ old_cr4) & pdptr_bits)
                kvm_mmu_reset_context(vcpu);
 
        if ((cr4 ^ old_cr4) & X86_CR4_OSXSAVE)
-               update_cpuid(vcpu);
+               kvm_update_cpuid(vcpu);
 
        return 0;
 }
@@ -785,12 +769,12 @@ EXPORT_SYMBOL_GPL(kvm_get_dr);
  * kvm-specific. Those are put in the beginning of the list.
  */
 
-#define KVM_SAVE_MSRS_BEGIN    8
+#define KVM_SAVE_MSRS_BEGIN    9
 static u32 msrs_to_save[] = {
        MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
        MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
        HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
-       HV_X64_MSR_APIC_ASSIST_PAGE, MSR_KVM_ASYNC_PF_EN,
+       HV_X64_MSR_APIC_ASSIST_PAGE, MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
        MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
        MSR_STAR,
 #ifdef CONFIG_X86_64
@@ -802,6 +786,7 @@ static u32 msrs_to_save[] = {
 static unsigned num_msrs_to_save;
 
 static u32 emulated_msrs[] = {
+       MSR_IA32_TSCDEADLINE,
        MSR_IA32_MISC_ENABLE,
        MSR_IA32_MCG_STATUS,
        MSR_IA32_MCG_CTL,
@@ -977,7 +962,7 @@ static inline int kvm_tsc_changes_freq(void)
        return ret;
 }
 
-static u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu)
+u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu)
 {
        if (vcpu->arch.virtual_tsc_khz)
                return vcpu->arch.virtual_tsc_khz;
@@ -1075,7 +1060,7 @@ static int kvm_guest_time_update(struct kvm_vcpu *v)
 
        /* Keep irq disabled to prevent changes to the clock */
        local_irq_save(flags);
-       kvm_get_msr(v, MSR_IA32_TSC, &tsc_timestamp);
+       tsc_timestamp = kvm_x86_ops->read_l1_tsc(v);
        kernel_ns = get_kernel_ns();
        this_tsc_khz = vcpu_tsc_khz(v);
        if (unlikely(this_tsc_khz == 0)) {
@@ -1324,12 +1309,11 @@ static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
        if (page_num >= blob_size)
                goto out;
        r = -ENOMEM;
-       page = kzalloc(PAGE_SIZE, GFP_KERNEL);
-       if (!page)
+       page = memdup_user(blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE);
+       if (IS_ERR(page)) {
+               r = PTR_ERR(page);
                goto out;
-       r = -EFAULT;
-       if (copy_from_user(page, blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE))
-               goto out_free;
+       }
        if (kvm_write_guest(kvm, page_addr, page, PAGE_SIZE))
                goto out_free;
        r = 0;
@@ -1386,7 +1370,7 @@ static int set_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data)
                        return 1;
                kvm_x86_ops->patch_hypercall(vcpu, instructions);
                ((unsigned char *)instructions)[3] = 0xc3; /* ret */
-               if (copy_to_user((void __user *)addr, instructions, 4))
+               if (__copy_to_user((void __user *)addr, instructions, 4))
                        return 1;
                kvm->arch.hv_hypercall = data;
                break;
@@ -1413,7 +1397,7 @@ static int set_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 data)
                                  HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT);
                if (kvm_is_error_hva(addr))
                        return 1;
-               if (clear_user((void __user *)addr, PAGE_SIZE))
+               if (__clear_user((void __user *)addr, PAGE_SIZE))
                        return 1;
                vcpu->arch.hv_vapic = data;
                break;
@@ -1465,6 +1449,35 @@ static void kvmclock_reset(struct kvm_vcpu *vcpu)
        }
 }
 
+static void accumulate_steal_time(struct kvm_vcpu *vcpu)
+{
+       u64 delta;
+
+       if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
+               return;
+
+       delta = current->sched_info.run_delay - vcpu->arch.st.last_steal;
+       vcpu->arch.st.last_steal = current->sched_info.run_delay;
+       vcpu->arch.st.accum_steal = delta;
+}
+
+static void record_steal_time(struct kvm_vcpu *vcpu)
+{
+       if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
+               return;
+
+       if (unlikely(kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
+               &vcpu->arch.st.steal, sizeof(struct kvm_steal_time))))
+               return;
+
+       vcpu->arch.st.steal.steal += vcpu->arch.st.accum_steal;
+       vcpu->arch.st.steal.version += 2;
+       vcpu->arch.st.accum_steal = 0;
+
+       kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
+               &vcpu->arch.st.steal, sizeof(struct kvm_steal_time));
+}
+
 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 {
        switch (msr) {
@@ -1512,6 +1525,9 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
                break;
        case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
                return kvm_x2apic_msr_write(vcpu, msr, data);
+       case MSR_IA32_TSCDEADLINE:
+               kvm_set_lapic_tscdeadline_msr(vcpu, data);
+               break;
        case MSR_IA32_MISC_ENABLE:
                vcpu->arch.ia32_misc_enable_msr = data;
                break;
@@ -1547,6 +1563,33 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
                if (kvm_pv_enable_async_pf(vcpu, data))
                        return 1;
                break;
+       case MSR_KVM_STEAL_TIME:
+
+               if (unlikely(!sched_info_on()))
+                       return 1;
+
+               if (data & KVM_STEAL_RESERVED_MASK)
+                       return 1;
+
+               if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.st.stime,
+                                                       data & KVM_STEAL_VALID_BITS))
+                       return 1;
+
+               vcpu->arch.st.msr_val = data;
+
+               if (!(data & KVM_MSR_ENABLED))
+                       break;
+
+               vcpu->arch.st.last_steal = current->sched_info.run_delay;
+
+               preempt_disable();
+               accumulate_steal_time(vcpu);
+               preempt_enable();
+
+               kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
+
+               break;
+
        case MSR_IA32_MCG_CTL:
        case MSR_IA32_MCG_STATUS:
        case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
@@ -1746,6 +1789,9 @@ static int get_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
                return kvm_hv_vapic_msr_read(vcpu, APIC_ICR, pdata);
        case HV_X64_MSR_TPR:
                return kvm_hv_vapic_msr_read(vcpu, APIC_TASKPRI, pdata);
+       case HV_X64_MSR_APIC_ASSIST_PAGE:
+               data = vcpu->arch.hv_vapic;
+               break;
        default:
                pr_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
                return 1;
@@ -1760,7 +1806,6 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
 
        switch (msr) {
        case MSR_IA32_PLATFORM_ID:
-       case MSR_IA32_UCODE_REV:
        case MSR_IA32_EBL_CR_POWERON:
        case MSR_IA32_DEBUGCTLMSR:
        case MSR_IA32_LASTBRANCHFROMIP:
@@ -1781,6 +1826,9 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
        case MSR_FAM10H_MMIO_CONF_BASE:
                data = 0;
                break;
+       case MSR_IA32_UCODE_REV:
+               data = 0x100000000ULL;
+               break;
        case MSR_MTRRcap:
                data = 0x500 | KVM_NR_VAR_MTRR;
                break;
@@ -1809,6 +1857,9 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
        case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
                return kvm_x2apic_msr_read(vcpu, msr, pdata);
                break;
+       case MSR_IA32_TSCDEADLINE:
+               data = kvm_get_lapic_tscdeadline_msr(vcpu);
+               break;
        case MSR_IA32_MISC_ENABLE:
                data = vcpu->arch.ia32_misc_enable_msr;
                break;
@@ -1832,6 +1883,9 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
        case MSR_KVM_ASYNC_PF_EN:
                data = vcpu->arch.apf.msr_val;
                break;
+       case MSR_KVM_STEAL_TIME:
+               data = vcpu->arch.st.msr_val;
+               break;
        case MSR_IA32_P5_MC_ADDR:
        case MSR_IA32_P5_MC_TYPE:
        case MSR_IA32_MCG_CAP:
@@ -1933,15 +1987,12 @@ static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
        if (msrs.nmsrs >= MAX_IO_MSRS)
                goto out;
 
-       r = -ENOMEM;
        size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
-       entries = kmalloc(size, GFP_KERNEL);
-       if (!entries)
+       entries = memdup_user(user_msrs->entries, size);
+       if (IS_ERR(entries)) {
+               r = PTR_ERR(entries);
                goto out;
-
-       r = -EFAULT;
-       if (copy_from_user(entries, user_msrs->entries, size))
-               goto out_free;
+       }
 
        r = n = __msr_io(vcpu, &msrs, entries, do_msr);
        if (r < 0)
@@ -2004,6 +2055,9 @@ int kvm_dev_ioctl_check_extension(long ext)
                r = !kvm_x86_ops->cpu_has_accelerated_tpr();
                break;
        case KVM_CAP_NR_VCPUS:
+               r = KVM_SOFT_MAX_VCPUS;
+               break;
+       case KVM_CAP_MAX_VCPUS:
                r = KVM_MAX_VCPUS;
                break;
        case KVM_CAP_NR_MEMSLOTS:
@@ -2013,7 +2067,7 @@ int kvm_dev_ioctl_check_extension(long ext)
                r = 0;
                break;
        case KVM_CAP_IOMMU:
-               r = iommu_found();
+               r = iommu_present(&pci_bus_type);
                break;
        case KVM_CAP_MCE:
                r = KVM_MAX_MCE_BANKS;
@@ -2024,6 +2078,9 @@ int kvm_dev_ioctl_check_extension(long ext)
        case KVM_CAP_TSC_CONTROL:
                r = kvm_has_tsc_control;
                break;
+       case KVM_CAP_TSC_DEADLINE_TIMER:
+               r = boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER);
+               break;
        default:
                r = 0;
                break;
@@ -2128,7 +2185,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
                s64 tsc_delta;
                u64 tsc;
 
-               kvm_get_msr(vcpu, MSR_IA32_TSC, &tsc);
+               tsc = kvm_x86_ops->read_l1_tsc(vcpu);
                tsc_delta = !vcpu->arch.last_guest_tsc ? 0 :
                             tsc - vcpu->arch.last_guest_tsc;
 
@@ -2143,377 +2200,16 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
                        kvm_migrate_timers(vcpu);
                vcpu->cpu = cpu;
        }
+
+       accumulate_steal_time(vcpu);
+       kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
 }
 
 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 {
        kvm_x86_ops->vcpu_put(vcpu);
        kvm_put_guest_fpu(vcpu);
-       kvm_get_msr(vcpu, MSR_IA32_TSC, &vcpu->arch.last_guest_tsc);
-}
-
-static int is_efer_nx(void)
-{
-       unsigned long long efer = 0;
-
-       rdmsrl_safe(MSR_EFER, &efer);
-       return efer & EFER_NX;
-}
-
-static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
-{
-       int i;
-       struct kvm_cpuid_entry2 *e, *entry;
-
-       entry = NULL;
-       for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
-               e = &vcpu->arch.cpuid_entries[i];
-               if (e->function == 0x80000001) {
-                       entry = e;
-                       break;
-               }
-       }
-       if (entry && (entry->edx & (1 << 20)) && !is_efer_nx()) {
-               entry->edx &= ~(1 << 20);
-               printk(KERN_INFO "kvm: guest NX capability removed\n");
-       }
-}
-
-/* when an old userspace process fills a new kernel module */
-static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
-                                   struct kvm_cpuid *cpuid,
-                                   struct kvm_cpuid_entry __user *entries)
-{
-       int r, i;
-       struct kvm_cpuid_entry *cpuid_entries;
-
-       r = -E2BIG;
-       if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
-               goto out;
-       r = -ENOMEM;
-       cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry) * cpuid->nent);
-       if (!cpuid_entries)
-               goto out;
-       r = -EFAULT;
-       if (copy_from_user(cpuid_entries, entries,
-                          cpuid->nent * sizeof(struct kvm_cpuid_entry)))
-               goto out_free;
-       for (i = 0; i < cpuid->nent; i++) {
-               vcpu->arch.cpuid_entries[i].function = cpuid_entries[i].function;
-               vcpu->arch.cpuid_entries[i].eax = cpuid_entries[i].eax;
-               vcpu->arch.cpuid_entries[i].ebx = cpuid_entries[i].ebx;
-               vcpu->arch.cpuid_entries[i].ecx = cpuid_entries[i].ecx;
-               vcpu->arch.cpuid_entries[i].edx = cpuid_entries[i].edx;
-               vcpu->arch.cpuid_entries[i].index = 0;
-               vcpu->arch.cpuid_entries[i].flags = 0;
-               vcpu->arch.cpuid_entries[i].padding[0] = 0;
-               vcpu->arch.cpuid_entries[i].padding[1] = 0;
-               vcpu->arch.cpuid_entries[i].padding[2] = 0;
-       }
-       vcpu->arch.cpuid_nent = cpuid->nent;
-       cpuid_fix_nx_cap(vcpu);
-       r = 0;
-       kvm_apic_set_version(vcpu);
-       kvm_x86_ops->cpuid_update(vcpu);
-       update_cpuid(vcpu);
-
-out_free:
-       vfree(cpuid_entries);
-out:
-       return r;
-}
-
-static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu,
-                                    struct kvm_cpuid2 *cpuid,
-                                    struct kvm_cpuid_entry2 __user *entries)
-{
-       int r;
-
-       r = -E2BIG;
-       if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
-               goto out;
-       r = -EFAULT;
-       if (copy_from_user(&vcpu->arch.cpuid_entries, entries,
-                          cpuid->nent * sizeof(struct kvm_cpuid_entry2)))
-               goto out;
-       vcpu->arch.cpuid_nent = cpuid->nent;
-       kvm_apic_set_version(vcpu);
-       kvm_x86_ops->cpuid_update(vcpu);
-       update_cpuid(vcpu);
-       return 0;
-
-out:
-       return r;
-}
-
-static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu,
-                                    struct kvm_cpuid2 *cpuid,
-                                    struct kvm_cpuid_entry2 __user *entries)
-{
-       int r;
-
-       r = -E2BIG;
-       if (cpuid->nent < vcpu->arch.cpuid_nent)
-               goto out;
-       r = -EFAULT;
-       if (copy_to_user(entries, &vcpu->arch.cpuid_entries,
-                        vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2)))
-               goto out;
-       return 0;
-
-out:
-       cpuid->nent = vcpu->arch.cpuid_nent;
-       return r;
-}
-
-static void cpuid_mask(u32 *word, int wordnum)
-{
-       *word &= boot_cpu_data.x86_capability[wordnum];
-}
-
-static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function,
-                          u32 index)
-{
-       entry->function = function;
-       entry->index = index;
-       cpuid_count(entry->function, entry->index,
-                   &entry->eax, &entry->ebx, &entry->ecx, &entry->edx);
-       entry->flags = 0;
-}
-
-#define F(x) bit(X86_FEATURE_##x)
-
-static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
-                        u32 index, int *nent, int maxnent)
-{
-       unsigned f_nx = is_efer_nx() ? F(NX) : 0;
-#ifdef CONFIG_X86_64
-       unsigned f_gbpages = (kvm_x86_ops->get_lpage_level() == PT_PDPE_LEVEL)
-                               ? F(GBPAGES) : 0;
-       unsigned f_lm = F(LM);
-#else
-       unsigned f_gbpages = 0;
-       unsigned f_lm = 0;
-#endif
-       unsigned f_rdtscp = kvm_x86_ops->rdtscp_supported() ? F(RDTSCP) : 0;
-
-       /* cpuid 1.edx */
-       const u32 kvm_supported_word0_x86_features =
-               F(FPU) | F(VME) | F(DE) | F(PSE) |
-               F(TSC) | F(MSR) | F(PAE) | F(MCE) |
-               F(CX8) | F(APIC) | 0 /* Reserved */ | F(SEP) |
-               F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
-               F(PAT) | F(PSE36) | 0 /* PSN */ | F(CLFLSH) |
-               0 /* Reserved, DS, ACPI */ | F(MMX) |
-               F(FXSR) | F(XMM) | F(XMM2) | F(SELFSNOOP) |
-               0 /* HTT, TM, Reserved, PBE */;
-       /* cpuid 0x80000001.edx */
-       const u32 kvm_supported_word1_x86_features =
-               F(FPU) | F(VME) | F(DE) | F(PSE) |
-               F(TSC) | F(MSR) | F(PAE) | F(MCE) |
-               F(CX8) | F(APIC) | 0 /* Reserved */ | F(SYSCALL) |
-               F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
-               F(PAT) | F(PSE36) | 0 /* Reserved */ |
-               f_nx | 0 /* Reserved */ | F(MMXEXT) | F(MMX) |
-               F(FXSR) | F(FXSR_OPT) | f_gbpages | f_rdtscp |
-               0 /* Reserved */ | f_lm | F(3DNOWEXT) | F(3DNOW);
-       /* cpuid 1.ecx */
-       const u32 kvm_supported_word4_x86_features =
-               F(XMM3) | F(PCLMULQDQ) | 0 /* DTES64, MONITOR */ |
-               0 /* DS-CPL, VMX, SMX, EST */ |
-               0 /* TM2 */ | F(SSSE3) | 0 /* CNXT-ID */ | 0 /* Reserved */ |
-               0 /* Reserved */ | F(CX16) | 0 /* xTPR Update, PDCM */ |
-               0 /* Reserved, DCA */ | F(XMM4_1) |
-               F(XMM4_2) | F(X2APIC) | F(MOVBE) | F(POPCNT) |
-               0 /* Reserved*/ | F(AES) | F(XSAVE) | 0 /* OSXSAVE */ | F(AVX) |
-               F(F16C);
-       /* cpuid 0x80000001.ecx */
-       const u32 kvm_supported_word6_x86_features =
-               F(LAHF_LM) | F(CMP_LEGACY) | 0 /*SVM*/ | 0 /* ExtApicSpace */ |
-               F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
-               F(3DNOWPREFETCH) | 0 /* OSVW */ | 0 /* IBS */ | F(XOP) |
-               0 /* SKINIT, WDT, LWP */ | F(FMA4) | F(TBM);
-
-       /* all calls to cpuid_count() should be made on the same cpu */
-       get_cpu();
-       do_cpuid_1_ent(entry, function, index);
-       ++*nent;
-
-       switch (function) {
-       case 0:
-               entry->eax = min(entry->eax, (u32)0xd);
-               break;
-       case 1:
-               entry->edx &= kvm_supported_word0_x86_features;
-               cpuid_mask(&entry->edx, 0);
-               entry->ecx &= kvm_supported_word4_x86_features;
-               cpuid_mask(&entry->ecx, 4);
-               /* we support x2apic emulation even if host does not support
-                * it since we emulate x2apic in software */
-               entry->ecx |= F(X2APIC);
-               break;
-       /* function 2 entries are STATEFUL. That is, repeated cpuid commands
-        * may return different values. This forces us to get_cpu() before
-        * issuing the first command, and also to emulate this annoying behavior
-        * in kvm_emulate_cpuid() using KVM_CPUID_FLAG_STATE_READ_NEXT */
-       case 2: {
-               int t, times = entry->eax & 0xff;
-
-               entry->flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
-               entry->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT;
-               for (t = 1; t < times && *nent < maxnent; ++t) {
-                       do_cpuid_1_ent(&entry[t], function, 0);
-                       entry[t].flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
-                       ++*nent;
-               }
-               break;
-       }
-       /* function 4 and 0xb have additional index. */
-       case 4: {
-               int i, cache_type;
-
-               entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
-               /* read more entries until cache_type is zero */
-               for (i = 1; *nent < maxnent; ++i) {
-                       cache_type = entry[i - 1].eax & 0x1f;
-                       if (!cache_type)
-                               break;
-                       do_cpuid_1_ent(&entry[i], function, i);
-                       entry[i].flags |=
-                              KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
-                       ++*nent;
-               }
-               break;
-       }
-       case 0xb: {
-               int i, level_type;
-
-               entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
-               /* read more entries until level_type is zero */
-               for (i = 1; *nent < maxnent; ++i) {
-                       level_type = entry[i - 1].ecx & 0xff00;
-                       if (!level_type)
-                               break;
-                       do_cpuid_1_ent(&entry[i], function, i);
-                       entry[i].flags |=
-                              KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
-                       ++*nent;
-               }
-               break;
-       }
-       case 0xd: {
-               int i;
-
-               entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
-               for (i = 1; *nent < maxnent && i < 64; ++i) {
-                       if (entry[i].eax == 0)
-                               continue;
-                       do_cpuid_1_ent(&entry[i], function, i);
-                       entry[i].flags |=
-                              KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
-                       ++*nent;
-               }
-               break;
-       }
-       case KVM_CPUID_SIGNATURE: {
-               char signature[12] = "KVMKVMKVM\0\0";
-               u32 *sigptr = (u32 *)signature;
-               entry->eax = 0;
-               entry->ebx = sigptr[0];
-               entry->ecx = sigptr[1];
-               entry->edx = sigptr[2];
-               break;
-       }
-       case KVM_CPUID_FEATURES:
-               entry->eax = (1 << KVM_FEATURE_CLOCKSOURCE) |
-                            (1 << KVM_FEATURE_NOP_IO_DELAY) |
-                            (1 << KVM_FEATURE_CLOCKSOURCE2) |
-                            (1 << KVM_FEATURE_ASYNC_PF) |
-                            (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT);
-               entry->ebx = 0;
-               entry->ecx = 0;
-               entry->edx = 0;
-               break;
-       case 0x80000000:
-               entry->eax = min(entry->eax, 0x8000001a);
-               break;
-       case 0x80000001:
-               entry->edx &= kvm_supported_word1_x86_features;
-               cpuid_mask(&entry->edx, 1);
-               entry->ecx &= kvm_supported_word6_x86_features;
-               cpuid_mask(&entry->ecx, 6);
-               break;
-       }
-
-       kvm_x86_ops->set_supported_cpuid(function, entry);
-
-       put_cpu();
-}
-
-#undef F
-
-static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
-                                    struct kvm_cpuid_entry2 __user *entries)
-{
-       struct kvm_cpuid_entry2 *cpuid_entries;
-       int limit, nent = 0, r = -E2BIG;
-       u32 func;
-
-       if (cpuid->nent < 1)
-               goto out;
-       if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
-               cpuid->nent = KVM_MAX_CPUID_ENTRIES;
-       r = -ENOMEM;
-       cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry2) * cpuid->nent);
-       if (!cpuid_entries)
-               goto out;
-
-       do_cpuid_ent(&cpuid_entries[0], 0, 0, &nent, cpuid->nent);
-       limit = cpuid_entries[0].eax;
-       for (func = 1; func <= limit && nent < cpuid->nent; ++func)
-               do_cpuid_ent(&cpuid_entries[nent], func, 0,
-                            &nent, cpuid->nent);
-       r = -E2BIG;
-       if (nent >= cpuid->nent)
-               goto out_free;
-
-       do_cpuid_ent(&cpuid_entries[nent], 0x80000000, 0, &nent, cpuid->nent);
-       limit = cpuid_entries[nent - 1].eax;
-       for (func = 0x80000001; func <= limit && nent < cpuid->nent; ++func)
-               do_cpuid_ent(&cpuid_entries[nent], func, 0,
-                            &nent, cpuid->nent);
-
-
-
-       r = -E2BIG;
-       if (nent >= cpuid->nent)
-               goto out_free;
-
-       do_cpuid_ent(&cpuid_entries[nent], KVM_CPUID_SIGNATURE, 0, &nent,
-                    cpuid->nent);
-
-       r = -E2BIG;
-       if (nent >= cpuid->nent)
-               goto out_free;
-
-       do_cpuid_ent(&cpuid_entries[nent], KVM_CPUID_FEATURES, 0, &nent,
-                    cpuid->nent);
-
-       r = -E2BIG;
-       if (nent >= cpuid->nent)
-               goto out_free;
-
-       r = -EFAULT;
-       if (copy_to_user(entries, cpuid_entries,
-                        nent * sizeof(struct kvm_cpuid_entry2)))
-               goto out_free;
-       cpuid->nent = nent;
-       r = 0;
-
-out_free:
-       vfree(cpuid_entries);
-out:
-       return r;
+       vcpu->arch.last_guest_tsc = kvm_x86_ops->read_l1_tsc(vcpu);
 }
 
 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
@@ -2638,6 +2334,7 @@ static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
                                               struct kvm_vcpu_events *events)
 {
+       process_nmi(vcpu);
        events->exception.injected =
                vcpu->arch.exception.pending &&
                !kvm_exception_is_soft(vcpu->arch.exception.nr);
@@ -2655,7 +2352,7 @@ static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
                        KVM_X86_SHADOW_INT_MOV_SS | KVM_X86_SHADOW_INT_STI);
 
        events->nmi.injected = vcpu->arch.nmi_injected;
-       events->nmi.pending = vcpu->arch.nmi_pending;
+       events->nmi.pending = vcpu->arch.nmi_pending != 0;
        events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu);
        events->nmi.pad = 0;
 
@@ -2675,6 +2372,7 @@ static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
                              | KVM_VCPUEVENT_VALID_SHADOW))
                return -EINVAL;
 
+       process_nmi(vcpu);
        vcpu->arch.exception.pending = events->exception.injected;
        vcpu->arch.exception.nr = events->exception.nr;
        vcpu->arch.exception.has_error_code = events->exception.has_error_code;
@@ -2831,13 +2529,12 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
                r = -EINVAL;
                if (!vcpu->arch.apic)
                        goto out;
-               u.lapic = kmalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL);
-               r = -ENOMEM;
-               if (!u.lapic)
-                       goto out;
-               r = -EFAULT;
-               if (copy_from_user(u.lapic, argp, sizeof(struct kvm_lapic_state)))
+               u.lapic = memdup_user(argp, sizeof(*u.lapic));
+               if (IS_ERR(u.lapic)) {
+                       r = PTR_ERR(u.lapic);
                        goto out;
+               }
+
                r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
                if (r)
                        goto out;
@@ -3016,14 +2713,11 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
                break;
        }
        case KVM_SET_XSAVE: {
-               u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL);
-               r = -ENOMEM;
-               if (!u.xsave)
-                       break;
-
-               r = -EFAULT;
-               if (copy_from_user(u.xsave, argp, sizeof(struct kvm_xsave)))
-                       break;
+               u.xsave = memdup_user(argp, sizeof(*u.xsave));
+               if (IS_ERR(u.xsave)) {
+                       r = PTR_ERR(u.xsave);
+                       goto out;
+               }
 
                r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
                break;
@@ -3044,15 +2738,11 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
                break;
        }
        case KVM_SET_XCRS: {
-               u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL);
-               r = -ENOMEM;
-               if (!u.xcrs)
-                       break;
-
-               r = -EFAULT;
-               if (copy_from_user(u.xcrs, argp,
-                                  sizeof(struct kvm_xcrs)))
-                       break;
+               u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
+               if (IS_ERR(u.xcrs)) {
+                       r = PTR_ERR(u.xcrs);
+                       goto out;
+               }
 
                r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
                break;
@@ -3249,16 +2939,59 @@ static int kvm_vm_ioctl_reinject(struct kvm *kvm,
        return 0;
 }
 
+/**
+ * write_protect_slot - write protect a slot for dirty logging
+ * @kvm: the kvm instance
+ * @memslot: the slot we protect
+ * @dirty_bitmap: the bitmap indicating which pages are dirty
+ * @nr_dirty_pages: the number of dirty pages
+ *
+ * We have two ways to find all sptes to protect:
+ * 1. Use kvm_mmu_slot_remove_write_access() which walks all shadow pages and
+ *    checks ones that have a spte mapping a page in the slot.
+ * 2. Use kvm_mmu_rmap_write_protect() for each gfn found in the bitmap.
+ *
+ * Generally speaking, if there are not so many dirty pages compared to the
+ * number of shadow pages, we should use the latter.
+ *
+ * Note that letting others write into a page marked dirty in the old bitmap
+ * by using the remaining tlb entry is not a problem.  That page will become
+ * write protected again when we flush the tlb and then be reported dirty to
+ * the user space by copying the old bitmap.
+ */
+static void write_protect_slot(struct kvm *kvm,
+                              struct kvm_memory_slot *memslot,
+                              unsigned long *dirty_bitmap,
+                              unsigned long nr_dirty_pages)
+{
+       /* Not many dirty pages compared to # of shadow pages. */
+       if (nr_dirty_pages < kvm->arch.n_used_mmu_pages) {
+               unsigned long gfn_offset;
+
+               for_each_set_bit(gfn_offset, dirty_bitmap, memslot->npages) {
+                       unsigned long gfn = memslot->base_gfn + gfn_offset;
+
+                       spin_lock(&kvm->mmu_lock);
+                       kvm_mmu_rmap_write_protect(kvm, gfn, memslot);
+                       spin_unlock(&kvm->mmu_lock);
+               }
+               kvm_flush_remote_tlbs(kvm);
+       } else {
+               spin_lock(&kvm->mmu_lock);
+               kvm_mmu_slot_remove_write_access(kvm, memslot->id);
+               spin_unlock(&kvm->mmu_lock);
+       }
+}
+
 /*
  * Get (and clear) the dirty memory log for a memory slot.
  */
 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
                                      struct kvm_dirty_log *log)
 {
-       int r, i;
+       int r;
        struct kvm_memory_slot *memslot;
-       unsigned long n;
-       unsigned long is_dirty = 0;
+       unsigned long n, nr_dirty_pages;
 
        mutex_lock(&kvm->slots_lock);
 
@@ -3266,43 +2999,41 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
        if (log->slot >= KVM_MEMORY_SLOTS)
                goto out;
 
-       memslot = &kvm->memslots->memslots[log->slot];
+       memslot = id_to_memslot(kvm->memslots, log->slot);
        r = -ENOENT;
        if (!memslot->dirty_bitmap)
                goto out;
 
        n = kvm_dirty_bitmap_bytes(memslot);
-
-       for (i = 0; !is_dirty && i < n/sizeof(long); i++)
-               is_dirty = memslot->dirty_bitmap[i];
+       nr_dirty_pages = memslot->nr_dirty_pages;
 
        /* If nothing is dirty, don't bother messing with page tables. */
-       if (is_dirty) {
+       if (nr_dirty_pages) {
                struct kvm_memslots *slots, *old_slots;
-               unsigned long *dirty_bitmap;
+               unsigned long *dirty_bitmap, *dirty_bitmap_head;
 
-               dirty_bitmap = memslot->dirty_bitmap_head;
-               if (memslot->dirty_bitmap == dirty_bitmap)
-                       dirty_bitmap += n / sizeof(long);
-               memset(dirty_bitmap, 0, n);
+               dirty_bitmap = memslot->dirty_bitmap;
+               dirty_bitmap_head = memslot->dirty_bitmap_head;
+               if (dirty_bitmap == dirty_bitmap_head)
+                       dirty_bitmap_head += n / sizeof(long);
+               memset(dirty_bitmap_head, 0, n);
 
                r = -ENOMEM;
-               slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
+               slots = kmemdup(kvm->memslots, sizeof(*kvm->memslots), GFP_KERNEL);
                if (!slots)
                        goto out;
-               memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
-               slots->memslots[log->slot].dirty_bitmap = dirty_bitmap;
-               slots->generation++;
+
+               memslot = id_to_memslot(slots, log->slot);
+               memslot->nr_dirty_pages = 0;
+               memslot->dirty_bitmap = dirty_bitmap_head;
+               update_memslots(slots, NULL);
 
                old_slots = kvm->memslots;
                rcu_assign_pointer(kvm->memslots, slots);
                synchronize_srcu_expedited(&kvm->srcu);
-               dirty_bitmap = old_slots->memslots[log->slot].dirty_bitmap;
                kfree(old_slots);
 
-               spin_lock(&kvm->mmu_lock);
-               kvm_mmu_slot_remove_write_access(kvm, log->slot);
-               spin_unlock(&kvm->mmu_lock);
+               write_protect_slot(kvm, memslot, dirty_bitmap, nr_dirty_pages);
 
                r = -EFAULT;
                if (copy_to_user(log->dirty_bitmap, dirty_bitmap, n))
@@ -3375,7 +3106,11 @@ long kvm_arch_vm_ioctl(struct file *filp,
                        if (r) {
                                mutex_lock(&kvm->slots_lock);
                                kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
-                                                         &vpic->dev);
+                                                         &vpic->dev_master);
+                               kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
+                                                         &vpic->dev_slave);
+                               kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
+                                                         &vpic->dev_eclr);
                                mutex_unlock(&kvm->slots_lock);
                                kfree(vpic);
                                goto create_irqchip_unlock;
@@ -3443,14 +3178,14 @@ long kvm_arch_vm_ioctl(struct file *filp,
        }
        case KVM_GET_IRQCHIP: {
                /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
-               struct kvm_irqchip *chip = kmalloc(sizeof(*chip), GFP_KERNEL);
+               struct kvm_irqchip *chip;
 
-               r = -ENOMEM;
-               if (!chip)
+               chip = memdup_user(argp, sizeof(*chip));
+               if (IS_ERR(chip)) {
+                       r = PTR_ERR(chip);
                        goto out;
-               r = -EFAULT;
-               if (copy_from_user(chip, argp, sizeof *chip))
-                       goto get_irqchip_out;
+               }
+
                r = -ENXIO;
                if (!irqchip_in_kernel(kvm))
                        goto get_irqchip_out;
@@ -3469,14 +3204,14 @@ long kvm_arch_vm_ioctl(struct file *filp,
        }
        case KVM_SET_IRQCHIP: {
                /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
-               struct kvm_irqchip *chip = kmalloc(sizeof(*chip), GFP_KERNEL);
+               struct kvm_irqchip *chip;
 
-               r = -ENOMEM;
-               if (!chip)
+               chip = memdup_user(argp, sizeof(*chip));
+               if (IS_ERR(chip)) {
+                       r = PTR_ERR(chip);
                        goto out;
-               r = -EFAULT;
-               if (copy_from_user(chip, argp, sizeof *chip))
-                       goto set_irqchip_out;
+               }
+
                r = -ENXIO;
                if (!irqchip_in_kernel(kvm))
                        goto set_irqchip_out;
@@ -3683,12 +3418,7 @@ void kvm_get_segment(struct kvm_vcpu *vcpu,
        kvm_x86_ops->get_segment(vcpu, var, seg);
 }
 
-static gpa_t translate_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access)
-{
-       return gpa;
-}
-
-static gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access)
+gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access)
 {
        gpa_t t_gpa;
        struct x86_exception exception;
@@ -3775,7 +3505,7 @@ static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
                                          exception);
 }
 
-static int kvm_read_guest_virt(struct x86_emulate_ctxt *ctxt,
+int kvm_read_guest_virt(struct x86_emulate_ctxt *ctxt,
                               gva_t addr, void *val, unsigned int bytes,
                               struct x86_exception *exception)
 {
@@ -3785,6 +3515,7 @@ static int kvm_read_guest_virt(struct x86_emulate_ctxt *ctxt,
        return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
                                          exception);
 }
+EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
 
 static int kvm_read_guest_virt_system(struct x86_emulate_ctxt *ctxt,
                                      gva_t addr, void *val, unsigned int bytes,
@@ -3794,7 +3525,7 @@ static int kvm_read_guest_virt_system(struct x86_emulate_ctxt *ctxt,
        return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, exception);
 }
 
-static int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt,
+int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt,
                                       gva_t addr, void *val,
                                       unsigned int bytes,
                                       struct x86_exception *exception)
@@ -3826,62 +3557,41 @@ static int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt,
 out:
        return r;
 }
+EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
 
-static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
-                                 unsigned long addr,
-                                 void *val,
-                                 unsigned int bytes,
-                                 struct x86_exception *exception)
+static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
+                               gpa_t *gpa, struct x86_exception *exception,
+                               bool write)
 {
-       struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
-       gpa_t                 gpa;
-       int handled;
+       u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
 
-       if (vcpu->mmio_read_completed) {
-               memcpy(val, vcpu->mmio_data, bytes);
-               trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
-                              vcpu->mmio_phys_addr, *(u64 *)val);
-               vcpu->mmio_read_completed = 0;
-               return X86EMUL_CONTINUE;
+       if (vcpu_match_mmio_gva(vcpu, gva) &&
+                 check_write_user_access(vcpu, write, access,
+                 vcpu->arch.access)) {
+               *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
+                                       (gva & (PAGE_SIZE - 1));
+               trace_vcpu_match_mmio(gva, *gpa, write, false);
+               return 1;
        }
 
-       gpa = kvm_mmu_gva_to_gpa_read(vcpu, addr, exception);
-
-       if (gpa == UNMAPPED_GVA)
-               return X86EMUL_PROPAGATE_FAULT;
+       if (write)
+               access |= PFERR_WRITE_MASK;
 
-       /* For APIC access vmexit */
-       if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
-               goto mmio;
+       *gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
 
-       if (kvm_read_guest_virt(ctxt, addr, val, bytes, exception)
-           == X86EMUL_CONTINUE)
-               return X86EMUL_CONTINUE;
-
-mmio:
-       /*
-        * Is this MMIO handled locally?
-        */
-       handled = vcpu_mmio_read(vcpu, gpa, bytes, val);
-
-       if (handled == bytes)
-               return X86EMUL_CONTINUE;
-
-       gpa += handled;
-       bytes -= handled;
-       val += handled;
+       if (*gpa == UNMAPPED_GVA)
+               return -1;
 
-       trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, 0);
+       /* For APIC access vmexit */
+       if ((*gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
+               return 1;
 
-       vcpu->mmio_needed = 1;
-       vcpu->run->exit_reason = KVM_EXIT_MMIO;
-       vcpu->run->mmio.phys_addr = vcpu->mmio_phys_addr = gpa;
-       vcpu->mmio_size = bytes;
-       vcpu->run->mmio.len = min(vcpu->mmio_size, 8);
-       vcpu->run->mmio.is_write = vcpu->mmio_is_write = 0;
-       vcpu->mmio_index = 0;
+       if (vcpu_match_mmio_gpa(vcpu, *gpa)) {
+               trace_vcpu_match_mmio(gva, *gpa, write, true);
+               return 1;
+       }
 
-       return X86EMUL_IO_NEEDED;
+       return 0;
 }
 
 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
@@ -3892,37 +3602,113 @@ int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
        ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes);
        if (ret < 0)
                return 0;
-       kvm_mmu_pte_write(vcpu, gpa, val, bytes, 1);
+       kvm_mmu_pte_write(vcpu, gpa, val, bytes);
        return 1;
 }
 
-static int emulator_write_emulated_onepage(unsigned long addr,
-                                          const void *val,
-                                          unsigned int bytes,
-                                          struct x86_exception *exception,
-                                          struct kvm_vcpu *vcpu)
+struct read_write_emulator_ops {
+       int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
+                                 int bytes);
+       int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
+                                 void *val, int bytes);
+       int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
+                              int bytes, void *val);
+       int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
+                                   void *val, int bytes);
+       bool write;
+};
+
+static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
+{
+       if (vcpu->mmio_read_completed) {
+               memcpy(val, vcpu->mmio_data, bytes);
+               trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
+                              vcpu->mmio_phys_addr, *(u64 *)val);
+               vcpu->mmio_read_completed = 0;
+               return 1;
+       }
+
+       return 0;
+}
+
+static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
+                       void *val, int bytes)
 {
-       gpa_t                 gpa;
-       int handled;
+       return !kvm_read_guest(vcpu->kvm, gpa, val, bytes);
+}
 
-       gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, exception);
+static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
+                        void *val, int bytes)
+{
+       return emulator_write_phys(vcpu, gpa, val, bytes);
+}
 
-       if (gpa == UNMAPPED_GVA)
+static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
+{
+       trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, *(u64 *)val);
+       return vcpu_mmio_write(vcpu, gpa, bytes, val);
+}
+
+static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
+                         void *val, int bytes)
+{
+       trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, 0);
+       return X86EMUL_IO_NEEDED;
+}
+
+static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
+                          void *val, int bytes)
+{
+       memcpy(vcpu->mmio_data, val, bytes);
+       memcpy(vcpu->run->mmio.data, vcpu->mmio_data, 8);
+       return X86EMUL_CONTINUE;
+}
+
+static struct read_write_emulator_ops read_emultor = {
+       .read_write_prepare = read_prepare,
+       .read_write_emulate = read_emulate,
+       .read_write_mmio = vcpu_mmio_read,
+       .read_write_exit_mmio = read_exit_mmio,
+};
+
+static struct read_write_emulator_ops write_emultor = {
+       .read_write_emulate = write_emulate,
+       .read_write_mmio = write_mmio,
+       .read_write_exit_mmio = write_exit_mmio,
+       .write = true,
+};
+
+static int emulator_read_write_onepage(unsigned long addr, void *val,
+                                      unsigned int bytes,
+                                      struct x86_exception *exception,
+                                      struct kvm_vcpu *vcpu,
+                                      struct read_write_emulator_ops *ops)
+{
+       gpa_t gpa;
+       int handled, ret;
+       bool write = ops->write;
+
+       if (ops->read_write_prepare &&
+                 ops->read_write_prepare(vcpu, val, bytes))
+               return X86EMUL_CONTINUE;
+
+       ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
+
+       if (ret < 0)
                return X86EMUL_PROPAGATE_FAULT;
 
        /* For APIC access vmexit */
-       if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
+       if (ret)
                goto mmio;
 
-       if (emulator_write_phys(vcpu, gpa, val, bytes))
+       if (ops->read_write_emulate(vcpu, gpa, val, bytes))
                return X86EMUL_CONTINUE;
 
 mmio:
-       trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, *(u64 *)val);
        /*
         * Is this MMIO handled locally?
         */
-       handled = vcpu_mmio_write(vcpu, gpa, bytes, val);
+       handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
        if (handled == bytes)
                return X86EMUL_CONTINUE;
 
@@ -3931,23 +3717,20 @@ mmio:
        val += handled;
 
        vcpu->mmio_needed = 1;
-       memcpy(vcpu->mmio_data, val, bytes);
        vcpu->run->exit_reason = KVM_EXIT_MMIO;
        vcpu->run->mmio.phys_addr = vcpu->mmio_phys_addr = gpa;
        vcpu->mmio_size = bytes;
        vcpu->run->mmio.len = min(vcpu->mmio_size, 8);
-       vcpu->run->mmio.is_write = vcpu->mmio_is_write = 1;
-       memcpy(vcpu->run->mmio.data, vcpu->mmio_data, 8);
+       vcpu->run->mmio.is_write = vcpu->mmio_is_write = write;
        vcpu->mmio_index = 0;
 
-       return X86EMUL_CONTINUE;
+       return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
 }
 
-int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
-                           unsigned long addr,
-                           const void *val,
-                           unsigned int bytes,
-                           struct x86_exception *exception)
+int emulator_read_write(struct x86_emulate_ctxt *ctxt, unsigned long addr,
+                       void *val, unsigned int bytes,
+                       struct x86_exception *exception,
+                       struct read_write_emulator_ops *ops)
 {
        struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
 
@@ -3956,16 +3739,38 @@ int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
                int rc, now;
 
                now = -addr & ~PAGE_MASK;
-               rc = emulator_write_emulated_onepage(addr, val, now, exception,
-                                                    vcpu);
+               rc = emulator_read_write_onepage(addr, val, now, exception,
+                                                vcpu, ops);
+
                if (rc != X86EMUL_CONTINUE)
                        return rc;
                addr += now;
                val += now;
                bytes -= now;
        }
-       return emulator_write_emulated_onepage(addr, val, bytes, exception,
-                                              vcpu);
+
+       return emulator_read_write_onepage(addr, val, bytes, exception,
+                                          vcpu, ops);
+}
+
+static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
+                                 unsigned long addr,
+                                 void *val,
+                                 unsigned int bytes,
+                                 struct x86_exception *exception)
+{
+       return emulator_read_write(ctxt, addr, val, bytes,
+                                  exception, &read_emultor);
+}
+
+int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
+                           unsigned long addr,
+                           const void *val,
+                           unsigned int bytes,
+                           struct x86_exception *exception)
+{
+       return emulator_read_write(ctxt, addr, (void *)val, bytes,
+                                  exception, &write_emultor);
 }
 
 #define CMPXCHG_TYPE(t, ptr, old, new) \
@@ -4034,7 +3839,7 @@ static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
        if (!exchanged)
                return X86EMUL_CMPXCHG_FAILED;
 
-       kvm_mmu_pte_write(vcpu, gpa, new, bytes, 1);
+       kvm_mmu_pte_write(vcpu, gpa, new, bytes);
 
        return X86EMUL_CONTINUE;
 
@@ -4059,29 +3864,24 @@ static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
        return r;
 }
 
-
-static int emulator_pio_in_emulated(int size, unsigned short port, void *val,
-                            unsigned int count, struct kvm_vcpu *vcpu)
+static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
+                              unsigned short port, void *val,
+                              unsigned int count, bool in)
 {
-       if (vcpu->arch.pio.count)
-               goto data_avail;
-
-       trace_kvm_pio(0, port, size, count);
+       trace_kvm_pio(!in, port, size, count);
 
        vcpu->arch.pio.port = port;
-       vcpu->arch.pio.in = 1;
+       vcpu->arch.pio.in = in;
        vcpu->arch.pio.count  = count;
        vcpu->arch.pio.size = size;
 
        if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
-       data_avail:
-               memcpy(val, vcpu->arch.pio_data, size * count);
                vcpu->arch.pio.count = 0;
                return 1;
        }
 
        vcpu->run->exit_reason = KVM_EXIT_IO;
-       vcpu->run->io.direction = KVM_EXIT_IO_IN;
+       vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
        vcpu->run->io.size = size;
        vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
        vcpu->run->io.count = count;
@@ -4090,43 +3890,45 @@ static int emulator_pio_in_emulated(int size, unsigned short port, void *val,
        return 0;
 }
 
-static int emulator_pio_out_emulated(int size, unsigned short port,
-                             const void *val, unsigned int count,
-                             struct kvm_vcpu *vcpu)
+static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
+                                   int size, unsigned short port, void *val,
+                                   unsigned int count)
 {
-       trace_kvm_pio(1, port, size, count);
-
-       vcpu->arch.pio.port = port;
-       vcpu->arch.pio.in = 0;
-       vcpu->arch.pio.count = count;
-       vcpu->arch.pio.size = size;
+       struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
+       int ret;
 
-       memcpy(vcpu->arch.pio_data, val, size * count);
+       if (vcpu->arch.pio.count)
+               goto data_avail;
 
-       if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
+       ret = emulator_pio_in_out(vcpu, size, port, val, count, true);
+       if (ret) {
+data_avail:
+               memcpy(val, vcpu->arch.pio_data, size * count);
                vcpu->arch.pio.count = 0;
                return 1;
        }
 
-       vcpu->run->exit_reason = KVM_EXIT_IO;
-       vcpu->run->io.direction = KVM_EXIT_IO_OUT;
-       vcpu->run->io.size = size;
-       vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
-       vcpu->run->io.count = count;
-       vcpu->run->io.port = port;
-
        return 0;
 }
 
+static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
+                                    int size, unsigned short port,
+                                    const void *val, unsigned int count)
+{
+       struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
+
+       memcpy(vcpu->arch.pio_data, val, size * count);
+       return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
+}
+
 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
 {
        return kvm_x86_ops->get_segment_base(vcpu, seg);
 }
 
-int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
+static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
 {
-       kvm_mmu_invlpg(vcpu, address);
-       return X86EMUL_CONTINUE;
+       kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
 }
 
 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
@@ -4148,22 +3950,20 @@ int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);
 
-int emulate_clts(struct kvm_vcpu *vcpu)
+static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
 {
-       kvm_x86_ops->set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
-       kvm_x86_ops->fpu_activate(vcpu);
-       return X86EMUL_CONTINUE;
+       kvm_emulate_wbinvd(emul_to_vcpu(ctxt));
 }
 
-int emulator_get_dr(int dr, unsigned long *dest, struct kvm_vcpu *vcpu)
+int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest)
 {
-       return _kvm_get_dr(vcpu, dr, dest);
+       return _kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
 }
 
-int emulator_set_dr(int dr, unsigned long value, struct kvm_vcpu *vcpu)
+int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
 {
 
-       return __kvm_set_dr(vcpu, dr, value);
+       return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
 }
 
 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
@@ -4171,8 +3971,9 @@ static u64 mk_cr_64(u64 curr_cr, u32 new_val)
        return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
 }
 
-static unsigned long emulator_get_cr(int cr, struct kvm_vcpu *vcpu)
+static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)
 {
+       struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
        unsigned long value;
 
        switch (cr) {
@@ -4199,8 +4000,9 @@ static unsigned long emulator_get_cr(int cr, struct kvm_vcpu *vcpu)
        return value;
 }
 
-static int emulator_set_cr(int cr, unsigned long val, struct kvm_vcpu *vcpu)
+static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)
 {
+       struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
        int res = 0;
 
        switch (cr) {
@@ -4227,33 +4029,45 @@ static int emulator_set_cr(int cr, unsigned long val, struct kvm_vcpu *vcpu)
        return res;
 }
 
-static int emulator_get_cpl(struct kvm_vcpu *vcpu)
+static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
+{
+       return kvm_x86_ops->get_cpl(emul_to_vcpu(ctxt));
+}
+
+static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
+{
+       kvm_x86_ops->get_gdt(emul_to_vcpu(ctxt), dt);
+}
+
+static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
 {
-       return kvm_x86_ops->get_cpl(vcpu);
+       kvm_x86_ops->get_idt(emul_to_vcpu(ctxt), dt);
 }
 
-static void emulator_get_gdt(struct desc_ptr *dt, struct kvm_vcpu *vcpu)
+static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
 {
-       kvm_x86_ops->get_gdt(vcpu, dt);
+       kvm_x86_ops->set_gdt(emul_to_vcpu(ctxt), dt);
 }
 
-static void emulator_get_idt(struct desc_ptr *dt, struct kvm_vcpu *vcpu)
+static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
 {
-       kvm_x86_ops->get_idt(vcpu, dt);
+       kvm_x86_ops->set_idt(emul_to_vcpu(ctxt), dt);
 }
 
-static unsigned long emulator_get_cached_segment_base(int seg,
-                                                     struct kvm_vcpu *vcpu)
+static unsigned long emulator_get_cached_segment_base(
+       struct x86_emulate_ctxt *ctxt, int seg)
 {
-       return get_segment_base(vcpu, seg);
+       return get_segment_base(emul_to_vcpu(ctxt), seg);
 }
 
-static bool emulator_get_cached_descriptor(struct desc_struct *desc, u32 *base3,
-                                          int seg, struct kvm_vcpu *vcpu)
+static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
+                                struct desc_struct *desc, u32 *base3,
+                                int seg)
 {
        struct kvm_segment var;
 
-       kvm_get_segment(vcpu, &var, seg);
+       kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
+       *selector = var.selector;
 
        if (var.unusable)
                return false;
@@ -4278,14 +4092,14 @@ static bool emulator_get_cached_descriptor(struct desc_struct *desc, u32 *base3,
        return true;
 }
 
-static void emulator_set_cached_descriptor(struct desc_struct *desc, u32 base3,
-                                          int seg, struct kvm_vcpu *vcpu)
+static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,
+                                struct desc_struct *desc, u32 base3,
+                                int seg)
 {
+       struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
        struct kvm_segment var;
 
-       /* needed to preserve selector */
-       kvm_get_segment(vcpu, &var, seg);
-
+       var.selector = selector;
        var.base = get_desc_base(desc);
 #ifdef CONFIG_X86_64
        var.base |= ((u64)base3) << 32;
@@ -4309,28 +4123,27 @@ static void emulator_set_cached_descriptor(struct desc_struct *desc, u32 base3,
        return;
 }
 
-static u16 emulator_get_segment_selector(int seg, struct kvm_vcpu *vcpu)
+static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
+                           u32 msr_index, u64 *pdata)
 {
-       struct kvm_segment kvm_seg;
-
-       kvm_get_segment(vcpu, &kvm_seg, seg);
-       return kvm_seg.selector;
+       return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
 }
 
-static void emulator_set_segment_selector(u16 sel, int seg,
-                                         struct kvm_vcpu *vcpu)
+static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
+                           u32 msr_index, u64 data)
 {
-       struct kvm_segment kvm_seg;
+       return kvm_set_msr(emul_to_vcpu(ctxt), msr_index, data);
+}
 
-       kvm_get_segment(vcpu, &kvm_seg, seg);
-       kvm_seg.selector = sel;
-       kvm_set_segment(vcpu, &kvm_seg, seg);
+static void emulator_halt(struct x86_emulate_ctxt *ctxt)
+{
+       emul_to_vcpu(ctxt)->arch.halt_request = 1;
 }
 
 static void emulator_get_fpu(struct x86_emulate_ctxt *ctxt)
 {
        preempt_disable();
-       kvm_load_guest_fpu(ctxt->vcpu);
+       kvm_load_guest_fpu(emul_to_vcpu(ctxt));
        /*
         * CR0.TS may reference the host fpu state, not the guest fpu state,
         * so it may be clear at this point.
@@ -4343,11 +4156,11 @@ static void emulator_put_fpu(struct x86_emulate_ctxt *ctxt)
        preempt_enable();
 }
 
-static int emulator_intercept(struct kvm_vcpu *vcpu,
+static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
                              struct x86_instruction_info *info,
                              enum x86_intercept_stage stage)
 {
-       return kvm_x86_ops->check_intercept(vcpu, info, stage);
+       return kvm_x86_ops->check_intercept(emul_to_vcpu(ctxt), info, stage);
 }
 
 static struct x86_emulate_ops emulate_ops = {
@@ -4357,22 +4170,26 @@ static struct x86_emulate_ops emulate_ops = {
        .read_emulated       = emulator_read_emulated,
        .write_emulated      = emulator_write_emulated,
        .cmpxchg_emulated    = emulator_cmpxchg_emulated,
+       .invlpg              = emulator_invlpg,
        .pio_in_emulated     = emulator_pio_in_emulated,
        .pio_out_emulated    = emulator_pio_out_emulated,
-       .get_cached_descriptor = emulator_get_cached_descriptor,
-       .set_cached_descriptor = emulator_set_cached_descriptor,
-       .get_segment_selector = emulator_get_segment_selector,
-       .set_segment_selector = emulator_set_segment_selector,
+       .get_segment         = emulator_get_segment,
+       .set_segment         = emulator_set_segment,
        .get_cached_segment_base = emulator_get_cached_segment_base,
        .get_gdt             = emulator_get_gdt,
        .get_idt             = emulator_get_idt,
+       .set_gdt             = emulator_set_gdt,
+       .set_idt             = emulator_set_idt,
        .get_cr              = emulator_get_cr,
        .set_cr              = emulator_set_cr,
        .cpl                 = emulator_get_cpl,
        .get_dr              = emulator_get_dr,
        .set_dr              = emulator_set_dr,
-       .set_msr             = kvm_set_msr,
-       .get_msr             = kvm_get_msr,
+       .set_msr             = emulator_set_msr,
+       .get_msr             = emulator_get_msr,
+       .halt                = emulator_halt,
+       .wbinvd              = emulator_wbinvd,
+       .fix_hypercall       = emulator_fix_hypercall,
        .get_fpu             = emulator_get_fpu,
        .put_fpu             = emulator_put_fpu,
        .intercept           = emulator_intercept,
@@ -4412,53 +4229,71 @@ static void inject_emulated_exception(struct kvm_vcpu *vcpu)
                kvm_queue_exception(vcpu, ctxt->exception.vector);
 }
 
+static void init_decode_cache(struct x86_emulate_ctxt *ctxt,
+                             const unsigned long *regs)
+{
+       memset(&ctxt->twobyte, 0,
+              (void *)&ctxt->regs - (void *)&ctxt->twobyte);
+       memcpy(ctxt->regs, regs, sizeof(ctxt->regs));
+
+       ctxt->fetch.start = 0;
+       ctxt->fetch.end = 0;
+       ctxt->io_read.pos = 0;
+       ctxt->io_read.end = 0;
+       ctxt->mem_read.pos = 0;
+       ctxt->mem_read.end = 0;
+}
+
 static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
 {
-       struct decode_cache *c = &vcpu->arch.emulate_ctxt.decode;
+       struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
        int cs_db, cs_l;
 
+       /*
+        * TODO: fix emulate.c to use guest_read/write_register
+        * instead of direct ->regs accesses, can save hundred cycles
+        * on Intel for instructions that don't read/change RSP, for
+        * for example.
+        */
        cache_all_regs(vcpu);
 
        kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 
-       vcpu->arch.emulate_ctxt.vcpu = vcpu;
-       vcpu->arch.emulate_ctxt.eflags = kvm_get_rflags(vcpu);
-       vcpu->arch.emulate_ctxt.eip = kvm_rip_read(vcpu);
-       vcpu->arch.emulate_ctxt.mode =
-               (!is_protmode(vcpu)) ? X86EMUL_MODE_REAL :
-               (vcpu->arch.emulate_ctxt.eflags & X86_EFLAGS_VM)
-               ? X86EMUL_MODE_VM86 : cs_l
-               ? X86EMUL_MODE_PROT64 : cs_db
-               ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
-       vcpu->arch.emulate_ctxt.guest_mode = is_guest_mode(vcpu);
-       memset(c, 0, sizeof(struct decode_cache));
-       memcpy(c->regs, vcpu->arch.regs, sizeof c->regs);
+       ctxt->eflags = kvm_get_rflags(vcpu);
+       ctxt->eip = kvm_rip_read(vcpu);
+       ctxt->mode = (!is_protmode(vcpu))               ? X86EMUL_MODE_REAL :
+                    (ctxt->eflags & X86_EFLAGS_VM)     ? X86EMUL_MODE_VM86 :
+                    cs_l                               ? X86EMUL_MODE_PROT64 :
+                    cs_db                              ? X86EMUL_MODE_PROT32 :
+                                                         X86EMUL_MODE_PROT16;
+       ctxt->guest_mode = is_guest_mode(vcpu);
+
+       init_decode_cache(ctxt, vcpu->arch.regs);
        vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
 }
 
 int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
 {
-       struct decode_cache *c = &vcpu->arch.emulate_ctxt.decode;
+       struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
        int ret;
 
        init_emulate_ctxt(vcpu);
 
-       vcpu->arch.emulate_ctxt.decode.op_bytes = 2;
-       vcpu->arch.emulate_ctxt.decode.ad_bytes = 2;
-       vcpu->arch.emulate_ctxt.decode.eip = vcpu->arch.emulate_ctxt.eip +
-                                                                inc_eip;
-       ret = emulate_int_real(&vcpu->arch.emulate_ctxt, &emulate_ops, irq);
+       ctxt->op_bytes = 2;
+       ctxt->ad_bytes = 2;
+       ctxt->_eip = ctxt->eip + inc_eip;
+       ret = emulate_int_real(ctxt, irq);
 
        if (ret != X86EMUL_CONTINUE)
                return EMULATE_FAIL;
 
-       vcpu->arch.emulate_ctxt.eip = c->eip;
-       memcpy(vcpu->arch.regs, c->regs, sizeof c->regs);
-       kvm_rip_write(vcpu, vcpu->arch.emulate_ctxt.eip);
-       kvm_set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags);
+       ctxt->eip = ctxt->_eip;
+       memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
+       kvm_rip_write(vcpu, ctxt->eip);
+       kvm_set_rflags(vcpu, ctxt->eflags);
 
        if (irq == NMI_VECTOR)
-               vcpu->arch.nmi_pending = false;
+               vcpu->arch.nmi_pending = 0;
        else
                vcpu->arch.interrupt.pending = false;
 
@@ -4509,6 +4344,50 @@ static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva)
        return false;
 }
 
+static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
+                             unsigned long cr2,  int emulation_type)
+{
+       struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
+       unsigned long last_retry_eip, last_retry_addr, gpa = cr2;
+
+       last_retry_eip = vcpu->arch.last_retry_eip;
+       last_retry_addr = vcpu->arch.last_retry_addr;
+
+       /*
+        * If the emulation is caused by #PF and it is non-page_table
+        * writing instruction, it means the VM-EXIT is caused by shadow
+        * page protected, we can zap the shadow page and retry this
+        * instruction directly.
+        *
+        * Note: if the guest uses a non-page-table modifying instruction
+        * on the PDE that points to the instruction, then we will unmap
+        * the instruction and go to an infinite loop. So, we cache the
+        * last retried eip and the last fault address, if we meet the eip
+        * and the address again, we can break out of the potential infinite
+        * loop.
+        */
+       vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;
+
+       if (!(emulation_type & EMULTYPE_RETRY))
+               return false;
+
+       if (x86_page_table_writing_insn(ctxt))
+               return false;
+
+       if (ctxt->eip == last_retry_eip && last_retry_addr == cr2)
+               return false;
+
+       vcpu->arch.last_retry_eip = ctxt->eip;
+       vcpu->arch.last_retry_addr = cr2;
+
+       if (!vcpu->arch.mmu.direct_map)
+               gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL);
+
+       kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
+
+       return true;
+}
+
 int x86_emulate_instruction(struct kvm_vcpu *vcpu,
                            unsigned long cr2,
                            int emulation_type,
@@ -4516,33 +4395,25 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu,
                            int insn_len)
 {
        int r;
-       struct decode_cache *c = &vcpu->arch.emulate_ctxt.decode;
+       struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
        bool writeback = true;
 
        kvm_clear_exception_queue(vcpu);
-       vcpu->arch.mmio_fault_cr2 = cr2;
-       /*
-        * TODO: fix emulate.c to use guest_read/write_register
-        * instead of direct ->regs accesses, can save hundred cycles
-        * on Intel for instructions that don't read/change RSP, for
-        * for example.
-        */
-       cache_all_regs(vcpu);
 
        if (!(emulation_type & EMULTYPE_NO_DECODE)) {
                init_emulate_ctxt(vcpu);
-               vcpu->arch.emulate_ctxt.interruptibility = 0;
-               vcpu->arch.emulate_ctxt.have_exception = false;
-               vcpu->arch.emulate_ctxt.perm_ok = false;
+               ctxt->interruptibility = 0;
+               ctxt->have_exception = false;
+               ctxt->perm_ok = false;
 
-               vcpu->arch.emulate_ctxt.only_vendor_specific_insn
+               ctxt->only_vendor_specific_insn
                        = emulation_type & EMULTYPE_TRAP_UD;
 
-               r = x86_decode_insn(&vcpu->arch.emulate_ctxt, insn, insn_len);
+               r = x86_decode_insn(ctxt, insn, insn_len);
 
                trace_kvm_emulate_insn_start(vcpu);
                ++vcpu->stat.insn_emulation;
-               if (r)  {
+               if (r != EMULATION_OK)  {
                        if (emulation_type & EMULTYPE_TRAP_UD)
                                return EMULATE_FAIL;
                        if (reexecute_instruction(vcpu, cr2))
@@ -4554,19 +4425,22 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu,
        }
 
        if (emulation_type & EMULTYPE_SKIP) {
-               kvm_rip_write(vcpu, vcpu->arch.emulate_ctxt.decode.eip);
+               kvm_rip_write(vcpu, ctxt->_eip);
                return EMULATE_DONE;
        }
 
+       if (retry_instruction(ctxt, cr2, emulation_type))
+               return EMULATE_DONE;
+
        /* this is needed for vmware backdoor interface to work since it
           changes registers values  during IO operation */
        if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
                vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
-               memcpy(c->regs, vcpu->arch.regs, sizeof c->regs);
+               memcpy(ctxt->regs, vcpu->arch.regs, sizeof ctxt->regs);
        }
 
 restart:
-       r = x86_emulate_insn(&vcpu->arch.emulate_ctxt);
+       r = x86_emulate_insn(ctxt);
 
        if (r == EMULATION_INTERCEPTED)
                return EMULATE_DONE;
@@ -4578,7 +4452,7 @@ restart:
                return handle_emulation_failure(vcpu);
        }
 
-       if (vcpu->arch.emulate_ctxt.have_exception) {
+       if (ctxt->have_exception) {
                inject_emulated_exception(vcpu);
                r = EMULATE_DONE;
        } else if (vcpu->arch.pio.count) {
@@ -4597,13 +4471,12 @@ restart:
                r = EMULATE_DONE;
 
        if (writeback) {
-               toggle_interruptibility(vcpu,
-                               vcpu->arch.emulate_ctxt.interruptibility);
-               kvm_set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags);
+               toggle_interruptibility(vcpu, ctxt->interruptibility);
+               kvm_set_rflags(vcpu, ctxt->eflags);
                kvm_make_request(KVM_REQ_EVENT, vcpu);
-               memcpy(vcpu->arch.regs, c->regs, sizeof c->regs);
+               memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
                vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
-               kvm_rip_write(vcpu, vcpu->arch.emulate_ctxt.eip);
+               kvm_rip_write(vcpu, ctxt->eip);
        } else
                vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
 
@@ -4614,7 +4487,8 @@ EXPORT_SYMBOL_GPL(x86_emulate_instruction);
 int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port)
 {
        unsigned long val = kvm_register_read(vcpu, VCPU_REGS_RAX);
-       int ret = emulator_pio_out_emulated(size, port, &val, 1, vcpu);
+       int ret = emulator_pio_out_emulated(&vcpu->arch.emulate_ctxt,
+                                           size, port, &val, 1);
        /* do not return to emulator after return from userspace */
        vcpu->arch.pio.count = 0;
        return ret;
@@ -4778,15 +4652,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
 
 static int kvm_is_in_guest(void)
 {
-       return percpu_read(current_vcpu) != NULL;
+       return __this_cpu_read(current_vcpu) != NULL;
 }
 
 static int kvm_is_user_mode(void)
 {
        int user_mode = 3;
 
-       if (percpu_read(current_vcpu))
-               user_mode = kvm_x86_ops->get_cpl(percpu_read(current_vcpu));
+       if (__this_cpu_read(current_vcpu))
+               user_mode = kvm_x86_ops->get_cpl(__this_cpu_read(current_vcpu));
 
        return user_mode != 0;
 }
@@ -4795,8 +4669,8 @@ static unsigned long kvm_get_guest_ip(void)
 {
        unsigned long ip = 0;
 
-       if (percpu_read(current_vcpu))
-               ip = kvm_rip_read(percpu_read(current_vcpu));
+       if (__this_cpu_read(current_vcpu))
+               ip = kvm_rip_read(__this_cpu_read(current_vcpu));
 
        return ip;
 }
@@ -4809,16 +4683,40 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = {
 
 void kvm_before_handle_nmi(struct kvm_vcpu *vcpu)
 {
-       percpu_write(current_vcpu, vcpu);
+       __this_cpu_write(current_vcpu, vcpu);
 }
 EXPORT_SYMBOL_GPL(kvm_before_handle_nmi);
 
 void kvm_after_handle_nmi(struct kvm_vcpu *vcpu)
 {
-       percpu_write(current_vcpu, NULL);
+       __this_cpu_write(current_vcpu, NULL);
 }
 EXPORT_SYMBOL_GPL(kvm_after_handle_nmi);
 
+static void kvm_set_mmio_spte_mask(void)
+{
+       u64 mask;
+       int maxphyaddr = boot_cpu_data.x86_phys_bits;
+
+       /*
+        * Set the reserved bits and the present bit of an paging-structure
+        * entry to generate page fault with PFER.RSV = 1.
+        */
+       mask = ((1ull << (62 - maxphyaddr + 1)) - 1) << maxphyaddr;
+       mask |= 1ull;
+
+#ifdef CONFIG_X86_64
+       /*
+        * If reserved bit is not supported, clear the present bit to disable
+        * mmio page fault.
+        */
+       if (maxphyaddr == 52)
+               mask &= ~1ull;
+#endif
+
+       kvm_mmu_set_mmio_spte_mask(mask);
+}
+
 int kvm_arch_init(void *opaque)
 {
        int r;
@@ -4845,10 +4743,10 @@ int kvm_arch_init(void *opaque)
        if (r)
                goto out;
 
+       kvm_set_mmio_spte_mask();
        kvm_init_msr_list();
 
        kvm_x86_ops = ops;
-       kvm_mmu_set_nonpresent_ptes(0ull, 0ull);
        kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
                        PT_DIRTY_MASK, PT64_NX_MASK, 0);
 
@@ -4890,15 +4788,6 @@ int kvm_emulate_halt(struct kvm_vcpu *vcpu)
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
 
-static inline gpa_t hc_gpa(struct kvm_vcpu *vcpu, unsigned long a0,
-                          unsigned long a1)
-{
-       if (is_long_mode(vcpu))
-               return a0;
-       else
-               return a0 | ((gpa_t)a1 << 32);
-}
-
 int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
 {
        u64 param, ingpa, outgpa, ret;
@@ -4994,9 +4883,6 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
        case KVM_HC_VAPIC_POLL_IRQ:
                ret = 0;
                break;
-       case KVM_HC_MMU_OP:
-               r = kvm_pv_mmu_op(vcpu, a0, hc_gpa(vcpu, a1, a2), &ret);
-               break;
        default:
                ret = -KVM_ENOSYS;
                break;
@@ -5008,8 +4894,9 @@ out:
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);
 
-int kvm_fix_hypercall(struct kvm_vcpu *vcpu)
+int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)
 {
+       struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
        char instruction[3];
        unsigned long rip = kvm_rip_read(vcpu);
 
@@ -5022,144 +4909,10 @@ int kvm_fix_hypercall(struct kvm_vcpu *vcpu)
 
        kvm_x86_ops->patch_hypercall(vcpu, instruction);
 
-       return emulator_write_emulated(&vcpu->arch.emulate_ctxt,
-                                      rip, instruction, 3, NULL);
-}
-
-void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
-{
-       struct desc_ptr dt = { limit, base };
-
-       kvm_x86_ops->set_gdt(vcpu, &dt);
-}
-
-void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
-{
-       struct desc_ptr dt = { limit, base };
-
-       kvm_x86_ops->set_idt(vcpu, &dt);
-}
-
-static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i)
-{
-       struct kvm_cpuid_entry2 *e = &vcpu->arch.cpuid_entries[i];
-       int j, nent = vcpu->arch.cpuid_nent;
-
-       e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT;
-       /* when no next entry is found, the current entry[i] is reselected */
-       for (j = i + 1; ; j = (j + 1) % nent) {
-               struct kvm_cpuid_entry2 *ej = &vcpu->arch.cpuid_entries[j];
-               if (ej->function == e->function) {
-                       ej->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT;
-                       return j;
-               }
-       }
-       return 0; /* silence gcc, even though control never reaches here */
-}
-
-/* find an entry with matching function, matching index (if needed), and that
- * should be read next (if it's stateful) */
-static int is_matching_cpuid_entry(struct kvm_cpuid_entry2 *e,
-       u32 function, u32 index)
-{
-       if (e->function != function)
-               return 0;
-       if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index)
-               return 0;
-       if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) &&
-           !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT))
-               return 0;
-       return 1;
-}
-
-struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu,
-                                             u32 function, u32 index)
-{
-       int i;
-       struct kvm_cpuid_entry2 *best = NULL;
-
-       for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
-               struct kvm_cpuid_entry2 *e;
-
-               e = &vcpu->arch.cpuid_entries[i];
-               if (is_matching_cpuid_entry(e, function, index)) {
-                       if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC)
-                               move_to_next_stateful_cpuid_entry(vcpu, i);
-                       best = e;
-                       break;
-               }
-       }
-       return best;
-}
-EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry);
-
-int cpuid_maxphyaddr(struct kvm_vcpu *vcpu)
-{
-       struct kvm_cpuid_entry2 *best;
-
-       best = kvm_find_cpuid_entry(vcpu, 0x80000000, 0);
-       if (!best || best->eax < 0x80000008)
-               goto not_found;
-       best = kvm_find_cpuid_entry(vcpu, 0x80000008, 0);
-       if (best)
-               return best->eax & 0xff;
-not_found:
-       return 36;
+       return emulator_write_emulated(ctxt, rip, instruction, 3, NULL);
 }
 
 /*
- * If no match is found, check whether we exceed the vCPU's limit
- * and return the content of the highest valid _standard_ leaf instead.
- * This is to satisfy the CPUID specification.
- */
-static struct kvm_cpuid_entry2* check_cpuid_limit(struct kvm_vcpu *vcpu,
-                                                  u32 function, u32 index)
-{
-       struct kvm_cpuid_entry2 *maxlevel;
-
-       maxlevel = kvm_find_cpuid_entry(vcpu, function & 0x80000000, 0);
-       if (!maxlevel || maxlevel->eax >= function)
-               return NULL;
-       if (function & 0x80000000) {
-               maxlevel = kvm_find_cpuid_entry(vcpu, 0, 0);
-               if (!maxlevel)
-                       return NULL;
-       }
-       return kvm_find_cpuid_entry(vcpu, maxlevel->eax, index);
-}
-
-void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
-{
-       u32 function, index;
-       struct kvm_cpuid_entry2 *best;
-
-       function = kvm_register_read(vcpu, VCPU_REGS_RAX);
-       index = kvm_register_read(vcpu, VCPU_REGS_RCX);
-       kvm_register_write(vcpu, VCPU_REGS_RAX, 0);
-       kvm_register_write(vcpu, VCPU_REGS_RBX, 0);
-       kvm_register_write(vcpu, VCPU_REGS_RCX, 0);
-       kvm_register_write(vcpu, VCPU_REGS_RDX, 0);
-       best = kvm_find_cpuid_entry(vcpu, function, index);
-
-       if (!best)
-               best = check_cpuid_limit(vcpu, function, index);
-
-       if (best) {
-               kvm_register_write(vcpu, VCPU_REGS_RAX, best->eax);
-               kvm_register_write(vcpu, VCPU_REGS_RBX, best->ebx);
-               kvm_register_write(vcpu, VCPU_REGS_RCX, best->ecx);
-               kvm_register_write(vcpu, VCPU_REGS_RDX, best->edx);
-       }
-       kvm_x86_ops->skip_emulated_instruction(vcpu);
-       trace_kvm_cpuid(function,
-                       kvm_register_read(vcpu, VCPU_REGS_RAX),
-                       kvm_register_read(vcpu, VCPU_REGS_RBX),
-                       kvm_register_read(vcpu, VCPU_REGS_RCX),
-                       kvm_register_read(vcpu, VCPU_REGS_RDX));
-}
-EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
-
-/*
  * Check if userspace requested an interrupt window, and that the
  * interrupt window is open.
  *
@@ -5265,7 +5018,7 @@ static void inject_pending_event(struct kvm_vcpu *vcpu)
        /* try to inject new event if pending */
        if (vcpu->arch.nmi_pending) {
                if (kvm_x86_ops->nmi_allowed(vcpu)) {
-                       vcpu->arch.nmi_pending = false;
+                       --vcpu->arch.nmi_pending;
                        vcpu->arch.nmi_injected = true;
                        kvm_x86_ops->set_nmi(vcpu);
                }
@@ -5297,12 +5050,29 @@ static void kvm_put_guest_xcr0(struct kvm_vcpu *vcpu)
        }
 }
 
+static void process_nmi(struct kvm_vcpu *vcpu)
+{
+       unsigned limit = 2;
+
+       /*
+        * x86 is limited to one NMI running, and one NMI pending after it.
+        * If an NMI is already in progress, limit further NMIs to just one.
+        * Otherwise, allow two (and we'll inject the first one immediately).
+        */
+       if (kvm_x86_ops->get_nmi_mask(vcpu) || vcpu->arch.nmi_injected)
+               limit = 1;
+
+       vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
+       vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
+       kvm_make_request(KVM_REQ_EVENT, vcpu);
+}
+
 static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 {
        int r;
-       bool nmi_pending;
        bool req_int_win = !irqchip_in_kernel(vcpu->kvm) &&
                vcpu->run->request_interrupt_window;
+       bool req_immediate_exit = 0;
 
        if (vcpu->requests) {
                if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu))
@@ -5338,25 +5108,23 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
                        r = 1;
                        goto out;
                }
+               if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
+                       record_steal_time(vcpu);
+               if (kvm_check_request(KVM_REQ_NMI, vcpu))
+                       process_nmi(vcpu);
+               req_immediate_exit =
+                       kvm_check_request(KVM_REQ_IMMEDIATE_EXIT, vcpu);
        }
 
        r = kvm_mmu_reload(vcpu);
        if (unlikely(r))
                goto out;
 
-       /*
-        * An NMI can be injected between local nmi_pending read and
-        * vcpu->arch.nmi_pending read inside inject_pending_event().
-        * But in that case, KVM_REQ_EVENT will be set, which makes
-        * the race described above benign.
-        */
-       nmi_pending = ACCESS_ONCE(vcpu->arch.nmi_pending);
-
        if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) {
                inject_pending_event(vcpu);
 
                /* enable NMI/IRQ window open exits if needed */
-               if (nmi_pending)
+               if (vcpu->arch.nmi_pending)
                        kvm_x86_ops->enable_nmi_window(vcpu);
                else if (kvm_cpu_has_interrupt(vcpu) || req_int_win)
                        kvm_x86_ops->enable_irq_window(vcpu);
@@ -5396,6 +5164,9 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 
        srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
 
+       if (req_immediate_exit)
+               smp_send_reschedule(vcpu->cpu);
+
        kvm_guest_enter();
 
        if (unlikely(vcpu->arch.switch_db_regs)) {
@@ -5419,7 +5190,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
        if (hw_breakpoint_active())
                hw_breakpoint_restore();
 
-       kvm_get_msr(vcpu, MSR_IA32_TSC, &vcpu->arch.last_guest_tsc);
+       vcpu->arch.last_guest_tsc = kvm_x86_ops->read_l1_tsc(vcpu);
 
        vcpu->mode = OUTSIDE_GUEST_MODE;
        smp_wmb();
@@ -5547,7 +5318,8 @@ static int complete_mmio(struct kvm_vcpu *vcpu)
        if (vcpu->mmio_needed) {
                vcpu->mmio_needed = 0;
                if (!vcpu->mmio_is_write)
-                       memcpy(vcpu->mmio_data, run->mmio.data, 8);
+                       memcpy(vcpu->mmio_data + vcpu->mmio_index,
+                              run->mmio.data, 8);
                vcpu->mmio_index += 8;
                if (vcpu->mmio_index < vcpu->mmio_size) {
                        run->exit_reason = KVM_EXIT_MMIO;
@@ -5600,10 +5372,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
        if (r <= 0)
                goto out;
 
-       if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL)
-               kvm_register_write(vcpu, VCPU_REGS_RAX,
-                                    kvm_run->hypercall.ret);
-
        r = __vcpu_run(vcpu);
 
 out:
@@ -5624,8 +5392,8 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
                 * that usually, but some bad designed PV devices (vmware
                 * backdoor interface) need this to work
                 */
-               struct decode_cache *c = &vcpu->arch.emulate_ctxt.decode;
-               memcpy(vcpu->arch.regs, c->regs, sizeof c->regs);
+               struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
+               memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
                vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
        }
        regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX);
@@ -5754,21 +5522,20 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason,
                    bool has_error_code, u32 error_code)
 {
-       struct decode_cache *c = &vcpu->arch.emulate_ctxt.decode;
+       struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
        int ret;
 
        init_emulate_ctxt(vcpu);
 
-       ret = emulator_task_switch(&vcpu->arch.emulate_ctxt,
-                                  tss_selector, reason, has_error_code,
-                                  error_code);
+       ret = emulator_task_switch(ctxt, tss_selector, reason,
+                                  has_error_code, error_code);
 
        if (ret)
                return EMULATE_FAIL;
 
-       memcpy(vcpu->arch.regs, c->regs, sizeof c->regs);
-       kvm_rip_write(vcpu, vcpu->arch.emulate_ctxt.eip);
-       kvm_set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags);
+       memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
+       kvm_rip_write(vcpu, ctxt->eip);
+       kvm_set_rflags(vcpu, ctxt->eflags);
        kvm_make_request(KVM_REQ_EVENT, vcpu);
        return EMULATE_DONE;
 }
@@ -5806,7 +5573,7 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
        mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
        kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
        if (sregs->cr4 & X86_CR4_OSXSAVE)
-               update_cpuid(vcpu);
+               kvm_update_cpuid(vcpu);
 
        idx = srcu_read_lock(&vcpu->kvm->srcu);
        if (!is_long_mode(vcpu) && is_pae(vcpu)) {
@@ -6046,12 +5813,7 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
        if (r == 0)
                r = kvm_mmu_setup(vcpu);
        vcpu_put(vcpu);
-       if (r < 0)
-               goto free_vcpu;
 
-       return 0;
-free_vcpu:
-       kvm_x86_ops->vcpu_free(vcpu);
        return r;
 }
 
@@ -6069,7 +5831,8 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 
 int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu)
 {
-       vcpu->arch.nmi_pending = false;
+       atomic_set(&vcpu->arch.nmi_queued, 0);
+       vcpu->arch.nmi_pending = 0;
        vcpu->arch.nmi_injected = false;
 
        vcpu->arch.switch_db_regs = 0;
@@ -6079,6 +5842,7 @@ int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu)
 
        kvm_make_request(KVM_REQ_EVENT, vcpu);
        vcpu->arch.apf.msr_val = 0;
+       vcpu->arch.st.msr_val = 0;
 
        kvmclock_reset(vcpu);
 
@@ -6134,10 +5898,6 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
        kvm = vcpu->kvm;
 
        vcpu->arch.emulate_ctxt.ops = &emulate_ops;
-       vcpu->arch.walk_mmu = &vcpu->arch.mmu;
-       vcpu->arch.mmu.root_hpa = INVALID_PAGE;
-       vcpu->arch.mmu.translate_gpa = translate_gpa;
-       vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
        if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_bsp(vcpu))
                vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
        else
@@ -6343,7 +6103,7 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
                !vcpu->arch.apf.halted)
                || !list_empty_careful(&vcpu->async_pf.done)
                || vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED
-               || vcpu->arch.nmi_pending ||
+               || atomic_read(&vcpu->arch.nmi_queued) ||
                (kvm_arch_interrupt_allowed(vcpu) &&
                 kvm_cpu_has_interrupt(vcpu));
 }