mm: Remove slab destructors from kmem_cache_create().
[linux-2.6.git] / drivers / kvm / mmu.c
index 1dcbbd5..1199d3f 100644 (file)
  * the COPYING file in the top-level directory.
  *
  */
+
+#include "vmx.h"
+#include "kvm.h"
+
 #include <linux/types.h>
 #include <linux/string.h>
-#include <asm/page.h>
 #include <linux/mm.h>
 #include <linux/highmem.h>
 #include <linux/module.h>
 
-#include "vmx.h"
-#include "kvm.h"
+#include <asm/page.h>
+#include <asm/cmpxchg.h>
+
+#undef MMU_DEBUG
+
+#undef AUDIT
+
+#ifdef AUDIT
+static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg);
+#else
+static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
+#endif
+
+#ifdef MMU_DEBUG
+
+#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
+#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
+
+#else
 
 #define pgprintk(x...) do { } while (0)
 #define rmap_printk(x...) do { } while (0)
 
+#endif
+
+#if defined(MMU_DEBUG) || defined(AUDIT)
+static int dbg = 1;
+#endif
+
+#ifndef MMU_DEBUG
+#define ASSERT(x) do { } while (0)
+#else
 #define ASSERT(x)                                                      \
        if (!(x)) {                                                     \
                printk(KERN_WARNING "assertion failed %s:%d: %s\n",     \
                       __FILE__, __LINE__, #x);                         \
        }
+#endif
 
-#define PT64_ENT_PER_PAGE 512
-#define PT32_ENT_PER_PAGE 1024
+#define PT64_PT_BITS 9
+#define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS)
+#define PT32_PT_BITS 10
+#define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS)
 
 #define PT_WRITABLE_SHIFT 1
 
 #define PT32_DIR_PSE36_MASK (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)
 
 
-#define PT32_PTE_COPY_MASK \
-       (PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_DIRTY_MASK | PT_GLOBAL_MASK)
-
-#define PT64_PTE_COPY_MASK (PT64_NX_MASK | PT32_PTE_COPY_MASK)
-
 #define PT_FIRST_AVAIL_BITS_SHIFT 9
 #define PT64_SECOND_AVAIL_BITS_SHIFT 52
 
-#define PT_SHADOW_PS_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
 #define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
 
-#define PT_SHADOW_WRITABLE_SHIFT (PT_FIRST_AVAIL_BITS_SHIFT + 1)
-#define PT_SHADOW_WRITABLE_MASK (1ULL << PT_SHADOW_WRITABLE_SHIFT)
-
-#define PT_SHADOW_USER_SHIFT (PT_SHADOW_WRITABLE_SHIFT + 1)
-#define PT_SHADOW_USER_MASK (1ULL << (PT_SHADOW_USER_SHIFT))
-
-#define PT_SHADOW_BITS_OFFSET (PT_SHADOW_WRITABLE_SHIFT - PT_WRITABLE_SHIFT)
-
 #define VALID_PAGE(x) ((x) != INVALID_PAGE)
 
 #define PT64_LEVEL_BITS 9
        (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
 
 
-#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & PAGE_MASK)
+#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
 #define PT64_DIR_BASE_ADDR_MASK \
        (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
 
 #define PFERR_PRESENT_MASK (1U << 0)
 #define PFERR_WRITE_MASK (1U << 1)
 #define PFERR_USER_MASK (1U << 2)
+#define PFERR_FETCH_MASK (1U << 4)
 
 #define PT64_ROOT_LEVEL 4
 #define PT32_ROOT_LEVEL 2
@@ -133,6 +152,11 @@ struct kvm_rmap_desc {
        struct kvm_rmap_desc *more;
 };
 
+static struct kmem_cache *pte_chain_cache;
+static struct kmem_cache *rmap_desc_cache;
+static struct kmem_cache *mmu_page_cache;
+static struct kmem_cache *mmu_page_header_cache;
+
 static int is_write_protection(struct kvm_vcpu *vcpu)
 {
        return vcpu->cr0 & CR0_WP_MASK;
@@ -143,6 +167,11 @@ static int is_cpuid_PSE36(void)
        return 1;
 }
 
+static int is_nx(struct kvm_vcpu *vcpu)
+{
+       return vcpu->shadow_efer & EFER_NX;
+}
+
 static int is_present_pte(unsigned long pte)
 {
        return pte & PT_PRESENT_MASK;
@@ -164,6 +193,126 @@ static int is_rmap_pte(u64 pte)
                == (PT_WRITABLE_MASK | PT_PRESENT_MASK);
 }
 
+static void set_shadow_pte(u64 *sptep, u64 spte)
+{
+#ifdef CONFIG_X86_64
+       set_64bit((unsigned long *)sptep, spte);
+#else
+       set_64bit((unsigned long long *)sptep, spte);
+#endif
+}
+
+static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
+                                 struct kmem_cache *base_cache, int min,
+                                 gfp_t gfp_flags)
+{
+       void *obj;
+
+       if (cache->nobjs >= min)
+               return 0;
+       while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
+               obj = kmem_cache_zalloc(base_cache, gfp_flags);
+               if (!obj)
+                       return -ENOMEM;
+               cache->objects[cache->nobjs++] = obj;
+       }
+       return 0;
+}
+
+static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
+{
+       while (mc->nobjs)
+               kfree(mc->objects[--mc->nobjs]);
+}
+
+static int __mmu_topup_memory_caches(struct kvm_vcpu *vcpu, gfp_t gfp_flags)
+{
+       int r;
+
+       r = mmu_topup_memory_cache(&vcpu->mmu_pte_chain_cache,
+                                  pte_chain_cache, 4, gfp_flags);
+       if (r)
+               goto out;
+       r = mmu_topup_memory_cache(&vcpu->mmu_rmap_desc_cache,
+                                  rmap_desc_cache, 1, gfp_flags);
+       if (r)
+               goto out;
+       r = mmu_topup_memory_cache(&vcpu->mmu_page_cache,
+                                  mmu_page_cache, 4, gfp_flags);
+       if (r)
+               goto out;
+       r = mmu_topup_memory_cache(&vcpu->mmu_page_header_cache,
+                                  mmu_page_header_cache, 4, gfp_flags);
+out:
+       return r;
+}
+
+static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
+{
+       int r;
+
+       r = __mmu_topup_memory_caches(vcpu, GFP_NOWAIT);
+       if (r < 0) {
+               spin_unlock(&vcpu->kvm->lock);
+               kvm_arch_ops->vcpu_put(vcpu);
+               r = __mmu_topup_memory_caches(vcpu, GFP_KERNEL);
+               kvm_arch_ops->vcpu_load(vcpu);
+               spin_lock(&vcpu->kvm->lock);
+       }
+       return r;
+}
+
+static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
+{
+       mmu_free_memory_cache(&vcpu->mmu_pte_chain_cache);
+       mmu_free_memory_cache(&vcpu->mmu_rmap_desc_cache);
+       mmu_free_memory_cache(&vcpu->mmu_page_cache);
+       mmu_free_memory_cache(&vcpu->mmu_page_header_cache);
+}
+
+static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
+                                   size_t size)
+{
+       void *p;
+
+       BUG_ON(!mc->nobjs);
+       p = mc->objects[--mc->nobjs];
+       memset(p, 0, size);
+       return p;
+}
+
+static void mmu_memory_cache_free(struct kvm_mmu_memory_cache *mc, void *obj)
+{
+       if (mc->nobjs < KVM_NR_MEM_OBJS)
+               mc->objects[mc->nobjs++] = obj;
+       else
+               kfree(obj);
+}
+
+static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
+{
+       return mmu_memory_cache_alloc(&vcpu->mmu_pte_chain_cache,
+                                     sizeof(struct kvm_pte_chain));
+}
+
+static void mmu_free_pte_chain(struct kvm_vcpu *vcpu,
+                              struct kvm_pte_chain *pc)
+{
+       mmu_memory_cache_free(&vcpu->mmu_pte_chain_cache, pc);
+}
+
+static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
+{
+       return mmu_memory_cache_alloc(&vcpu->mmu_rmap_desc_cache,
+                                     sizeof(struct kvm_rmap_desc));
+}
+
+static void mmu_free_rmap_desc(struct kvm_vcpu *vcpu,
+                              struct kvm_rmap_desc *rd)
+{
+       mmu_memory_cache_free(&vcpu->mmu_rmap_desc_cache, rd);
+}
+
 /*
  * Reverse mapping data structures:
  *
@@ -173,7 +322,7 @@ static int is_rmap_pte(u64 pte)
  * If page->private bit zero is one, (then page->private & ~1) points
  * to a struct kvm_rmap_desc containing more mappings.
  */
-static void rmap_add(struct kvm *kvm, u64 *spte)
+static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte)
 {
        struct page *page;
        struct kvm_rmap_desc *desc;
@@ -182,26 +331,22 @@ static void rmap_add(struct kvm *kvm, u64 *spte)
        if (!is_rmap_pte(*spte))
                return;
        page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
-       if (!page->private) {
+       if (!page_private(page)) {
                rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
-               page->private = (unsigned long)spte;
-       } else if (!(page->private & 1)) {
+               set_page_private(page,(unsigned long)spte);
+       } else if (!(page_private(page) & 1)) {
                rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
-               desc = kzalloc(sizeof *desc, GFP_NOWAIT);
-               if (!desc)
-                       BUG(); /* FIXME: return error */
-               desc->shadow_ptes[0] = (u64 *)page->private;
+               desc = mmu_alloc_rmap_desc(vcpu);
+               desc->shadow_ptes[0] = (u64 *)page_private(page);
                desc->shadow_ptes[1] = spte;
-               page->private = (unsigned long)desc | 1;
+               set_page_private(page,(unsigned long)desc | 1);
        } else {
                rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
-               desc = (struct kvm_rmap_desc *)(page->private & ~1ul);
+               desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
                while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
                        desc = desc->more;
                if (desc->shadow_ptes[RMAP_EXT-1]) {
-                       desc->more = kzalloc(sizeof *desc->more, GFP_NOWAIT);
-                       if (!desc->more)
-                               BUG(); /* FIXME: return error */
+                       desc->more = mmu_alloc_rmap_desc(vcpu);
                        desc = desc->more;
                }
                for (i = 0; desc->shadow_ptes[i]; ++i)
@@ -210,7 +355,8 @@ static void rmap_add(struct kvm *kvm, u64 *spte)
        }
 }
 
-static void rmap_desc_remove_entry(struct page *page,
+static void rmap_desc_remove_entry(struct kvm_vcpu *vcpu,
+                                  struct page *page,
                                   struct kvm_rmap_desc *desc,
                                   int i,
                                   struct kvm_rmap_desc *prev_desc)
@@ -220,20 +366,20 @@ static void rmap_desc_remove_entry(struct page *page,
        for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
                ;
        desc->shadow_ptes[i] = desc->shadow_ptes[j];
-       desc->shadow_ptes[j] = 0;
+       desc->shadow_ptes[j] = NULL;
        if (j != 0)
                return;
        if (!prev_desc && !desc->more)
-               page->private = (unsigned long)desc->shadow_ptes[0];
+               set_page_private(page,(unsigned long)desc->shadow_ptes[0]);
        else
                if (prev_desc)
                        prev_desc->more = desc->more;
                else
-                       page->private = (unsigned long)desc->more | 1;
-       kfree(desc);
+                       set_page_private(page,(unsigned long)desc->more | 1);
+       mmu_free_rmap_desc(vcpu, desc);
 }
 
-static void rmap_remove(struct kvm *kvm, u64 *spte)
+static void rmap_remove(struct kvm_vcpu *vcpu, u64 *spte)
 {
        struct page *page;
        struct kvm_rmap_desc *desc;
@@ -243,25 +389,26 @@ static void rmap_remove(struct kvm *kvm, u64 *spte)
        if (!is_rmap_pte(*spte))
                return;
        page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
-       if (!page->private) {
+       if (!page_private(page)) {
                printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
                BUG();
-       } else if (!(page->private & 1)) {
+       } else if (!(page_private(page) & 1)) {
                rmap_printk("rmap_remove:  %p %llx 1->0\n", spte, *spte);
-               if ((u64 *)page->private != spte) {
+               if ((u64 *)page_private(page) != spte) {
                        printk(KERN_ERR "rmap_remove:  %p %llx 1->BUG\n",
                               spte, *spte);
                        BUG();
                }
-               page->private = 0;
+               set_page_private(page,0);
        } else {
                rmap_printk("rmap_remove:  %p %llx many->many\n", spte, *spte);
-               desc = (struct kvm_rmap_desc *)(page->private & ~1ul);
+               desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
                prev_desc = NULL;
                while (desc) {
                        for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
                                if (desc->shadow_ptes[i] == spte) {
-                                       rmap_desc_remove_entry(page, desc, i,
+                                       rmap_desc_remove_entry(vcpu, page,
+                                                              desc, i,
                                                               prev_desc);
                                        return;
                                }
@@ -272,41 +419,326 @@ static void rmap_remove(struct kvm *kvm, u64 *spte)
        }
 }
 
-static void kvm_mmu_free_page(struct kvm_vcpu *vcpu, hpa_t page_hpa)
+static void rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn)
 {
-       struct kvm_mmu_page *page_head = page_header(page_hpa);
+       struct kvm *kvm = vcpu->kvm;
+       struct page *page;
+       struct kvm_rmap_desc *desc;
+       u64 *spte;
 
-       list_del(&page_head->link);
-       page_head->page_hpa = page_hpa;
-       list_add(&page_head->link, &vcpu->free_pages);
+       page = gfn_to_page(kvm, gfn);
+       BUG_ON(!page);
+
+       while (page_private(page)) {
+               if (!(page_private(page) & 1))
+                       spte = (u64 *)page_private(page);
+               else {
+                       desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
+                       spte = desc->shadow_ptes[0];
+               }
+               BUG_ON(!spte);
+               BUG_ON((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT
+                      != page_to_pfn(page));
+               BUG_ON(!(*spte & PT_PRESENT_MASK));
+               BUG_ON(!(*spte & PT_WRITABLE_MASK));
+               rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
+               rmap_remove(vcpu, spte);
+               set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK);
+               kvm_flush_remote_tlbs(vcpu->kvm);
+       }
 }
 
-static int is_empty_shadow_page(hpa_t page_hpa)
+#ifdef MMU_DEBUG
+static int is_empty_shadow_page(u64 *spt)
 {
-       u32 *pos;
-       u32 *end;
-       for (pos = __va(page_hpa), end = pos + PAGE_SIZE / sizeof(u32);
-                     pos != end; pos++)
-               if (*pos != 0)
+       u64 *pos;
+       u64 *end;
+
+       for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
+               if (*pos != 0) {
+                       printk(KERN_ERR "%s: %p %llx\n", __FUNCTION__,
+                              pos, *pos);
                        return 0;
+               }
        return 1;
 }
+#endif
+
+static void kvm_mmu_free_page(struct kvm_vcpu *vcpu,
+                             struct kvm_mmu_page *page_head)
+{
+       ASSERT(is_empty_shadow_page(page_head->spt));
+       list_del(&page_head->link);
+       mmu_memory_cache_free(&vcpu->mmu_page_cache, page_head->spt);
+       mmu_memory_cache_free(&vcpu->mmu_page_header_cache, page_head);
+       ++vcpu->kvm->n_free_mmu_pages;
+}
+
+static unsigned kvm_page_table_hashfn(gfn_t gfn)
+{
+       return gfn;
+}
 
-static hpa_t kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, u64 *parent_pte)
+static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
+                                              u64 *parent_pte)
 {
        struct kvm_mmu_page *page;
 
-       if (list_empty(&vcpu->free_pages))
-               return INVALID_PAGE;
+       if (!vcpu->kvm->n_free_mmu_pages)
+               return NULL;
 
-       page = list_entry(vcpu->free_pages.next, struct kvm_mmu_page, link);
-       list_del(&page->link);
+       page = mmu_memory_cache_alloc(&vcpu->mmu_page_header_cache,
+                                     sizeof *page);
+       page->spt = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE);
+       set_page_private(virt_to_page(page->spt), (unsigned long)page);
        list_add(&page->link, &vcpu->kvm->active_mmu_pages);
-       ASSERT(is_empty_shadow_page(page->page_hpa));
+       ASSERT(is_empty_shadow_page(page->spt));
        page->slot_bitmap = 0;
-       page->global = 1;
+       page->multimapped = 0;
        page->parent_pte = parent_pte;
-       return page->page_hpa;
+       --vcpu->kvm->n_free_mmu_pages;
+       return page;
+}
+
+static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
+                                   struct kvm_mmu_page *page, u64 *parent_pte)
+{
+       struct kvm_pte_chain *pte_chain;
+       struct hlist_node *node;
+       int i;
+
+       if (!parent_pte)
+               return;
+       if (!page->multimapped) {
+               u64 *old = page->parent_pte;
+
+               if (!old) {
+                       page->parent_pte = parent_pte;
+                       return;
+               }
+               page->multimapped = 1;
+               pte_chain = mmu_alloc_pte_chain(vcpu);
+               INIT_HLIST_HEAD(&page->parent_ptes);
+               hlist_add_head(&pte_chain->link, &page->parent_ptes);
+               pte_chain->parent_ptes[0] = old;
+       }
+       hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link) {
+               if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
+                       continue;
+               for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
+                       if (!pte_chain->parent_ptes[i]) {
+                               pte_chain->parent_ptes[i] = parent_pte;
+                               return;
+                       }
+       }
+       pte_chain = mmu_alloc_pte_chain(vcpu);
+       BUG_ON(!pte_chain);
+       hlist_add_head(&pte_chain->link, &page->parent_ptes);
+       pte_chain->parent_ptes[0] = parent_pte;
+}
+
+static void mmu_page_remove_parent_pte(struct kvm_vcpu *vcpu,
+                                      struct kvm_mmu_page *page,
+                                      u64 *parent_pte)
+{
+       struct kvm_pte_chain *pte_chain;
+       struct hlist_node *node;
+       int i;
+
+       if (!page->multimapped) {
+               BUG_ON(page->parent_pte != parent_pte);
+               page->parent_pte = NULL;
+               return;
+       }
+       hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link)
+               for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
+                       if (!pte_chain->parent_ptes[i])
+                               break;
+                       if (pte_chain->parent_ptes[i] != parent_pte)
+                               continue;
+                       while (i + 1 < NR_PTE_CHAIN_ENTRIES
+                               && pte_chain->parent_ptes[i + 1]) {
+                               pte_chain->parent_ptes[i]
+                                       = pte_chain->parent_ptes[i + 1];
+                               ++i;
+                       }
+                       pte_chain->parent_ptes[i] = NULL;
+                       if (i == 0) {
+                               hlist_del(&pte_chain->link);
+                               mmu_free_pte_chain(vcpu, pte_chain);
+                               if (hlist_empty(&page->parent_ptes)) {
+                                       page->multimapped = 0;
+                                       page->parent_pte = NULL;
+                               }
+                       }
+                       return;
+               }
+       BUG();
+}
+
+static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm_vcpu *vcpu,
+                                               gfn_t gfn)
+{
+       unsigned index;
+       struct hlist_head *bucket;
+       struct kvm_mmu_page *page;
+       struct hlist_node *node;
+
+       pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
+       index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
+       bucket = &vcpu->kvm->mmu_page_hash[index];
+       hlist_for_each_entry(page, node, bucket, hash_link)
+               if (page->gfn == gfn && !page->role.metaphysical) {
+                       pgprintk("%s: found role %x\n",
+                                __FUNCTION__, page->role.word);
+                       return page;
+               }
+       return NULL;
+}
+
+static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
+                                            gfn_t gfn,
+                                            gva_t gaddr,
+                                            unsigned level,
+                                            int metaphysical,
+                                            unsigned hugepage_access,
+                                            u64 *parent_pte)
+{
+       union kvm_mmu_page_role role;
+       unsigned index;
+       unsigned quadrant;
+       struct hlist_head *bucket;
+       struct kvm_mmu_page *page;
+       struct hlist_node *node;
+
+       role.word = 0;
+       role.glevels = vcpu->mmu.root_level;
+       role.level = level;
+       role.metaphysical = metaphysical;
+       role.hugepage_access = hugepage_access;
+       if (vcpu->mmu.root_level <= PT32_ROOT_LEVEL) {
+               quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
+               quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
+               role.quadrant = quadrant;
+       }
+       pgprintk("%s: looking gfn %lx role %x\n", __FUNCTION__,
+                gfn, role.word);
+       index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
+       bucket = &vcpu->kvm->mmu_page_hash[index];
+       hlist_for_each_entry(page, node, bucket, hash_link)
+               if (page->gfn == gfn && page->role.word == role.word) {
+                       mmu_page_add_parent_pte(vcpu, page, parent_pte);
+                       pgprintk("%s: found\n", __FUNCTION__);
+                       return page;
+               }
+       page = kvm_mmu_alloc_page(vcpu, parent_pte);
+       if (!page)
+               return page;
+       pgprintk("%s: adding gfn %lx role %x\n", __FUNCTION__, gfn, role.word);
+       page->gfn = gfn;
+       page->role = role;
+       hlist_add_head(&page->hash_link, bucket);
+       if (!metaphysical)
+               rmap_write_protect(vcpu, gfn);
+       return page;
+}
+
+static void kvm_mmu_page_unlink_children(struct kvm_vcpu *vcpu,
+                                        struct kvm_mmu_page *page)
+{
+       unsigned i;
+       u64 *pt;
+       u64 ent;
+
+       pt = page->spt;
+
+       if (page->role.level == PT_PAGE_TABLE_LEVEL) {
+               for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
+                       if (pt[i] & PT_PRESENT_MASK)
+                               rmap_remove(vcpu, &pt[i]);
+                       pt[i] = 0;
+               }
+               kvm_flush_remote_tlbs(vcpu->kvm);
+               return;
+       }
+
+       for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
+               ent = pt[i];
+
+               pt[i] = 0;
+               if (!(ent & PT_PRESENT_MASK))
+                       continue;
+               ent &= PT64_BASE_ADDR_MASK;
+               mmu_page_remove_parent_pte(vcpu, page_header(ent), &pt[i]);
+       }
+       kvm_flush_remote_tlbs(vcpu->kvm);
+}
+
+static void kvm_mmu_put_page(struct kvm_vcpu *vcpu,
+                            struct kvm_mmu_page *page,
+                            u64 *parent_pte)
+{
+       mmu_page_remove_parent_pte(vcpu, page, parent_pte);
+}
+
+static void kvm_mmu_zap_page(struct kvm_vcpu *vcpu,
+                            struct kvm_mmu_page *page)
+{
+       u64 *parent_pte;
+
+       while (page->multimapped || page->parent_pte) {
+               if (!page->multimapped)
+                       parent_pte = page->parent_pte;
+               else {
+                       struct kvm_pte_chain *chain;
+
+                       chain = container_of(page->parent_ptes.first,
+                                            struct kvm_pte_chain, link);
+                       parent_pte = chain->parent_ptes[0];
+               }
+               BUG_ON(!parent_pte);
+               kvm_mmu_put_page(vcpu, page, parent_pte);
+               set_shadow_pte(parent_pte, 0);
+       }
+       kvm_mmu_page_unlink_children(vcpu, page);
+       if (!page->root_count) {
+               hlist_del(&page->hash_link);
+               kvm_mmu_free_page(vcpu, page);
+       } else
+               list_move(&page->link, &vcpu->kvm->active_mmu_pages);
+}
+
+static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn)
+{
+       unsigned index;
+       struct hlist_head *bucket;
+       struct kvm_mmu_page *page;
+       struct hlist_node *node, *n;
+       int r;
+
+       pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
+       r = 0;
+       index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
+       bucket = &vcpu->kvm->mmu_page_hash[index];
+       hlist_for_each_entry_safe(page, node, n, bucket, hash_link)
+               if (page->gfn == gfn && !page->role.metaphysical) {
+                       pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn,
+                                page->role.word);
+                       kvm_mmu_zap_page(vcpu, page);
+                       r = 1;
+               }
+       return r;
+}
+
+static void mmu_unshadow(struct kvm_vcpu *vcpu, gfn_t gfn)
+{
+       struct kvm_mmu_page *page;
+
+       while ((page = kvm_mmu_lookup_page(vcpu, gfn)) != NULL) {
+               pgprintk("%s: zap %lx %x\n",
+                        __FUNCTION__, gfn, page->role.word);
+               kvm_mmu_zap_page(vcpu, page);
+       }
 }
 
 static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa)
@@ -326,14 +758,12 @@ hpa_t safe_gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
 
 hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
 {
-       struct kvm_memory_slot *slot;
        struct page *page;
 
        ASSERT((gpa & HPA_ERR_MASK) == 0);
-       slot = gfn_to_memslot(vcpu->kvm, gpa >> PAGE_SHIFT);
-       if (!slot)
+       page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
+       if (!page)
                return gpa | HPA_ERR_MASK;
-       page = gfn_to_page(slot, gpa >> PAGE_SHIFT);
        return ((hpa_t)page_to_pfn(page) << PAGE_SHIFT)
                | (gpa & (PAGE_SIZE-1));
 }
@@ -347,33 +777,13 @@ hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva)
        return gpa_to_hpa(vcpu, gpa);
 }
 
-
-static void release_pt_page_64(struct kvm_vcpu *vcpu, hpa_t page_hpa,
-                              int level)
+struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva)
 {
-       u64 *pos;
-       u64 *end;
+       gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
 
-       ASSERT(vcpu);
-       ASSERT(VALID_PAGE(page_hpa));
-       ASSERT(level <= PT64_ROOT_LEVEL && level > 0);
-
-       for (pos = __va(page_hpa), end = pos + PT64_ENT_PER_PAGE;
-            pos != end; pos++) {
-               u64 current_ent = *pos;
-
-               if (is_present_pte(current_ent)) {
-                       if (level != 1)
-                               release_pt_page_64(vcpu,
-                                                 current_ent &
-                                                 PT64_BASE_ADDR_MASK,
-                                                 level - 1);
-                       else
-                               rmap_remove(vcpu->kvm, pos);
-               }
-               *pos = 0;
-       }
-       kvm_mmu_free_page(vcpu, page_hpa);
+       if (gpa == UNMAPPED_GVA)
+               return NULL;
+       return pfn_to_page(gpa_to_hpa(vcpu, gpa) >> PAGE_SHIFT);
 }
 
 static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
@@ -388,33 +798,39 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p)
        for (; ; level--) {
                u32 index = PT64_INDEX(v, level);
                u64 *table;
+               u64 pte;
 
                ASSERT(VALID_PAGE(table_addr));
                table = __va(table_addr);
 
                if (level == 1) {
+                       pte = table[index];
+                       if (is_present_pte(pte) && is_writeble_pte(pte))
+                               return 0;
                        mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT);
                        page_header_update_slot(vcpu->kvm, table, v);
                        table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK |
                                                                PT_USER_MASK;
-                       rmap_add(vcpu->kvm, &table[index]);
+                       rmap_add(vcpu, &table[index]);
                        return 0;
                }
 
                if (table[index] == 0) {
-                       hpa_t new_table = kvm_mmu_alloc_page(vcpu,
-                                                            &table[index]);
-
-                       if (!VALID_PAGE(new_table)) {
+                       struct kvm_mmu_page *new_table;
+                       gfn_t pseudo_gfn;
+
+                       pseudo_gfn = (v & PT64_DIR_BASE_ADDR_MASK)
+                               >> PAGE_SHIFT;
+                       new_table = kvm_mmu_get_page(vcpu, pseudo_gfn,
+                                                    v, level - 1,
+                                                    1, 0, &table[index]);
+                       if (!new_table) {
                                pgprintk("nonpaging_map: ENOMEM\n");
                                return -ENOMEM;
                        }
 
-                       if (level == PT32E_ROOT_LEVEL)
-                               table[index] = new_table | PT_PRESENT_MASK;
-                       else
-                               table[index] = new_table | PT_PRESENT_MASK |
-                                               PT_WRITABLE_MASK | PT_USER_MASK;
+                       table[index] = __pa(new_table->spt) | PT_PRESENT_MASK
+                               | PT_WRITABLE_MASK | PT_USER_MASK;
                }
                table_addr = table[index] & PT64_BASE_ADDR_MASK;
        }
@@ -423,13 +839,16 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p)
 static void mmu_free_roots(struct kvm_vcpu *vcpu)
 {
        int i;
+       struct kvm_mmu_page *page;
 
+       if (!VALID_PAGE(vcpu->mmu.root_hpa))
+               return;
 #ifdef CONFIG_X86_64
        if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
                hpa_t root = vcpu->mmu.root_hpa;
 
-               ASSERT(VALID_PAGE(root));
-               release_pt_page_64(vcpu, root, PT64_ROOT_LEVEL);
+               page = page_header(root);
+               --page->root_count;
                vcpu->mmu.root_hpa = INVALID_PAGE;
                return;
        }
@@ -437,9 +856,11 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu)
        for (i = 0; i < 4; ++i) {
                hpa_t root = vcpu->mmu.pae_root[i];
 
-               ASSERT(VALID_PAGE(root));
-               root &= PT64_BASE_ADDR_MASK;
-               release_pt_page_64(vcpu, root, PT32E_ROOT_LEVEL - 1);
+               if (root) {
+                       root &= PT64_BASE_ADDR_MASK;
+                       page = page_header(root);
+                       --page->root_count;
+               }
                vcpu->mmu.pae_root[i] = INVALID_PAGE;
        }
        vcpu->mmu.root_hpa = INVALID_PAGE;
@@ -448,13 +869,20 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu)
 static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
 {
        int i;
+       gfn_t root_gfn;
+       struct kvm_mmu_page *page;
+
+       root_gfn = vcpu->cr3 >> PAGE_SHIFT;
 
 #ifdef CONFIG_X86_64
        if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
                hpa_t root = vcpu->mmu.root_hpa;
 
                ASSERT(!VALID_PAGE(root));
-               root = kvm_mmu_alloc_page(vcpu, NULL);
+               page = kvm_mmu_get_page(vcpu, root_gfn, 0,
+                                       PT64_ROOT_LEVEL, 0, 0, NULL);
+               root = __pa(page->spt);
+               ++page->root_count;
                vcpu->mmu.root_hpa = root;
                return;
        }
@@ -463,24 +891,24 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
                hpa_t root = vcpu->mmu.pae_root[i];
 
                ASSERT(!VALID_PAGE(root));
-               root = kvm_mmu_alloc_page(vcpu, NULL);
+               if (vcpu->mmu.root_level == PT32E_ROOT_LEVEL) {
+                       if (!is_present_pte(vcpu->pdptrs[i])) {
+                               vcpu->mmu.pae_root[i] = 0;
+                               continue;
+                       }
+                       root_gfn = vcpu->pdptrs[i] >> PAGE_SHIFT;
+               } else if (vcpu->mmu.root_level == 0)
+                       root_gfn = 0;
+               page = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
+                                       PT32_ROOT_LEVEL, !is_paging(vcpu),
+                                       0, NULL);
+               root = __pa(page->spt);
+               ++page->root_count;
                vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK;
        }
        vcpu->mmu.root_hpa = __pa(vcpu->mmu.pae_root);
 }
 
-static void nonpaging_flush(struct kvm_vcpu *vcpu)
-{
-       hpa_t root = vcpu->mmu.root_hpa;
-
-       ++kvm_stat.tlb_flush;
-       pgprintk("nonpaging_flush\n");
-       mmu_free_roots(vcpu);
-       mmu_alloc_roots(vcpu);
-       kvm_arch_ops->set_cr3(vcpu, root);
-       kvm_arch_ops->tlb_flush(vcpu);
-}
-
 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
 {
        return vaddr;
@@ -489,32 +917,24 @@ static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
                               u32 error_code)
 {
-       int ret;
        gpa_t addr = gva;
+       hpa_t paddr;
+       int r;
+
+       r = mmu_topup_memory_caches(vcpu);
+       if (r)
+               return r;
 
        ASSERT(vcpu);
        ASSERT(VALID_PAGE(vcpu->mmu.root_hpa));
 
-       for (;;) {
-            hpa_t paddr;
-
-            paddr = gpa_to_hpa(vcpu , addr & PT64_BASE_ADDR_MASK);
 
-            if (is_error_hpa(paddr))
-                    return 1;
+       paddr = gpa_to_hpa(vcpu , addr & PT64_BASE_ADDR_MASK);
 
-            ret = nonpaging_map(vcpu, addr & PAGE_MASK, paddr);
-            if (ret) {
-                    nonpaging_flush(vcpu);
-                    continue;
-            }
-            break;
-       }
-       return ret;
-}
+       if (is_error_hpa(paddr))
+               return 1;
 
-static void nonpaging_inval_page(struct kvm_vcpu *vcpu, gva_t addr)
-{
+       return nonpaging_map(vcpu, addr & PAGE_MASK, paddr);
 }
 
 static void nonpaging_free(struct kvm_vcpu *vcpu)
@@ -528,78 +948,24 @@ static int nonpaging_init_context(struct kvm_vcpu *vcpu)
 
        context->new_cr3 = nonpaging_new_cr3;
        context->page_fault = nonpaging_page_fault;
-       context->inval_page = nonpaging_inval_page;
        context->gva_to_gpa = nonpaging_gva_to_gpa;
        context->free = nonpaging_free;
-       context->root_level = PT32E_ROOT_LEVEL;
+       context->root_level = 0;
        context->shadow_root_level = PT32E_ROOT_LEVEL;
-       mmu_alloc_roots(vcpu);
-       ASSERT(VALID_PAGE(context->root_hpa));
-       kvm_arch_ops->set_cr3(vcpu, context->root_hpa);
+       context->root_hpa = INVALID_PAGE;
        return 0;
 }
 
-
 static void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
 {
-       struct kvm_mmu_page *page, *npage;
-
-       list_for_each_entry_safe(page, npage, &vcpu->kvm->active_mmu_pages,
-                                link) {
-               if (page->global)
-                       continue;
-
-               if (!page->parent_pte)
-                       continue;
-
-               *page->parent_pte = 0;
-               release_pt_page_64(vcpu, page->page_hpa, 1);
-       }
-       ++kvm_stat.tlb_flush;
+       ++vcpu->stat.tlb_flush;
        kvm_arch_ops->tlb_flush(vcpu);
 }
 
 static void paging_new_cr3(struct kvm_vcpu *vcpu)
 {
-       kvm_mmu_flush_tlb(vcpu);
-}
-
-static void mark_pagetable_nonglobal(void *shadow_pte)
-{
-       page_header(__pa(shadow_pte))->global = 0;
-}
-
-static inline void set_pte_common(struct kvm_vcpu *vcpu,
-                            u64 *shadow_pte,
-                            gpa_t gaddr,
-                            int dirty,
-                            u64 access_bits)
-{
-       hpa_t paddr;
-
-       *shadow_pte |= access_bits << PT_SHADOW_BITS_OFFSET;
-       if (!dirty)
-               access_bits &= ~PT_WRITABLE_MASK;
-
-       if (access_bits & PT_WRITABLE_MASK)
-               mark_page_dirty(vcpu->kvm, gaddr >> PAGE_SHIFT);
-
-       *shadow_pte |= access_bits;
-
-       paddr = gpa_to_hpa(vcpu, gaddr & PT64_BASE_ADDR_MASK);
-
-       if (!(*shadow_pte & PT_GLOBAL_MASK))
-               mark_pagetable_nonglobal(shadow_pte);
-
-       if (is_error_hpa(paddr)) {
-               *shadow_pte |= gaddr;
-               *shadow_pte |= PT_SHADOW_IO_MARK;
-               *shadow_pte &= ~PT_PRESENT_MASK;
-       } else {
-               *shadow_pte |= paddr;
-               page_header_update_slot(vcpu->kvm, shadow_pte, gaddr);
-               rmap_add(vcpu->kvm, shadow_pte);
-       }
+       pgprintk("%s: cr3 %lx\n", __FUNCTION__, vcpu->cr3);
+       mmu_free_roots(vcpu);
 }
 
 static void inject_page_fault(struct kvm_vcpu *vcpu,
@@ -609,69 +975,6 @@ static void inject_page_fault(struct kvm_vcpu *vcpu,
        kvm_arch_ops->inject_page_fault(vcpu, addr, err_code);
 }
 
-static inline int fix_read_pf(u64 *shadow_ent)
-{
-       if ((*shadow_ent & PT_SHADOW_USER_MASK) &&
-           !(*shadow_ent & PT_USER_MASK)) {
-               /*
-                * If supervisor write protect is disabled, we shadow kernel
-                * pages as user pages so we can trap the write access.
-                */
-               *shadow_ent |= PT_USER_MASK;
-               *shadow_ent &= ~PT_WRITABLE_MASK;
-
-               return 1;
-
-       }
-       return 0;
-}
-
-static int may_access(u64 pte, int write, int user)
-{
-
-       if (user && !(pte & PT_USER_MASK))
-               return 0;
-       if (write && !(pte & PT_WRITABLE_MASK))
-               return 0;
-       return 1;
-}
-
-/*
- * Remove a shadow pte.
- */
-static void paging_inval_page(struct kvm_vcpu *vcpu, gva_t addr)
-{
-       hpa_t page_addr = vcpu->mmu.root_hpa;
-       int level = vcpu->mmu.shadow_root_level;
-
-       ++kvm_stat.invlpg;
-
-       for (; ; level--) {
-               u32 index = PT64_INDEX(addr, level);
-               u64 *table = __va(page_addr);
-
-               if (level == PT_PAGE_TABLE_LEVEL ) {
-                       rmap_remove(vcpu->kvm, &table[index]);
-                       table[index] = 0;
-                       return;
-               }
-
-               if (!is_present_pte(table[index]))
-                       return;
-
-               page_addr = table[index] & PT64_BASE_ADDR_MASK;
-
-               if (level == PT_DIRECTORY_LEVEL &&
-                         (table[index] & PT_SHADOW_PS_MARK)) {
-                       table[index] = 0;
-                       release_pt_page_64(vcpu, page_addr, PT_PAGE_TABLE_LEVEL);
-
-                       kvm_arch_ops->tlb_flush(vcpu);
-                       return;
-               }
-       }
-}
-
 static void paging_free(struct kvm_vcpu *vcpu)
 {
        nonpaging_free(vcpu);
@@ -692,15 +995,11 @@ static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
        ASSERT(is_pae(vcpu));
        context->new_cr3 = paging_new_cr3;
        context->page_fault = paging64_page_fault;
-       context->inval_page = paging_inval_page;
        context->gva_to_gpa = paging64_gva_to_gpa;
        context->free = paging_free;
        context->root_level = level;
        context->shadow_root_level = level;
-       mmu_alloc_roots(vcpu);
-       ASSERT(VALID_PAGE(context->root_hpa));
-       kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
-                   (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
+       context->root_hpa = INVALID_PAGE;
        return 0;
 }
 
@@ -715,15 +1014,11 @@ static int paging32_init_context(struct kvm_vcpu *vcpu)
 
        context->new_cr3 = paging_new_cr3;
        context->page_fault = paging32_page_fault;
-       context->inval_page = paging_inval_page;
        context->gva_to_gpa = paging32_gva_to_gpa;
        context->free = paging_free;
        context->root_level = PT32_ROOT_LEVEL;
        context->shadow_root_level = PT32E_ROOT_LEVEL;
-       mmu_alloc_roots(vcpu);
-       ASSERT(VALID_PAGE(context->root_hpa));
-       kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
-                   (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
+       context->root_hpa = INVALID_PAGE;
        return 0;
 }
 
@@ -762,16 +1057,169 @@ int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
        return init_kvm_mmu(vcpu);
 }
 
-static void free_mmu_pages(struct kvm_vcpu *vcpu)
+int kvm_mmu_load(struct kvm_vcpu *vcpu)
+{
+       int r;
+
+       spin_lock(&vcpu->kvm->lock);
+       r = mmu_topup_memory_caches(vcpu);
+       if (r)
+               goto out;
+       mmu_alloc_roots(vcpu);
+       kvm_arch_ops->set_cr3(vcpu, vcpu->mmu.root_hpa);
+       kvm_mmu_flush_tlb(vcpu);
+out:
+       spin_unlock(&vcpu->kvm->lock);
+       return r;
+}
+EXPORT_SYMBOL_GPL(kvm_mmu_load);
+
+void kvm_mmu_unload(struct kvm_vcpu *vcpu)
+{
+       mmu_free_roots(vcpu);
+}
+
+static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
+                                 struct kvm_mmu_page *page,
+                                 u64 *spte)
+{
+       u64 pte;
+       struct kvm_mmu_page *child;
+
+       pte = *spte;
+       if (is_present_pte(pte)) {
+               if (page->role.level == PT_PAGE_TABLE_LEVEL)
+                       rmap_remove(vcpu, spte);
+               else {
+                       child = page_header(pte & PT64_BASE_ADDR_MASK);
+                       mmu_page_remove_parent_pte(vcpu, child, spte);
+               }
+       }
+       *spte = 0;
+       kvm_flush_remote_tlbs(vcpu->kvm);
+}
+
+static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
+                                 struct kvm_mmu_page *page,
+                                 u64 *spte,
+                                 const void *new, int bytes)
+{
+       if (page->role.level != PT_PAGE_TABLE_LEVEL)
+               return;
+
+       if (page->role.glevels == PT32_ROOT_LEVEL)
+               paging32_update_pte(vcpu, page, spte, new, bytes);
+       else
+               paging64_update_pte(vcpu, page, spte, new, bytes);
+}
+
+void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
+                      const u8 *old, const u8 *new, int bytes)
+{
+       gfn_t gfn = gpa >> PAGE_SHIFT;
+       struct kvm_mmu_page *page;
+       struct hlist_node *node, *n;
+       struct hlist_head *bucket;
+       unsigned index;
+       u64 *spte;
+       unsigned offset = offset_in_page(gpa);
+       unsigned pte_size;
+       unsigned page_offset;
+       unsigned misaligned;
+       unsigned quadrant;
+       int level;
+       int flooded = 0;
+       int npte;
+
+       pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes);
+       if (gfn == vcpu->last_pt_write_gfn) {
+               ++vcpu->last_pt_write_count;
+               if (vcpu->last_pt_write_count >= 3)
+                       flooded = 1;
+       } else {
+               vcpu->last_pt_write_gfn = gfn;
+               vcpu->last_pt_write_count = 1;
+       }
+       index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
+       bucket = &vcpu->kvm->mmu_page_hash[index];
+       hlist_for_each_entry_safe(page, node, n, bucket, hash_link) {
+               if (page->gfn != gfn || page->role.metaphysical)
+                       continue;
+               pte_size = page->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
+               misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
+               misaligned |= bytes < 4;
+               if (misaligned || flooded) {
+                       /*
+                        * Misaligned accesses are too much trouble to fix
+                        * up; also, they usually indicate a page is not used
+                        * as a page table.
+                        *
+                        * If we're seeing too many writes to a page,
+                        * it may no longer be a page table, or we may be
+                        * forking, in which case it is better to unmap the
+                        * page.
+                        */
+                       pgprintk("misaligned: gpa %llx bytes %d role %x\n",
+                                gpa, bytes, page->role.word);
+                       kvm_mmu_zap_page(vcpu, page);
+                       continue;
+               }
+               page_offset = offset;
+               level = page->role.level;
+               npte = 1;
+               if (page->role.glevels == PT32_ROOT_LEVEL) {
+                       page_offset <<= 1;      /* 32->64 */
+                       /*
+                        * A 32-bit pde maps 4MB while the shadow pdes map
+                        * only 2MB.  So we need to double the offset again
+                        * and zap two pdes instead of one.
+                        */
+                       if (level == PT32_ROOT_LEVEL) {
+                               page_offset &= ~7; /* kill rounding error */
+                               page_offset <<= 1;
+                               npte = 2;
+                       }
+                       quadrant = page_offset >> PAGE_SHIFT;
+                       page_offset &= ~PAGE_MASK;
+                       if (quadrant != page->role.quadrant)
+                               continue;
+               }
+               spte = &page->spt[page_offset / sizeof(*spte)];
+               while (npte--) {
+                       mmu_pte_write_zap_pte(vcpu, page, spte);
+                       mmu_pte_write_new_pte(vcpu, page, spte, new, bytes);
+                       ++spte;
+               }
+       }
+}
+
+int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
 {
-       while (!list_empty(&vcpu->free_pages)) {
+       gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
+
+       return kvm_mmu_unprotect_page(vcpu, gpa >> PAGE_SHIFT);
+}
+
+void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
+{
+       while (vcpu->kvm->n_free_mmu_pages < KVM_REFILL_PAGES) {
                struct kvm_mmu_page *page;
 
-               page = list_entry(vcpu->free_pages.next,
-                                 struct kvm_mmu_page, link);
-               list_del(&page->link);
-               __free_page(pfn_to_page(page->page_hpa >> PAGE_SHIFT));
-               page->page_hpa = INVALID_PAGE;
+               page = container_of(vcpu->kvm->active_mmu_pages.prev,
+                                   struct kvm_mmu_page, link);
+               kvm_mmu_zap_page(vcpu, page);
+       }
+}
+EXPORT_SYMBOL_GPL(kvm_mmu_free_some_pages);
+
+static void free_mmu_pages(struct kvm_vcpu *vcpu)
+{
+       struct kvm_mmu_page *page;
+
+       while (!list_empty(&vcpu->kvm->active_mmu_pages)) {
+               page = container_of(vcpu->kvm->active_mmu_pages.next,
+                                   struct kvm_mmu_page, link);
+               kvm_mmu_zap_page(vcpu, page);
        }
        free_page((unsigned long)vcpu->mmu.pae_root);
 }
@@ -783,17 +1231,7 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
 
        ASSERT(vcpu);
 
-       for (i = 0; i < KVM_NUM_MMU_PAGES; i++) {
-               struct kvm_mmu_page *page_header = &vcpu->page_header_buf[i];
-
-               INIT_LIST_HEAD(&page_header->link);
-               if ((page = alloc_page(GFP_KERNEL)) == NULL)
-                       goto error_1;
-               page->private = (unsigned long)page_header;
-               page_header->page_hpa = (hpa_t)page_to_pfn(page) << PAGE_SHIFT;
-               memset(__va(page_header->page_hpa), 0, PAGE_SIZE);
-               list_add(&page_header->link, &vcpu->free_pages);
-       }
+       vcpu->kvm->n_free_mmu_pages = KVM_NUM_MMU_PAGES;
 
        /*
         * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
@@ -818,7 +1256,6 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu)
 {
        ASSERT(vcpu);
        ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
-       ASSERT(list_empty(&vcpu->free_pages));
 
        return alloc_mmu_pages(vcpu);
 }
@@ -827,7 +1264,6 @@ int kvm_mmu_setup(struct kvm_vcpu *vcpu)
 {
        ASSERT(vcpu);
        ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
-       ASSERT(!list_empty(&vcpu->free_pages));
 
        return init_kvm_mmu(vcpu);
 }
@@ -838,10 +1274,12 @@ void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
 
        destroy_kvm_mmu(vcpu);
        free_mmu_pages(vcpu);
+       mmu_free_memory_caches(vcpu);
 }
 
-void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
+void kvm_mmu_slot_remove_write_access(struct kvm_vcpu *vcpu, int slot)
 {
+       struct kvm *kvm = vcpu->kvm;
        struct kvm_mmu_page *page;
 
        list_for_each_entry(page, &kvm->active_mmu_pages, link) {
@@ -851,12 +1289,233 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
                if (!test_bit(slot, &page->slot_bitmap))
                        continue;
 
-               pt = __va(page->page_hpa);
+               pt = page->spt;
                for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
                        /* avoid RMW */
                        if (pt[i] & PT_WRITABLE_MASK) {
-                               rmap_remove(kvm, &pt[i]);
+                               rmap_remove(vcpu, &pt[i]);
                                pt[i] &= ~PT_WRITABLE_MASK;
                        }
        }
 }
+
+void kvm_mmu_zap_all(struct kvm_vcpu *vcpu)
+{
+       destroy_kvm_mmu(vcpu);
+
+       while (!list_empty(&vcpu->kvm->active_mmu_pages)) {
+               struct kvm_mmu_page *page;
+
+               page = container_of(vcpu->kvm->active_mmu_pages.next,
+                                   struct kvm_mmu_page, link);
+               kvm_mmu_zap_page(vcpu, page);
+       }
+
+       mmu_free_memory_caches(vcpu);
+       kvm_flush_remote_tlbs(vcpu->kvm);
+       init_kvm_mmu(vcpu);
+}
+
+void kvm_mmu_module_exit(void)
+{
+       if (pte_chain_cache)
+               kmem_cache_destroy(pte_chain_cache);
+       if (rmap_desc_cache)
+               kmem_cache_destroy(rmap_desc_cache);
+       if (mmu_page_cache)
+               kmem_cache_destroy(mmu_page_cache);
+       if (mmu_page_header_cache)
+               kmem_cache_destroy(mmu_page_header_cache);
+}
+
+int kvm_mmu_module_init(void)
+{
+       pte_chain_cache = kmem_cache_create("kvm_pte_chain",
+                                           sizeof(struct kvm_pte_chain),
+                                           0, 0, NULL);
+       if (!pte_chain_cache)
+               goto nomem;
+       rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
+                                           sizeof(struct kvm_rmap_desc),
+                                           0, 0, NULL);
+       if (!rmap_desc_cache)
+               goto nomem;
+
+       mmu_page_cache = kmem_cache_create("kvm_mmu_page",
+                                          PAGE_SIZE,
+                                          PAGE_SIZE, 0, NULL);
+       if (!mmu_page_cache)
+               goto nomem;
+
+       mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
+                                                 sizeof(struct kvm_mmu_page),
+                                                 0, 0, NULL);
+       if (!mmu_page_header_cache)
+               goto nomem;
+
+       return 0;
+
+nomem:
+       kvm_mmu_module_exit();
+       return -ENOMEM;
+}
+
+#ifdef AUDIT
+
+static const char *audit_msg;
+
+static gva_t canonicalize(gva_t gva)
+{
+#ifdef CONFIG_X86_64
+       gva = (long long)(gva << 16) >> 16;
+#endif
+       return gva;
+}
+
+static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
+                               gva_t va, int level)
+{
+       u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK);
+       int i;
+       gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1));
+
+       for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
+               u64 ent = pt[i];
+
+               if (!(ent & PT_PRESENT_MASK))
+                       continue;
+
+               va = canonicalize(va);
+               if (level > 1)
+                       audit_mappings_page(vcpu, ent, va, level - 1);
+               else {
+                       gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, va);
+                       hpa_t hpa = gpa_to_hpa(vcpu, gpa);
+
+                       if ((ent & PT_PRESENT_MASK)
+                           && (ent & PT64_BASE_ADDR_MASK) != hpa)
+                               printk(KERN_ERR "audit error: (%s) levels %d"
+                                      " gva %lx gpa %llx hpa %llx ent %llx\n",
+                                      audit_msg, vcpu->mmu.root_level,
+                                      va, gpa, hpa, ent);
+               }
+       }
+}
+
+static void audit_mappings(struct kvm_vcpu *vcpu)
+{
+       unsigned i;
+
+       if (vcpu->mmu.root_level == 4)
+               audit_mappings_page(vcpu, vcpu->mmu.root_hpa, 0, 4);
+       else
+               for (i = 0; i < 4; ++i)
+                       if (vcpu->mmu.pae_root[i] & PT_PRESENT_MASK)
+                               audit_mappings_page(vcpu,
+                                                   vcpu->mmu.pae_root[i],
+                                                   i << 30,
+                                                   2);
+}
+
+static int count_rmaps(struct kvm_vcpu *vcpu)
+{
+       int nmaps = 0;
+       int i, j, k;
+
+       for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
+               struct kvm_memory_slot *m = &vcpu->kvm->memslots[i];
+               struct kvm_rmap_desc *d;
+
+               for (j = 0; j < m->npages; ++j) {
+                       struct page *page = m->phys_mem[j];
+
+                       if (!page->private)
+                               continue;
+                       if (!(page->private & 1)) {
+                               ++nmaps;
+                               continue;
+                       }
+                       d = (struct kvm_rmap_desc *)(page->private & ~1ul);
+                       while (d) {
+                               for (k = 0; k < RMAP_EXT; ++k)
+                                       if (d->shadow_ptes[k])
+                                               ++nmaps;
+                                       else
+                                               break;
+                               d = d->more;
+                       }
+               }
+       }
+       return nmaps;
+}
+
+static int count_writable_mappings(struct kvm_vcpu *vcpu)
+{
+       int nmaps = 0;
+       struct kvm_mmu_page *page;
+       int i;
+
+       list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
+               u64 *pt = page->spt;
+
+               if (page->role.level != PT_PAGE_TABLE_LEVEL)
+                       continue;
+
+               for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
+                       u64 ent = pt[i];
+
+                       if (!(ent & PT_PRESENT_MASK))
+                               continue;
+                       if (!(ent & PT_WRITABLE_MASK))
+                               continue;
+                       ++nmaps;
+               }
+       }
+       return nmaps;
+}
+
+static void audit_rmap(struct kvm_vcpu *vcpu)
+{
+       int n_rmap = count_rmaps(vcpu);
+       int n_actual = count_writable_mappings(vcpu);
+
+       if (n_rmap != n_actual)
+               printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
+                      __FUNCTION__, audit_msg, n_rmap, n_actual);
+}
+
+static void audit_write_protection(struct kvm_vcpu *vcpu)
+{
+       struct kvm_mmu_page *page;
+
+       list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
+               hfn_t hfn;
+               struct page *pg;
+
+               if (page->role.metaphysical)
+                       continue;
+
+               hfn = gpa_to_hpa(vcpu, (gpa_t)page->gfn << PAGE_SHIFT)
+                       >> PAGE_SHIFT;
+               pg = pfn_to_page(hfn);
+               if (pg->private)
+                       printk(KERN_ERR "%s: (%s) shadow page has writable"
+                              " mappings: gfn %lx role %x\n",
+                              __FUNCTION__, audit_msg, page->gfn,
+                              page->role.word);
+       }
+}
+
+static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
+{
+       int olddbg = dbg;
+
+       dbg = 0;
+       audit_msg = msg;
+       audit_rmap(vcpu);
+       audit_write_protection(vcpu);
+       audit_mappings(vcpu);
+       dbg = olddbg;
+}
+
+#endif