mm: Remove slab destructors from kmem_cache_create().
[linux-2.6.git] / drivers / kvm / mmu.c
index 8bdb9ca..1199d3f 100644 (file)
  * the COPYING file in the top-level directory.
  *
  */
+
+#include "vmx.h"
+#include "kvm.h"
+
 #include <linux/types.h>
 #include <linux/string.h>
-#include <asm/page.h>
 #include <linux/mm.h>
 #include <linux/highmem.h>
 #include <linux/module.h>
 
-#include "vmx.h"
-#include "kvm.h"
+#include <asm/page.h>
+#include <asm/cmpxchg.h>
 
 #undef MMU_DEBUG
 
@@ -52,11 +55,15 @@ static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
 static int dbg = 1;
 #endif
 
+#ifndef MMU_DEBUG
+#define ASSERT(x) do { } while (0)
+#else
 #define ASSERT(x)                                                      \
        if (!(x)) {                                                     \
                printk(KERN_WARNING "assertion failed %s:%d: %s\n",     \
                       __FILE__, __LINE__, #x);                         \
        }
+#endif
 
 #define PT64_PT_BITS 9
 #define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS)
@@ -86,25 +93,11 @@ static int dbg = 1;
 #define PT32_DIR_PSE36_MASK (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)
 
 
-#define PT32_PTE_COPY_MASK \
-       (PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_DIRTY_MASK | PT_GLOBAL_MASK)
-
-#define PT64_PTE_COPY_MASK (PT64_NX_MASK | PT32_PTE_COPY_MASK)
-
 #define PT_FIRST_AVAIL_BITS_SHIFT 9
 #define PT64_SECOND_AVAIL_BITS_SHIFT 52
 
-#define PT_SHADOW_PS_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
 #define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
 
-#define PT_SHADOW_WRITABLE_SHIFT (PT_FIRST_AVAIL_BITS_SHIFT + 1)
-#define PT_SHADOW_WRITABLE_MASK (1ULL << PT_SHADOW_WRITABLE_SHIFT)
-
-#define PT_SHADOW_USER_SHIFT (PT_SHADOW_WRITABLE_SHIFT + 1)
-#define PT_SHADOW_USER_MASK (1ULL << (PT_SHADOW_USER_SHIFT))
-
-#define PT_SHADOW_BITS_OFFSET (PT_SHADOW_WRITABLE_SHIFT - PT_WRITABLE_SHIFT)
-
 #define VALID_PAGE(x) ((x) != INVALID_PAGE)
 
 #define PT64_LEVEL_BITS 9
@@ -159,6 +152,11 @@ struct kvm_rmap_desc {
        struct kvm_rmap_desc *more;
 };
 
+static struct kmem_cache *pte_chain_cache;
+static struct kmem_cache *rmap_desc_cache;
+static struct kmem_cache *mmu_page_cache;
+static struct kmem_cache *mmu_page_header_cache;
+
 static int is_write_protection(struct kvm_vcpu *vcpu)
 {
        return vcpu->cr0 & CR0_WP_MASK;
@@ -195,15 +193,25 @@ static int is_rmap_pte(u64 pte)
                == (PT_WRITABLE_MASK | PT_PRESENT_MASK);
 }
 
+static void set_shadow_pte(u64 *sptep, u64 spte)
+{
+#ifdef CONFIG_X86_64
+       set_64bit((unsigned long *)sptep, spte);
+#else
+       set_64bit((unsigned long long *)sptep, spte);
+#endif
+}
+
 static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
-                                 size_t objsize, int min)
+                                 struct kmem_cache *base_cache, int min,
+                                 gfp_t gfp_flags)
 {
        void *obj;
 
        if (cache->nobjs >= min)
                return 0;
        while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
-               obj = kzalloc(objsize, GFP_NOWAIT);
+               obj = kmem_cache_zalloc(base_cache, gfp_flags);
                if (!obj)
                        return -ENOMEM;
                cache->objects[cache->nobjs++] = obj;
@@ -217,24 +225,49 @@ static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
                kfree(mc->objects[--mc->nobjs]);
 }
 
-static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
+static int __mmu_topup_memory_caches(struct kvm_vcpu *vcpu, gfp_t gfp_flags)
 {
        int r;
 
        r = mmu_topup_memory_cache(&vcpu->mmu_pte_chain_cache,
-                                  sizeof(struct kvm_pte_chain), 4);
+                                  pte_chain_cache, 4, gfp_flags);
        if (r)
                goto out;
        r = mmu_topup_memory_cache(&vcpu->mmu_rmap_desc_cache,
-                                  sizeof(struct kvm_rmap_desc), 1);
+                                  rmap_desc_cache, 1, gfp_flags);
+       if (r)
+               goto out;
+       r = mmu_topup_memory_cache(&vcpu->mmu_page_cache,
+                                  mmu_page_cache, 4, gfp_flags);
+       if (r)
+               goto out;
+       r = mmu_topup_memory_cache(&vcpu->mmu_page_header_cache,
+                                  mmu_page_header_cache, 4, gfp_flags);
 out:
        return r;
 }
 
+static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
+{
+       int r;
+
+       r = __mmu_topup_memory_caches(vcpu, GFP_NOWAIT);
+       if (r < 0) {
+               spin_unlock(&vcpu->kvm->lock);
+               kvm_arch_ops->vcpu_put(vcpu);
+               r = __mmu_topup_memory_caches(vcpu, GFP_KERNEL);
+               kvm_arch_ops->vcpu_load(vcpu);
+               spin_lock(&vcpu->kvm->lock);
+       }
+       return r;
+}
+
 static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
 {
        mmu_free_memory_cache(&vcpu->mmu_pte_chain_cache);
        mmu_free_memory_cache(&vcpu->mmu_rmap_desc_cache);
+       mmu_free_memory_cache(&vcpu->mmu_page_cache);
+       mmu_free_memory_cache(&vcpu->mmu_page_header_cache);
 }
 
 static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
@@ -410,18 +443,18 @@ static void rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn)
                BUG_ON(!(*spte & PT_WRITABLE_MASK));
                rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
                rmap_remove(vcpu, spte);
-               kvm_arch_ops->tlb_flush(vcpu);
-               *spte &= ~(u64)PT_WRITABLE_MASK;
+               set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK);
+               kvm_flush_remote_tlbs(vcpu->kvm);
        }
 }
 
-static int is_empty_shadow_page(hpa_t page_hpa)
+#ifdef MMU_DEBUG
+static int is_empty_shadow_page(u64 *spt)
 {
        u64 *pos;
        u64 *end;
 
-       for (pos = __va(page_hpa), end = pos + PAGE_SIZE / sizeof(u64);
-                     pos != end; pos++)
+       for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
                if (*pos != 0) {
                        printk(KERN_ERR "%s: %p %llx\n", __FUNCTION__,
                               pos, *pos);
@@ -429,14 +462,15 @@ static int is_empty_shadow_page(hpa_t page_hpa)
                }
        return 1;
 }
+#endif
 
-static void kvm_mmu_free_page(struct kvm_vcpu *vcpu, hpa_t page_hpa)
+static void kvm_mmu_free_page(struct kvm_vcpu *vcpu,
+                             struct kvm_mmu_page *page_head)
 {
-       struct kvm_mmu_page *page_head = page_header(page_hpa);
-
-       ASSERT(is_empty_shadow_page(page_hpa));
-       page_head->page_hpa = page_hpa;
-       list_move(&page_head->link, &vcpu->free_pages);
+       ASSERT(is_empty_shadow_page(page_head->spt));
+       list_del(&page_head->link);
+       mmu_memory_cache_free(&vcpu->mmu_page_cache, page_head->spt);
+       mmu_memory_cache_free(&vcpu->mmu_page_header_cache, page_head);
        ++vcpu->kvm->n_free_mmu_pages;
 }
 
@@ -450,12 +484,15 @@ static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
 {
        struct kvm_mmu_page *page;
 
-       if (list_empty(&vcpu->free_pages))
+       if (!vcpu->kvm->n_free_mmu_pages)
                return NULL;
 
-       page = list_entry(vcpu->free_pages.next, struct kvm_mmu_page, link);
-       list_move(&page->link, &vcpu->kvm->active_mmu_pages);
-       ASSERT(is_empty_shadow_page(page->page_hpa));
+       page = mmu_memory_cache_alloc(&vcpu->mmu_page_header_cache,
+                                     sizeof *page);
+       page->spt = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE);
+       set_page_private(virt_to_page(page->spt), (unsigned long)page);
+       list_add(&page->link, &vcpu->kvm->active_mmu_pages);
+       ASSERT(is_empty_shadow_page(page->spt));
        page->slot_bitmap = 0;
        page->multimapped = 0;
        page->parent_pte = parent_pte;
@@ -613,7 +650,7 @@ static void kvm_mmu_page_unlink_children(struct kvm_vcpu *vcpu,
        u64 *pt;
        u64 ent;
 
-       pt = __va(page->page_hpa);
+       pt = page->spt;
 
        if (page->role.level == PT_PAGE_TABLE_LEVEL) {
                for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
@@ -621,7 +658,7 @@ static void kvm_mmu_page_unlink_children(struct kvm_vcpu *vcpu,
                                rmap_remove(vcpu, &pt[i]);
                        pt[i] = 0;
                }
-               kvm_arch_ops->tlb_flush(vcpu);
+               kvm_flush_remote_tlbs(vcpu->kvm);
                return;
        }
 
@@ -634,6 +671,7 @@ static void kvm_mmu_page_unlink_children(struct kvm_vcpu *vcpu,
                ent &= PT64_BASE_ADDR_MASK;
                mmu_page_remove_parent_pte(vcpu, page_header(ent), &pt[i]);
        }
+       kvm_flush_remote_tlbs(vcpu->kvm);
 }
 
 static void kvm_mmu_put_page(struct kvm_vcpu *vcpu,
@@ -660,12 +698,12 @@ static void kvm_mmu_zap_page(struct kvm_vcpu *vcpu,
                }
                BUG_ON(!parent_pte);
                kvm_mmu_put_page(vcpu, page, parent_pte);
-               *parent_pte = 0;
+               set_shadow_pte(parent_pte, 0);
        }
        kvm_mmu_page_unlink_children(vcpu, page);
        if (!page->root_count) {
                hlist_del(&page->hash_link);
-               kvm_mmu_free_page(vcpu, page->page_hpa);
+               kvm_mmu_free_page(vcpu, page);
        } else
                list_move(&page->link, &vcpu->kvm->active_mmu_pages);
 }
@@ -692,6 +730,17 @@ static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn)
        return r;
 }
 
+static void mmu_unshadow(struct kvm_vcpu *vcpu, gfn_t gfn)
+{
+       struct kvm_mmu_page *page;
+
+       while ((page = kvm_mmu_lookup_page(vcpu, gfn)) != NULL) {
+               pgprintk("%s: zap %lx %x\n",
+                        __FUNCTION__, gfn, page->role.word);
+               kvm_mmu_zap_page(vcpu, page);
+       }
+}
+
 static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa)
 {
        int slot = memslot_id(kvm, gfn_to_memslot(kvm, gpa >> PAGE_SHIFT));
@@ -780,7 +829,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p)
                                return -ENOMEM;
                        }
 
-                       table[index] = new_table->page_hpa | PT_PRESENT_MASK
+                       table[index] = __pa(new_table->spt) | PT_PRESENT_MASK
                                | PT_WRITABLE_MASK | PT_USER_MASK;
                }
                table_addr = table[index] & PT64_BASE_ADDR_MASK;
@@ -792,11 +841,12 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu)
        int i;
        struct kvm_mmu_page *page;
 
+       if (!VALID_PAGE(vcpu->mmu.root_hpa))
+               return;
 #ifdef CONFIG_X86_64
        if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
                hpa_t root = vcpu->mmu.root_hpa;
 
-               ASSERT(VALID_PAGE(root));
                page = page_header(root);
                --page->root_count;
                vcpu->mmu.root_hpa = INVALID_PAGE;
@@ -806,10 +856,11 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu)
        for (i = 0; i < 4; ++i) {
                hpa_t root = vcpu->mmu.pae_root[i];
 
-               ASSERT(VALID_PAGE(root));
-               root &= PT64_BASE_ADDR_MASK;
-               page = page_header(root);
-               --page->root_count;
+               if (root) {
+                       root &= PT64_BASE_ADDR_MASK;
+                       page = page_header(root);
+                       --page->root_count;
+               }
                vcpu->mmu.pae_root[i] = INVALID_PAGE;
        }
        vcpu->mmu.root_hpa = INVALID_PAGE;
@@ -830,7 +881,7 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
                ASSERT(!VALID_PAGE(root));
                page = kvm_mmu_get_page(vcpu, root_gfn, 0,
                                        PT64_ROOT_LEVEL, 0, 0, NULL);
-               root = page->page_hpa;
+               root = __pa(page->spt);
                ++page->root_count;
                vcpu->mmu.root_hpa = root;
                return;
@@ -840,14 +891,18 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
                hpa_t root = vcpu->mmu.pae_root[i];
 
                ASSERT(!VALID_PAGE(root));
-               if (vcpu->mmu.root_level == PT32E_ROOT_LEVEL)
+               if (vcpu->mmu.root_level == PT32E_ROOT_LEVEL) {
+                       if (!is_present_pte(vcpu->pdptrs[i])) {
+                               vcpu->mmu.pae_root[i] = 0;
+                               continue;
+                       }
                        root_gfn = vcpu->pdptrs[i] >> PAGE_SHIFT;
-               else if (vcpu->mmu.root_level == 0)
+               } else if (vcpu->mmu.root_level == 0)
                        root_gfn = 0;
                page = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
                                        PT32_ROOT_LEVEL, !is_paging(vcpu),
                                        0, NULL);
-               root = page->page_hpa;
+               root = __pa(page->spt);
                ++page->root_count;
                vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK;
        }
@@ -897,15 +952,13 @@ static int nonpaging_init_context(struct kvm_vcpu *vcpu)
        context->free = nonpaging_free;
        context->root_level = 0;
        context->shadow_root_level = PT32E_ROOT_LEVEL;
-       mmu_alloc_roots(vcpu);
-       ASSERT(VALID_PAGE(context->root_hpa));
-       kvm_arch_ops->set_cr3(vcpu, context->root_hpa);
+       context->root_hpa = INVALID_PAGE;
        return 0;
 }
 
 static void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
 {
-       ++kvm_stat.tlb_flush;
+       ++vcpu->stat.tlb_flush;
        kvm_arch_ops->tlb_flush(vcpu);
 }
 
@@ -913,59 +966,6 @@ static void paging_new_cr3(struct kvm_vcpu *vcpu)
 {
        pgprintk("%s: cr3 %lx\n", __FUNCTION__, vcpu->cr3);
        mmu_free_roots(vcpu);
-       if (unlikely(vcpu->kvm->n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES))
-               kvm_mmu_free_some_pages(vcpu);
-       mmu_alloc_roots(vcpu);
-       kvm_mmu_flush_tlb(vcpu);
-       kvm_arch_ops->set_cr3(vcpu, vcpu->mmu.root_hpa);
-}
-
-static inline void set_pte_common(struct kvm_vcpu *vcpu,
-                            u64 *shadow_pte,
-                            gpa_t gaddr,
-                            int dirty,
-                            u64 access_bits,
-                            gfn_t gfn)
-{
-       hpa_t paddr;
-
-       *shadow_pte |= access_bits << PT_SHADOW_BITS_OFFSET;
-       if (!dirty)
-               access_bits &= ~PT_WRITABLE_MASK;
-
-       paddr = gpa_to_hpa(vcpu, gaddr & PT64_BASE_ADDR_MASK);
-
-       *shadow_pte |= access_bits;
-
-       if (is_error_hpa(paddr)) {
-               *shadow_pte |= gaddr;
-               *shadow_pte |= PT_SHADOW_IO_MARK;
-               *shadow_pte &= ~PT_PRESENT_MASK;
-               return;
-       }
-
-       *shadow_pte |= paddr;
-
-       if (access_bits & PT_WRITABLE_MASK) {
-               struct kvm_mmu_page *shadow;
-
-               shadow = kvm_mmu_lookup_page(vcpu, gfn);
-               if (shadow) {
-                       pgprintk("%s: found shadow page for %lx, marking ro\n",
-                                __FUNCTION__, gfn);
-                       access_bits &= ~PT_WRITABLE_MASK;
-                       if (is_writeble_pte(*shadow_pte)) {
-                                   *shadow_pte &= ~PT_WRITABLE_MASK;
-                                   kvm_arch_ops->tlb_flush(vcpu);
-                       }
-               }
-       }
-
-       if (access_bits & PT_WRITABLE_MASK)
-               mark_page_dirty(vcpu->kvm, gaddr >> PAGE_SHIFT);
-
-       page_header_update_slot(vcpu->kvm, shadow_pte, gaddr);
-       rmap_add(vcpu, shadow_pte);
 }
 
 static void inject_page_fault(struct kvm_vcpu *vcpu,
@@ -975,23 +975,6 @@ static void inject_page_fault(struct kvm_vcpu *vcpu,
        kvm_arch_ops->inject_page_fault(vcpu, addr, err_code);
 }
 
-static inline int fix_read_pf(u64 *shadow_ent)
-{
-       if ((*shadow_ent & PT_SHADOW_USER_MASK) &&
-           !(*shadow_ent & PT_USER_MASK)) {
-               /*
-                * If supervisor write protect is disabled, we shadow kernel
-                * pages as user pages so we can trap the write access.
-                */
-               *shadow_ent |= PT_USER_MASK;
-               *shadow_ent &= ~PT_WRITABLE_MASK;
-
-               return 1;
-
-       }
-       return 0;
-}
-
 static void paging_free(struct kvm_vcpu *vcpu)
 {
        nonpaging_free(vcpu);
@@ -1016,10 +999,7 @@ static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
        context->free = paging_free;
        context->root_level = level;
        context->shadow_root_level = level;
-       mmu_alloc_roots(vcpu);
-       ASSERT(VALID_PAGE(context->root_hpa));
-       kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
-                   (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
+       context->root_hpa = INVALID_PAGE;
        return 0;
 }
 
@@ -1038,10 +1018,7 @@ static int paging32_init_context(struct kvm_vcpu *vcpu)
        context->free = paging_free;
        context->root_level = PT32_ROOT_LEVEL;
        context->shadow_root_level = PT32E_ROOT_LEVEL;
-       mmu_alloc_roots(vcpu);
-       ASSERT(VALID_PAGE(context->root_hpa));
-       kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
-                   (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
+       context->root_hpa = INVALID_PAGE;
        return 0;
 }
 
@@ -1076,18 +1053,33 @@ static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
 
 int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
 {
+       destroy_kvm_mmu(vcpu);
+       return init_kvm_mmu(vcpu);
+}
+
+int kvm_mmu_load(struct kvm_vcpu *vcpu)
+{
        int r;
 
-       destroy_kvm_mmu(vcpu);
-       r = init_kvm_mmu(vcpu);
-       if (r < 0)
-               goto out;
+       spin_lock(&vcpu->kvm->lock);
        r = mmu_topup_memory_caches(vcpu);
+       if (r)
+               goto out;
+       mmu_alloc_roots(vcpu);
+       kvm_arch_ops->set_cr3(vcpu, vcpu->mmu.root_hpa);
+       kvm_mmu_flush_tlb(vcpu);
 out:
+       spin_unlock(&vcpu->kvm->lock);
        return r;
 }
+EXPORT_SYMBOL_GPL(kvm_mmu_load);
 
-static void mmu_pre_write_zap_pte(struct kvm_vcpu *vcpu,
+void kvm_mmu_unload(struct kvm_vcpu *vcpu)
+{
+       mmu_free_roots(vcpu);
+}
+
+static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
                                  struct kvm_mmu_page *page,
                                  u64 *spte)
 {
@@ -1104,9 +1096,25 @@ static void mmu_pre_write_zap_pte(struct kvm_vcpu *vcpu,
                }
        }
        *spte = 0;
+       kvm_flush_remote_tlbs(vcpu->kvm);
+}
+
+static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
+                                 struct kvm_mmu_page *page,
+                                 u64 *spte,
+                                 const void *new, int bytes)
+{
+       if (page->role.level != PT_PAGE_TABLE_LEVEL)
+               return;
+
+       if (page->role.glevels == PT32_ROOT_LEVEL)
+               paging32_update_pte(vcpu, page, spte, new, bytes);
+       else
+               paging64_update_pte(vcpu, page, spte, new, bytes);
 }
 
-void kvm_mmu_pre_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes)
+void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
+                      const u8 *old, const u8 *new, int bytes)
 {
        gfn_t gfn = gpa >> PAGE_SHIFT;
        struct kvm_mmu_page *page;
@@ -1118,6 +1126,7 @@ void kvm_mmu_pre_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes)
        unsigned pte_size;
        unsigned page_offset;
        unsigned misaligned;
+       unsigned quadrant;
        int level;
        int flooded = 0;
        int npte;
@@ -1138,6 +1147,7 @@ void kvm_mmu_pre_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes)
                        continue;
                pte_size = page->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
                misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
+               misaligned |= bytes < 4;
                if (misaligned || flooded) {
                        /*
                         * Misaligned accesses are too much trouble to fix
@@ -1169,21 +1179,20 @@ void kvm_mmu_pre_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes)
                                page_offset <<= 1;
                                npte = 2;
                        }
+                       quadrant = page_offset >> PAGE_SHIFT;
                        page_offset &= ~PAGE_MASK;
+                       if (quadrant != page->role.quadrant)
+                               continue;
                }
-               spte = __va(page->page_hpa);
-               spte += page_offset / sizeof(*spte);
+               spte = &page->spt[page_offset / sizeof(*spte)];
                while (npte--) {
-                       mmu_pre_write_zap_pte(vcpu, page, spte);
+                       mmu_pte_write_zap_pte(vcpu, page, spte);
+                       mmu_pte_write_new_pte(vcpu, page, spte, new, bytes);
                        ++spte;
                }
        }
 }
 
-void kvm_mmu_post_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes)
-{
-}
-
 int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
 {
        gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
@@ -1212,13 +1221,6 @@ static void free_mmu_pages(struct kvm_vcpu *vcpu)
                                    struct kvm_mmu_page, link);
                kvm_mmu_zap_page(vcpu, page);
        }
-       while (!list_empty(&vcpu->free_pages)) {
-               page = list_entry(vcpu->free_pages.next,
-                                 struct kvm_mmu_page, link);
-               list_del(&page->link);
-               __free_page(pfn_to_page(page->page_hpa >> PAGE_SHIFT));
-               page->page_hpa = INVALID_PAGE;
-       }
        free_page((unsigned long)vcpu->mmu.pae_root);
 }
 
@@ -1229,18 +1231,7 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
 
        ASSERT(vcpu);
 
-       for (i = 0; i < KVM_NUM_MMU_PAGES; i++) {
-               struct kvm_mmu_page *page_header = &vcpu->page_header_buf[i];
-
-               INIT_LIST_HEAD(&page_header->link);
-               if ((page = alloc_page(GFP_KERNEL)) == NULL)
-                       goto error_1;
-               set_page_private(page, (unsigned long)page_header);
-               page_header->page_hpa = (hpa_t)page_to_pfn(page) << PAGE_SHIFT;
-               memset(__va(page_header->page_hpa), 0, PAGE_SIZE);
-               list_add(&page_header->link, &vcpu->free_pages);
-               ++vcpu->kvm->n_free_mmu_pages;
-       }
+       vcpu->kvm->n_free_mmu_pages = KVM_NUM_MMU_PAGES;
 
        /*
         * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
@@ -1265,7 +1256,6 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu)
 {
        ASSERT(vcpu);
        ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
-       ASSERT(list_empty(&vcpu->free_pages));
 
        return alloc_mmu_pages(vcpu);
 }
@@ -1274,7 +1264,6 @@ int kvm_mmu_setup(struct kvm_vcpu *vcpu)
 {
        ASSERT(vcpu);
        ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
-       ASSERT(!list_empty(&vcpu->free_pages));
 
        return init_kvm_mmu(vcpu);
 }
@@ -1300,7 +1289,7 @@ void kvm_mmu_slot_remove_write_access(struct kvm_vcpu *vcpu, int slot)
                if (!test_bit(slot, &page->slot_bitmap))
                        continue;
 
-               pt = __va(page->page_hpa);
+               pt = page->spt;
                for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
                        /* avoid RMW */
                        if (pt[i] & PT_WRITABLE_MASK) {
@@ -1323,10 +1312,54 @@ void kvm_mmu_zap_all(struct kvm_vcpu *vcpu)
        }
 
        mmu_free_memory_caches(vcpu);
-       kvm_arch_ops->tlb_flush(vcpu);
+       kvm_flush_remote_tlbs(vcpu->kvm);
        init_kvm_mmu(vcpu);
 }
 
+void kvm_mmu_module_exit(void)
+{
+       if (pte_chain_cache)
+               kmem_cache_destroy(pte_chain_cache);
+       if (rmap_desc_cache)
+               kmem_cache_destroy(rmap_desc_cache);
+       if (mmu_page_cache)
+               kmem_cache_destroy(mmu_page_cache);
+       if (mmu_page_header_cache)
+               kmem_cache_destroy(mmu_page_header_cache);
+}
+
+int kvm_mmu_module_init(void)
+{
+       pte_chain_cache = kmem_cache_create("kvm_pte_chain",
+                                           sizeof(struct kvm_pte_chain),
+                                           0, 0, NULL);
+       if (!pte_chain_cache)
+               goto nomem;
+       rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
+                                           sizeof(struct kvm_rmap_desc),
+                                           0, 0, NULL);
+       if (!rmap_desc_cache)
+               goto nomem;
+
+       mmu_page_cache = kmem_cache_create("kvm_mmu_page",
+                                          PAGE_SIZE,
+                                          PAGE_SIZE, 0, NULL);
+       if (!mmu_page_cache)
+               goto nomem;
+
+       mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
+                                                 sizeof(struct kvm_mmu_page),
+                                                 0, 0, NULL);
+       if (!mmu_page_header_cache)
+               goto nomem;
+
+       return 0;
+
+nomem:
+       kvm_mmu_module_exit();
+       return -ENOMEM;
+}
+
 #ifdef AUDIT
 
 static const char *audit_msg;
@@ -1349,7 +1382,7 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
        for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
                u64 ent = pt[i];
 
-               if (!ent & PT_PRESENT_MASK)
+               if (!(ent & PT_PRESENT_MASK))
                        continue;
 
                va = canonicalize(va);
@@ -1423,7 +1456,7 @@ static int count_writable_mappings(struct kvm_vcpu *vcpu)
        int i;
 
        list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
-               u64 *pt = __va(page->page_hpa);
+               u64 *pt = page->spt;
 
                if (page->role.level != PT_PAGE_TABLE_LEVEL)
                        continue;