[PATCH] KVM: Simplify is_long_mode()
[linux-2.6.git] / drivers / kvm / mmu.c
1 /*
2  * Kernel-based Virtual Machine driver for Linux
3  *
4  * This module enables machines with Intel VT-x extensions to run virtual
5  * machines without emulation or binary translation.
6  *
7  * MMU support
8  *
9  * Copyright (C) 2006 Qumranet, Inc.
10  *
11  * Authors:
12  *   Yaniv Kamay  <yaniv@qumranet.com>
13  *   Avi Kivity   <avi@qumranet.com>
14  *
15  * This work is licensed under the terms of the GNU GPL, version 2.  See
16  * the COPYING file in the top-level directory.
17  *
18  */
19 #include <linux/types.h>
20 #include <linux/string.h>
21 #include <asm/page.h>
22 #include <linux/mm.h>
23 #include <linux/highmem.h>
24 #include <linux/module.h>
25
26 #include "vmx.h"
27 #include "kvm.h"
28
29 #define pgprintk(x...) do { } while (0)
30
31 #define ASSERT(x)                                                       \
32         if (!(x)) {                                                     \
33                 printk(KERN_WARNING "assertion failed %s:%d: %s\n",     \
34                        __FILE__, __LINE__, #x);                         \
35         }
36
37 #define PT64_ENT_PER_PAGE 512
38 #define PT32_ENT_PER_PAGE 1024
39
40 #define PT_WRITABLE_SHIFT 1
41
42 #define PT_PRESENT_MASK (1ULL << 0)
43 #define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
44 #define PT_USER_MASK (1ULL << 2)
45 #define PT_PWT_MASK (1ULL << 3)
46 #define PT_PCD_MASK (1ULL << 4)
47 #define PT_ACCESSED_MASK (1ULL << 5)
48 #define PT_DIRTY_MASK (1ULL << 6)
49 #define PT_PAGE_SIZE_MASK (1ULL << 7)
50 #define PT_PAT_MASK (1ULL << 7)
51 #define PT_GLOBAL_MASK (1ULL << 8)
52 #define PT64_NX_MASK (1ULL << 63)
53
54 #define PT_PAT_SHIFT 7
55 #define PT_DIR_PAT_SHIFT 12
56 #define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT)
57
58 #define PT32_DIR_PSE36_SIZE 4
59 #define PT32_DIR_PSE36_SHIFT 13
60 #define PT32_DIR_PSE36_MASK (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)
61
62
63 #define PT32_PTE_COPY_MASK \
64         (PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_DIRTY_MASK | PT_GLOBAL_MASK)
65
66 #define PT64_PTE_COPY_MASK (PT64_NX_MASK | PT32_PTE_COPY_MASK)
67
68 #define PT_FIRST_AVAIL_BITS_SHIFT 9
69 #define PT64_SECOND_AVAIL_BITS_SHIFT 52
70
71 #define PT_SHADOW_PS_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
72 #define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
73
74 #define PT_SHADOW_WRITABLE_SHIFT (PT_FIRST_AVAIL_BITS_SHIFT + 1)
75 #define PT_SHADOW_WRITABLE_MASK (1ULL << PT_SHADOW_WRITABLE_SHIFT)
76
77 #define PT_SHADOW_USER_SHIFT (PT_SHADOW_WRITABLE_SHIFT + 1)
78 #define PT_SHADOW_USER_MASK (1ULL << (PT_SHADOW_USER_SHIFT))
79
80 #define PT_SHADOW_BITS_OFFSET (PT_SHADOW_WRITABLE_SHIFT - PT_WRITABLE_SHIFT)
81
82 #define VALID_PAGE(x) ((x) != INVALID_PAGE)
83
84 #define PT64_LEVEL_BITS 9
85
86 #define PT64_LEVEL_SHIFT(level) \
87                 ( PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS )
88
89 #define PT64_LEVEL_MASK(level) \
90                 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
91
92 #define PT64_INDEX(address, level)\
93         (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
94
95
96 #define PT32_LEVEL_BITS 10
97
98 #define PT32_LEVEL_SHIFT(level) \
99                 ( PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS )
100
101 #define PT32_LEVEL_MASK(level) \
102                 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
103
104 #define PT32_INDEX(address, level)\
105         (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
106
107
108 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & PAGE_MASK)
109 #define PT64_DIR_BASE_ADDR_MASK \
110         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
111
112 #define PT32_BASE_ADDR_MASK PAGE_MASK
113 #define PT32_DIR_BASE_ADDR_MASK \
114         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
115
116
117 #define PFERR_PRESENT_MASK (1U << 0)
118 #define PFERR_WRITE_MASK (1U << 1)
119 #define PFERR_USER_MASK (1U << 2)
120
121 #define PT64_ROOT_LEVEL 4
122 #define PT32_ROOT_LEVEL 2
123 #define PT32E_ROOT_LEVEL 3
124
125 #define PT_DIRECTORY_LEVEL 2
126 #define PT_PAGE_TABLE_LEVEL 1
127
128 static int is_write_protection(struct kvm_vcpu *vcpu)
129 {
130         return vcpu->cr0 & CR0_WP_MASK;
131 }
132
133 static int is_cpuid_PSE36(void)
134 {
135         return 1;
136 }
137
138 static int is_present_pte(unsigned long pte)
139 {
140         return pte & PT_PRESENT_MASK;
141 }
142
143 static int is_writeble_pte(unsigned long pte)
144 {
145         return pte & PT_WRITABLE_MASK;
146 }
147
148 static int is_io_pte(unsigned long pte)
149 {
150         return pte & PT_SHADOW_IO_MARK;
151 }
152
153 static void kvm_mmu_free_page(struct kvm_vcpu *vcpu, hpa_t page_hpa)
154 {
155         struct kvm_mmu_page *page_head = page_header(page_hpa);
156
157         list_del(&page_head->link);
158         page_head->page_hpa = page_hpa;
159         list_add(&page_head->link, &vcpu->free_pages);
160 }
161
162 static int is_empty_shadow_page(hpa_t page_hpa)
163 {
164         u32 *pos;
165         u32 *end;
166         for (pos = __va(page_hpa), end = pos + PAGE_SIZE / sizeof(u32);
167                       pos != end; pos++)
168                 if (*pos != 0)
169                         return 0;
170         return 1;
171 }
172
173 static hpa_t kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, u64 *parent_pte)
174 {
175         struct kvm_mmu_page *page;
176
177         if (list_empty(&vcpu->free_pages))
178                 return INVALID_PAGE;
179
180         page = list_entry(vcpu->free_pages.next, struct kvm_mmu_page, link);
181         list_del(&page->link);
182         list_add(&page->link, &vcpu->kvm->active_mmu_pages);
183         ASSERT(is_empty_shadow_page(page->page_hpa));
184         page->slot_bitmap = 0;
185         page->global = 1;
186         page->parent_pte = parent_pte;
187         return page->page_hpa;
188 }
189
190 static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa)
191 {
192         int slot = memslot_id(kvm, gfn_to_memslot(kvm, gpa >> PAGE_SHIFT));
193         struct kvm_mmu_page *page_head = page_header(__pa(pte));
194
195         __set_bit(slot, &page_head->slot_bitmap);
196 }
197
198 hpa_t safe_gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
199 {
200         hpa_t hpa = gpa_to_hpa(vcpu, gpa);
201
202         return is_error_hpa(hpa) ? bad_page_address | (gpa & ~PAGE_MASK): hpa;
203 }
204
205 hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
206 {
207         struct kvm_memory_slot *slot;
208         struct page *page;
209
210         ASSERT((gpa & HPA_ERR_MASK) == 0);
211         slot = gfn_to_memslot(vcpu->kvm, gpa >> PAGE_SHIFT);
212         if (!slot)
213                 return gpa | HPA_ERR_MASK;
214         page = gfn_to_page(slot, gpa >> PAGE_SHIFT);
215         return ((hpa_t)page_to_pfn(page) << PAGE_SHIFT)
216                 | (gpa & (PAGE_SIZE-1));
217 }
218
219 hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva)
220 {
221         gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
222
223         if (gpa == UNMAPPED_GVA)
224                 return UNMAPPED_GVA;
225         return gpa_to_hpa(vcpu, gpa);
226 }
227
228
229 static void release_pt_page_64(struct kvm_vcpu *vcpu, hpa_t page_hpa,
230                                int level)
231 {
232         ASSERT(vcpu);
233         ASSERT(VALID_PAGE(page_hpa));
234         ASSERT(level <= PT64_ROOT_LEVEL && level > 0);
235
236         if (level == 1)
237                 memset(__va(page_hpa), 0, PAGE_SIZE);
238         else {
239                 u64 *pos;
240                 u64 *end;
241
242                 for (pos = __va(page_hpa), end = pos + PT64_ENT_PER_PAGE;
243                      pos != end; pos++) {
244                         u64 current_ent = *pos;
245
246                         *pos = 0;
247                         if (is_present_pte(current_ent))
248                                 release_pt_page_64(vcpu,
249                                                   current_ent &
250                                                   PT64_BASE_ADDR_MASK,
251                                                   level - 1);
252                 }
253         }
254         kvm_mmu_free_page(vcpu, page_hpa);
255 }
256
257 static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
258 {
259 }
260
261 static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p)
262 {
263         int level = PT32E_ROOT_LEVEL;
264         hpa_t table_addr = vcpu->mmu.root_hpa;
265
266         for (; ; level--) {
267                 u32 index = PT64_INDEX(v, level);
268                 u64 *table;
269
270                 ASSERT(VALID_PAGE(table_addr));
271                 table = __va(table_addr);
272
273                 if (level == 1) {
274                         mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT);
275                         page_header_update_slot(vcpu->kvm, table, v);
276                         table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK |
277                                                                 PT_USER_MASK;
278                         return 0;
279                 }
280
281                 if (table[index] == 0) {
282                         hpa_t new_table = kvm_mmu_alloc_page(vcpu,
283                                                              &table[index]);
284
285                         if (!VALID_PAGE(new_table)) {
286                                 pgprintk("nonpaging_map: ENOMEM\n");
287                                 return -ENOMEM;
288                         }
289
290                         if (level == PT32E_ROOT_LEVEL)
291                                 table[index] = new_table | PT_PRESENT_MASK;
292                         else
293                                 table[index] = new_table | PT_PRESENT_MASK |
294                                                 PT_WRITABLE_MASK | PT_USER_MASK;
295                 }
296                 table_addr = table[index] & PT64_BASE_ADDR_MASK;
297         }
298 }
299
300 static void nonpaging_flush(struct kvm_vcpu *vcpu)
301 {
302         hpa_t root = vcpu->mmu.root_hpa;
303
304         ++kvm_stat.tlb_flush;
305         pgprintk("nonpaging_flush\n");
306         ASSERT(VALID_PAGE(root));
307         release_pt_page_64(vcpu, root, vcpu->mmu.shadow_root_level);
308         root = kvm_mmu_alloc_page(vcpu, NULL);
309         ASSERT(VALID_PAGE(root));
310         vcpu->mmu.root_hpa = root;
311         if (is_paging(vcpu))
312                 root |= (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK));
313         kvm_arch_ops->set_cr3(vcpu, root);
314         kvm_arch_ops->tlb_flush(vcpu);
315 }
316
317 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
318 {
319         return vaddr;
320 }
321
322 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
323                                u32 error_code)
324 {
325         int ret;
326         gpa_t addr = gva;
327
328         ASSERT(vcpu);
329         ASSERT(VALID_PAGE(vcpu->mmu.root_hpa));
330
331         for (;;) {
332              hpa_t paddr;
333
334              paddr = gpa_to_hpa(vcpu , addr & PT64_BASE_ADDR_MASK);
335
336              if (is_error_hpa(paddr))
337                      return 1;
338
339              ret = nonpaging_map(vcpu, addr & PAGE_MASK, paddr);
340              if (ret) {
341                      nonpaging_flush(vcpu);
342                      continue;
343              }
344              break;
345         }
346         return ret;
347 }
348
349 static void nonpaging_inval_page(struct kvm_vcpu *vcpu, gva_t addr)
350 {
351 }
352
353 static void nonpaging_free(struct kvm_vcpu *vcpu)
354 {
355         hpa_t root;
356
357         ASSERT(vcpu);
358         root = vcpu->mmu.root_hpa;
359         if (VALID_PAGE(root))
360                 release_pt_page_64(vcpu, root, vcpu->mmu.shadow_root_level);
361         vcpu->mmu.root_hpa = INVALID_PAGE;
362 }
363
364 static int nonpaging_init_context(struct kvm_vcpu *vcpu)
365 {
366         struct kvm_mmu *context = &vcpu->mmu;
367
368         context->new_cr3 = nonpaging_new_cr3;
369         context->page_fault = nonpaging_page_fault;
370         context->inval_page = nonpaging_inval_page;
371         context->gva_to_gpa = nonpaging_gva_to_gpa;
372         context->free = nonpaging_free;
373         context->root_level = PT32E_ROOT_LEVEL;
374         context->shadow_root_level = PT32E_ROOT_LEVEL;
375         context->root_hpa = kvm_mmu_alloc_page(vcpu, NULL);
376         ASSERT(VALID_PAGE(context->root_hpa));
377         kvm_arch_ops->set_cr3(vcpu, context->root_hpa);
378         return 0;
379 }
380
381
382 static void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
383 {
384         struct kvm_mmu_page *page, *npage;
385
386         list_for_each_entry_safe(page, npage, &vcpu->kvm->active_mmu_pages,
387                                  link) {
388                 if (page->global)
389                         continue;
390
391                 if (!page->parent_pte)
392                         continue;
393
394                 *page->parent_pte = 0;
395                 release_pt_page_64(vcpu, page->page_hpa, 1);
396         }
397         ++kvm_stat.tlb_flush;
398         kvm_arch_ops->tlb_flush(vcpu);
399 }
400
401 static void paging_new_cr3(struct kvm_vcpu *vcpu)
402 {
403         kvm_mmu_flush_tlb(vcpu);
404 }
405
406 static void mark_pagetable_nonglobal(void *shadow_pte)
407 {
408         page_header(__pa(shadow_pte))->global = 0;
409 }
410
411 static inline void set_pte_common(struct kvm_vcpu *vcpu,
412                              u64 *shadow_pte,
413                              gpa_t gaddr,
414                              int dirty,
415                              u64 access_bits)
416 {
417         hpa_t paddr;
418
419         *shadow_pte |= access_bits << PT_SHADOW_BITS_OFFSET;
420         if (!dirty)
421                 access_bits &= ~PT_WRITABLE_MASK;
422
423         if (access_bits & PT_WRITABLE_MASK)
424                 mark_page_dirty(vcpu->kvm, gaddr >> PAGE_SHIFT);
425
426         *shadow_pte |= access_bits;
427
428         paddr = gpa_to_hpa(vcpu, gaddr & PT64_BASE_ADDR_MASK);
429
430         if (!(*shadow_pte & PT_GLOBAL_MASK))
431                 mark_pagetable_nonglobal(shadow_pte);
432
433         if (is_error_hpa(paddr)) {
434                 *shadow_pte |= gaddr;
435                 *shadow_pte |= PT_SHADOW_IO_MARK;
436                 *shadow_pte &= ~PT_PRESENT_MASK;
437         } else {
438                 *shadow_pte |= paddr;
439                 page_header_update_slot(vcpu->kvm, shadow_pte, gaddr);
440         }
441 }
442
443 static void inject_page_fault(struct kvm_vcpu *vcpu,
444                               u64 addr,
445                               u32 err_code)
446 {
447         kvm_arch_ops->inject_page_fault(vcpu, addr, err_code);
448 }
449
450 static inline int fix_read_pf(u64 *shadow_ent)
451 {
452         if ((*shadow_ent & PT_SHADOW_USER_MASK) &&
453             !(*shadow_ent & PT_USER_MASK)) {
454                 /*
455                  * If supervisor write protect is disabled, we shadow kernel
456                  * pages as user pages so we can trap the write access.
457                  */
458                 *shadow_ent |= PT_USER_MASK;
459                 *shadow_ent &= ~PT_WRITABLE_MASK;
460
461                 return 1;
462
463         }
464         return 0;
465 }
466
467 static int may_access(u64 pte, int write, int user)
468 {
469
470         if (user && !(pte & PT_USER_MASK))
471                 return 0;
472         if (write && !(pte & PT_WRITABLE_MASK))
473                 return 0;
474         return 1;
475 }
476
477 /*
478  * Remove a shadow pte.
479  */
480 static void paging_inval_page(struct kvm_vcpu *vcpu, gva_t addr)
481 {
482         hpa_t page_addr = vcpu->mmu.root_hpa;
483         int level = vcpu->mmu.shadow_root_level;
484
485         ++kvm_stat.invlpg;
486
487         for (; ; level--) {
488                 u32 index = PT64_INDEX(addr, level);
489                 u64 *table = __va(page_addr);
490
491                 if (level == PT_PAGE_TABLE_LEVEL ) {
492                         table[index] = 0;
493                         return;
494                 }
495
496                 if (!is_present_pte(table[index]))
497                         return;
498
499                 page_addr = table[index] & PT64_BASE_ADDR_MASK;
500
501                 if (level == PT_DIRECTORY_LEVEL &&
502                           (table[index] & PT_SHADOW_PS_MARK)) {
503                         table[index] = 0;
504                         release_pt_page_64(vcpu, page_addr, PT_PAGE_TABLE_LEVEL);
505
506                         kvm_arch_ops->tlb_flush(vcpu);
507                         return;
508                 }
509         }
510 }
511
512 static void paging_free(struct kvm_vcpu *vcpu)
513 {
514         nonpaging_free(vcpu);
515 }
516
517 #define PTTYPE 64
518 #include "paging_tmpl.h"
519 #undef PTTYPE
520
521 #define PTTYPE 32
522 #include "paging_tmpl.h"
523 #undef PTTYPE
524
525 static int paging64_init_context(struct kvm_vcpu *vcpu)
526 {
527         struct kvm_mmu *context = &vcpu->mmu;
528
529         ASSERT(is_pae(vcpu));
530         context->new_cr3 = paging_new_cr3;
531         context->page_fault = paging64_page_fault;
532         context->inval_page = paging_inval_page;
533         context->gva_to_gpa = paging64_gva_to_gpa;
534         context->free = paging_free;
535         context->root_level = PT64_ROOT_LEVEL;
536         context->shadow_root_level = PT64_ROOT_LEVEL;
537         context->root_hpa = kvm_mmu_alloc_page(vcpu, NULL);
538         ASSERT(VALID_PAGE(context->root_hpa));
539         kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
540                     (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
541         return 0;
542 }
543
544 static int paging32_init_context(struct kvm_vcpu *vcpu)
545 {
546         struct kvm_mmu *context = &vcpu->mmu;
547
548         context->new_cr3 = paging_new_cr3;
549         context->page_fault = paging32_page_fault;
550         context->inval_page = paging_inval_page;
551         context->gva_to_gpa = paging32_gva_to_gpa;
552         context->free = paging_free;
553         context->root_level = PT32_ROOT_LEVEL;
554         context->shadow_root_level = PT32E_ROOT_LEVEL;
555         context->root_hpa = kvm_mmu_alloc_page(vcpu, NULL);
556         ASSERT(VALID_PAGE(context->root_hpa));
557         kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
558                     (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
559         return 0;
560 }
561
562 static int paging32E_init_context(struct kvm_vcpu *vcpu)
563 {
564         int ret;
565
566         if ((ret = paging64_init_context(vcpu)))
567                 return ret;
568
569         vcpu->mmu.root_level = PT32E_ROOT_LEVEL;
570         vcpu->mmu.shadow_root_level = PT32E_ROOT_LEVEL;
571         return 0;
572 }
573
574 static int init_kvm_mmu(struct kvm_vcpu *vcpu)
575 {
576         ASSERT(vcpu);
577         ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
578
579         if (!is_paging(vcpu))
580                 return nonpaging_init_context(vcpu);
581         else if (is_long_mode(vcpu))
582                 return paging64_init_context(vcpu);
583         else if (is_pae(vcpu))
584                 return paging32E_init_context(vcpu);
585         else
586                 return paging32_init_context(vcpu);
587 }
588
589 static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
590 {
591         ASSERT(vcpu);
592         if (VALID_PAGE(vcpu->mmu.root_hpa)) {
593                 vcpu->mmu.free(vcpu);
594                 vcpu->mmu.root_hpa = INVALID_PAGE;
595         }
596 }
597
598 int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
599 {
600         destroy_kvm_mmu(vcpu);
601         return init_kvm_mmu(vcpu);
602 }
603
604 static void free_mmu_pages(struct kvm_vcpu *vcpu)
605 {
606         while (!list_empty(&vcpu->free_pages)) {
607                 struct kvm_mmu_page *page;
608
609                 page = list_entry(vcpu->free_pages.next,
610                                   struct kvm_mmu_page, link);
611                 list_del(&page->link);
612                 __free_page(pfn_to_page(page->page_hpa >> PAGE_SHIFT));
613                 page->page_hpa = INVALID_PAGE;
614         }
615 }
616
617 static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
618 {
619         int i;
620
621         ASSERT(vcpu);
622
623         for (i = 0; i < KVM_NUM_MMU_PAGES; i++) {
624                 struct page *page;
625                 struct kvm_mmu_page *page_header = &vcpu->page_header_buf[i];
626
627                 INIT_LIST_HEAD(&page_header->link);
628                 if ((page = alloc_page(GFP_KVM_MMU)) == NULL)
629                         goto error_1;
630                 page->private = (unsigned long)page_header;
631                 page_header->page_hpa = (hpa_t)page_to_pfn(page) << PAGE_SHIFT;
632                 memset(__va(page_header->page_hpa), 0, PAGE_SIZE);
633                 list_add(&page_header->link, &vcpu->free_pages);
634         }
635         return 0;
636
637 error_1:
638         free_mmu_pages(vcpu);
639         return -ENOMEM;
640 }
641
642 int kvm_mmu_init(struct kvm_vcpu *vcpu)
643 {
644         int r;
645
646         ASSERT(vcpu);
647         ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
648         ASSERT(list_empty(&vcpu->free_pages));
649
650         r = alloc_mmu_pages(vcpu);
651         if (r)
652                 goto out;
653
654         r = init_kvm_mmu(vcpu);
655         if (r)
656                 goto out_free_pages;
657
658         return 0;
659
660 out_free_pages:
661         free_mmu_pages(vcpu);
662 out:
663         return r;
664 }
665
666 void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
667 {
668         ASSERT(vcpu);
669
670         destroy_kvm_mmu(vcpu);
671         free_mmu_pages(vcpu);
672 }
673
674 void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
675 {
676         struct kvm_mmu_page *page;
677
678         list_for_each_entry(page, &kvm->active_mmu_pages, link) {
679                 int i;
680                 u64 *pt;
681
682                 if (!test_bit(slot, &page->slot_bitmap))
683                         continue;
684
685                 pt = __va(page->page_hpa);
686                 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
687                         /* avoid RMW */
688                         if (pt[i] & PT_WRITABLE_MASK)
689                                 pt[i] &= ~PT_WRITABLE_MASK;
690
691         }
692 }