[PATCH] KVM: MMU: Make kvm_mmu_alloc_page() return a kvm_mmu_page pointer
[linux-2.6.git] / drivers / kvm / mmu.c
1 /*
2  * Kernel-based Virtual Machine driver for Linux
3  *
4  * This module enables machines with Intel VT-x extensions to run virtual
5  * machines without emulation or binary translation.
6  *
7  * MMU support
8  *
9  * Copyright (C) 2006 Qumranet, Inc.
10  *
11  * Authors:
12  *   Yaniv Kamay  <yaniv@qumranet.com>
13  *   Avi Kivity   <avi@qumranet.com>
14  *
15  * This work is licensed under the terms of the GNU GPL, version 2.  See
16  * the COPYING file in the top-level directory.
17  *
18  */
19 #include <linux/types.h>
20 #include <linux/string.h>
21 #include <asm/page.h>
22 #include <linux/mm.h>
23 #include <linux/highmem.h>
24 #include <linux/module.h>
25
26 #include "vmx.h"
27 #include "kvm.h"
28
29 #define pgprintk(x...) do { } while (0)
30 #define rmap_printk(x...) do { } while (0)
31
32 #define ASSERT(x)                                                       \
33         if (!(x)) {                                                     \
34                 printk(KERN_WARNING "assertion failed %s:%d: %s\n",     \
35                        __FILE__, __LINE__, #x);                         \
36         }
37
38 #define PT64_ENT_PER_PAGE 512
39 #define PT32_ENT_PER_PAGE 1024
40
41 #define PT_WRITABLE_SHIFT 1
42
43 #define PT_PRESENT_MASK (1ULL << 0)
44 #define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
45 #define PT_USER_MASK (1ULL << 2)
46 #define PT_PWT_MASK (1ULL << 3)
47 #define PT_PCD_MASK (1ULL << 4)
48 #define PT_ACCESSED_MASK (1ULL << 5)
49 #define PT_DIRTY_MASK (1ULL << 6)
50 #define PT_PAGE_SIZE_MASK (1ULL << 7)
51 #define PT_PAT_MASK (1ULL << 7)
52 #define PT_GLOBAL_MASK (1ULL << 8)
53 #define PT64_NX_MASK (1ULL << 63)
54
55 #define PT_PAT_SHIFT 7
56 #define PT_DIR_PAT_SHIFT 12
57 #define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT)
58
59 #define PT32_DIR_PSE36_SIZE 4
60 #define PT32_DIR_PSE36_SHIFT 13
61 #define PT32_DIR_PSE36_MASK (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)
62
63
64 #define PT32_PTE_COPY_MASK \
65         (PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_DIRTY_MASK | PT_GLOBAL_MASK)
66
67 #define PT64_PTE_COPY_MASK (PT64_NX_MASK | PT32_PTE_COPY_MASK)
68
69 #define PT_FIRST_AVAIL_BITS_SHIFT 9
70 #define PT64_SECOND_AVAIL_BITS_SHIFT 52
71
72 #define PT_SHADOW_PS_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
73 #define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
74
75 #define PT_SHADOW_WRITABLE_SHIFT (PT_FIRST_AVAIL_BITS_SHIFT + 1)
76 #define PT_SHADOW_WRITABLE_MASK (1ULL << PT_SHADOW_WRITABLE_SHIFT)
77
78 #define PT_SHADOW_USER_SHIFT (PT_SHADOW_WRITABLE_SHIFT + 1)
79 #define PT_SHADOW_USER_MASK (1ULL << (PT_SHADOW_USER_SHIFT))
80
81 #define PT_SHADOW_BITS_OFFSET (PT_SHADOW_WRITABLE_SHIFT - PT_WRITABLE_SHIFT)
82
83 #define VALID_PAGE(x) ((x) != INVALID_PAGE)
84
85 #define PT64_LEVEL_BITS 9
86
87 #define PT64_LEVEL_SHIFT(level) \
88                 ( PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS )
89
90 #define PT64_LEVEL_MASK(level) \
91                 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
92
93 #define PT64_INDEX(address, level)\
94         (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
95
96
97 #define PT32_LEVEL_BITS 10
98
99 #define PT32_LEVEL_SHIFT(level) \
100                 ( PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS )
101
102 #define PT32_LEVEL_MASK(level) \
103                 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
104
105 #define PT32_INDEX(address, level)\
106         (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
107
108
109 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & PAGE_MASK)
110 #define PT64_DIR_BASE_ADDR_MASK \
111         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
112
113 #define PT32_BASE_ADDR_MASK PAGE_MASK
114 #define PT32_DIR_BASE_ADDR_MASK \
115         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
116
117
118 #define PFERR_PRESENT_MASK (1U << 0)
119 #define PFERR_WRITE_MASK (1U << 1)
120 #define PFERR_USER_MASK (1U << 2)
121
122 #define PT64_ROOT_LEVEL 4
123 #define PT32_ROOT_LEVEL 2
124 #define PT32E_ROOT_LEVEL 3
125
126 #define PT_DIRECTORY_LEVEL 2
127 #define PT_PAGE_TABLE_LEVEL 1
128
129 #define RMAP_EXT 4
130
131 struct kvm_rmap_desc {
132         u64 *shadow_ptes[RMAP_EXT];
133         struct kvm_rmap_desc *more;
134 };
135
136 static int is_write_protection(struct kvm_vcpu *vcpu)
137 {
138         return vcpu->cr0 & CR0_WP_MASK;
139 }
140
141 static int is_cpuid_PSE36(void)
142 {
143         return 1;
144 }
145
146 static int is_present_pte(unsigned long pte)
147 {
148         return pte & PT_PRESENT_MASK;
149 }
150
151 static int is_writeble_pte(unsigned long pte)
152 {
153         return pte & PT_WRITABLE_MASK;
154 }
155
156 static int is_io_pte(unsigned long pte)
157 {
158         return pte & PT_SHADOW_IO_MARK;
159 }
160
161 static int is_rmap_pte(u64 pte)
162 {
163         return (pte & (PT_WRITABLE_MASK | PT_PRESENT_MASK))
164                 == (PT_WRITABLE_MASK | PT_PRESENT_MASK);
165 }
166
167 /*
168  * Reverse mapping data structures:
169  *
170  * If page->private bit zero is zero, then page->private points to the
171  * shadow page table entry that points to page_address(page).
172  *
173  * If page->private bit zero is one, (then page->private & ~1) points
174  * to a struct kvm_rmap_desc containing more mappings.
175  */
176 static void rmap_add(struct kvm *kvm, u64 *spte)
177 {
178         struct page *page;
179         struct kvm_rmap_desc *desc;
180         int i;
181
182         if (!is_rmap_pte(*spte))
183                 return;
184         page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
185         if (!page->private) {
186                 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
187                 page->private = (unsigned long)spte;
188         } else if (!(page->private & 1)) {
189                 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
190                 desc = kzalloc(sizeof *desc, GFP_NOWAIT);
191                 if (!desc)
192                         BUG(); /* FIXME: return error */
193                 desc->shadow_ptes[0] = (u64 *)page->private;
194                 desc->shadow_ptes[1] = spte;
195                 page->private = (unsigned long)desc | 1;
196         } else {
197                 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
198                 desc = (struct kvm_rmap_desc *)(page->private & ~1ul);
199                 while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
200                         desc = desc->more;
201                 if (desc->shadow_ptes[RMAP_EXT-1]) {
202                         desc->more = kzalloc(sizeof *desc->more, GFP_NOWAIT);
203                         if (!desc->more)
204                                 BUG(); /* FIXME: return error */
205                         desc = desc->more;
206                 }
207                 for (i = 0; desc->shadow_ptes[i]; ++i)
208                         ;
209                 desc->shadow_ptes[i] = spte;
210         }
211 }
212
213 static void rmap_desc_remove_entry(struct page *page,
214                                    struct kvm_rmap_desc *desc,
215                                    int i,
216                                    struct kvm_rmap_desc *prev_desc)
217 {
218         int j;
219
220         for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
221                 ;
222         desc->shadow_ptes[i] = desc->shadow_ptes[j];
223         desc->shadow_ptes[j] = 0;
224         if (j != 0)
225                 return;
226         if (!prev_desc && !desc->more)
227                 page->private = (unsigned long)desc->shadow_ptes[0];
228         else
229                 if (prev_desc)
230                         prev_desc->more = desc->more;
231                 else
232                         page->private = (unsigned long)desc->more | 1;
233         kfree(desc);
234 }
235
236 static void rmap_remove(struct kvm *kvm, u64 *spte)
237 {
238         struct page *page;
239         struct kvm_rmap_desc *desc;
240         struct kvm_rmap_desc *prev_desc;
241         int i;
242
243         if (!is_rmap_pte(*spte))
244                 return;
245         page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
246         if (!page->private) {
247                 printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
248                 BUG();
249         } else if (!(page->private & 1)) {
250                 rmap_printk("rmap_remove:  %p %llx 1->0\n", spte, *spte);
251                 if ((u64 *)page->private != spte) {
252                         printk(KERN_ERR "rmap_remove:  %p %llx 1->BUG\n",
253                                spte, *spte);
254                         BUG();
255                 }
256                 page->private = 0;
257         } else {
258                 rmap_printk("rmap_remove:  %p %llx many->many\n", spte, *spte);
259                 desc = (struct kvm_rmap_desc *)(page->private & ~1ul);
260                 prev_desc = NULL;
261                 while (desc) {
262                         for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
263                                 if (desc->shadow_ptes[i] == spte) {
264                                         rmap_desc_remove_entry(page, desc, i,
265                                                                prev_desc);
266                                         return;
267                                 }
268                         prev_desc = desc;
269                         desc = desc->more;
270                 }
271                 BUG();
272         }
273 }
274
275 static void kvm_mmu_free_page(struct kvm_vcpu *vcpu, hpa_t page_hpa)
276 {
277         struct kvm_mmu_page *page_head = page_header(page_hpa);
278
279         list_del(&page_head->link);
280         page_head->page_hpa = page_hpa;
281         list_add(&page_head->link, &vcpu->free_pages);
282 }
283
284 static int is_empty_shadow_page(hpa_t page_hpa)
285 {
286         u32 *pos;
287         u32 *end;
288         for (pos = __va(page_hpa), end = pos + PAGE_SIZE / sizeof(u32);
289                       pos != end; pos++)
290                 if (*pos != 0)
291                         return 0;
292         return 1;
293 }
294
295 static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
296                                                u64 *parent_pte)
297 {
298         struct kvm_mmu_page *page;
299
300         if (list_empty(&vcpu->free_pages))
301                 return NULL;
302
303         page = list_entry(vcpu->free_pages.next, struct kvm_mmu_page, link);
304         list_del(&page->link);
305         list_add(&page->link, &vcpu->kvm->active_mmu_pages);
306         ASSERT(is_empty_shadow_page(page->page_hpa));
307         page->slot_bitmap = 0;
308         page->global = 1;
309         page->parent_pte = parent_pte;
310         return page;
311 }
312
313 static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa)
314 {
315         int slot = memslot_id(kvm, gfn_to_memslot(kvm, gpa >> PAGE_SHIFT));
316         struct kvm_mmu_page *page_head = page_header(__pa(pte));
317
318         __set_bit(slot, &page_head->slot_bitmap);
319 }
320
321 hpa_t safe_gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
322 {
323         hpa_t hpa = gpa_to_hpa(vcpu, gpa);
324
325         return is_error_hpa(hpa) ? bad_page_address | (gpa & ~PAGE_MASK): hpa;
326 }
327
328 hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
329 {
330         struct kvm_memory_slot *slot;
331         struct page *page;
332
333         ASSERT((gpa & HPA_ERR_MASK) == 0);
334         slot = gfn_to_memslot(vcpu->kvm, gpa >> PAGE_SHIFT);
335         if (!slot)
336                 return gpa | HPA_ERR_MASK;
337         page = gfn_to_page(slot, gpa >> PAGE_SHIFT);
338         return ((hpa_t)page_to_pfn(page) << PAGE_SHIFT)
339                 | (gpa & (PAGE_SIZE-1));
340 }
341
342 hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva)
343 {
344         gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
345
346         if (gpa == UNMAPPED_GVA)
347                 return UNMAPPED_GVA;
348         return gpa_to_hpa(vcpu, gpa);
349 }
350
351
352 static void release_pt_page_64(struct kvm_vcpu *vcpu, hpa_t page_hpa,
353                                int level)
354 {
355         u64 *pos;
356         u64 *end;
357
358         ASSERT(vcpu);
359         ASSERT(VALID_PAGE(page_hpa));
360         ASSERT(level <= PT64_ROOT_LEVEL && level > 0);
361
362         for (pos = __va(page_hpa), end = pos + PT64_ENT_PER_PAGE;
363              pos != end; pos++) {
364                 u64 current_ent = *pos;
365
366                 if (is_present_pte(current_ent)) {
367                         if (level != 1)
368                                 release_pt_page_64(vcpu,
369                                                   current_ent &
370                                                   PT64_BASE_ADDR_MASK,
371                                                   level - 1);
372                         else
373                                 rmap_remove(vcpu->kvm, pos);
374                 }
375                 *pos = 0;
376         }
377         kvm_mmu_free_page(vcpu, page_hpa);
378 }
379
380 static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
381 {
382 }
383
384 static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p)
385 {
386         int level = PT32E_ROOT_LEVEL;
387         hpa_t table_addr = vcpu->mmu.root_hpa;
388
389         for (; ; level--) {
390                 u32 index = PT64_INDEX(v, level);
391                 u64 *table;
392
393                 ASSERT(VALID_PAGE(table_addr));
394                 table = __va(table_addr);
395
396                 if (level == 1) {
397                         mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT);
398                         page_header_update_slot(vcpu->kvm, table, v);
399                         table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK |
400                                                                 PT_USER_MASK;
401                         rmap_add(vcpu->kvm, &table[index]);
402                         return 0;
403                 }
404
405                 if (table[index] == 0) {
406                         struct kvm_mmu_page *new_table;
407
408                         new_table = kvm_mmu_alloc_page(vcpu, &table[index]);
409                         if (!new_table) {
410                                 pgprintk("nonpaging_map: ENOMEM\n");
411                                 return -ENOMEM;
412                         }
413
414                         table[index] = new_table->page_hpa | PT_PRESENT_MASK
415                                 | PT_WRITABLE_MASK | PT_USER_MASK;
416                 }
417                 table_addr = table[index] & PT64_BASE_ADDR_MASK;
418         }
419 }
420
421 static void mmu_free_roots(struct kvm_vcpu *vcpu)
422 {
423         int i;
424
425 #ifdef CONFIG_X86_64
426         if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
427                 hpa_t root = vcpu->mmu.root_hpa;
428
429                 ASSERT(VALID_PAGE(root));
430                 release_pt_page_64(vcpu, root, PT64_ROOT_LEVEL);
431                 vcpu->mmu.root_hpa = INVALID_PAGE;
432                 return;
433         }
434 #endif
435         for (i = 0; i < 4; ++i) {
436                 hpa_t root = vcpu->mmu.pae_root[i];
437
438                 ASSERT(VALID_PAGE(root));
439                 root &= PT64_BASE_ADDR_MASK;
440                 release_pt_page_64(vcpu, root, PT32E_ROOT_LEVEL - 1);
441                 vcpu->mmu.pae_root[i] = INVALID_PAGE;
442         }
443         vcpu->mmu.root_hpa = INVALID_PAGE;
444 }
445
446 static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
447 {
448         int i;
449
450 #ifdef CONFIG_X86_64
451         if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
452                 hpa_t root = vcpu->mmu.root_hpa;
453
454                 ASSERT(!VALID_PAGE(root));
455                 root = kvm_mmu_alloc_page(vcpu, NULL)->page_hpa;
456                 vcpu->mmu.root_hpa = root;
457                 return;
458         }
459 #endif
460         for (i = 0; i < 4; ++i) {
461                 hpa_t root = vcpu->mmu.pae_root[i];
462
463                 ASSERT(!VALID_PAGE(root));
464                 root = kvm_mmu_alloc_page(vcpu, NULL)->page_hpa;
465                 vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK;
466         }
467         vcpu->mmu.root_hpa = __pa(vcpu->mmu.pae_root);
468 }
469
470 static void nonpaging_flush(struct kvm_vcpu *vcpu)
471 {
472         hpa_t root = vcpu->mmu.root_hpa;
473
474         ++kvm_stat.tlb_flush;
475         pgprintk("nonpaging_flush\n");
476         mmu_free_roots(vcpu);
477         mmu_alloc_roots(vcpu);
478         kvm_arch_ops->set_cr3(vcpu, root);
479         kvm_arch_ops->tlb_flush(vcpu);
480 }
481
482 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
483 {
484         return vaddr;
485 }
486
487 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
488                                u32 error_code)
489 {
490         int ret;
491         gpa_t addr = gva;
492
493         ASSERT(vcpu);
494         ASSERT(VALID_PAGE(vcpu->mmu.root_hpa));
495
496         for (;;) {
497              hpa_t paddr;
498
499              paddr = gpa_to_hpa(vcpu , addr & PT64_BASE_ADDR_MASK);
500
501              if (is_error_hpa(paddr))
502                      return 1;
503
504              ret = nonpaging_map(vcpu, addr & PAGE_MASK, paddr);
505              if (ret) {
506                      nonpaging_flush(vcpu);
507                      continue;
508              }
509              break;
510         }
511         return ret;
512 }
513
514 static void nonpaging_inval_page(struct kvm_vcpu *vcpu, gva_t addr)
515 {
516 }
517
518 static void nonpaging_free(struct kvm_vcpu *vcpu)
519 {
520         mmu_free_roots(vcpu);
521 }
522
523 static int nonpaging_init_context(struct kvm_vcpu *vcpu)
524 {
525         struct kvm_mmu *context = &vcpu->mmu;
526
527         context->new_cr3 = nonpaging_new_cr3;
528         context->page_fault = nonpaging_page_fault;
529         context->inval_page = nonpaging_inval_page;
530         context->gva_to_gpa = nonpaging_gva_to_gpa;
531         context->free = nonpaging_free;
532         context->root_level = PT32E_ROOT_LEVEL;
533         context->shadow_root_level = PT32E_ROOT_LEVEL;
534         mmu_alloc_roots(vcpu);
535         ASSERT(VALID_PAGE(context->root_hpa));
536         kvm_arch_ops->set_cr3(vcpu, context->root_hpa);
537         return 0;
538 }
539
540
541 static void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
542 {
543         struct kvm_mmu_page *page, *npage;
544
545         list_for_each_entry_safe(page, npage, &vcpu->kvm->active_mmu_pages,
546                                  link) {
547                 if (page->global)
548                         continue;
549
550                 if (!page->parent_pte)
551                         continue;
552
553                 *page->parent_pte = 0;
554                 release_pt_page_64(vcpu, page->page_hpa, 1);
555         }
556         ++kvm_stat.tlb_flush;
557         kvm_arch_ops->tlb_flush(vcpu);
558 }
559
560 static void paging_new_cr3(struct kvm_vcpu *vcpu)
561 {
562         kvm_mmu_flush_tlb(vcpu);
563 }
564
565 static void mark_pagetable_nonglobal(void *shadow_pte)
566 {
567         page_header(__pa(shadow_pte))->global = 0;
568 }
569
570 static inline void set_pte_common(struct kvm_vcpu *vcpu,
571                              u64 *shadow_pte,
572                              gpa_t gaddr,
573                              int dirty,
574                              u64 access_bits)
575 {
576         hpa_t paddr;
577
578         *shadow_pte |= access_bits << PT_SHADOW_BITS_OFFSET;
579         if (!dirty)
580                 access_bits &= ~PT_WRITABLE_MASK;
581
582         if (access_bits & PT_WRITABLE_MASK)
583                 mark_page_dirty(vcpu->kvm, gaddr >> PAGE_SHIFT);
584
585         *shadow_pte |= access_bits;
586
587         paddr = gpa_to_hpa(vcpu, gaddr & PT64_BASE_ADDR_MASK);
588
589         if (!(*shadow_pte & PT_GLOBAL_MASK))
590                 mark_pagetable_nonglobal(shadow_pte);
591
592         if (is_error_hpa(paddr)) {
593                 *shadow_pte |= gaddr;
594                 *shadow_pte |= PT_SHADOW_IO_MARK;
595                 *shadow_pte &= ~PT_PRESENT_MASK;
596         } else {
597                 *shadow_pte |= paddr;
598                 page_header_update_slot(vcpu->kvm, shadow_pte, gaddr);
599                 rmap_add(vcpu->kvm, shadow_pte);
600         }
601 }
602
603 static void inject_page_fault(struct kvm_vcpu *vcpu,
604                               u64 addr,
605                               u32 err_code)
606 {
607         kvm_arch_ops->inject_page_fault(vcpu, addr, err_code);
608 }
609
610 static inline int fix_read_pf(u64 *shadow_ent)
611 {
612         if ((*shadow_ent & PT_SHADOW_USER_MASK) &&
613             !(*shadow_ent & PT_USER_MASK)) {
614                 /*
615                  * If supervisor write protect is disabled, we shadow kernel
616                  * pages as user pages so we can trap the write access.
617                  */
618                 *shadow_ent |= PT_USER_MASK;
619                 *shadow_ent &= ~PT_WRITABLE_MASK;
620
621                 return 1;
622
623         }
624         return 0;
625 }
626
627 static int may_access(u64 pte, int write, int user)
628 {
629
630         if (user && !(pte & PT_USER_MASK))
631                 return 0;
632         if (write && !(pte & PT_WRITABLE_MASK))
633                 return 0;
634         return 1;
635 }
636
637 /*
638  * Remove a shadow pte.
639  */
640 static void paging_inval_page(struct kvm_vcpu *vcpu, gva_t addr)
641 {
642         hpa_t page_addr = vcpu->mmu.root_hpa;
643         int level = vcpu->mmu.shadow_root_level;
644
645         ++kvm_stat.invlpg;
646
647         for (; ; level--) {
648                 u32 index = PT64_INDEX(addr, level);
649                 u64 *table = __va(page_addr);
650
651                 if (level == PT_PAGE_TABLE_LEVEL ) {
652                         rmap_remove(vcpu->kvm, &table[index]);
653                         table[index] = 0;
654                         return;
655                 }
656
657                 if (!is_present_pte(table[index]))
658                         return;
659
660                 page_addr = table[index] & PT64_BASE_ADDR_MASK;
661
662                 if (level == PT_DIRECTORY_LEVEL &&
663                           (table[index] & PT_SHADOW_PS_MARK)) {
664                         table[index] = 0;
665                         release_pt_page_64(vcpu, page_addr, PT_PAGE_TABLE_LEVEL);
666
667                         kvm_arch_ops->tlb_flush(vcpu);
668                         return;
669                 }
670         }
671 }
672
673 static void paging_free(struct kvm_vcpu *vcpu)
674 {
675         nonpaging_free(vcpu);
676 }
677
678 #define PTTYPE 64
679 #include "paging_tmpl.h"
680 #undef PTTYPE
681
682 #define PTTYPE 32
683 #include "paging_tmpl.h"
684 #undef PTTYPE
685
686 static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
687 {
688         struct kvm_mmu *context = &vcpu->mmu;
689
690         ASSERT(is_pae(vcpu));
691         context->new_cr3 = paging_new_cr3;
692         context->page_fault = paging64_page_fault;
693         context->inval_page = paging_inval_page;
694         context->gva_to_gpa = paging64_gva_to_gpa;
695         context->free = paging_free;
696         context->root_level = level;
697         context->shadow_root_level = level;
698         mmu_alloc_roots(vcpu);
699         ASSERT(VALID_PAGE(context->root_hpa));
700         kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
701                     (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
702         return 0;
703 }
704
705 static int paging64_init_context(struct kvm_vcpu *vcpu)
706 {
707         return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
708 }
709
710 static int paging32_init_context(struct kvm_vcpu *vcpu)
711 {
712         struct kvm_mmu *context = &vcpu->mmu;
713
714         context->new_cr3 = paging_new_cr3;
715         context->page_fault = paging32_page_fault;
716         context->inval_page = paging_inval_page;
717         context->gva_to_gpa = paging32_gva_to_gpa;
718         context->free = paging_free;
719         context->root_level = PT32_ROOT_LEVEL;
720         context->shadow_root_level = PT32E_ROOT_LEVEL;
721         mmu_alloc_roots(vcpu);
722         ASSERT(VALID_PAGE(context->root_hpa));
723         kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
724                     (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
725         return 0;
726 }
727
728 static int paging32E_init_context(struct kvm_vcpu *vcpu)
729 {
730         return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
731 }
732
733 static int init_kvm_mmu(struct kvm_vcpu *vcpu)
734 {
735         ASSERT(vcpu);
736         ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
737
738         if (!is_paging(vcpu))
739                 return nonpaging_init_context(vcpu);
740         else if (is_long_mode(vcpu))
741                 return paging64_init_context(vcpu);
742         else if (is_pae(vcpu))
743                 return paging32E_init_context(vcpu);
744         else
745                 return paging32_init_context(vcpu);
746 }
747
748 static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
749 {
750         ASSERT(vcpu);
751         if (VALID_PAGE(vcpu->mmu.root_hpa)) {
752                 vcpu->mmu.free(vcpu);
753                 vcpu->mmu.root_hpa = INVALID_PAGE;
754         }
755 }
756
757 int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
758 {
759         destroy_kvm_mmu(vcpu);
760         return init_kvm_mmu(vcpu);
761 }
762
763 static void free_mmu_pages(struct kvm_vcpu *vcpu)
764 {
765         while (!list_empty(&vcpu->free_pages)) {
766                 struct kvm_mmu_page *page;
767
768                 page = list_entry(vcpu->free_pages.next,
769                                   struct kvm_mmu_page, link);
770                 list_del(&page->link);
771                 __free_page(pfn_to_page(page->page_hpa >> PAGE_SHIFT));
772                 page->page_hpa = INVALID_PAGE;
773         }
774         free_page((unsigned long)vcpu->mmu.pae_root);
775 }
776
777 static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
778 {
779         struct page *page;
780         int i;
781
782         ASSERT(vcpu);
783
784         for (i = 0; i < KVM_NUM_MMU_PAGES; i++) {
785                 struct kvm_mmu_page *page_header = &vcpu->page_header_buf[i];
786
787                 INIT_LIST_HEAD(&page_header->link);
788                 if ((page = alloc_page(GFP_KERNEL)) == NULL)
789                         goto error_1;
790                 page->private = (unsigned long)page_header;
791                 page_header->page_hpa = (hpa_t)page_to_pfn(page) << PAGE_SHIFT;
792                 memset(__va(page_header->page_hpa), 0, PAGE_SIZE);
793                 list_add(&page_header->link, &vcpu->free_pages);
794         }
795
796         /*
797          * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
798          * Therefore we need to allocate shadow page tables in the first
799          * 4GB of memory, which happens to fit the DMA32 zone.
800          */
801         page = alloc_page(GFP_KERNEL | __GFP_DMA32);
802         if (!page)
803                 goto error_1;
804         vcpu->mmu.pae_root = page_address(page);
805         for (i = 0; i < 4; ++i)
806                 vcpu->mmu.pae_root[i] = INVALID_PAGE;
807
808         return 0;
809
810 error_1:
811         free_mmu_pages(vcpu);
812         return -ENOMEM;
813 }
814
815 int kvm_mmu_create(struct kvm_vcpu *vcpu)
816 {
817         ASSERT(vcpu);
818         ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
819         ASSERT(list_empty(&vcpu->free_pages));
820
821         return alloc_mmu_pages(vcpu);
822 }
823
824 int kvm_mmu_setup(struct kvm_vcpu *vcpu)
825 {
826         ASSERT(vcpu);
827         ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
828         ASSERT(!list_empty(&vcpu->free_pages));
829
830         return init_kvm_mmu(vcpu);
831 }
832
833 void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
834 {
835         ASSERT(vcpu);
836
837         destroy_kvm_mmu(vcpu);
838         free_mmu_pages(vcpu);
839 }
840
841 void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
842 {
843         struct kvm_mmu_page *page;
844
845         list_for_each_entry(page, &kvm->active_mmu_pages, link) {
846                 int i;
847                 u64 *pt;
848
849                 if (!test_bit(slot, &page->slot_bitmap))
850                         continue;
851
852                 pt = __va(page->page_hpa);
853                 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
854                         /* avoid RMW */
855                         if (pt[i] & PT_WRITABLE_MASK) {
856                                 rmap_remove(kvm, &pt[i]);
857                                 pt[i] &= ~PT_WRITABLE_MASK;
858                         }
859         }
860 }