KVM: introduce gfn_to_pfn_memslot_atomic
[linux-3.10.git] / virt / kvm / kvm_main.c
1 /*
2  * Kernel-based Virtual Machine driver for Linux
3  *
4  * This module enables machines with Intel VT-x extensions to run virtual
5  * machines without emulation or binary translation.
6  *
7  * Copyright (C) 2006 Qumranet, Inc.
8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
9  *
10  * Authors:
11  *   Avi Kivity   <avi@qumranet.com>
12  *   Yaniv Kamay  <yaniv@qumranet.com>
13  *
14  * This work is licensed under the terms of the GNU GPL, version 2.  See
15  * the COPYING file in the top-level directory.
16  *
17  */
18
19 #include "iodev.h"
20
21 #include <linux/kvm_host.h>
22 #include <linux/kvm.h>
23 #include <linux/module.h>
24 #include <linux/errno.h>
25 #include <linux/percpu.h>
26 #include <linux/mm.h>
27 #include <linux/miscdevice.h>
28 #include <linux/vmalloc.h>
29 #include <linux/reboot.h>
30 #include <linux/debugfs.h>
31 #include <linux/highmem.h>
32 #include <linux/file.h>
33 #include <linux/syscore_ops.h>
34 #include <linux/cpu.h>
35 #include <linux/sched.h>
36 #include <linux/cpumask.h>
37 #include <linux/smp.h>
38 #include <linux/anon_inodes.h>
39 #include <linux/profile.h>
40 #include <linux/kvm_para.h>
41 #include <linux/pagemap.h>
42 #include <linux/mman.h>
43 #include <linux/swap.h>
44 #include <linux/bitops.h>
45 #include <linux/spinlock.h>
46 #include <linux/compat.h>
47 #include <linux/srcu.h>
48 #include <linux/hugetlb.h>
49 #include <linux/slab.h>
50 #include <linux/sort.h>
51 #include <linux/bsearch.h>
52
53 #include <asm/processor.h>
54 #include <asm/io.h>
55 #include <asm/uaccess.h>
56 #include <asm/pgtable.h>
57
58 #include "coalesced_mmio.h"
59 #include "async_pf.h"
60
61 #define CREATE_TRACE_POINTS
62 #include <trace/events/kvm.h>
63
64 MODULE_AUTHOR("Qumranet");
65 MODULE_LICENSE("GPL");
66
67 /*
68  * Ordering of locks:
69  *
70  *              kvm->lock --> kvm->slots_lock --> kvm->irq_lock
71  */
72
73 DEFINE_RAW_SPINLOCK(kvm_lock);
74 LIST_HEAD(vm_list);
75
76 static cpumask_var_t cpus_hardware_enabled;
77 static int kvm_usage_count = 0;
78 static atomic_t hardware_enable_failed;
79
80 struct kmem_cache *kvm_vcpu_cache;
81 EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
82
83 static __read_mostly struct preempt_ops kvm_preempt_ops;
84
85 struct dentry *kvm_debugfs_dir;
86
87 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
88                            unsigned long arg);
89 #ifdef CONFIG_COMPAT
90 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
91                                   unsigned long arg);
92 #endif
93 static int hardware_enable_all(void);
94 static void hardware_disable_all(void);
95
96 static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
97
98 bool kvm_rebooting;
99 EXPORT_SYMBOL_GPL(kvm_rebooting);
100
101 static bool largepages_enabled = true;
102
103 bool kvm_is_mmio_pfn(pfn_t pfn)
104 {
105         if (pfn_valid(pfn)) {
106                 int reserved;
107                 struct page *tail = pfn_to_page(pfn);
108                 struct page *head = compound_trans_head(tail);
109                 reserved = PageReserved(head);
110                 if (head != tail) {
111                         /*
112                          * "head" is not a dangling pointer
113                          * (compound_trans_head takes care of that)
114                          * but the hugepage may have been splitted
115                          * from under us (and we may not hold a
116                          * reference count on the head page so it can
117                          * be reused before we run PageReferenced), so
118                          * we've to check PageTail before returning
119                          * what we just read.
120                          */
121                         smp_rmb();
122                         if (PageTail(tail))
123                                 return reserved;
124                 }
125                 return PageReserved(tail);
126         }
127
128         return true;
129 }
130
131 /*
132  * Switches to specified vcpu, until a matching vcpu_put()
133  */
134 void vcpu_load(struct kvm_vcpu *vcpu)
135 {
136         int cpu;
137
138         mutex_lock(&vcpu->mutex);
139         if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
140                 /* The thread running this VCPU changed. */
141                 struct pid *oldpid = vcpu->pid;
142                 struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
143                 rcu_assign_pointer(vcpu->pid, newpid);
144                 synchronize_rcu();
145                 put_pid(oldpid);
146         }
147         cpu = get_cpu();
148         preempt_notifier_register(&vcpu->preempt_notifier);
149         kvm_arch_vcpu_load(vcpu, cpu);
150         put_cpu();
151 }
152
153 void vcpu_put(struct kvm_vcpu *vcpu)
154 {
155         preempt_disable();
156         kvm_arch_vcpu_put(vcpu);
157         preempt_notifier_unregister(&vcpu->preempt_notifier);
158         preempt_enable();
159         mutex_unlock(&vcpu->mutex);
160 }
161
162 static void ack_flush(void *_completed)
163 {
164 }
165
166 static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
167 {
168         int i, cpu, me;
169         cpumask_var_t cpus;
170         bool called = true;
171         struct kvm_vcpu *vcpu;
172
173         zalloc_cpumask_var(&cpus, GFP_ATOMIC);
174
175         me = get_cpu();
176         kvm_for_each_vcpu(i, vcpu, kvm) {
177                 kvm_make_request(req, vcpu);
178                 cpu = vcpu->cpu;
179
180                 /* Set ->requests bit before we read ->mode */
181                 smp_mb();
182
183                 if (cpus != NULL && cpu != -1 && cpu != me &&
184                       kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
185                         cpumask_set_cpu(cpu, cpus);
186         }
187         if (unlikely(cpus == NULL))
188                 smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
189         else if (!cpumask_empty(cpus))
190                 smp_call_function_many(cpus, ack_flush, NULL, 1);
191         else
192                 called = false;
193         put_cpu();
194         free_cpumask_var(cpus);
195         return called;
196 }
197
198 void kvm_flush_remote_tlbs(struct kvm *kvm)
199 {
200         long dirty_count = kvm->tlbs_dirty;
201
202         smp_mb();
203         if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
204                 ++kvm->stat.remote_tlb_flush;
205         cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
206 }
207
208 void kvm_reload_remote_mmus(struct kvm *kvm)
209 {
210         make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
211 }
212
213 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
214 {
215         struct page *page;
216         int r;
217
218         mutex_init(&vcpu->mutex);
219         vcpu->cpu = -1;
220         vcpu->kvm = kvm;
221         vcpu->vcpu_id = id;
222         vcpu->pid = NULL;
223         init_waitqueue_head(&vcpu->wq);
224         kvm_async_pf_vcpu_init(vcpu);
225
226         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
227         if (!page) {
228                 r = -ENOMEM;
229                 goto fail;
230         }
231         vcpu->run = page_address(page);
232
233         kvm_vcpu_set_in_spin_loop(vcpu, false);
234         kvm_vcpu_set_dy_eligible(vcpu, false);
235
236         r = kvm_arch_vcpu_init(vcpu);
237         if (r < 0)
238                 goto fail_free_run;
239         return 0;
240
241 fail_free_run:
242         free_page((unsigned long)vcpu->run);
243 fail:
244         return r;
245 }
246 EXPORT_SYMBOL_GPL(kvm_vcpu_init);
247
248 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
249 {
250         put_pid(vcpu->pid);
251         kvm_arch_vcpu_uninit(vcpu);
252         free_page((unsigned long)vcpu->run);
253 }
254 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
255
256 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
257 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
258 {
259         return container_of(mn, struct kvm, mmu_notifier);
260 }
261
262 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
263                                              struct mm_struct *mm,
264                                              unsigned long address)
265 {
266         struct kvm *kvm = mmu_notifier_to_kvm(mn);
267         int need_tlb_flush, idx;
268
269         /*
270          * When ->invalidate_page runs, the linux pte has been zapped
271          * already but the page is still allocated until
272          * ->invalidate_page returns. So if we increase the sequence
273          * here the kvm page fault will notice if the spte can't be
274          * established because the page is going to be freed. If
275          * instead the kvm page fault establishes the spte before
276          * ->invalidate_page runs, kvm_unmap_hva will release it
277          * before returning.
278          *
279          * The sequence increase only need to be seen at spin_unlock
280          * time, and not at spin_lock time.
281          *
282          * Increasing the sequence after the spin_unlock would be
283          * unsafe because the kvm page fault could then establish the
284          * pte after kvm_unmap_hva returned, without noticing the page
285          * is going to be freed.
286          */
287         idx = srcu_read_lock(&kvm->srcu);
288         spin_lock(&kvm->mmu_lock);
289
290         kvm->mmu_notifier_seq++;
291         need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
292         /* we've to flush the tlb before the pages can be freed */
293         if (need_tlb_flush)
294                 kvm_flush_remote_tlbs(kvm);
295
296         spin_unlock(&kvm->mmu_lock);
297         srcu_read_unlock(&kvm->srcu, idx);
298 }
299
300 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
301                                         struct mm_struct *mm,
302                                         unsigned long address,
303                                         pte_t pte)
304 {
305         struct kvm *kvm = mmu_notifier_to_kvm(mn);
306         int idx;
307
308         idx = srcu_read_lock(&kvm->srcu);
309         spin_lock(&kvm->mmu_lock);
310         kvm->mmu_notifier_seq++;
311         kvm_set_spte_hva(kvm, address, pte);
312         spin_unlock(&kvm->mmu_lock);
313         srcu_read_unlock(&kvm->srcu, idx);
314 }
315
316 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
317                                                     struct mm_struct *mm,
318                                                     unsigned long start,
319                                                     unsigned long end)
320 {
321         struct kvm *kvm = mmu_notifier_to_kvm(mn);
322         int need_tlb_flush = 0, idx;
323
324         idx = srcu_read_lock(&kvm->srcu);
325         spin_lock(&kvm->mmu_lock);
326         /*
327          * The count increase must become visible at unlock time as no
328          * spte can be established without taking the mmu_lock and
329          * count is also read inside the mmu_lock critical section.
330          */
331         kvm->mmu_notifier_count++;
332         need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
333         need_tlb_flush |= kvm->tlbs_dirty;
334         /* we've to flush the tlb before the pages can be freed */
335         if (need_tlb_flush)
336                 kvm_flush_remote_tlbs(kvm);
337
338         spin_unlock(&kvm->mmu_lock);
339         srcu_read_unlock(&kvm->srcu, idx);
340 }
341
342 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
343                                                   struct mm_struct *mm,
344                                                   unsigned long start,
345                                                   unsigned long end)
346 {
347         struct kvm *kvm = mmu_notifier_to_kvm(mn);
348
349         spin_lock(&kvm->mmu_lock);
350         /*
351          * This sequence increase will notify the kvm page fault that
352          * the page that is going to be mapped in the spte could have
353          * been freed.
354          */
355         kvm->mmu_notifier_seq++;
356         smp_wmb();
357         /*
358          * The above sequence increase must be visible before the
359          * below count decrease, which is ensured by the smp_wmb above
360          * in conjunction with the smp_rmb in mmu_notifier_retry().
361          */
362         kvm->mmu_notifier_count--;
363         spin_unlock(&kvm->mmu_lock);
364
365         BUG_ON(kvm->mmu_notifier_count < 0);
366 }
367
368 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
369                                               struct mm_struct *mm,
370                                               unsigned long address)
371 {
372         struct kvm *kvm = mmu_notifier_to_kvm(mn);
373         int young, idx;
374
375         idx = srcu_read_lock(&kvm->srcu);
376         spin_lock(&kvm->mmu_lock);
377
378         young = kvm_age_hva(kvm, address);
379         if (young)
380                 kvm_flush_remote_tlbs(kvm);
381
382         spin_unlock(&kvm->mmu_lock);
383         srcu_read_unlock(&kvm->srcu, idx);
384
385         return young;
386 }
387
388 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
389                                        struct mm_struct *mm,
390                                        unsigned long address)
391 {
392         struct kvm *kvm = mmu_notifier_to_kvm(mn);
393         int young, idx;
394
395         idx = srcu_read_lock(&kvm->srcu);
396         spin_lock(&kvm->mmu_lock);
397         young = kvm_test_age_hva(kvm, address);
398         spin_unlock(&kvm->mmu_lock);
399         srcu_read_unlock(&kvm->srcu, idx);
400
401         return young;
402 }
403
404 static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
405                                      struct mm_struct *mm)
406 {
407         struct kvm *kvm = mmu_notifier_to_kvm(mn);
408         int idx;
409
410         idx = srcu_read_lock(&kvm->srcu);
411         kvm_arch_flush_shadow(kvm);
412         srcu_read_unlock(&kvm->srcu, idx);
413 }
414
415 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
416         .invalidate_page        = kvm_mmu_notifier_invalidate_page,
417         .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
418         .invalidate_range_end   = kvm_mmu_notifier_invalidate_range_end,
419         .clear_flush_young      = kvm_mmu_notifier_clear_flush_young,
420         .test_young             = kvm_mmu_notifier_test_young,
421         .change_pte             = kvm_mmu_notifier_change_pte,
422         .release                = kvm_mmu_notifier_release,
423 };
424
425 static int kvm_init_mmu_notifier(struct kvm *kvm)
426 {
427         kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
428         return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
429 }
430
431 #else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
432
433 static int kvm_init_mmu_notifier(struct kvm *kvm)
434 {
435         return 0;
436 }
437
438 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
439
440 static void kvm_init_memslots_id(struct kvm *kvm)
441 {
442         int i;
443         struct kvm_memslots *slots = kvm->memslots;
444
445         for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
446                 slots->id_to_index[i] = slots->memslots[i].id = i;
447 }
448
449 static struct kvm *kvm_create_vm(unsigned long type)
450 {
451         int r, i;
452         struct kvm *kvm = kvm_arch_alloc_vm();
453
454         if (!kvm)
455                 return ERR_PTR(-ENOMEM);
456
457         r = kvm_arch_init_vm(kvm, type);
458         if (r)
459                 goto out_err_nodisable;
460
461         r = hardware_enable_all();
462         if (r)
463                 goto out_err_nodisable;
464
465 #ifdef CONFIG_HAVE_KVM_IRQCHIP
466         INIT_HLIST_HEAD(&kvm->mask_notifier_list);
467         INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
468 #endif
469
470         r = -ENOMEM;
471         kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
472         if (!kvm->memslots)
473                 goto out_err_nosrcu;
474         kvm_init_memslots_id(kvm);
475         if (init_srcu_struct(&kvm->srcu))
476                 goto out_err_nosrcu;
477         for (i = 0; i < KVM_NR_BUSES; i++) {
478                 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
479                                         GFP_KERNEL);
480                 if (!kvm->buses[i])
481                         goto out_err;
482         }
483
484         spin_lock_init(&kvm->mmu_lock);
485         kvm->mm = current->mm;
486         atomic_inc(&kvm->mm->mm_count);
487         kvm_eventfd_init(kvm);
488         mutex_init(&kvm->lock);
489         mutex_init(&kvm->irq_lock);
490         mutex_init(&kvm->slots_lock);
491         atomic_set(&kvm->users_count, 1);
492
493         r = kvm_init_mmu_notifier(kvm);
494         if (r)
495                 goto out_err;
496
497         raw_spin_lock(&kvm_lock);
498         list_add(&kvm->vm_list, &vm_list);
499         raw_spin_unlock(&kvm_lock);
500
501         return kvm;
502
503 out_err:
504         cleanup_srcu_struct(&kvm->srcu);
505 out_err_nosrcu:
506         hardware_disable_all();
507 out_err_nodisable:
508         for (i = 0; i < KVM_NR_BUSES; i++)
509                 kfree(kvm->buses[i]);
510         kfree(kvm->memslots);
511         kvm_arch_free_vm(kvm);
512         return ERR_PTR(r);
513 }
514
515 /*
516  * Avoid using vmalloc for a small buffer.
517  * Should not be used when the size is statically known.
518  */
519 void *kvm_kvzalloc(unsigned long size)
520 {
521         if (size > PAGE_SIZE)
522                 return vzalloc(size);
523         else
524                 return kzalloc(size, GFP_KERNEL);
525 }
526
527 void kvm_kvfree(const void *addr)
528 {
529         if (is_vmalloc_addr(addr))
530                 vfree(addr);
531         else
532                 kfree(addr);
533 }
534
535 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
536 {
537         if (!memslot->dirty_bitmap)
538                 return;
539
540         kvm_kvfree(memslot->dirty_bitmap);
541         memslot->dirty_bitmap = NULL;
542 }
543
544 /*
545  * Free any memory in @free but not in @dont.
546  */
547 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
548                                   struct kvm_memory_slot *dont)
549 {
550         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
551                 kvm_destroy_dirty_bitmap(free);
552
553         kvm_arch_free_memslot(free, dont);
554
555         free->npages = 0;
556 }
557
558 void kvm_free_physmem(struct kvm *kvm)
559 {
560         struct kvm_memslots *slots = kvm->memslots;
561         struct kvm_memory_slot *memslot;
562
563         kvm_for_each_memslot(memslot, slots)
564                 kvm_free_physmem_slot(memslot, NULL);
565
566         kfree(kvm->memslots);
567 }
568
569 static void kvm_destroy_vm(struct kvm *kvm)
570 {
571         int i;
572         struct mm_struct *mm = kvm->mm;
573
574         kvm_arch_sync_events(kvm);
575         raw_spin_lock(&kvm_lock);
576         list_del(&kvm->vm_list);
577         raw_spin_unlock(&kvm_lock);
578         kvm_free_irq_routing(kvm);
579         for (i = 0; i < KVM_NR_BUSES; i++)
580                 kvm_io_bus_destroy(kvm->buses[i]);
581         kvm_coalesced_mmio_free(kvm);
582 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
583         mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
584 #else
585         kvm_arch_flush_shadow(kvm);
586 #endif
587         kvm_arch_destroy_vm(kvm);
588         kvm_free_physmem(kvm);
589         cleanup_srcu_struct(&kvm->srcu);
590         kvm_arch_free_vm(kvm);
591         hardware_disable_all();
592         mmdrop(mm);
593 }
594
595 void kvm_get_kvm(struct kvm *kvm)
596 {
597         atomic_inc(&kvm->users_count);
598 }
599 EXPORT_SYMBOL_GPL(kvm_get_kvm);
600
601 void kvm_put_kvm(struct kvm *kvm)
602 {
603         if (atomic_dec_and_test(&kvm->users_count))
604                 kvm_destroy_vm(kvm);
605 }
606 EXPORT_SYMBOL_GPL(kvm_put_kvm);
607
608
609 static int kvm_vm_release(struct inode *inode, struct file *filp)
610 {
611         struct kvm *kvm = filp->private_data;
612
613         kvm_irqfd_release(kvm);
614
615         kvm_put_kvm(kvm);
616         return 0;
617 }
618
619 /*
620  * Allocation size is twice as large as the actual dirty bitmap size.
621  * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
622  */
623 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
624 {
625 #ifndef CONFIG_S390
626         unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
627
628         memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes);
629         if (!memslot->dirty_bitmap)
630                 return -ENOMEM;
631
632 #endif /* !CONFIG_S390 */
633         return 0;
634 }
635
636 static int cmp_memslot(const void *slot1, const void *slot2)
637 {
638         struct kvm_memory_slot *s1, *s2;
639
640         s1 = (struct kvm_memory_slot *)slot1;
641         s2 = (struct kvm_memory_slot *)slot2;
642
643         if (s1->npages < s2->npages)
644                 return 1;
645         if (s1->npages > s2->npages)
646                 return -1;
647
648         return 0;
649 }
650
651 /*
652  * Sort the memslots base on its size, so the larger slots
653  * will get better fit.
654  */
655 static void sort_memslots(struct kvm_memslots *slots)
656 {
657         int i;
658
659         sort(slots->memslots, KVM_MEM_SLOTS_NUM,
660               sizeof(struct kvm_memory_slot), cmp_memslot, NULL);
661
662         for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
663                 slots->id_to_index[slots->memslots[i].id] = i;
664 }
665
666 void update_memslots(struct kvm_memslots *slots, struct kvm_memory_slot *new)
667 {
668         if (new) {
669                 int id = new->id;
670                 struct kvm_memory_slot *old = id_to_memslot(slots, id);
671                 unsigned long npages = old->npages;
672
673                 *old = *new;
674                 if (new->npages != npages)
675                         sort_memslots(slots);
676         }
677
678         slots->generation++;
679 }
680
681 static int check_memory_region_flags(struct kvm_userspace_memory_region *mem)
682 {
683         if (mem->flags & ~KVM_MEM_LOG_DIRTY_PAGES)
684                 return -EINVAL;
685
686         return 0;
687 }
688
689 /*
690  * Allocate some memory and give it an address in the guest physical address
691  * space.
692  *
693  * Discontiguous memory is allowed, mostly for framebuffers.
694  *
695  * Must be called holding mmap_sem for write.
696  */
697 int __kvm_set_memory_region(struct kvm *kvm,
698                             struct kvm_userspace_memory_region *mem,
699                             int user_alloc)
700 {
701         int r;
702         gfn_t base_gfn;
703         unsigned long npages;
704         unsigned long i;
705         struct kvm_memory_slot *memslot;
706         struct kvm_memory_slot old, new;
707         struct kvm_memslots *slots, *old_memslots;
708
709         r = check_memory_region_flags(mem);
710         if (r)
711                 goto out;
712
713         r = -EINVAL;
714         /* General sanity checks */
715         if (mem->memory_size & (PAGE_SIZE - 1))
716                 goto out;
717         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
718                 goto out;
719         /* We can read the guest memory with __xxx_user() later on. */
720         if (user_alloc &&
721             ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
722              !access_ok(VERIFY_WRITE,
723                         (void __user *)(unsigned long)mem->userspace_addr,
724                         mem->memory_size)))
725                 goto out;
726         if (mem->slot >= KVM_MEM_SLOTS_NUM)
727                 goto out;
728         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
729                 goto out;
730
731         memslot = id_to_memslot(kvm->memslots, mem->slot);
732         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
733         npages = mem->memory_size >> PAGE_SHIFT;
734
735         r = -EINVAL;
736         if (npages > KVM_MEM_MAX_NR_PAGES)
737                 goto out;
738
739         if (!npages)
740                 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
741
742         new = old = *memslot;
743
744         new.id = mem->slot;
745         new.base_gfn = base_gfn;
746         new.npages = npages;
747         new.flags = mem->flags;
748
749         /* Disallow changing a memory slot's size. */
750         r = -EINVAL;
751         if (npages && old.npages && npages != old.npages)
752                 goto out_free;
753
754         /* Check for overlaps */
755         r = -EEXIST;
756         for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
757                 struct kvm_memory_slot *s = &kvm->memslots->memslots[i];
758
759                 if (s == memslot || !s->npages)
760                         continue;
761                 if (!((base_gfn + npages <= s->base_gfn) ||
762                       (base_gfn >= s->base_gfn + s->npages)))
763                         goto out_free;
764         }
765
766         /* Free page dirty bitmap if unneeded */
767         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
768                 new.dirty_bitmap = NULL;
769
770         r = -ENOMEM;
771
772         /* Allocate if a slot is being created */
773         if (npages && !old.npages) {
774                 new.user_alloc = user_alloc;
775                 new.userspace_addr = mem->userspace_addr;
776
777                 if (kvm_arch_create_memslot(&new, npages))
778                         goto out_free;
779         }
780
781         /* Allocate page dirty bitmap if needed */
782         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
783                 if (kvm_create_dirty_bitmap(&new) < 0)
784                         goto out_free;
785                 /* destroy any largepage mappings for dirty tracking */
786         }
787
788         if (!npages) {
789                 struct kvm_memory_slot *slot;
790
791                 r = -ENOMEM;
792                 slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
793                                 GFP_KERNEL);
794                 if (!slots)
795                         goto out_free;
796                 slot = id_to_memslot(slots, mem->slot);
797                 slot->flags |= KVM_MEMSLOT_INVALID;
798
799                 update_memslots(slots, NULL);
800
801                 old_memslots = kvm->memslots;
802                 rcu_assign_pointer(kvm->memslots, slots);
803                 synchronize_srcu_expedited(&kvm->srcu);
804                 /* From this point no new shadow pages pointing to a deleted
805                  * memslot will be created.
806                  *
807                  * validation of sp->gfn happens in:
808                  *      - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
809                  *      - kvm_is_visible_gfn (mmu_check_roots)
810                  */
811                 kvm_arch_flush_shadow(kvm);
812                 kfree(old_memslots);
813         }
814
815         r = kvm_arch_prepare_memory_region(kvm, &new, old, mem, user_alloc);
816         if (r)
817                 goto out_free;
818
819         /* map/unmap the pages in iommu page table */
820         if (npages) {
821                 r = kvm_iommu_map_pages(kvm, &new);
822                 if (r)
823                         goto out_free;
824         } else
825                 kvm_iommu_unmap_pages(kvm, &old);
826
827         r = -ENOMEM;
828         slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
829                         GFP_KERNEL);
830         if (!slots)
831                 goto out_free;
832
833         /* actual memory is freed via old in kvm_free_physmem_slot below */
834         if (!npages) {
835                 new.dirty_bitmap = NULL;
836                 memset(&new.arch, 0, sizeof(new.arch));
837         }
838
839         update_memslots(slots, &new);
840         old_memslots = kvm->memslots;
841         rcu_assign_pointer(kvm->memslots, slots);
842         synchronize_srcu_expedited(&kvm->srcu);
843
844         kvm_arch_commit_memory_region(kvm, mem, old, user_alloc);
845
846         /*
847          * If the new memory slot is created, we need to clear all
848          * mmio sptes.
849          */
850         if (npages && old.base_gfn != mem->guest_phys_addr >> PAGE_SHIFT)
851                 kvm_arch_flush_shadow(kvm);
852
853         kvm_free_physmem_slot(&old, &new);
854         kfree(old_memslots);
855
856         return 0;
857
858 out_free:
859         kvm_free_physmem_slot(&new, &old);
860 out:
861         return r;
862
863 }
864 EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
865
866 int kvm_set_memory_region(struct kvm *kvm,
867                           struct kvm_userspace_memory_region *mem,
868                           int user_alloc)
869 {
870         int r;
871
872         mutex_lock(&kvm->slots_lock);
873         r = __kvm_set_memory_region(kvm, mem, user_alloc);
874         mutex_unlock(&kvm->slots_lock);
875         return r;
876 }
877 EXPORT_SYMBOL_GPL(kvm_set_memory_region);
878
879 int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
880                                    struct
881                                    kvm_userspace_memory_region *mem,
882                                    int user_alloc)
883 {
884         if (mem->slot >= KVM_MEMORY_SLOTS)
885                 return -EINVAL;
886         return kvm_set_memory_region(kvm, mem, user_alloc);
887 }
888
889 int kvm_get_dirty_log(struct kvm *kvm,
890                         struct kvm_dirty_log *log, int *is_dirty)
891 {
892         struct kvm_memory_slot *memslot;
893         int r, i;
894         unsigned long n;
895         unsigned long any = 0;
896
897         r = -EINVAL;
898         if (log->slot >= KVM_MEMORY_SLOTS)
899                 goto out;
900
901         memslot = id_to_memslot(kvm->memslots, log->slot);
902         r = -ENOENT;
903         if (!memslot->dirty_bitmap)
904                 goto out;
905
906         n = kvm_dirty_bitmap_bytes(memslot);
907
908         for (i = 0; !any && i < n/sizeof(long); ++i)
909                 any = memslot->dirty_bitmap[i];
910
911         r = -EFAULT;
912         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
913                 goto out;
914
915         if (any)
916                 *is_dirty = 1;
917
918         r = 0;
919 out:
920         return r;
921 }
922
923 bool kvm_largepages_enabled(void)
924 {
925         return largepages_enabled;
926 }
927
928 void kvm_disable_largepages(void)
929 {
930         largepages_enabled = false;
931 }
932 EXPORT_SYMBOL_GPL(kvm_disable_largepages);
933
934 static inline unsigned long bad_hva(void)
935 {
936         return PAGE_OFFSET;
937 }
938
939 int kvm_is_error_hva(unsigned long addr)
940 {
941         return addr == bad_hva();
942 }
943 EXPORT_SYMBOL_GPL(kvm_is_error_hva);
944
945 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
946 {
947         return __gfn_to_memslot(kvm_memslots(kvm), gfn);
948 }
949 EXPORT_SYMBOL_GPL(gfn_to_memslot);
950
951 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
952 {
953         struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
954
955         if (!memslot || memslot->id >= KVM_MEMORY_SLOTS ||
956               memslot->flags & KVM_MEMSLOT_INVALID)
957                 return 0;
958
959         return 1;
960 }
961 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
962
963 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
964 {
965         struct vm_area_struct *vma;
966         unsigned long addr, size;
967
968         size = PAGE_SIZE;
969
970         addr = gfn_to_hva(kvm, gfn);
971         if (kvm_is_error_hva(addr))
972                 return PAGE_SIZE;
973
974         down_read(&current->mm->mmap_sem);
975         vma = find_vma(current->mm, addr);
976         if (!vma)
977                 goto out;
978
979         size = vma_kernel_pagesize(vma);
980
981 out:
982         up_read(&current->mm->mmap_sem);
983
984         return size;
985 }
986
987 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
988                                      gfn_t *nr_pages)
989 {
990         if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
991                 return bad_hva();
992
993         if (nr_pages)
994                 *nr_pages = slot->npages - (gfn - slot->base_gfn);
995
996         return gfn_to_hva_memslot(slot, gfn);
997 }
998
999 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
1000 {
1001         return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1002 }
1003 EXPORT_SYMBOL_GPL(gfn_to_hva);
1004
1005 int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
1006         unsigned long start, int write, struct page **page)
1007 {
1008         int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;
1009
1010         if (write)
1011                 flags |= FOLL_WRITE;
1012
1013         return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL);
1014 }
1015
1016 static inline int check_user_page_hwpoison(unsigned long addr)
1017 {
1018         int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;
1019
1020         rc = __get_user_pages(current, current->mm, addr, 1,
1021                               flags, NULL, NULL, NULL);
1022         return rc == -EHWPOISON;
1023 }
1024
1025 static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
1026                         bool write_fault, bool *writable)
1027 {
1028         struct page *page[1];
1029         int npages = 0;
1030         pfn_t pfn;
1031
1032         /* we can do it either atomically or asynchronously, not both */
1033         BUG_ON(atomic && async);
1034
1035         BUG_ON(!write_fault && !writable);
1036
1037         if (writable)
1038                 *writable = true;
1039
1040         if (atomic || async)
1041                 npages = __get_user_pages_fast(addr, 1, 1, page);
1042
1043         if (unlikely(npages != 1) && !atomic) {
1044                 might_sleep();
1045
1046                 if (writable)
1047                         *writable = write_fault;
1048
1049                 if (async) {
1050                         down_read(&current->mm->mmap_sem);
1051                         npages = get_user_page_nowait(current, current->mm,
1052                                                      addr, write_fault, page);
1053                         up_read(&current->mm->mmap_sem);
1054                 } else
1055                         npages = get_user_pages_fast(addr, 1, write_fault,
1056                                                      page);
1057
1058                 /* map read fault as writable if possible */
1059                 if (unlikely(!write_fault) && npages == 1) {
1060                         struct page *wpage[1];
1061
1062                         npages = __get_user_pages_fast(addr, 1, 1, wpage);
1063                         if (npages == 1) {
1064                                 *writable = true;
1065                                 put_page(page[0]);
1066                                 page[0] = wpage[0];
1067                         }
1068                         npages = 1;
1069                 }
1070         }
1071
1072         if (unlikely(npages != 1)) {
1073                 struct vm_area_struct *vma;
1074
1075                 if (atomic)
1076                         return KVM_PFN_ERR_FAULT;
1077
1078                 down_read(&current->mm->mmap_sem);
1079                 if (npages == -EHWPOISON ||
1080                         (!async && check_user_page_hwpoison(addr))) {
1081                         up_read(&current->mm->mmap_sem);
1082                         return KVM_PFN_ERR_HWPOISON;
1083                 }
1084
1085                 vma = find_vma_intersection(current->mm, addr, addr+1);
1086
1087                 if (vma == NULL)
1088                         pfn = KVM_PFN_ERR_FAULT;
1089                 else if ((vma->vm_flags & VM_PFNMAP)) {
1090                         pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
1091                                 vma->vm_pgoff;
1092                         BUG_ON(!kvm_is_mmio_pfn(pfn));
1093                 } else {
1094                         if (async && (vma->vm_flags & VM_WRITE))
1095                                 *async = true;
1096                         pfn = KVM_PFN_ERR_FAULT;
1097                 }
1098                 up_read(&current->mm->mmap_sem);
1099         } else
1100                 pfn = page_to_pfn(page[0]);
1101
1102         return pfn;
1103 }
1104
1105 static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async,
1106                           bool write_fault, bool *writable)
1107 {
1108         unsigned long addr;
1109
1110         if (async)
1111                 *async = false;
1112
1113         addr = gfn_to_hva(kvm, gfn);
1114         if (kvm_is_error_hva(addr))
1115                 return KVM_PFN_ERR_BAD;
1116
1117         return hva_to_pfn(addr, atomic, async, write_fault, writable);
1118 }
1119
1120 pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
1121 {
1122         return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL);
1123 }
1124 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
1125
1126 pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
1127                        bool write_fault, bool *writable)
1128 {
1129         return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable);
1130 }
1131 EXPORT_SYMBOL_GPL(gfn_to_pfn_async);
1132
1133 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
1134 {
1135         return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL);
1136 }
1137 EXPORT_SYMBOL_GPL(gfn_to_pfn);
1138
1139 pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1140                       bool *writable)
1141 {
1142         return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable);
1143 }
1144 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
1145
1146 pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1147 {
1148         unsigned long addr = gfn_to_hva_memslot(slot, gfn);
1149         return hva_to_pfn(addr, false, NULL, true, NULL);
1150 }
1151
1152 pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1153 {
1154         unsigned long addr = gfn_to_hva_memslot(slot, gfn);
1155
1156         return hva_to_pfn(addr, true, NULL, true, NULL);
1157 }
1158 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1159
1160 int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
1161                                                                   int nr_pages)
1162 {
1163         unsigned long addr;
1164         gfn_t entry;
1165
1166         addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry);
1167         if (kvm_is_error_hva(addr))
1168                 return -1;
1169
1170         if (entry < nr_pages)
1171                 return 0;
1172
1173         return __get_user_pages_fast(addr, nr_pages, 1, pages);
1174 }
1175 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
1176
1177 static struct page *kvm_pfn_to_page(pfn_t pfn)
1178 {
1179         if (is_error_pfn(pfn))
1180                 return KVM_ERR_PTR_BAD_PAGE;
1181
1182         if (kvm_is_mmio_pfn(pfn)) {
1183                 WARN_ON(1);
1184                 return KVM_ERR_PTR_BAD_PAGE;
1185         }
1186
1187         return pfn_to_page(pfn);
1188 }
1189
1190 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1191 {
1192         pfn_t pfn;
1193
1194         pfn = gfn_to_pfn(kvm, gfn);
1195
1196         return kvm_pfn_to_page(pfn);
1197 }
1198
1199 EXPORT_SYMBOL_GPL(gfn_to_page);
1200
1201 void kvm_release_page_clean(struct page *page)
1202 {
1203         WARN_ON(is_error_page(page));
1204
1205         kvm_release_pfn_clean(page_to_pfn(page));
1206 }
1207 EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1208
1209 void kvm_release_pfn_clean(pfn_t pfn)
1210 {
1211         WARN_ON(is_error_pfn(pfn));
1212
1213         if (!kvm_is_mmio_pfn(pfn))
1214                 put_page(pfn_to_page(pfn));
1215 }
1216 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1217
1218 void kvm_release_page_dirty(struct page *page)
1219 {
1220         WARN_ON(is_error_page(page));
1221
1222         kvm_release_pfn_dirty(page_to_pfn(page));
1223 }
1224 EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
1225
1226 void kvm_release_pfn_dirty(pfn_t pfn)
1227 {
1228         kvm_set_pfn_dirty(pfn);
1229         kvm_release_pfn_clean(pfn);
1230 }
1231 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
1232
1233 void kvm_set_page_dirty(struct page *page)
1234 {
1235         kvm_set_pfn_dirty(page_to_pfn(page));
1236 }
1237 EXPORT_SYMBOL_GPL(kvm_set_page_dirty);
1238
1239 void kvm_set_pfn_dirty(pfn_t pfn)
1240 {
1241         if (!kvm_is_mmio_pfn(pfn)) {
1242                 struct page *page = pfn_to_page(pfn);
1243                 if (!PageReserved(page))
1244                         SetPageDirty(page);
1245         }
1246 }
1247 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
1248
1249 void kvm_set_pfn_accessed(pfn_t pfn)
1250 {
1251         if (!kvm_is_mmio_pfn(pfn))
1252                 mark_page_accessed(pfn_to_page(pfn));
1253 }
1254 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
1255
1256 void kvm_get_pfn(pfn_t pfn)
1257 {
1258         if (!kvm_is_mmio_pfn(pfn))
1259                 get_page(pfn_to_page(pfn));
1260 }
1261 EXPORT_SYMBOL_GPL(kvm_get_pfn);
1262
1263 static int next_segment(unsigned long len, int offset)
1264 {
1265         if (len > PAGE_SIZE - offset)
1266                 return PAGE_SIZE - offset;
1267         else
1268                 return len;
1269 }
1270
1271 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1272                         int len)
1273 {
1274         int r;
1275         unsigned long addr;
1276
1277         addr = gfn_to_hva(kvm, gfn);
1278         if (kvm_is_error_hva(addr))
1279                 return -EFAULT;
1280         r = __copy_from_user(data, (void __user *)addr + offset, len);
1281         if (r)
1282                 return -EFAULT;
1283         return 0;
1284 }
1285 EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1286
1287 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1288 {
1289         gfn_t gfn = gpa >> PAGE_SHIFT;
1290         int seg;
1291         int offset = offset_in_page(gpa);
1292         int ret;
1293
1294         while ((seg = next_segment(len, offset)) != 0) {
1295                 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
1296                 if (ret < 0)
1297                         return ret;
1298                 offset = 0;
1299                 len -= seg;
1300                 data += seg;
1301                 ++gfn;
1302         }
1303         return 0;
1304 }
1305 EXPORT_SYMBOL_GPL(kvm_read_guest);
1306
1307 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1308                           unsigned long len)
1309 {
1310         int r;
1311         unsigned long addr;
1312         gfn_t gfn = gpa >> PAGE_SHIFT;
1313         int offset = offset_in_page(gpa);
1314
1315         addr = gfn_to_hva(kvm, gfn);
1316         if (kvm_is_error_hva(addr))
1317                 return -EFAULT;
1318         pagefault_disable();
1319         r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1320         pagefault_enable();
1321         if (r)
1322                 return -EFAULT;
1323         return 0;
1324 }
1325 EXPORT_SYMBOL(kvm_read_guest_atomic);
1326
1327 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
1328                          int offset, int len)
1329 {
1330         int r;
1331         unsigned long addr;
1332
1333         addr = gfn_to_hva(kvm, gfn);
1334         if (kvm_is_error_hva(addr))
1335                 return -EFAULT;
1336         r = __copy_to_user((void __user *)addr + offset, data, len);
1337         if (r)
1338                 return -EFAULT;
1339         mark_page_dirty(kvm, gfn);
1340         return 0;
1341 }
1342 EXPORT_SYMBOL_GPL(kvm_write_guest_page);
1343
1344 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
1345                     unsigned long len)
1346 {
1347         gfn_t gfn = gpa >> PAGE_SHIFT;
1348         int seg;
1349         int offset = offset_in_page(gpa);
1350         int ret;
1351
1352         while ((seg = next_segment(len, offset)) != 0) {
1353                 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
1354                 if (ret < 0)
1355                         return ret;
1356                 offset = 0;
1357                 len -= seg;
1358                 data += seg;
1359                 ++gfn;
1360         }
1361         return 0;
1362 }
1363
1364 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1365                               gpa_t gpa)
1366 {
1367         struct kvm_memslots *slots = kvm_memslots(kvm);
1368         int offset = offset_in_page(gpa);
1369         gfn_t gfn = gpa >> PAGE_SHIFT;
1370
1371         ghc->gpa = gpa;
1372         ghc->generation = slots->generation;
1373         ghc->memslot = gfn_to_memslot(kvm, gfn);
1374         ghc->hva = gfn_to_hva_many(ghc->memslot, gfn, NULL);
1375         if (!kvm_is_error_hva(ghc->hva))
1376                 ghc->hva += offset;
1377         else
1378                 return -EFAULT;
1379
1380         return 0;
1381 }
1382 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
1383
1384 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1385                            void *data, unsigned long len)
1386 {
1387         struct kvm_memslots *slots = kvm_memslots(kvm);
1388         int r;
1389
1390         if (slots->generation != ghc->generation)
1391                 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa);
1392
1393         if (kvm_is_error_hva(ghc->hva))
1394                 return -EFAULT;
1395
1396         r = __copy_to_user((void __user *)ghc->hva, data, len);
1397         if (r)
1398                 return -EFAULT;
1399         mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT);
1400
1401         return 0;
1402 }
1403 EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
1404
1405 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1406                            void *data, unsigned long len)
1407 {
1408         struct kvm_memslots *slots = kvm_memslots(kvm);
1409         int r;
1410
1411         if (slots->generation != ghc->generation)
1412                 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa);
1413
1414         if (kvm_is_error_hva(ghc->hva))
1415                 return -EFAULT;
1416
1417         r = __copy_from_user(data, (void __user *)ghc->hva, len);
1418         if (r)
1419                 return -EFAULT;
1420
1421         return 0;
1422 }
1423 EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
1424
1425 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
1426 {
1427         return kvm_write_guest_page(kvm, gfn, (const void *) empty_zero_page,
1428                                     offset, len);
1429 }
1430 EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
1431
1432 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
1433 {
1434         gfn_t gfn = gpa >> PAGE_SHIFT;
1435         int seg;
1436         int offset = offset_in_page(gpa);
1437         int ret;
1438
1439         while ((seg = next_segment(len, offset)) != 0) {
1440                 ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
1441                 if (ret < 0)
1442                         return ret;
1443                 offset = 0;
1444                 len -= seg;
1445                 ++gfn;
1446         }
1447         return 0;
1448 }
1449 EXPORT_SYMBOL_GPL(kvm_clear_guest);
1450
1451 void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot,
1452                              gfn_t gfn)
1453 {
1454         if (memslot && memslot->dirty_bitmap) {
1455                 unsigned long rel_gfn = gfn - memslot->base_gfn;
1456
1457                 /* TODO: introduce set_bit_le() and use it */
1458                 test_and_set_bit_le(rel_gfn, memslot->dirty_bitmap);
1459         }
1460 }
1461
1462 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
1463 {
1464         struct kvm_memory_slot *memslot;
1465
1466         memslot = gfn_to_memslot(kvm, gfn);
1467         mark_page_dirty_in_slot(kvm, memslot, gfn);
1468 }
1469
1470 /*
1471  * The vCPU has executed a HLT instruction with in-kernel mode enabled.
1472  */
1473 void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1474 {
1475         DEFINE_WAIT(wait);
1476
1477         for (;;) {
1478                 prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
1479
1480                 if (kvm_arch_vcpu_runnable(vcpu)) {
1481                         kvm_make_request(KVM_REQ_UNHALT, vcpu);
1482                         break;
1483                 }
1484                 if (kvm_cpu_has_pending_timer(vcpu))
1485                         break;
1486                 if (signal_pending(current))
1487                         break;
1488
1489                 schedule();
1490         }
1491
1492         finish_wait(&vcpu->wq, &wait);
1493 }
1494
1495 #ifndef CONFIG_S390
1496 /*
1497  * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
1498  */
1499 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
1500 {
1501         int me;
1502         int cpu = vcpu->cpu;
1503         wait_queue_head_t *wqp;
1504
1505         wqp = kvm_arch_vcpu_wq(vcpu);
1506         if (waitqueue_active(wqp)) {
1507                 wake_up_interruptible(wqp);
1508                 ++vcpu->stat.halt_wakeup;
1509         }
1510
1511         me = get_cpu();
1512         if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
1513                 if (kvm_arch_vcpu_should_kick(vcpu))
1514                         smp_send_reschedule(cpu);
1515         put_cpu();
1516 }
1517 #endif /* !CONFIG_S390 */
1518
1519 void kvm_resched(struct kvm_vcpu *vcpu)
1520 {
1521         if (!need_resched())
1522                 return;
1523         cond_resched();
1524 }
1525 EXPORT_SYMBOL_GPL(kvm_resched);
1526
1527 bool kvm_vcpu_yield_to(struct kvm_vcpu *target)
1528 {
1529         struct pid *pid;
1530         struct task_struct *task = NULL;
1531
1532         rcu_read_lock();
1533         pid = rcu_dereference(target->pid);
1534         if (pid)
1535                 task = get_pid_task(target->pid, PIDTYPE_PID);
1536         rcu_read_unlock();
1537         if (!task)
1538                 return false;
1539         if (task->flags & PF_VCPU) {
1540                 put_task_struct(task);
1541                 return false;
1542         }
1543         if (yield_to(task, 1)) {
1544                 put_task_struct(task);
1545                 return true;
1546         }
1547         put_task_struct(task);
1548         return false;
1549 }
1550 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
1551
1552 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
1553 /*
1554  * Helper that checks whether a VCPU is eligible for directed yield.
1555  * Most eligible candidate to yield is decided by following heuristics:
1556  *
1557  *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
1558  *  (preempted lock holder), indicated by @in_spin_loop.
1559  *  Set at the beiginning and cleared at the end of interception/PLE handler.
1560  *
1561  *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
1562  *  chance last time (mostly it has become eligible now since we have probably
1563  *  yielded to lockholder in last iteration. This is done by toggling
1564  *  @dy_eligible each time a VCPU checked for eligibility.)
1565  *
1566  *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
1567  *  to preempted lock-holder could result in wrong VCPU selection and CPU
1568  *  burning. Giving priority for a potential lock-holder increases lock
1569  *  progress.
1570  *
1571  *  Since algorithm is based on heuristics, accessing another VCPU data without
1572  *  locking does not harm. It may result in trying to yield to  same VCPU, fail
1573  *  and continue with next VCPU and so on.
1574  */
1575 bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
1576 {
1577         bool eligible;
1578
1579         eligible = !vcpu->spin_loop.in_spin_loop ||
1580                         (vcpu->spin_loop.in_spin_loop &&
1581                          vcpu->spin_loop.dy_eligible);
1582
1583         if (vcpu->spin_loop.in_spin_loop)
1584                 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
1585
1586         return eligible;
1587 }
1588 #endif
1589 void kvm_vcpu_on_spin(struct kvm_vcpu *me)
1590 {
1591         struct kvm *kvm = me->kvm;
1592         struct kvm_vcpu *vcpu;
1593         int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
1594         int yielded = 0;
1595         int pass;
1596         int i;
1597
1598         kvm_vcpu_set_in_spin_loop(me, true);
1599         /*
1600          * We boost the priority of a VCPU that is runnable but not
1601          * currently running, because it got preempted by something
1602          * else and called schedule in __vcpu_run.  Hopefully that
1603          * VCPU is holding the lock that we need and will release it.
1604          * We approximate round-robin by starting at the last boosted VCPU.
1605          */
1606         for (pass = 0; pass < 2 && !yielded; pass++) {
1607                 kvm_for_each_vcpu(i, vcpu, kvm) {
1608                         if (!pass && i <= last_boosted_vcpu) {
1609                                 i = last_boosted_vcpu;
1610                                 continue;
1611                         } else if (pass && i > last_boosted_vcpu)
1612                                 break;
1613                         if (vcpu == me)
1614                                 continue;
1615                         if (waitqueue_active(&vcpu->wq))
1616                                 continue;
1617                         if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
1618                                 continue;
1619                         if (kvm_vcpu_yield_to(vcpu)) {
1620                                 kvm->last_boosted_vcpu = i;
1621                                 yielded = 1;
1622                                 break;
1623                         }
1624                 }
1625         }
1626         kvm_vcpu_set_in_spin_loop(me, false);
1627
1628         /* Ensure vcpu is not eligible during next spinloop */
1629         kvm_vcpu_set_dy_eligible(me, false);
1630 }
1631 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
1632
1633 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1634 {
1635         struct kvm_vcpu *vcpu = vma->vm_file->private_data;
1636         struct page *page;
1637
1638         if (vmf->pgoff == 0)
1639                 page = virt_to_page(vcpu->run);
1640 #ifdef CONFIG_X86
1641         else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
1642                 page = virt_to_page(vcpu->arch.pio_data);
1643 #endif
1644 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
1645         else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
1646                 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
1647 #endif
1648         else
1649                 return kvm_arch_vcpu_fault(vcpu, vmf);
1650         get_page(page);
1651         vmf->page = page;
1652         return 0;
1653 }
1654
1655 static const struct vm_operations_struct kvm_vcpu_vm_ops = {
1656         .fault = kvm_vcpu_fault,
1657 };
1658
1659 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
1660 {
1661         vma->vm_ops = &kvm_vcpu_vm_ops;
1662         return 0;
1663 }
1664
1665 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
1666 {
1667         struct kvm_vcpu *vcpu = filp->private_data;
1668
1669         kvm_put_kvm(vcpu->kvm);
1670         return 0;
1671 }
1672
1673 static struct file_operations kvm_vcpu_fops = {
1674         .release        = kvm_vcpu_release,
1675         .unlocked_ioctl = kvm_vcpu_ioctl,
1676 #ifdef CONFIG_COMPAT
1677         .compat_ioctl   = kvm_vcpu_compat_ioctl,
1678 #endif
1679         .mmap           = kvm_vcpu_mmap,
1680         .llseek         = noop_llseek,
1681 };
1682
1683 /*
1684  * Allocates an inode for the vcpu.
1685  */
1686 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
1687 {
1688         return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR);
1689 }
1690
1691 /*
1692  * Creates some virtual cpus.  Good luck creating more than one.
1693  */
1694 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
1695 {
1696         int r;
1697         struct kvm_vcpu *vcpu, *v;
1698
1699         vcpu = kvm_arch_vcpu_create(kvm, id);
1700         if (IS_ERR(vcpu))
1701                 return PTR_ERR(vcpu);
1702
1703         preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
1704
1705         r = kvm_arch_vcpu_setup(vcpu);
1706         if (r)
1707                 goto vcpu_destroy;
1708
1709         mutex_lock(&kvm->lock);
1710         if (!kvm_vcpu_compatible(vcpu)) {
1711                 r = -EINVAL;
1712                 goto unlock_vcpu_destroy;
1713         }
1714         if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
1715                 r = -EINVAL;
1716                 goto unlock_vcpu_destroy;
1717         }
1718
1719         kvm_for_each_vcpu(r, v, kvm)
1720                 if (v->vcpu_id == id) {
1721                         r = -EEXIST;
1722                         goto unlock_vcpu_destroy;
1723                 }
1724
1725         BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
1726
1727         /* Now it's all set up, let userspace reach it */
1728         kvm_get_kvm(kvm);
1729         r = create_vcpu_fd(vcpu);
1730         if (r < 0) {
1731                 kvm_put_kvm(kvm);
1732                 goto unlock_vcpu_destroy;
1733         }
1734
1735         kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
1736         smp_wmb();
1737         atomic_inc(&kvm->online_vcpus);
1738
1739         mutex_unlock(&kvm->lock);
1740         return r;
1741
1742 unlock_vcpu_destroy:
1743         mutex_unlock(&kvm->lock);
1744 vcpu_destroy:
1745         kvm_arch_vcpu_destroy(vcpu);
1746         return r;
1747 }
1748
1749 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
1750 {
1751         if (sigset) {
1752                 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
1753                 vcpu->sigset_active = 1;
1754                 vcpu->sigset = *sigset;
1755         } else
1756                 vcpu->sigset_active = 0;
1757         return 0;
1758 }
1759
1760 static long kvm_vcpu_ioctl(struct file *filp,
1761                            unsigned int ioctl, unsigned long arg)
1762 {
1763         struct kvm_vcpu *vcpu = filp->private_data;
1764         void __user *argp = (void __user *)arg;
1765         int r;
1766         struct kvm_fpu *fpu = NULL;
1767         struct kvm_sregs *kvm_sregs = NULL;
1768
1769         if (vcpu->kvm->mm != current->mm)
1770                 return -EIO;
1771
1772 #if defined(CONFIG_S390) || defined(CONFIG_PPC)
1773         /*
1774          * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
1775          * so vcpu_load() would break it.
1776          */
1777         if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT)
1778                 return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
1779 #endif
1780
1781
1782         vcpu_load(vcpu);
1783         switch (ioctl) {
1784         case KVM_RUN:
1785                 r = -EINVAL;
1786                 if (arg)
1787                         goto out;
1788                 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
1789                 trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
1790                 break;
1791         case KVM_GET_REGS: {
1792                 struct kvm_regs *kvm_regs;
1793
1794                 r = -ENOMEM;
1795                 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
1796                 if (!kvm_regs)
1797                         goto out;
1798                 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
1799                 if (r)
1800                         goto out_free1;
1801                 r = -EFAULT;
1802                 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
1803                         goto out_free1;
1804                 r = 0;
1805 out_free1:
1806                 kfree(kvm_regs);
1807                 break;
1808         }
1809         case KVM_SET_REGS: {
1810                 struct kvm_regs *kvm_regs;
1811
1812                 r = -ENOMEM;
1813                 kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
1814                 if (IS_ERR(kvm_regs)) {
1815                         r = PTR_ERR(kvm_regs);
1816                         goto out;
1817                 }
1818                 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
1819                 if (r)
1820                         goto out_free2;
1821                 r = 0;
1822 out_free2:
1823                 kfree(kvm_regs);
1824                 break;
1825         }
1826         case KVM_GET_SREGS: {
1827                 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
1828                 r = -ENOMEM;
1829                 if (!kvm_sregs)
1830                         goto out;
1831                 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
1832                 if (r)
1833                         goto out;
1834                 r = -EFAULT;
1835                 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
1836                         goto out;
1837                 r = 0;
1838                 break;
1839         }
1840         case KVM_SET_SREGS: {
1841                 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
1842                 if (IS_ERR(kvm_sregs)) {
1843                         r = PTR_ERR(kvm_sregs);
1844                         goto out;
1845                 }
1846                 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
1847                 if (r)
1848                         goto out;
1849                 r = 0;
1850                 break;
1851         }
1852         case KVM_GET_MP_STATE: {
1853                 struct kvm_mp_state mp_state;
1854
1855                 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
1856                 if (r)
1857                         goto out;
1858                 r = -EFAULT;
1859                 if (copy_to_user(argp, &mp_state, sizeof mp_state))
1860                         goto out;
1861                 r = 0;
1862                 break;
1863         }
1864         case KVM_SET_MP_STATE: {
1865                 struct kvm_mp_state mp_state;
1866
1867                 r = -EFAULT;
1868                 if (copy_from_user(&mp_state, argp, sizeof mp_state))
1869                         goto out;
1870                 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
1871                 if (r)
1872                         goto out;
1873                 r = 0;
1874                 break;
1875         }
1876         case KVM_TRANSLATE: {
1877                 struct kvm_translation tr;
1878
1879                 r = -EFAULT;
1880                 if (copy_from_user(&tr, argp, sizeof tr))
1881                         goto out;
1882                 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
1883                 if (r)
1884                         goto out;
1885                 r = -EFAULT;
1886                 if (copy_to_user(argp, &tr, sizeof tr))
1887                         goto out;
1888                 r = 0;
1889                 break;
1890         }
1891         case KVM_SET_GUEST_DEBUG: {
1892                 struct kvm_guest_debug dbg;
1893
1894                 r = -EFAULT;
1895                 if (copy_from_user(&dbg, argp, sizeof dbg))
1896                         goto out;
1897                 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
1898                 if (r)
1899                         goto out;
1900                 r = 0;
1901                 break;
1902         }
1903         case KVM_SET_SIGNAL_MASK: {
1904                 struct kvm_signal_mask __user *sigmask_arg = argp;
1905                 struct kvm_signal_mask kvm_sigmask;
1906                 sigset_t sigset, *p;
1907
1908                 p = NULL;
1909                 if (argp) {
1910                         r = -EFAULT;
1911                         if (copy_from_user(&kvm_sigmask, argp,
1912                                            sizeof kvm_sigmask))
1913                                 goto out;
1914                         r = -EINVAL;
1915                         if (kvm_sigmask.len != sizeof sigset)
1916                                 goto out;
1917                         r = -EFAULT;
1918                         if (copy_from_user(&sigset, sigmask_arg->sigset,
1919                                            sizeof sigset))
1920                                 goto out;
1921                         p = &sigset;
1922                 }
1923                 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
1924                 break;
1925         }
1926         case KVM_GET_FPU: {
1927                 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
1928                 r = -ENOMEM;
1929                 if (!fpu)
1930                         goto out;
1931                 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
1932                 if (r)
1933                         goto out;
1934                 r = -EFAULT;
1935                 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
1936                         goto out;
1937                 r = 0;
1938                 break;
1939         }
1940         case KVM_SET_FPU: {
1941                 fpu = memdup_user(argp, sizeof(*fpu));
1942                 if (IS_ERR(fpu)) {
1943                         r = PTR_ERR(fpu);
1944                         goto out;
1945                 }
1946                 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
1947                 if (r)
1948                         goto out;
1949                 r = 0;
1950                 break;
1951         }
1952         default:
1953                 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
1954         }
1955 out:
1956         vcpu_put(vcpu);
1957         kfree(fpu);
1958         kfree(kvm_sregs);
1959         return r;
1960 }
1961
1962 #ifdef CONFIG_COMPAT
1963 static long kvm_vcpu_compat_ioctl(struct file *filp,
1964                                   unsigned int ioctl, unsigned long arg)
1965 {
1966         struct kvm_vcpu *vcpu = filp->private_data;
1967         void __user *argp = compat_ptr(arg);
1968         int r;
1969
1970         if (vcpu->kvm->mm != current->mm)
1971                 return -EIO;
1972
1973         switch (ioctl) {
1974         case KVM_SET_SIGNAL_MASK: {
1975                 struct kvm_signal_mask __user *sigmask_arg = argp;
1976                 struct kvm_signal_mask kvm_sigmask;
1977                 compat_sigset_t csigset;
1978                 sigset_t sigset;
1979
1980                 if (argp) {
1981                         r = -EFAULT;
1982                         if (copy_from_user(&kvm_sigmask, argp,
1983                                            sizeof kvm_sigmask))
1984                                 goto out;
1985                         r = -EINVAL;
1986                         if (kvm_sigmask.len != sizeof csigset)
1987                                 goto out;
1988                         r = -EFAULT;
1989                         if (copy_from_user(&csigset, sigmask_arg->sigset,
1990                                            sizeof csigset))
1991                                 goto out;
1992                 }
1993                 sigset_from_compat(&sigset, &csigset);
1994                 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
1995                 break;
1996         }
1997         default:
1998                 r = kvm_vcpu_ioctl(filp, ioctl, arg);
1999         }
2000
2001 out:
2002         return r;
2003 }
2004 #endif
2005
2006 static long kvm_vm_ioctl(struct file *filp,
2007                            unsigned int ioctl, unsigned long arg)
2008 {
2009         struct kvm *kvm = filp->private_data;
2010         void __user *argp = (void __user *)arg;
2011         int r;
2012
2013         if (kvm->mm != current->mm)
2014                 return -EIO;
2015         switch (ioctl) {
2016         case KVM_CREATE_VCPU:
2017                 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
2018                 if (r < 0)
2019                         goto out;
2020                 break;
2021         case KVM_SET_USER_MEMORY_REGION: {
2022                 struct kvm_userspace_memory_region kvm_userspace_mem;
2023
2024                 r = -EFAULT;
2025                 if (copy_from_user(&kvm_userspace_mem, argp,
2026                                                 sizeof kvm_userspace_mem))
2027                         goto out;
2028
2029                 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
2030                 if (r)
2031                         goto out;
2032                 break;
2033         }
2034         case KVM_GET_DIRTY_LOG: {
2035                 struct kvm_dirty_log log;
2036
2037                 r = -EFAULT;
2038                 if (copy_from_user(&log, argp, sizeof log))
2039                         goto out;
2040                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2041                 if (r)
2042                         goto out;
2043                 break;
2044         }
2045 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2046         case KVM_REGISTER_COALESCED_MMIO: {
2047                 struct kvm_coalesced_mmio_zone zone;
2048                 r = -EFAULT;
2049                 if (copy_from_user(&zone, argp, sizeof zone))
2050                         goto out;
2051                 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
2052                 if (r)
2053                         goto out;
2054                 r = 0;
2055                 break;
2056         }
2057         case KVM_UNREGISTER_COALESCED_MMIO: {
2058                 struct kvm_coalesced_mmio_zone zone;
2059                 r = -EFAULT;
2060                 if (copy_from_user(&zone, argp, sizeof zone))
2061                         goto out;
2062                 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
2063                 if (r)
2064                         goto out;
2065                 r = 0;
2066                 break;
2067         }
2068 #endif
2069         case KVM_IRQFD: {
2070                 struct kvm_irqfd data;
2071
2072                 r = -EFAULT;
2073                 if (copy_from_user(&data, argp, sizeof data))
2074                         goto out;
2075                 r = kvm_irqfd(kvm, &data);
2076                 break;
2077         }
2078         case KVM_IOEVENTFD: {
2079                 struct kvm_ioeventfd data;
2080
2081                 r = -EFAULT;
2082                 if (copy_from_user(&data, argp, sizeof data))
2083                         goto out;
2084                 r = kvm_ioeventfd(kvm, &data);
2085                 break;
2086         }
2087 #ifdef CONFIG_KVM_APIC_ARCHITECTURE
2088         case KVM_SET_BOOT_CPU_ID:
2089                 r = 0;
2090                 mutex_lock(&kvm->lock);
2091                 if (atomic_read(&kvm->online_vcpus) != 0)
2092                         r = -EBUSY;
2093                 else
2094                         kvm->bsp_vcpu_id = arg;
2095                 mutex_unlock(&kvm->lock);
2096                 break;
2097 #endif
2098 #ifdef CONFIG_HAVE_KVM_MSI
2099         case KVM_SIGNAL_MSI: {
2100                 struct kvm_msi msi;
2101
2102                 r = -EFAULT;
2103                 if (copy_from_user(&msi, argp, sizeof msi))
2104                         goto out;
2105                 r = kvm_send_userspace_msi(kvm, &msi);
2106                 break;
2107         }
2108 #endif
2109 #ifdef __KVM_HAVE_IRQ_LINE
2110         case KVM_IRQ_LINE_STATUS:
2111         case KVM_IRQ_LINE: {
2112                 struct kvm_irq_level irq_event;
2113
2114                 r = -EFAULT;
2115                 if (copy_from_user(&irq_event, argp, sizeof irq_event))
2116                         goto out;
2117
2118                 r = kvm_vm_ioctl_irq_line(kvm, &irq_event);
2119                 if (r)
2120                         goto out;
2121
2122                 r = -EFAULT;
2123                 if (ioctl == KVM_IRQ_LINE_STATUS) {
2124                         if (copy_to_user(argp, &irq_event, sizeof irq_event))
2125                                 goto out;
2126                 }
2127
2128                 r = 0;
2129                 break;
2130         }
2131 #endif
2132         default:
2133                 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
2134                 if (r == -ENOTTY)
2135                         r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
2136         }
2137 out:
2138         return r;
2139 }
2140
2141 #ifdef CONFIG_COMPAT
2142 struct compat_kvm_dirty_log {
2143         __u32 slot;
2144         __u32 padding1;
2145         union {
2146                 compat_uptr_t dirty_bitmap; /* one bit per page */
2147                 __u64 padding2;
2148         };
2149 };
2150
2151 static long kvm_vm_compat_ioctl(struct file *filp,
2152                            unsigned int ioctl, unsigned long arg)
2153 {
2154         struct kvm *kvm = filp->private_data;
2155         int r;
2156
2157         if (kvm->mm != current->mm)
2158                 return -EIO;
2159         switch (ioctl) {
2160         case KVM_GET_DIRTY_LOG: {
2161                 struct compat_kvm_dirty_log compat_log;
2162                 struct kvm_dirty_log log;
2163
2164                 r = -EFAULT;
2165                 if (copy_from_user(&compat_log, (void __user *)arg,
2166                                    sizeof(compat_log)))
2167                         goto out;
2168                 log.slot         = compat_log.slot;
2169                 log.padding1     = compat_log.padding1;
2170                 log.padding2     = compat_log.padding2;
2171                 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
2172
2173                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2174                 if (r)
2175                         goto out;
2176                 break;
2177         }
2178         default:
2179                 r = kvm_vm_ioctl(filp, ioctl, arg);
2180         }
2181
2182 out:
2183         return r;
2184 }
2185 #endif
2186
2187 static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2188 {
2189         struct page *page[1];
2190         unsigned long addr;
2191         int npages;
2192         gfn_t gfn = vmf->pgoff;
2193         struct kvm *kvm = vma->vm_file->private_data;
2194
2195         addr = gfn_to_hva(kvm, gfn);
2196         if (kvm_is_error_hva(addr))
2197                 return VM_FAULT_SIGBUS;
2198
2199         npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
2200                                 NULL);
2201         if (unlikely(npages != 1))
2202                 return VM_FAULT_SIGBUS;
2203
2204         vmf->page = page[0];
2205         return 0;
2206 }
2207
2208 static const struct vm_operations_struct kvm_vm_vm_ops = {
2209         .fault = kvm_vm_fault,
2210 };
2211
2212 static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
2213 {
2214         vma->vm_ops = &kvm_vm_vm_ops;
2215         return 0;
2216 }
2217
2218 static struct file_operations kvm_vm_fops = {
2219         .release        = kvm_vm_release,
2220         .unlocked_ioctl = kvm_vm_ioctl,
2221 #ifdef CONFIG_COMPAT
2222         .compat_ioctl   = kvm_vm_compat_ioctl,
2223 #endif
2224         .mmap           = kvm_vm_mmap,
2225         .llseek         = noop_llseek,
2226 };
2227
2228 static int kvm_dev_ioctl_create_vm(unsigned long type)
2229 {
2230         int r;
2231         struct kvm *kvm;
2232
2233         kvm = kvm_create_vm(type);
2234         if (IS_ERR(kvm))
2235                 return PTR_ERR(kvm);
2236 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2237         r = kvm_coalesced_mmio_init(kvm);
2238         if (r < 0) {
2239                 kvm_put_kvm(kvm);
2240                 return r;
2241         }
2242 #endif
2243         r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
2244         if (r < 0)
2245                 kvm_put_kvm(kvm);
2246
2247         return r;
2248 }
2249
2250 static long kvm_dev_ioctl_check_extension_generic(long arg)
2251 {
2252         switch (arg) {
2253         case KVM_CAP_USER_MEMORY:
2254         case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
2255         case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
2256 #ifdef CONFIG_KVM_APIC_ARCHITECTURE
2257         case KVM_CAP_SET_BOOT_CPU_ID:
2258 #endif
2259         case KVM_CAP_INTERNAL_ERROR_DATA:
2260 #ifdef CONFIG_HAVE_KVM_MSI
2261         case KVM_CAP_SIGNAL_MSI:
2262 #endif
2263                 return 1;
2264 #ifdef KVM_CAP_IRQ_ROUTING
2265         case KVM_CAP_IRQ_ROUTING:
2266                 return KVM_MAX_IRQ_ROUTES;
2267 #endif
2268         default:
2269                 break;
2270         }
2271         return kvm_dev_ioctl_check_extension(arg);
2272 }
2273
2274 static long kvm_dev_ioctl(struct file *filp,
2275                           unsigned int ioctl, unsigned long arg)
2276 {
2277         long r = -EINVAL;
2278
2279         switch (ioctl) {
2280         case KVM_GET_API_VERSION:
2281                 r = -EINVAL;
2282                 if (arg)
2283                         goto out;
2284                 r = KVM_API_VERSION;
2285                 break;
2286         case KVM_CREATE_VM:
2287                 r = kvm_dev_ioctl_create_vm(arg);
2288                 break;
2289         case KVM_CHECK_EXTENSION:
2290                 r = kvm_dev_ioctl_check_extension_generic(arg);
2291                 break;
2292         case KVM_GET_VCPU_MMAP_SIZE:
2293                 r = -EINVAL;
2294                 if (arg)
2295                         goto out;
2296                 r = PAGE_SIZE;     /* struct kvm_run */
2297 #ifdef CONFIG_X86
2298                 r += PAGE_SIZE;    /* pio data page */
2299 #endif
2300 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2301                 r += PAGE_SIZE;    /* coalesced mmio ring page */
2302 #endif
2303                 break;
2304         case KVM_TRACE_ENABLE:
2305         case KVM_TRACE_PAUSE:
2306         case KVM_TRACE_DISABLE:
2307                 r = -EOPNOTSUPP;
2308                 break;
2309         default:
2310                 return kvm_arch_dev_ioctl(filp, ioctl, arg);
2311         }
2312 out:
2313         return r;
2314 }
2315
2316 static struct file_operations kvm_chardev_ops = {
2317         .unlocked_ioctl = kvm_dev_ioctl,
2318         .compat_ioctl   = kvm_dev_ioctl,
2319         .llseek         = noop_llseek,
2320 };
2321
2322 static struct miscdevice kvm_dev = {
2323         KVM_MINOR,
2324         "kvm",
2325         &kvm_chardev_ops,
2326 };
2327
2328 static void hardware_enable_nolock(void *junk)
2329 {
2330         int cpu = raw_smp_processor_id();
2331         int r;
2332
2333         if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
2334                 return;
2335
2336         cpumask_set_cpu(cpu, cpus_hardware_enabled);
2337
2338         r = kvm_arch_hardware_enable(NULL);
2339
2340         if (r) {
2341                 cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2342                 atomic_inc(&hardware_enable_failed);
2343                 printk(KERN_INFO "kvm: enabling virtualization on "
2344                                  "CPU%d failed\n", cpu);
2345         }
2346 }
2347
2348 static void hardware_enable(void *junk)
2349 {
2350         raw_spin_lock(&kvm_lock);
2351         hardware_enable_nolock(junk);
2352         raw_spin_unlock(&kvm_lock);
2353 }
2354
2355 static void hardware_disable_nolock(void *junk)
2356 {
2357         int cpu = raw_smp_processor_id();
2358
2359         if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
2360                 return;
2361         cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2362         kvm_arch_hardware_disable(NULL);
2363 }
2364
2365 static void hardware_disable(void *junk)
2366 {
2367         raw_spin_lock(&kvm_lock);
2368         hardware_disable_nolock(junk);
2369         raw_spin_unlock(&kvm_lock);
2370 }
2371
2372 static void hardware_disable_all_nolock(void)
2373 {
2374         BUG_ON(!kvm_usage_count);
2375
2376         kvm_usage_count--;
2377         if (!kvm_usage_count)
2378                 on_each_cpu(hardware_disable_nolock, NULL, 1);
2379 }
2380
2381 static void hardware_disable_all(void)
2382 {
2383         raw_spin_lock(&kvm_lock);
2384         hardware_disable_all_nolock();
2385         raw_spin_unlock(&kvm_lock);
2386 }
2387
2388 static int hardware_enable_all(void)
2389 {
2390         int r = 0;
2391
2392         raw_spin_lock(&kvm_lock);
2393
2394         kvm_usage_count++;
2395         if (kvm_usage_count == 1) {
2396                 atomic_set(&hardware_enable_failed, 0);
2397                 on_each_cpu(hardware_enable_nolock, NULL, 1);
2398
2399                 if (atomic_read(&hardware_enable_failed)) {
2400                         hardware_disable_all_nolock();
2401                         r = -EBUSY;
2402                 }
2403         }
2404
2405         raw_spin_unlock(&kvm_lock);
2406
2407         return r;
2408 }
2409
2410 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
2411                            void *v)
2412 {
2413         int cpu = (long)v;
2414
2415         if (!kvm_usage_count)
2416                 return NOTIFY_OK;
2417
2418         val &= ~CPU_TASKS_FROZEN;
2419         switch (val) {
2420         case CPU_DYING:
2421                 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
2422                        cpu);
2423                 hardware_disable(NULL);
2424                 break;
2425         case CPU_STARTING:
2426                 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
2427                        cpu);
2428                 hardware_enable(NULL);
2429                 break;
2430         }
2431         return NOTIFY_OK;
2432 }
2433
2434
2435 asmlinkage void kvm_spurious_fault(void)
2436 {
2437         /* Fault while not rebooting.  We want the trace. */
2438         BUG();
2439 }
2440 EXPORT_SYMBOL_GPL(kvm_spurious_fault);
2441
2442 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
2443                       void *v)
2444 {
2445         /*
2446          * Some (well, at least mine) BIOSes hang on reboot if
2447          * in vmx root mode.
2448          *
2449          * And Intel TXT required VMX off for all cpu when system shutdown.
2450          */
2451         printk(KERN_INFO "kvm: exiting hardware virtualization\n");
2452         kvm_rebooting = true;
2453         on_each_cpu(hardware_disable_nolock, NULL, 1);
2454         return NOTIFY_OK;
2455 }
2456
2457 static struct notifier_block kvm_reboot_notifier = {
2458         .notifier_call = kvm_reboot,
2459         .priority = 0,
2460 };
2461
2462 static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
2463 {
2464         int i;
2465
2466         for (i = 0; i < bus->dev_count; i++) {
2467                 struct kvm_io_device *pos = bus->range[i].dev;
2468
2469                 kvm_iodevice_destructor(pos);
2470         }
2471         kfree(bus);
2472 }
2473
2474 int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
2475 {
2476         const struct kvm_io_range *r1 = p1;
2477         const struct kvm_io_range *r2 = p2;
2478
2479         if (r1->addr < r2->addr)
2480                 return -1;
2481         if (r1->addr + r1->len > r2->addr + r2->len)
2482                 return 1;
2483         return 0;
2484 }
2485
2486 int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev,
2487                           gpa_t addr, int len)
2488 {
2489         bus->range[bus->dev_count++] = (struct kvm_io_range) {
2490                 .addr = addr,
2491                 .len = len,
2492                 .dev = dev,
2493         };
2494
2495         sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range),
2496                 kvm_io_bus_sort_cmp, NULL);
2497
2498         return 0;
2499 }
2500
2501 int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
2502                              gpa_t addr, int len)
2503 {
2504         struct kvm_io_range *range, key;
2505         int off;
2506
2507         key = (struct kvm_io_range) {
2508                 .addr = addr,
2509                 .len = len,
2510         };
2511
2512         range = bsearch(&key, bus->range, bus->dev_count,
2513                         sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
2514         if (range == NULL)
2515                 return -ENOENT;
2516
2517         off = range - bus->range;
2518
2519         while (off > 0 && kvm_io_bus_sort_cmp(&key, &bus->range[off-1]) == 0)
2520                 off--;
2521
2522         return off;
2523 }
2524
2525 /* kvm_io_bus_write - called under kvm->slots_lock */
2526 int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2527                      int len, const void *val)
2528 {
2529         int idx;
2530         struct kvm_io_bus *bus;
2531         struct kvm_io_range range;
2532
2533         range = (struct kvm_io_range) {
2534                 .addr = addr,
2535                 .len = len,
2536         };
2537
2538         bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2539         idx = kvm_io_bus_get_first_dev(bus, addr, len);
2540         if (idx < 0)
2541                 return -EOPNOTSUPP;
2542
2543         while (idx < bus->dev_count &&
2544                 kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) {
2545                 if (!kvm_iodevice_write(bus->range[idx].dev, addr, len, val))
2546                         return 0;
2547                 idx++;
2548         }
2549
2550         return -EOPNOTSUPP;
2551 }
2552
2553 /* kvm_io_bus_read - called under kvm->slots_lock */
2554 int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2555                     int len, void *val)
2556 {
2557         int idx;
2558         struct kvm_io_bus *bus;
2559         struct kvm_io_range range;
2560
2561         range = (struct kvm_io_range) {
2562                 .addr = addr,
2563                 .len = len,
2564         };
2565
2566         bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2567         idx = kvm_io_bus_get_first_dev(bus, addr, len);
2568         if (idx < 0)
2569                 return -EOPNOTSUPP;
2570
2571         while (idx < bus->dev_count &&
2572                 kvm_io_bus_sort_cmp(&range, &bus->range[idx]) == 0) {
2573                 if (!kvm_iodevice_read(bus->range[idx].dev, addr, len, val))
2574                         return 0;
2575                 idx++;
2576         }
2577
2578         return -EOPNOTSUPP;
2579 }
2580
2581 /* Caller must hold slots_lock. */
2582 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2583                             int len, struct kvm_io_device *dev)
2584 {
2585         struct kvm_io_bus *new_bus, *bus;
2586
2587         bus = kvm->buses[bus_idx];
2588         if (bus->dev_count > NR_IOBUS_DEVS - 1)
2589                 return -ENOSPC;
2590
2591         new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count + 1) *
2592                           sizeof(struct kvm_io_range)), GFP_KERNEL);
2593         if (!new_bus)
2594                 return -ENOMEM;
2595         memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count *
2596                sizeof(struct kvm_io_range)));
2597         kvm_io_bus_insert_dev(new_bus, dev, addr, len);
2598         rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
2599         synchronize_srcu_expedited(&kvm->srcu);
2600         kfree(bus);
2601
2602         return 0;
2603 }
2604
2605 /* Caller must hold slots_lock. */
2606 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
2607                               struct kvm_io_device *dev)
2608 {
2609         int i, r;
2610         struct kvm_io_bus *new_bus, *bus;
2611
2612         bus = kvm->buses[bus_idx];
2613         r = -ENOENT;
2614         for (i = 0; i < bus->dev_count; i++)
2615                 if (bus->range[i].dev == dev) {
2616                         r = 0;
2617                         break;
2618                 }
2619
2620         if (r)
2621                 return r;
2622
2623         new_bus = kzalloc(sizeof(*bus) + ((bus->dev_count - 1) *
2624                           sizeof(struct kvm_io_range)), GFP_KERNEL);
2625         if (!new_bus)
2626                 return -ENOMEM;
2627
2628         memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
2629         new_bus->dev_count--;
2630         memcpy(new_bus->range + i, bus->range + i + 1,
2631                (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
2632
2633         rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
2634         synchronize_srcu_expedited(&kvm->srcu);
2635         kfree(bus);
2636         return r;
2637 }
2638
2639 static struct notifier_block kvm_cpu_notifier = {
2640         .notifier_call = kvm_cpu_hotplug,
2641 };
2642
2643 static int vm_stat_get(void *_offset, u64 *val)
2644 {
2645         unsigned offset = (long)_offset;
2646         struct kvm *kvm;
2647
2648         *val = 0;
2649         raw_spin_lock(&kvm_lock);
2650         list_for_each_entry(kvm, &vm_list, vm_list)
2651                 *val += *(u32 *)((void *)kvm + offset);
2652         raw_spin_unlock(&kvm_lock);
2653         return 0;
2654 }
2655
2656 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
2657
2658 static int vcpu_stat_get(void *_offset, u64 *val)
2659 {
2660         unsigned offset = (long)_offset;
2661         struct kvm *kvm;
2662         struct kvm_vcpu *vcpu;
2663         int i;
2664
2665         *val = 0;
2666         raw_spin_lock(&kvm_lock);
2667         list_for_each_entry(kvm, &vm_list, vm_list)
2668                 kvm_for_each_vcpu(i, vcpu, kvm)
2669                         *val += *(u32 *)((void *)vcpu + offset);
2670
2671         raw_spin_unlock(&kvm_lock);
2672         return 0;
2673 }
2674
2675 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
2676
2677 static const struct file_operations *stat_fops[] = {
2678         [KVM_STAT_VCPU] = &vcpu_stat_fops,
2679         [KVM_STAT_VM]   = &vm_stat_fops,
2680 };
2681
2682 static int kvm_init_debug(void)
2683 {
2684         int r = -EFAULT;
2685         struct kvm_stats_debugfs_item *p;
2686
2687         kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
2688         if (kvm_debugfs_dir == NULL)
2689                 goto out;
2690
2691         for (p = debugfs_entries; p->name; ++p) {
2692                 p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
2693                                                 (void *)(long)p->offset,
2694                                                 stat_fops[p->kind]);
2695                 if (p->dentry == NULL)
2696                         goto out_dir;
2697         }
2698
2699         return 0;
2700
2701 out_dir:
2702         debugfs_remove_recursive(kvm_debugfs_dir);
2703 out:
2704         return r;
2705 }
2706
2707 static void kvm_exit_debug(void)
2708 {
2709         struct kvm_stats_debugfs_item *p;
2710
2711         for (p = debugfs_entries; p->name; ++p)
2712                 debugfs_remove(p->dentry);
2713         debugfs_remove(kvm_debugfs_dir);
2714 }
2715
2716 static int kvm_suspend(void)
2717 {
2718         if (kvm_usage_count)
2719                 hardware_disable_nolock(NULL);
2720         return 0;
2721 }
2722
2723 static void kvm_resume(void)
2724 {
2725         if (kvm_usage_count) {
2726                 WARN_ON(raw_spin_is_locked(&kvm_lock));
2727                 hardware_enable_nolock(NULL);
2728         }
2729 }
2730
2731 static struct syscore_ops kvm_syscore_ops = {
2732         .suspend = kvm_suspend,
2733         .resume = kvm_resume,
2734 };
2735
2736 static inline
2737 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
2738 {
2739         return container_of(pn, struct kvm_vcpu, preempt_notifier);
2740 }
2741
2742 static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
2743 {
2744         struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
2745
2746         kvm_arch_vcpu_load(vcpu, cpu);
2747 }
2748
2749 static void kvm_sched_out(struct preempt_notifier *pn,
2750                           struct task_struct *next)
2751 {
2752         struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
2753
2754         kvm_arch_vcpu_put(vcpu);
2755 }
2756
2757 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
2758                   struct module *module)
2759 {
2760         int r;
2761         int cpu;
2762
2763         r = kvm_arch_init(opaque);
2764         if (r)
2765                 goto out_fail;
2766
2767         if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
2768                 r = -ENOMEM;
2769                 goto out_free_0;
2770         }
2771
2772         r = kvm_arch_hardware_setup();
2773         if (r < 0)
2774                 goto out_free_0a;
2775
2776         for_each_online_cpu(cpu) {
2777                 smp_call_function_single(cpu,
2778                                 kvm_arch_check_processor_compat,
2779                                 &r, 1);
2780                 if (r < 0)
2781                         goto out_free_1;
2782         }
2783
2784         r = register_cpu_notifier(&kvm_cpu_notifier);
2785         if (r)
2786                 goto out_free_2;
2787         register_reboot_notifier(&kvm_reboot_notifier);
2788
2789         /* A kmem cache lets us meet the alignment requirements of fx_save. */
2790         if (!vcpu_align)
2791                 vcpu_align = __alignof__(struct kvm_vcpu);
2792         kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
2793                                            0, NULL);
2794         if (!kvm_vcpu_cache) {
2795                 r = -ENOMEM;
2796                 goto out_free_3;
2797         }
2798
2799         r = kvm_async_pf_init();
2800         if (r)
2801                 goto out_free;
2802
2803         kvm_chardev_ops.owner = module;
2804         kvm_vm_fops.owner = module;
2805         kvm_vcpu_fops.owner = module;
2806
2807         r = misc_register(&kvm_dev);
2808         if (r) {
2809                 printk(KERN_ERR "kvm: misc device register failed\n");
2810                 goto out_unreg;
2811         }
2812
2813         register_syscore_ops(&kvm_syscore_ops);
2814
2815         kvm_preempt_ops.sched_in = kvm_sched_in;
2816         kvm_preempt_ops.sched_out = kvm_sched_out;
2817
2818         r = kvm_init_debug();
2819         if (r) {
2820                 printk(KERN_ERR "kvm: create debugfs files failed\n");
2821                 goto out_undebugfs;
2822         }
2823
2824         return 0;
2825
2826 out_undebugfs:
2827         unregister_syscore_ops(&kvm_syscore_ops);
2828 out_unreg:
2829         kvm_async_pf_deinit();
2830 out_free:
2831         kmem_cache_destroy(kvm_vcpu_cache);
2832 out_free_3:
2833         unregister_reboot_notifier(&kvm_reboot_notifier);
2834         unregister_cpu_notifier(&kvm_cpu_notifier);
2835 out_free_2:
2836 out_free_1:
2837         kvm_arch_hardware_unsetup();
2838 out_free_0a:
2839         free_cpumask_var(cpus_hardware_enabled);
2840 out_free_0:
2841         kvm_arch_exit();
2842 out_fail:
2843         return r;
2844 }
2845 EXPORT_SYMBOL_GPL(kvm_init);
2846
2847 void kvm_exit(void)
2848 {
2849         kvm_exit_debug();
2850         misc_deregister(&kvm_dev);
2851         kmem_cache_destroy(kvm_vcpu_cache);
2852         kvm_async_pf_deinit();
2853         unregister_syscore_ops(&kvm_syscore_ops);
2854         unregister_reboot_notifier(&kvm_reboot_notifier);
2855         unregister_cpu_notifier(&kvm_cpu_notifier);
2856         on_each_cpu(hardware_disable_nolock, NULL, 1);
2857         kvm_arch_hardware_unsetup();
2858         kvm_arch_exit();
2859         free_cpumask_var(cpus_hardware_enabled);
2860 }
2861 EXPORT_SYMBOL_GPL(kvm_exit);