KVM: Per-vcpu inodes
[linux-2.6.git] / drivers / kvm / kvm_main.c
1 /*
2  * Kernel-based Virtual Machine driver for Linux
3  *
4  * This module enables machines with Intel VT-x extensions to run virtual
5  * machines without emulation or binary translation.
6  *
7  * Copyright (C) 2006 Qumranet, Inc.
8  *
9  * Authors:
10  *   Avi Kivity   <avi@qumranet.com>
11  *   Yaniv Kamay  <yaniv@qumranet.com>
12  *
13  * This work is licensed under the terms of the GNU GPL, version 2.  See
14  * the COPYING file in the top-level directory.
15  *
16  */
17
18 #include "kvm.h"
19
20 #include <linux/kvm.h>
21 #include <linux/module.h>
22 #include <linux/errno.h>
23 #include <asm/processor.h>
24 #include <linux/percpu.h>
25 #include <linux/gfp.h>
26 #include <asm/msr.h>
27 #include <linux/mm.h>
28 #include <linux/miscdevice.h>
29 #include <linux/vmalloc.h>
30 #include <asm/uaccess.h>
31 #include <linux/reboot.h>
32 #include <asm/io.h>
33 #include <linux/debugfs.h>
34 #include <linux/highmem.h>
35 #include <linux/file.h>
36 #include <asm/desc.h>
37 #include <linux/sysdev.h>
38 #include <linux/cpu.h>
39 #include <linux/file.h>
40 #include <linux/fs.h>
41 #include <linux/mount.h>
42
43 #include "x86_emulate.h"
44 #include "segment_descriptor.h"
45
46 MODULE_AUTHOR("Qumranet");
47 MODULE_LICENSE("GPL");
48
49 static DEFINE_SPINLOCK(kvm_lock);
50 static LIST_HEAD(vm_list);
51
52 struct kvm_arch_ops *kvm_arch_ops;
53 struct kvm_stat kvm_stat;
54 EXPORT_SYMBOL_GPL(kvm_stat);
55
56 static struct kvm_stats_debugfs_item {
57         const char *name;
58         u32 *data;
59         struct dentry *dentry;
60 } debugfs_entries[] = {
61         { "pf_fixed", &kvm_stat.pf_fixed },
62         { "pf_guest", &kvm_stat.pf_guest },
63         { "tlb_flush", &kvm_stat.tlb_flush },
64         { "invlpg", &kvm_stat.invlpg },
65         { "exits", &kvm_stat.exits },
66         { "io_exits", &kvm_stat.io_exits },
67         { "mmio_exits", &kvm_stat.mmio_exits },
68         { "signal_exits", &kvm_stat.signal_exits },
69         { "irq_window", &kvm_stat.irq_window_exits },
70         { "halt_exits", &kvm_stat.halt_exits },
71         { "request_irq", &kvm_stat.request_irq_exits },
72         { "irq_exits", &kvm_stat.irq_exits },
73         { NULL, NULL }
74 };
75
76 static struct dentry *debugfs_dir;
77
78 #define KVMFS_MAGIC 0x19700426
79 struct vfsmount *kvmfs_mnt;
80
81 #define MAX_IO_MSRS 256
82
83 #define CR0_RESEVED_BITS 0xffffffff1ffaffc0ULL
84 #define LMSW_GUEST_MASK 0x0eULL
85 #define CR4_RESEVED_BITS (~((1ULL << 11) - 1))
86 #define CR8_RESEVED_BITS (~0x0fULL)
87 #define EFER_RESERVED_BITS 0xfffffffffffff2fe
88
89 #ifdef CONFIG_X86_64
90 // LDT or TSS descriptor in the GDT. 16 bytes.
91 struct segment_descriptor_64 {
92         struct segment_descriptor s;
93         u32 base_higher;
94         u32 pad_zero;
95 };
96
97 #endif
98
99 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
100                            unsigned long arg);
101
102 static struct inode *kvmfs_inode(struct file_operations *fops)
103 {
104         int error = -ENOMEM;
105         struct inode *inode = new_inode(kvmfs_mnt->mnt_sb);
106
107         if (!inode)
108                 goto eexit_1;
109
110         inode->i_fop = fops;
111
112         /*
113          * Mark the inode dirty from the very beginning,
114          * that way it will never be moved to the dirty
115          * list because mark_inode_dirty() will think
116          * that it already _is_ on the dirty list.
117          */
118         inode->i_state = I_DIRTY;
119         inode->i_mode = S_IRUSR | S_IWUSR;
120         inode->i_uid = current->fsuid;
121         inode->i_gid = current->fsgid;
122         inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
123         return inode;
124
125 eexit_1:
126         return ERR_PTR(error);
127 }
128
129 static struct file *kvmfs_file(struct inode *inode, void *private_data)
130 {
131         struct file *file = get_empty_filp();
132
133         if (!file)
134                 return ERR_PTR(-ENFILE);
135
136         file->f_path.mnt = mntget(kvmfs_mnt);
137         file->f_path.dentry = d_alloc_anon(inode);
138         if (!file->f_path.dentry)
139                 return ERR_PTR(-ENOMEM);
140         file->f_mapping = inode->i_mapping;
141
142         file->f_pos = 0;
143         file->f_flags = O_RDWR;
144         file->f_op = inode->i_fop;
145         file->f_mode = FMODE_READ | FMODE_WRITE;
146         file->f_version = 0;
147         file->private_data = private_data;
148         return file;
149 }
150
151 unsigned long segment_base(u16 selector)
152 {
153         struct descriptor_table gdt;
154         struct segment_descriptor *d;
155         unsigned long table_base;
156         typedef unsigned long ul;
157         unsigned long v;
158
159         if (selector == 0)
160                 return 0;
161
162         asm ("sgdt %0" : "=m"(gdt));
163         table_base = gdt.base;
164
165         if (selector & 4) {           /* from ldt */
166                 u16 ldt_selector;
167
168                 asm ("sldt %0" : "=g"(ldt_selector));
169                 table_base = segment_base(ldt_selector);
170         }
171         d = (struct segment_descriptor *)(table_base + (selector & ~7));
172         v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24);
173 #ifdef CONFIG_X86_64
174         if (d->system == 0
175             && (d->type == 2 || d->type == 9 || d->type == 11))
176                 v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32;
177 #endif
178         return v;
179 }
180 EXPORT_SYMBOL_GPL(segment_base);
181
182 static inline int valid_vcpu(int n)
183 {
184         return likely(n >= 0 && n < KVM_MAX_VCPUS);
185 }
186
187 int kvm_read_guest(struct kvm_vcpu *vcpu, gva_t addr, unsigned long size,
188                    void *dest)
189 {
190         unsigned char *host_buf = dest;
191         unsigned long req_size = size;
192
193         while (size) {
194                 hpa_t paddr;
195                 unsigned now;
196                 unsigned offset;
197                 hva_t guest_buf;
198
199                 paddr = gva_to_hpa(vcpu, addr);
200
201                 if (is_error_hpa(paddr))
202                         break;
203
204                 guest_buf = (hva_t)kmap_atomic(
205                                         pfn_to_page(paddr >> PAGE_SHIFT),
206                                         KM_USER0);
207                 offset = addr & ~PAGE_MASK;
208                 guest_buf |= offset;
209                 now = min(size, PAGE_SIZE - offset);
210                 memcpy(host_buf, (void*)guest_buf, now);
211                 host_buf += now;
212                 addr += now;
213                 size -= now;
214                 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
215         }
216         return req_size - size;
217 }
218 EXPORT_SYMBOL_GPL(kvm_read_guest);
219
220 int kvm_write_guest(struct kvm_vcpu *vcpu, gva_t addr, unsigned long size,
221                     void *data)
222 {
223         unsigned char *host_buf = data;
224         unsigned long req_size = size;
225
226         while (size) {
227                 hpa_t paddr;
228                 unsigned now;
229                 unsigned offset;
230                 hva_t guest_buf;
231
232                 paddr = gva_to_hpa(vcpu, addr);
233
234                 if (is_error_hpa(paddr))
235                         break;
236
237                 guest_buf = (hva_t)kmap_atomic(
238                                 pfn_to_page(paddr >> PAGE_SHIFT), KM_USER0);
239                 offset = addr & ~PAGE_MASK;
240                 guest_buf |= offset;
241                 now = min(size, PAGE_SIZE - offset);
242                 memcpy((void*)guest_buf, host_buf, now);
243                 host_buf += now;
244                 addr += now;
245                 size -= now;
246                 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
247         }
248         return req_size - size;
249 }
250 EXPORT_SYMBOL_GPL(kvm_write_guest);
251
252 /*
253  * Switches to specified vcpu, until a matching vcpu_put()
254  */
255 static void vcpu_load(struct kvm_vcpu *vcpu)
256 {
257         mutex_lock(&vcpu->mutex);
258         kvm_arch_ops->vcpu_load(vcpu);
259 }
260
261 /*
262  * Switches to specified vcpu, until a matching vcpu_put(). Will return NULL
263  * if the slot is not populated.
264  */
265 static struct kvm_vcpu *vcpu_load_slot(struct kvm *kvm, int slot)
266 {
267         struct kvm_vcpu *vcpu = &kvm->vcpus[slot];
268
269         mutex_lock(&vcpu->mutex);
270         if (!vcpu->vmcs) {
271                 mutex_unlock(&vcpu->mutex);
272                 return NULL;
273         }
274         kvm_arch_ops->vcpu_load(vcpu);
275         return vcpu;
276 }
277
278 static void vcpu_put(struct kvm_vcpu *vcpu)
279 {
280         kvm_arch_ops->vcpu_put(vcpu);
281         mutex_unlock(&vcpu->mutex);
282 }
283
284 static struct kvm *kvm_create_vm(void)
285 {
286         struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
287         int i;
288
289         if (!kvm)
290                 return ERR_PTR(-ENOMEM);
291
292         spin_lock_init(&kvm->lock);
293         INIT_LIST_HEAD(&kvm->active_mmu_pages);
294         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
295                 struct kvm_vcpu *vcpu = &kvm->vcpus[i];
296
297                 mutex_init(&vcpu->mutex);
298                 vcpu->cpu = -1;
299                 vcpu->kvm = kvm;
300                 vcpu->mmu.root_hpa = INVALID_PAGE;
301                 INIT_LIST_HEAD(&vcpu->free_pages);
302                 spin_lock(&kvm_lock);
303                 list_add(&kvm->vm_list, &vm_list);
304                 spin_unlock(&kvm_lock);
305         }
306         return kvm;
307 }
308
309 static int kvm_dev_open(struct inode *inode, struct file *filp)
310 {
311         return 0;
312 }
313
314 /*
315  * Free any memory in @free but not in @dont.
316  */
317 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
318                                   struct kvm_memory_slot *dont)
319 {
320         int i;
321
322         if (!dont || free->phys_mem != dont->phys_mem)
323                 if (free->phys_mem) {
324                         for (i = 0; i < free->npages; ++i)
325                                 if (free->phys_mem[i])
326                                         __free_page(free->phys_mem[i]);
327                         vfree(free->phys_mem);
328                 }
329
330         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
331                 vfree(free->dirty_bitmap);
332
333         free->phys_mem = NULL;
334         free->npages = 0;
335         free->dirty_bitmap = NULL;
336 }
337
338 static void kvm_free_physmem(struct kvm *kvm)
339 {
340         int i;
341
342         for (i = 0; i < kvm->nmemslots; ++i)
343                 kvm_free_physmem_slot(&kvm->memslots[i], NULL);
344 }
345
346 static void kvm_free_vcpu(struct kvm_vcpu *vcpu)
347 {
348         if (!vcpu->vmcs)
349                 return;
350
351         vcpu_load(vcpu);
352         kvm_mmu_destroy(vcpu);
353         vcpu_put(vcpu);
354         kvm_arch_ops->vcpu_free(vcpu);
355 }
356
357 static void kvm_free_vcpus(struct kvm *kvm)
358 {
359         unsigned int i;
360
361         for (i = 0; i < KVM_MAX_VCPUS; ++i)
362                 kvm_free_vcpu(&kvm->vcpus[i]);
363 }
364
365 static int kvm_dev_release(struct inode *inode, struct file *filp)
366 {
367         return 0;
368 }
369
370 static void kvm_destroy_vm(struct kvm *kvm)
371 {
372         spin_lock(&kvm_lock);
373         list_del(&kvm->vm_list);
374         spin_unlock(&kvm_lock);
375         kvm_free_vcpus(kvm);
376         kvm_free_physmem(kvm);
377         kfree(kvm);
378 }
379
380 static int kvm_vm_release(struct inode *inode, struct file *filp)
381 {
382         struct kvm *kvm = filp->private_data;
383
384         kvm_destroy_vm(kvm);
385         return 0;
386 }
387
388 static void inject_gp(struct kvm_vcpu *vcpu)
389 {
390         kvm_arch_ops->inject_gp(vcpu, 0);
391 }
392
393 /*
394  * Load the pae pdptrs.  Return true is they are all valid.
395  */
396 static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
397 {
398         gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
399         unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
400         int i;
401         u64 pdpte;
402         u64 *pdpt;
403         int ret;
404         struct kvm_memory_slot *memslot;
405
406         spin_lock(&vcpu->kvm->lock);
407         memslot = gfn_to_memslot(vcpu->kvm, pdpt_gfn);
408         /* FIXME: !memslot - emulate? 0xff? */
409         pdpt = kmap_atomic(gfn_to_page(memslot, pdpt_gfn), KM_USER0);
410
411         ret = 1;
412         for (i = 0; i < 4; ++i) {
413                 pdpte = pdpt[offset + i];
414                 if ((pdpte & 1) && (pdpte & 0xfffffff0000001e6ull)) {
415                         ret = 0;
416                         goto out;
417                 }
418         }
419
420         for (i = 0; i < 4; ++i)
421                 vcpu->pdptrs[i] = pdpt[offset + i];
422
423 out:
424         kunmap_atomic(pdpt, KM_USER0);
425         spin_unlock(&vcpu->kvm->lock);
426
427         return ret;
428 }
429
430 void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
431 {
432         if (cr0 & CR0_RESEVED_BITS) {
433                 printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
434                        cr0, vcpu->cr0);
435                 inject_gp(vcpu);
436                 return;
437         }
438
439         if ((cr0 & CR0_NW_MASK) && !(cr0 & CR0_CD_MASK)) {
440                 printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
441                 inject_gp(vcpu);
442                 return;
443         }
444
445         if ((cr0 & CR0_PG_MASK) && !(cr0 & CR0_PE_MASK)) {
446                 printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
447                        "and a clear PE flag\n");
448                 inject_gp(vcpu);
449                 return;
450         }
451
452         if (!is_paging(vcpu) && (cr0 & CR0_PG_MASK)) {
453 #ifdef CONFIG_X86_64
454                 if ((vcpu->shadow_efer & EFER_LME)) {
455                         int cs_db, cs_l;
456
457                         if (!is_pae(vcpu)) {
458                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
459                                        "in long mode while PAE is disabled\n");
460                                 inject_gp(vcpu);
461                                 return;
462                         }
463                         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
464                         if (cs_l) {
465                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
466                                        "in long mode while CS.L == 1\n");
467                                 inject_gp(vcpu);
468                                 return;
469
470                         }
471                 } else
472 #endif
473                 if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) {
474                         printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
475                                "reserved bits\n");
476                         inject_gp(vcpu);
477                         return;
478                 }
479
480         }
481
482         kvm_arch_ops->set_cr0(vcpu, cr0);
483         vcpu->cr0 = cr0;
484
485         spin_lock(&vcpu->kvm->lock);
486         kvm_mmu_reset_context(vcpu);
487         spin_unlock(&vcpu->kvm->lock);
488         return;
489 }
490 EXPORT_SYMBOL_GPL(set_cr0);
491
492 void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
493 {
494         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
495         set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
496 }
497 EXPORT_SYMBOL_GPL(lmsw);
498
499 void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
500 {
501         if (cr4 & CR4_RESEVED_BITS) {
502                 printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
503                 inject_gp(vcpu);
504                 return;
505         }
506
507         if (is_long_mode(vcpu)) {
508                 if (!(cr4 & CR4_PAE_MASK)) {
509                         printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
510                                "in long mode\n");
511                         inject_gp(vcpu);
512                         return;
513                 }
514         } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & CR4_PAE_MASK)
515                    && !load_pdptrs(vcpu, vcpu->cr3)) {
516                 printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
517                 inject_gp(vcpu);
518         }
519
520         if (cr4 & CR4_VMXE_MASK) {
521                 printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
522                 inject_gp(vcpu);
523                 return;
524         }
525         kvm_arch_ops->set_cr4(vcpu, cr4);
526         spin_lock(&vcpu->kvm->lock);
527         kvm_mmu_reset_context(vcpu);
528         spin_unlock(&vcpu->kvm->lock);
529 }
530 EXPORT_SYMBOL_GPL(set_cr4);
531
532 void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
533 {
534         if (is_long_mode(vcpu)) {
535                 if (cr3 & CR3_L_MODE_RESEVED_BITS) {
536                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
537                         inject_gp(vcpu);
538                         return;
539                 }
540         } else {
541                 if (cr3 & CR3_RESEVED_BITS) {
542                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
543                         inject_gp(vcpu);
544                         return;
545                 }
546                 if (is_paging(vcpu) && is_pae(vcpu) &&
547                     !load_pdptrs(vcpu, cr3)) {
548                         printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
549                                "reserved bits\n");
550                         inject_gp(vcpu);
551                         return;
552                 }
553         }
554
555         vcpu->cr3 = cr3;
556         spin_lock(&vcpu->kvm->lock);
557         /*
558          * Does the new cr3 value map to physical memory? (Note, we
559          * catch an invalid cr3 even in real-mode, because it would
560          * cause trouble later on when we turn on paging anyway.)
561          *
562          * A real CPU would silently accept an invalid cr3 and would
563          * attempt to use it - with largely undefined (and often hard
564          * to debug) behavior on the guest side.
565          */
566         if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
567                 inject_gp(vcpu);
568         else
569                 vcpu->mmu.new_cr3(vcpu);
570         spin_unlock(&vcpu->kvm->lock);
571 }
572 EXPORT_SYMBOL_GPL(set_cr3);
573
574 void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
575 {
576         if ( cr8 & CR8_RESEVED_BITS) {
577                 printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
578                 inject_gp(vcpu);
579                 return;
580         }
581         vcpu->cr8 = cr8;
582 }
583 EXPORT_SYMBOL_GPL(set_cr8);
584
585 void fx_init(struct kvm_vcpu *vcpu)
586 {
587         struct __attribute__ ((__packed__)) fx_image_s {
588                 u16 control; //fcw
589                 u16 status; //fsw
590                 u16 tag; // ftw
591                 u16 opcode; //fop
592                 u64 ip; // fpu ip
593                 u64 operand;// fpu dp
594                 u32 mxcsr;
595                 u32 mxcsr_mask;
596
597         } *fx_image;
598
599         fx_save(vcpu->host_fx_image);
600         fpu_init();
601         fx_save(vcpu->guest_fx_image);
602         fx_restore(vcpu->host_fx_image);
603
604         fx_image = (struct fx_image_s *)vcpu->guest_fx_image;
605         fx_image->mxcsr = 0x1f80;
606         memset(vcpu->guest_fx_image + sizeof(struct fx_image_s),
607                0, FX_IMAGE_SIZE - sizeof(struct fx_image_s));
608 }
609 EXPORT_SYMBOL_GPL(fx_init);
610
611 /*
612  * Allocate some memory and give it an address in the guest physical address
613  * space.
614  *
615  * Discontiguous memory is allowed, mostly for framebuffers.
616  */
617 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
618                                           struct kvm_memory_region *mem)
619 {
620         int r;
621         gfn_t base_gfn;
622         unsigned long npages;
623         unsigned long i;
624         struct kvm_memory_slot *memslot;
625         struct kvm_memory_slot old, new;
626         int memory_config_version;
627
628         r = -EINVAL;
629         /* General sanity checks */
630         if (mem->memory_size & (PAGE_SIZE - 1))
631                 goto out;
632         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
633                 goto out;
634         if (mem->slot >= KVM_MEMORY_SLOTS)
635                 goto out;
636         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
637                 goto out;
638
639         memslot = &kvm->memslots[mem->slot];
640         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
641         npages = mem->memory_size >> PAGE_SHIFT;
642
643         if (!npages)
644                 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
645
646 raced:
647         spin_lock(&kvm->lock);
648
649         memory_config_version = kvm->memory_config_version;
650         new = old = *memslot;
651
652         new.base_gfn = base_gfn;
653         new.npages = npages;
654         new.flags = mem->flags;
655
656         /* Disallow changing a memory slot's size. */
657         r = -EINVAL;
658         if (npages && old.npages && npages != old.npages)
659                 goto out_unlock;
660
661         /* Check for overlaps */
662         r = -EEXIST;
663         for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
664                 struct kvm_memory_slot *s = &kvm->memslots[i];
665
666                 if (s == memslot)
667                         continue;
668                 if (!((base_gfn + npages <= s->base_gfn) ||
669                       (base_gfn >= s->base_gfn + s->npages)))
670                         goto out_unlock;
671         }
672         /*
673          * Do memory allocations outside lock.  memory_config_version will
674          * detect any races.
675          */
676         spin_unlock(&kvm->lock);
677
678         /* Deallocate if slot is being removed */
679         if (!npages)
680                 new.phys_mem = NULL;
681
682         /* Free page dirty bitmap if unneeded */
683         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
684                 new.dirty_bitmap = NULL;
685
686         r = -ENOMEM;
687
688         /* Allocate if a slot is being created */
689         if (npages && !new.phys_mem) {
690                 new.phys_mem = vmalloc(npages * sizeof(struct page *));
691
692                 if (!new.phys_mem)
693                         goto out_free;
694
695                 memset(new.phys_mem, 0, npages * sizeof(struct page *));
696                 for (i = 0; i < npages; ++i) {
697                         new.phys_mem[i] = alloc_page(GFP_HIGHUSER
698                                                      | __GFP_ZERO);
699                         if (!new.phys_mem[i])
700                                 goto out_free;
701                         set_page_private(new.phys_mem[i],0);
702                 }
703         }
704
705         /* Allocate page dirty bitmap if needed */
706         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
707                 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
708
709                 new.dirty_bitmap = vmalloc(dirty_bytes);
710                 if (!new.dirty_bitmap)
711                         goto out_free;
712                 memset(new.dirty_bitmap, 0, dirty_bytes);
713         }
714
715         spin_lock(&kvm->lock);
716
717         if (memory_config_version != kvm->memory_config_version) {
718                 spin_unlock(&kvm->lock);
719                 kvm_free_physmem_slot(&new, &old);
720                 goto raced;
721         }
722
723         r = -EAGAIN;
724         if (kvm->busy)
725                 goto out_unlock;
726
727         if (mem->slot >= kvm->nmemslots)
728                 kvm->nmemslots = mem->slot + 1;
729
730         *memslot = new;
731         ++kvm->memory_config_version;
732
733         spin_unlock(&kvm->lock);
734
735         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
736                 struct kvm_vcpu *vcpu;
737
738                 vcpu = vcpu_load_slot(kvm, i);
739                 if (!vcpu)
740                         continue;
741                 kvm_mmu_reset_context(vcpu);
742                 vcpu_put(vcpu);
743         }
744
745         kvm_free_physmem_slot(&old, &new);
746         return 0;
747
748 out_unlock:
749         spin_unlock(&kvm->lock);
750 out_free:
751         kvm_free_physmem_slot(&new, &old);
752 out:
753         return r;
754 }
755
756 static void do_remove_write_access(struct kvm_vcpu *vcpu, int slot)
757 {
758         spin_lock(&vcpu->kvm->lock);
759         kvm_mmu_slot_remove_write_access(vcpu, slot);
760         spin_unlock(&vcpu->kvm->lock);
761 }
762
763 /*
764  * Get (and clear) the dirty memory log for a memory slot.
765  */
766 static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
767                                       struct kvm_dirty_log *log)
768 {
769         struct kvm_memory_slot *memslot;
770         int r, i;
771         int n;
772         int cleared;
773         unsigned long any = 0;
774
775         spin_lock(&kvm->lock);
776
777         /*
778          * Prevent changes to guest memory configuration even while the lock
779          * is not taken.
780          */
781         ++kvm->busy;
782         spin_unlock(&kvm->lock);
783         r = -EINVAL;
784         if (log->slot >= KVM_MEMORY_SLOTS)
785                 goto out;
786
787         memslot = &kvm->memslots[log->slot];
788         r = -ENOENT;
789         if (!memslot->dirty_bitmap)
790                 goto out;
791
792         n = ALIGN(memslot->npages, 8) / 8;
793
794         for (i = 0; !any && i < n; ++i)
795                 any = memslot->dirty_bitmap[i];
796
797         r = -EFAULT;
798         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
799                 goto out;
800
801         if (any) {
802                 cleared = 0;
803                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
804                         struct kvm_vcpu *vcpu;
805
806                         vcpu = vcpu_load_slot(kvm, i);
807                         if (!vcpu)
808                                 continue;
809                         if (!cleared) {
810                                 do_remove_write_access(vcpu, log->slot);
811                                 memset(memslot->dirty_bitmap, 0, n);
812                                 cleared = 1;
813                         }
814                         kvm_arch_ops->tlb_flush(vcpu);
815                         vcpu_put(vcpu);
816                 }
817         }
818
819         r = 0;
820
821 out:
822         spin_lock(&kvm->lock);
823         --kvm->busy;
824         spin_unlock(&kvm->lock);
825         return r;
826 }
827
828 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
829 {
830         int i;
831
832         for (i = 0; i < kvm->nmemslots; ++i) {
833                 struct kvm_memory_slot *memslot = &kvm->memslots[i];
834
835                 if (gfn >= memslot->base_gfn
836                     && gfn < memslot->base_gfn + memslot->npages)
837                         return memslot;
838         }
839         return NULL;
840 }
841 EXPORT_SYMBOL_GPL(gfn_to_memslot);
842
843 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
844 {
845         int i;
846         struct kvm_memory_slot *memslot = NULL;
847         unsigned long rel_gfn;
848
849         for (i = 0; i < kvm->nmemslots; ++i) {
850                 memslot = &kvm->memslots[i];
851
852                 if (gfn >= memslot->base_gfn
853                     && gfn < memslot->base_gfn + memslot->npages) {
854
855                         if (!memslot || !memslot->dirty_bitmap)
856                                 return;
857
858                         rel_gfn = gfn - memslot->base_gfn;
859
860                         /* avoid RMW */
861                         if (!test_bit(rel_gfn, memslot->dirty_bitmap))
862                                 set_bit(rel_gfn, memslot->dirty_bitmap);
863                         return;
864                 }
865         }
866 }
867
868 static int emulator_read_std(unsigned long addr,
869                              unsigned long *val,
870                              unsigned int bytes,
871                              struct x86_emulate_ctxt *ctxt)
872 {
873         struct kvm_vcpu *vcpu = ctxt->vcpu;
874         void *data = val;
875
876         while (bytes) {
877                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
878                 unsigned offset = addr & (PAGE_SIZE-1);
879                 unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
880                 unsigned long pfn;
881                 struct kvm_memory_slot *memslot;
882                 void *page;
883
884                 if (gpa == UNMAPPED_GVA)
885                         return X86EMUL_PROPAGATE_FAULT;
886                 pfn = gpa >> PAGE_SHIFT;
887                 memslot = gfn_to_memslot(vcpu->kvm, pfn);
888                 if (!memslot)
889                         return X86EMUL_UNHANDLEABLE;
890                 page = kmap_atomic(gfn_to_page(memslot, pfn), KM_USER0);
891
892                 memcpy(data, page + offset, tocopy);
893
894                 kunmap_atomic(page, KM_USER0);
895
896                 bytes -= tocopy;
897                 data += tocopy;
898                 addr += tocopy;
899         }
900
901         return X86EMUL_CONTINUE;
902 }
903
904 static int emulator_write_std(unsigned long addr,
905                               unsigned long val,
906                               unsigned int bytes,
907                               struct x86_emulate_ctxt *ctxt)
908 {
909         printk(KERN_ERR "emulator_write_std: addr %lx n %d\n",
910                addr, bytes);
911         return X86EMUL_UNHANDLEABLE;
912 }
913
914 static int emulator_read_emulated(unsigned long addr,
915                                   unsigned long *val,
916                                   unsigned int bytes,
917                                   struct x86_emulate_ctxt *ctxt)
918 {
919         struct kvm_vcpu *vcpu = ctxt->vcpu;
920
921         if (vcpu->mmio_read_completed) {
922                 memcpy(val, vcpu->mmio_data, bytes);
923                 vcpu->mmio_read_completed = 0;
924                 return X86EMUL_CONTINUE;
925         } else if (emulator_read_std(addr, val, bytes, ctxt)
926                    == X86EMUL_CONTINUE)
927                 return X86EMUL_CONTINUE;
928         else {
929                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
930
931                 if (gpa == UNMAPPED_GVA)
932                         return X86EMUL_PROPAGATE_FAULT;
933                 vcpu->mmio_needed = 1;
934                 vcpu->mmio_phys_addr = gpa;
935                 vcpu->mmio_size = bytes;
936                 vcpu->mmio_is_write = 0;
937
938                 return X86EMUL_UNHANDLEABLE;
939         }
940 }
941
942 static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
943                                unsigned long val, int bytes)
944 {
945         struct kvm_memory_slot *m;
946         struct page *page;
947         void *virt;
948
949         if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT))
950                 return 0;
951         m = gfn_to_memslot(vcpu->kvm, gpa >> PAGE_SHIFT);
952         if (!m)
953                 return 0;
954         page = gfn_to_page(m, gpa >> PAGE_SHIFT);
955         kvm_mmu_pre_write(vcpu, gpa, bytes);
956         virt = kmap_atomic(page, KM_USER0);
957         memcpy(virt + offset_in_page(gpa), &val, bytes);
958         kunmap_atomic(virt, KM_USER0);
959         kvm_mmu_post_write(vcpu, gpa, bytes);
960         return 1;
961 }
962
963 static int emulator_write_emulated(unsigned long addr,
964                                    unsigned long val,
965                                    unsigned int bytes,
966                                    struct x86_emulate_ctxt *ctxt)
967 {
968         struct kvm_vcpu *vcpu = ctxt->vcpu;
969         gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
970
971         if (gpa == UNMAPPED_GVA)
972                 return X86EMUL_PROPAGATE_FAULT;
973
974         if (emulator_write_phys(vcpu, gpa, val, bytes))
975                 return X86EMUL_CONTINUE;
976
977         vcpu->mmio_needed = 1;
978         vcpu->mmio_phys_addr = gpa;
979         vcpu->mmio_size = bytes;
980         vcpu->mmio_is_write = 1;
981         memcpy(vcpu->mmio_data, &val, bytes);
982
983         return X86EMUL_CONTINUE;
984 }
985
986 static int emulator_cmpxchg_emulated(unsigned long addr,
987                                      unsigned long old,
988                                      unsigned long new,
989                                      unsigned int bytes,
990                                      struct x86_emulate_ctxt *ctxt)
991 {
992         static int reported;
993
994         if (!reported) {
995                 reported = 1;
996                 printk(KERN_WARNING "kvm: emulating exchange as write\n");
997         }
998         return emulator_write_emulated(addr, new, bytes, ctxt);
999 }
1000
1001 #ifdef CONFIG_X86_32
1002
1003 static int emulator_cmpxchg8b_emulated(unsigned long addr,
1004                                        unsigned long old_lo,
1005                                        unsigned long old_hi,
1006                                        unsigned long new_lo,
1007                                        unsigned long new_hi,
1008                                        struct x86_emulate_ctxt *ctxt)
1009 {
1010         static int reported;
1011         int r;
1012
1013         if (!reported) {
1014                 reported = 1;
1015                 printk(KERN_WARNING "kvm: emulating exchange8b as write\n");
1016         }
1017         r = emulator_write_emulated(addr, new_lo, 4, ctxt);
1018         if (r != X86EMUL_CONTINUE)
1019                 return r;
1020         return emulator_write_emulated(addr+4, new_hi, 4, ctxt);
1021 }
1022
1023 #endif
1024
1025 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
1026 {
1027         return kvm_arch_ops->get_segment_base(vcpu, seg);
1028 }
1029
1030 int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
1031 {
1032         return X86EMUL_CONTINUE;
1033 }
1034
1035 int emulate_clts(struct kvm_vcpu *vcpu)
1036 {
1037         unsigned long cr0;
1038
1039         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1040         cr0 = vcpu->cr0 & ~CR0_TS_MASK;
1041         kvm_arch_ops->set_cr0(vcpu, cr0);
1042         return X86EMUL_CONTINUE;
1043 }
1044
1045 int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest)
1046 {
1047         struct kvm_vcpu *vcpu = ctxt->vcpu;
1048
1049         switch (dr) {
1050         case 0 ... 3:
1051                 *dest = kvm_arch_ops->get_dr(vcpu, dr);
1052                 return X86EMUL_CONTINUE;
1053         default:
1054                 printk(KERN_DEBUG "%s: unexpected dr %u\n",
1055                        __FUNCTION__, dr);
1056                 return X86EMUL_UNHANDLEABLE;
1057         }
1058 }
1059
1060 int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
1061 {
1062         unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
1063         int exception;
1064
1065         kvm_arch_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
1066         if (exception) {
1067                 /* FIXME: better handling */
1068                 return X86EMUL_UNHANDLEABLE;
1069         }
1070         return X86EMUL_CONTINUE;
1071 }
1072
1073 static void report_emulation_failure(struct x86_emulate_ctxt *ctxt)
1074 {
1075         static int reported;
1076         u8 opcodes[4];
1077         unsigned long rip = ctxt->vcpu->rip;
1078         unsigned long rip_linear;
1079
1080         rip_linear = rip + get_segment_base(ctxt->vcpu, VCPU_SREG_CS);
1081
1082         if (reported)
1083                 return;
1084
1085         emulator_read_std(rip_linear, (void *)opcodes, 4, ctxt);
1086
1087         printk(KERN_ERR "emulation failed but !mmio_needed?"
1088                " rip %lx %02x %02x %02x %02x\n",
1089                rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
1090         reported = 1;
1091 }
1092
1093 struct x86_emulate_ops emulate_ops = {
1094         .read_std            = emulator_read_std,
1095         .write_std           = emulator_write_std,
1096         .read_emulated       = emulator_read_emulated,
1097         .write_emulated      = emulator_write_emulated,
1098         .cmpxchg_emulated    = emulator_cmpxchg_emulated,
1099 #ifdef CONFIG_X86_32
1100         .cmpxchg8b_emulated  = emulator_cmpxchg8b_emulated,
1101 #endif
1102 };
1103
1104 int emulate_instruction(struct kvm_vcpu *vcpu,
1105                         struct kvm_run *run,
1106                         unsigned long cr2,
1107                         u16 error_code)
1108 {
1109         struct x86_emulate_ctxt emulate_ctxt;
1110         int r;
1111         int cs_db, cs_l;
1112
1113         kvm_arch_ops->cache_regs(vcpu);
1114
1115         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
1116
1117         emulate_ctxt.vcpu = vcpu;
1118         emulate_ctxt.eflags = kvm_arch_ops->get_rflags(vcpu);
1119         emulate_ctxt.cr2 = cr2;
1120         emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM)
1121                 ? X86EMUL_MODE_REAL : cs_l
1122                 ? X86EMUL_MODE_PROT64 : cs_db
1123                 ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
1124
1125         if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
1126                 emulate_ctxt.cs_base = 0;
1127                 emulate_ctxt.ds_base = 0;
1128                 emulate_ctxt.es_base = 0;
1129                 emulate_ctxt.ss_base = 0;
1130         } else {
1131                 emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS);
1132                 emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS);
1133                 emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES);
1134                 emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS);
1135         }
1136
1137         emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS);
1138         emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS);
1139
1140         vcpu->mmio_is_write = 0;
1141         r = x86_emulate_memop(&emulate_ctxt, &emulate_ops);
1142
1143         if ((r || vcpu->mmio_is_write) && run) {
1144                 run->mmio.phys_addr = vcpu->mmio_phys_addr;
1145                 memcpy(run->mmio.data, vcpu->mmio_data, 8);
1146                 run->mmio.len = vcpu->mmio_size;
1147                 run->mmio.is_write = vcpu->mmio_is_write;
1148         }
1149
1150         if (r) {
1151                 if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
1152                         return EMULATE_DONE;
1153                 if (!vcpu->mmio_needed) {
1154                         report_emulation_failure(&emulate_ctxt);
1155                         return EMULATE_FAIL;
1156                 }
1157                 return EMULATE_DO_MMIO;
1158         }
1159
1160         kvm_arch_ops->decache_regs(vcpu);
1161         kvm_arch_ops->set_rflags(vcpu, emulate_ctxt.eflags);
1162
1163         if (vcpu->mmio_is_write)
1164                 return EMULATE_DO_MMIO;
1165
1166         return EMULATE_DONE;
1167 }
1168 EXPORT_SYMBOL_GPL(emulate_instruction);
1169
1170 int kvm_hypercall(struct kvm_vcpu *vcpu, struct kvm_run *run)
1171 {
1172         unsigned long nr, a0, a1, a2, a3, a4, a5, ret;
1173
1174         kvm_arch_ops->decache_regs(vcpu);
1175         ret = -KVM_EINVAL;
1176 #ifdef CONFIG_X86_64
1177         if (is_long_mode(vcpu)) {
1178                 nr = vcpu->regs[VCPU_REGS_RAX];
1179                 a0 = vcpu->regs[VCPU_REGS_RDI];
1180                 a1 = vcpu->regs[VCPU_REGS_RSI];
1181                 a2 = vcpu->regs[VCPU_REGS_RDX];
1182                 a3 = vcpu->regs[VCPU_REGS_RCX];
1183                 a4 = vcpu->regs[VCPU_REGS_R8];
1184                 a5 = vcpu->regs[VCPU_REGS_R9];
1185         } else
1186 #endif
1187         {
1188                 nr = vcpu->regs[VCPU_REGS_RBX] & -1u;
1189                 a0 = vcpu->regs[VCPU_REGS_RAX] & -1u;
1190                 a1 = vcpu->regs[VCPU_REGS_RCX] & -1u;
1191                 a2 = vcpu->regs[VCPU_REGS_RDX] & -1u;
1192                 a3 = vcpu->regs[VCPU_REGS_RSI] & -1u;
1193                 a4 = vcpu->regs[VCPU_REGS_RDI] & -1u;
1194                 a5 = vcpu->regs[VCPU_REGS_RBP] & -1u;
1195         }
1196         switch (nr) {
1197         default:
1198                 ;
1199         }
1200         vcpu->regs[VCPU_REGS_RAX] = ret;
1201         kvm_arch_ops->cache_regs(vcpu);
1202         return 1;
1203 }
1204 EXPORT_SYMBOL_GPL(kvm_hypercall);
1205
1206 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
1207 {
1208         return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
1209 }
1210
1211 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1212 {
1213         struct descriptor_table dt = { limit, base };
1214
1215         kvm_arch_ops->set_gdt(vcpu, &dt);
1216 }
1217
1218 void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1219 {
1220         struct descriptor_table dt = { limit, base };
1221
1222         kvm_arch_ops->set_idt(vcpu, &dt);
1223 }
1224
1225 void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
1226                    unsigned long *rflags)
1227 {
1228         lmsw(vcpu, msw);
1229         *rflags = kvm_arch_ops->get_rflags(vcpu);
1230 }
1231
1232 unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
1233 {
1234         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1235         switch (cr) {
1236         case 0:
1237                 return vcpu->cr0;
1238         case 2:
1239                 return vcpu->cr2;
1240         case 3:
1241                 return vcpu->cr3;
1242         case 4:
1243                 return vcpu->cr4;
1244         default:
1245                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1246                 return 0;
1247         }
1248 }
1249
1250 void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
1251                      unsigned long *rflags)
1252 {
1253         switch (cr) {
1254         case 0:
1255                 set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
1256                 *rflags = kvm_arch_ops->get_rflags(vcpu);
1257                 break;
1258         case 2:
1259                 vcpu->cr2 = val;
1260                 break;
1261         case 3:
1262                 set_cr3(vcpu, val);
1263                 break;
1264         case 4:
1265                 set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
1266                 break;
1267         default:
1268                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1269         }
1270 }
1271
1272 /*
1273  * Register the para guest with the host:
1274  */
1275 static int vcpu_register_para(struct kvm_vcpu *vcpu, gpa_t para_state_gpa)
1276 {
1277         struct kvm_vcpu_para_state *para_state;
1278         hpa_t para_state_hpa, hypercall_hpa;
1279         struct page *para_state_page;
1280         unsigned char *hypercall;
1281         gpa_t hypercall_gpa;
1282
1283         printk(KERN_DEBUG "kvm: guest trying to enter paravirtual mode\n");
1284         printk(KERN_DEBUG ".... para_state_gpa: %08Lx\n", para_state_gpa);
1285
1286         /*
1287          * Needs to be page aligned:
1288          */
1289         if (para_state_gpa != PAGE_ALIGN(para_state_gpa))
1290                 goto err_gp;
1291
1292         para_state_hpa = gpa_to_hpa(vcpu, para_state_gpa);
1293         printk(KERN_DEBUG ".... para_state_hpa: %08Lx\n", para_state_hpa);
1294         if (is_error_hpa(para_state_hpa))
1295                 goto err_gp;
1296
1297         para_state_page = pfn_to_page(para_state_hpa >> PAGE_SHIFT);
1298         para_state = kmap_atomic(para_state_page, KM_USER0);
1299
1300         printk(KERN_DEBUG "....  guest version: %d\n", para_state->guest_version);
1301         printk(KERN_DEBUG "....           size: %d\n", para_state->size);
1302
1303         para_state->host_version = KVM_PARA_API_VERSION;
1304         /*
1305          * We cannot support guests that try to register themselves
1306          * with a newer API version than the host supports:
1307          */
1308         if (para_state->guest_version > KVM_PARA_API_VERSION) {
1309                 para_state->ret = -KVM_EINVAL;
1310                 goto err_kunmap_skip;
1311         }
1312
1313         hypercall_gpa = para_state->hypercall_gpa;
1314         hypercall_hpa = gpa_to_hpa(vcpu, hypercall_gpa);
1315         printk(KERN_DEBUG ".... hypercall_hpa: %08Lx\n", hypercall_hpa);
1316         if (is_error_hpa(hypercall_hpa)) {
1317                 para_state->ret = -KVM_EINVAL;
1318                 goto err_kunmap_skip;
1319         }
1320
1321         printk(KERN_DEBUG "kvm: para guest successfully registered.\n");
1322         vcpu->para_state_page = para_state_page;
1323         vcpu->para_state_gpa = para_state_gpa;
1324         vcpu->hypercall_gpa = hypercall_gpa;
1325
1326         hypercall = kmap_atomic(pfn_to_page(hypercall_hpa >> PAGE_SHIFT),
1327                                 KM_USER1) + (hypercall_hpa & ~PAGE_MASK);
1328         kvm_arch_ops->patch_hypercall(vcpu, hypercall);
1329         kunmap_atomic(hypercall, KM_USER1);
1330
1331         para_state->ret = 0;
1332 err_kunmap_skip:
1333         kunmap_atomic(para_state, KM_USER0);
1334         return 0;
1335 err_gp:
1336         return 1;
1337 }
1338
1339 int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
1340 {
1341         u64 data;
1342
1343         switch (msr) {
1344         case 0xc0010010: /* SYSCFG */
1345         case 0xc0010015: /* HWCR */
1346         case MSR_IA32_PLATFORM_ID:
1347         case MSR_IA32_P5_MC_ADDR:
1348         case MSR_IA32_P5_MC_TYPE:
1349         case MSR_IA32_MC0_CTL:
1350         case MSR_IA32_MCG_STATUS:
1351         case MSR_IA32_MCG_CAP:
1352         case MSR_IA32_MC0_MISC:
1353         case MSR_IA32_MC0_MISC+4:
1354         case MSR_IA32_MC0_MISC+8:
1355         case MSR_IA32_MC0_MISC+12:
1356         case MSR_IA32_MC0_MISC+16:
1357         case MSR_IA32_UCODE_REV:
1358         case MSR_IA32_PERF_STATUS:
1359                 /* MTRR registers */
1360         case 0xfe:
1361         case 0x200 ... 0x2ff:
1362                 data = 0;
1363                 break;
1364         case 0xcd: /* fsb frequency */
1365                 data = 3;
1366                 break;
1367         case MSR_IA32_APICBASE:
1368                 data = vcpu->apic_base;
1369                 break;
1370         case MSR_IA32_MISC_ENABLE:
1371                 data = vcpu->ia32_misc_enable_msr;
1372                 break;
1373 #ifdef CONFIG_X86_64
1374         case MSR_EFER:
1375                 data = vcpu->shadow_efer;
1376                 break;
1377 #endif
1378         default:
1379                 printk(KERN_ERR "kvm: unhandled rdmsr: 0x%x\n", msr);
1380                 return 1;
1381         }
1382         *pdata = data;
1383         return 0;
1384 }
1385 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
1386
1387 /*
1388  * Reads an msr value (of 'msr_index') into 'pdata'.
1389  * Returns 0 on success, non-0 otherwise.
1390  * Assumes vcpu_load() was already called.
1391  */
1392 static int get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
1393 {
1394         return kvm_arch_ops->get_msr(vcpu, msr_index, pdata);
1395 }
1396
1397 #ifdef CONFIG_X86_64
1398
1399 static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
1400 {
1401         if (efer & EFER_RESERVED_BITS) {
1402                 printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
1403                        efer);
1404                 inject_gp(vcpu);
1405                 return;
1406         }
1407
1408         if (is_paging(vcpu)
1409             && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
1410                 printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
1411                 inject_gp(vcpu);
1412                 return;
1413         }
1414
1415         kvm_arch_ops->set_efer(vcpu, efer);
1416
1417         efer &= ~EFER_LMA;
1418         efer |= vcpu->shadow_efer & EFER_LMA;
1419
1420         vcpu->shadow_efer = efer;
1421 }
1422
1423 #endif
1424
1425 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1426 {
1427         switch (msr) {
1428 #ifdef CONFIG_X86_64
1429         case MSR_EFER:
1430                 set_efer(vcpu, data);
1431                 break;
1432 #endif
1433         case MSR_IA32_MC0_STATUS:
1434                 printk(KERN_WARNING "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
1435                        __FUNCTION__, data);
1436                 break;
1437         case MSR_IA32_UCODE_REV:
1438         case MSR_IA32_UCODE_WRITE:
1439         case 0x200 ... 0x2ff: /* MTRRs */
1440                 break;
1441         case MSR_IA32_APICBASE:
1442                 vcpu->apic_base = data;
1443                 break;
1444         case MSR_IA32_MISC_ENABLE:
1445                 vcpu->ia32_misc_enable_msr = data;
1446                 break;
1447         /*
1448          * This is the 'probe whether the host is KVM' logic:
1449          */
1450         case MSR_KVM_API_MAGIC:
1451                 return vcpu_register_para(vcpu, data);
1452
1453         default:
1454                 printk(KERN_ERR "kvm: unhandled wrmsr: 0x%x\n", msr);
1455                 return 1;
1456         }
1457         return 0;
1458 }
1459 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
1460
1461 /*
1462  * Writes msr value into into the appropriate "register".
1463  * Returns 0 on success, non-0 otherwise.
1464  * Assumes vcpu_load() was already called.
1465  */
1466 static int set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
1467 {
1468         return kvm_arch_ops->set_msr(vcpu, msr_index, data);
1469 }
1470
1471 void kvm_resched(struct kvm_vcpu *vcpu)
1472 {
1473         vcpu_put(vcpu);
1474         cond_resched();
1475         vcpu_load(vcpu);
1476 }
1477 EXPORT_SYMBOL_GPL(kvm_resched);
1478
1479 void load_msrs(struct vmx_msr_entry *e, int n)
1480 {
1481         int i;
1482
1483         for (i = 0; i < n; ++i)
1484                 wrmsrl(e[i].index, e[i].data);
1485 }
1486 EXPORT_SYMBOL_GPL(load_msrs);
1487
1488 void save_msrs(struct vmx_msr_entry *e, int n)
1489 {
1490         int i;
1491
1492         for (i = 0; i < n; ++i)
1493                 rdmsrl(e[i].index, e[i].data);
1494 }
1495 EXPORT_SYMBOL_GPL(save_msrs);
1496
1497 static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1498 {
1499         int r;
1500
1501         vcpu_load(vcpu);
1502
1503         /* re-sync apic's tpr */
1504         vcpu->cr8 = kvm_run->cr8;
1505
1506         if (kvm_run->emulated) {
1507                 kvm_arch_ops->skip_emulated_instruction(vcpu);
1508                 kvm_run->emulated = 0;
1509         }
1510
1511         if (kvm_run->mmio_completed) {
1512                 memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
1513                 vcpu->mmio_read_completed = 1;
1514         }
1515
1516         vcpu->mmio_needed = 0;
1517
1518         r = kvm_arch_ops->run(vcpu, kvm_run);
1519
1520         vcpu_put(vcpu);
1521         return r;
1522 }
1523
1524 static int kvm_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu,
1525                                    struct kvm_regs *regs)
1526 {
1527         vcpu_load(vcpu);
1528
1529         kvm_arch_ops->cache_regs(vcpu);
1530
1531         regs->rax = vcpu->regs[VCPU_REGS_RAX];
1532         regs->rbx = vcpu->regs[VCPU_REGS_RBX];
1533         regs->rcx = vcpu->regs[VCPU_REGS_RCX];
1534         regs->rdx = vcpu->regs[VCPU_REGS_RDX];
1535         regs->rsi = vcpu->regs[VCPU_REGS_RSI];
1536         regs->rdi = vcpu->regs[VCPU_REGS_RDI];
1537         regs->rsp = vcpu->regs[VCPU_REGS_RSP];
1538         regs->rbp = vcpu->regs[VCPU_REGS_RBP];
1539 #ifdef CONFIG_X86_64
1540         regs->r8 = vcpu->regs[VCPU_REGS_R8];
1541         regs->r9 = vcpu->regs[VCPU_REGS_R9];
1542         regs->r10 = vcpu->regs[VCPU_REGS_R10];
1543         regs->r11 = vcpu->regs[VCPU_REGS_R11];
1544         regs->r12 = vcpu->regs[VCPU_REGS_R12];
1545         regs->r13 = vcpu->regs[VCPU_REGS_R13];
1546         regs->r14 = vcpu->regs[VCPU_REGS_R14];
1547         regs->r15 = vcpu->regs[VCPU_REGS_R15];
1548 #endif
1549
1550         regs->rip = vcpu->rip;
1551         regs->rflags = kvm_arch_ops->get_rflags(vcpu);
1552
1553         /*
1554          * Don't leak debug flags in case they were set for guest debugging
1555          */
1556         if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
1557                 regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1558
1559         vcpu_put(vcpu);
1560
1561         return 0;
1562 }
1563
1564 static int kvm_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu,
1565                                    struct kvm_regs *regs)
1566 {
1567         vcpu_load(vcpu);
1568
1569         vcpu->regs[VCPU_REGS_RAX] = regs->rax;
1570         vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
1571         vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
1572         vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
1573         vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
1574         vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
1575         vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
1576         vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
1577 #ifdef CONFIG_X86_64
1578         vcpu->regs[VCPU_REGS_R8] = regs->r8;
1579         vcpu->regs[VCPU_REGS_R9] = regs->r9;
1580         vcpu->regs[VCPU_REGS_R10] = regs->r10;
1581         vcpu->regs[VCPU_REGS_R11] = regs->r11;
1582         vcpu->regs[VCPU_REGS_R12] = regs->r12;
1583         vcpu->regs[VCPU_REGS_R13] = regs->r13;
1584         vcpu->regs[VCPU_REGS_R14] = regs->r14;
1585         vcpu->regs[VCPU_REGS_R15] = regs->r15;
1586 #endif
1587
1588         vcpu->rip = regs->rip;
1589         kvm_arch_ops->set_rflags(vcpu, regs->rflags);
1590
1591         kvm_arch_ops->decache_regs(vcpu);
1592
1593         vcpu_put(vcpu);
1594
1595         return 0;
1596 }
1597
1598 static void get_segment(struct kvm_vcpu *vcpu,
1599                         struct kvm_segment *var, int seg)
1600 {
1601         return kvm_arch_ops->get_segment(vcpu, var, seg);
1602 }
1603
1604 static int kvm_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1605                                     struct kvm_sregs *sregs)
1606 {
1607         struct descriptor_table dt;
1608
1609         vcpu_load(vcpu);
1610
1611         get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
1612         get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
1613         get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
1614         get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
1615         get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
1616         get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
1617
1618         get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1619         get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1620
1621         kvm_arch_ops->get_idt(vcpu, &dt);
1622         sregs->idt.limit = dt.limit;
1623         sregs->idt.base = dt.base;
1624         kvm_arch_ops->get_gdt(vcpu, &dt);
1625         sregs->gdt.limit = dt.limit;
1626         sregs->gdt.base = dt.base;
1627
1628         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1629         sregs->cr0 = vcpu->cr0;
1630         sregs->cr2 = vcpu->cr2;
1631         sregs->cr3 = vcpu->cr3;
1632         sregs->cr4 = vcpu->cr4;
1633         sregs->cr8 = vcpu->cr8;
1634         sregs->efer = vcpu->shadow_efer;
1635         sregs->apic_base = vcpu->apic_base;
1636
1637         memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
1638                sizeof sregs->interrupt_bitmap);
1639
1640         vcpu_put(vcpu);
1641
1642         return 0;
1643 }
1644
1645 static void set_segment(struct kvm_vcpu *vcpu,
1646                         struct kvm_segment *var, int seg)
1647 {
1648         return kvm_arch_ops->set_segment(vcpu, var, seg);
1649 }
1650
1651 static int kvm_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1652                                     struct kvm_sregs *sregs)
1653 {
1654         int mmu_reset_needed = 0;
1655         int i;
1656         struct descriptor_table dt;
1657
1658         vcpu_load(vcpu);
1659
1660         set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
1661         set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
1662         set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
1663         set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
1664         set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
1665         set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
1666
1667         set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1668         set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1669
1670         dt.limit = sregs->idt.limit;
1671         dt.base = sregs->idt.base;
1672         kvm_arch_ops->set_idt(vcpu, &dt);
1673         dt.limit = sregs->gdt.limit;
1674         dt.base = sregs->gdt.base;
1675         kvm_arch_ops->set_gdt(vcpu, &dt);
1676
1677         vcpu->cr2 = sregs->cr2;
1678         mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
1679         vcpu->cr3 = sregs->cr3;
1680
1681         vcpu->cr8 = sregs->cr8;
1682
1683         mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
1684 #ifdef CONFIG_X86_64
1685         kvm_arch_ops->set_efer(vcpu, sregs->efer);
1686 #endif
1687         vcpu->apic_base = sregs->apic_base;
1688
1689         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1690
1691         mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
1692         kvm_arch_ops->set_cr0_no_modeswitch(vcpu, sregs->cr0);
1693
1694         mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
1695         kvm_arch_ops->set_cr4(vcpu, sregs->cr4);
1696         if (!is_long_mode(vcpu) && is_pae(vcpu))
1697                 load_pdptrs(vcpu, vcpu->cr3);
1698
1699         if (mmu_reset_needed)
1700                 kvm_mmu_reset_context(vcpu);
1701
1702         memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
1703                sizeof vcpu->irq_pending);
1704         vcpu->irq_summary = 0;
1705         for (i = 0; i < NR_IRQ_WORDS; ++i)
1706                 if (vcpu->irq_pending[i])
1707                         __set_bit(i, &vcpu->irq_summary);
1708
1709         vcpu_put(vcpu);
1710
1711         return 0;
1712 }
1713
1714 /*
1715  * List of msr numbers which we expose to userspace through KVM_GET_MSRS
1716  * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
1717  *
1718  * This list is modified at module load time to reflect the
1719  * capabilities of the host cpu.
1720  */
1721 static u32 msrs_to_save[] = {
1722         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
1723         MSR_K6_STAR,
1724 #ifdef CONFIG_X86_64
1725         MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
1726 #endif
1727         MSR_IA32_TIME_STAMP_COUNTER,
1728 };
1729
1730 static unsigned num_msrs_to_save;
1731
1732 static u32 emulated_msrs[] = {
1733         MSR_IA32_MISC_ENABLE,
1734 };
1735
1736 static __init void kvm_init_msr_list(void)
1737 {
1738         u32 dummy[2];
1739         unsigned i, j;
1740
1741         for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
1742                 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
1743                         continue;
1744                 if (j < i)
1745                         msrs_to_save[j] = msrs_to_save[i];
1746                 j++;
1747         }
1748         num_msrs_to_save = j;
1749 }
1750
1751 /*
1752  * Adapt set_msr() to msr_io()'s calling convention
1753  */
1754 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1755 {
1756         return set_msr(vcpu, index, *data);
1757 }
1758
1759 /*
1760  * Read or write a bunch of msrs. All parameters are kernel addresses.
1761  *
1762  * @return number of msrs set successfully.
1763  */
1764 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
1765                     struct kvm_msr_entry *entries,
1766                     int (*do_msr)(struct kvm_vcpu *vcpu,
1767                                   unsigned index, u64 *data))
1768 {
1769         int i;
1770
1771         vcpu_load(vcpu);
1772
1773         for (i = 0; i < msrs->nmsrs; ++i)
1774                 if (do_msr(vcpu, entries[i].index, &entries[i].data))
1775                         break;
1776
1777         vcpu_put(vcpu);
1778
1779         return i;
1780 }
1781
1782 /*
1783  * Read or write a bunch of msrs. Parameters are user addresses.
1784  *
1785  * @return number of msrs set successfully.
1786  */
1787 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
1788                   int (*do_msr)(struct kvm_vcpu *vcpu,
1789                                 unsigned index, u64 *data),
1790                   int writeback)
1791 {
1792         struct kvm_msrs msrs;
1793         struct kvm_msr_entry *entries;
1794         int r, n;
1795         unsigned size;
1796
1797         r = -EFAULT;
1798         if (copy_from_user(&msrs, user_msrs, sizeof msrs))
1799                 goto out;
1800
1801         r = -E2BIG;
1802         if (msrs.nmsrs >= MAX_IO_MSRS)
1803                 goto out;
1804
1805         r = -ENOMEM;
1806         size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
1807         entries = vmalloc(size);
1808         if (!entries)
1809                 goto out;
1810
1811         r = -EFAULT;
1812         if (copy_from_user(entries, user_msrs->entries, size))
1813                 goto out_free;
1814
1815         r = n = __msr_io(vcpu, &msrs, entries, do_msr);
1816         if (r < 0)
1817                 goto out_free;
1818
1819         r = -EFAULT;
1820         if (writeback && copy_to_user(user_msrs->entries, entries, size))
1821                 goto out_free;
1822
1823         r = n;
1824
1825 out_free:
1826         vfree(entries);
1827 out:
1828         return r;
1829 }
1830
1831 /*
1832  * Translate a guest virtual address to a guest physical address.
1833  */
1834 static int kvm_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1835                                     struct kvm_translation *tr)
1836 {
1837         unsigned long vaddr = tr->linear_address;
1838         gpa_t gpa;
1839
1840         vcpu_load(vcpu);
1841         spin_lock(&vcpu->kvm->lock);
1842         gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
1843         tr->physical_address = gpa;
1844         tr->valid = gpa != UNMAPPED_GVA;
1845         tr->writeable = 1;
1846         tr->usermode = 0;
1847         spin_unlock(&vcpu->kvm->lock);
1848         vcpu_put(vcpu);
1849
1850         return 0;
1851 }
1852
1853 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
1854                                     struct kvm_interrupt *irq)
1855 {
1856         if (irq->irq < 0 || irq->irq >= 256)
1857                 return -EINVAL;
1858         vcpu_load(vcpu);
1859
1860         set_bit(irq->irq, vcpu->irq_pending);
1861         set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);
1862
1863         vcpu_put(vcpu);
1864
1865         return 0;
1866 }
1867
1868 static int kvm_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu,
1869                                       struct kvm_debug_guest *dbg)
1870 {
1871         int r;
1872
1873         vcpu_load(vcpu);
1874
1875         r = kvm_arch_ops->set_guest_debug(vcpu, dbg);
1876
1877         vcpu_put(vcpu);
1878
1879         return r;
1880 }
1881
1882 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
1883 {
1884         struct kvm_vcpu *vcpu = filp->private_data;
1885
1886         fput(vcpu->kvm->filp);
1887         return 0;
1888 }
1889
1890 static struct file_operations kvm_vcpu_fops = {
1891         .release        = kvm_vcpu_release,
1892         .unlocked_ioctl = kvm_vcpu_ioctl,
1893         .compat_ioctl   = kvm_vcpu_ioctl,
1894 };
1895
1896 /*
1897  * Allocates an inode for the vcpu.
1898  */
1899 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
1900 {
1901         int fd, r;
1902         struct inode *inode;
1903         struct file *file;
1904
1905         atomic_inc(&vcpu->kvm->filp->f_count);
1906         inode = kvmfs_inode(&kvm_vcpu_fops);
1907         if (IS_ERR(inode)) {
1908                 r = PTR_ERR(inode);
1909                 goto out1;
1910         }
1911
1912         file = kvmfs_file(inode, vcpu);
1913         if (IS_ERR(file)) {
1914                 r = PTR_ERR(file);
1915                 goto out2;
1916         }
1917
1918         r = get_unused_fd();
1919         if (r < 0)
1920                 goto out3;
1921         fd = r;
1922         fd_install(fd, file);
1923
1924         return fd;
1925
1926 out3:
1927         fput(file);
1928 out2:
1929         iput(inode);
1930 out1:
1931         fput(vcpu->kvm->filp);
1932         return r;
1933 }
1934
1935 /*
1936  * Creates some virtual cpus.  Good luck creating more than one.
1937  */
1938 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
1939 {
1940         int r;
1941         struct kvm_vcpu *vcpu;
1942
1943         r = -EINVAL;
1944         if (!valid_vcpu(n))
1945                 goto out;
1946
1947         vcpu = &kvm->vcpus[n];
1948
1949         mutex_lock(&vcpu->mutex);
1950
1951         if (vcpu->vmcs) {
1952                 mutex_unlock(&vcpu->mutex);
1953                 return -EEXIST;
1954         }
1955
1956         vcpu->host_fx_image = (char*)ALIGN((hva_t)vcpu->fx_buf,
1957                                            FX_IMAGE_ALIGN);
1958         vcpu->guest_fx_image = vcpu->host_fx_image + FX_IMAGE_SIZE;
1959
1960         r = kvm_arch_ops->vcpu_create(vcpu);
1961         if (r < 0)
1962                 goto out_free_vcpus;
1963
1964         r = kvm_mmu_create(vcpu);
1965         if (r < 0)
1966                 goto out_free_vcpus;
1967
1968         kvm_arch_ops->vcpu_load(vcpu);
1969         r = kvm_mmu_setup(vcpu);
1970         if (r >= 0)
1971                 r = kvm_arch_ops->vcpu_setup(vcpu);
1972         vcpu_put(vcpu);
1973
1974         if (r < 0)
1975                 goto out_free_vcpus;
1976
1977         r = create_vcpu_fd(vcpu);
1978         if (r < 0)
1979                 goto out_free_vcpus;
1980
1981         return r;
1982
1983 out_free_vcpus:
1984         kvm_free_vcpu(vcpu);
1985         mutex_unlock(&vcpu->mutex);
1986 out:
1987         return r;
1988 }
1989
1990 static long kvm_vcpu_ioctl(struct file *filp,
1991                            unsigned int ioctl, unsigned long arg)
1992 {
1993         struct kvm_vcpu *vcpu = filp->private_data;
1994         void __user *argp = (void __user *)arg;
1995         int r = -EINVAL;
1996
1997         switch (ioctl) {
1998         case KVM_RUN: {
1999                 struct kvm_run kvm_run;
2000
2001                 r = -EFAULT;
2002                 if (copy_from_user(&kvm_run, argp, sizeof kvm_run))
2003                         goto out;
2004                 r = kvm_vcpu_ioctl_run(vcpu, &kvm_run);
2005                 if (r < 0 &&  r != -EINTR)
2006                         goto out;
2007                 if (copy_to_user(argp, &kvm_run, sizeof kvm_run)) {
2008                         r = -EFAULT;
2009                         goto out;
2010                 }
2011                 break;
2012         }
2013         case KVM_GET_REGS: {
2014                 struct kvm_regs kvm_regs;
2015
2016                 memset(&kvm_regs, 0, sizeof kvm_regs);
2017                 r = kvm_vcpu_ioctl_get_regs(vcpu, &kvm_regs);
2018                 if (r)
2019                         goto out;
2020                 r = -EFAULT;
2021                 if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
2022                         goto out;
2023                 r = 0;
2024                 break;
2025         }
2026         case KVM_SET_REGS: {
2027                 struct kvm_regs kvm_regs;
2028
2029                 r = -EFAULT;
2030                 if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
2031                         goto out;
2032                 r = kvm_vcpu_ioctl_set_regs(vcpu, &kvm_regs);
2033                 if (r)
2034                         goto out;
2035                 r = 0;
2036                 break;
2037         }
2038         case KVM_GET_SREGS: {
2039                 struct kvm_sregs kvm_sregs;
2040
2041                 memset(&kvm_sregs, 0, sizeof kvm_sregs);
2042                 r = kvm_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
2043                 if (r)
2044                         goto out;
2045                 r = -EFAULT;
2046                 if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
2047                         goto out;
2048                 r = 0;
2049                 break;
2050         }
2051         case KVM_SET_SREGS: {
2052                 struct kvm_sregs kvm_sregs;
2053
2054                 r = -EFAULT;
2055                 if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
2056                         goto out;
2057                 r = kvm_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
2058                 if (r)
2059                         goto out;
2060                 r = 0;
2061                 break;
2062         }
2063         case KVM_TRANSLATE: {
2064                 struct kvm_translation tr;
2065
2066                 r = -EFAULT;
2067                 if (copy_from_user(&tr, argp, sizeof tr))
2068                         goto out;
2069                 r = kvm_vcpu_ioctl_translate(vcpu, &tr);
2070                 if (r)
2071                         goto out;
2072                 r = -EFAULT;
2073                 if (copy_to_user(argp, &tr, sizeof tr))
2074                         goto out;
2075                 r = 0;
2076                 break;
2077         }
2078         case KVM_INTERRUPT: {
2079                 struct kvm_interrupt irq;
2080
2081                 r = -EFAULT;
2082                 if (copy_from_user(&irq, argp, sizeof irq))
2083                         goto out;
2084                 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
2085                 if (r)
2086                         goto out;
2087                 r = 0;
2088                 break;
2089         }
2090         case KVM_DEBUG_GUEST: {
2091                 struct kvm_debug_guest dbg;
2092
2093                 r = -EFAULT;
2094                 if (copy_from_user(&dbg, argp, sizeof dbg))
2095                         goto out;
2096                 r = kvm_vcpu_ioctl_debug_guest(vcpu, &dbg);
2097                 if (r)
2098                         goto out;
2099                 r = 0;
2100                 break;
2101         }
2102         case KVM_GET_MSRS:
2103                 r = msr_io(vcpu, argp, get_msr, 1);
2104                 break;
2105         case KVM_SET_MSRS:
2106                 r = msr_io(vcpu, argp, do_set_msr, 0);
2107                 break;
2108         default:
2109                 ;
2110         }
2111 out:
2112         return r;
2113 }
2114
2115 static long kvm_vm_ioctl(struct file *filp,
2116                            unsigned int ioctl, unsigned long arg)
2117 {
2118         struct kvm *kvm = filp->private_data;
2119         void __user *argp = (void __user *)arg;
2120         int r = -EINVAL;
2121
2122         switch (ioctl) {
2123         case KVM_CREATE_VCPU:
2124                 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
2125                 if (r < 0)
2126                         goto out;
2127                 break;
2128         case KVM_SET_MEMORY_REGION: {
2129                 struct kvm_memory_region kvm_mem;
2130
2131                 r = -EFAULT;
2132                 if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
2133                         goto out;
2134                 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_mem);
2135                 if (r)
2136                         goto out;
2137                 break;
2138         }
2139         case KVM_GET_DIRTY_LOG: {
2140                 struct kvm_dirty_log log;
2141
2142                 r = -EFAULT;
2143                 if (copy_from_user(&log, argp, sizeof log))
2144                         goto out;
2145                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2146                 if (r)
2147                         goto out;
2148                 break;
2149         }
2150         default:
2151                 ;
2152         }
2153 out:
2154         return r;
2155 }
2156
2157 static struct page *kvm_vm_nopage(struct vm_area_struct *vma,
2158                                   unsigned long address,
2159                                   int *type)
2160 {
2161         struct kvm *kvm = vma->vm_file->private_data;
2162         unsigned long pgoff;
2163         struct kvm_memory_slot *slot;
2164         struct page *page;
2165
2166         *type = VM_FAULT_MINOR;
2167         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2168         slot = gfn_to_memslot(kvm, pgoff);
2169         if (!slot)
2170                 return NOPAGE_SIGBUS;
2171         page = gfn_to_page(slot, pgoff);
2172         if (!page)
2173                 return NOPAGE_SIGBUS;
2174         get_page(page);
2175         return page;
2176 }
2177
2178 static struct vm_operations_struct kvm_vm_vm_ops = {
2179         .nopage = kvm_vm_nopage,
2180 };
2181
2182 static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
2183 {
2184         vma->vm_ops = &kvm_vm_vm_ops;
2185         return 0;
2186 }
2187
2188 static struct file_operations kvm_vm_fops = {
2189         .release        = kvm_vm_release,
2190         .unlocked_ioctl = kvm_vm_ioctl,
2191         .compat_ioctl   = kvm_vm_ioctl,
2192         .mmap           = kvm_vm_mmap,
2193 };
2194
2195 static int kvm_dev_ioctl_create_vm(void)
2196 {
2197         int fd, r;
2198         struct inode *inode;
2199         struct file *file;
2200         struct kvm *kvm;
2201
2202         inode = kvmfs_inode(&kvm_vm_fops);
2203         if (IS_ERR(inode)) {
2204                 r = PTR_ERR(inode);
2205                 goto out1;
2206         }
2207
2208         kvm = kvm_create_vm();
2209         if (IS_ERR(kvm)) {
2210                 r = PTR_ERR(kvm);
2211                 goto out2;
2212         }
2213
2214         file = kvmfs_file(inode, kvm);
2215         if (IS_ERR(file)) {
2216                 r = PTR_ERR(file);
2217                 goto out3;
2218         }
2219         kvm->filp = file;
2220
2221         r = get_unused_fd();
2222         if (r < 0)
2223                 goto out4;
2224         fd = r;
2225         fd_install(fd, file);
2226
2227         return fd;
2228
2229 out4:
2230         fput(file);
2231 out3:
2232         kvm_destroy_vm(kvm);
2233 out2:
2234         iput(inode);
2235 out1:
2236         return r;
2237 }
2238
2239 static long kvm_dev_ioctl(struct file *filp,
2240                           unsigned int ioctl, unsigned long arg)
2241 {
2242         void __user *argp = (void __user *)arg;
2243         int r = -EINVAL;
2244
2245         switch (ioctl) {
2246         case KVM_GET_API_VERSION:
2247                 r = KVM_API_VERSION;
2248                 break;
2249         case KVM_CREATE_VM:
2250                 r = kvm_dev_ioctl_create_vm();
2251                 break;
2252         case KVM_GET_MSR_INDEX_LIST: {
2253                 struct kvm_msr_list __user *user_msr_list = argp;
2254                 struct kvm_msr_list msr_list;
2255                 unsigned n;
2256
2257                 r = -EFAULT;
2258                 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
2259                         goto out;
2260                 n = msr_list.nmsrs;
2261                 msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
2262                 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
2263                         goto out;
2264                 r = -E2BIG;
2265                 if (n < num_msrs_to_save)
2266                         goto out;
2267                 r = -EFAULT;
2268                 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
2269                                  num_msrs_to_save * sizeof(u32)))
2270                         goto out;
2271                 if (copy_to_user(user_msr_list->indices
2272                                  + num_msrs_to_save * sizeof(u32),
2273                                  &emulated_msrs,
2274                                  ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
2275                         goto out;
2276                 r = 0;
2277                 break;
2278         }
2279         default:
2280                 ;
2281         }
2282 out:
2283         return r;
2284 }
2285
2286 static struct file_operations kvm_chardev_ops = {
2287         .open           = kvm_dev_open,
2288         .release        = kvm_dev_release,
2289         .unlocked_ioctl = kvm_dev_ioctl,
2290         .compat_ioctl   = kvm_dev_ioctl,
2291 };
2292
2293 static struct miscdevice kvm_dev = {
2294         MISC_DYNAMIC_MINOR,
2295         "kvm",
2296         &kvm_chardev_ops,
2297 };
2298
2299 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
2300                        void *v)
2301 {
2302         if (val == SYS_RESTART) {
2303                 /*
2304                  * Some (well, at least mine) BIOSes hang on reboot if
2305                  * in vmx root mode.
2306                  */
2307                 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
2308                 on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2309         }
2310         return NOTIFY_OK;
2311 }
2312
2313 static struct notifier_block kvm_reboot_notifier = {
2314         .notifier_call = kvm_reboot,
2315         .priority = 0,
2316 };
2317
2318 /*
2319  * Make sure that a cpu that is being hot-unplugged does not have any vcpus
2320  * cached on it.
2321  */
2322 static void decache_vcpus_on_cpu(int cpu)
2323 {
2324         struct kvm *vm;
2325         struct kvm_vcpu *vcpu;
2326         int i;
2327
2328         spin_lock(&kvm_lock);
2329         list_for_each_entry(vm, &vm_list, vm_list)
2330                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
2331                         vcpu = &vm->vcpus[i];
2332                         /*
2333                          * If the vcpu is locked, then it is running on some
2334                          * other cpu and therefore it is not cached on the
2335                          * cpu in question.
2336                          *
2337                          * If it's not locked, check the last cpu it executed
2338                          * on.
2339                          */
2340                         if (mutex_trylock(&vcpu->mutex)) {
2341                                 if (vcpu->cpu == cpu) {
2342                                         kvm_arch_ops->vcpu_decache(vcpu);
2343                                         vcpu->cpu = -1;
2344                                 }
2345                                 mutex_unlock(&vcpu->mutex);
2346                         }
2347                 }
2348         spin_unlock(&kvm_lock);
2349 }
2350
2351 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
2352                            void *v)
2353 {
2354         int cpu = (long)v;
2355
2356         switch (val) {
2357         case CPU_DOWN_PREPARE:
2358         case CPU_UP_CANCELED:
2359                 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
2360                        cpu);
2361                 decache_vcpus_on_cpu(cpu);
2362                 smp_call_function_single(cpu, kvm_arch_ops->hardware_disable,
2363                                          NULL, 0, 1);
2364                 break;
2365         case CPU_ONLINE:
2366                 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
2367                        cpu);
2368                 smp_call_function_single(cpu, kvm_arch_ops->hardware_enable,
2369                                          NULL, 0, 1);
2370                 break;
2371         }
2372         return NOTIFY_OK;
2373 }
2374
2375 static struct notifier_block kvm_cpu_notifier = {
2376         .notifier_call = kvm_cpu_hotplug,
2377         .priority = 20, /* must be > scheduler priority */
2378 };
2379
2380 static __init void kvm_init_debug(void)
2381 {
2382         struct kvm_stats_debugfs_item *p;
2383
2384         debugfs_dir = debugfs_create_dir("kvm", NULL);
2385         for (p = debugfs_entries; p->name; ++p)
2386                 p->dentry = debugfs_create_u32(p->name, 0444, debugfs_dir,
2387                                                p->data);
2388 }
2389
2390 static void kvm_exit_debug(void)
2391 {
2392         struct kvm_stats_debugfs_item *p;
2393
2394         for (p = debugfs_entries; p->name; ++p)
2395                 debugfs_remove(p->dentry);
2396         debugfs_remove(debugfs_dir);
2397 }
2398
2399 static int kvm_suspend(struct sys_device *dev, pm_message_t state)
2400 {
2401         decache_vcpus_on_cpu(raw_smp_processor_id());
2402         on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2403         return 0;
2404 }
2405
2406 static int kvm_resume(struct sys_device *dev)
2407 {
2408         on_each_cpu(kvm_arch_ops->hardware_enable, NULL, 0, 1);
2409         return 0;
2410 }
2411
2412 static struct sysdev_class kvm_sysdev_class = {
2413         set_kset_name("kvm"),
2414         .suspend = kvm_suspend,
2415         .resume = kvm_resume,
2416 };
2417
2418 static struct sys_device kvm_sysdev = {
2419         .id = 0,
2420         .cls = &kvm_sysdev_class,
2421 };
2422
2423 hpa_t bad_page_address;
2424
2425 static int kvmfs_get_sb(struct file_system_type *fs_type, int flags,
2426                         const char *dev_name, void *data, struct vfsmount *mnt)
2427 {
2428         return get_sb_pseudo(fs_type, "kvm:", NULL, KVMFS_MAGIC, mnt);
2429 }
2430
2431 static struct file_system_type kvm_fs_type = {
2432         .name           = "kvmfs",
2433         .get_sb         = kvmfs_get_sb,
2434         .kill_sb        = kill_anon_super,
2435 };
2436
2437 int kvm_init_arch(struct kvm_arch_ops *ops, struct module *module)
2438 {
2439         int r;
2440
2441         if (kvm_arch_ops) {
2442                 printk(KERN_ERR "kvm: already loaded the other module\n");
2443                 return -EEXIST;
2444         }
2445
2446         if (!ops->cpu_has_kvm_support()) {
2447                 printk(KERN_ERR "kvm: no hardware support\n");
2448                 return -EOPNOTSUPP;
2449         }
2450         if (ops->disabled_by_bios()) {
2451                 printk(KERN_ERR "kvm: disabled by bios\n");
2452                 return -EOPNOTSUPP;
2453         }
2454
2455         kvm_arch_ops = ops;
2456
2457         r = kvm_arch_ops->hardware_setup();
2458         if (r < 0)
2459             return r;
2460
2461         on_each_cpu(kvm_arch_ops->hardware_enable, NULL, 0, 1);
2462         r = register_cpu_notifier(&kvm_cpu_notifier);
2463         if (r)
2464                 goto out_free_1;
2465         register_reboot_notifier(&kvm_reboot_notifier);
2466
2467         r = sysdev_class_register(&kvm_sysdev_class);
2468         if (r)
2469                 goto out_free_2;
2470
2471         r = sysdev_register(&kvm_sysdev);
2472         if (r)
2473                 goto out_free_3;
2474
2475         kvm_chardev_ops.owner = module;
2476
2477         r = misc_register(&kvm_dev);
2478         if (r) {
2479                 printk (KERN_ERR "kvm: misc device register failed\n");
2480                 goto out_free;
2481         }
2482
2483         return r;
2484
2485 out_free:
2486         sysdev_unregister(&kvm_sysdev);
2487 out_free_3:
2488         sysdev_class_unregister(&kvm_sysdev_class);
2489 out_free_2:
2490         unregister_reboot_notifier(&kvm_reboot_notifier);
2491         unregister_cpu_notifier(&kvm_cpu_notifier);
2492 out_free_1:
2493         on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2494         kvm_arch_ops->hardware_unsetup();
2495         return r;
2496 }
2497
2498 void kvm_exit_arch(void)
2499 {
2500         misc_deregister(&kvm_dev);
2501         sysdev_unregister(&kvm_sysdev);
2502         sysdev_class_unregister(&kvm_sysdev_class);
2503         unregister_reboot_notifier(&kvm_reboot_notifier);
2504         unregister_cpu_notifier(&kvm_cpu_notifier);
2505         on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2506         kvm_arch_ops->hardware_unsetup();
2507         kvm_arch_ops = NULL;
2508 }
2509
2510 static __init int kvm_init(void)
2511 {
2512         static struct page *bad_page;
2513         int r;
2514
2515         r = register_filesystem(&kvm_fs_type);
2516         if (r)
2517                 goto out3;
2518
2519         kvmfs_mnt = kern_mount(&kvm_fs_type);
2520         r = PTR_ERR(kvmfs_mnt);
2521         if (IS_ERR(kvmfs_mnt))
2522                 goto out2;
2523         kvm_init_debug();
2524
2525         kvm_init_msr_list();
2526
2527         if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) {
2528                 r = -ENOMEM;
2529                 goto out;
2530         }
2531
2532         bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT;
2533         memset(__va(bad_page_address), 0, PAGE_SIZE);
2534
2535         return r;
2536
2537 out:
2538         kvm_exit_debug();
2539         mntput(kvmfs_mnt);
2540 out2:
2541         unregister_filesystem(&kvm_fs_type);
2542 out3:
2543         return r;
2544 }
2545
2546 static __exit void kvm_exit(void)
2547 {
2548         kvm_exit_debug();
2549         __free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT));
2550         mntput(kvmfs_mnt);
2551         unregister_filesystem(&kvm_fs_type);
2552 }
2553
2554 module_init(kvm_init)
2555 module_exit(kvm_exit)
2556
2557 EXPORT_SYMBOL_GPL(kvm_init_arch);
2558 EXPORT_SYMBOL_GPL(kvm_exit_arch);