Merge branch 'x86/apic' into x86-v28-for-linus-phase4-B
[linux-2.6.git] / arch / x86 / xen / enlighten.c
1 /*
2  * Core of Xen paravirt_ops implementation.
3  *
4  * This file contains the xen_paravirt_ops structure itself, and the
5  * implementations for:
6  * - privileged instructions
7  * - interrupt flags
8  * - segment operations
9  * - booting and setup
10  *
11  * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
12  */
13
14 #include <linux/kernel.h>
15 #include <linux/init.h>
16 #include <linux/smp.h>
17 #include <linux/preempt.h>
18 #include <linux/hardirq.h>
19 #include <linux/percpu.h>
20 #include <linux/delay.h>
21 #include <linux/start_kernel.h>
22 #include <linux/sched.h>
23 #include <linux/bootmem.h>
24 #include <linux/module.h>
25 #include <linux/mm.h>
26 #include <linux/page-flags.h>
27 #include <linux/highmem.h>
28 #include <linux/console.h>
29
30 #include <xen/interface/xen.h>
31 #include <xen/interface/physdev.h>
32 #include <xen/interface/vcpu.h>
33 #include <xen/interface/sched.h>
34 #include <xen/features.h>
35 #include <xen/page.h>
36 #include <xen/hvc-console.h>
37
38 #include <asm/paravirt.h>
39 #include <asm/apic.h>
40 #include <asm/page.h>
41 #include <asm/xen/hypercall.h>
42 #include <asm/xen/hypervisor.h>
43 #include <asm/fixmap.h>
44 #include <asm/processor.h>
45 #include <asm/msr-index.h>
46 #include <asm/setup.h>
47 #include <asm/desc.h>
48 #include <asm/pgtable.h>
49 #include <asm/tlbflush.h>
50 #include <asm/reboot.h>
51
52 #include "xen-ops.h"
53 #include "mmu.h"
54 #include "multicalls.h"
55
56 EXPORT_SYMBOL_GPL(hypercall_page);
57
58 DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu);
59 DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info);
60
61 /*
62  * Identity map, in addition to plain kernel map.  This needs to be
63  * large enough to allocate page table pages to allocate the rest.
64  * Each page can map 2MB.
65  */
66 static pte_t level1_ident_pgt[PTRS_PER_PTE * 4] __page_aligned_bss;
67
68 #ifdef CONFIG_X86_64
69 /* l3 pud for userspace vsyscall mapping */
70 static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss;
71 #endif /* CONFIG_X86_64 */
72
73 /*
74  * Note about cr3 (pagetable base) values:
75  *
76  * xen_cr3 contains the current logical cr3 value; it contains the
77  * last set cr3.  This may not be the current effective cr3, because
78  * its update may be being lazily deferred.  However, a vcpu looking
79  * at its own cr3 can use this value knowing that it everything will
80  * be self-consistent.
81  *
82  * xen_current_cr3 contains the actual vcpu cr3; it is set once the
83  * hypercall to set the vcpu cr3 is complete (so it may be a little
84  * out of date, but it will never be set early).  If one vcpu is
85  * looking at another vcpu's cr3 value, it should use this variable.
86  */
87 DEFINE_PER_CPU(unsigned long, xen_cr3);  /* cr3 stored as physaddr */
88 DEFINE_PER_CPU(unsigned long, xen_current_cr3);  /* actual vcpu cr3 */
89
90 struct start_info *xen_start_info;
91 EXPORT_SYMBOL_GPL(xen_start_info);
92
93 struct shared_info xen_dummy_shared_info;
94
95 /*
96  * Point at some empty memory to start with. We map the real shared_info
97  * page as soon as fixmap is up and running.
98  */
99 struct shared_info *HYPERVISOR_shared_info = (void *)&xen_dummy_shared_info;
100
101 /*
102  * Flag to determine whether vcpu info placement is available on all
103  * VCPUs.  We assume it is to start with, and then set it to zero on
104  * the first failure.  This is because it can succeed on some VCPUs
105  * and not others, since it can involve hypervisor memory allocation,
106  * or because the guest failed to guarantee all the appropriate
107  * constraints on all VCPUs (ie buffer can't cross a page boundary).
108  *
109  * Note that any particular CPU may be using a placed vcpu structure,
110  * but we can only optimise if the all are.
111  *
112  * 0: not available, 1: available
113  */
114 static int have_vcpu_info_placement = 1;
115
116 static void xen_vcpu_setup(int cpu)
117 {
118         struct vcpu_register_vcpu_info info;
119         int err;
120         struct vcpu_info *vcpup;
121
122         BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info);
123         per_cpu(xen_vcpu, cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu];
124
125         if (!have_vcpu_info_placement)
126                 return;         /* already tested, not available */
127
128         vcpup = &per_cpu(xen_vcpu_info, cpu);
129
130         info.mfn = virt_to_mfn(vcpup);
131         info.offset = offset_in_page(vcpup);
132
133         printk(KERN_DEBUG "trying to map vcpu_info %d at %p, mfn %llx, offset %d\n",
134                cpu, vcpup, info.mfn, info.offset);
135
136         /* Check to see if the hypervisor will put the vcpu_info
137            structure where we want it, which allows direct access via
138            a percpu-variable. */
139         err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, cpu, &info);
140
141         if (err) {
142                 printk(KERN_DEBUG "register_vcpu_info failed: err=%d\n", err);
143                 have_vcpu_info_placement = 0;
144         } else {
145                 /* This cpu is using the registered vcpu info, even if
146                    later ones fail to. */
147                 per_cpu(xen_vcpu, cpu) = vcpup;
148
149                 printk(KERN_DEBUG "cpu %d using vcpu_info at %p\n",
150                        cpu, vcpup);
151         }
152 }
153
154 /*
155  * On restore, set the vcpu placement up again.
156  * If it fails, then we're in a bad state, since
157  * we can't back out from using it...
158  */
159 void xen_vcpu_restore(void)
160 {
161         if (have_vcpu_info_placement) {
162                 int cpu;
163
164                 for_each_online_cpu(cpu) {
165                         bool other_cpu = (cpu != smp_processor_id());
166
167                         if (other_cpu &&
168                             HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL))
169                                 BUG();
170
171                         xen_vcpu_setup(cpu);
172
173                         if (other_cpu &&
174                             HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL))
175                                 BUG();
176                 }
177
178                 BUG_ON(!have_vcpu_info_placement);
179         }
180 }
181
182 static void __init xen_banner(void)
183 {
184         unsigned version = HYPERVISOR_xen_version(XENVER_version, NULL);
185         struct xen_extraversion extra;
186         HYPERVISOR_xen_version(XENVER_extraversion, &extra);
187
188         printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
189                pv_info.name);
190         printk(KERN_INFO "Xen version: %d.%d%s%s\n",
191                version >> 16, version & 0xffff, extra.extraversion,
192                xen_feature(XENFEAT_mmu_pt_update_preserve_ad) ? " (preserve-AD)" : "");
193 }
194
195 static void xen_cpuid(unsigned int *ax, unsigned int *bx,
196                       unsigned int *cx, unsigned int *dx)
197 {
198         unsigned maskedx = ~0;
199
200         /*
201          * Mask out inconvenient features, to try and disable as many
202          * unsupported kernel subsystems as possible.
203          */
204         if (*ax == 1)
205                 maskedx = ~((1 << X86_FEATURE_APIC) |  /* disable APIC */
206                             (1 << X86_FEATURE_ACPI) |  /* disable ACPI */
207                             (1 << X86_FEATURE_MCE)  |  /* disable MCE */
208                             (1 << X86_FEATURE_MCA)  |  /* disable MCA */
209                             (1 << X86_FEATURE_ACC));   /* thermal monitoring */
210
211         asm(XEN_EMULATE_PREFIX "cpuid"
212                 : "=a" (*ax),
213                   "=b" (*bx),
214                   "=c" (*cx),
215                   "=d" (*dx)
216                 : "0" (*ax), "2" (*cx));
217         *dx &= maskedx;
218 }
219
220 static void xen_set_debugreg(int reg, unsigned long val)
221 {
222         HYPERVISOR_set_debugreg(reg, val);
223 }
224
225 static unsigned long xen_get_debugreg(int reg)
226 {
227         return HYPERVISOR_get_debugreg(reg);
228 }
229
230 static unsigned long xen_save_fl(void)
231 {
232         struct vcpu_info *vcpu;
233         unsigned long flags;
234
235         vcpu = x86_read_percpu(xen_vcpu);
236
237         /* flag has opposite sense of mask */
238         flags = !vcpu->evtchn_upcall_mask;
239
240         /* convert to IF type flag
241            -0 -> 0x00000000
242            -1 -> 0xffffffff
243         */
244         return (-flags) & X86_EFLAGS_IF;
245 }
246
247 static void xen_restore_fl(unsigned long flags)
248 {
249         struct vcpu_info *vcpu;
250
251         /* convert from IF type flag */
252         flags = !(flags & X86_EFLAGS_IF);
253
254         /* There's a one instruction preempt window here.  We need to
255            make sure we're don't switch CPUs between getting the vcpu
256            pointer and updating the mask. */
257         preempt_disable();
258         vcpu = x86_read_percpu(xen_vcpu);
259         vcpu->evtchn_upcall_mask = flags;
260         preempt_enable_no_resched();
261
262         /* Doesn't matter if we get preempted here, because any
263            pending event will get dealt with anyway. */
264
265         if (flags == 0) {
266                 preempt_check_resched();
267                 barrier(); /* unmask then check (avoid races) */
268                 if (unlikely(vcpu->evtchn_upcall_pending))
269                         force_evtchn_callback();
270         }
271 }
272
273 static void xen_irq_disable(void)
274 {
275         /* There's a one instruction preempt window here.  We need to
276            make sure we're don't switch CPUs between getting the vcpu
277            pointer and updating the mask. */
278         preempt_disable();
279         x86_read_percpu(xen_vcpu)->evtchn_upcall_mask = 1;
280         preempt_enable_no_resched();
281 }
282
283 static void xen_irq_enable(void)
284 {
285         struct vcpu_info *vcpu;
286
287         /* We don't need to worry about being preempted here, since
288            either a) interrupts are disabled, so no preemption, or b)
289            the caller is confused and is trying to re-enable interrupts
290            on an indeterminate processor. */
291
292         vcpu = x86_read_percpu(xen_vcpu);
293         vcpu->evtchn_upcall_mask = 0;
294
295         /* Doesn't matter if we get preempted here, because any
296            pending event will get dealt with anyway. */
297
298         barrier(); /* unmask then check (avoid races) */
299         if (unlikely(vcpu->evtchn_upcall_pending))
300                 force_evtchn_callback();
301 }
302
303 static void xen_safe_halt(void)
304 {
305         /* Blocking includes an implicit local_irq_enable(). */
306         if (HYPERVISOR_sched_op(SCHEDOP_block, NULL) != 0)
307                 BUG();
308 }
309
310 static void xen_halt(void)
311 {
312         if (irqs_disabled())
313                 HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL);
314         else
315                 xen_safe_halt();
316 }
317
318 static void xen_leave_lazy(void)
319 {
320         paravirt_leave_lazy(paravirt_get_lazy_mode());
321         xen_mc_flush();
322 }
323
324 static unsigned long xen_store_tr(void)
325 {
326         return 0;
327 }
328
329 static void xen_set_ldt(const void *addr, unsigned entries)
330 {
331         struct mmuext_op *op;
332         struct multicall_space mcs = xen_mc_entry(sizeof(*op));
333
334         op = mcs.args;
335         op->cmd = MMUEXT_SET_LDT;
336         op->arg1.linear_addr = (unsigned long)addr;
337         op->arg2.nr_ents = entries;
338
339         MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
340
341         xen_mc_issue(PARAVIRT_LAZY_CPU);
342 }
343
344 static void xen_load_gdt(const struct desc_ptr *dtr)
345 {
346         unsigned long *frames;
347         unsigned long va = dtr->address;
348         unsigned int size = dtr->size + 1;
349         unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
350         int f;
351         struct multicall_space mcs;
352
353         /* A GDT can be up to 64k in size, which corresponds to 8192
354            8-byte entries, or 16 4k pages.. */
355
356         BUG_ON(size > 65536);
357         BUG_ON(va & ~PAGE_MASK);
358
359         mcs = xen_mc_entry(sizeof(*frames) * pages);
360         frames = mcs.args;
361
362         for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
363                 frames[f] = virt_to_mfn(va);
364                 make_lowmem_page_readonly((void *)va);
365         }
366
367         MULTI_set_gdt(mcs.mc, frames, size / sizeof(struct desc_struct));
368
369         xen_mc_issue(PARAVIRT_LAZY_CPU);
370 }
371
372 static void load_TLS_descriptor(struct thread_struct *t,
373                                 unsigned int cpu, unsigned int i)
374 {
375         struct desc_struct *gdt = get_cpu_gdt_table(cpu);
376         xmaddr_t maddr = virt_to_machine(&gdt[GDT_ENTRY_TLS_MIN+i]);
377         struct multicall_space mc = __xen_mc_entry(0);
378
379         MULTI_update_descriptor(mc.mc, maddr.maddr, t->tls_array[i]);
380 }
381
382 static void xen_load_tls(struct thread_struct *t, unsigned int cpu)
383 {
384         /*
385          * XXX sleazy hack: If we're being called in a lazy-cpu zone,
386          * it means we're in a context switch, and %gs has just been
387          * saved.  This means we can zero it out to prevent faults on
388          * exit from the hypervisor if the next process has no %gs.
389          * Either way, it has been saved, and the new value will get
390          * loaded properly.  This will go away as soon as Xen has been
391          * modified to not save/restore %gs for normal hypercalls.
392          *
393          * On x86_64, this hack is not used for %gs, because gs points
394          * to KERNEL_GS_BASE (and uses it for PDA references), so we
395          * must not zero %gs on x86_64
396          *
397          * For x86_64, we need to zero %fs, otherwise we may get an
398          * exception between the new %fs descriptor being loaded and
399          * %fs being effectively cleared at __switch_to().
400          */
401         if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_CPU) {
402 #ifdef CONFIG_X86_32
403                 loadsegment(gs, 0);
404 #else
405                 loadsegment(fs, 0);
406 #endif
407         }
408
409         xen_mc_batch();
410
411         load_TLS_descriptor(t, cpu, 0);
412         load_TLS_descriptor(t, cpu, 1);
413         load_TLS_descriptor(t, cpu, 2);
414
415         xen_mc_issue(PARAVIRT_LAZY_CPU);
416 }
417
418 #ifdef CONFIG_X86_64
419 static void xen_load_gs_index(unsigned int idx)
420 {
421         if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, idx))
422                 BUG();
423 }
424 #endif
425
426 static void xen_write_ldt_entry(struct desc_struct *dt, int entrynum,
427                                 const void *ptr)
428 {
429         unsigned long lp = (unsigned long)&dt[entrynum];
430         xmaddr_t mach_lp = virt_to_machine(lp);
431         u64 entry = *(u64 *)ptr;
432
433         preempt_disable();
434
435         xen_mc_flush();
436         if (HYPERVISOR_update_descriptor(mach_lp.maddr, entry))
437                 BUG();
438
439         preempt_enable();
440 }
441
442 static int cvt_gate_to_trap(int vector, const gate_desc *val,
443                             struct trap_info *info)
444 {
445         if (val->type != 0xf && val->type != 0xe)
446                 return 0;
447
448         info->vector = vector;
449         info->address = gate_offset(*val);
450         info->cs = gate_segment(*val);
451         info->flags = val->dpl;
452         /* interrupt gates clear IF */
453         if (val->type == 0xe)
454                 info->flags |= 4;
455
456         return 1;
457 }
458
459 /* Locations of each CPU's IDT */
460 static DEFINE_PER_CPU(struct desc_ptr, idt_desc);
461
462 /* Set an IDT entry.  If the entry is part of the current IDT, then
463    also update Xen. */
464 static void xen_write_idt_entry(gate_desc *dt, int entrynum, const gate_desc *g)
465 {
466         unsigned long p = (unsigned long)&dt[entrynum];
467         unsigned long start, end;
468
469         preempt_disable();
470
471         start = __get_cpu_var(idt_desc).address;
472         end = start + __get_cpu_var(idt_desc).size + 1;
473
474         xen_mc_flush();
475
476         native_write_idt_entry(dt, entrynum, g);
477
478         if (p >= start && (p + 8) <= end) {
479                 struct trap_info info[2];
480
481                 info[1].address = 0;
482
483                 if (cvt_gate_to_trap(entrynum, g, &info[0]))
484                         if (HYPERVISOR_set_trap_table(info))
485                                 BUG();
486         }
487
488         preempt_enable();
489 }
490
491 static void xen_convert_trap_info(const struct desc_ptr *desc,
492                                   struct trap_info *traps)
493 {
494         unsigned in, out, count;
495
496         count = (desc->size+1) / sizeof(gate_desc);
497         BUG_ON(count > 256);
498
499         for (in = out = 0; in < count; in++) {
500                 gate_desc *entry = (gate_desc*)(desc->address) + in;
501
502                 if (cvt_gate_to_trap(in, entry, &traps[out]))
503                         out++;
504         }
505         traps[out].address = 0;
506 }
507
508 void xen_copy_trap_info(struct trap_info *traps)
509 {
510         const struct desc_ptr *desc = &__get_cpu_var(idt_desc);
511
512         xen_convert_trap_info(desc, traps);
513 }
514
515 /* Load a new IDT into Xen.  In principle this can be per-CPU, so we
516    hold a spinlock to protect the static traps[] array (static because
517    it avoids allocation, and saves stack space). */
518 static void xen_load_idt(const struct desc_ptr *desc)
519 {
520         static DEFINE_SPINLOCK(lock);
521         static struct trap_info traps[257];
522
523         spin_lock(&lock);
524
525         __get_cpu_var(idt_desc) = *desc;
526
527         xen_convert_trap_info(desc, traps);
528
529         xen_mc_flush();
530         if (HYPERVISOR_set_trap_table(traps))
531                 BUG();
532
533         spin_unlock(&lock);
534 }
535
536 /* Write a GDT descriptor entry.  Ignore LDT descriptors, since
537    they're handled differently. */
538 static void xen_write_gdt_entry(struct desc_struct *dt, int entry,
539                                 const void *desc, int type)
540 {
541         preempt_disable();
542
543         switch (type) {
544         case DESC_LDT:
545         case DESC_TSS:
546                 /* ignore */
547                 break;
548
549         default: {
550                 xmaddr_t maddr = virt_to_machine(&dt[entry]);
551
552                 xen_mc_flush();
553                 if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
554                         BUG();
555         }
556
557         }
558
559         preempt_enable();
560 }
561
562 static void xen_load_sp0(struct tss_struct *tss,
563                           struct thread_struct *thread)
564 {
565         struct multicall_space mcs = xen_mc_entry(0);
566         MULTI_stack_switch(mcs.mc, __KERNEL_DS, thread->sp0);
567         xen_mc_issue(PARAVIRT_LAZY_CPU);
568 }
569
570 static void xen_set_iopl_mask(unsigned mask)
571 {
572         struct physdev_set_iopl set_iopl;
573
574         /* Force the change at ring 0. */
575         set_iopl.iopl = (mask == 0) ? 1 : (mask >> 12) & 3;
576         HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
577 }
578
579 static void xen_io_delay(void)
580 {
581 }
582
583 #ifdef CONFIG_X86_LOCAL_APIC
584 static u32 xen_apic_read(u32 reg)
585 {
586         return 0;
587 }
588
589 static void xen_apic_write(u32 reg, u32 val)
590 {
591         /* Warn to see if there's any stray references */
592         WARN_ON(1);
593 }
594
595 static u64 xen_apic_icr_read(void)
596 {
597         return 0;
598 }
599
600 static void xen_apic_icr_write(u32 low, u32 id)
601 {
602         /* Warn to see if there's any stray references */
603         WARN_ON(1);
604 }
605
606 static void xen_apic_wait_icr_idle(void)
607 {
608         return;
609 }
610
611 static u32 xen_safe_apic_wait_icr_idle(void)
612 {
613         return 0;
614 }
615
616 static struct apic_ops xen_basic_apic_ops = {
617         .read = xen_apic_read,
618         .write = xen_apic_write,
619         .icr_read = xen_apic_icr_read,
620         .icr_write = xen_apic_icr_write,
621         .wait_icr_idle = xen_apic_wait_icr_idle,
622         .safe_wait_icr_idle = xen_safe_apic_wait_icr_idle,
623 };
624
625 #endif
626
627 static void xen_flush_tlb(void)
628 {
629         struct mmuext_op *op;
630         struct multicall_space mcs;
631
632         preempt_disable();
633
634         mcs = xen_mc_entry(sizeof(*op));
635
636         op = mcs.args;
637         op->cmd = MMUEXT_TLB_FLUSH_LOCAL;
638         MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
639
640         xen_mc_issue(PARAVIRT_LAZY_MMU);
641
642         preempt_enable();
643 }
644
645 static void xen_flush_tlb_single(unsigned long addr)
646 {
647         struct mmuext_op *op;
648         struct multicall_space mcs;
649
650         preempt_disable();
651
652         mcs = xen_mc_entry(sizeof(*op));
653         op = mcs.args;
654         op->cmd = MMUEXT_INVLPG_LOCAL;
655         op->arg1.linear_addr = addr & PAGE_MASK;
656         MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
657
658         xen_mc_issue(PARAVIRT_LAZY_MMU);
659
660         preempt_enable();
661 }
662
663 static void xen_flush_tlb_others(const cpumask_t *cpus, struct mm_struct *mm,
664                                  unsigned long va)
665 {
666         struct {
667                 struct mmuext_op op;
668                 cpumask_t mask;
669         } *args;
670         cpumask_t cpumask = *cpus;
671         struct multicall_space mcs;
672
673         /*
674          * A couple of (to be removed) sanity checks:
675          *
676          * - current CPU must not be in mask
677          * - mask must exist :)
678          */
679         BUG_ON(cpus_empty(cpumask));
680         BUG_ON(cpu_isset(smp_processor_id(), cpumask));
681         BUG_ON(!mm);
682
683         /* If a CPU which we ran on has gone down, OK. */
684         cpus_and(cpumask, cpumask, cpu_online_map);
685         if (cpus_empty(cpumask))
686                 return;
687
688         mcs = xen_mc_entry(sizeof(*args));
689         args = mcs.args;
690         args->mask = cpumask;
691         args->op.arg2.vcpumask = &args->mask;
692
693         if (va == TLB_FLUSH_ALL) {
694                 args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
695         } else {
696                 args->op.cmd = MMUEXT_INVLPG_MULTI;
697                 args->op.arg1.linear_addr = va;
698         }
699
700         MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);
701
702         xen_mc_issue(PARAVIRT_LAZY_MMU);
703 }
704
705 static void xen_clts(void)
706 {
707         struct multicall_space mcs;
708
709         mcs = xen_mc_entry(0);
710
711         MULTI_fpu_taskswitch(mcs.mc, 0);
712
713         xen_mc_issue(PARAVIRT_LAZY_CPU);
714 }
715
716 static void xen_write_cr0(unsigned long cr0)
717 {
718         struct multicall_space mcs;
719
720         /* Only pay attention to cr0.TS; everything else is
721            ignored. */
722         mcs = xen_mc_entry(0);
723
724         MULTI_fpu_taskswitch(mcs.mc, (cr0 & X86_CR0_TS) != 0);
725
726         xen_mc_issue(PARAVIRT_LAZY_CPU);
727 }
728
729 static void xen_write_cr2(unsigned long cr2)
730 {
731         x86_read_percpu(xen_vcpu)->arch.cr2 = cr2;
732 }
733
734 static unsigned long xen_read_cr2(void)
735 {
736         return x86_read_percpu(xen_vcpu)->arch.cr2;
737 }
738
739 static unsigned long xen_read_cr2_direct(void)
740 {
741         return x86_read_percpu(xen_vcpu_info.arch.cr2);
742 }
743
744 static void xen_write_cr4(unsigned long cr4)
745 {
746         cr4 &= ~X86_CR4_PGE;
747         cr4 &= ~X86_CR4_PSE;
748
749         native_write_cr4(cr4);
750 }
751
752 static unsigned long xen_read_cr3(void)
753 {
754         return x86_read_percpu(xen_cr3);
755 }
756
757 static void set_current_cr3(void *v)
758 {
759         x86_write_percpu(xen_current_cr3, (unsigned long)v);
760 }
761
762 static void __xen_write_cr3(bool kernel, unsigned long cr3)
763 {
764         struct mmuext_op *op;
765         struct multicall_space mcs;
766         unsigned long mfn;
767
768         if (cr3)
769                 mfn = pfn_to_mfn(PFN_DOWN(cr3));
770         else
771                 mfn = 0;
772
773         WARN_ON(mfn == 0 && kernel);
774
775         mcs = __xen_mc_entry(sizeof(*op));
776
777         op = mcs.args;
778         op->cmd = kernel ? MMUEXT_NEW_BASEPTR : MMUEXT_NEW_USER_BASEPTR;
779         op->arg1.mfn = mfn;
780
781         MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
782
783         if (kernel) {
784                 x86_write_percpu(xen_cr3, cr3);
785
786                 /* Update xen_current_cr3 once the batch has actually
787                    been submitted. */
788                 xen_mc_callback(set_current_cr3, (void *)cr3);
789         }
790 }
791
792 static void xen_write_cr3(unsigned long cr3)
793 {
794         BUG_ON(preemptible());
795
796         xen_mc_batch();  /* disables interrupts */
797
798         /* Update while interrupts are disabled, so its atomic with
799            respect to ipis */
800         x86_write_percpu(xen_cr3, cr3);
801
802         __xen_write_cr3(true, cr3);
803
804 #ifdef CONFIG_X86_64
805         {
806                 pgd_t *user_pgd = xen_get_user_pgd(__va(cr3));
807                 if (user_pgd)
808                         __xen_write_cr3(false, __pa(user_pgd));
809                 else
810                         __xen_write_cr3(false, 0);
811         }
812 #endif
813
814         xen_mc_issue(PARAVIRT_LAZY_CPU);  /* interrupts restored */
815 }
816
817 static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high)
818 {
819         int ret;
820
821         ret = 0;
822
823         switch(msr) {
824 #ifdef CONFIG_X86_64
825                 unsigned which;
826                 u64 base;
827
828         case MSR_FS_BASE:               which = SEGBASE_FS; goto set;
829         case MSR_KERNEL_GS_BASE:        which = SEGBASE_GS_USER; goto set;
830         case MSR_GS_BASE:               which = SEGBASE_GS_KERNEL; goto set;
831
832         set:
833                 base = ((u64)high << 32) | low;
834                 if (HYPERVISOR_set_segment_base(which, base) != 0)
835                         ret = -EFAULT;
836                 break;
837 #endif
838         default:
839                 ret = native_write_msr_safe(msr, low, high);
840         }
841
842         return ret;
843 }
844
845 /* Early in boot, while setting up the initial pagetable, assume
846    everything is pinned. */
847 static __init void xen_alloc_pte_init(struct mm_struct *mm, unsigned long pfn)
848 {
849 #ifdef CONFIG_FLATMEM
850         BUG_ON(mem_map);        /* should only be used early */
851 #endif
852         make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
853 }
854
855 /* Early release_pte assumes that all pts are pinned, since there's
856    only init_mm and anything attached to that is pinned. */
857 static void xen_release_pte_init(unsigned long pfn)
858 {
859         make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
860 }
861
862 static void pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
863 {
864         struct mmuext_op op;
865         op.cmd = cmd;
866         op.arg1.mfn = pfn_to_mfn(pfn);
867         if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF))
868                 BUG();
869 }
870
871 /* This needs to make sure the new pte page is pinned iff its being
872    attached to a pinned pagetable. */
873 static void xen_alloc_ptpage(struct mm_struct *mm, unsigned long pfn, unsigned level)
874 {
875         struct page *page = pfn_to_page(pfn);
876
877         if (PagePinned(virt_to_page(mm->pgd))) {
878                 SetPagePinned(page);
879
880                 if (!PageHighMem(page)) {
881                         make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
882                         if (level == PT_PTE)
883                                 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
884                 } else
885                         /* make sure there are no stray mappings of
886                            this page */
887                         kmap_flush_unused();
888         }
889 }
890
891 static void xen_alloc_pte(struct mm_struct *mm, unsigned long pfn)
892 {
893         xen_alloc_ptpage(mm, pfn, PT_PTE);
894 }
895
896 static void xen_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
897 {
898         xen_alloc_ptpage(mm, pfn, PT_PMD);
899 }
900
901 static int xen_pgd_alloc(struct mm_struct *mm)
902 {
903         pgd_t *pgd = mm->pgd;
904         int ret = 0;
905
906         BUG_ON(PagePinned(virt_to_page(pgd)));
907
908 #ifdef CONFIG_X86_64
909         {
910                 struct page *page = virt_to_page(pgd);
911                 pgd_t *user_pgd;
912
913                 BUG_ON(page->private != 0);
914
915                 ret = -ENOMEM;
916
917                 user_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
918                 page->private = (unsigned long)user_pgd;
919
920                 if (user_pgd != NULL) {
921                         user_pgd[pgd_index(VSYSCALL_START)] =
922                                 __pgd(__pa(level3_user_vsyscall) | _PAGE_TABLE);
923                         ret = 0;
924                 }
925
926                 BUG_ON(PagePinned(virt_to_page(xen_get_user_pgd(pgd))));
927         }
928 #endif
929
930         return ret;
931 }
932
933 static void xen_pgd_free(struct mm_struct *mm, pgd_t *pgd)
934 {
935 #ifdef CONFIG_X86_64
936         pgd_t *user_pgd = xen_get_user_pgd(pgd);
937
938         if (user_pgd)
939                 free_page((unsigned long)user_pgd);
940 #endif
941 }
942
943 /* This should never happen until we're OK to use struct page */
944 static void xen_release_ptpage(unsigned long pfn, unsigned level)
945 {
946         struct page *page = pfn_to_page(pfn);
947
948         if (PagePinned(page)) {
949                 if (!PageHighMem(page)) {
950                         if (level == PT_PTE)
951                                 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
952                         make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
953                 }
954                 ClearPagePinned(page);
955         }
956 }
957
958 static void xen_release_pte(unsigned long pfn)
959 {
960         xen_release_ptpage(pfn, PT_PTE);
961 }
962
963 static void xen_release_pmd(unsigned long pfn)
964 {
965         xen_release_ptpage(pfn, PT_PMD);
966 }
967
968 #if PAGETABLE_LEVELS == 4
969 static void xen_alloc_pud(struct mm_struct *mm, unsigned long pfn)
970 {
971         xen_alloc_ptpage(mm, pfn, PT_PUD);
972 }
973
974 static void xen_release_pud(unsigned long pfn)
975 {
976         xen_release_ptpage(pfn, PT_PUD);
977 }
978 #endif
979
980 #ifdef CONFIG_HIGHPTE
981 static void *xen_kmap_atomic_pte(struct page *page, enum km_type type)
982 {
983         pgprot_t prot = PAGE_KERNEL;
984
985         if (PagePinned(page))
986                 prot = PAGE_KERNEL_RO;
987
988         if (0 && PageHighMem(page))
989                 printk("mapping highpte %lx type %d prot %s\n",
990                        page_to_pfn(page), type,
991                        (unsigned long)pgprot_val(prot) & _PAGE_RW ? "WRITE" : "READ");
992
993         return kmap_atomic_prot(page, type, prot);
994 }
995 #endif
996
997 static __init pte_t mask_rw_pte(pte_t *ptep, pte_t pte)
998 {
999         /* If there's an existing pte, then don't allow _PAGE_RW to be set */
1000         if (pte_val_ma(*ptep) & _PAGE_PRESENT)
1001                 pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) &
1002                                pte_val_ma(pte));
1003
1004         return pte;
1005 }
1006
1007 /* Init-time set_pte while constructing initial pagetables, which
1008    doesn't allow RO pagetable pages to be remapped RW */
1009 static __init void xen_set_pte_init(pte_t *ptep, pte_t pte)
1010 {
1011         pte = mask_rw_pte(ptep, pte);
1012
1013         xen_set_pte(ptep, pte);
1014 }
1015
1016 static __init void xen_pagetable_setup_start(pgd_t *base)
1017 {
1018 }
1019
1020 void xen_setup_shared_info(void)
1021 {
1022         if (!xen_feature(XENFEAT_auto_translated_physmap)) {
1023                 set_fixmap(FIX_PARAVIRT_BOOTMAP,
1024                            xen_start_info->shared_info);
1025
1026                 HYPERVISOR_shared_info =
1027                         (struct shared_info *)fix_to_virt(FIX_PARAVIRT_BOOTMAP);
1028         } else
1029                 HYPERVISOR_shared_info =
1030                         (struct shared_info *)__va(xen_start_info->shared_info);
1031
1032 #ifndef CONFIG_SMP
1033         /* In UP this is as good a place as any to set up shared info */
1034         xen_setup_vcpu_info_placement();
1035 #endif
1036
1037         xen_setup_mfn_list_list();
1038 }
1039
1040 static __init void xen_pagetable_setup_done(pgd_t *base)
1041 {
1042         xen_setup_shared_info();
1043 }
1044
1045 static __init void xen_post_allocator_init(void)
1046 {
1047         pv_mmu_ops.set_pte = xen_set_pte;
1048         pv_mmu_ops.set_pmd = xen_set_pmd;
1049         pv_mmu_ops.set_pud = xen_set_pud;
1050 #if PAGETABLE_LEVELS == 4
1051         pv_mmu_ops.set_pgd = xen_set_pgd;
1052 #endif
1053
1054         /* This will work as long as patching hasn't happened yet
1055            (which it hasn't) */
1056         pv_mmu_ops.alloc_pte = xen_alloc_pte;
1057         pv_mmu_ops.alloc_pmd = xen_alloc_pmd;
1058         pv_mmu_ops.release_pte = xen_release_pte;
1059         pv_mmu_ops.release_pmd = xen_release_pmd;
1060 #if PAGETABLE_LEVELS == 4
1061         pv_mmu_ops.alloc_pud = xen_alloc_pud;
1062         pv_mmu_ops.release_pud = xen_release_pud;
1063 #endif
1064
1065 #ifdef CONFIG_X86_64
1066         SetPagePinned(virt_to_page(level3_user_vsyscall));
1067 #endif
1068         xen_mark_init_mm_pinned();
1069 }
1070
1071 /* This is called once we have the cpu_possible_map */
1072 void xen_setup_vcpu_info_placement(void)
1073 {
1074         int cpu;
1075
1076         for_each_possible_cpu(cpu)
1077                 xen_vcpu_setup(cpu);
1078
1079         /* xen_vcpu_setup managed to place the vcpu_info within the
1080            percpu area for all cpus, so make use of it */
1081 #ifdef CONFIG_X86_32
1082         if (have_vcpu_info_placement) {
1083                 printk(KERN_INFO "Xen: using vcpu_info placement\n");
1084
1085                 pv_irq_ops.save_fl = xen_save_fl_direct;
1086                 pv_irq_ops.restore_fl = xen_restore_fl_direct;
1087                 pv_irq_ops.irq_disable = xen_irq_disable_direct;
1088                 pv_irq_ops.irq_enable = xen_irq_enable_direct;
1089                 pv_mmu_ops.read_cr2 = xen_read_cr2_direct;
1090         }
1091 #endif
1092 }
1093
1094 static unsigned xen_patch(u8 type, u16 clobbers, void *insnbuf,
1095                           unsigned long addr, unsigned len)
1096 {
1097         char *start, *end, *reloc;
1098         unsigned ret;
1099
1100         start = end = reloc = NULL;
1101
1102 #define SITE(op, x)                                                     \
1103         case PARAVIRT_PATCH(op.x):                                      \
1104         if (have_vcpu_info_placement) {                                 \
1105                 start = (char *)xen_##x##_direct;                       \
1106                 end = xen_##x##_direct_end;                             \
1107                 reloc = xen_##x##_direct_reloc;                         \
1108         }                                                               \
1109         goto patch_site
1110
1111         switch (type) {
1112 #ifdef CONFIG_X86_32
1113                 SITE(pv_irq_ops, irq_enable);
1114                 SITE(pv_irq_ops, irq_disable);
1115                 SITE(pv_irq_ops, save_fl);
1116                 SITE(pv_irq_ops, restore_fl);
1117 #endif /* CONFIG_X86_32 */
1118 #undef SITE
1119
1120         patch_site:
1121                 if (start == NULL || (end-start) > len)
1122                         goto default_patch;
1123
1124                 ret = paravirt_patch_insns(insnbuf, len, start, end);
1125
1126                 /* Note: because reloc is assigned from something that
1127                    appears to be an array, gcc assumes it's non-null,
1128                    but doesn't know its relationship with start and
1129                    end. */
1130                 if (reloc > start && reloc < end) {
1131                         int reloc_off = reloc - start;
1132                         long *relocp = (long *)(insnbuf + reloc_off);
1133                         long delta = start - (char *)addr;
1134
1135                         *relocp += delta;
1136                 }
1137                 break;
1138
1139         default_patch:
1140         default:
1141                 ret = paravirt_patch_default(type, clobbers, insnbuf,
1142                                              addr, len);
1143                 break;
1144         }
1145
1146         return ret;
1147 }
1148
1149 static void xen_set_fixmap(unsigned idx, unsigned long phys, pgprot_t prot)
1150 {
1151         pte_t pte;
1152
1153         phys >>= PAGE_SHIFT;
1154
1155         switch (idx) {
1156         case FIX_BTMAP_END ... FIX_BTMAP_BEGIN:
1157 #ifdef CONFIG_X86_F00F_BUG
1158         case FIX_F00F_IDT:
1159 #endif
1160 #ifdef CONFIG_X86_32
1161         case FIX_WP_TEST:
1162         case FIX_VDSO:
1163 # ifdef CONFIG_HIGHMEM
1164         case FIX_KMAP_BEGIN ... FIX_KMAP_END:
1165 # endif
1166 #else
1167         case VSYSCALL_LAST_PAGE ... VSYSCALL_FIRST_PAGE:
1168 #endif
1169 #ifdef CONFIG_X86_LOCAL_APIC
1170         case FIX_APIC_BASE:     /* maps dummy local APIC */
1171 #endif
1172                 pte = pfn_pte(phys, prot);
1173                 break;
1174
1175         default:
1176                 pte = mfn_pte(phys, prot);
1177                 break;
1178         }
1179
1180         __native_set_fixmap(idx, pte);
1181
1182 #ifdef CONFIG_X86_64
1183         /* Replicate changes to map the vsyscall page into the user
1184            pagetable vsyscall mapping. */
1185         if (idx >= VSYSCALL_LAST_PAGE && idx <= VSYSCALL_FIRST_PAGE) {
1186                 unsigned long vaddr = __fix_to_virt(idx);
1187                 set_pte_vaddr_pud(level3_user_vsyscall, vaddr, pte);
1188         }
1189 #endif
1190 }
1191
1192 static const struct pv_info xen_info __initdata = {
1193         .paravirt_enabled = 1,
1194         .shared_kernel_pmd = 0,
1195
1196         .name = "Xen",
1197 };
1198
1199 static const struct pv_init_ops xen_init_ops __initdata = {
1200         .patch = xen_patch,
1201
1202         .banner = xen_banner,
1203         .memory_setup = xen_memory_setup,
1204         .arch_setup = xen_arch_setup,
1205         .post_allocator_init = xen_post_allocator_init,
1206 };
1207
1208 static const struct pv_time_ops xen_time_ops __initdata = {
1209         .time_init = xen_time_init,
1210
1211         .set_wallclock = xen_set_wallclock,
1212         .get_wallclock = xen_get_wallclock,
1213         .get_tsc_khz = xen_tsc_khz,
1214         .sched_clock = xen_sched_clock,
1215 };
1216
1217 static const struct pv_cpu_ops xen_cpu_ops __initdata = {
1218         .cpuid = xen_cpuid,
1219
1220         .set_debugreg = xen_set_debugreg,
1221         .get_debugreg = xen_get_debugreg,
1222
1223         .clts = xen_clts,
1224
1225         .read_cr0 = native_read_cr0,
1226         .write_cr0 = xen_write_cr0,
1227
1228         .read_cr4 = native_read_cr4,
1229         .read_cr4_safe = native_read_cr4_safe,
1230         .write_cr4 = xen_write_cr4,
1231
1232         .wbinvd = native_wbinvd,
1233
1234         .read_msr = native_read_msr_safe,
1235         .write_msr = xen_write_msr_safe,
1236         .read_tsc = native_read_tsc,
1237         .read_pmc = native_read_pmc,
1238
1239         .iret = xen_iret,
1240         .irq_enable_sysexit = xen_sysexit,
1241 #ifdef CONFIG_X86_64
1242         .usergs_sysret32 = xen_sysret32,
1243         .usergs_sysret64 = xen_sysret64,
1244 #endif
1245
1246         .load_tr_desc = paravirt_nop,
1247         .set_ldt = xen_set_ldt,
1248         .load_gdt = xen_load_gdt,
1249         .load_idt = xen_load_idt,
1250         .load_tls = xen_load_tls,
1251 #ifdef CONFIG_X86_64
1252         .load_gs_index = xen_load_gs_index,
1253 #endif
1254
1255         .store_gdt = native_store_gdt,
1256         .store_idt = native_store_idt,
1257         .store_tr = xen_store_tr,
1258
1259         .write_ldt_entry = xen_write_ldt_entry,
1260         .write_gdt_entry = xen_write_gdt_entry,
1261         .write_idt_entry = xen_write_idt_entry,
1262         .load_sp0 = xen_load_sp0,
1263
1264         .set_iopl_mask = xen_set_iopl_mask,
1265         .io_delay = xen_io_delay,
1266
1267         /* Xen takes care of %gs when switching to usermode for us */
1268         .swapgs = paravirt_nop,
1269
1270         .lazy_mode = {
1271                 .enter = paravirt_enter_lazy_cpu,
1272                 .leave = xen_leave_lazy,
1273         },
1274 };
1275
1276 static void __init __xen_init_IRQ(void)
1277 {
1278 #ifdef CONFIG_X86_64
1279         int i;
1280
1281         /* Create identity vector->irq map */
1282         for(i = 0; i < NR_VECTORS; i++) {
1283                 int cpu;
1284
1285                 for_each_possible_cpu(cpu)
1286                         per_cpu(vector_irq, cpu)[i] = i;
1287         }
1288 #endif  /* CONFIG_X86_64 */
1289
1290         xen_init_IRQ();
1291 }
1292
1293 static const struct pv_irq_ops xen_irq_ops __initdata = {
1294         .init_IRQ = __xen_init_IRQ,
1295         .save_fl = xen_save_fl,
1296         .restore_fl = xen_restore_fl,
1297         .irq_disable = xen_irq_disable,
1298         .irq_enable = xen_irq_enable,
1299         .safe_halt = xen_safe_halt,
1300         .halt = xen_halt,
1301 #ifdef CONFIG_X86_64
1302         .adjust_exception_frame = xen_adjust_exception_frame,
1303 #endif
1304 };
1305
1306 static const struct pv_apic_ops xen_apic_ops __initdata = {
1307 #ifdef CONFIG_X86_LOCAL_APIC
1308         .setup_boot_clock = paravirt_nop,
1309         .setup_secondary_clock = paravirt_nop,
1310         .startup_ipi_hook = paravirt_nop,
1311 #endif
1312 };
1313
1314 static const struct pv_mmu_ops xen_mmu_ops __initdata = {
1315         .pagetable_setup_start = xen_pagetable_setup_start,
1316         .pagetable_setup_done = xen_pagetable_setup_done,
1317
1318         .read_cr2 = xen_read_cr2,
1319         .write_cr2 = xen_write_cr2,
1320
1321         .read_cr3 = xen_read_cr3,
1322         .write_cr3 = xen_write_cr3,
1323
1324         .flush_tlb_user = xen_flush_tlb,
1325         .flush_tlb_kernel = xen_flush_tlb,
1326         .flush_tlb_single = xen_flush_tlb_single,
1327         .flush_tlb_others = xen_flush_tlb_others,
1328
1329         .pte_update = paravirt_nop,
1330         .pte_update_defer = paravirt_nop,
1331
1332         .pgd_alloc = xen_pgd_alloc,
1333         .pgd_free = xen_pgd_free,
1334
1335         .alloc_pte = xen_alloc_pte_init,
1336         .release_pte = xen_release_pte_init,
1337         .alloc_pmd = xen_alloc_pte_init,
1338         .alloc_pmd_clone = paravirt_nop,
1339         .release_pmd = xen_release_pte_init,
1340
1341 #ifdef CONFIG_HIGHPTE
1342         .kmap_atomic_pte = xen_kmap_atomic_pte,
1343 #endif
1344
1345 #ifdef CONFIG_X86_64
1346         .set_pte = xen_set_pte,
1347 #else
1348         .set_pte = xen_set_pte_init,
1349 #endif
1350         .set_pte_at = xen_set_pte_at,
1351         .set_pmd = xen_set_pmd_hyper,
1352
1353         .ptep_modify_prot_start = __ptep_modify_prot_start,
1354         .ptep_modify_prot_commit = __ptep_modify_prot_commit,
1355
1356         .pte_val = xen_pte_val,
1357         .pte_flags = native_pte_flags,
1358         .pgd_val = xen_pgd_val,
1359
1360         .make_pte = xen_make_pte,
1361         .make_pgd = xen_make_pgd,
1362
1363 #ifdef CONFIG_X86_PAE
1364         .set_pte_atomic = xen_set_pte_atomic,
1365         .set_pte_present = xen_set_pte_at,
1366         .pte_clear = xen_pte_clear,
1367         .pmd_clear = xen_pmd_clear,
1368 #endif  /* CONFIG_X86_PAE */
1369         .set_pud = xen_set_pud_hyper,
1370
1371         .make_pmd = xen_make_pmd,
1372         .pmd_val = xen_pmd_val,
1373
1374 #if PAGETABLE_LEVELS == 4
1375         .pud_val = xen_pud_val,
1376         .make_pud = xen_make_pud,
1377         .set_pgd = xen_set_pgd_hyper,
1378
1379         .alloc_pud = xen_alloc_pte_init,
1380         .release_pud = xen_release_pte_init,
1381 #endif  /* PAGETABLE_LEVELS == 4 */
1382
1383         .activate_mm = xen_activate_mm,
1384         .dup_mmap = xen_dup_mmap,
1385         .exit_mmap = xen_exit_mmap,
1386
1387         .lazy_mode = {
1388                 .enter = paravirt_enter_lazy_mmu,
1389                 .leave = xen_leave_lazy,
1390         },
1391
1392         .set_fixmap = xen_set_fixmap,
1393 };
1394
1395 static void xen_reboot(int reason)
1396 {
1397         struct sched_shutdown r = { .reason = reason };
1398
1399 #ifdef CONFIG_SMP
1400         smp_send_stop();
1401 #endif
1402
1403         if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))
1404                 BUG();
1405 }
1406
1407 static void xen_restart(char *msg)
1408 {
1409         xen_reboot(SHUTDOWN_reboot);
1410 }
1411
1412 static void xen_emergency_restart(void)
1413 {
1414         xen_reboot(SHUTDOWN_reboot);
1415 }
1416
1417 static void xen_machine_halt(void)
1418 {
1419         xen_reboot(SHUTDOWN_poweroff);
1420 }
1421
1422 static void xen_crash_shutdown(struct pt_regs *regs)
1423 {
1424         xen_reboot(SHUTDOWN_crash);
1425 }
1426
1427 static const struct machine_ops __initdata xen_machine_ops = {
1428         .restart = xen_restart,
1429         .halt = xen_machine_halt,
1430         .power_off = xen_machine_halt,
1431         .shutdown = xen_machine_halt,
1432         .crash_shutdown = xen_crash_shutdown,
1433         .emergency_restart = xen_emergency_restart,
1434 };
1435
1436
1437 static void __init xen_reserve_top(void)
1438 {
1439 #ifdef CONFIG_X86_32
1440         unsigned long top = HYPERVISOR_VIRT_START;
1441         struct xen_platform_parameters pp;
1442
1443         if (HYPERVISOR_xen_version(XENVER_platform_parameters, &pp) == 0)
1444                 top = pp.virt_start;
1445
1446         reserve_top_address(-top + 2 * PAGE_SIZE);
1447 #endif  /* CONFIG_X86_32 */
1448 }
1449
1450 /*
1451  * Like __va(), but returns address in the kernel mapping (which is
1452  * all we have until the physical memory mapping has been set up.
1453  */
1454 static void *__ka(phys_addr_t paddr)
1455 {
1456 #ifdef CONFIG_X86_64
1457         return (void *)(paddr + __START_KERNEL_map);
1458 #else
1459         return __va(paddr);
1460 #endif
1461 }
1462
1463 /* Convert a machine address to physical address */
1464 static unsigned long m2p(phys_addr_t maddr)
1465 {
1466         phys_addr_t paddr;
1467
1468         maddr &= PTE_PFN_MASK;
1469         paddr = mfn_to_pfn(maddr >> PAGE_SHIFT) << PAGE_SHIFT;
1470
1471         return paddr;
1472 }
1473
1474 /* Convert a machine address to kernel virtual */
1475 static void *m2v(phys_addr_t maddr)
1476 {
1477         return __ka(m2p(maddr));
1478 }
1479
1480 #ifdef CONFIG_X86_64
1481 static void walk(pgd_t *pgd, unsigned long addr)
1482 {
1483         unsigned l4idx = pgd_index(addr);
1484         unsigned l3idx = pud_index(addr);
1485         unsigned l2idx = pmd_index(addr);
1486         unsigned l1idx = pte_index(addr);
1487         pgd_t l4;
1488         pud_t l3;
1489         pmd_t l2;
1490         pte_t l1;
1491
1492         xen_raw_printk("walk %p, %lx -> %d %d %d %d\n",
1493                        pgd, addr, l4idx, l3idx, l2idx, l1idx);
1494
1495         l4 = pgd[l4idx];
1496         xen_raw_printk("  l4: %016lx\n", l4.pgd);
1497         xen_raw_printk("      %016lx\n", pgd_val(l4));
1498
1499         l3 = ((pud_t *)(m2v(l4.pgd)))[l3idx];
1500         xen_raw_printk("  l3: %016lx\n", l3.pud);
1501         xen_raw_printk("      %016lx\n", pud_val(l3));
1502
1503         l2 = ((pmd_t *)(m2v(l3.pud)))[l2idx];
1504         xen_raw_printk("  l2: %016lx\n", l2.pmd);
1505         xen_raw_printk("      %016lx\n", pmd_val(l2));
1506
1507         l1 = ((pte_t *)(m2v(l2.pmd)))[l1idx];
1508         xen_raw_printk("  l1: %016lx\n", l1.pte);
1509         xen_raw_printk("      %016lx\n", pte_val(l1));
1510 }
1511 #endif
1512
1513 static void set_page_prot(void *addr, pgprot_t prot)
1514 {
1515         unsigned long pfn = __pa(addr) >> PAGE_SHIFT;
1516         pte_t pte = pfn_pte(pfn, prot);
1517
1518         xen_raw_printk("addr=%p pfn=%lx mfn=%lx prot=%016llx pte=%016llx\n",
1519                        addr, pfn, get_phys_to_machine(pfn),
1520                        pgprot_val(prot), pte.pte);
1521
1522         if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, 0))
1523                 BUG();
1524 }
1525
1526 static __init void xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn)
1527 {
1528         unsigned pmdidx, pteidx;
1529         unsigned ident_pte;
1530         unsigned long pfn;
1531
1532         ident_pte = 0;
1533         pfn = 0;
1534         for(pmdidx = 0; pmdidx < PTRS_PER_PMD && pfn < max_pfn; pmdidx++) {
1535                 pte_t *pte_page;
1536
1537                 /* Reuse or allocate a page of ptes */
1538                 if (pmd_present(pmd[pmdidx]))
1539                         pte_page = m2v(pmd[pmdidx].pmd);
1540                 else {
1541                         /* Check for free pte pages */
1542                         if (ident_pte == ARRAY_SIZE(level1_ident_pgt))
1543                                 break;
1544
1545                         pte_page = &level1_ident_pgt[ident_pte];
1546                         ident_pte += PTRS_PER_PTE;
1547
1548                         pmd[pmdidx] = __pmd(__pa(pte_page) | _PAGE_TABLE);
1549                 }
1550
1551                 /* Install mappings */
1552                 for(pteidx = 0; pteidx < PTRS_PER_PTE; pteidx++, pfn++) {
1553                         pte_t pte;
1554
1555                         if (pfn > max_pfn_mapped)
1556                                 max_pfn_mapped = pfn;
1557
1558                         if (!pte_none(pte_page[pteidx]))
1559                                 continue;
1560
1561                         pte = pfn_pte(pfn, PAGE_KERNEL_EXEC);
1562                         pte_page[pteidx] = pte;
1563                 }
1564         }
1565
1566         for(pteidx = 0; pteidx < ident_pte; pteidx += PTRS_PER_PTE)
1567                 set_page_prot(&level1_ident_pgt[pteidx], PAGE_KERNEL_RO);
1568
1569         set_page_prot(pmd, PAGE_KERNEL_RO);
1570 }
1571
1572 #ifdef CONFIG_X86_64
1573 static void convert_pfn_mfn(void *v)
1574 {
1575         pte_t *pte = v;
1576         int i;
1577
1578         /* All levels are converted the same way, so just treat them
1579            as ptes. */
1580         for(i = 0; i < PTRS_PER_PTE; i++)
1581                 pte[i] = xen_make_pte(pte[i].pte);
1582 }
1583
1584 /*
1585  * Set up the inital kernel pagetable.
1586  *
1587  * We can construct this by grafting the Xen provided pagetable into
1588  * head_64.S's preconstructed pagetables.  We copy the Xen L2's into
1589  * level2_ident_pgt, level2_kernel_pgt and level2_fixmap_pgt.  This
1590  * means that only the kernel has a physical mapping to start with -
1591  * but that's enough to get __va working.  We need to fill in the rest
1592  * of the physical mapping once some sort of allocator has been set
1593  * up.
1594  */
1595 static __init pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
1596 {
1597         pud_t *l3;
1598         pmd_t *l2;
1599
1600         /* Zap identity mapping */
1601         init_level4_pgt[0] = __pgd(0);
1602
1603         /* Pre-constructed entries are in pfn, so convert to mfn */
1604         convert_pfn_mfn(init_level4_pgt);
1605         convert_pfn_mfn(level3_ident_pgt);
1606         convert_pfn_mfn(level3_kernel_pgt);
1607
1608         l3 = m2v(pgd[pgd_index(__START_KERNEL_map)].pgd);
1609         l2 = m2v(l3[pud_index(__START_KERNEL_map)].pud);
1610
1611         memcpy(level2_ident_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
1612         memcpy(level2_kernel_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
1613
1614         l3 = m2v(pgd[pgd_index(__START_KERNEL_map + PMD_SIZE)].pgd);
1615         l2 = m2v(l3[pud_index(__START_KERNEL_map + PMD_SIZE)].pud);
1616         memcpy(level2_fixmap_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
1617
1618         /* Set up identity map */
1619         xen_map_identity_early(level2_ident_pgt, max_pfn);
1620
1621         /* Make pagetable pieces RO */
1622         set_page_prot(init_level4_pgt, PAGE_KERNEL_RO);
1623         set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO);
1624         set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO);
1625         set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO);
1626         set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
1627         set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO);
1628
1629         /* Pin down new L4 */
1630         pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE,
1631                           PFN_DOWN(__pa_symbol(init_level4_pgt)));
1632
1633         /* Unpin Xen-provided one */
1634         pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
1635
1636         /* Switch over */
1637         pgd = init_level4_pgt;
1638
1639         /*
1640          * At this stage there can be no user pgd, and no page
1641          * structure to attach it to, so make sure we just set kernel
1642          * pgd.
1643          */
1644         xen_mc_batch();
1645         __xen_write_cr3(true, __pa(pgd));
1646         xen_mc_issue(PARAVIRT_LAZY_CPU);
1647
1648         reserve_early(__pa(xen_start_info->pt_base),
1649                       __pa(xen_start_info->pt_base +
1650                            xen_start_info->nr_pt_frames * PAGE_SIZE),
1651                       "XEN PAGETABLES");
1652
1653         return pgd;
1654 }
1655 #else   /* !CONFIG_X86_64 */
1656 static pmd_t level2_kernel_pgt[PTRS_PER_PMD] __page_aligned_bss;
1657
1658 static __init pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
1659 {
1660         pmd_t *kernel_pmd;
1661
1662         init_pg_tables_start = __pa(pgd);
1663         init_pg_tables_end = __pa(pgd) + xen_start_info->nr_pt_frames*PAGE_SIZE;
1664         max_pfn_mapped = PFN_DOWN(init_pg_tables_end + 512*1024);
1665
1666         kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd);
1667         memcpy(level2_kernel_pgt, kernel_pmd, sizeof(pmd_t) * PTRS_PER_PMD);
1668
1669         xen_map_identity_early(level2_kernel_pgt, max_pfn);
1670
1671         memcpy(swapper_pg_dir, pgd, sizeof(pgd_t) * PTRS_PER_PGD);
1672         set_pgd(&swapper_pg_dir[KERNEL_PGD_BOUNDARY],
1673                         __pgd(__pa(level2_kernel_pgt) | _PAGE_PRESENT));
1674
1675         set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
1676         set_page_prot(swapper_pg_dir, PAGE_KERNEL_RO);
1677         set_page_prot(empty_zero_page, PAGE_KERNEL_RO);
1678
1679         pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
1680
1681         xen_write_cr3(__pa(swapper_pg_dir));
1682
1683         pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(swapper_pg_dir)));
1684
1685         return swapper_pg_dir;
1686 }
1687 #endif  /* CONFIG_X86_64 */
1688
1689 /* First C function to be called on Xen boot */
1690 asmlinkage void __init xen_start_kernel(void)
1691 {
1692         pgd_t *pgd;
1693
1694         if (!xen_start_info)
1695                 return;
1696
1697         BUG_ON(memcmp(xen_start_info->magic, "xen-3", 5) != 0);
1698
1699         xen_setup_features();
1700
1701         /* Install Xen paravirt ops */
1702         pv_info = xen_info;
1703         pv_init_ops = xen_init_ops;
1704         pv_time_ops = xen_time_ops;
1705         pv_cpu_ops = xen_cpu_ops;
1706         pv_irq_ops = xen_irq_ops;
1707         pv_apic_ops = xen_apic_ops;
1708         pv_mmu_ops = xen_mmu_ops;
1709
1710 #ifdef CONFIG_X86_LOCAL_APIC
1711         /*
1712          * set up the basic apic ops.
1713          */
1714         apic_ops = &xen_basic_apic_ops;
1715 #endif
1716
1717         if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) {
1718                 pv_mmu_ops.ptep_modify_prot_start = xen_ptep_modify_prot_start;
1719                 pv_mmu_ops.ptep_modify_prot_commit = xen_ptep_modify_prot_commit;
1720         }
1721
1722         machine_ops = xen_machine_ops;
1723
1724 #ifdef CONFIG_X86_64
1725         /* Disable until direct per-cpu data access. */
1726         have_vcpu_info_placement = 0;
1727         x86_64_init_pda();
1728 #endif
1729
1730         xen_smp_init();
1731
1732         /* Get mfn list */
1733         if (!xen_feature(XENFEAT_auto_translated_physmap))
1734                 xen_build_dynamic_phys_to_machine();
1735
1736         pgd = (pgd_t *)xen_start_info->pt_base;
1737
1738         /* Prevent unwanted bits from being set in PTEs. */
1739         __supported_pte_mask &= ~_PAGE_GLOBAL;
1740         if (!is_initial_xendomain())
1741                 __supported_pte_mask &= ~(_PAGE_PWT | _PAGE_PCD);
1742
1743         /* Don't do the full vcpu_info placement stuff until we have a
1744            possible map and a non-dummy shared_info. */
1745         per_cpu(xen_vcpu, 0) = &HYPERVISOR_shared_info->vcpu_info[0];
1746
1747         xen_raw_console_write("mapping kernel into physical memory\n");
1748         pgd = xen_setup_kernel_pagetable(pgd, xen_start_info->nr_pages);
1749
1750         init_mm.pgd = pgd;
1751
1752         /* keep using Xen gdt for now; no urgent need to change it */
1753
1754         pv_info.kernel_rpl = 1;
1755         if (xen_feature(XENFEAT_supervisor_mode_kernel))
1756                 pv_info.kernel_rpl = 0;
1757
1758         /* set the limit of our address space */
1759         xen_reserve_top();
1760
1761 #ifdef CONFIG_X86_32
1762         /* set up basic CPUID stuff */
1763         cpu_detect(&new_cpu_data);
1764         new_cpu_data.hard_math = 1;
1765         new_cpu_data.x86_capability[0] = cpuid_edx(1);
1766 #endif
1767
1768         /* Poke various useful things into boot_params */
1769         boot_params.hdr.type_of_loader = (9 << 4) | 0;
1770         boot_params.hdr.ramdisk_image = xen_start_info->mod_start
1771                 ? __pa(xen_start_info->mod_start) : 0;
1772         boot_params.hdr.ramdisk_size = xen_start_info->mod_len;
1773         boot_params.hdr.cmd_line_ptr = __pa(xen_start_info->cmd_line);
1774
1775         if (!is_initial_xendomain()) {
1776                 add_preferred_console("xenboot", 0, NULL);
1777                 add_preferred_console("tty", 0, NULL);
1778                 add_preferred_console("hvc", 0, NULL);
1779         }
1780
1781         xen_raw_console_write("about to get started...\n");
1782
1783 #if 0
1784         xen_raw_printk("&boot_params=%p __pa(&boot_params)=%lx __va(__pa(&boot_params))=%lx\n",
1785                        &boot_params, __pa_symbol(&boot_params),
1786                        __va(__pa_symbol(&boot_params)));
1787
1788         walk(pgd, &boot_params);
1789         walk(pgd, __va(__pa(&boot_params)));
1790 #endif
1791
1792         /* Start the world */
1793 #ifdef CONFIG_X86_32
1794         i386_start_kernel();
1795 #else
1796         x86_64_start_reservations((char *)__pa_symbol(&boot_params));
1797 #endif
1798 }