x86: Move xen_post_allocator_init into xen_pagetable_setup_done
[linux-2.6.git] / arch / x86 / include / asm / paravirt.h
1 #ifndef _ASM_X86_PARAVIRT_H
2 #define _ASM_X86_PARAVIRT_H
3 /* Various instructions on x86 need to be replaced for
4  * para-virtualization: those hooks are defined here. */
5
6 #ifdef CONFIG_PARAVIRT
7 #include <asm/pgtable_types.h>
8 #include <asm/asm.h>
9
10 #include <asm/paravirt_types.h>
11
12 #ifndef __ASSEMBLY__
13 #include <linux/types.h>
14 #include <linux/cpumask.h>
15
16 static inline int paravirt_enabled(void)
17 {
18         return pv_info.paravirt_enabled;
19 }
20
21 static inline void load_sp0(struct tss_struct *tss,
22                              struct thread_struct *thread)
23 {
24         PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread);
25 }
26
27 static inline unsigned long get_wallclock(void)
28 {
29         return PVOP_CALL0(unsigned long, pv_time_ops.get_wallclock);
30 }
31
32 static inline int set_wallclock(unsigned long nowtime)
33 {
34         return PVOP_CALL1(int, pv_time_ops.set_wallclock, nowtime);
35 }
36
37 static inline void (*choose_time_init(void))(void)
38 {
39         return pv_time_ops.time_init;
40 }
41
42 /* The paravirtualized CPUID instruction. */
43 static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
44                            unsigned int *ecx, unsigned int *edx)
45 {
46         PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx);
47 }
48
49 /*
50  * These special macros can be used to get or set a debugging register
51  */
52 static inline unsigned long paravirt_get_debugreg(int reg)
53 {
54         return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg);
55 }
56 #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
57 static inline void set_debugreg(unsigned long val, int reg)
58 {
59         PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val);
60 }
61
62 static inline void clts(void)
63 {
64         PVOP_VCALL0(pv_cpu_ops.clts);
65 }
66
67 static inline unsigned long read_cr0(void)
68 {
69         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0);
70 }
71
72 static inline void write_cr0(unsigned long x)
73 {
74         PVOP_VCALL1(pv_cpu_ops.write_cr0, x);
75 }
76
77 static inline unsigned long read_cr2(void)
78 {
79         return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2);
80 }
81
82 static inline void write_cr2(unsigned long x)
83 {
84         PVOP_VCALL1(pv_mmu_ops.write_cr2, x);
85 }
86
87 static inline unsigned long read_cr3(void)
88 {
89         return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3);
90 }
91
92 static inline void write_cr3(unsigned long x)
93 {
94         PVOP_VCALL1(pv_mmu_ops.write_cr3, x);
95 }
96
97 static inline unsigned long read_cr4(void)
98 {
99         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4);
100 }
101 static inline unsigned long read_cr4_safe(void)
102 {
103         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe);
104 }
105
106 static inline void write_cr4(unsigned long x)
107 {
108         PVOP_VCALL1(pv_cpu_ops.write_cr4, x);
109 }
110
111 #ifdef CONFIG_X86_64
112 static inline unsigned long read_cr8(void)
113 {
114         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8);
115 }
116
117 static inline void write_cr8(unsigned long x)
118 {
119         PVOP_VCALL1(pv_cpu_ops.write_cr8, x);
120 }
121 #endif
122
123 static inline void raw_safe_halt(void)
124 {
125         PVOP_VCALL0(pv_irq_ops.safe_halt);
126 }
127
128 static inline void halt(void)
129 {
130         PVOP_VCALL0(pv_irq_ops.safe_halt);
131 }
132
133 static inline void wbinvd(void)
134 {
135         PVOP_VCALL0(pv_cpu_ops.wbinvd);
136 }
137
138 #define get_kernel_rpl()  (pv_info.kernel_rpl)
139
140 static inline u64 paravirt_read_msr(unsigned msr, int *err)
141 {
142         return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err);
143 }
144 static inline u64 paravirt_read_msr_amd(unsigned msr, int *err)
145 {
146         return PVOP_CALL2(u64, pv_cpu_ops.read_msr_amd, msr, err);
147 }
148 static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high)
149 {
150         return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high);
151 }
152
153 /* These should all do BUG_ON(_err), but our headers are too tangled. */
154 #define rdmsr(msr, val1, val2)                  \
155 do {                                            \
156         int _err;                               \
157         u64 _l = paravirt_read_msr(msr, &_err); \
158         val1 = (u32)_l;                         \
159         val2 = _l >> 32;                        \
160 } while (0)
161
162 #define wrmsr(msr, val1, val2)                  \
163 do {                                            \
164         paravirt_write_msr(msr, val1, val2);    \
165 } while (0)
166
167 #define rdmsrl(msr, val)                        \
168 do {                                            \
169         int _err;                               \
170         val = paravirt_read_msr(msr, &_err);    \
171 } while (0)
172
173 #define wrmsrl(msr, val)        wrmsr(msr, (u32)((u64)(val)), ((u64)(val))>>32)
174 #define wrmsr_safe(msr, a, b)   paravirt_write_msr(msr, a, b)
175
176 /* rdmsr with exception handling */
177 #define rdmsr_safe(msr, a, b)                   \
178 ({                                              \
179         int _err;                               \
180         u64 _l = paravirt_read_msr(msr, &_err); \
181         (*a) = (u32)_l;                         \
182         (*b) = _l >> 32;                        \
183         _err;                                   \
184 })
185
186 static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
187 {
188         int err;
189
190         *p = paravirt_read_msr(msr, &err);
191         return err;
192 }
193 static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
194 {
195         int err;
196
197         *p = paravirt_read_msr_amd(msr, &err);
198         return err;
199 }
200
201 static inline u64 paravirt_read_tsc(void)
202 {
203         return PVOP_CALL0(u64, pv_cpu_ops.read_tsc);
204 }
205
206 #define rdtscl(low)                             \
207 do {                                            \
208         u64 _l = paravirt_read_tsc();           \
209         low = (int)_l;                          \
210 } while (0)
211
212 #define rdtscll(val) (val = paravirt_read_tsc())
213
214 static inline unsigned long long paravirt_sched_clock(void)
215 {
216         return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
217 }
218 #define calibrate_tsc() (pv_time_ops.get_tsc_khz())
219
220 static inline unsigned long long paravirt_read_pmc(int counter)
221 {
222         return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter);
223 }
224
225 #define rdpmc(counter, low, high)               \
226 do {                                            \
227         u64 _l = paravirt_read_pmc(counter);    \
228         low = (u32)_l;                          \
229         high = _l >> 32;                        \
230 } while (0)
231
232 static inline unsigned long long paravirt_rdtscp(unsigned int *aux)
233 {
234         return PVOP_CALL1(u64, pv_cpu_ops.read_tscp, aux);
235 }
236
237 #define rdtscp(low, high, aux)                          \
238 do {                                                    \
239         int __aux;                                      \
240         unsigned long __val = paravirt_rdtscp(&__aux);  \
241         (low) = (u32)__val;                             \
242         (high) = (u32)(__val >> 32);                    \
243         (aux) = __aux;                                  \
244 } while (0)
245
246 #define rdtscpll(val, aux)                              \
247 do {                                                    \
248         unsigned long __aux;                            \
249         val = paravirt_rdtscp(&__aux);                  \
250         (aux) = __aux;                                  \
251 } while (0)
252
253 static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
254 {
255         PVOP_VCALL2(pv_cpu_ops.alloc_ldt, ldt, entries);
256 }
257
258 static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries)
259 {
260         PVOP_VCALL2(pv_cpu_ops.free_ldt, ldt, entries);
261 }
262
263 static inline void load_TR_desc(void)
264 {
265         PVOP_VCALL0(pv_cpu_ops.load_tr_desc);
266 }
267 static inline void load_gdt(const struct desc_ptr *dtr)
268 {
269         PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr);
270 }
271 static inline void load_idt(const struct desc_ptr *dtr)
272 {
273         PVOP_VCALL1(pv_cpu_ops.load_idt, dtr);
274 }
275 static inline void set_ldt(const void *addr, unsigned entries)
276 {
277         PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries);
278 }
279 static inline void store_gdt(struct desc_ptr *dtr)
280 {
281         PVOP_VCALL1(pv_cpu_ops.store_gdt, dtr);
282 }
283 static inline void store_idt(struct desc_ptr *dtr)
284 {
285         PVOP_VCALL1(pv_cpu_ops.store_idt, dtr);
286 }
287 static inline unsigned long paravirt_store_tr(void)
288 {
289         return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr);
290 }
291 #define store_tr(tr)    ((tr) = paravirt_store_tr())
292 static inline void load_TLS(struct thread_struct *t, unsigned cpu)
293 {
294         PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu);
295 }
296
297 #ifdef CONFIG_X86_64
298 static inline void load_gs_index(unsigned int gs)
299 {
300         PVOP_VCALL1(pv_cpu_ops.load_gs_index, gs);
301 }
302 #endif
303
304 static inline void write_ldt_entry(struct desc_struct *dt, int entry,
305                                    const void *desc)
306 {
307         PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc);
308 }
309
310 static inline void write_gdt_entry(struct desc_struct *dt, int entry,
311                                    void *desc, int type)
312 {
313         PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type);
314 }
315
316 static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g)
317 {
318         PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g);
319 }
320 static inline void set_iopl_mask(unsigned mask)
321 {
322         PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask);
323 }
324
325 /* The paravirtualized I/O functions */
326 static inline void slow_down_io(void)
327 {
328         pv_cpu_ops.io_delay();
329 #ifdef REALLY_SLOW_IO
330         pv_cpu_ops.io_delay();
331         pv_cpu_ops.io_delay();
332         pv_cpu_ops.io_delay();
333 #endif
334 }
335
336 #ifdef CONFIG_X86_LOCAL_APIC
337 static inline void setup_boot_clock(void)
338 {
339         PVOP_VCALL0(pv_apic_ops.setup_boot_clock);
340 }
341
342 static inline void setup_secondary_clock(void)
343 {
344         PVOP_VCALL0(pv_apic_ops.setup_secondary_clock);
345 }
346 #endif
347
348 #ifdef CONFIG_SMP
349 static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip,
350                                     unsigned long start_esp)
351 {
352         PVOP_VCALL3(pv_apic_ops.startup_ipi_hook,
353                     phys_apicid, start_eip, start_esp);
354 }
355 #endif
356
357 static inline void paravirt_activate_mm(struct mm_struct *prev,
358                                         struct mm_struct *next)
359 {
360         PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next);
361 }
362
363 static inline void arch_dup_mmap(struct mm_struct *oldmm,
364                                  struct mm_struct *mm)
365 {
366         PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm);
367 }
368
369 static inline void arch_exit_mmap(struct mm_struct *mm)
370 {
371         PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm);
372 }
373
374 static inline void __flush_tlb(void)
375 {
376         PVOP_VCALL0(pv_mmu_ops.flush_tlb_user);
377 }
378 static inline void __flush_tlb_global(void)
379 {
380         PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel);
381 }
382 static inline void __flush_tlb_single(unsigned long addr)
383 {
384         PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr);
385 }
386
387 static inline void flush_tlb_others(const struct cpumask *cpumask,
388                                     struct mm_struct *mm,
389                                     unsigned long va)
390 {
391         PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, cpumask, mm, va);
392 }
393
394 static inline int paravirt_pgd_alloc(struct mm_struct *mm)
395 {
396         return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm);
397 }
398
399 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd)
400 {
401         PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd);
402 }
403
404 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn)
405 {
406         PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn);
407 }
408 static inline void paravirt_release_pte(unsigned long pfn)
409 {
410         PVOP_VCALL1(pv_mmu_ops.release_pte, pfn);
411 }
412
413 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
414 {
415         PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn);
416 }
417
418 static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn,
419                                             unsigned long start, unsigned long count)
420 {
421         PVOP_VCALL4(pv_mmu_ops.alloc_pmd_clone, pfn, clonepfn, start, count);
422 }
423 static inline void paravirt_release_pmd(unsigned long pfn)
424 {
425         PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn);
426 }
427
428 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn)
429 {
430         PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn);
431 }
432 static inline void paravirt_release_pud(unsigned long pfn)
433 {
434         PVOP_VCALL1(pv_mmu_ops.release_pud, pfn);
435 }
436
437 #ifdef CONFIG_HIGHPTE
438 static inline void *kmap_atomic_pte(struct page *page, enum km_type type)
439 {
440         unsigned long ret;
441         ret = PVOP_CALL2(unsigned long, pv_mmu_ops.kmap_atomic_pte, page, type);
442         return (void *)ret;
443 }
444 #endif
445
446 static inline void pte_update(struct mm_struct *mm, unsigned long addr,
447                               pte_t *ptep)
448 {
449         PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep);
450 }
451
452 static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr,
453                                     pte_t *ptep)
454 {
455         PVOP_VCALL3(pv_mmu_ops.pte_update_defer, mm, addr, ptep);
456 }
457
458 static inline pte_t __pte(pteval_t val)
459 {
460         pteval_t ret;
461
462         if (sizeof(pteval_t) > sizeof(long))
463                 ret = PVOP_CALLEE2(pteval_t,
464                                    pv_mmu_ops.make_pte,
465                                    val, (u64)val >> 32);
466         else
467                 ret = PVOP_CALLEE1(pteval_t,
468                                    pv_mmu_ops.make_pte,
469                                    val);
470
471         return (pte_t) { .pte = ret };
472 }
473
474 static inline pteval_t pte_val(pte_t pte)
475 {
476         pteval_t ret;
477
478         if (sizeof(pteval_t) > sizeof(long))
479                 ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val,
480                                    pte.pte, (u64)pte.pte >> 32);
481         else
482                 ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val,
483                                    pte.pte);
484
485         return ret;
486 }
487
488 static inline pgd_t __pgd(pgdval_t val)
489 {
490         pgdval_t ret;
491
492         if (sizeof(pgdval_t) > sizeof(long))
493                 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd,
494                                    val, (u64)val >> 32);
495         else
496                 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd,
497                                    val);
498
499         return (pgd_t) { ret };
500 }
501
502 static inline pgdval_t pgd_val(pgd_t pgd)
503 {
504         pgdval_t ret;
505
506         if (sizeof(pgdval_t) > sizeof(long))
507                 ret =  PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val,
508                                     pgd.pgd, (u64)pgd.pgd >> 32);
509         else
510                 ret =  PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val,
511                                     pgd.pgd);
512
513         return ret;
514 }
515
516 #define  __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
517 static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
518                                            pte_t *ptep)
519 {
520         pteval_t ret;
521
522         ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start,
523                          mm, addr, ptep);
524
525         return (pte_t) { .pte = ret };
526 }
527
528 static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
529                                            pte_t *ptep, pte_t pte)
530 {
531         if (sizeof(pteval_t) > sizeof(long))
532                 /* 5 arg words */
533                 pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte);
534         else
535                 PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit,
536                             mm, addr, ptep, pte.pte);
537 }
538
539 static inline void set_pte(pte_t *ptep, pte_t pte)
540 {
541         if (sizeof(pteval_t) > sizeof(long))
542                 PVOP_VCALL3(pv_mmu_ops.set_pte, ptep,
543                             pte.pte, (u64)pte.pte >> 32);
544         else
545                 PVOP_VCALL2(pv_mmu_ops.set_pte, ptep,
546                             pte.pte);
547 }
548
549 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
550                               pte_t *ptep, pte_t pte)
551 {
552         if (sizeof(pteval_t) > sizeof(long))
553                 /* 5 arg words */
554                 pv_mmu_ops.set_pte_at(mm, addr, ptep, pte);
555         else
556                 PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte);
557 }
558
559 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
560 {
561         pmdval_t val = native_pmd_val(pmd);
562
563         if (sizeof(pmdval_t) > sizeof(long))
564                 PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32);
565         else
566                 PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val);
567 }
568
569 #if PAGETABLE_LEVELS >= 3
570 static inline pmd_t __pmd(pmdval_t val)
571 {
572         pmdval_t ret;
573
574         if (sizeof(pmdval_t) > sizeof(long))
575                 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd,
576                                    val, (u64)val >> 32);
577         else
578                 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd,
579                                    val);
580
581         return (pmd_t) { ret };
582 }
583
584 static inline pmdval_t pmd_val(pmd_t pmd)
585 {
586         pmdval_t ret;
587
588         if (sizeof(pmdval_t) > sizeof(long))
589                 ret =  PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val,
590                                     pmd.pmd, (u64)pmd.pmd >> 32);
591         else
592                 ret =  PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val,
593                                     pmd.pmd);
594
595         return ret;
596 }
597
598 static inline void set_pud(pud_t *pudp, pud_t pud)
599 {
600         pudval_t val = native_pud_val(pud);
601
602         if (sizeof(pudval_t) > sizeof(long))
603                 PVOP_VCALL3(pv_mmu_ops.set_pud, pudp,
604                             val, (u64)val >> 32);
605         else
606                 PVOP_VCALL2(pv_mmu_ops.set_pud, pudp,
607                             val);
608 }
609 #if PAGETABLE_LEVELS == 4
610 static inline pud_t __pud(pudval_t val)
611 {
612         pudval_t ret;
613
614         if (sizeof(pudval_t) > sizeof(long))
615                 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud,
616                                    val, (u64)val >> 32);
617         else
618                 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud,
619                                    val);
620
621         return (pud_t) { ret };
622 }
623
624 static inline pudval_t pud_val(pud_t pud)
625 {
626         pudval_t ret;
627
628         if (sizeof(pudval_t) > sizeof(long))
629                 ret =  PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val,
630                                     pud.pud, (u64)pud.pud >> 32);
631         else
632                 ret =  PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val,
633                                     pud.pud);
634
635         return ret;
636 }
637
638 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
639 {
640         pgdval_t val = native_pgd_val(pgd);
641
642         if (sizeof(pgdval_t) > sizeof(long))
643                 PVOP_VCALL3(pv_mmu_ops.set_pgd, pgdp,
644                             val, (u64)val >> 32);
645         else
646                 PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp,
647                             val);
648 }
649
650 static inline void pgd_clear(pgd_t *pgdp)
651 {
652         set_pgd(pgdp, __pgd(0));
653 }
654
655 static inline void pud_clear(pud_t *pudp)
656 {
657         set_pud(pudp, __pud(0));
658 }
659
660 #endif  /* PAGETABLE_LEVELS == 4 */
661
662 #endif  /* PAGETABLE_LEVELS >= 3 */
663
664 #ifdef CONFIG_X86_PAE
665 /* Special-case pte-setting operations for PAE, which can't update a
666    64-bit pte atomically */
667 static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
668 {
669         PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep,
670                     pte.pte, pte.pte >> 32);
671 }
672
673 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
674                              pte_t *ptep)
675 {
676         PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep);
677 }
678
679 static inline void pmd_clear(pmd_t *pmdp)
680 {
681         PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp);
682 }
683 #else  /* !CONFIG_X86_PAE */
684 static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
685 {
686         set_pte(ptep, pte);
687 }
688
689 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
690                              pte_t *ptep)
691 {
692         set_pte_at(mm, addr, ptep, __pte(0));
693 }
694
695 static inline void pmd_clear(pmd_t *pmdp)
696 {
697         set_pmd(pmdp, __pmd(0));
698 }
699 #endif  /* CONFIG_X86_PAE */
700
701 #define  __HAVE_ARCH_START_CONTEXT_SWITCH
702 static inline void arch_start_context_switch(struct task_struct *prev)
703 {
704         PVOP_VCALL1(pv_cpu_ops.start_context_switch, prev);
705 }
706
707 static inline void arch_end_context_switch(struct task_struct *next)
708 {
709         PVOP_VCALL1(pv_cpu_ops.end_context_switch, next);
710 }
711
712 #define  __HAVE_ARCH_ENTER_LAZY_MMU_MODE
713 static inline void arch_enter_lazy_mmu_mode(void)
714 {
715         PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter);
716 }
717
718 static inline void arch_leave_lazy_mmu_mode(void)
719 {
720         PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave);
721 }
722
723 void arch_flush_lazy_mmu_mode(void);
724
725 static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
726                                 phys_addr_t phys, pgprot_t flags)
727 {
728         pv_mmu_ops.set_fixmap(idx, phys, flags);
729 }
730
731 #if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)
732
733 static inline int __raw_spin_is_locked(struct raw_spinlock *lock)
734 {
735         return PVOP_CALL1(int, pv_lock_ops.spin_is_locked, lock);
736 }
737
738 static inline int __raw_spin_is_contended(struct raw_spinlock *lock)
739 {
740         return PVOP_CALL1(int, pv_lock_ops.spin_is_contended, lock);
741 }
742 #define __raw_spin_is_contended __raw_spin_is_contended
743
744 static __always_inline void __raw_spin_lock(struct raw_spinlock *lock)
745 {
746         PVOP_VCALL1(pv_lock_ops.spin_lock, lock);
747 }
748
749 static __always_inline void __raw_spin_lock_flags(struct raw_spinlock *lock,
750                                                   unsigned long flags)
751 {
752         PVOP_VCALL2(pv_lock_ops.spin_lock_flags, lock, flags);
753 }
754
755 static __always_inline int __raw_spin_trylock(struct raw_spinlock *lock)
756 {
757         return PVOP_CALL1(int, pv_lock_ops.spin_trylock, lock);
758 }
759
760 static __always_inline void __raw_spin_unlock(struct raw_spinlock *lock)
761 {
762         PVOP_VCALL1(pv_lock_ops.spin_unlock, lock);
763 }
764
765 #endif
766
767 #ifdef CONFIG_X86_32
768 #define PV_SAVE_REGS "pushl %ecx; pushl %edx;"
769 #define PV_RESTORE_REGS "popl %edx; popl %ecx;"
770
771 /* save and restore all caller-save registers, except return value */
772 #define PV_SAVE_ALL_CALLER_REGS         "pushl %ecx;"
773 #define PV_RESTORE_ALL_CALLER_REGS      "popl  %ecx;"
774
775 #define PV_FLAGS_ARG "0"
776 #define PV_EXTRA_CLOBBERS
777 #define PV_VEXTRA_CLOBBERS
778 #else
779 /* save and restore all caller-save registers, except return value */
780 #define PV_SAVE_ALL_CALLER_REGS                                         \
781         "push %rcx;"                                                    \
782         "push %rdx;"                                                    \
783         "push %rsi;"                                                    \
784         "push %rdi;"                                                    \
785         "push %r8;"                                                     \
786         "push %r9;"                                                     \
787         "push %r10;"                                                    \
788         "push %r11;"
789 #define PV_RESTORE_ALL_CALLER_REGS                                      \
790         "pop %r11;"                                                     \
791         "pop %r10;"                                                     \
792         "pop %r9;"                                                      \
793         "pop %r8;"                                                      \
794         "pop %rdi;"                                                     \
795         "pop %rsi;"                                                     \
796         "pop %rdx;"                                                     \
797         "pop %rcx;"
798
799 /* We save some registers, but all of them, that's too much. We clobber all
800  * caller saved registers but the argument parameter */
801 #define PV_SAVE_REGS "pushq %%rdi;"
802 #define PV_RESTORE_REGS "popq %%rdi;"
803 #define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx", "rsi"
804 #define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx", "rsi"
805 #define PV_FLAGS_ARG "D"
806 #endif
807
808 /*
809  * Generate a thunk around a function which saves all caller-save
810  * registers except for the return value.  This allows C functions to
811  * be called from assembler code where fewer than normal registers are
812  * available.  It may also help code generation around calls from C
813  * code if the common case doesn't use many registers.
814  *
815  * When a callee is wrapped in a thunk, the caller can assume that all
816  * arg regs and all scratch registers are preserved across the
817  * call. The return value in rax/eax will not be saved, even for void
818  * functions.
819  */
820 #define PV_CALLEE_SAVE_REGS_THUNK(func)                                 \
821         extern typeof(func) __raw_callee_save_##func;                   \
822         static void *__##func##__ __used = func;                        \
823                                                                         \
824         asm(".pushsection .text;"                                       \
825             "__raw_callee_save_" #func ": "                             \
826             PV_SAVE_ALL_CALLER_REGS                                     \
827             "call " #func ";"                                           \
828             PV_RESTORE_ALL_CALLER_REGS                                  \
829             "ret;"                                                      \
830             ".popsection")
831
832 /* Get a reference to a callee-save function */
833 #define PV_CALLEE_SAVE(func)                                            \
834         ((struct paravirt_callee_save) { __raw_callee_save_##func })
835
836 /* Promise that "func" already uses the right calling convention */
837 #define __PV_IS_CALLEE_SAVE(func)                       \
838         ((struct paravirt_callee_save) { func })
839
840 static inline unsigned long __raw_local_save_flags(void)
841 {
842         unsigned long f;
843
844         asm volatile(paravirt_alt(PARAVIRT_CALL)
845                      : "=a"(f)
846                      : paravirt_type(pv_irq_ops.save_fl),
847                        paravirt_clobber(CLBR_EAX)
848                      : "memory", "cc");
849         return f;
850 }
851
852 static inline void raw_local_irq_restore(unsigned long f)
853 {
854         asm volatile(paravirt_alt(PARAVIRT_CALL)
855                      : "=a"(f)
856                      : PV_FLAGS_ARG(f),
857                        paravirt_type(pv_irq_ops.restore_fl),
858                        paravirt_clobber(CLBR_EAX)
859                      : "memory", "cc");
860 }
861
862 static inline void raw_local_irq_disable(void)
863 {
864         asm volatile(paravirt_alt(PARAVIRT_CALL)
865                      :
866                      : paravirt_type(pv_irq_ops.irq_disable),
867                        paravirt_clobber(CLBR_EAX)
868                      : "memory", "eax", "cc");
869 }
870
871 static inline void raw_local_irq_enable(void)
872 {
873         asm volatile(paravirt_alt(PARAVIRT_CALL)
874                      :
875                      : paravirt_type(pv_irq_ops.irq_enable),
876                        paravirt_clobber(CLBR_EAX)
877                      : "memory", "eax", "cc");
878 }
879
880 static inline unsigned long __raw_local_irq_save(void)
881 {
882         unsigned long f;
883
884         f = __raw_local_save_flags();
885         raw_local_irq_disable();
886         return f;
887 }
888
889
890 /* Make sure as little as possible of this mess escapes. */
891 #undef PARAVIRT_CALL
892 #undef __PVOP_CALL
893 #undef __PVOP_VCALL
894 #undef PVOP_VCALL0
895 #undef PVOP_CALL0
896 #undef PVOP_VCALL1
897 #undef PVOP_CALL1
898 #undef PVOP_VCALL2
899 #undef PVOP_CALL2
900 #undef PVOP_VCALL3
901 #undef PVOP_CALL3
902 #undef PVOP_VCALL4
903 #undef PVOP_CALL4
904
905 extern void default_banner(void);
906
907 #else  /* __ASSEMBLY__ */
908
909 #define _PVSITE(ptype, clobbers, ops, word, algn)       \
910 771:;                                           \
911         ops;                                    \
912 772:;                                           \
913         .pushsection .parainstructions,"a";     \
914          .align algn;                           \
915          word 771b;                             \
916          .byte ptype;                           \
917          .byte 772b-771b;                       \
918          .short clobbers;                       \
919         .popsection
920
921
922 #define COND_PUSH(set, mask, reg)                       \
923         .if ((~(set)) & mask); push %reg; .endif
924 #define COND_POP(set, mask, reg)                        \
925         .if ((~(set)) & mask); pop %reg; .endif
926
927 #ifdef CONFIG_X86_64
928
929 #define PV_SAVE_REGS(set)                       \
930         COND_PUSH(set, CLBR_RAX, rax);          \
931         COND_PUSH(set, CLBR_RCX, rcx);          \
932         COND_PUSH(set, CLBR_RDX, rdx);          \
933         COND_PUSH(set, CLBR_RSI, rsi);          \
934         COND_PUSH(set, CLBR_RDI, rdi);          \
935         COND_PUSH(set, CLBR_R8, r8);            \
936         COND_PUSH(set, CLBR_R9, r9);            \
937         COND_PUSH(set, CLBR_R10, r10);          \
938         COND_PUSH(set, CLBR_R11, r11)
939 #define PV_RESTORE_REGS(set)                    \
940         COND_POP(set, CLBR_R11, r11);           \
941         COND_POP(set, CLBR_R10, r10);           \
942         COND_POP(set, CLBR_R9, r9);             \
943         COND_POP(set, CLBR_R8, r8);             \
944         COND_POP(set, CLBR_RDI, rdi);           \
945         COND_POP(set, CLBR_RSI, rsi);           \
946         COND_POP(set, CLBR_RDX, rdx);           \
947         COND_POP(set, CLBR_RCX, rcx);           \
948         COND_POP(set, CLBR_RAX, rax)
949
950 #define PARA_PATCH(struct, off)        ((PARAVIRT_PATCH_##struct + (off)) / 8)
951 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)
952 #define PARA_INDIRECT(addr)     *addr(%rip)
953 #else
954 #define PV_SAVE_REGS(set)                       \
955         COND_PUSH(set, CLBR_EAX, eax);          \
956         COND_PUSH(set, CLBR_EDI, edi);          \
957         COND_PUSH(set, CLBR_ECX, ecx);          \
958         COND_PUSH(set, CLBR_EDX, edx)
959 #define PV_RESTORE_REGS(set)                    \
960         COND_POP(set, CLBR_EDX, edx);           \
961         COND_POP(set, CLBR_ECX, ecx);           \
962         COND_POP(set, CLBR_EDI, edi);           \
963         COND_POP(set, CLBR_EAX, eax)
964
965 #define PARA_PATCH(struct, off)        ((PARAVIRT_PATCH_##struct + (off)) / 4)
966 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
967 #define PARA_INDIRECT(addr)     *%cs:addr
968 #endif
969
970 #define INTERRUPT_RETURN                                                \
971         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE,       \
972                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret))
973
974 #define DISABLE_INTERRUPTS(clobbers)                                    \
975         PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
976                   PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);            \
977                   call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable);    \
978                   PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
979
980 #define ENABLE_INTERRUPTS(clobbers)                                     \
981         PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers,  \
982                   PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);            \
983                   call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable);     \
984                   PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
985
986 #define USERGS_SYSRET32                                                 \
987         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret32),       \
988                   CLBR_NONE,                                            \
989                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret32))
990
991 #ifdef CONFIG_X86_32
992 #define GET_CR0_INTO_EAX                                \
993         push %ecx; push %edx;                           \
994         call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0); \
995         pop %edx; pop %ecx
996
997 #define ENABLE_INTERRUPTS_SYSEXIT                                       \
998         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit),    \
999                   CLBR_NONE,                                            \
1000                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
1001
1002
1003 #else   /* !CONFIG_X86_32 */
1004
1005 /*
1006  * If swapgs is used while the userspace stack is still current,
1007  * there's no way to call a pvop.  The PV replacement *must* be
1008  * inlined, or the swapgs instruction must be trapped and emulated.
1009  */
1010 #define SWAPGS_UNSAFE_STACK                                             \
1011         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,     \
1012                   swapgs)
1013
1014 /*
1015  * Note: swapgs is very special, and in practise is either going to be
1016  * implemented with a single "swapgs" instruction or something very
1017  * special.  Either way, we don't need to save any registers for
1018  * it.
1019  */
1020 #define SWAPGS                                                          \
1021         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,     \
1022                   call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs)          \
1023                  )
1024
1025 #define GET_CR2_INTO_RCX                                \
1026         call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2); \
1027         movq %rax, %rcx;                                \
1028         xorq %rax, %rax;
1029
1030 #define PARAVIRT_ADJUST_EXCEPTION_FRAME                                 \
1031         PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_adjust_exception_frame), \
1032                   CLBR_NONE,                                            \
1033                   call PARA_INDIRECT(pv_irq_ops+PV_IRQ_adjust_exception_frame))
1034
1035 #define USERGS_SYSRET64                                                 \
1036         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64),       \
1037                   CLBR_NONE,                                            \
1038                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64))
1039
1040 #define ENABLE_INTERRUPTS_SYSEXIT32                                     \
1041         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit),    \
1042                   CLBR_NONE,                                            \
1043                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
1044 #endif  /* CONFIG_X86_32 */
1045
1046 #endif /* __ASSEMBLY__ */
1047 #else  /* CONFIG_PARAVIRT */
1048 # define default_banner x86_init_noop
1049 #endif /* !CONFIG_PARAVIRT */
1050 #endif /* _ASM_X86_PARAVIRT_H */