e2f43768723a57452b15200f9f0f209e4d6622f6
[linux-2.6.git] / arch / x86 / kernel / paravirt.c
1 /*  Paravirtualization interfaces
2     Copyright (C) 2006 Rusty Russell IBM Corporation
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17
18     2007 - x86_64 support added by Glauber de Oliveira Costa, Red Hat Inc
19 */
20
21 #include <linux/errno.h>
22 #include <linux/module.h>
23 #include <linux/efi.h>
24 #include <linux/bcd.h>
25 #include <linux/highmem.h>
26
27 #include <asm/bug.h>
28 #include <asm/paravirt.h>
29 #include <asm/desc.h>
30 #include <asm/setup.h>
31 #include <asm/arch_hooks.h>
32 #include <asm/pgtable.h>
33 #include <asm/time.h>
34 #include <asm/pgalloc.h>
35 #include <asm/irq.h>
36 #include <asm/delay.h>
37 #include <asm/fixmap.h>
38 #include <asm/apic.h>
39 #include <asm/tlbflush.h>
40 #include <asm/timer.h>
41
42 /* nop stub */
43 void _paravirt_nop(void)
44 {
45 }
46
47 static void __init default_banner(void)
48 {
49         printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
50                pv_info.name);
51 }
52
53 char *memory_setup(void)
54 {
55         return pv_init_ops.memory_setup();
56 }
57
58 /* Simple instruction patching code. */
59 #define DEF_NATIVE(ops, name, code)                                     \
60         extern const char start_##ops##_##name[], end_##ops##_##name[]; \
61         asm("start_" #ops "_" #name ": " code "; end_" #ops "_" #name ":")
62
63 /* Undefined instruction for dealing with missing ops pointers. */
64 static const unsigned char ud2a[] = { 0x0f, 0x0b };
65
66 unsigned paravirt_patch_nop(void)
67 {
68         return 0;
69 }
70
71 unsigned paravirt_patch_ignore(unsigned len)
72 {
73         return len;
74 }
75
76 struct branch {
77         unsigned char opcode;
78         u32 delta;
79 } __attribute__((packed));
80
81 unsigned paravirt_patch_call(void *insnbuf,
82                              const void *target, u16 tgt_clobbers,
83                              unsigned long addr, u16 site_clobbers,
84                              unsigned len)
85 {
86         struct branch *b = insnbuf;
87         unsigned long delta = (unsigned long)target - (addr+5);
88
89         if (tgt_clobbers & ~site_clobbers)
90                 return len;     /* target would clobber too much for this site */
91         if (len < 5)
92                 return len;     /* call too long for patch site */
93
94         b->opcode = 0xe8; /* call */
95         b->delta = delta;
96         BUILD_BUG_ON(sizeof(*b) != 5);
97
98         return 5;
99 }
100
101 unsigned paravirt_patch_jmp(void *insnbuf, const void *target,
102                             unsigned long addr, unsigned len)
103 {
104         struct branch *b = insnbuf;
105         unsigned long delta = (unsigned long)target - (addr+5);
106
107         if (len < 5)
108                 return len;     /* call too long for patch site */
109
110         b->opcode = 0xe9;       /* jmp */
111         b->delta = delta;
112
113         return 5;
114 }
115
116 /* Neat trick to map patch type back to the call within the
117  * corresponding structure. */
118 static void *get_call_destination(u8 type)
119 {
120         struct paravirt_patch_template tmpl = {
121                 .pv_init_ops = pv_init_ops,
122                 .pv_time_ops = pv_time_ops,
123                 .pv_cpu_ops = pv_cpu_ops,
124                 .pv_irq_ops = pv_irq_ops,
125                 .pv_apic_ops = pv_apic_ops,
126                 .pv_mmu_ops = pv_mmu_ops,
127                 .pv_lock_ops = pv_lock_ops,
128         };
129         return *((void **)&tmpl + type);
130 }
131
132 unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
133                                 unsigned long addr, unsigned len)
134 {
135         void *opfunc = get_call_destination(type);
136         unsigned ret;
137
138         if (opfunc == NULL)
139                 /* If there's no function, patch it with a ud2a (BUG) */
140                 ret = paravirt_patch_insns(insnbuf, len, ud2a, ud2a+sizeof(ud2a));
141         else if (opfunc == paravirt_nop)
142                 /* If the operation is a nop, then nop the callsite */
143                 ret = paravirt_patch_nop();
144         else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) ||
145                  type == PARAVIRT_PATCH(pv_cpu_ops.irq_enable_sysexit) ||
146                  type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret32) ||
147                  type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret64))
148                 /* If operation requires a jmp, then jmp */
149                 ret = paravirt_patch_jmp(insnbuf, opfunc, addr, len);
150         else
151                 /* Otherwise call the function; assume target could
152                    clobber any caller-save reg */
153                 ret = paravirt_patch_call(insnbuf, opfunc, CLBR_ANY,
154                                           addr, clobbers, len);
155
156         return ret;
157 }
158
159 unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
160                               const char *start, const char *end)
161 {
162         unsigned insn_len = end - start;
163
164         if (insn_len > len || start == NULL)
165                 insn_len = len;
166         else
167                 memcpy(insnbuf, start, insn_len);
168
169         return insn_len;
170 }
171
172 void init_IRQ(void)
173 {
174         pv_irq_ops.init_IRQ();
175 }
176
177 static void native_flush_tlb(void)
178 {
179         __native_flush_tlb();
180 }
181
182 /*
183  * Global pages have to be flushed a bit differently. Not a real
184  * performance problem because this does not happen often.
185  */
186 static void native_flush_tlb_global(void)
187 {
188         __native_flush_tlb_global();
189 }
190
191 static void native_flush_tlb_single(unsigned long addr)
192 {
193         __native_flush_tlb_single(addr);
194 }
195
196 /* These are in entry.S */
197 extern void native_iret(void);
198 extern void native_irq_enable_sysexit(void);
199 extern void native_usergs_sysret32(void);
200 extern void native_usergs_sysret64(void);
201
202 static int __init print_banner(void)
203 {
204         pv_init_ops.banner();
205         return 0;
206 }
207 core_initcall(print_banner);
208
209 static struct resource reserve_ioports = {
210         .start = 0,
211         .end = IO_SPACE_LIMIT,
212         .name = "paravirt-ioport",
213         .flags = IORESOURCE_IO | IORESOURCE_BUSY,
214 };
215
216 /*
217  * Reserve the whole legacy IO space to prevent any legacy drivers
218  * from wasting time probing for their hardware.  This is a fairly
219  * brute-force approach to disabling all non-virtual drivers.
220  *
221  * Note that this must be called very early to have any effect.
222  */
223 int paravirt_disable_iospace(void)
224 {
225         return request_resource(&ioport_resource, &reserve_ioports);
226 }
227
228 static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAZY_NONE;
229
230 static inline void enter_lazy(enum paravirt_lazy_mode mode)
231 {
232         BUG_ON(__get_cpu_var(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
233         BUG_ON(preemptible());
234
235         __get_cpu_var(paravirt_lazy_mode) = mode;
236 }
237
238 void paravirt_leave_lazy(enum paravirt_lazy_mode mode)
239 {
240         BUG_ON(__get_cpu_var(paravirt_lazy_mode) != mode);
241         BUG_ON(preemptible());
242
243         __get_cpu_var(paravirt_lazy_mode) = PARAVIRT_LAZY_NONE;
244 }
245
246 void paravirt_enter_lazy_mmu(void)
247 {
248         enter_lazy(PARAVIRT_LAZY_MMU);
249 }
250
251 void paravirt_leave_lazy_mmu(void)
252 {
253         paravirt_leave_lazy(PARAVIRT_LAZY_MMU);
254 }
255
256 void paravirt_enter_lazy_cpu(void)
257 {
258         enter_lazy(PARAVIRT_LAZY_CPU);
259 }
260
261 void paravirt_leave_lazy_cpu(void)
262 {
263         paravirt_leave_lazy(PARAVIRT_LAZY_CPU);
264 }
265
266 enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
267 {
268         return __get_cpu_var(paravirt_lazy_mode);
269 }
270
271 void __init paravirt_use_bytelocks(void)
272 {
273 #ifdef CONFIG_SMP
274         pv_lock_ops.spin_is_locked = __byte_spin_is_locked;
275         pv_lock_ops.spin_is_contended = __byte_spin_is_contended;
276         pv_lock_ops.spin_lock = __byte_spin_lock;
277         pv_lock_ops.spin_trylock = __byte_spin_trylock;
278         pv_lock_ops.spin_unlock = __byte_spin_unlock;
279 #endif
280 }
281
282 struct pv_info pv_info = {
283         .name = "bare hardware",
284         .paravirt_enabled = 0,
285         .kernel_rpl = 0,
286         .shared_kernel_pmd = 1, /* Only used when CONFIG_X86_PAE is set */
287 };
288
289 struct pv_init_ops pv_init_ops = {
290         .patch = native_patch,
291         .banner = default_banner,
292         .arch_setup = paravirt_nop,
293         .memory_setup = machine_specific_memory_setup,
294 };
295
296 struct pv_time_ops pv_time_ops = {
297         .time_init = hpet_time_init,
298         .get_wallclock = native_get_wallclock,
299         .set_wallclock = native_set_wallclock,
300         .sched_clock = native_sched_clock,
301         .get_tsc_khz = native_calibrate_tsc,
302 };
303
304 struct pv_irq_ops pv_irq_ops = {
305         .init_IRQ = native_init_IRQ,
306         .save_fl = native_save_fl,
307         .restore_fl = native_restore_fl,
308         .irq_disable = native_irq_disable,
309         .irq_enable = native_irq_enable,
310         .safe_halt = native_safe_halt,
311         .halt = native_halt,
312 #ifdef CONFIG_X86_64
313         .adjust_exception_frame = paravirt_nop,
314 #endif
315 };
316
317 struct pv_cpu_ops pv_cpu_ops = {
318         .cpuid = native_cpuid,
319         .get_debugreg = native_get_debugreg,
320         .set_debugreg = native_set_debugreg,
321         .clts = native_clts,
322         .read_cr0 = native_read_cr0,
323         .write_cr0 = native_write_cr0,
324         .read_cr4 = native_read_cr4,
325         .read_cr4_safe = native_read_cr4_safe,
326         .write_cr4 = native_write_cr4,
327 #ifdef CONFIG_X86_64
328         .read_cr8 = native_read_cr8,
329         .write_cr8 = native_write_cr8,
330 #endif
331         .wbinvd = native_wbinvd,
332         .read_msr = native_read_msr_safe,
333         .read_msr_amd = native_read_msr_amd_safe,
334         .write_msr = native_write_msr_safe,
335         .read_tsc = native_read_tsc,
336         .read_pmc = native_read_pmc,
337         .read_tscp = native_read_tscp,
338         .load_tr_desc = native_load_tr_desc,
339         .set_ldt = native_set_ldt,
340         .load_gdt = native_load_gdt,
341         .load_idt = native_load_idt,
342         .store_gdt = native_store_gdt,
343         .store_idt = native_store_idt,
344         .store_tr = native_store_tr,
345         .load_tls = native_load_tls,
346 #ifdef CONFIG_X86_64
347         .load_gs_index = native_load_gs_index,
348 #endif
349         .write_ldt_entry = native_write_ldt_entry,
350         .write_gdt_entry = native_write_gdt_entry,
351         .write_idt_entry = native_write_idt_entry,
352         .load_sp0 = native_load_sp0,
353
354 #if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
355         .irq_enable_sysexit = native_irq_enable_sysexit,
356 #endif
357 #ifdef CONFIG_X86_64
358 #ifdef CONFIG_IA32_EMULATION
359         .usergs_sysret32 = native_usergs_sysret32,
360 #endif
361         .usergs_sysret64 = native_usergs_sysret64,
362 #endif
363         .iret = native_iret,
364         .swapgs = native_swapgs,
365
366         .set_iopl_mask = native_set_iopl_mask,
367         .io_delay = native_io_delay,
368
369         .lazy_mode = {
370                 .enter = paravirt_nop,
371                 .leave = paravirt_nop,
372         },
373 };
374
375 struct pv_apic_ops pv_apic_ops = {
376 #ifdef CONFIG_X86_LOCAL_APIC
377         .apic_write = native_apic_write,
378         .apic_read = native_apic_read,
379         .setup_boot_clock = setup_boot_APIC_clock,
380         .setup_secondary_clock = setup_secondary_APIC_clock,
381         .startup_ipi_hook = paravirt_nop,
382 #endif
383 };
384
385 struct pv_mmu_ops pv_mmu_ops = {
386 #ifndef CONFIG_X86_64
387         .pagetable_setup_start = native_pagetable_setup_start,
388         .pagetable_setup_done = native_pagetable_setup_done,
389 #else
390         .pagetable_setup_start = paravirt_nop,
391         .pagetable_setup_done = paravirt_nop,
392 #endif
393
394         .read_cr2 = native_read_cr2,
395         .write_cr2 = native_write_cr2,
396         .read_cr3 = native_read_cr3,
397         .write_cr3 = native_write_cr3,
398
399         .flush_tlb_user = native_flush_tlb,
400         .flush_tlb_kernel = native_flush_tlb_global,
401         .flush_tlb_single = native_flush_tlb_single,
402         .flush_tlb_others = native_flush_tlb_others,
403
404         .pgd_alloc = __paravirt_pgd_alloc,
405         .pgd_free = paravirt_nop,
406
407         .alloc_pte = paravirt_nop,
408         .alloc_pmd = paravirt_nop,
409         .alloc_pmd_clone = paravirt_nop,
410         .alloc_pud = paravirt_nop,
411         .release_pte = paravirt_nop,
412         .release_pmd = paravirt_nop,
413         .release_pud = paravirt_nop,
414
415         .set_pte = native_set_pte,
416         .set_pte_at = native_set_pte_at,
417         .set_pmd = native_set_pmd,
418         .pte_update = paravirt_nop,
419         .pte_update_defer = paravirt_nop,
420
421         .ptep_modify_prot_start = __ptep_modify_prot_start,
422         .ptep_modify_prot_commit = __ptep_modify_prot_commit,
423
424 #ifdef CONFIG_HIGHPTE
425         .kmap_atomic_pte = kmap_atomic,
426 #endif
427
428 #if PAGETABLE_LEVELS >= 3
429 #ifdef CONFIG_X86_PAE
430         .set_pte_atomic = native_set_pte_atomic,
431         .set_pte_present = native_set_pte_present,
432         .pte_clear = native_pte_clear,
433         .pmd_clear = native_pmd_clear,
434 #endif
435         .set_pud = native_set_pud,
436         .pmd_val = native_pmd_val,
437         .make_pmd = native_make_pmd,
438
439 #if PAGETABLE_LEVELS == 4
440         .pud_val = native_pud_val,
441         .make_pud = native_make_pud,
442         .set_pgd = native_set_pgd,
443 #endif
444 #endif /* PAGETABLE_LEVELS >= 3 */
445
446         .pte_val = native_pte_val,
447         .pte_flags = native_pte_flags,
448         .pgd_val = native_pgd_val,
449
450         .make_pte = native_make_pte,
451         .make_pgd = native_make_pgd,
452
453         .dup_mmap = paravirt_nop,
454         .exit_mmap = paravirt_nop,
455         .activate_mm = paravirt_nop,
456
457         .lazy_mode = {
458                 .enter = paravirt_nop,
459                 .leave = paravirt_nop,
460         },
461
462         .set_fixmap = native_set_fixmap,
463 };
464
465 struct pv_lock_ops pv_lock_ops = {
466 #ifdef CONFIG_SMP
467         .spin_is_locked = __ticket_spin_is_locked,
468         .spin_is_contended = __ticket_spin_is_contended,
469
470         .spin_lock = __ticket_spin_lock,
471         .spin_trylock = __ticket_spin_trylock,
472         .spin_unlock = __ticket_spin_unlock,
473 #endif
474 };
475 EXPORT_SYMBOL(pv_lock_ops);
476
477 EXPORT_SYMBOL_GPL(pv_time_ops);
478 EXPORT_SYMBOL    (pv_cpu_ops);
479 EXPORT_SYMBOL    (pv_mmu_ops);
480 EXPORT_SYMBOL_GPL(pv_apic_ops);
481 EXPORT_SYMBOL_GPL(pv_info);
482 EXPORT_SYMBOL    (pv_irq_ops);