[PATCH] paravirt: Patch inline replacements for paravirt intercepts
[linux-2.6.git] / arch / i386 / kernel / paravirt.c
1 /*  Paravirtualization interfaces
2     Copyright (C) 2006 Rusty Russell IBM Corporation
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17 */
18 #include <linux/errno.h>
19 #include <linux/module.h>
20 #include <linux/efi.h>
21 #include <linux/bcd.h>
22
23 #include <asm/bug.h>
24 #include <asm/paravirt.h>
25 #include <asm/desc.h>
26 #include <asm/setup.h>
27 #include <asm/arch_hooks.h>
28 #include <asm/time.h>
29 #include <asm/irq.h>
30 #include <asm/delay.h>
31
32 /* nop stub */
33 static void native_nop(void)
34 {
35 }
36
37 static void __init default_banner(void)
38 {
39         printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
40                paravirt_ops.name);
41 }
42
43 char *memory_setup(void)
44 {
45         return paravirt_ops.memory_setup();
46 }
47
48 /* Simple instruction patching code. */
49 #define DEF_NATIVE(name, code)                                  \
50         extern const char start_##name[], end_##name[];         \
51         asm("start_" #name ": " code "; end_" #name ":")
52 DEF_NATIVE(cli, "cli");
53 DEF_NATIVE(sti, "sti");
54 DEF_NATIVE(popf, "push %eax; popf");
55 DEF_NATIVE(pushf, "pushf; pop %eax");
56 DEF_NATIVE(pushf_cli, "pushf; pop %eax; cli");
57 DEF_NATIVE(iret, "iret");
58 DEF_NATIVE(sti_sysexit, "sti; sysexit");
59
60 static const struct native_insns
61 {
62         const char *start, *end;
63 } native_insns[] = {
64         [PARAVIRT_IRQ_DISABLE] = { start_cli, end_cli },
65         [PARAVIRT_IRQ_ENABLE] = { start_sti, end_sti },
66         [PARAVIRT_RESTORE_FLAGS] = { start_popf, end_popf },
67         [PARAVIRT_SAVE_FLAGS] = { start_pushf, end_pushf },
68         [PARAVIRT_SAVE_FLAGS_IRQ_DISABLE] = { start_pushf_cli, end_pushf_cli },
69         [PARAVIRT_INTERRUPT_RETURN] = { start_iret, end_iret },
70         [PARAVIRT_STI_SYSEXIT] = { start_sti_sysexit, end_sti_sysexit },
71 };
72
73 static unsigned native_patch(u8 type, u16 clobbers, void *insns, unsigned len)
74 {
75         unsigned int insn_len;
76
77         /* Don't touch it if we don't have a replacement */
78         if (type >= ARRAY_SIZE(native_insns) || !native_insns[type].start)
79                 return len;
80
81         insn_len = native_insns[type].end - native_insns[type].start;
82
83         /* Similarly if we can't fit replacement. */
84         if (len < insn_len)
85                 return len;
86
87         memcpy(insns, native_insns[type].start, insn_len);
88         return insn_len;
89 }
90
91 static fastcall unsigned long native_get_debugreg(int regno)
92 {
93         unsigned long val = 0;  /* Damn you, gcc! */
94
95         switch (regno) {
96         case 0:
97                 asm("movl %%db0, %0" :"=r" (val)); break;
98         case 1:
99                 asm("movl %%db1, %0" :"=r" (val)); break;
100         case 2:
101                 asm("movl %%db2, %0" :"=r" (val)); break;
102         case 3:
103                 asm("movl %%db3, %0" :"=r" (val)); break;
104         case 6:
105                 asm("movl %%db6, %0" :"=r" (val)); break;
106         case 7:
107                 asm("movl %%db7, %0" :"=r" (val)); break;
108         default:
109                 BUG();
110         }
111         return val;
112 }
113
114 static fastcall void native_set_debugreg(int regno, unsigned long value)
115 {
116         switch (regno) {
117         case 0:
118                 asm("movl %0,%%db0"     : /* no output */ :"r" (value));
119                 break;
120         case 1:
121                 asm("movl %0,%%db1"     : /* no output */ :"r" (value));
122                 break;
123         case 2:
124                 asm("movl %0,%%db2"     : /* no output */ :"r" (value));
125                 break;
126         case 3:
127                 asm("movl %0,%%db3"     : /* no output */ :"r" (value));
128                 break;
129         case 6:
130                 asm("movl %0,%%db6"     : /* no output */ :"r" (value));
131                 break;
132         case 7:
133                 asm("movl %0,%%db7"     : /* no output */ :"r" (value));
134                 break;
135         default:
136                 BUG();
137         }
138 }
139
140 void init_IRQ(void)
141 {
142         paravirt_ops.init_IRQ();
143 }
144
145 static fastcall void native_clts(void)
146 {
147         asm volatile ("clts");
148 }
149
150 static fastcall unsigned long native_read_cr0(void)
151 {
152         unsigned long val;
153         asm volatile("movl %%cr0,%0\n\t" :"=r" (val));
154         return val;
155 }
156
157 static fastcall void native_write_cr0(unsigned long val)
158 {
159         asm volatile("movl %0,%%cr0": :"r" (val));
160 }
161
162 static fastcall unsigned long native_read_cr2(void)
163 {
164         unsigned long val;
165         asm volatile("movl %%cr2,%0\n\t" :"=r" (val));
166         return val;
167 }
168
169 static fastcall void native_write_cr2(unsigned long val)
170 {
171         asm volatile("movl %0,%%cr2": :"r" (val));
172 }
173
174 static fastcall unsigned long native_read_cr3(void)
175 {
176         unsigned long val;
177         asm volatile("movl %%cr3,%0\n\t" :"=r" (val));
178         return val;
179 }
180
181 static fastcall void native_write_cr3(unsigned long val)
182 {
183         asm volatile("movl %0,%%cr3": :"r" (val));
184 }
185
186 static fastcall unsigned long native_read_cr4(void)
187 {
188         unsigned long val;
189         asm volatile("movl %%cr4,%0\n\t" :"=r" (val));
190         return val;
191 }
192
193 static fastcall unsigned long native_read_cr4_safe(void)
194 {
195         unsigned long val;
196         /* This could fault if %cr4 does not exist */
197         asm("1: movl %%cr4, %0          \n"
198                 "2:                             \n"
199                 ".section __ex_table,\"a\"      \n"
200                 ".long 1b,2b                    \n"
201                 ".previous                      \n"
202                 : "=r" (val): "0" (0));
203         return val;
204 }
205
206 static fastcall void native_write_cr4(unsigned long val)
207 {
208         asm volatile("movl %0,%%cr4": :"r" (val));
209 }
210
211 static fastcall unsigned long native_save_fl(void)
212 {
213         unsigned long f;
214         asm volatile("pushfl ; popl %0":"=g" (f): /* no input */);
215         return f;
216 }
217
218 static fastcall void native_restore_fl(unsigned long f)
219 {
220         asm volatile("pushl %0 ; popfl": /* no output */
221                              :"g" (f)
222                              :"memory", "cc");
223 }
224
225 static fastcall void native_irq_disable(void)
226 {
227         asm volatile("cli": : :"memory");
228 }
229
230 static fastcall void native_irq_enable(void)
231 {
232         asm volatile("sti": : :"memory");
233 }
234
235 static fastcall void native_safe_halt(void)
236 {
237         asm volatile("sti; hlt": : :"memory");
238 }
239
240 static fastcall void native_halt(void)
241 {
242         asm volatile("hlt": : :"memory");
243 }
244
245 static fastcall void native_wbinvd(void)
246 {
247         asm volatile("wbinvd": : :"memory");
248 }
249
250 static fastcall unsigned long long native_read_msr(unsigned int msr, int *err)
251 {
252         unsigned long long val;
253
254         asm volatile("2: rdmsr ; xorl %0,%0\n"
255                      "1:\n\t"
256                      ".section .fixup,\"ax\"\n\t"
257                      "3:  movl %3,%0 ; jmp 1b\n\t"
258                      ".previous\n\t"
259                      ".section __ex_table,\"a\"\n"
260                      "   .align 4\n\t"
261                      "   .long  2b,3b\n\t"
262                      ".previous"
263                      : "=r" (*err), "=A" (val)
264                      : "c" (msr), "i" (-EFAULT));
265
266         return val;
267 }
268
269 static fastcall int native_write_msr(unsigned int msr, unsigned long long val)
270 {
271         int err;
272         asm volatile("2: wrmsr ; xorl %0,%0\n"
273                      "1:\n\t"
274                      ".section .fixup,\"ax\"\n\t"
275                      "3:  movl %4,%0 ; jmp 1b\n\t"
276                      ".previous\n\t"
277                      ".section __ex_table,\"a\"\n"
278                      "   .align 4\n\t"
279                      "   .long  2b,3b\n\t"
280                      ".previous"
281                      : "=a" (err)
282                      : "c" (msr), "0" ((u32)val), "d" ((u32)(val>>32)),
283                        "i" (-EFAULT));
284         return err;
285 }
286
287 static fastcall unsigned long long native_read_tsc(void)
288 {
289         unsigned long long val;
290         asm volatile("rdtsc" : "=A" (val));
291         return val;
292 }
293
294 static fastcall unsigned long long native_read_pmc(void)
295 {
296         unsigned long long val;
297         asm volatile("rdpmc" : "=A" (val));
298         return val;
299 }
300
301 static fastcall void native_load_tr_desc(void)
302 {
303         asm volatile("ltr %w0"::"q" (GDT_ENTRY_TSS*8));
304 }
305
306 static fastcall void native_load_gdt(const struct Xgt_desc_struct *dtr)
307 {
308         asm volatile("lgdt %0"::"m" (*dtr));
309 }
310
311 static fastcall void native_load_idt(const struct Xgt_desc_struct *dtr)
312 {
313         asm volatile("lidt %0"::"m" (*dtr));
314 }
315
316 static fastcall void native_store_gdt(struct Xgt_desc_struct *dtr)
317 {
318         asm ("sgdt %0":"=m" (*dtr));
319 }
320
321 static fastcall void native_store_idt(struct Xgt_desc_struct *dtr)
322 {
323         asm ("sidt %0":"=m" (*dtr));
324 }
325
326 static fastcall unsigned long native_store_tr(void)
327 {
328         unsigned long tr;
329         asm ("str %0":"=r" (tr));
330         return tr;
331 }
332
333 static fastcall void native_load_tls(struct thread_struct *t, unsigned int cpu)
334 {
335 #define C(i) get_cpu_gdt_table(cpu)[GDT_ENTRY_TLS_MIN + i] = t->tls_array[i]
336         C(0); C(1); C(2);
337 #undef C
338 }
339
340 static inline void native_write_dt_entry(void *dt, int entry, u32 entry_low, u32 entry_high)
341 {
342         u32 *lp = (u32 *)((char *)dt + entry*8);
343         lp[0] = entry_low;
344         lp[1] = entry_high;
345 }
346
347 static fastcall void native_write_ldt_entry(void *dt, int entrynum, u32 low, u32 high)
348 {
349         native_write_dt_entry(dt, entrynum, low, high);
350 }
351
352 static fastcall void native_write_gdt_entry(void *dt, int entrynum, u32 low, u32 high)
353 {
354         native_write_dt_entry(dt, entrynum, low, high);
355 }
356
357 static fastcall void native_write_idt_entry(void *dt, int entrynum, u32 low, u32 high)
358 {
359         native_write_dt_entry(dt, entrynum, low, high);
360 }
361
362 static fastcall void native_load_esp0(struct tss_struct *tss,
363                                       struct thread_struct *thread)
364 {
365         tss->esp0 = thread->esp0;
366
367         /* This can only happen when SEP is enabled, no need to test "SEP"arately */
368         if (unlikely(tss->ss1 != thread->sysenter_cs)) {
369                 tss->ss1 = thread->sysenter_cs;
370                 wrmsr(MSR_IA32_SYSENTER_CS, thread->sysenter_cs, 0);
371         }
372 }
373
374 static fastcall void native_io_delay(void)
375 {
376         asm volatile("outb %al,$0x80");
377 }
378
379 /* These are in entry.S */
380 extern fastcall void native_iret(void);
381 extern fastcall void native_irq_enable_sysexit(void);
382
383 static int __init print_banner(void)
384 {
385         paravirt_ops.banner();
386         return 0;
387 }
388 core_initcall(print_banner);
389
390 struct paravirt_ops paravirt_ops = {
391         .name = "bare hardware",
392         .paravirt_enabled = 0,
393         .kernel_rpl = 0,
394
395         .patch = native_patch,
396         .banner = default_banner,
397         .arch_setup = native_nop,
398         .memory_setup = machine_specific_memory_setup,
399         .get_wallclock = native_get_wallclock,
400         .set_wallclock = native_set_wallclock,
401         .time_init = time_init_hook,
402         .init_IRQ = native_init_IRQ,
403
404         .cpuid = native_cpuid,
405         .get_debugreg = native_get_debugreg,
406         .set_debugreg = native_set_debugreg,
407         .clts = native_clts,
408         .read_cr0 = native_read_cr0,
409         .write_cr0 = native_write_cr0,
410         .read_cr2 = native_read_cr2,
411         .write_cr2 = native_write_cr2,
412         .read_cr3 = native_read_cr3,
413         .write_cr3 = native_write_cr3,
414         .read_cr4 = native_read_cr4,
415         .read_cr4_safe = native_read_cr4_safe,
416         .write_cr4 = native_write_cr4,
417         .save_fl = native_save_fl,
418         .restore_fl = native_restore_fl,
419         .irq_disable = native_irq_disable,
420         .irq_enable = native_irq_enable,
421         .safe_halt = native_safe_halt,
422         .halt = native_halt,
423         .wbinvd = native_wbinvd,
424         .read_msr = native_read_msr,
425         .write_msr = native_write_msr,
426         .read_tsc = native_read_tsc,
427         .read_pmc = native_read_pmc,
428         .load_tr_desc = native_load_tr_desc,
429         .set_ldt = native_set_ldt,
430         .load_gdt = native_load_gdt,
431         .load_idt = native_load_idt,
432         .store_gdt = native_store_gdt,
433         .store_idt = native_store_idt,
434         .store_tr = native_store_tr,
435         .load_tls = native_load_tls,
436         .write_ldt_entry = native_write_ldt_entry,
437         .write_gdt_entry = native_write_gdt_entry,
438         .write_idt_entry = native_write_idt_entry,
439         .load_esp0 = native_load_esp0,
440
441         .set_iopl_mask = native_set_iopl_mask,
442         .io_delay = native_io_delay,
443         .const_udelay = __const_udelay,
444
445         .irq_enable_sysexit = native_irq_enable_sysexit,
446         .iret = native_iret,
447 };
448 EXPORT_SYMBOL(paravirt_ops);