Merge branch 'stable/bug.fixes' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux-2.6.git] / arch / x86 / xen / smp.c
1 /*
2  * Xen SMP support
3  *
4  * This file implements the Xen versions of smp_ops.  SMP under Xen is
5  * very straightforward.  Bringing a CPU up is simply a matter of
6  * loading its initial context and setting it running.
7  *
8  * IPIs are handled through the Xen event mechanism.
9  *
10  * Because virtual CPUs can be scheduled onto any real CPU, there's no
11  * useful topology information for the kernel to make use of.  As a
12  * result, all CPUs are treated as if they're single-core and
13  * single-threaded.
14  */
15 #include <linux/sched.h>
16 #include <linux/err.h>
17 #include <linux/slab.h>
18 #include <linux/smp.h>
19
20 #include <asm/paravirt.h>
21 #include <asm/desc.h>
22 #include <asm/pgtable.h>
23 #include <asm/cpu.h>
24
25 #include <xen/interface/xen.h>
26 #include <xen/interface/vcpu.h>
27
28 #include <asm/xen/interface.h>
29 #include <asm/xen/hypercall.h>
30
31 #include <xen/xen.h>
32 #include <xen/page.h>
33 #include <xen/events.h>
34
35 #include "xen-ops.h"
36 #include "mmu.h"
37
38 cpumask_var_t xen_cpu_initialized_map;
39
40 static DEFINE_PER_CPU(int, xen_resched_irq);
41 static DEFINE_PER_CPU(int, xen_callfunc_irq);
42 static DEFINE_PER_CPU(int, xen_callfuncsingle_irq);
43 static DEFINE_PER_CPU(int, xen_debug_irq) = -1;
44
45 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
46 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
47
48 /*
49  * Reschedule call back.
50  */
51 static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
52 {
53         inc_irq_stat(irq_resched_count);
54         scheduler_ipi();
55
56         return IRQ_HANDLED;
57 }
58
59 static void __cpuinit cpu_bringup(void)
60 {
61         int cpu = smp_processor_id();
62
63         cpu_init();
64         touch_softlockup_watchdog();
65         preempt_disable();
66
67         xen_enable_sysenter();
68         xen_enable_syscall();
69
70         cpu = smp_processor_id();
71         smp_store_cpu_info(cpu);
72         cpu_data(cpu).x86_max_cores = 1;
73         set_cpu_sibling_map(cpu);
74
75         xen_setup_cpu_clockevents();
76
77         set_cpu_online(cpu, true);
78         percpu_write(cpu_state, CPU_ONLINE);
79         wmb();
80
81         /* We can take interrupts now: we're officially "up". */
82         local_irq_enable();
83
84         wmb();                  /* make sure everything is out */
85 }
86
87 static void __cpuinit cpu_bringup_and_idle(void)
88 {
89         cpu_bringup();
90         cpu_idle();
91 }
92
93 static int xen_smp_intr_init(unsigned int cpu)
94 {
95         int rc;
96         const char *resched_name, *callfunc_name, *debug_name;
97
98         resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu);
99         rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR,
100                                     cpu,
101                                     xen_reschedule_interrupt,
102                                     IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
103                                     resched_name,
104                                     NULL);
105         if (rc < 0)
106                 goto fail;
107         per_cpu(xen_resched_irq, cpu) = rc;
108
109         callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu);
110         rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR,
111                                     cpu,
112                                     xen_call_function_interrupt,
113                                     IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
114                                     callfunc_name,
115                                     NULL);
116         if (rc < 0)
117                 goto fail;
118         per_cpu(xen_callfunc_irq, cpu) = rc;
119
120         debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu);
121         rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt,
122                                      IRQF_DISABLED | IRQF_PERCPU | IRQF_NOBALANCING,
123                                      debug_name, NULL);
124         if (rc < 0)
125                 goto fail;
126         per_cpu(xen_debug_irq, cpu) = rc;
127
128         callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
129         rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
130                                     cpu,
131                                     xen_call_function_single_interrupt,
132                                     IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
133                                     callfunc_name,
134                                     NULL);
135         if (rc < 0)
136                 goto fail;
137         per_cpu(xen_callfuncsingle_irq, cpu) = rc;
138
139         return 0;
140
141  fail:
142         if (per_cpu(xen_resched_irq, cpu) >= 0)
143                 unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu), NULL);
144         if (per_cpu(xen_callfunc_irq, cpu) >= 0)
145                 unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu), NULL);
146         if (per_cpu(xen_debug_irq, cpu) >= 0)
147                 unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu), NULL);
148         if (per_cpu(xen_callfuncsingle_irq, cpu) >= 0)
149                 unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu),
150                                        NULL);
151
152         return rc;
153 }
154
155 static void __init xen_fill_possible_map(void)
156 {
157         int i, rc;
158
159         if (xen_initial_domain())
160                 return;
161
162         for (i = 0; i < nr_cpu_ids; i++) {
163                 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
164                 if (rc >= 0) {
165                         num_processors++;
166                         set_cpu_possible(i, true);
167                 }
168         }
169 }
170
171 static void __init xen_filter_cpu_maps(void)
172 {
173         int i, rc;
174
175         if (!xen_initial_domain())
176                 return;
177
178         num_processors = 0;
179         disabled_cpus = 0;
180         for (i = 0; i < nr_cpu_ids; i++) {
181                 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
182                 if (rc >= 0) {
183                         num_processors++;
184                         set_cpu_possible(i, true);
185                 } else {
186                         set_cpu_possible(i, false);
187                         set_cpu_present(i, false);
188                 }
189         }
190 }
191
192 static void __init xen_smp_prepare_boot_cpu(void)
193 {
194         BUG_ON(smp_processor_id() != 0);
195         native_smp_prepare_boot_cpu();
196
197         /* We've switched to the "real" per-cpu gdt, so make sure the
198            old memory can be recycled */
199         make_lowmem_page_readwrite(xen_initial_gdt);
200
201         xen_filter_cpu_maps();
202         xen_setup_vcpu_info_placement();
203 }
204
205 static void __init xen_smp_prepare_cpus(unsigned int max_cpus)
206 {
207         unsigned cpu;
208         unsigned int i;
209
210         xen_init_lock_cpu(0);
211
212         smp_store_cpu_info(0);
213         cpu_data(0).x86_max_cores = 1;
214
215         for_each_possible_cpu(i) {
216                 zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
217                 zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
218                 zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
219         }
220         set_cpu_sibling_map(0);
221
222         if (xen_smp_intr_init(0))
223                 BUG();
224
225         if (!alloc_cpumask_var(&xen_cpu_initialized_map, GFP_KERNEL))
226                 panic("could not allocate xen_cpu_initialized_map\n");
227
228         cpumask_copy(xen_cpu_initialized_map, cpumask_of(0));
229
230         /* Restrict the possible_map according to max_cpus. */
231         while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
232                 for (cpu = nr_cpu_ids - 1; !cpu_possible(cpu); cpu--)
233                         continue;
234                 set_cpu_possible(cpu, false);
235         }
236
237         for_each_possible_cpu (cpu) {
238                 struct task_struct *idle;
239
240                 if (cpu == 0)
241                         continue;
242
243                 idle = fork_idle(cpu);
244                 if (IS_ERR(idle))
245                         panic("failed fork for CPU %d", cpu);
246
247                 set_cpu_present(cpu, true);
248         }
249 }
250
251 static int __cpuinit
252 cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
253 {
254         struct vcpu_guest_context *ctxt;
255         struct desc_struct *gdt;
256         unsigned long gdt_mfn;
257
258         if (cpumask_test_and_set_cpu(cpu, xen_cpu_initialized_map))
259                 return 0;
260
261         ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
262         if (ctxt == NULL)
263                 return -ENOMEM;
264
265         gdt = get_cpu_gdt_table(cpu);
266
267         ctxt->flags = VGCF_IN_KERNEL;
268         ctxt->user_regs.ds = __USER_DS;
269         ctxt->user_regs.es = __USER_DS;
270         ctxt->user_regs.ss = __KERNEL_DS;
271 #ifdef CONFIG_X86_32
272         ctxt->user_regs.fs = __KERNEL_PERCPU;
273         ctxt->user_regs.gs = __KERNEL_STACK_CANARY;
274 #else
275         ctxt->gs_base_kernel = per_cpu_offset(cpu);
276 #endif
277         ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
278         ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
279
280         memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
281
282         xen_copy_trap_info(ctxt->trap_ctxt);
283
284         ctxt->ldt_ents = 0;
285
286         BUG_ON((unsigned long)gdt & ~PAGE_MASK);
287
288         gdt_mfn = arbitrary_virt_to_mfn(gdt);
289         make_lowmem_page_readonly(gdt);
290         make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
291
292         ctxt->gdt_frames[0] = gdt_mfn;
293         ctxt->gdt_ents      = GDT_ENTRIES;
294
295         ctxt->user_regs.cs = __KERNEL_CS;
296         ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
297
298         ctxt->kernel_ss = __KERNEL_DS;
299         ctxt->kernel_sp = idle->thread.sp0;
300
301 #ifdef CONFIG_X86_32
302         ctxt->event_callback_cs     = __KERNEL_CS;
303         ctxt->failsafe_callback_cs  = __KERNEL_CS;
304 #endif
305         ctxt->event_callback_eip    = (unsigned long)xen_hypervisor_callback;
306         ctxt->failsafe_callback_eip = (unsigned long)xen_failsafe_callback;
307
308         per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
309         ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(swapper_pg_dir));
310
311         if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt))
312                 BUG();
313
314         kfree(ctxt);
315         return 0;
316 }
317
318 static int __cpuinit xen_cpu_up(unsigned int cpu)
319 {
320         struct task_struct *idle = idle_task(cpu);
321         int rc;
322
323         per_cpu(current_task, cpu) = idle;
324 #ifdef CONFIG_X86_32
325         irq_ctx_init(cpu);
326 #else
327         clear_tsk_thread_flag(idle, TIF_FORK);
328         per_cpu(kernel_stack, cpu) =
329                 (unsigned long)task_stack_page(idle) -
330                 KERNEL_STACK_OFFSET + THREAD_SIZE;
331 #endif
332         xen_setup_runstate_info(cpu);
333         xen_setup_timer(cpu);
334         xen_init_lock_cpu(cpu);
335
336         per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
337
338         /* make sure interrupts start blocked */
339         per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1;
340
341         rc = cpu_initialize_context(cpu, idle);
342         if (rc)
343                 return rc;
344
345         if (num_online_cpus() == 1)
346                 alternatives_smp_switch(1);
347
348         rc = xen_smp_intr_init(cpu);
349         if (rc)
350                 return rc;
351
352         rc = HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL);
353         BUG_ON(rc);
354
355         while(per_cpu(cpu_state, cpu) != CPU_ONLINE) {
356                 HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
357                 barrier();
358         }
359
360         return 0;
361 }
362
363 static void xen_smp_cpus_done(unsigned int max_cpus)
364 {
365 }
366
367 #ifdef CONFIG_HOTPLUG_CPU
368 static int xen_cpu_disable(void)
369 {
370         unsigned int cpu = smp_processor_id();
371         if (cpu == 0)
372                 return -EBUSY;
373
374         cpu_disable_common();
375
376         load_cr3(swapper_pg_dir);
377         return 0;
378 }
379
380 static void xen_cpu_die(unsigned int cpu)
381 {
382         while (HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) {
383                 current->state = TASK_UNINTERRUPTIBLE;
384                 schedule_timeout(HZ/10);
385         }
386         unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu), NULL);
387         unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu), NULL);
388         unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu), NULL);
389         unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu), NULL);
390         xen_uninit_lock_cpu(cpu);
391         xen_teardown_timer(cpu);
392
393         if (num_online_cpus() == 1)
394                 alternatives_smp_switch(0);
395 }
396
397 static void __cpuinit xen_play_dead(void) /* used only with HOTPLUG_CPU */
398 {
399         play_dead_common();
400         HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL);
401         cpu_bringup();
402 }
403
404 #else /* !CONFIG_HOTPLUG_CPU */
405 static int xen_cpu_disable(void)
406 {
407         return -ENOSYS;
408 }
409
410 static void xen_cpu_die(unsigned int cpu)
411 {
412         BUG();
413 }
414
415 static void xen_play_dead(void)
416 {
417         BUG();
418 }
419
420 #endif
421 static void stop_self(void *v)
422 {
423         int cpu = smp_processor_id();
424
425         /* make sure we're not pinning something down */
426         load_cr3(swapper_pg_dir);
427         /* should set up a minimal gdt */
428
429         set_cpu_online(cpu, false);
430
431         HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL);
432         BUG();
433 }
434
435 static void xen_stop_other_cpus(int wait)
436 {
437         smp_call_function(stop_self, NULL, wait);
438 }
439
440 static void xen_smp_send_reschedule(int cpu)
441 {
442         xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
443 }
444
445 static void xen_send_IPI_mask(const struct cpumask *mask,
446                               enum ipi_vector vector)
447 {
448         unsigned cpu;
449
450         for_each_cpu_and(cpu, mask, cpu_online_mask)
451                 xen_send_IPI_one(cpu, vector);
452 }
453
454 static void xen_smp_send_call_function_ipi(const struct cpumask *mask)
455 {
456         int cpu;
457
458         xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
459
460         /* Make sure other vcpus get a chance to run if they need to. */
461         for_each_cpu(cpu, mask) {
462                 if (xen_vcpu_stolen(cpu)) {
463                         HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
464                         break;
465                 }
466         }
467 }
468
469 static void xen_smp_send_call_function_single_ipi(int cpu)
470 {
471         xen_send_IPI_mask(cpumask_of(cpu),
472                           XEN_CALL_FUNCTION_SINGLE_VECTOR);
473 }
474
475 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
476 {
477         irq_enter();
478         generic_smp_call_function_interrupt();
479         inc_irq_stat(irq_call_count);
480         irq_exit();
481
482         return IRQ_HANDLED;
483 }
484
485 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
486 {
487         irq_enter();
488         generic_smp_call_function_single_interrupt();
489         inc_irq_stat(irq_call_count);
490         irq_exit();
491
492         return IRQ_HANDLED;
493 }
494
495 static const struct smp_ops xen_smp_ops __initconst = {
496         .smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu,
497         .smp_prepare_cpus = xen_smp_prepare_cpus,
498         .smp_cpus_done = xen_smp_cpus_done,
499
500         .cpu_up = xen_cpu_up,
501         .cpu_die = xen_cpu_die,
502         .cpu_disable = xen_cpu_disable,
503         .play_dead = xen_play_dead,
504
505         .stop_other_cpus = xen_stop_other_cpus,
506         .smp_send_reschedule = xen_smp_send_reschedule,
507
508         .send_call_func_ipi = xen_smp_send_call_function_ipi,
509         .send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
510 };
511
512 void __init xen_smp_init(void)
513 {
514         smp_ops = xen_smp_ops;
515         xen_fill_possible_map();
516         xen_init_spinlocks();
517 }
518
519 static void __init xen_hvm_smp_prepare_cpus(unsigned int max_cpus)
520 {
521         native_smp_prepare_cpus(max_cpus);
522         WARN_ON(xen_smp_intr_init(0));
523
524         if (!xen_have_vector_callback)
525                 return;
526         xen_init_lock_cpu(0);
527         xen_init_spinlocks();
528 }
529
530 static int __cpuinit xen_hvm_cpu_up(unsigned int cpu)
531 {
532         int rc;
533         rc = native_cpu_up(cpu);
534         WARN_ON (xen_smp_intr_init(cpu));
535         return rc;
536 }
537
538 static void xen_hvm_cpu_die(unsigned int cpu)
539 {
540         unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu), NULL);
541         unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu), NULL);
542         unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu), NULL);
543         unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu), NULL);
544         native_cpu_die(cpu);
545 }
546
547 void __init xen_hvm_smp_init(void)
548 {
549         smp_ops.smp_prepare_cpus = xen_hvm_smp_prepare_cpus;
550         smp_ops.smp_send_reschedule = xen_smp_send_reschedule;
551         smp_ops.cpu_up = xen_hvm_cpu_up;
552         smp_ops.cpu_die = xen_hvm_cpu_die;
553         smp_ops.send_call_func_ipi = xen_smp_send_call_function_ipi;
554         smp_ops.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi;
555 }