Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/paulus/powerpc
[linux-2.6.git] / arch / powerpc / kernel / smp.c
1 /*
2  * SMP support for ppc.
3  *
4  * Written by Cort Dougan (cort@cs.nmt.edu) borrowing a great
5  * deal of code from the sparc and intel versions.
6  *
7  * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
8  *
9  * PowerPC-64 Support added by Dave Engebretsen, Peter Bergner, and
10  * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com
11  *
12  *      This program is free software; you can redistribute it and/or
13  *      modify it under the terms of the GNU General Public License
14  *      as published by the Free Software Foundation; either version
15  *      2 of the License, or (at your option) any later version.
16  */
17
18 #undef DEBUG
19
20 #include <linux/kernel.h>
21 #include <linux/module.h>
22 #include <linux/sched.h>
23 #include <linux/smp.h>
24 #include <linux/interrupt.h>
25 #include <linux/delay.h>
26 #include <linux/init.h>
27 #include <linux/spinlock.h>
28 #include <linux/cache.h>
29 #include <linux/err.h>
30 #include <linux/sysdev.h>
31 #include <linux/cpu.h>
32 #include <linux/notifier.h>
33 #include <linux/topology.h>
34
35 #include <asm/ptrace.h>
36 #include <asm/atomic.h>
37 #include <asm/irq.h>
38 #include <asm/page.h>
39 #include <asm/pgtable.h>
40 #include <asm/prom.h>
41 #include <asm/smp.h>
42 #include <asm/time.h>
43 #include <asm/machdep.h>
44 #include <asm/cputhreads.h>
45 #include <asm/cputable.h>
46 #include <asm/system.h>
47 #include <asm/mpic.h>
48 #include <asm/vdso_datapage.h>
49 #ifdef CONFIG_PPC64
50 #include <asm/paca.h>
51 #endif
52
53 #ifdef DEBUG
54 #include <asm/udbg.h>
55 #define DBG(fmt...) udbg_printf(fmt)
56 #else
57 #define DBG(fmt...)
58 #endif
59
60 struct thread_info *secondary_ti;
61
62 cpumask_t cpu_possible_map = CPU_MASK_NONE;
63 cpumask_t cpu_online_map = CPU_MASK_NONE;
64 DEFINE_PER_CPU(cpumask_t, cpu_sibling_map) = CPU_MASK_NONE;
65 DEFINE_PER_CPU(cpumask_t, cpu_core_map) = CPU_MASK_NONE;
66
67 EXPORT_SYMBOL(cpu_online_map);
68 EXPORT_SYMBOL(cpu_possible_map);
69 EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
70 EXPORT_PER_CPU_SYMBOL(cpu_core_map);
71
72 /* SMP operations for this machine */
73 struct smp_ops_t *smp_ops;
74
75 static volatile unsigned int cpu_callin_map[NR_CPUS];
76
77 int smt_enabled_at_boot = 1;
78
79 static void (*crash_ipi_function_ptr)(struct pt_regs *) = NULL;
80
81 #ifdef CONFIG_PPC64
82 void __devinit smp_generic_kick_cpu(int nr)
83 {
84         BUG_ON(nr < 0 || nr >= NR_CPUS);
85
86         /*
87          * The processor is currently spinning, waiting for the
88          * cpu_start field to become non-zero After we set cpu_start,
89          * the processor will continue on to secondary_start
90          */
91         paca[nr].cpu_start = 1;
92         smp_mb();
93 }
94 #endif
95
96 void smp_message_recv(int msg)
97 {
98         switch(msg) {
99         case PPC_MSG_CALL_FUNCTION:
100                 generic_smp_call_function_interrupt();
101                 break;
102         case PPC_MSG_RESCHEDULE:
103                 /* we notice need_resched on exit */
104                 break;
105         case PPC_MSG_CALL_FUNC_SINGLE:
106                 generic_smp_call_function_single_interrupt();
107                 break;
108         case PPC_MSG_DEBUGGER_BREAK:
109                 if (crash_ipi_function_ptr) {
110                         crash_ipi_function_ptr(get_irq_regs());
111                         break;
112                 }
113 #ifdef CONFIG_DEBUGGER
114                 debugger_ipi(get_irq_regs());
115                 break;
116 #endif /* CONFIG_DEBUGGER */
117                 /* FALLTHROUGH */
118         default:
119                 printk("SMP %d: smp_message_recv(): unknown msg %d\n",
120                        smp_processor_id(), msg);
121                 break;
122         }
123 }
124
125 static irqreturn_t call_function_action(int irq, void *data)
126 {
127         generic_smp_call_function_interrupt();
128         return IRQ_HANDLED;
129 }
130
131 static irqreturn_t reschedule_action(int irq, void *data)
132 {
133         /* we just need the return path side effect of checking need_resched */
134         return IRQ_HANDLED;
135 }
136
137 static irqreturn_t call_function_single_action(int irq, void *data)
138 {
139         generic_smp_call_function_single_interrupt();
140         return IRQ_HANDLED;
141 }
142
143 static irqreturn_t debug_ipi_action(int irq, void *data)
144 {
145         smp_message_recv(PPC_MSG_DEBUGGER_BREAK);
146         return IRQ_HANDLED;
147 }
148
149 static irq_handler_t smp_ipi_action[] = {
150         [PPC_MSG_CALL_FUNCTION] =  call_function_action,
151         [PPC_MSG_RESCHEDULE] = reschedule_action,
152         [PPC_MSG_CALL_FUNC_SINGLE] = call_function_single_action,
153         [PPC_MSG_DEBUGGER_BREAK] = debug_ipi_action,
154 };
155
156 const char *smp_ipi_name[] = {
157         [PPC_MSG_CALL_FUNCTION] =  "ipi call function",
158         [PPC_MSG_RESCHEDULE] = "ipi reschedule",
159         [PPC_MSG_CALL_FUNC_SINGLE] = "ipi call function single",
160         [PPC_MSG_DEBUGGER_BREAK] = "ipi debugger",
161 };
162
163 /* optional function to request ipi, for controllers with >= 4 ipis */
164 int smp_request_message_ipi(int virq, int msg)
165 {
166         int err;
167
168         if (msg < 0 || msg > PPC_MSG_DEBUGGER_BREAK) {
169                 return -EINVAL;
170         }
171 #if !defined(CONFIG_DEBUGGER) && !defined(CONFIG_KEXEC)
172         if (msg == PPC_MSG_DEBUGGER_BREAK) {
173                 return 1;
174         }
175 #endif
176         err = request_irq(virq, smp_ipi_action[msg], IRQF_DISABLED|IRQF_PERCPU,
177                           smp_ipi_name[msg], 0);
178         WARN(err < 0, "unable to request_irq %d for %s (rc %d)\n",
179                 virq, smp_ipi_name[msg], err);
180
181         return err;
182 }
183
184 void smp_send_reschedule(int cpu)
185 {
186         if (likely(smp_ops))
187                 smp_ops->message_pass(cpu, PPC_MSG_RESCHEDULE);
188 }
189
190 void arch_send_call_function_single_ipi(int cpu)
191 {
192         smp_ops->message_pass(cpu, PPC_MSG_CALL_FUNC_SINGLE);
193 }
194
195 void arch_send_call_function_ipi(cpumask_t mask)
196 {
197         unsigned int cpu;
198
199         for_each_cpu_mask(cpu, mask)
200                 smp_ops->message_pass(cpu, PPC_MSG_CALL_FUNCTION);
201 }
202
203 #ifdef CONFIG_DEBUGGER
204 void smp_send_debugger_break(int cpu)
205 {
206         if (likely(smp_ops))
207                 smp_ops->message_pass(cpu, PPC_MSG_DEBUGGER_BREAK);
208 }
209 #endif
210
211 #ifdef CONFIG_KEXEC
212 void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *))
213 {
214         crash_ipi_function_ptr = crash_ipi_callback;
215         if (crash_ipi_callback && smp_ops) {
216                 mb();
217                 smp_ops->message_pass(MSG_ALL_BUT_SELF, PPC_MSG_DEBUGGER_BREAK);
218         }
219 }
220 #endif
221
222 static void stop_this_cpu(void *dummy)
223 {
224         local_irq_disable();
225         while (1)
226                 ;
227 }
228
229 void smp_send_stop(void)
230 {
231         smp_call_function(stop_this_cpu, NULL, 0);
232 }
233
234 struct thread_info *current_set[NR_CPUS];
235
236 static void __devinit smp_store_cpu_info(int id)
237 {
238         per_cpu(pvr, id) = mfspr(SPRN_PVR);
239 }
240
241 static void __init smp_create_idle(unsigned int cpu)
242 {
243         struct task_struct *p;
244
245         /* create a process for the processor */
246         p = fork_idle(cpu);
247         if (IS_ERR(p))
248                 panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p));
249 #ifdef CONFIG_PPC64
250         paca[cpu].__current = p;
251         paca[cpu].kstack = (unsigned long) task_thread_info(p)
252                 + THREAD_SIZE - STACK_FRAME_OVERHEAD;
253 #endif
254         current_set[cpu] = task_thread_info(p);
255         task_thread_info(p)->cpu = cpu;
256 }
257
258 void __init smp_prepare_cpus(unsigned int max_cpus)
259 {
260         unsigned int cpu;
261
262         DBG("smp_prepare_cpus\n");
263
264         /* 
265          * setup_cpu may need to be called on the boot cpu. We havent
266          * spun any cpus up but lets be paranoid.
267          */
268         BUG_ON(boot_cpuid != smp_processor_id());
269
270         /* Fixup boot cpu */
271         smp_store_cpu_info(boot_cpuid);
272         cpu_callin_map[boot_cpuid] = 1;
273
274         if (smp_ops)
275                 max_cpus = smp_ops->probe();
276         else
277                 max_cpus = 1;
278  
279         smp_space_timers(max_cpus);
280
281         for_each_possible_cpu(cpu)
282                 if (cpu != boot_cpuid)
283                         smp_create_idle(cpu);
284 }
285
286 void __devinit smp_prepare_boot_cpu(void)
287 {
288         BUG_ON(smp_processor_id() != boot_cpuid);
289
290         cpu_set(boot_cpuid, cpu_online_map);
291         cpu_set(boot_cpuid, per_cpu(cpu_sibling_map, boot_cpuid));
292         cpu_set(boot_cpuid, per_cpu(cpu_core_map, boot_cpuid));
293 #ifdef CONFIG_PPC64
294         paca[boot_cpuid].__current = current;
295 #endif
296         current_set[boot_cpuid] = task_thread_info(current);
297 }
298
299 #ifdef CONFIG_HOTPLUG_CPU
300 /* State of each CPU during hotplug phases */
301 DEFINE_PER_CPU(int, cpu_state) = { 0 };
302
303 int generic_cpu_disable(void)
304 {
305         unsigned int cpu = smp_processor_id();
306
307         if (cpu == boot_cpuid)
308                 return -EBUSY;
309
310         cpu_clear(cpu, cpu_online_map);
311 #ifdef CONFIG_PPC64
312         vdso_data->processorCount--;
313         fixup_irqs(cpu_online_map);
314 #endif
315         return 0;
316 }
317
318 int generic_cpu_enable(unsigned int cpu)
319 {
320         /* Do the normal bootup if we haven't
321          * already bootstrapped. */
322         if (system_state != SYSTEM_RUNNING)
323                 return -ENOSYS;
324
325         /* get the target out of it's holding state */
326         per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
327         smp_wmb();
328
329         while (!cpu_online(cpu))
330                 cpu_relax();
331
332 #ifdef CONFIG_PPC64
333         fixup_irqs(cpu_online_map);
334         /* counter the irq disable in fixup_irqs */
335         local_irq_enable();
336 #endif
337         return 0;
338 }
339
340 void generic_cpu_die(unsigned int cpu)
341 {
342         int i;
343
344         for (i = 0; i < 100; i++) {
345                 smp_rmb();
346                 if (per_cpu(cpu_state, cpu) == CPU_DEAD)
347                         return;
348                 msleep(100);
349         }
350         printk(KERN_ERR "CPU%d didn't die...\n", cpu);
351 }
352
353 void generic_mach_cpu_die(void)
354 {
355         unsigned int cpu;
356
357         local_irq_disable();
358         cpu = smp_processor_id();
359         printk(KERN_DEBUG "CPU%d offline\n", cpu);
360         __get_cpu_var(cpu_state) = CPU_DEAD;
361         smp_wmb();
362         while (__get_cpu_var(cpu_state) != CPU_UP_PREPARE)
363                 cpu_relax();
364         cpu_set(cpu, cpu_online_map);
365         local_irq_enable();
366 }
367 #endif
368
369 static int __devinit cpu_enable(unsigned int cpu)
370 {
371         if (smp_ops && smp_ops->cpu_enable)
372                 return smp_ops->cpu_enable(cpu);
373
374         return -ENOSYS;
375 }
376
377 int __cpuinit __cpu_up(unsigned int cpu)
378 {
379         int c;
380
381         secondary_ti = current_set[cpu];
382         if (!cpu_enable(cpu))
383                 return 0;
384
385         if (smp_ops == NULL ||
386             (smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu)))
387                 return -EINVAL;
388
389         /* Make sure callin-map entry is 0 (can be leftover a CPU
390          * hotplug
391          */
392         cpu_callin_map[cpu] = 0;
393
394         /* The information for processor bringup must
395          * be written out to main store before we release
396          * the processor.
397          */
398         smp_mb();
399
400         /* wake up cpus */
401         DBG("smp: kicking cpu %d\n", cpu);
402         smp_ops->kick_cpu(cpu);
403
404         /*
405          * wait to see if the cpu made a callin (is actually up).
406          * use this value that I found through experimentation.
407          * -- Cort
408          */
409         if (system_state < SYSTEM_RUNNING)
410                 for (c = 50000; c && !cpu_callin_map[cpu]; c--)
411                         udelay(100);
412 #ifdef CONFIG_HOTPLUG_CPU
413         else
414                 /*
415                  * CPUs can take much longer to come up in the
416                  * hotplug case.  Wait five seconds.
417                  */
418                 for (c = 25; c && !cpu_callin_map[cpu]; c--) {
419                         msleep(200);
420                 }
421 #endif
422
423         if (!cpu_callin_map[cpu]) {
424                 printk("Processor %u is stuck.\n", cpu);
425                 return -ENOENT;
426         }
427
428         printk("Processor %u found.\n", cpu);
429
430         if (smp_ops->give_timebase)
431                 smp_ops->give_timebase();
432
433         /* Wait until cpu puts itself in the online map */
434         while (!cpu_online(cpu))
435                 cpu_relax();
436
437         return 0;
438 }
439
440 /* Return the value of the reg property corresponding to the given
441  * logical cpu.
442  */
443 int cpu_to_core_id(int cpu)
444 {
445         struct device_node *np;
446         const int *reg;
447         int id = -1;
448
449         np = of_get_cpu_node(cpu, NULL);
450         if (!np)
451                 goto out;
452
453         reg = of_get_property(np, "reg", NULL);
454         if (!reg)
455                 goto out;
456
457         id = *reg;
458 out:
459         of_node_put(np);
460         return id;
461 }
462
463 /* Must be called when no change can occur to cpu_present_map,
464  * i.e. during cpu online or offline.
465  */
466 static struct device_node *cpu_to_l2cache(int cpu)
467 {
468         struct device_node *np;
469         struct device_node *cache;
470
471         if (!cpu_present(cpu))
472                 return NULL;
473
474         np = of_get_cpu_node(cpu, NULL);
475         if (np == NULL)
476                 return NULL;
477
478         cache = of_find_next_cache_node(np);
479
480         of_node_put(np);
481
482         return cache;
483 }
484
485 /* Activate a secondary processor. */
486 int __devinit start_secondary(void *unused)
487 {
488         unsigned int cpu = smp_processor_id();
489         struct device_node *l2_cache;
490         int i, base;
491
492         atomic_inc(&init_mm.mm_count);
493         current->active_mm = &init_mm;
494
495         smp_store_cpu_info(cpu);
496         set_dec(tb_ticks_per_jiffy);
497         preempt_disable();
498         cpu_callin_map[cpu] = 1;
499
500         smp_ops->setup_cpu(cpu);
501         if (smp_ops->take_timebase)
502                 smp_ops->take_timebase();
503
504         if (system_state > SYSTEM_BOOTING)
505                 snapshot_timebase();
506
507         secondary_cpu_time_init();
508
509         ipi_call_lock();
510         notify_cpu_starting(cpu);
511         cpu_set(cpu, cpu_online_map);
512         /* Update sibling maps */
513         base = cpu_first_thread_in_core(cpu);
514         for (i = 0; i < threads_per_core; i++) {
515                 if (cpu_is_offline(base + i))
516                         continue;
517                 cpu_set(cpu, per_cpu(cpu_sibling_map, base + i));
518                 cpu_set(base + i, per_cpu(cpu_sibling_map, cpu));
519
520                 /* cpu_core_map should be a superset of
521                  * cpu_sibling_map even if we don't have cache
522                  * information, so update the former here, too.
523                  */
524                 cpu_set(cpu, per_cpu(cpu_core_map, base +i));
525                 cpu_set(base + i, per_cpu(cpu_core_map, cpu));
526         }
527         l2_cache = cpu_to_l2cache(cpu);
528         for_each_online_cpu(i) {
529                 struct device_node *np = cpu_to_l2cache(i);
530                 if (!np)
531                         continue;
532                 if (np == l2_cache) {
533                         cpu_set(cpu, per_cpu(cpu_core_map, i));
534                         cpu_set(i, per_cpu(cpu_core_map, cpu));
535                 }
536                 of_node_put(np);
537         }
538         of_node_put(l2_cache);
539         ipi_call_unlock();
540
541         local_irq_enable();
542
543         cpu_idle();
544         return 0;
545 }
546
547 int setup_profiling_timer(unsigned int multiplier)
548 {
549         return 0;
550 }
551
552 void __init smp_cpus_done(unsigned int max_cpus)
553 {
554         cpumask_t old_mask;
555
556         /* We want the setup_cpu() here to be called from CPU 0, but our
557          * init thread may have been "borrowed" by another CPU in the meantime
558          * se we pin us down to CPU 0 for a short while
559          */
560         old_mask = current->cpus_allowed;
561         set_cpus_allowed(current, cpumask_of_cpu(boot_cpuid));
562         
563         if (smp_ops)
564                 smp_ops->setup_cpu(boot_cpuid);
565
566         set_cpus_allowed(current, old_mask);
567
568         snapshot_timebases();
569
570         dump_numa_cpu_topology();
571 }
572
573 #ifdef CONFIG_HOTPLUG_CPU
574 int __cpu_disable(void)
575 {
576         struct device_node *l2_cache;
577         int cpu = smp_processor_id();
578         int base, i;
579         int err;
580
581         if (!smp_ops->cpu_disable)
582                 return -ENOSYS;
583
584         err = smp_ops->cpu_disable();
585         if (err)
586                 return err;
587
588         /* Update sibling maps */
589         base = cpu_first_thread_in_core(cpu);
590         for (i = 0; i < threads_per_core; i++) {
591                 cpu_clear(cpu, per_cpu(cpu_sibling_map, base + i));
592                 cpu_clear(base + i, per_cpu(cpu_sibling_map, cpu));
593                 cpu_clear(cpu, per_cpu(cpu_core_map, base +i));
594                 cpu_clear(base + i, per_cpu(cpu_core_map, cpu));
595         }
596
597         l2_cache = cpu_to_l2cache(cpu);
598         for_each_present_cpu(i) {
599                 struct device_node *np = cpu_to_l2cache(i);
600                 if (!np)
601                         continue;
602                 if (np == l2_cache) {
603                         cpu_clear(cpu, per_cpu(cpu_core_map, i));
604                         cpu_clear(i, per_cpu(cpu_core_map, cpu));
605                 }
606                 of_node_put(np);
607         }
608         of_node_put(l2_cache);
609
610
611         return 0;
612 }
613
614 void __cpu_die(unsigned int cpu)
615 {
616         if (smp_ops->cpu_die)
617                 smp_ops->cpu_die(cpu);
618 }
619 #endif