include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit...
[linux-3.10.git] / arch / blackfin / mach-common / smp.c
1 /*
2  * IPI management based on arch/arm/kernel/smp.c (Copyright 2002 ARM Limited)
3  *
4  * Copyright 2007-2009 Analog Devices Inc.
5  *                         Philippe Gerum <rpm@xenomai.org>
6  *
7  * Licensed under the GPL-2.
8  */
9
10 #include <linux/module.h>
11 #include <linux/delay.h>
12 #include <linux/init.h>
13 #include <linux/spinlock.h>
14 #include <linux/sched.h>
15 #include <linux/interrupt.h>
16 #include <linux/cache.h>
17 #include <linux/profile.h>
18 #include <linux/errno.h>
19 #include <linux/mm.h>
20 #include <linux/cpu.h>
21 #include <linux/smp.h>
22 #include <linux/seq_file.h>
23 #include <linux/irq.h>
24 #include <linux/slab.h>
25 #include <asm/atomic.h>
26 #include <asm/cacheflush.h>
27 #include <asm/mmu_context.h>
28 #include <asm/pgtable.h>
29 #include <asm/pgalloc.h>
30 #include <asm/processor.h>
31 #include <asm/ptrace.h>
32 #include <asm/cpu.h>
33 #include <asm/time.h>
34 #include <linux/err.h>
35
36 /*
37  * Anomaly notes:
38  * 05000120 - we always define corelock as 32-bit integer in L2
39  */
40 struct corelock_slot corelock __attribute__ ((__section__(".l2.bss")));
41
42 void __cpuinitdata *init_retx_coreb, *init_saved_retx_coreb,
43         *init_saved_seqstat_coreb, *init_saved_icplb_fault_addr_coreb,
44         *init_saved_dcplb_fault_addr_coreb;
45
46 cpumask_t cpu_possible_map;
47 EXPORT_SYMBOL(cpu_possible_map);
48
49 cpumask_t cpu_online_map;
50 EXPORT_SYMBOL(cpu_online_map);
51
52 #define BFIN_IPI_RESCHEDULE   0
53 #define BFIN_IPI_CALL_FUNC    1
54 #define BFIN_IPI_CPU_STOP     2
55
56 struct blackfin_flush_data {
57         unsigned long start;
58         unsigned long end;
59 };
60
61 void *secondary_stack;
62
63
64 struct smp_call_struct {
65         void (*func)(void *info);
66         void *info;
67         int wait;
68         cpumask_t pending;
69         cpumask_t waitmask;
70 };
71
72 static struct blackfin_flush_data smp_flush_data;
73
74 static DEFINE_SPINLOCK(stop_lock);
75
76 struct ipi_message {
77         struct list_head list;
78         unsigned long type;
79         struct smp_call_struct call_struct;
80 };
81
82 struct ipi_message_queue {
83         struct list_head head;
84         spinlock_t lock;
85         unsigned long count;
86 };
87
88 static DEFINE_PER_CPU(struct ipi_message_queue, ipi_msg_queue);
89
90 static void ipi_cpu_stop(unsigned int cpu)
91 {
92         spin_lock(&stop_lock);
93         printk(KERN_CRIT "CPU%u: stopping\n", cpu);
94         dump_stack();
95         spin_unlock(&stop_lock);
96
97         cpu_clear(cpu, cpu_online_map);
98
99         local_irq_disable();
100
101         while (1)
102                 SSYNC();
103 }
104
105 static void ipi_flush_icache(void *info)
106 {
107         struct blackfin_flush_data *fdata = info;
108
109         /* Invalidate the memory holding the bounds of the flushed region. */
110         blackfin_dcache_invalidate_range((unsigned long)fdata,
111                                          (unsigned long)fdata + sizeof(*fdata));
112
113         blackfin_icache_flush_range(fdata->start, fdata->end);
114 }
115
116 static void ipi_call_function(unsigned int cpu, struct ipi_message *msg)
117 {
118         int wait;
119         void (*func)(void *info);
120         void *info;
121         func = msg->call_struct.func;
122         info = msg->call_struct.info;
123         wait = msg->call_struct.wait;
124         cpu_clear(cpu, msg->call_struct.pending);
125         func(info);
126         if (wait) {
127 #ifdef __ARCH_SYNC_CORE_DCACHE
128                 /*
129                  * 'wait' usually means synchronization between CPUs.
130                  * Invalidate D cache in case shared data was changed
131                  * by func() to ensure cache coherence.
132                  */
133                 resync_core_dcache();
134 #endif
135                 cpu_clear(cpu, msg->call_struct.waitmask);
136         } else
137                 kfree(msg);
138 }
139
140 static irqreturn_t ipi_handler(int irq, void *dev_instance)
141 {
142         struct ipi_message *msg;
143         struct ipi_message_queue *msg_queue;
144         unsigned int cpu = smp_processor_id();
145
146         platform_clear_ipi(cpu);
147
148         msg_queue = &__get_cpu_var(ipi_msg_queue);
149         msg_queue->count++;
150
151         spin_lock(&msg_queue->lock);
152         while (!list_empty(&msg_queue->head)) {
153                 msg = list_entry(msg_queue->head.next, typeof(*msg), list);
154                 list_del(&msg->list);
155                 switch (msg->type) {
156                 case BFIN_IPI_RESCHEDULE:
157                         /* That's the easiest one; leave it to
158                          * return_from_int. */
159                         kfree(msg);
160                         break;
161                 case BFIN_IPI_CALL_FUNC:
162                         spin_unlock(&msg_queue->lock);
163                         ipi_call_function(cpu, msg);
164                         spin_lock(&msg_queue->lock);
165                         break;
166                 case BFIN_IPI_CPU_STOP:
167                         spin_unlock(&msg_queue->lock);
168                         ipi_cpu_stop(cpu);
169                         spin_lock(&msg_queue->lock);
170                         kfree(msg);
171                         break;
172                 default:
173                         printk(KERN_CRIT "CPU%u: Unknown IPI message \
174                         0x%lx\n", cpu, msg->type);
175                         kfree(msg);
176                         break;
177                 }
178         }
179         spin_unlock(&msg_queue->lock);
180         return IRQ_HANDLED;
181 }
182
183 static void ipi_queue_init(void)
184 {
185         unsigned int cpu;
186         struct ipi_message_queue *msg_queue;
187         for_each_possible_cpu(cpu) {
188                 msg_queue = &per_cpu(ipi_msg_queue, cpu);
189                 INIT_LIST_HEAD(&msg_queue->head);
190                 spin_lock_init(&msg_queue->lock);
191                 msg_queue->count = 0;
192         }
193 }
194
195 int smp_call_function(void (*func)(void *info), void *info, int wait)
196 {
197         unsigned int cpu;
198         cpumask_t callmap;
199         unsigned long flags;
200         struct ipi_message_queue *msg_queue;
201         struct ipi_message *msg;
202
203         callmap = cpu_online_map;
204         cpu_clear(smp_processor_id(), callmap);
205         if (cpus_empty(callmap))
206                 return 0;
207
208         msg = kmalloc(sizeof(*msg), GFP_ATOMIC);
209         if (!msg)
210                 return -ENOMEM;
211         INIT_LIST_HEAD(&msg->list);
212         msg->call_struct.func = func;
213         msg->call_struct.info = info;
214         msg->call_struct.wait = wait;
215         msg->call_struct.pending = callmap;
216         msg->call_struct.waitmask = callmap;
217         msg->type = BFIN_IPI_CALL_FUNC;
218
219         for_each_cpu_mask(cpu, callmap) {
220                 msg_queue = &per_cpu(ipi_msg_queue, cpu);
221                 spin_lock_irqsave(&msg_queue->lock, flags);
222                 list_add_tail(&msg->list, &msg_queue->head);
223                 spin_unlock_irqrestore(&msg_queue->lock, flags);
224                 platform_send_ipi_cpu(cpu);
225         }
226         if (wait) {
227                 while (!cpus_empty(msg->call_struct.waitmask))
228                         blackfin_dcache_invalidate_range(
229                                 (unsigned long)(&msg->call_struct.waitmask),
230                                 (unsigned long)(&msg->call_struct.waitmask));
231 #ifdef __ARCH_SYNC_CORE_DCACHE
232                 /*
233                  * Invalidate D cache in case shared data was changed by
234                  * other processors to ensure cache coherence.
235                  */
236                 resync_core_dcache();
237 #endif
238                 kfree(msg);
239         }
240         return 0;
241 }
242 EXPORT_SYMBOL_GPL(smp_call_function);
243
244 int smp_call_function_single(int cpuid, void (*func) (void *info), void *info,
245                                 int wait)
246 {
247         unsigned int cpu = cpuid;
248         cpumask_t callmap;
249         unsigned long flags;
250         struct ipi_message_queue *msg_queue;
251         struct ipi_message *msg;
252
253         if (cpu_is_offline(cpu))
254                 return 0;
255         cpus_clear(callmap);
256         cpu_set(cpu, callmap);
257
258         msg = kmalloc(sizeof(*msg), GFP_ATOMIC);
259         if (!msg)
260                 return -ENOMEM;
261         INIT_LIST_HEAD(&msg->list);
262         msg->call_struct.func = func;
263         msg->call_struct.info = info;
264         msg->call_struct.wait = wait;
265         msg->call_struct.pending = callmap;
266         msg->call_struct.waitmask = callmap;
267         msg->type = BFIN_IPI_CALL_FUNC;
268
269         msg_queue = &per_cpu(ipi_msg_queue, cpu);
270         spin_lock_irqsave(&msg_queue->lock, flags);
271         list_add_tail(&msg->list, &msg_queue->head);
272         spin_unlock_irqrestore(&msg_queue->lock, flags);
273         platform_send_ipi_cpu(cpu);
274
275         if (wait) {
276                 while (!cpus_empty(msg->call_struct.waitmask))
277                         blackfin_dcache_invalidate_range(
278                                 (unsigned long)(&msg->call_struct.waitmask),
279                                 (unsigned long)(&msg->call_struct.waitmask));
280 #ifdef __ARCH_SYNC_CORE_DCACHE
281                 /*
282                  * Invalidate D cache in case shared data was changed by
283                  * other processors to ensure cache coherence.
284                  */
285                 resync_core_dcache();
286 #endif
287                 kfree(msg);
288         }
289         return 0;
290 }
291 EXPORT_SYMBOL_GPL(smp_call_function_single);
292
293 void smp_send_reschedule(int cpu)
294 {
295         unsigned long flags;
296         struct ipi_message_queue *msg_queue;
297         struct ipi_message *msg;
298
299         if (cpu_is_offline(cpu))
300                 return;
301
302         msg = kzalloc(sizeof(*msg), GFP_ATOMIC);
303         if (!msg)
304                 return;
305         INIT_LIST_HEAD(&msg->list);
306         msg->type = BFIN_IPI_RESCHEDULE;
307
308         msg_queue = &per_cpu(ipi_msg_queue, cpu);
309         spin_lock_irqsave(&msg_queue->lock, flags);
310         list_add_tail(&msg->list, &msg_queue->head);
311         spin_unlock_irqrestore(&msg_queue->lock, flags);
312         platform_send_ipi_cpu(cpu);
313
314         return;
315 }
316
317 void smp_send_stop(void)
318 {
319         unsigned int cpu;
320         cpumask_t callmap;
321         unsigned long flags;
322         struct ipi_message_queue *msg_queue;
323         struct ipi_message *msg;
324
325         callmap = cpu_online_map;
326         cpu_clear(smp_processor_id(), callmap);
327         if (cpus_empty(callmap))
328                 return;
329
330         msg = kzalloc(sizeof(*msg), GFP_ATOMIC);
331         if (!msg)
332                 return;
333         INIT_LIST_HEAD(&msg->list);
334         msg->type = BFIN_IPI_CPU_STOP;
335
336         for_each_cpu_mask(cpu, callmap) {
337                 msg_queue = &per_cpu(ipi_msg_queue, cpu);
338                 spin_lock_irqsave(&msg_queue->lock, flags);
339                 list_add_tail(&msg->list, &msg_queue->head);
340                 spin_unlock_irqrestore(&msg_queue->lock, flags);
341                 platform_send_ipi_cpu(cpu);
342         }
343         return;
344 }
345
346 int __cpuinit __cpu_up(unsigned int cpu)
347 {
348         int ret;
349         static struct task_struct *idle;
350
351         if (idle)
352                 free_task(idle);
353
354         idle = fork_idle(cpu);
355         if (IS_ERR(idle)) {
356                 printk(KERN_ERR "CPU%u: fork() failed\n", cpu);
357                 return PTR_ERR(idle);
358         }
359
360         secondary_stack = task_stack_page(idle) + THREAD_SIZE;
361
362         ret = platform_boot_secondary(cpu, idle);
363
364         secondary_stack = NULL;
365
366         return ret;
367 }
368
369 static void __cpuinit setup_secondary(unsigned int cpu)
370 {
371         unsigned long ilat;
372
373         bfin_write_IMASK(0);
374         CSYNC();
375         ilat = bfin_read_ILAT();
376         CSYNC();
377         bfin_write_ILAT(ilat);
378         CSYNC();
379
380         /* Enable interrupt levels IVG7-15. IARs have been already
381          * programmed by the boot CPU.  */
382         bfin_irq_flags |= IMASK_IVG15 |
383             IMASK_IVG14 | IMASK_IVG13 | IMASK_IVG12 | IMASK_IVG11 |
384             IMASK_IVG10 | IMASK_IVG9 | IMASK_IVG8 | IMASK_IVG7 | IMASK_IVGHW;
385 }
386
387 void __cpuinit secondary_start_kernel(void)
388 {
389         unsigned int cpu = smp_processor_id();
390         struct mm_struct *mm = &init_mm;
391
392         if (_bfin_swrst & SWRST_DBL_FAULT_B) {
393                 printk(KERN_EMERG "CoreB Recovering from DOUBLE FAULT event\n");
394 #ifdef CONFIG_DEBUG_DOUBLEFAULT
395                 printk(KERN_EMERG " While handling exception (EXCAUSE = 0x%x) at %pF\n",
396                         (int)init_saved_seqstat_coreb & SEQSTAT_EXCAUSE, init_saved_retx_coreb);
397                 printk(KERN_NOTICE "   DCPLB_FAULT_ADDR: %pF\n", init_saved_dcplb_fault_addr_coreb);
398                 printk(KERN_NOTICE "   ICPLB_FAULT_ADDR: %pF\n", init_saved_icplb_fault_addr_coreb);
399 #endif
400                 printk(KERN_NOTICE " The instruction at %pF caused a double exception\n",
401                         init_retx_coreb);
402         }
403
404         /*
405          * We want the D-cache to be enabled early, in case the atomic
406          * support code emulates cache coherence (see
407          * __ARCH_SYNC_CORE_DCACHE).
408          */
409         init_exception_vectors();
410
411         bfin_setup_caches(cpu);
412
413         local_irq_disable();
414
415         /* Attach the new idle task to the global mm. */
416         atomic_inc(&mm->mm_users);
417         atomic_inc(&mm->mm_count);
418         current->active_mm = mm;
419
420         preempt_disable();
421
422         setup_secondary(cpu);
423
424         platform_secondary_init(cpu);
425
426         /* setup local core timer */
427         bfin_local_timer_setup();
428
429         local_irq_enable();
430
431         /*
432          * Calibrate loops per jiffy value.
433          * IRQs need to be enabled here - D-cache can be invalidated
434          * in timer irq handler, so core B can read correct jiffies.
435          */
436         calibrate_delay();
437
438         cpu_idle();
439 }
440
441 void __init smp_prepare_boot_cpu(void)
442 {
443 }
444
445 void __init smp_prepare_cpus(unsigned int max_cpus)
446 {
447         platform_prepare_cpus(max_cpus);
448         ipi_queue_init();
449         platform_request_ipi(&ipi_handler);
450 }
451
452 void __init smp_cpus_done(unsigned int max_cpus)
453 {
454         unsigned long bogosum = 0;
455         unsigned int cpu;
456
457         for_each_online_cpu(cpu)
458                 bogosum += loops_per_jiffy;
459
460         printk(KERN_INFO "SMP: Total of %d processors activated "
461                "(%lu.%02lu BogoMIPS).\n",
462                num_online_cpus(),
463                bogosum / (500000/HZ),
464                (bogosum / (5000/HZ)) % 100);
465 }
466
467 void smp_icache_flush_range_others(unsigned long start, unsigned long end)
468 {
469         smp_flush_data.start = start;
470         smp_flush_data.end = end;
471
472         if (smp_call_function(&ipi_flush_icache, &smp_flush_data, 0))
473                 printk(KERN_WARNING "SMP: failed to run I-cache flush request on other CPUs\n");
474 }
475 EXPORT_SYMBOL_GPL(smp_icache_flush_range_others);
476
477 #ifdef __ARCH_SYNC_CORE_ICACHE
478 unsigned long icache_invld_count[NR_CPUS];
479 void resync_core_icache(void)
480 {
481         unsigned int cpu = get_cpu();
482         blackfin_invalidate_entire_icache();
483         icache_invld_count[cpu]++;
484         put_cpu();
485 }
486 EXPORT_SYMBOL(resync_core_icache);
487 #endif
488
489 #ifdef __ARCH_SYNC_CORE_DCACHE
490 unsigned long dcache_invld_count[NR_CPUS];
491 unsigned long barrier_mask __attribute__ ((__section__(".l2.bss")));
492
493 void resync_core_dcache(void)
494 {
495         unsigned int cpu = get_cpu();
496         blackfin_invalidate_entire_dcache();
497         dcache_invld_count[cpu]++;
498         put_cpu();
499 }
500 EXPORT_SYMBOL(resync_core_dcache);
501 #endif
502
503 #ifdef CONFIG_HOTPLUG_CPU
504 int __cpuexit __cpu_disable(void)
505 {
506         unsigned int cpu = smp_processor_id();
507
508         if (cpu == 0)
509                 return -EPERM;
510
511         set_cpu_online(cpu, false);
512         return 0;
513 }
514
515 static DECLARE_COMPLETION(cpu_killed);
516
517 int __cpuexit __cpu_die(unsigned int cpu)
518 {
519         return wait_for_completion_timeout(&cpu_killed, 5000);
520 }
521
522 void cpu_die(void)
523 {
524         complete(&cpu_killed);
525
526         atomic_dec(&init_mm.mm_users);
527         atomic_dec(&init_mm.mm_count);
528
529         local_irq_disable();
530         platform_cpu_die();
531 }
532 #endif