Blackfin: SMP: tweak platform_request_ipi() usage
[linux-3.10.git] / arch / blackfin / mach-common / smp.c
1 /*
2  * IPI management based on arch/arm/kernel/smp.c (Copyright 2002 ARM Limited)
3  *
4  * Copyright 2007-2009 Analog Devices Inc.
5  *                         Philippe Gerum <rpm@xenomai.org>
6  *
7  * Licensed under the GPL-2.
8  */
9
10 #include <linux/module.h>
11 #include <linux/delay.h>
12 #include <linux/init.h>
13 #include <linux/spinlock.h>
14 #include <linux/sched.h>
15 #include <linux/interrupt.h>
16 #include <linux/cache.h>
17 #include <linux/profile.h>
18 #include <linux/errno.h>
19 #include <linux/mm.h>
20 #include <linux/cpu.h>
21 #include <linux/smp.h>
22 #include <linux/cpumask.h>
23 #include <linux/seq_file.h>
24 #include <linux/irq.h>
25 #include <linux/slab.h>
26 #include <asm/atomic.h>
27 #include <asm/cacheflush.h>
28 #include <asm/mmu_context.h>
29 #include <asm/pgtable.h>
30 #include <asm/pgalloc.h>
31 #include <asm/processor.h>
32 #include <asm/ptrace.h>
33 #include <asm/cpu.h>
34 #include <asm/time.h>
35 #include <linux/err.h>
36
37 /*
38  * Anomaly notes:
39  * 05000120 - we always define corelock as 32-bit integer in L2
40  */
41 struct corelock_slot corelock __attribute__ ((__section__(".l2.bss")));
42
43 void __cpuinitdata *init_retx_coreb, *init_saved_retx_coreb,
44         *init_saved_seqstat_coreb, *init_saved_icplb_fault_addr_coreb,
45         *init_saved_dcplb_fault_addr_coreb;
46
47 #define BFIN_IPI_RESCHEDULE   0
48 #define BFIN_IPI_CALL_FUNC    1
49 #define BFIN_IPI_CPU_STOP     2
50
51 struct blackfin_flush_data {
52         unsigned long start;
53         unsigned long end;
54 };
55
56 void *secondary_stack;
57
58
59 struct smp_call_struct {
60         void (*func)(void *info);
61         void *info;
62         int wait;
63         cpumask_t pending;
64         cpumask_t waitmask;
65 };
66
67 static struct blackfin_flush_data smp_flush_data;
68
69 static DEFINE_SPINLOCK(stop_lock);
70
71 struct ipi_message {
72         struct list_head list;
73         unsigned long type;
74         struct smp_call_struct call_struct;
75 };
76
77 struct ipi_message_queue {
78         struct list_head head;
79         spinlock_t lock;
80         unsigned long count;
81 };
82
83 static DEFINE_PER_CPU(struct ipi_message_queue, ipi_msg_queue);
84
85 static void ipi_cpu_stop(unsigned int cpu)
86 {
87         spin_lock(&stop_lock);
88         printk(KERN_CRIT "CPU%u: stopping\n", cpu);
89         dump_stack();
90         spin_unlock(&stop_lock);
91
92         cpu_clear(cpu, cpu_online_map);
93
94         local_irq_disable();
95
96         while (1)
97                 SSYNC();
98 }
99
100 static void ipi_flush_icache(void *info)
101 {
102         struct blackfin_flush_data *fdata = info;
103
104         /* Invalidate the memory holding the bounds of the flushed region. */
105         blackfin_dcache_invalidate_range((unsigned long)fdata,
106                                          (unsigned long)fdata + sizeof(*fdata));
107
108         blackfin_icache_flush_range(fdata->start, fdata->end);
109 }
110
111 static void ipi_call_function(unsigned int cpu, struct ipi_message *msg)
112 {
113         int wait;
114         void (*func)(void *info);
115         void *info;
116         func = msg->call_struct.func;
117         info = msg->call_struct.info;
118         wait = msg->call_struct.wait;
119         cpu_clear(cpu, msg->call_struct.pending);
120         func(info);
121         if (wait) {
122 #ifdef __ARCH_SYNC_CORE_DCACHE
123                 /*
124                  * 'wait' usually means synchronization between CPUs.
125                  * Invalidate D cache in case shared data was changed
126                  * by func() to ensure cache coherence.
127                  */
128                 resync_core_dcache();
129 #endif
130                 cpu_clear(cpu, msg->call_struct.waitmask);
131         } else
132                 kfree(msg);
133 }
134
135 static irqreturn_t ipi_handler(int irq, void *dev_instance)
136 {
137         struct ipi_message *msg;
138         struct ipi_message_queue *msg_queue;
139         unsigned int cpu = smp_processor_id();
140
141         platform_clear_ipi(cpu);
142
143         msg_queue = &__get_cpu_var(ipi_msg_queue);
144         msg_queue->count++;
145
146         spin_lock(&msg_queue->lock);
147         while (!list_empty(&msg_queue->head)) {
148                 msg = list_entry(msg_queue->head.next, typeof(*msg), list);
149                 list_del(&msg->list);
150                 switch (msg->type) {
151                 case BFIN_IPI_RESCHEDULE:
152                         /* That's the easiest one; leave it to
153                          * return_from_int. */
154                         kfree(msg);
155                         break;
156                 case BFIN_IPI_CALL_FUNC:
157                         spin_unlock(&msg_queue->lock);
158                         ipi_call_function(cpu, msg);
159                         spin_lock(&msg_queue->lock);
160                         break;
161                 case BFIN_IPI_CPU_STOP:
162                         spin_unlock(&msg_queue->lock);
163                         ipi_cpu_stop(cpu);
164                         spin_lock(&msg_queue->lock);
165                         kfree(msg);
166                         break;
167                 default:
168                         printk(KERN_CRIT "CPU%u: Unknown IPI message 0x%lx\n",
169                                cpu, msg->type);
170                         kfree(msg);
171                         break;
172                 }
173         }
174         spin_unlock(&msg_queue->lock);
175         return IRQ_HANDLED;
176 }
177
178 static void ipi_queue_init(void)
179 {
180         unsigned int cpu;
181         struct ipi_message_queue *msg_queue;
182         for_each_possible_cpu(cpu) {
183                 msg_queue = &per_cpu(ipi_msg_queue, cpu);
184                 INIT_LIST_HEAD(&msg_queue->head);
185                 spin_lock_init(&msg_queue->lock);
186                 msg_queue->count = 0;
187         }
188 }
189
190 int smp_call_function(void (*func)(void *info), void *info, int wait)
191 {
192         unsigned int cpu;
193         cpumask_t callmap;
194         unsigned long flags;
195         struct ipi_message_queue *msg_queue;
196         struct ipi_message *msg;
197
198         callmap = cpu_online_map;
199         cpu_clear(smp_processor_id(), callmap);
200         if (cpus_empty(callmap))
201                 return 0;
202
203         msg = kmalloc(sizeof(*msg), GFP_ATOMIC);
204         if (!msg)
205                 return -ENOMEM;
206         INIT_LIST_HEAD(&msg->list);
207         msg->call_struct.func = func;
208         msg->call_struct.info = info;
209         msg->call_struct.wait = wait;
210         msg->call_struct.pending = callmap;
211         msg->call_struct.waitmask = callmap;
212         msg->type = BFIN_IPI_CALL_FUNC;
213
214         for_each_cpu_mask(cpu, callmap) {
215                 msg_queue = &per_cpu(ipi_msg_queue, cpu);
216                 spin_lock_irqsave(&msg_queue->lock, flags);
217                 list_add_tail(&msg->list, &msg_queue->head);
218                 spin_unlock_irqrestore(&msg_queue->lock, flags);
219                 platform_send_ipi_cpu(cpu);
220         }
221         if (wait) {
222                 while (!cpus_empty(msg->call_struct.waitmask))
223                         blackfin_dcache_invalidate_range(
224                                 (unsigned long)(&msg->call_struct.waitmask),
225                                 (unsigned long)(&msg->call_struct.waitmask));
226 #ifdef __ARCH_SYNC_CORE_DCACHE
227                 /*
228                  * Invalidate D cache in case shared data was changed by
229                  * other processors to ensure cache coherence.
230                  */
231                 resync_core_dcache();
232 #endif
233                 kfree(msg);
234         }
235         return 0;
236 }
237 EXPORT_SYMBOL_GPL(smp_call_function);
238
239 int smp_call_function_single(int cpuid, void (*func) (void *info), void *info,
240                                 int wait)
241 {
242         unsigned int cpu = cpuid;
243         cpumask_t callmap;
244         unsigned long flags;
245         struct ipi_message_queue *msg_queue;
246         struct ipi_message *msg;
247
248         if (cpu_is_offline(cpu))
249                 return 0;
250         cpus_clear(callmap);
251         cpu_set(cpu, callmap);
252
253         msg = kmalloc(sizeof(*msg), GFP_ATOMIC);
254         if (!msg)
255                 return -ENOMEM;
256         INIT_LIST_HEAD(&msg->list);
257         msg->call_struct.func = func;
258         msg->call_struct.info = info;
259         msg->call_struct.wait = wait;
260         msg->call_struct.pending = callmap;
261         msg->call_struct.waitmask = callmap;
262         msg->type = BFIN_IPI_CALL_FUNC;
263
264         msg_queue = &per_cpu(ipi_msg_queue, cpu);
265         spin_lock_irqsave(&msg_queue->lock, flags);
266         list_add_tail(&msg->list, &msg_queue->head);
267         spin_unlock_irqrestore(&msg_queue->lock, flags);
268         platform_send_ipi_cpu(cpu);
269
270         if (wait) {
271                 while (!cpus_empty(msg->call_struct.waitmask))
272                         blackfin_dcache_invalidate_range(
273                                 (unsigned long)(&msg->call_struct.waitmask),
274                                 (unsigned long)(&msg->call_struct.waitmask));
275 #ifdef __ARCH_SYNC_CORE_DCACHE
276                 /*
277                  * Invalidate D cache in case shared data was changed by
278                  * other processors to ensure cache coherence.
279                  */
280                 resync_core_dcache();
281 #endif
282                 kfree(msg);
283         }
284         return 0;
285 }
286 EXPORT_SYMBOL_GPL(smp_call_function_single);
287
288 void smp_send_reschedule(int cpu)
289 {
290         unsigned long flags;
291         struct ipi_message_queue *msg_queue;
292         struct ipi_message *msg;
293
294         if (cpu_is_offline(cpu))
295                 return;
296
297         msg = kzalloc(sizeof(*msg), GFP_ATOMIC);
298         if (!msg)
299                 return;
300         INIT_LIST_HEAD(&msg->list);
301         msg->type = BFIN_IPI_RESCHEDULE;
302
303         msg_queue = &per_cpu(ipi_msg_queue, cpu);
304         spin_lock_irqsave(&msg_queue->lock, flags);
305         list_add_tail(&msg->list, &msg_queue->head);
306         spin_unlock_irqrestore(&msg_queue->lock, flags);
307         platform_send_ipi_cpu(cpu);
308
309         return;
310 }
311
312 void smp_send_stop(void)
313 {
314         unsigned int cpu;
315         cpumask_t callmap;
316         unsigned long flags;
317         struct ipi_message_queue *msg_queue;
318         struct ipi_message *msg;
319
320         callmap = cpu_online_map;
321         cpu_clear(smp_processor_id(), callmap);
322         if (cpus_empty(callmap))
323                 return;
324
325         msg = kzalloc(sizeof(*msg), GFP_ATOMIC);
326         if (!msg)
327                 return;
328         INIT_LIST_HEAD(&msg->list);
329         msg->type = BFIN_IPI_CPU_STOP;
330
331         for_each_cpu_mask(cpu, callmap) {
332                 msg_queue = &per_cpu(ipi_msg_queue, cpu);
333                 spin_lock_irqsave(&msg_queue->lock, flags);
334                 list_add_tail(&msg->list, &msg_queue->head);
335                 spin_unlock_irqrestore(&msg_queue->lock, flags);
336                 platform_send_ipi_cpu(cpu);
337         }
338         return;
339 }
340
341 int __cpuinit __cpu_up(unsigned int cpu)
342 {
343         int ret;
344         static struct task_struct *idle;
345
346         if (idle)
347                 free_task(idle);
348
349         idle = fork_idle(cpu);
350         if (IS_ERR(idle)) {
351                 printk(KERN_ERR "CPU%u: fork() failed\n", cpu);
352                 return PTR_ERR(idle);
353         }
354
355         secondary_stack = task_stack_page(idle) + THREAD_SIZE;
356
357         ret = platform_boot_secondary(cpu, idle);
358
359         secondary_stack = NULL;
360
361         return ret;
362 }
363
364 static void __cpuinit setup_secondary(unsigned int cpu)
365 {
366         unsigned long ilat;
367
368         bfin_write_IMASK(0);
369         CSYNC();
370         ilat = bfin_read_ILAT();
371         CSYNC();
372         bfin_write_ILAT(ilat);
373         CSYNC();
374
375         /* Enable interrupt levels IVG7-15. IARs have been already
376          * programmed by the boot CPU.  */
377         bfin_irq_flags |= IMASK_IVG15 |
378             IMASK_IVG14 | IMASK_IVG13 | IMASK_IVG12 | IMASK_IVG11 |
379             IMASK_IVG10 | IMASK_IVG9 | IMASK_IVG8 | IMASK_IVG7 | IMASK_IVGHW;
380 }
381
382 void __cpuinit secondary_start_kernel(void)
383 {
384         unsigned int cpu = smp_processor_id();
385         struct mm_struct *mm = &init_mm;
386
387         if (_bfin_swrst & SWRST_DBL_FAULT_B) {
388                 printk(KERN_EMERG "CoreB Recovering from DOUBLE FAULT event\n");
389 #ifdef CONFIG_DEBUG_DOUBLEFAULT
390                 printk(KERN_EMERG " While handling exception (EXCAUSE = 0x%x) at %pF\n",
391                         (int)init_saved_seqstat_coreb & SEQSTAT_EXCAUSE, init_saved_retx_coreb);
392                 printk(KERN_NOTICE "   DCPLB_FAULT_ADDR: %pF\n", init_saved_dcplb_fault_addr_coreb);
393                 printk(KERN_NOTICE "   ICPLB_FAULT_ADDR: %pF\n", init_saved_icplb_fault_addr_coreb);
394 #endif
395                 printk(KERN_NOTICE " The instruction at %pF caused a double exception\n",
396                         init_retx_coreb);
397         }
398
399         /*
400          * We want the D-cache to be enabled early, in case the atomic
401          * support code emulates cache coherence (see
402          * __ARCH_SYNC_CORE_DCACHE).
403          */
404         init_exception_vectors();
405
406         bfin_setup_caches(cpu);
407
408         local_irq_disable();
409
410         /* Attach the new idle task to the global mm. */
411         atomic_inc(&mm->mm_users);
412         atomic_inc(&mm->mm_count);
413         current->active_mm = mm;
414
415         preempt_disable();
416
417         setup_secondary(cpu);
418
419         platform_secondary_init(cpu);
420
421         /* setup local core timer */
422         bfin_local_timer_setup();
423
424         local_irq_enable();
425
426         /*
427          * Calibrate loops per jiffy value.
428          * IRQs need to be enabled here - D-cache can be invalidated
429          * in timer irq handler, so core B can read correct jiffies.
430          */
431         calibrate_delay();
432
433         cpu_idle();
434 }
435
436 void __init smp_prepare_boot_cpu(void)
437 {
438 }
439
440 void __init smp_prepare_cpus(unsigned int max_cpus)
441 {
442         platform_prepare_cpus(max_cpus);
443         ipi_queue_init();
444         platform_request_ipi(ipi_handler);
445 }
446
447 void __init smp_cpus_done(unsigned int max_cpus)
448 {
449         unsigned long bogosum = 0;
450         unsigned int cpu;
451
452         for_each_online_cpu(cpu)
453                 bogosum += loops_per_jiffy;
454
455         printk(KERN_INFO "SMP: Total of %d processors activated "
456                "(%lu.%02lu BogoMIPS).\n",
457                num_online_cpus(),
458                bogosum / (500000/HZ),
459                (bogosum / (5000/HZ)) % 100);
460 }
461
462 void smp_icache_flush_range_others(unsigned long start, unsigned long end)
463 {
464         smp_flush_data.start = start;
465         smp_flush_data.end = end;
466
467         if (smp_call_function(&ipi_flush_icache, &smp_flush_data, 0))
468                 printk(KERN_WARNING "SMP: failed to run I-cache flush request on other CPUs\n");
469 }
470 EXPORT_SYMBOL_GPL(smp_icache_flush_range_others);
471
472 #ifdef __ARCH_SYNC_CORE_ICACHE
473 unsigned long icache_invld_count[NR_CPUS];
474 void resync_core_icache(void)
475 {
476         unsigned int cpu = get_cpu();
477         blackfin_invalidate_entire_icache();
478         icache_invld_count[cpu]++;
479         put_cpu();
480 }
481 EXPORT_SYMBOL(resync_core_icache);
482 #endif
483
484 #ifdef __ARCH_SYNC_CORE_DCACHE
485 unsigned long dcache_invld_count[NR_CPUS];
486 unsigned long barrier_mask __attribute__ ((__section__(".l2.bss")));
487
488 void resync_core_dcache(void)
489 {
490         unsigned int cpu = get_cpu();
491         blackfin_invalidate_entire_dcache();
492         dcache_invld_count[cpu]++;
493         put_cpu();
494 }
495 EXPORT_SYMBOL(resync_core_dcache);
496 #endif
497
498 #ifdef CONFIG_HOTPLUG_CPU
499 int __cpuexit __cpu_disable(void)
500 {
501         unsigned int cpu = smp_processor_id();
502
503         if (cpu == 0)
504                 return -EPERM;
505
506         set_cpu_online(cpu, false);
507         return 0;
508 }
509
510 static DECLARE_COMPLETION(cpu_killed);
511
512 int __cpuexit __cpu_die(unsigned int cpu)
513 {
514         return wait_for_completion_timeout(&cpu_killed, 5000);
515 }
516
517 void cpu_die(void)
518 {
519         complete(&cpu_killed);
520
521         atomic_dec(&init_mm.mm_users);
522         atomic_dec(&init_mm.mm_count);
523
524         local_irq_disable();
525         platform_cpu_die();
526 }
527 #endif