tracing/events: move trace point headers into include/trace/events
[linux-2.6.git] / kernel / softirq.c
1 /*
2  *      linux/kernel/softirq.c
3  *
4  *      Copyright (C) 1992 Linus Torvalds
5  *
6  *      Distribute under GPLv2.
7  *
8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9  *
10  *      Remote softirq infrastructure is by Jens Axboe.
11  */
12
13 #include <linux/module.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/interrupt.h>
16 #include <linux/init.h>
17 #include <linux/mm.h>
18 #include <linux/notifier.h>
19 #include <linux/percpu.h>
20 #include <linux/cpu.h>
21 #include <linux/freezer.h>
22 #include <linux/kthread.h>
23 #include <linux/rcupdate.h>
24 #include <linux/ftrace.h>
25 #include <linux/smp.h>
26 #include <linux/tick.h>
27 #include <trace/events/irq.h>
28
29 #include <asm/irq.h>
30 /*
31    - No shared variables, all the data are CPU local.
32    - If a softirq needs serialization, let it serialize itself
33      by its own spinlocks.
34    - Even if softirq is serialized, only local cpu is marked for
35      execution. Hence, we get something sort of weak cpu binding.
36      Though it is still not clear, will it result in better locality
37      or will not.
38
39    Examples:
40    - NET RX softirq. It is multithreaded and does not require
41      any global serialization.
42    - NET TX softirq. It kicks software netdevice queues, hence
43      it is logically serialized per device, but this serialization
44      is invisible to common code.
45    - Tasklets: serialized wrt itself.
46  */
47
48 #ifndef __ARCH_IRQ_STAT
49 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
50 EXPORT_SYMBOL(irq_stat);
51 #endif
52
53 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
54
55 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
56
57 char *softirq_to_name[NR_SOFTIRQS] = {
58         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK",
59         "TASKLET", "SCHED", "HRTIMER",  "RCU"
60 };
61
62 /*
63  * we cannot loop indefinitely here to avoid userspace starvation,
64  * but we also don't want to introduce a worst case 1/HZ latency
65  * to the pending events, so lets the scheduler to balance
66  * the softirq load for us.
67  */
68 void wakeup_softirqd(void)
69 {
70         /* Interrupts are disabled: no need to stop preemption */
71         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
72
73         if (tsk && tsk->state != TASK_RUNNING)
74                 wake_up_process(tsk);
75 }
76
77 /*
78  * This one is for softirq.c-internal use,
79  * where hardirqs are disabled legitimately:
80  */
81 #ifdef CONFIG_TRACE_IRQFLAGS
82 static void __local_bh_disable(unsigned long ip)
83 {
84         unsigned long flags;
85
86         WARN_ON_ONCE(in_irq());
87
88         raw_local_irq_save(flags);
89         /*
90          * The preempt tracer hooks into add_preempt_count and will break
91          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
92          * is set and before current->softirq_enabled is cleared.
93          * We must manually increment preempt_count here and manually
94          * call the trace_preempt_off later.
95          */
96         preempt_count() += SOFTIRQ_OFFSET;
97         /*
98          * Were softirqs turned off above:
99          */
100         if (softirq_count() == SOFTIRQ_OFFSET)
101                 trace_softirqs_off(ip);
102         raw_local_irq_restore(flags);
103
104         if (preempt_count() == SOFTIRQ_OFFSET)
105                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
106 }
107 #else /* !CONFIG_TRACE_IRQFLAGS */
108 static inline void __local_bh_disable(unsigned long ip)
109 {
110         add_preempt_count(SOFTIRQ_OFFSET);
111         barrier();
112 }
113 #endif /* CONFIG_TRACE_IRQFLAGS */
114
115 void local_bh_disable(void)
116 {
117         __local_bh_disable((unsigned long)__builtin_return_address(0));
118 }
119
120 EXPORT_SYMBOL(local_bh_disable);
121
122 /*
123  * Special-case - softirqs can safely be enabled in
124  * cond_resched_softirq(), or by __do_softirq(),
125  * without processing still-pending softirqs:
126  */
127 void _local_bh_enable(void)
128 {
129         WARN_ON_ONCE(in_irq());
130         WARN_ON_ONCE(!irqs_disabled());
131
132         if (softirq_count() == SOFTIRQ_OFFSET)
133                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
134         sub_preempt_count(SOFTIRQ_OFFSET);
135 }
136
137 EXPORT_SYMBOL(_local_bh_enable);
138
139 static inline void _local_bh_enable_ip(unsigned long ip)
140 {
141         WARN_ON_ONCE(in_irq() || irqs_disabled());
142 #ifdef CONFIG_TRACE_IRQFLAGS
143         local_irq_disable();
144 #endif
145         /*
146          * Are softirqs going to be turned on now:
147          */
148         if (softirq_count() == SOFTIRQ_OFFSET)
149                 trace_softirqs_on(ip);
150         /*
151          * Keep preemption disabled until we are done with
152          * softirq processing:
153          */
154         sub_preempt_count(SOFTIRQ_OFFSET - 1);
155
156         if (unlikely(!in_interrupt() && local_softirq_pending()))
157                 do_softirq();
158
159         dec_preempt_count();
160 #ifdef CONFIG_TRACE_IRQFLAGS
161         local_irq_enable();
162 #endif
163         preempt_check_resched();
164 }
165
166 void local_bh_enable(void)
167 {
168         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
169 }
170 EXPORT_SYMBOL(local_bh_enable);
171
172 void local_bh_enable_ip(unsigned long ip)
173 {
174         _local_bh_enable_ip(ip);
175 }
176 EXPORT_SYMBOL(local_bh_enable_ip);
177
178 /*
179  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
180  * and we fall back to softirqd after that.
181  *
182  * This number has been established via experimentation.
183  * The two things to balance is latency against fairness -
184  * we want to handle softirqs as soon as possible, but they
185  * should not be able to lock up the box.
186  */
187 #define MAX_SOFTIRQ_RESTART 10
188
189 asmlinkage void __do_softirq(void)
190 {
191         struct softirq_action *h;
192         __u32 pending;
193         int max_restart = MAX_SOFTIRQ_RESTART;
194         int cpu;
195
196         pending = local_softirq_pending();
197         account_system_vtime(current);
198
199         __local_bh_disable((unsigned long)__builtin_return_address(0));
200         lockdep_softirq_enter();
201
202         cpu = smp_processor_id();
203 restart:
204         /* Reset the pending bitmask before enabling irqs */
205         set_softirq_pending(0);
206
207         local_irq_enable();
208
209         h = softirq_vec;
210
211         do {
212                 if (pending & 1) {
213                         int prev_count = preempt_count();
214
215                         trace_softirq_entry(h, softirq_vec);
216                         h->action(h);
217                         trace_softirq_exit(h, softirq_vec);
218                         if (unlikely(prev_count != preempt_count())) {
219                                 printk(KERN_ERR "huh, entered softirq %td %s %p"
220                                        "with preempt_count %08x,"
221                                        " exited with %08x?\n", h - softirq_vec,
222                                        softirq_to_name[h - softirq_vec],
223                                        h->action, prev_count, preempt_count());
224                                 preempt_count() = prev_count;
225                         }
226
227                         rcu_bh_qsctr_inc(cpu);
228                 }
229                 h++;
230                 pending >>= 1;
231         } while (pending);
232
233         local_irq_disable();
234
235         pending = local_softirq_pending();
236         if (pending && --max_restart)
237                 goto restart;
238
239         if (pending)
240                 wakeup_softirqd();
241
242         lockdep_softirq_exit();
243
244         account_system_vtime(current);
245         _local_bh_enable();
246 }
247
248 #ifndef __ARCH_HAS_DO_SOFTIRQ
249
250 asmlinkage void do_softirq(void)
251 {
252         __u32 pending;
253         unsigned long flags;
254
255         if (in_interrupt())
256                 return;
257
258         local_irq_save(flags);
259
260         pending = local_softirq_pending();
261
262         if (pending)
263                 __do_softirq();
264
265         local_irq_restore(flags);
266 }
267
268 #endif
269
270 /*
271  * Enter an interrupt context.
272  */
273 void irq_enter(void)
274 {
275         int cpu = smp_processor_id();
276
277         rcu_irq_enter();
278         if (idle_cpu(cpu) && !in_interrupt()) {
279                 __irq_enter();
280                 tick_check_idle(cpu);
281         } else
282                 __irq_enter();
283 }
284
285 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
286 # define invoke_softirq()       __do_softirq()
287 #else
288 # define invoke_softirq()       do_softirq()
289 #endif
290
291 /*
292  * Exit an interrupt context. Process softirqs if needed and possible:
293  */
294 void irq_exit(void)
295 {
296         account_system_vtime(current);
297         trace_hardirq_exit();
298         sub_preempt_count(IRQ_EXIT_OFFSET);
299         if (!in_interrupt() && local_softirq_pending())
300                 invoke_softirq();
301
302 #ifdef CONFIG_NO_HZ
303         /* Make sure that timer wheel updates are propagated */
304         rcu_irq_exit();
305         if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
306                 tick_nohz_stop_sched_tick(0);
307 #endif
308         preempt_enable_no_resched();
309 }
310
311 /*
312  * This function must run with irqs disabled!
313  */
314 inline void raise_softirq_irqoff(unsigned int nr)
315 {
316         __raise_softirq_irqoff(nr);
317
318         /*
319          * If we're in an interrupt or softirq, we're done
320          * (this also catches softirq-disabled code). We will
321          * actually run the softirq once we return from
322          * the irq or softirq.
323          *
324          * Otherwise we wake up ksoftirqd to make sure we
325          * schedule the softirq soon.
326          */
327         if (!in_interrupt())
328                 wakeup_softirqd();
329 }
330
331 void raise_softirq(unsigned int nr)
332 {
333         unsigned long flags;
334
335         local_irq_save(flags);
336         raise_softirq_irqoff(nr);
337         local_irq_restore(flags);
338 }
339
340 void open_softirq(int nr, void (*action)(struct softirq_action *))
341 {
342         softirq_vec[nr].action = action;
343 }
344
345 /* Tasklets */
346 struct tasklet_head
347 {
348         struct tasklet_struct *head;
349         struct tasklet_struct **tail;
350 };
351
352 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
353 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
354
355 void __tasklet_schedule(struct tasklet_struct *t)
356 {
357         unsigned long flags;
358
359         local_irq_save(flags);
360         t->next = NULL;
361         *__get_cpu_var(tasklet_vec).tail = t;
362         __get_cpu_var(tasklet_vec).tail = &(t->next);
363         raise_softirq_irqoff(TASKLET_SOFTIRQ);
364         local_irq_restore(flags);
365 }
366
367 EXPORT_SYMBOL(__tasklet_schedule);
368
369 void __tasklet_hi_schedule(struct tasklet_struct *t)
370 {
371         unsigned long flags;
372
373         local_irq_save(flags);
374         t->next = NULL;
375         *__get_cpu_var(tasklet_hi_vec).tail = t;
376         __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
377         raise_softirq_irqoff(HI_SOFTIRQ);
378         local_irq_restore(flags);
379 }
380
381 EXPORT_SYMBOL(__tasklet_hi_schedule);
382
383 static void tasklet_action(struct softirq_action *a)
384 {
385         struct tasklet_struct *list;
386
387         local_irq_disable();
388         list = __get_cpu_var(tasklet_vec).head;
389         __get_cpu_var(tasklet_vec).head = NULL;
390         __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
391         local_irq_enable();
392
393         while (list) {
394                 struct tasklet_struct *t = list;
395
396                 list = list->next;
397
398                 if (tasklet_trylock(t)) {
399                         if (!atomic_read(&t->count)) {
400                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
401                                         BUG();
402                                 t->func(t->data);
403                                 tasklet_unlock(t);
404                                 continue;
405                         }
406                         tasklet_unlock(t);
407                 }
408
409                 local_irq_disable();
410                 t->next = NULL;
411                 *__get_cpu_var(tasklet_vec).tail = t;
412                 __get_cpu_var(tasklet_vec).tail = &(t->next);
413                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
414                 local_irq_enable();
415         }
416 }
417
418 static void tasklet_hi_action(struct softirq_action *a)
419 {
420         struct tasklet_struct *list;
421
422         local_irq_disable();
423         list = __get_cpu_var(tasklet_hi_vec).head;
424         __get_cpu_var(tasklet_hi_vec).head = NULL;
425         __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
426         local_irq_enable();
427
428         while (list) {
429                 struct tasklet_struct *t = list;
430
431                 list = list->next;
432
433                 if (tasklet_trylock(t)) {
434                         if (!atomic_read(&t->count)) {
435                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
436                                         BUG();
437                                 t->func(t->data);
438                                 tasklet_unlock(t);
439                                 continue;
440                         }
441                         tasklet_unlock(t);
442                 }
443
444                 local_irq_disable();
445                 t->next = NULL;
446                 *__get_cpu_var(tasklet_hi_vec).tail = t;
447                 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
448                 __raise_softirq_irqoff(HI_SOFTIRQ);
449                 local_irq_enable();
450         }
451 }
452
453
454 void tasklet_init(struct tasklet_struct *t,
455                   void (*func)(unsigned long), unsigned long data)
456 {
457         t->next = NULL;
458         t->state = 0;
459         atomic_set(&t->count, 0);
460         t->func = func;
461         t->data = data;
462 }
463
464 EXPORT_SYMBOL(tasklet_init);
465
466 void tasklet_kill(struct tasklet_struct *t)
467 {
468         if (in_interrupt())
469                 printk("Attempt to kill tasklet from interrupt\n");
470
471         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
472                 do
473                         yield();
474                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
475         }
476         tasklet_unlock_wait(t);
477         clear_bit(TASKLET_STATE_SCHED, &t->state);
478 }
479
480 EXPORT_SYMBOL(tasklet_kill);
481
482 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
483 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
484
485 static void __local_trigger(struct call_single_data *cp, int softirq)
486 {
487         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
488
489         list_add_tail(&cp->list, head);
490
491         /* Trigger the softirq only if the list was previously empty.  */
492         if (head->next == &cp->list)
493                 raise_softirq_irqoff(softirq);
494 }
495
496 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
497 static void remote_softirq_receive(void *data)
498 {
499         struct call_single_data *cp = data;
500         unsigned long flags;
501         int softirq;
502
503         softirq = cp->priv;
504
505         local_irq_save(flags);
506         __local_trigger(cp, softirq);
507         local_irq_restore(flags);
508 }
509
510 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
511 {
512         if (cpu_online(cpu)) {
513                 cp->func = remote_softirq_receive;
514                 cp->info = cp;
515                 cp->flags = 0;
516                 cp->priv = softirq;
517
518                 __smp_call_function_single(cpu, cp, 0);
519                 return 0;
520         }
521         return 1;
522 }
523 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
524 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
525 {
526         return 1;
527 }
528 #endif
529
530 /**
531  * __send_remote_softirq - try to schedule softirq work on a remote cpu
532  * @cp: private SMP call function data area
533  * @cpu: the remote cpu
534  * @this_cpu: the currently executing cpu
535  * @softirq: the softirq for the work
536  *
537  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
538  * done, the work is instead queued up on the local cpu.
539  *
540  * Interrupts must be disabled.
541  */
542 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
543 {
544         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
545                 __local_trigger(cp, softirq);
546 }
547 EXPORT_SYMBOL(__send_remote_softirq);
548
549 /**
550  * send_remote_softirq - try to schedule softirq work on a remote cpu
551  * @cp: private SMP call function data area
552  * @cpu: the remote cpu
553  * @softirq: the softirq for the work
554  *
555  * Like __send_remote_softirq except that disabling interrupts and
556  * computing the current cpu is done for the caller.
557  */
558 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
559 {
560         unsigned long flags;
561         int this_cpu;
562
563         local_irq_save(flags);
564         this_cpu = smp_processor_id();
565         __send_remote_softirq(cp, cpu, this_cpu, softirq);
566         local_irq_restore(flags);
567 }
568 EXPORT_SYMBOL(send_remote_softirq);
569
570 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
571                                                unsigned long action, void *hcpu)
572 {
573         /*
574          * If a CPU goes away, splice its entries to the current CPU
575          * and trigger a run of the softirq
576          */
577         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
578                 int cpu = (unsigned long) hcpu;
579                 int i;
580
581                 local_irq_disable();
582                 for (i = 0; i < NR_SOFTIRQS; i++) {
583                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
584                         struct list_head *local_head;
585
586                         if (list_empty(head))
587                                 continue;
588
589                         local_head = &__get_cpu_var(softirq_work_list[i]);
590                         list_splice_init(head, local_head);
591                         raise_softirq_irqoff(i);
592                 }
593                 local_irq_enable();
594         }
595
596         return NOTIFY_OK;
597 }
598
599 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
600         .notifier_call  = remote_softirq_cpu_notify,
601 };
602
603 void __init softirq_init(void)
604 {
605         int cpu;
606
607         for_each_possible_cpu(cpu) {
608                 int i;
609
610                 per_cpu(tasklet_vec, cpu).tail =
611                         &per_cpu(tasklet_vec, cpu).head;
612                 per_cpu(tasklet_hi_vec, cpu).tail =
613                         &per_cpu(tasklet_hi_vec, cpu).head;
614                 for (i = 0; i < NR_SOFTIRQS; i++)
615                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
616         }
617
618         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
619
620         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
621         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
622 }
623
624 static int ksoftirqd(void * __bind_cpu)
625 {
626         set_current_state(TASK_INTERRUPTIBLE);
627
628         while (!kthread_should_stop()) {
629                 preempt_disable();
630                 if (!local_softirq_pending()) {
631                         preempt_enable_no_resched();
632                         schedule();
633                         preempt_disable();
634                 }
635
636                 __set_current_state(TASK_RUNNING);
637
638                 while (local_softirq_pending()) {
639                         /* Preempt disable stops cpu going offline.
640                            If already offline, we'll be on wrong CPU:
641                            don't process */
642                         if (cpu_is_offline((long)__bind_cpu))
643                                 goto wait_to_die;
644                         do_softirq();
645                         preempt_enable_no_resched();
646                         cond_resched();
647                         preempt_disable();
648                         rcu_qsctr_inc((long)__bind_cpu);
649                 }
650                 preempt_enable();
651                 set_current_state(TASK_INTERRUPTIBLE);
652         }
653         __set_current_state(TASK_RUNNING);
654         return 0;
655
656 wait_to_die:
657         preempt_enable();
658         /* Wait for kthread_stop */
659         set_current_state(TASK_INTERRUPTIBLE);
660         while (!kthread_should_stop()) {
661                 schedule();
662                 set_current_state(TASK_INTERRUPTIBLE);
663         }
664         __set_current_state(TASK_RUNNING);
665         return 0;
666 }
667
668 #ifdef CONFIG_HOTPLUG_CPU
669 /*
670  * tasklet_kill_immediate is called to remove a tasklet which can already be
671  * scheduled for execution on @cpu.
672  *
673  * Unlike tasklet_kill, this function removes the tasklet
674  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
675  *
676  * When this function is called, @cpu must be in the CPU_DEAD state.
677  */
678 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
679 {
680         struct tasklet_struct **i;
681
682         BUG_ON(cpu_online(cpu));
683         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
684
685         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
686                 return;
687
688         /* CPU is dead, so no lock needed. */
689         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
690                 if (*i == t) {
691                         *i = t->next;
692                         /* If this was the tail element, move the tail ptr */
693                         if (*i == NULL)
694                                 per_cpu(tasklet_vec, cpu).tail = i;
695                         return;
696                 }
697         }
698         BUG();
699 }
700
701 static void takeover_tasklets(unsigned int cpu)
702 {
703         /* CPU is dead, so no lock needed. */
704         local_irq_disable();
705
706         /* Find end, append list for that CPU. */
707         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
708                 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
709                 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
710                 per_cpu(tasklet_vec, cpu).head = NULL;
711                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
712         }
713         raise_softirq_irqoff(TASKLET_SOFTIRQ);
714
715         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
716                 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
717                 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
718                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
719                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
720         }
721         raise_softirq_irqoff(HI_SOFTIRQ);
722
723         local_irq_enable();
724 }
725 #endif /* CONFIG_HOTPLUG_CPU */
726
727 static int __cpuinit cpu_callback(struct notifier_block *nfb,
728                                   unsigned long action,
729                                   void *hcpu)
730 {
731         int hotcpu = (unsigned long)hcpu;
732         struct task_struct *p;
733
734         switch (action) {
735         case CPU_UP_PREPARE:
736         case CPU_UP_PREPARE_FROZEN:
737                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
738                 if (IS_ERR(p)) {
739                         printk("ksoftirqd for %i failed\n", hotcpu);
740                         return NOTIFY_BAD;
741                 }
742                 kthread_bind(p, hotcpu);
743                 per_cpu(ksoftirqd, hotcpu) = p;
744                 break;
745         case CPU_ONLINE:
746         case CPU_ONLINE_FROZEN:
747                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
748                 break;
749 #ifdef CONFIG_HOTPLUG_CPU
750         case CPU_UP_CANCELED:
751         case CPU_UP_CANCELED_FROZEN:
752                 if (!per_cpu(ksoftirqd, hotcpu))
753                         break;
754                 /* Unbind so it can run.  Fall thru. */
755                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
756                              cpumask_any(cpu_online_mask));
757         case CPU_DEAD:
758         case CPU_DEAD_FROZEN: {
759                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
760
761                 p = per_cpu(ksoftirqd, hotcpu);
762                 per_cpu(ksoftirqd, hotcpu) = NULL;
763                 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
764                 kthread_stop(p);
765                 takeover_tasklets(hotcpu);
766                 break;
767         }
768 #endif /* CONFIG_HOTPLUG_CPU */
769         }
770         return NOTIFY_OK;
771 }
772
773 static struct notifier_block __cpuinitdata cpu_nfb = {
774         .notifier_call = cpu_callback
775 };
776
777 static __init int spawn_ksoftirqd(void)
778 {
779         void *cpu = (void *)(long)smp_processor_id();
780         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
781
782         BUG_ON(err == NOTIFY_BAD);
783         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
784         register_cpu_notifier(&cpu_nfb);
785         return 0;
786 }
787 early_initcall(spawn_ksoftirqd);
788
789 #ifdef CONFIG_SMP
790 /*
791  * Call a function on all processors
792  */
793 int on_each_cpu(void (*func) (void *info), void *info, int wait)
794 {
795         int ret = 0;
796
797         preempt_disable();
798         ret = smp_call_function(func, info, wait);
799         local_irq_disable();
800         func(info);
801         local_irq_enable();
802         preempt_enable();
803         return ret;
804 }
805 EXPORT_SYMBOL(on_each_cpu);
806 #endif
807
808 /*
809  * [ These __weak aliases are kept in a separate compilation unit, so that
810  *   GCC does not inline them incorrectly. ]
811  */
812
813 int __init __weak early_irq_init(void)
814 {
815         return 0;
816 }
817
818 int __init __weak arch_probe_nr_irqs(void)
819 {
820         return 0;
821 }
822
823 int __init __weak arch_early_irq_init(void)
824 {
825         return 0;
826 }
827
828 int __weak arch_init_chip_data(struct irq_desc *desc, int cpu)
829 {
830         return 0;
831 }