tracing: fix build failure on s390
[linux-2.6.git] / kernel / softirq.c
1 /*
2  *      linux/kernel/softirq.c
3  *
4  *      Copyright (C) 1992 Linus Torvalds
5  *
6  *      Distribute under GPLv2.
7  *
8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9  *
10  *      Remote softirq infrastructure is by Jens Axboe.
11  */
12
13 #include <linux/module.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/interrupt.h>
16 #include <linux/init.h>
17 #include <linux/mm.h>
18 #include <linux/notifier.h>
19 #include <linux/percpu.h>
20 #include <linux/cpu.h>
21 #include <linux/freezer.h>
22 #include <linux/kthread.h>
23 #include <linux/rcupdate.h>
24 #include <linux/ftrace.h>
25 #include <linux/smp.h>
26 #include <linux/tick.h>
27
28 #define CREATE_TRACE_POINTS
29 #include <trace/events/irq.h>
30
31 #include <asm/irq.h>
32 /*
33    - No shared variables, all the data are CPU local.
34    - If a softirq needs serialization, let it serialize itself
35      by its own spinlocks.
36    - Even if softirq is serialized, only local cpu is marked for
37      execution. Hence, we get something sort of weak cpu binding.
38      Though it is still not clear, will it result in better locality
39      or will not.
40
41    Examples:
42    - NET RX softirq. It is multithreaded and does not require
43      any global serialization.
44    - NET TX softirq. It kicks software netdevice queues, hence
45      it is logically serialized per device, but this serialization
46      is invisible to common code.
47    - Tasklets: serialized wrt itself.
48  */
49
50 #ifndef __ARCH_IRQ_STAT
51 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
52 EXPORT_SYMBOL(irq_stat);
53 #endif
54
55 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
56
57 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
58
59 char *softirq_to_name[NR_SOFTIRQS] = {
60         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK",
61         "TASKLET", "SCHED", "HRTIMER",  "RCU"
62 };
63
64 /*
65  * we cannot loop indefinitely here to avoid userspace starvation,
66  * but we also don't want to introduce a worst case 1/HZ latency
67  * to the pending events, so lets the scheduler to balance
68  * the softirq load for us.
69  */
70 void wakeup_softirqd(void)
71 {
72         /* Interrupts are disabled: no need to stop preemption */
73         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
74
75         if (tsk && tsk->state != TASK_RUNNING)
76                 wake_up_process(tsk);
77 }
78
79 /*
80  * This one is for softirq.c-internal use,
81  * where hardirqs are disabled legitimately:
82  */
83 #ifdef CONFIG_TRACE_IRQFLAGS
84 static void __local_bh_disable(unsigned long ip)
85 {
86         unsigned long flags;
87
88         WARN_ON_ONCE(in_irq());
89
90         raw_local_irq_save(flags);
91         /*
92          * The preempt tracer hooks into add_preempt_count and will break
93          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
94          * is set and before current->softirq_enabled is cleared.
95          * We must manually increment preempt_count here and manually
96          * call the trace_preempt_off later.
97          */
98         preempt_count() += SOFTIRQ_OFFSET;
99         /*
100          * Were softirqs turned off above:
101          */
102         if (softirq_count() == SOFTIRQ_OFFSET)
103                 trace_softirqs_off(ip);
104         raw_local_irq_restore(flags);
105
106         if (preempt_count() == SOFTIRQ_OFFSET)
107                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
108 }
109 #else /* !CONFIG_TRACE_IRQFLAGS */
110 static inline void __local_bh_disable(unsigned long ip)
111 {
112         add_preempt_count(SOFTIRQ_OFFSET);
113         barrier();
114 }
115 #endif /* CONFIG_TRACE_IRQFLAGS */
116
117 void local_bh_disable(void)
118 {
119         __local_bh_disable((unsigned long)__builtin_return_address(0));
120 }
121
122 EXPORT_SYMBOL(local_bh_disable);
123
124 /*
125  * Special-case - softirqs can safely be enabled in
126  * cond_resched_softirq(), or by __do_softirq(),
127  * without processing still-pending softirqs:
128  */
129 void _local_bh_enable(void)
130 {
131         WARN_ON_ONCE(in_irq());
132         WARN_ON_ONCE(!irqs_disabled());
133
134         if (softirq_count() == SOFTIRQ_OFFSET)
135                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
136         sub_preempt_count(SOFTIRQ_OFFSET);
137 }
138
139 EXPORT_SYMBOL(_local_bh_enable);
140
141 static inline void _local_bh_enable_ip(unsigned long ip)
142 {
143         WARN_ON_ONCE(in_irq() || irqs_disabled());
144 #ifdef CONFIG_TRACE_IRQFLAGS
145         local_irq_disable();
146 #endif
147         /*
148          * Are softirqs going to be turned on now:
149          */
150         if (softirq_count() == SOFTIRQ_OFFSET)
151                 trace_softirqs_on(ip);
152         /*
153          * Keep preemption disabled until we are done with
154          * softirq processing:
155          */
156         sub_preempt_count(SOFTIRQ_OFFSET - 1);
157
158         if (unlikely(!in_interrupt() && local_softirq_pending()))
159                 do_softirq();
160
161         dec_preempt_count();
162 #ifdef CONFIG_TRACE_IRQFLAGS
163         local_irq_enable();
164 #endif
165         preempt_check_resched();
166 }
167
168 void local_bh_enable(void)
169 {
170         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
171 }
172 EXPORT_SYMBOL(local_bh_enable);
173
174 void local_bh_enable_ip(unsigned long ip)
175 {
176         _local_bh_enable_ip(ip);
177 }
178 EXPORT_SYMBOL(local_bh_enable_ip);
179
180 /*
181  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
182  * and we fall back to softirqd after that.
183  *
184  * This number has been established via experimentation.
185  * The two things to balance is latency against fairness -
186  * we want to handle softirqs as soon as possible, but they
187  * should not be able to lock up the box.
188  */
189 #define MAX_SOFTIRQ_RESTART 10
190
191 asmlinkage void __do_softirq(void)
192 {
193         struct softirq_action *h;
194         __u32 pending;
195         int max_restart = MAX_SOFTIRQ_RESTART;
196         int cpu;
197
198         pending = local_softirq_pending();
199         account_system_vtime(current);
200
201         __local_bh_disable((unsigned long)__builtin_return_address(0));
202         lockdep_softirq_enter();
203
204         cpu = smp_processor_id();
205 restart:
206         /* Reset the pending bitmask before enabling irqs */
207         set_softirq_pending(0);
208
209         local_irq_enable();
210
211         h = softirq_vec;
212
213         do {
214                 if (pending & 1) {
215                         int prev_count = preempt_count();
216
217                         trace_softirq_entry(h, softirq_vec);
218                         h->action(h);
219                         trace_softirq_exit(h, softirq_vec);
220                         if (unlikely(prev_count != preempt_count())) {
221                                 printk(KERN_ERR "huh, entered softirq %td %s %p"
222                                        "with preempt_count %08x,"
223                                        " exited with %08x?\n", h - softirq_vec,
224                                        softirq_to_name[h - softirq_vec],
225                                        h->action, prev_count, preempt_count());
226                                 preempt_count() = prev_count;
227                         }
228
229                         rcu_bh_qsctr_inc(cpu);
230                 }
231                 h++;
232                 pending >>= 1;
233         } while (pending);
234
235         local_irq_disable();
236
237         pending = local_softirq_pending();
238         if (pending && --max_restart)
239                 goto restart;
240
241         if (pending)
242                 wakeup_softirqd();
243
244         lockdep_softirq_exit();
245
246         account_system_vtime(current);
247         _local_bh_enable();
248 }
249
250 #ifndef __ARCH_HAS_DO_SOFTIRQ
251
252 asmlinkage void do_softirq(void)
253 {
254         __u32 pending;
255         unsigned long flags;
256
257         if (in_interrupt())
258                 return;
259
260         local_irq_save(flags);
261
262         pending = local_softirq_pending();
263
264         if (pending)
265                 __do_softirq();
266
267         local_irq_restore(flags);
268 }
269
270 #endif
271
272 /*
273  * Enter an interrupt context.
274  */
275 void irq_enter(void)
276 {
277         int cpu = smp_processor_id();
278
279         rcu_irq_enter();
280         if (idle_cpu(cpu) && !in_interrupt()) {
281                 __irq_enter();
282                 tick_check_idle(cpu);
283         } else
284                 __irq_enter();
285 }
286
287 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
288 # define invoke_softirq()       __do_softirq()
289 #else
290 # define invoke_softirq()       do_softirq()
291 #endif
292
293 /*
294  * Exit an interrupt context. Process softirqs if needed and possible:
295  */
296 void irq_exit(void)
297 {
298         account_system_vtime(current);
299         trace_hardirq_exit();
300         sub_preempt_count(IRQ_EXIT_OFFSET);
301         if (!in_interrupt() && local_softirq_pending())
302                 invoke_softirq();
303
304 #ifdef CONFIG_NO_HZ
305         /* Make sure that timer wheel updates are propagated */
306         rcu_irq_exit();
307         if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
308                 tick_nohz_stop_sched_tick(0);
309 #endif
310         preempt_enable_no_resched();
311 }
312
313 /*
314  * This function must run with irqs disabled!
315  */
316 inline void raise_softirq_irqoff(unsigned int nr)
317 {
318         __raise_softirq_irqoff(nr);
319
320         /*
321          * If we're in an interrupt or softirq, we're done
322          * (this also catches softirq-disabled code). We will
323          * actually run the softirq once we return from
324          * the irq or softirq.
325          *
326          * Otherwise we wake up ksoftirqd to make sure we
327          * schedule the softirq soon.
328          */
329         if (!in_interrupt())
330                 wakeup_softirqd();
331 }
332
333 void raise_softirq(unsigned int nr)
334 {
335         unsigned long flags;
336
337         local_irq_save(flags);
338         raise_softirq_irqoff(nr);
339         local_irq_restore(flags);
340 }
341
342 void open_softirq(int nr, void (*action)(struct softirq_action *))
343 {
344         softirq_vec[nr].action = action;
345 }
346
347 /* Tasklets */
348 struct tasklet_head
349 {
350         struct tasklet_struct *head;
351         struct tasklet_struct **tail;
352 };
353
354 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
355 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
356
357 void __tasklet_schedule(struct tasklet_struct *t)
358 {
359         unsigned long flags;
360
361         local_irq_save(flags);
362         t->next = NULL;
363         *__get_cpu_var(tasklet_vec).tail = t;
364         __get_cpu_var(tasklet_vec).tail = &(t->next);
365         raise_softirq_irqoff(TASKLET_SOFTIRQ);
366         local_irq_restore(flags);
367 }
368
369 EXPORT_SYMBOL(__tasklet_schedule);
370
371 void __tasklet_hi_schedule(struct tasklet_struct *t)
372 {
373         unsigned long flags;
374
375         local_irq_save(flags);
376         t->next = NULL;
377         *__get_cpu_var(tasklet_hi_vec).tail = t;
378         __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
379         raise_softirq_irqoff(HI_SOFTIRQ);
380         local_irq_restore(flags);
381 }
382
383 EXPORT_SYMBOL(__tasklet_hi_schedule);
384
385 static void tasklet_action(struct softirq_action *a)
386 {
387         struct tasklet_struct *list;
388
389         local_irq_disable();
390         list = __get_cpu_var(tasklet_vec).head;
391         __get_cpu_var(tasklet_vec).head = NULL;
392         __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
393         local_irq_enable();
394
395         while (list) {
396                 struct tasklet_struct *t = list;
397
398                 list = list->next;
399
400                 if (tasklet_trylock(t)) {
401                         if (!atomic_read(&t->count)) {
402                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
403                                         BUG();
404                                 t->func(t->data);
405                                 tasklet_unlock(t);
406                                 continue;
407                         }
408                         tasklet_unlock(t);
409                 }
410
411                 local_irq_disable();
412                 t->next = NULL;
413                 *__get_cpu_var(tasklet_vec).tail = t;
414                 __get_cpu_var(tasklet_vec).tail = &(t->next);
415                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
416                 local_irq_enable();
417         }
418 }
419
420 static void tasklet_hi_action(struct softirq_action *a)
421 {
422         struct tasklet_struct *list;
423
424         local_irq_disable();
425         list = __get_cpu_var(tasklet_hi_vec).head;
426         __get_cpu_var(tasklet_hi_vec).head = NULL;
427         __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
428         local_irq_enable();
429
430         while (list) {
431                 struct tasklet_struct *t = list;
432
433                 list = list->next;
434
435                 if (tasklet_trylock(t)) {
436                         if (!atomic_read(&t->count)) {
437                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
438                                         BUG();
439                                 t->func(t->data);
440                                 tasklet_unlock(t);
441                                 continue;
442                         }
443                         tasklet_unlock(t);
444                 }
445
446                 local_irq_disable();
447                 t->next = NULL;
448                 *__get_cpu_var(tasklet_hi_vec).tail = t;
449                 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
450                 __raise_softirq_irqoff(HI_SOFTIRQ);
451                 local_irq_enable();
452         }
453 }
454
455
456 void tasklet_init(struct tasklet_struct *t,
457                   void (*func)(unsigned long), unsigned long data)
458 {
459         t->next = NULL;
460         t->state = 0;
461         atomic_set(&t->count, 0);
462         t->func = func;
463         t->data = data;
464 }
465
466 EXPORT_SYMBOL(tasklet_init);
467
468 void tasklet_kill(struct tasklet_struct *t)
469 {
470         if (in_interrupt())
471                 printk("Attempt to kill tasklet from interrupt\n");
472
473         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
474                 do
475                         yield();
476                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
477         }
478         tasklet_unlock_wait(t);
479         clear_bit(TASKLET_STATE_SCHED, &t->state);
480 }
481
482 EXPORT_SYMBOL(tasklet_kill);
483
484 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
485 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
486
487 static void __local_trigger(struct call_single_data *cp, int softirq)
488 {
489         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
490
491         list_add_tail(&cp->list, head);
492
493         /* Trigger the softirq only if the list was previously empty.  */
494         if (head->next == &cp->list)
495                 raise_softirq_irqoff(softirq);
496 }
497
498 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
499 static void remote_softirq_receive(void *data)
500 {
501         struct call_single_data *cp = data;
502         unsigned long flags;
503         int softirq;
504
505         softirq = cp->priv;
506
507         local_irq_save(flags);
508         __local_trigger(cp, softirq);
509         local_irq_restore(flags);
510 }
511
512 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
513 {
514         if (cpu_online(cpu)) {
515                 cp->func = remote_softirq_receive;
516                 cp->info = cp;
517                 cp->flags = 0;
518                 cp->priv = softirq;
519
520                 __smp_call_function_single(cpu, cp, 0);
521                 return 0;
522         }
523         return 1;
524 }
525 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
526 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
527 {
528         return 1;
529 }
530 #endif
531
532 /**
533  * __send_remote_softirq - try to schedule softirq work on a remote cpu
534  * @cp: private SMP call function data area
535  * @cpu: the remote cpu
536  * @this_cpu: the currently executing cpu
537  * @softirq: the softirq for the work
538  *
539  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
540  * done, the work is instead queued up on the local cpu.
541  *
542  * Interrupts must be disabled.
543  */
544 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
545 {
546         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
547                 __local_trigger(cp, softirq);
548 }
549 EXPORT_SYMBOL(__send_remote_softirq);
550
551 /**
552  * send_remote_softirq - try to schedule softirq work on a remote cpu
553  * @cp: private SMP call function data area
554  * @cpu: the remote cpu
555  * @softirq: the softirq for the work
556  *
557  * Like __send_remote_softirq except that disabling interrupts and
558  * computing the current cpu is done for the caller.
559  */
560 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
561 {
562         unsigned long flags;
563         int this_cpu;
564
565         local_irq_save(flags);
566         this_cpu = smp_processor_id();
567         __send_remote_softirq(cp, cpu, this_cpu, softirq);
568         local_irq_restore(flags);
569 }
570 EXPORT_SYMBOL(send_remote_softirq);
571
572 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
573                                                unsigned long action, void *hcpu)
574 {
575         /*
576          * If a CPU goes away, splice its entries to the current CPU
577          * and trigger a run of the softirq
578          */
579         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
580                 int cpu = (unsigned long) hcpu;
581                 int i;
582
583                 local_irq_disable();
584                 for (i = 0; i < NR_SOFTIRQS; i++) {
585                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
586                         struct list_head *local_head;
587
588                         if (list_empty(head))
589                                 continue;
590
591                         local_head = &__get_cpu_var(softirq_work_list[i]);
592                         list_splice_init(head, local_head);
593                         raise_softirq_irqoff(i);
594                 }
595                 local_irq_enable();
596         }
597
598         return NOTIFY_OK;
599 }
600
601 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
602         .notifier_call  = remote_softirq_cpu_notify,
603 };
604
605 void __init softirq_init(void)
606 {
607         int cpu;
608
609         for_each_possible_cpu(cpu) {
610                 int i;
611
612                 per_cpu(tasklet_vec, cpu).tail =
613                         &per_cpu(tasklet_vec, cpu).head;
614                 per_cpu(tasklet_hi_vec, cpu).tail =
615                         &per_cpu(tasklet_hi_vec, cpu).head;
616                 for (i = 0; i < NR_SOFTIRQS; i++)
617                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
618         }
619
620         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
621
622         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
623         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
624 }
625
626 static int ksoftirqd(void * __bind_cpu)
627 {
628         set_current_state(TASK_INTERRUPTIBLE);
629
630         while (!kthread_should_stop()) {
631                 preempt_disable();
632                 if (!local_softirq_pending()) {
633                         preempt_enable_no_resched();
634                         schedule();
635                         preempt_disable();
636                 }
637
638                 __set_current_state(TASK_RUNNING);
639
640                 while (local_softirq_pending()) {
641                         /* Preempt disable stops cpu going offline.
642                            If already offline, we'll be on wrong CPU:
643                            don't process */
644                         if (cpu_is_offline((long)__bind_cpu))
645                                 goto wait_to_die;
646                         do_softirq();
647                         preempt_enable_no_resched();
648                         cond_resched();
649                         preempt_disable();
650                         rcu_qsctr_inc((long)__bind_cpu);
651                 }
652                 preempt_enable();
653                 set_current_state(TASK_INTERRUPTIBLE);
654         }
655         __set_current_state(TASK_RUNNING);
656         return 0;
657
658 wait_to_die:
659         preempt_enable();
660         /* Wait for kthread_stop */
661         set_current_state(TASK_INTERRUPTIBLE);
662         while (!kthread_should_stop()) {
663                 schedule();
664                 set_current_state(TASK_INTERRUPTIBLE);
665         }
666         __set_current_state(TASK_RUNNING);
667         return 0;
668 }
669
670 #ifdef CONFIG_HOTPLUG_CPU
671 /*
672  * tasklet_kill_immediate is called to remove a tasklet which can already be
673  * scheduled for execution on @cpu.
674  *
675  * Unlike tasklet_kill, this function removes the tasklet
676  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
677  *
678  * When this function is called, @cpu must be in the CPU_DEAD state.
679  */
680 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
681 {
682         struct tasklet_struct **i;
683
684         BUG_ON(cpu_online(cpu));
685         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
686
687         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
688                 return;
689
690         /* CPU is dead, so no lock needed. */
691         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
692                 if (*i == t) {
693                         *i = t->next;
694                         /* If this was the tail element, move the tail ptr */
695                         if (*i == NULL)
696                                 per_cpu(tasklet_vec, cpu).tail = i;
697                         return;
698                 }
699         }
700         BUG();
701 }
702
703 static void takeover_tasklets(unsigned int cpu)
704 {
705         /* CPU is dead, so no lock needed. */
706         local_irq_disable();
707
708         /* Find end, append list for that CPU. */
709         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
710                 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
711                 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
712                 per_cpu(tasklet_vec, cpu).head = NULL;
713                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
714         }
715         raise_softirq_irqoff(TASKLET_SOFTIRQ);
716
717         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
718                 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
719                 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
720                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
721                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
722         }
723         raise_softirq_irqoff(HI_SOFTIRQ);
724
725         local_irq_enable();
726 }
727 #endif /* CONFIG_HOTPLUG_CPU */
728
729 static int __cpuinit cpu_callback(struct notifier_block *nfb,
730                                   unsigned long action,
731                                   void *hcpu)
732 {
733         int hotcpu = (unsigned long)hcpu;
734         struct task_struct *p;
735
736         switch (action) {
737         case CPU_UP_PREPARE:
738         case CPU_UP_PREPARE_FROZEN:
739                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
740                 if (IS_ERR(p)) {
741                         printk("ksoftirqd for %i failed\n", hotcpu);
742                         return NOTIFY_BAD;
743                 }
744                 kthread_bind(p, hotcpu);
745                 per_cpu(ksoftirqd, hotcpu) = p;
746                 break;
747         case CPU_ONLINE:
748         case CPU_ONLINE_FROZEN:
749                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
750                 break;
751 #ifdef CONFIG_HOTPLUG_CPU
752         case CPU_UP_CANCELED:
753         case CPU_UP_CANCELED_FROZEN:
754                 if (!per_cpu(ksoftirqd, hotcpu))
755                         break;
756                 /* Unbind so it can run.  Fall thru. */
757                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
758                              cpumask_any(cpu_online_mask));
759         case CPU_DEAD:
760         case CPU_DEAD_FROZEN: {
761                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
762
763                 p = per_cpu(ksoftirqd, hotcpu);
764                 per_cpu(ksoftirqd, hotcpu) = NULL;
765                 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
766                 kthread_stop(p);
767                 takeover_tasklets(hotcpu);
768                 break;
769         }
770 #endif /* CONFIG_HOTPLUG_CPU */
771         }
772         return NOTIFY_OK;
773 }
774
775 static struct notifier_block __cpuinitdata cpu_nfb = {
776         .notifier_call = cpu_callback
777 };
778
779 static __init int spawn_ksoftirqd(void)
780 {
781         void *cpu = (void *)(long)smp_processor_id();
782         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
783
784         BUG_ON(err == NOTIFY_BAD);
785         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
786         register_cpu_notifier(&cpu_nfb);
787         return 0;
788 }
789 early_initcall(spawn_ksoftirqd);
790
791 #ifdef CONFIG_SMP
792 /*
793  * Call a function on all processors
794  */
795 int on_each_cpu(void (*func) (void *info), void *info, int wait)
796 {
797         int ret = 0;
798
799         preempt_disable();
800         ret = smp_call_function(func, info, wait);
801         local_irq_disable();
802         func(info);
803         local_irq_enable();
804         preempt_enable();
805         return ret;
806 }
807 EXPORT_SYMBOL(on_each_cpu);
808 #endif
809
810 /*
811  * [ These __weak aliases are kept in a separate compilation unit, so that
812  *   GCC does not inline them incorrectly. ]
813  */
814
815 int __init __weak early_irq_init(void)
816 {
817         return 0;
818 }
819
820 int __init __weak arch_probe_nr_irqs(void)
821 {
822         return 0;
823 }
824
825 int __init __weak arch_early_irq_init(void)
826 {
827         return 0;
828 }
829
830 int __weak arch_init_chip_data(struct irq_desc *desc, int cpu)
831 {
832         return 0;
833 }