e7c69a720d69a0a7d9166c21883e05d2521cb26a
[linux-2.6.git] / kernel / softirq.c
1 /*
2  *      linux/kernel/softirq.c
3  *
4  *      Copyright (C) 1992 Linus Torvalds
5  *
6  *      Distribute under GPLv2.
7  *
8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9  *
10  *      Remote softirq infrastructure is by Jens Axboe.
11  */
12
13 #include <linux/module.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/interrupt.h>
16 #include <linux/init.h>
17 #include <linux/mm.h>
18 #include <linux/notifier.h>
19 #include <linux/percpu.h>
20 #include <linux/cpu.h>
21 #include <linux/freezer.h>
22 #include <linux/kthread.h>
23 #include <linux/rcupdate.h>
24 #include <linux/smp.h>
25 #include <linux/tick.h>
26
27 #include <asm/irq.h>
28 /*
29    - No shared variables, all the data are CPU local.
30    - If a softirq needs serialization, let it serialize itself
31      by its own spinlocks.
32    - Even if softirq is serialized, only local cpu is marked for
33      execution. Hence, we get something sort of weak cpu binding.
34      Though it is still not clear, will it result in better locality
35      or will not.
36
37    Examples:
38    - NET RX softirq. It is multithreaded and does not require
39      any global serialization.
40    - NET TX softirq. It kicks software netdevice queues, hence
41      it is logically serialized per device, but this serialization
42      is invisible to common code.
43    - Tasklets: serialized wrt itself.
44  */
45
46 #ifndef __ARCH_IRQ_STAT
47 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
48 EXPORT_SYMBOL(irq_stat);
49 #endif
50
51 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
52
53 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
54
55 /*
56  * we cannot loop indefinitely here to avoid userspace starvation,
57  * but we also don't want to introduce a worst case 1/HZ latency
58  * to the pending events, so lets the scheduler to balance
59  * the softirq load for us.
60  */
61 static inline void wakeup_softirqd(void)
62 {
63         /* Interrupts are disabled: no need to stop preemption */
64         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
65
66         if (tsk && tsk->state != TASK_RUNNING)
67                 wake_up_process(tsk);
68 }
69
70 /*
71  * This one is for softirq.c-internal use,
72  * where hardirqs are disabled legitimately:
73  */
74 #ifdef CONFIG_TRACE_IRQFLAGS
75 static void __local_bh_disable(unsigned long ip)
76 {
77         unsigned long flags;
78
79         WARN_ON_ONCE(in_irq());
80
81         raw_local_irq_save(flags);
82         add_preempt_count(SOFTIRQ_OFFSET);
83         /*
84          * Were softirqs turned off above:
85          */
86         if (softirq_count() == SOFTIRQ_OFFSET)
87                 trace_softirqs_off(ip);
88         raw_local_irq_restore(flags);
89 }
90 #else /* !CONFIG_TRACE_IRQFLAGS */
91 static inline void __local_bh_disable(unsigned long ip)
92 {
93         add_preempt_count(SOFTIRQ_OFFSET);
94         barrier();
95 }
96 #endif /* CONFIG_TRACE_IRQFLAGS */
97
98 void local_bh_disable(void)
99 {
100         __local_bh_disable((unsigned long)__builtin_return_address(0));
101 }
102
103 EXPORT_SYMBOL(local_bh_disable);
104
105 void __local_bh_enable(void)
106 {
107         WARN_ON_ONCE(in_irq());
108
109         /*
110          * softirqs should never be enabled by __local_bh_enable(),
111          * it always nests inside local_bh_enable() sections:
112          */
113         WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
114
115         sub_preempt_count(SOFTIRQ_OFFSET);
116 }
117 EXPORT_SYMBOL_GPL(__local_bh_enable);
118
119 /*
120  * Special-case - softirqs can safely be enabled in
121  * cond_resched_softirq(), or by __do_softirq(),
122  * without processing still-pending softirqs:
123  */
124 void _local_bh_enable(void)
125 {
126         WARN_ON_ONCE(in_irq());
127         WARN_ON_ONCE(!irqs_disabled());
128
129         if (softirq_count() == SOFTIRQ_OFFSET)
130                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
131         sub_preempt_count(SOFTIRQ_OFFSET);
132 }
133
134 EXPORT_SYMBOL(_local_bh_enable);
135
136 static inline void _local_bh_enable_ip(unsigned long ip)
137 {
138         WARN_ON_ONCE(in_irq() || irqs_disabled());
139 #ifdef CONFIG_TRACE_IRQFLAGS
140         local_irq_disable();
141 #endif
142         /*
143          * Are softirqs going to be turned on now:
144          */
145         if (softirq_count() == SOFTIRQ_OFFSET)
146                 trace_softirqs_on(ip);
147         /*
148          * Keep preemption disabled until we are done with
149          * softirq processing:
150          */
151         sub_preempt_count(SOFTIRQ_OFFSET - 1);
152
153         if (unlikely(!in_interrupt() && local_softirq_pending()))
154                 do_softirq();
155
156         dec_preempt_count();
157 #ifdef CONFIG_TRACE_IRQFLAGS
158         local_irq_enable();
159 #endif
160         preempt_check_resched();
161 }
162
163 void local_bh_enable(void)
164 {
165         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
166 }
167 EXPORT_SYMBOL(local_bh_enable);
168
169 void local_bh_enable_ip(unsigned long ip)
170 {
171         _local_bh_enable_ip(ip);
172 }
173 EXPORT_SYMBOL(local_bh_enable_ip);
174
175 /*
176  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
177  * and we fall back to softirqd after that.
178  *
179  * This number has been established via experimentation.
180  * The two things to balance is latency against fairness -
181  * we want to handle softirqs as soon as possible, but they
182  * should not be able to lock up the box.
183  */
184 #define MAX_SOFTIRQ_RESTART 10
185
186 asmlinkage void __do_softirq(void)
187 {
188         struct softirq_action *h;
189         __u32 pending;
190         int max_restart = MAX_SOFTIRQ_RESTART;
191         int cpu;
192
193         pending = local_softirq_pending();
194         account_system_vtime(current);
195
196         __local_bh_disable((unsigned long)__builtin_return_address(0));
197         trace_softirq_enter();
198
199         cpu = smp_processor_id();
200 restart:
201         /* Reset the pending bitmask before enabling irqs */
202         set_softirq_pending(0);
203
204         local_irq_enable();
205
206         h = softirq_vec;
207
208         do {
209                 if (pending & 1) {
210                         int prev_count = preempt_count();
211
212                         h->action(h);
213
214                         if (unlikely(prev_count != preempt_count())) {
215                                 printk(KERN_ERR "huh, entered softirq %td %p"
216                                        "with preempt_count %08x,"
217                                        " exited with %08x?\n", h - softirq_vec,
218                                        h->action, prev_count, preempt_count());
219                                 preempt_count() = prev_count;
220                         }
221
222                         rcu_bh_qsctr_inc(cpu);
223                 }
224                 h++;
225                 pending >>= 1;
226         } while (pending);
227
228         local_irq_disable();
229
230         pending = local_softirq_pending();
231         if (pending && --max_restart)
232                 goto restart;
233
234         if (pending)
235                 wakeup_softirqd();
236
237         trace_softirq_exit();
238
239         account_system_vtime(current);
240         _local_bh_enable();
241 }
242
243 #ifndef __ARCH_HAS_DO_SOFTIRQ
244
245 asmlinkage void do_softirq(void)
246 {
247         __u32 pending;
248         unsigned long flags;
249
250         if (in_interrupt())
251                 return;
252
253         local_irq_save(flags);
254
255         pending = local_softirq_pending();
256
257         if (pending)
258                 __do_softirq();
259
260         local_irq_restore(flags);
261 }
262
263 #endif
264
265 /*
266  * Enter an interrupt context.
267  */
268 void irq_enter(void)
269 {
270         int cpu = smp_processor_id();
271
272         if (idle_cpu(cpu) && !in_interrupt()) {
273                 __irq_enter();
274                 tick_check_idle(cpu);
275         } else
276                 __irq_enter();
277 }
278
279 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
280 # define invoke_softirq()       __do_softirq()
281 #else
282 # define invoke_softirq()       do_softirq()
283 #endif
284
285 /*
286  * Exit an interrupt context. Process softirqs if needed and possible:
287  */
288 void irq_exit(void)
289 {
290         account_system_vtime(current);
291         trace_hardirq_exit();
292         sub_preempt_count(IRQ_EXIT_OFFSET);
293         if (!in_interrupt() && local_softirq_pending())
294                 invoke_softirq();
295
296 #ifdef CONFIG_NO_HZ
297         /* Make sure that timer wheel updates are propagated */
298         if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
299                 tick_nohz_stop_sched_tick(0);
300         rcu_irq_exit();
301 #endif
302         preempt_enable_no_resched();
303 }
304
305 /*
306  * This function must run with irqs disabled!
307  */
308 inline void raise_softirq_irqoff(unsigned int nr)
309 {
310         __raise_softirq_irqoff(nr);
311
312         /*
313          * If we're in an interrupt or softirq, we're done
314          * (this also catches softirq-disabled code). We will
315          * actually run the softirq once we return from
316          * the irq or softirq.
317          *
318          * Otherwise we wake up ksoftirqd to make sure we
319          * schedule the softirq soon.
320          */
321         if (!in_interrupt())
322                 wakeup_softirqd();
323 }
324
325 void raise_softirq(unsigned int nr)
326 {
327         unsigned long flags;
328
329         local_irq_save(flags);
330         raise_softirq_irqoff(nr);
331         local_irq_restore(flags);
332 }
333
334 void open_softirq(int nr, void (*action)(struct softirq_action *))
335 {
336         softirq_vec[nr].action = action;
337 }
338
339 /* Tasklets */
340 struct tasklet_head
341 {
342         struct tasklet_struct *head;
343         struct tasklet_struct **tail;
344 };
345
346 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
347 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
348
349 void __tasklet_schedule(struct tasklet_struct *t)
350 {
351         unsigned long flags;
352
353         local_irq_save(flags);
354         t->next = NULL;
355         *__get_cpu_var(tasklet_vec).tail = t;
356         __get_cpu_var(tasklet_vec).tail = &(t->next);
357         raise_softirq_irqoff(TASKLET_SOFTIRQ);
358         local_irq_restore(flags);
359 }
360
361 EXPORT_SYMBOL(__tasklet_schedule);
362
363 void __tasklet_hi_schedule(struct tasklet_struct *t)
364 {
365         unsigned long flags;
366
367         local_irq_save(flags);
368         t->next = NULL;
369         *__get_cpu_var(tasklet_hi_vec).tail = t;
370         __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
371         raise_softirq_irqoff(HI_SOFTIRQ);
372         local_irq_restore(flags);
373 }
374
375 EXPORT_SYMBOL(__tasklet_hi_schedule);
376
377 static void tasklet_action(struct softirq_action *a)
378 {
379         struct tasklet_struct *list;
380
381         local_irq_disable();
382         list = __get_cpu_var(tasklet_vec).head;
383         __get_cpu_var(tasklet_vec).head = NULL;
384         __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
385         local_irq_enable();
386
387         while (list) {
388                 struct tasklet_struct *t = list;
389
390                 list = list->next;
391
392                 if (tasklet_trylock(t)) {
393                         if (!atomic_read(&t->count)) {
394                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
395                                         BUG();
396                                 t->func(t->data);
397                                 tasklet_unlock(t);
398                                 continue;
399                         }
400                         tasklet_unlock(t);
401                 }
402
403                 local_irq_disable();
404                 t->next = NULL;
405                 *__get_cpu_var(tasklet_vec).tail = t;
406                 __get_cpu_var(tasklet_vec).tail = &(t->next);
407                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
408                 local_irq_enable();
409         }
410 }
411
412 static void tasklet_hi_action(struct softirq_action *a)
413 {
414         struct tasklet_struct *list;
415
416         local_irq_disable();
417         list = __get_cpu_var(tasklet_hi_vec).head;
418         __get_cpu_var(tasklet_hi_vec).head = NULL;
419         __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
420         local_irq_enable();
421
422         while (list) {
423                 struct tasklet_struct *t = list;
424
425                 list = list->next;
426
427                 if (tasklet_trylock(t)) {
428                         if (!atomic_read(&t->count)) {
429                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
430                                         BUG();
431                                 t->func(t->data);
432                                 tasklet_unlock(t);
433                                 continue;
434                         }
435                         tasklet_unlock(t);
436                 }
437
438                 local_irq_disable();
439                 t->next = NULL;
440                 *__get_cpu_var(tasklet_hi_vec).tail = t;
441                 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
442                 __raise_softirq_irqoff(HI_SOFTIRQ);
443                 local_irq_enable();
444         }
445 }
446
447
448 void tasklet_init(struct tasklet_struct *t,
449                   void (*func)(unsigned long), unsigned long data)
450 {
451         t->next = NULL;
452         t->state = 0;
453         atomic_set(&t->count, 0);
454         t->func = func;
455         t->data = data;
456 }
457
458 EXPORT_SYMBOL(tasklet_init);
459
460 void tasklet_kill(struct tasklet_struct *t)
461 {
462         if (in_interrupt())
463                 printk("Attempt to kill tasklet from interrupt\n");
464
465         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
466                 do
467                         yield();
468                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
469         }
470         tasklet_unlock_wait(t);
471         clear_bit(TASKLET_STATE_SCHED, &t->state);
472 }
473
474 EXPORT_SYMBOL(tasklet_kill);
475
476 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
477 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
478
479 static void __local_trigger(struct call_single_data *cp, int softirq)
480 {
481         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
482
483         list_add_tail(&cp->list, head);
484
485         /* Trigger the softirq only if the list was previously empty.  */
486         if (head->next == &cp->list)
487                 raise_softirq_irqoff(softirq);
488 }
489
490 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
491 static void remote_softirq_receive(void *data)
492 {
493         struct call_single_data *cp = data;
494         unsigned long flags;
495         int softirq;
496
497         softirq = cp->priv;
498
499         local_irq_save(flags);
500         __local_trigger(cp, softirq);
501         local_irq_restore(flags);
502 }
503
504 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
505 {
506         if (cpu_online(cpu)) {
507                 cp->func = remote_softirq_receive;
508                 cp->info = cp;
509                 cp->flags = 0;
510                 cp->priv = softirq;
511
512                 __smp_call_function_single(cpu, cp);
513                 return 0;
514         }
515         return 1;
516 }
517 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
518 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
519 {
520         return 1;
521 }
522 #endif
523
524 /**
525  * __send_remote_softirq - try to schedule softirq work on a remote cpu
526  * @cp: private SMP call function data area
527  * @cpu: the remote cpu
528  * @this_cpu: the currently executing cpu
529  * @softirq: the softirq for the work
530  *
531  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
532  * done, the work is instead queued up on the local cpu.
533  *
534  * Interrupts must be disabled.
535  */
536 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
537 {
538         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
539                 __local_trigger(cp, softirq);
540 }
541 EXPORT_SYMBOL(__send_remote_softirq);
542
543 /**
544  * send_remote_softirq - try to schedule softirq work on a remote cpu
545  * @cp: private SMP call function data area
546  * @cpu: the remote cpu
547  * @softirq: the softirq for the work
548  *
549  * Like __send_remote_softirq except that disabling interrupts and
550  * computing the current cpu is done for the caller.
551  */
552 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
553 {
554         unsigned long flags;
555         int this_cpu;
556
557         local_irq_save(flags);
558         this_cpu = smp_processor_id();
559         __send_remote_softirq(cp, cpu, this_cpu, softirq);
560         local_irq_restore(flags);
561 }
562 EXPORT_SYMBOL(send_remote_softirq);
563
564 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
565                                                unsigned long action, void *hcpu)
566 {
567         /*
568          * If a CPU goes away, splice its entries to the current CPU
569          * and trigger a run of the softirq
570          */
571         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
572                 int cpu = (unsigned long) hcpu;
573                 int i;
574
575                 local_irq_disable();
576                 for (i = 0; i < NR_SOFTIRQS; i++) {
577                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
578                         struct list_head *local_head;
579
580                         if (list_empty(head))
581                                 continue;
582
583                         local_head = &__get_cpu_var(softirq_work_list[i]);
584                         list_splice_init(head, local_head);
585                         raise_softirq_irqoff(i);
586                 }
587                 local_irq_enable();
588         }
589
590         return NOTIFY_OK;
591 }
592
593 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
594         .notifier_call  = remote_softirq_cpu_notify,
595 };
596
597 void __init softirq_init(void)
598 {
599         int cpu;
600
601         for_each_possible_cpu(cpu) {
602                 int i;
603
604                 per_cpu(tasklet_vec, cpu).tail =
605                         &per_cpu(tasklet_vec, cpu).head;
606                 per_cpu(tasklet_hi_vec, cpu).tail =
607                         &per_cpu(tasklet_hi_vec, cpu).head;
608                 for (i = 0; i < NR_SOFTIRQS; i++)
609                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
610         }
611
612         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
613
614         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
615         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
616 }
617
618 static int ksoftirqd(void * __bind_cpu)
619 {
620         set_current_state(TASK_INTERRUPTIBLE);
621
622         while (!kthread_should_stop()) {
623                 preempt_disable();
624                 if (!local_softirq_pending()) {
625                         preempt_enable_no_resched();
626                         schedule();
627                         preempt_disable();
628                 }
629
630                 __set_current_state(TASK_RUNNING);
631
632                 while (local_softirq_pending()) {
633                         /* Preempt disable stops cpu going offline.
634                            If already offline, we'll be on wrong CPU:
635                            don't process */
636                         if (cpu_is_offline((long)__bind_cpu))
637                                 goto wait_to_die;
638                         do_softirq();
639                         preempt_enable_no_resched();
640                         cond_resched();
641                         preempt_disable();
642                 }
643                 preempt_enable();
644                 set_current_state(TASK_INTERRUPTIBLE);
645         }
646         __set_current_state(TASK_RUNNING);
647         return 0;
648
649 wait_to_die:
650         preempt_enable();
651         /* Wait for kthread_stop */
652         set_current_state(TASK_INTERRUPTIBLE);
653         while (!kthread_should_stop()) {
654                 schedule();
655                 set_current_state(TASK_INTERRUPTIBLE);
656         }
657         __set_current_state(TASK_RUNNING);
658         return 0;
659 }
660
661 #ifdef CONFIG_HOTPLUG_CPU
662 /*
663  * tasklet_kill_immediate is called to remove a tasklet which can already be
664  * scheduled for execution on @cpu.
665  *
666  * Unlike tasklet_kill, this function removes the tasklet
667  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
668  *
669  * When this function is called, @cpu must be in the CPU_DEAD state.
670  */
671 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
672 {
673         struct tasklet_struct **i;
674
675         BUG_ON(cpu_online(cpu));
676         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
677
678         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
679                 return;
680
681         /* CPU is dead, so no lock needed. */
682         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
683                 if (*i == t) {
684                         *i = t->next;
685                         /* If this was the tail element, move the tail ptr */
686                         if (*i == NULL)
687                                 per_cpu(tasklet_vec, cpu).tail = i;
688                         return;
689                 }
690         }
691         BUG();
692 }
693
694 static void takeover_tasklets(unsigned int cpu)
695 {
696         /* CPU is dead, so no lock needed. */
697         local_irq_disable();
698
699         /* Find end, append list for that CPU. */
700         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
701                 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
702                 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
703                 per_cpu(tasklet_vec, cpu).head = NULL;
704                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
705         }
706         raise_softirq_irqoff(TASKLET_SOFTIRQ);
707
708         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
709                 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
710                 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
711                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
712                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
713         }
714         raise_softirq_irqoff(HI_SOFTIRQ);
715
716         local_irq_enable();
717 }
718 #endif /* CONFIG_HOTPLUG_CPU */
719
720 static int __cpuinit cpu_callback(struct notifier_block *nfb,
721                                   unsigned long action,
722                                   void *hcpu)
723 {
724         int hotcpu = (unsigned long)hcpu;
725         struct task_struct *p;
726
727         switch (action) {
728         case CPU_UP_PREPARE:
729         case CPU_UP_PREPARE_FROZEN:
730                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
731                 if (IS_ERR(p)) {
732                         printk("ksoftirqd for %i failed\n", hotcpu);
733                         return NOTIFY_BAD;
734                 }
735                 kthread_bind(p, hotcpu);
736                 per_cpu(ksoftirqd, hotcpu) = p;
737                 break;
738         case CPU_ONLINE:
739         case CPU_ONLINE_FROZEN:
740                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
741                 break;
742 #ifdef CONFIG_HOTPLUG_CPU
743         case CPU_UP_CANCELED:
744         case CPU_UP_CANCELED_FROZEN:
745                 if (!per_cpu(ksoftirqd, hotcpu))
746                         break;
747                 /* Unbind so it can run.  Fall thru. */
748                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
749                              any_online_cpu(cpu_online_map));
750         case CPU_DEAD:
751         case CPU_DEAD_FROZEN: {
752                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
753
754                 p = per_cpu(ksoftirqd, hotcpu);
755                 per_cpu(ksoftirqd, hotcpu) = NULL;
756                 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
757                 kthread_stop(p);
758                 takeover_tasklets(hotcpu);
759                 break;
760         }
761 #endif /* CONFIG_HOTPLUG_CPU */
762         }
763         return NOTIFY_OK;
764 }
765
766 static struct notifier_block __cpuinitdata cpu_nfb = {
767         .notifier_call = cpu_callback
768 };
769
770 static __init int spawn_ksoftirqd(void)
771 {
772         void *cpu = (void *)(long)smp_processor_id();
773         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
774
775         BUG_ON(err == NOTIFY_BAD);
776         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
777         register_cpu_notifier(&cpu_nfb);
778         return 0;
779 }
780 early_initcall(spawn_ksoftirqd);
781
782 #ifdef CONFIG_SMP
783 /*
784  * Call a function on all processors
785  */
786 int on_each_cpu(void (*func) (void *info), void *info, int wait)
787 {
788         int ret = 0;
789
790         preempt_disable();
791         ret = smp_call_function(func, info, wait);
792         local_irq_disable();
793         func(info);
794         local_irq_enable();
795         preempt_enable();
796         return ret;
797 }
798 EXPORT_SYMBOL(on_each_cpu);
799 #endif