tracing: tracepoints for softirq entry/exit - tracepoints
[linux-2.6.git] / kernel / softirq.c
1 /*
2  *      linux/kernel/softirq.c
3  *
4  *      Copyright (C) 1992 Linus Torvalds
5  *
6  *      Distribute under GPLv2.
7  *
8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9  *
10  *      Remote softirq infrastructure is by Jens Axboe.
11  */
12
13 #include <linux/module.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/interrupt.h>
16 #include <linux/init.h>
17 #include <linux/mm.h>
18 #include <linux/notifier.h>
19 #include <linux/percpu.h>
20 #include <linux/cpu.h>
21 #include <linux/freezer.h>
22 #include <linux/kthread.h>
23 #include <linux/rcupdate.h>
24 #include <linux/ftrace.h>
25 #include <linux/smp.h>
26 #include <linux/tick.h>
27 #include <trace/irq.h>
28
29 #include <asm/irq.h>
30 /*
31    - No shared variables, all the data are CPU local.
32    - If a softirq needs serialization, let it serialize itself
33      by its own spinlocks.
34    - Even if softirq is serialized, only local cpu is marked for
35      execution. Hence, we get something sort of weak cpu binding.
36      Though it is still not clear, will it result in better locality
37      or will not.
38
39    Examples:
40    - NET RX softirq. It is multithreaded and does not require
41      any global serialization.
42    - NET TX softirq. It kicks software netdevice queues, hence
43      it is logically serialized per device, but this serialization
44      is invisible to common code.
45    - Tasklets: serialized wrt itself.
46  */
47
48 #ifndef __ARCH_IRQ_STAT
49 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
50 EXPORT_SYMBOL(irq_stat);
51 #endif
52
53 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
54
55 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
56
57 char *softirq_to_name[NR_SOFTIRQS] = {
58         "HI_SOFTIRQ", "TIMER_SOFTIRQ", "NET_TX_SOFTIRQ", "NET_RX_SOFTIRQ",
59         "BLOCK_SOFTIRQ", "TASKLET_SOFTIRQ", "SCHED_SOFTIRQ", "HRTIMER_SOFTIRQ",
60         "RCU_SOFTIRQ"
61 };
62
63 /*
64  * we cannot loop indefinitely here to avoid userspace starvation,
65  * but we also don't want to introduce a worst case 1/HZ latency
66  * to the pending events, so lets the scheduler to balance
67  * the softirq load for us.
68  */
69 static inline void wakeup_softirqd(void)
70 {
71         /* Interrupts are disabled: no need to stop preemption */
72         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
73
74         if (tsk && tsk->state != TASK_RUNNING)
75                 wake_up_process(tsk);
76 }
77
78 /*
79  * This one is for softirq.c-internal use,
80  * where hardirqs are disabled legitimately:
81  */
82 #ifdef CONFIG_TRACE_IRQFLAGS
83 static void __local_bh_disable(unsigned long ip)
84 {
85         unsigned long flags;
86
87         WARN_ON_ONCE(in_irq());
88
89         raw_local_irq_save(flags);
90         /*
91          * The preempt tracer hooks into add_preempt_count and will break
92          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
93          * is set and before current->softirq_enabled is cleared.
94          * We must manually increment preempt_count here and manually
95          * call the trace_preempt_off later.
96          */
97         preempt_count() += SOFTIRQ_OFFSET;
98         /*
99          * Were softirqs turned off above:
100          */
101         if (softirq_count() == SOFTIRQ_OFFSET)
102                 trace_softirqs_off(ip);
103         raw_local_irq_restore(flags);
104
105         if (preempt_count() == SOFTIRQ_OFFSET)
106                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
107 }
108 #else /* !CONFIG_TRACE_IRQFLAGS */
109 static inline void __local_bh_disable(unsigned long ip)
110 {
111         add_preempt_count(SOFTIRQ_OFFSET);
112         barrier();
113 }
114 #endif /* CONFIG_TRACE_IRQFLAGS */
115
116 void local_bh_disable(void)
117 {
118         __local_bh_disable((unsigned long)__builtin_return_address(0));
119 }
120
121 EXPORT_SYMBOL(local_bh_disable);
122
123 /*
124  * Special-case - softirqs can safely be enabled in
125  * cond_resched_softirq(), or by __do_softirq(),
126  * without processing still-pending softirqs:
127  */
128 void _local_bh_enable(void)
129 {
130         WARN_ON_ONCE(in_irq());
131         WARN_ON_ONCE(!irqs_disabled());
132
133         if (softirq_count() == SOFTIRQ_OFFSET)
134                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
135         sub_preempt_count(SOFTIRQ_OFFSET);
136 }
137
138 EXPORT_SYMBOL(_local_bh_enable);
139
140 static inline void _local_bh_enable_ip(unsigned long ip)
141 {
142         WARN_ON_ONCE(in_irq() || irqs_disabled());
143 #ifdef CONFIG_TRACE_IRQFLAGS
144         local_irq_disable();
145 #endif
146         /*
147          * Are softirqs going to be turned on now:
148          */
149         if (softirq_count() == SOFTIRQ_OFFSET)
150                 trace_softirqs_on(ip);
151         /*
152          * Keep preemption disabled until we are done with
153          * softirq processing:
154          */
155         sub_preempt_count(SOFTIRQ_OFFSET - 1);
156
157         if (unlikely(!in_interrupt() && local_softirq_pending()))
158                 do_softirq();
159
160         dec_preempt_count();
161 #ifdef CONFIG_TRACE_IRQFLAGS
162         local_irq_enable();
163 #endif
164         preempt_check_resched();
165 }
166
167 void local_bh_enable(void)
168 {
169         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
170 }
171 EXPORT_SYMBOL(local_bh_enable);
172
173 void local_bh_enable_ip(unsigned long ip)
174 {
175         _local_bh_enable_ip(ip);
176 }
177 EXPORT_SYMBOL(local_bh_enable_ip);
178
179 /*
180  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
181  * and we fall back to softirqd after that.
182  *
183  * This number has been established via experimentation.
184  * The two things to balance is latency against fairness -
185  * we want to handle softirqs as soon as possible, but they
186  * should not be able to lock up the box.
187  */
188 #define MAX_SOFTIRQ_RESTART 10
189
190 DEFINE_TRACE(softirq_entry);
191 DEFINE_TRACE(softirq_exit);
192
193 asmlinkage void __do_softirq(void)
194 {
195         struct softirq_action *h;
196         __u32 pending;
197         int max_restart = MAX_SOFTIRQ_RESTART;
198         int cpu;
199
200         pending = local_softirq_pending();
201         account_system_vtime(current);
202
203         __local_bh_disable((unsigned long)__builtin_return_address(0));
204         lockdep_softirq_enter();
205
206         cpu = smp_processor_id();
207 restart:
208         /* Reset the pending bitmask before enabling irqs */
209         set_softirq_pending(0);
210
211         local_irq_enable();
212
213         h = softirq_vec;
214
215         do {
216                 if (pending & 1) {
217                         int prev_count = preempt_count();
218
219                         trace_softirq_entry(h, softirq_vec);
220                         h->action(h);
221                         trace_softirq_exit(h, softirq_vec);
222                         if (unlikely(prev_count != preempt_count())) {
223                                 printk(KERN_ERR "huh, entered softirq %td %s %p"
224                                        "with preempt_count %08x,"
225                                        " exited with %08x?\n", h - softirq_vec,
226                                        softirq_to_name[h - softirq_vec],
227                                        h->action, prev_count, preempt_count());
228                                 preempt_count() = prev_count;
229                         }
230
231                         rcu_bh_qsctr_inc(cpu);
232                 }
233                 h++;
234                 pending >>= 1;
235         } while (pending);
236
237         local_irq_disable();
238
239         pending = local_softirq_pending();
240         if (pending && --max_restart)
241                 goto restart;
242
243         if (pending)
244                 wakeup_softirqd();
245
246         lockdep_softirq_exit();
247
248         account_system_vtime(current);
249         _local_bh_enable();
250 }
251
252 #ifndef __ARCH_HAS_DO_SOFTIRQ
253
254 asmlinkage void do_softirq(void)
255 {
256         __u32 pending;
257         unsigned long flags;
258
259         if (in_interrupt())
260                 return;
261
262         local_irq_save(flags);
263
264         pending = local_softirq_pending();
265
266         if (pending)
267                 __do_softirq();
268
269         local_irq_restore(flags);
270 }
271
272 #endif
273
274 /*
275  * Enter an interrupt context.
276  */
277 void irq_enter(void)
278 {
279         int cpu = smp_processor_id();
280
281         rcu_irq_enter();
282         if (idle_cpu(cpu) && !in_interrupt()) {
283                 __irq_enter();
284                 tick_check_idle(cpu);
285         } else
286                 __irq_enter();
287 }
288
289 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
290 # define invoke_softirq()       __do_softirq()
291 #else
292 # define invoke_softirq()       do_softirq()
293 #endif
294
295 /*
296  * Exit an interrupt context. Process softirqs if needed and possible:
297  */
298 void irq_exit(void)
299 {
300         account_system_vtime(current);
301         trace_hardirq_exit();
302         sub_preempt_count(IRQ_EXIT_OFFSET);
303         if (!in_interrupt() && local_softirq_pending())
304                 invoke_softirq();
305
306 #ifdef CONFIG_NO_HZ
307         /* Make sure that timer wheel updates are propagated */
308         rcu_irq_exit();
309         if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
310                 tick_nohz_stop_sched_tick(0);
311 #endif
312         preempt_enable_no_resched();
313 }
314
315 /*
316  * This function must run with irqs disabled!
317  */
318 inline void raise_softirq_irqoff(unsigned int nr)
319 {
320         __raise_softirq_irqoff(nr);
321
322         /*
323          * If we're in an interrupt or softirq, we're done
324          * (this also catches softirq-disabled code). We will
325          * actually run the softirq once we return from
326          * the irq or softirq.
327          *
328          * Otherwise we wake up ksoftirqd to make sure we
329          * schedule the softirq soon.
330          */
331         if (!in_interrupt())
332                 wakeup_softirqd();
333 }
334
335 void raise_softirq(unsigned int nr)
336 {
337         unsigned long flags;
338
339         local_irq_save(flags);
340         raise_softirq_irqoff(nr);
341         local_irq_restore(flags);
342 }
343
344 void open_softirq(int nr, void (*action)(struct softirq_action *))
345 {
346         softirq_vec[nr].action = action;
347 }
348
349 /* Tasklets */
350 struct tasklet_head
351 {
352         struct tasklet_struct *head;
353         struct tasklet_struct **tail;
354 };
355
356 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
357 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
358
359 void __tasklet_schedule(struct tasklet_struct *t)
360 {
361         unsigned long flags;
362
363         local_irq_save(flags);
364         t->next = NULL;
365         *__get_cpu_var(tasklet_vec).tail = t;
366         __get_cpu_var(tasklet_vec).tail = &(t->next);
367         raise_softirq_irqoff(TASKLET_SOFTIRQ);
368         local_irq_restore(flags);
369 }
370
371 EXPORT_SYMBOL(__tasklet_schedule);
372
373 void __tasklet_hi_schedule(struct tasklet_struct *t)
374 {
375         unsigned long flags;
376
377         local_irq_save(flags);
378         t->next = NULL;
379         *__get_cpu_var(tasklet_hi_vec).tail = t;
380         __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
381         raise_softirq_irqoff(HI_SOFTIRQ);
382         local_irq_restore(flags);
383 }
384
385 EXPORT_SYMBOL(__tasklet_hi_schedule);
386
387 static void tasklet_action(struct softirq_action *a)
388 {
389         struct tasklet_struct *list;
390
391         local_irq_disable();
392         list = __get_cpu_var(tasklet_vec).head;
393         __get_cpu_var(tasklet_vec).head = NULL;
394         __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
395         local_irq_enable();
396
397         while (list) {
398                 struct tasklet_struct *t = list;
399
400                 list = list->next;
401
402                 if (tasklet_trylock(t)) {
403                         if (!atomic_read(&t->count)) {
404                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
405                                         BUG();
406                                 t->func(t->data);
407                                 tasklet_unlock(t);
408                                 continue;
409                         }
410                         tasklet_unlock(t);
411                 }
412
413                 local_irq_disable();
414                 t->next = NULL;
415                 *__get_cpu_var(tasklet_vec).tail = t;
416                 __get_cpu_var(tasklet_vec).tail = &(t->next);
417                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
418                 local_irq_enable();
419         }
420 }
421
422 static void tasklet_hi_action(struct softirq_action *a)
423 {
424         struct tasklet_struct *list;
425
426         local_irq_disable();
427         list = __get_cpu_var(tasklet_hi_vec).head;
428         __get_cpu_var(tasklet_hi_vec).head = NULL;
429         __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
430         local_irq_enable();
431
432         while (list) {
433                 struct tasklet_struct *t = list;
434
435                 list = list->next;
436
437                 if (tasklet_trylock(t)) {
438                         if (!atomic_read(&t->count)) {
439                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
440                                         BUG();
441                                 t->func(t->data);
442                                 tasklet_unlock(t);
443                                 continue;
444                         }
445                         tasklet_unlock(t);
446                 }
447
448                 local_irq_disable();
449                 t->next = NULL;
450                 *__get_cpu_var(tasklet_hi_vec).tail = t;
451                 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
452                 __raise_softirq_irqoff(HI_SOFTIRQ);
453                 local_irq_enable();
454         }
455 }
456
457
458 void tasklet_init(struct tasklet_struct *t,
459                   void (*func)(unsigned long), unsigned long data)
460 {
461         t->next = NULL;
462         t->state = 0;
463         atomic_set(&t->count, 0);
464         t->func = func;
465         t->data = data;
466 }
467
468 EXPORT_SYMBOL(tasklet_init);
469
470 void tasklet_kill(struct tasklet_struct *t)
471 {
472         if (in_interrupt())
473                 printk("Attempt to kill tasklet from interrupt\n");
474
475         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
476                 do
477                         yield();
478                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
479         }
480         tasklet_unlock_wait(t);
481         clear_bit(TASKLET_STATE_SCHED, &t->state);
482 }
483
484 EXPORT_SYMBOL(tasklet_kill);
485
486 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
487 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
488
489 static void __local_trigger(struct call_single_data *cp, int softirq)
490 {
491         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
492
493         list_add_tail(&cp->list, head);
494
495         /* Trigger the softirq only if the list was previously empty.  */
496         if (head->next == &cp->list)
497                 raise_softirq_irqoff(softirq);
498 }
499
500 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
501 static void remote_softirq_receive(void *data)
502 {
503         struct call_single_data *cp = data;
504         unsigned long flags;
505         int softirq;
506
507         softirq = cp->priv;
508
509         local_irq_save(flags);
510         __local_trigger(cp, softirq);
511         local_irq_restore(flags);
512 }
513
514 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
515 {
516         if (cpu_online(cpu)) {
517                 cp->func = remote_softirq_receive;
518                 cp->info = cp;
519                 cp->flags = 0;
520                 cp->priv = softirq;
521
522                 __smp_call_function_single(cpu, cp);
523                 return 0;
524         }
525         return 1;
526 }
527 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
528 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
529 {
530         return 1;
531 }
532 #endif
533
534 /**
535  * __send_remote_softirq - try to schedule softirq work on a remote cpu
536  * @cp: private SMP call function data area
537  * @cpu: the remote cpu
538  * @this_cpu: the currently executing cpu
539  * @softirq: the softirq for the work
540  *
541  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
542  * done, the work is instead queued up on the local cpu.
543  *
544  * Interrupts must be disabled.
545  */
546 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
547 {
548         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
549                 __local_trigger(cp, softirq);
550 }
551 EXPORT_SYMBOL(__send_remote_softirq);
552
553 /**
554  * send_remote_softirq - try to schedule softirq work on a remote cpu
555  * @cp: private SMP call function data area
556  * @cpu: the remote cpu
557  * @softirq: the softirq for the work
558  *
559  * Like __send_remote_softirq except that disabling interrupts and
560  * computing the current cpu is done for the caller.
561  */
562 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
563 {
564         unsigned long flags;
565         int this_cpu;
566
567         local_irq_save(flags);
568         this_cpu = smp_processor_id();
569         __send_remote_softirq(cp, cpu, this_cpu, softirq);
570         local_irq_restore(flags);
571 }
572 EXPORT_SYMBOL(send_remote_softirq);
573
574 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
575                                                unsigned long action, void *hcpu)
576 {
577         /*
578          * If a CPU goes away, splice its entries to the current CPU
579          * and trigger a run of the softirq
580          */
581         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
582                 int cpu = (unsigned long) hcpu;
583                 int i;
584
585                 local_irq_disable();
586                 for (i = 0; i < NR_SOFTIRQS; i++) {
587                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
588                         struct list_head *local_head;
589
590                         if (list_empty(head))
591                                 continue;
592
593                         local_head = &__get_cpu_var(softirq_work_list[i]);
594                         list_splice_init(head, local_head);
595                         raise_softirq_irqoff(i);
596                 }
597                 local_irq_enable();
598         }
599
600         return NOTIFY_OK;
601 }
602
603 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
604         .notifier_call  = remote_softirq_cpu_notify,
605 };
606
607 void __init softirq_init(void)
608 {
609         int cpu;
610
611         for_each_possible_cpu(cpu) {
612                 int i;
613
614                 per_cpu(tasklet_vec, cpu).tail =
615                         &per_cpu(tasklet_vec, cpu).head;
616                 per_cpu(tasklet_hi_vec, cpu).tail =
617                         &per_cpu(tasklet_hi_vec, cpu).head;
618                 for (i = 0; i < NR_SOFTIRQS; i++)
619                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
620         }
621
622         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
623
624         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
625         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
626 }
627
628 static int ksoftirqd(void * __bind_cpu)
629 {
630         set_current_state(TASK_INTERRUPTIBLE);
631
632         while (!kthread_should_stop()) {
633                 preempt_disable();
634                 if (!local_softirq_pending()) {
635                         preempt_enable_no_resched();
636                         schedule();
637                         preempt_disable();
638                 }
639
640                 __set_current_state(TASK_RUNNING);
641
642                 while (local_softirq_pending()) {
643                         /* Preempt disable stops cpu going offline.
644                            If already offline, we'll be on wrong CPU:
645                            don't process */
646                         if (cpu_is_offline((long)__bind_cpu))
647                                 goto wait_to_die;
648                         do_softirq();
649                         preempt_enable_no_resched();
650                         cond_resched();
651                         preempt_disable();
652                         rcu_qsctr_inc((long)__bind_cpu);
653                 }
654                 preempt_enable();
655                 set_current_state(TASK_INTERRUPTIBLE);
656         }
657         __set_current_state(TASK_RUNNING);
658         return 0;
659
660 wait_to_die:
661         preempt_enable();
662         /* Wait for kthread_stop */
663         set_current_state(TASK_INTERRUPTIBLE);
664         while (!kthread_should_stop()) {
665                 schedule();
666                 set_current_state(TASK_INTERRUPTIBLE);
667         }
668         __set_current_state(TASK_RUNNING);
669         return 0;
670 }
671
672 #ifdef CONFIG_HOTPLUG_CPU
673 /*
674  * tasklet_kill_immediate is called to remove a tasklet which can already be
675  * scheduled for execution on @cpu.
676  *
677  * Unlike tasklet_kill, this function removes the tasklet
678  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
679  *
680  * When this function is called, @cpu must be in the CPU_DEAD state.
681  */
682 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
683 {
684         struct tasklet_struct **i;
685
686         BUG_ON(cpu_online(cpu));
687         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
688
689         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
690                 return;
691
692         /* CPU is dead, so no lock needed. */
693         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
694                 if (*i == t) {
695                         *i = t->next;
696                         /* If this was the tail element, move the tail ptr */
697                         if (*i == NULL)
698                                 per_cpu(tasklet_vec, cpu).tail = i;
699                         return;
700                 }
701         }
702         BUG();
703 }
704
705 static void takeover_tasklets(unsigned int cpu)
706 {
707         /* CPU is dead, so no lock needed. */
708         local_irq_disable();
709
710         /* Find end, append list for that CPU. */
711         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
712                 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
713                 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
714                 per_cpu(tasklet_vec, cpu).head = NULL;
715                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
716         }
717         raise_softirq_irqoff(TASKLET_SOFTIRQ);
718
719         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
720                 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
721                 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
722                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
723                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
724         }
725         raise_softirq_irqoff(HI_SOFTIRQ);
726
727         local_irq_enable();
728 }
729 #endif /* CONFIG_HOTPLUG_CPU */
730
731 static int __cpuinit cpu_callback(struct notifier_block *nfb,
732                                   unsigned long action,
733                                   void *hcpu)
734 {
735         int hotcpu = (unsigned long)hcpu;
736         struct task_struct *p;
737
738         switch (action) {
739         case CPU_UP_PREPARE:
740         case CPU_UP_PREPARE_FROZEN:
741                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
742                 if (IS_ERR(p)) {
743                         printk("ksoftirqd for %i failed\n", hotcpu);
744                         return NOTIFY_BAD;
745                 }
746                 kthread_bind(p, hotcpu);
747                 per_cpu(ksoftirqd, hotcpu) = p;
748                 break;
749         case CPU_ONLINE:
750         case CPU_ONLINE_FROZEN:
751                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
752                 break;
753 #ifdef CONFIG_HOTPLUG_CPU
754         case CPU_UP_CANCELED:
755         case CPU_UP_CANCELED_FROZEN:
756                 if (!per_cpu(ksoftirqd, hotcpu))
757                         break;
758                 /* Unbind so it can run.  Fall thru. */
759                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
760                              cpumask_any(cpu_online_mask));
761         case CPU_DEAD:
762         case CPU_DEAD_FROZEN: {
763                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
764
765                 p = per_cpu(ksoftirqd, hotcpu);
766                 per_cpu(ksoftirqd, hotcpu) = NULL;
767                 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
768                 kthread_stop(p);
769                 takeover_tasklets(hotcpu);
770                 break;
771         }
772 #endif /* CONFIG_HOTPLUG_CPU */
773         }
774         return NOTIFY_OK;
775 }
776
777 static struct notifier_block __cpuinitdata cpu_nfb = {
778         .notifier_call = cpu_callback
779 };
780
781 static __init int spawn_ksoftirqd(void)
782 {
783         void *cpu = (void *)(long)smp_processor_id();
784         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
785
786         BUG_ON(err == NOTIFY_BAD);
787         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
788         register_cpu_notifier(&cpu_nfb);
789         return 0;
790 }
791 early_initcall(spawn_ksoftirqd);
792
793 #ifdef CONFIG_SMP
794 /*
795  * Call a function on all processors
796  */
797 int on_each_cpu(void (*func) (void *info), void *info, int wait)
798 {
799         int ret = 0;
800
801         preempt_disable();
802         ret = smp_call_function(func, info, wait);
803         local_irq_disable();
804         func(info);
805         local_irq_enable();
806         preempt_enable();
807         return ret;
808 }
809 EXPORT_SYMBOL(on_each_cpu);
810 #endif
811
812 /*
813  * [ These __weak aliases are kept in a separate compilation unit, so that
814  *   GCC does not inline them incorrectly. ]
815  */
816
817 int __init __weak early_irq_init(void)
818 {
819         return 0;
820 }
821
822 int __init __weak arch_probe_nr_irqs(void)
823 {
824         return 0;
825 }
826
827 int __init __weak arch_early_irq_init(void)
828 {
829         return 0;
830 }
831
832 int __weak arch_init_chip_data(struct irq_desc *desc, int cpu)
833 {
834         return 0;
835 }