genirq: Fix incorrect unlock in __setup_irq()
[linux-2.6.git] / kernel / irq / manage.c
1 /*
2  * linux/kernel/irq/manage.c
3  *
4  * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
5  * Copyright (C) 2005-2006 Thomas Gleixner
6  *
7  * This file contains driver APIs to the irq subsystem.
8  */
9
10 #include <linux/irq.h>
11 #include <linux/kthread.h>
12 #include <linux/module.h>
13 #include <linux/random.h>
14 #include <linux/interrupt.h>
15 #include <linux/slab.h>
16 #include <linux/sched.h>
17
18 #include "internals.h"
19
20 #ifdef CONFIG_IRQ_FORCED_THREADING
21 __read_mostly bool force_irqthreads;
22
23 static int __init setup_forced_irqthreads(char *arg)
24 {
25         force_irqthreads = true;
26         return 0;
27 }
28 early_param("threadirqs", setup_forced_irqthreads);
29 #endif
30
31 /**
32  *      synchronize_irq - wait for pending IRQ handlers (on other CPUs)
33  *      @irq: interrupt number to wait for
34  *
35  *      This function waits for any pending IRQ handlers for this interrupt
36  *      to complete before returning. If you use this function while
37  *      holding a resource the IRQ handler may need you will deadlock.
38  *
39  *      This function may be called - with care - from IRQ context.
40  */
41 void synchronize_irq(unsigned int irq)
42 {
43         struct irq_desc *desc = irq_to_desc(irq);
44         unsigned int state;
45
46         if (!desc)
47                 return;
48
49         do {
50                 unsigned long flags;
51
52                 /*
53                  * Wait until we're out of the critical section.  This might
54                  * give the wrong answer due to the lack of memory barriers.
55                  */
56                 while (desc->istate & IRQS_INPROGRESS)
57                         cpu_relax();
58
59                 /* Ok, that indicated we're done: double-check carefully. */
60                 raw_spin_lock_irqsave(&desc->lock, flags);
61                 state = desc->istate;
62                 raw_spin_unlock_irqrestore(&desc->lock, flags);
63
64                 /* Oops, that failed? */
65         } while (state & IRQS_INPROGRESS);
66
67         /*
68          * We made sure that no hardirq handler is running. Now verify
69          * that no threaded handlers are active.
70          */
71         wait_event(desc->wait_for_threads, !atomic_read(&desc->threads_active));
72 }
73 EXPORT_SYMBOL(synchronize_irq);
74
75 #ifdef CONFIG_SMP
76 cpumask_var_t irq_default_affinity;
77
78 /**
79  *      irq_can_set_affinity - Check if the affinity of a given irq can be set
80  *      @irq:           Interrupt to check
81  *
82  */
83 int irq_can_set_affinity(unsigned int irq)
84 {
85         struct irq_desc *desc = irq_to_desc(irq);
86
87         if (!desc || !irqd_can_balance(&desc->irq_data) ||
88             !desc->irq_data.chip || !desc->irq_data.chip->irq_set_affinity)
89                 return 0;
90
91         return 1;
92 }
93
94 /**
95  *      irq_set_thread_affinity - Notify irq threads to adjust affinity
96  *      @desc:          irq descriptor which has affitnity changed
97  *
98  *      We just set IRQTF_AFFINITY and delegate the affinity setting
99  *      to the interrupt thread itself. We can not call
100  *      set_cpus_allowed_ptr() here as we hold desc->lock and this
101  *      code can be called from hard interrupt context.
102  */
103 void irq_set_thread_affinity(struct irq_desc *desc)
104 {
105         struct irqaction *action = desc->action;
106
107         while (action) {
108                 if (action->thread)
109                         set_bit(IRQTF_AFFINITY, &action->thread_flags);
110                 action = action->next;
111         }
112 }
113
114 #ifdef CONFIG_GENERIC_PENDING_IRQ
115 static inline bool irq_can_move_pcntxt(struct irq_desc *desc)
116 {
117         return irq_settings_can_move_pcntxt(desc);
118 }
119 static inline bool irq_move_pending(struct irq_desc *desc)
120 {
121         return irqd_is_setaffinity_pending(&desc->irq_data);
122 }
123 static inline void
124 irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
125 {
126         cpumask_copy(desc->pending_mask, mask);
127 }
128 static inline void
129 irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
130 {
131         cpumask_copy(mask, desc->pending_mask);
132 }
133 #else
134 static inline bool irq_can_move_pcntxt(struct irq_desc *desc) { return true; }
135 static inline bool irq_move_pending(struct irq_desc *desc) { return false; }
136 static inline void
137 irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask) { }
138 static inline void
139 irq_get_pending(struct cpumask *mask, struct irq_desc *desc) { }
140 #endif
141
142 /**
143  *      irq_set_affinity - Set the irq affinity of a given irq
144  *      @irq:           Interrupt to set affinity
145  *      @cpumask:       cpumask
146  *
147  */
148 int irq_set_affinity(unsigned int irq, const struct cpumask *mask)
149 {
150         struct irq_desc *desc = irq_to_desc(irq);
151         struct irq_chip *chip = desc->irq_data.chip;
152         unsigned long flags;
153         int ret = 0;
154
155         if (!chip->irq_set_affinity)
156                 return -EINVAL;
157
158         raw_spin_lock_irqsave(&desc->lock, flags);
159
160         if (irq_can_move_pcntxt(desc)) {
161                 ret = chip->irq_set_affinity(&desc->irq_data, mask, false);
162                 switch (ret) {
163                 case IRQ_SET_MASK_OK:
164                         cpumask_copy(desc->irq_data.affinity, mask);
165                 case IRQ_SET_MASK_OK_NOCOPY:
166                         irq_set_thread_affinity(desc);
167                         ret = 0;
168                 }
169         } else {
170                 irqd_set_move_pending(&desc->irq_data);
171                 irq_copy_pending(desc, mask);
172         }
173
174         if (desc->affinity_notify) {
175                 kref_get(&desc->affinity_notify->kref);
176                 schedule_work(&desc->affinity_notify->work);
177         }
178         irq_compat_set_affinity(desc);
179         irqd_set(&desc->irq_data, IRQD_AFFINITY_SET);
180         raw_spin_unlock_irqrestore(&desc->lock, flags);
181         return ret;
182 }
183
184 int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m)
185 {
186         unsigned long flags;
187         struct irq_desc *desc = irq_get_desc_lock(irq, &flags);
188
189         if (!desc)
190                 return -EINVAL;
191         desc->affinity_hint = m;
192         irq_put_desc_unlock(desc, flags);
193         return 0;
194 }
195 EXPORT_SYMBOL_GPL(irq_set_affinity_hint);
196
197 static void irq_affinity_notify(struct work_struct *work)
198 {
199         struct irq_affinity_notify *notify =
200                 container_of(work, struct irq_affinity_notify, work);
201         struct irq_desc *desc = irq_to_desc(notify->irq);
202         cpumask_var_t cpumask;
203         unsigned long flags;
204
205         if (!desc || !alloc_cpumask_var(&cpumask, GFP_KERNEL))
206                 goto out;
207
208         raw_spin_lock_irqsave(&desc->lock, flags);
209         if (irq_move_pending(desc))
210                 irq_get_pending(cpumask, desc);
211         else
212                 cpumask_copy(cpumask, desc->irq_data.affinity);
213         raw_spin_unlock_irqrestore(&desc->lock, flags);
214
215         notify->notify(notify, cpumask);
216
217         free_cpumask_var(cpumask);
218 out:
219         kref_put(&notify->kref, notify->release);
220 }
221
222 /**
223  *      irq_set_affinity_notifier - control notification of IRQ affinity changes
224  *      @irq:           Interrupt for which to enable/disable notification
225  *      @notify:        Context for notification, or %NULL to disable
226  *                      notification.  Function pointers must be initialised;
227  *                      the other fields will be initialised by this function.
228  *
229  *      Must be called in process context.  Notification may only be enabled
230  *      after the IRQ is allocated and must be disabled before the IRQ is
231  *      freed using free_irq().
232  */
233 int
234 irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify)
235 {
236         struct irq_desc *desc = irq_to_desc(irq);
237         struct irq_affinity_notify *old_notify;
238         unsigned long flags;
239
240         /* The release function is promised process context */
241         might_sleep();
242
243         if (!desc)
244                 return -EINVAL;
245
246         /* Complete initialisation of *notify */
247         if (notify) {
248                 notify->irq = irq;
249                 kref_init(&notify->kref);
250                 INIT_WORK(&notify->work, irq_affinity_notify);
251         }
252
253         raw_spin_lock_irqsave(&desc->lock, flags);
254         old_notify = desc->affinity_notify;
255         desc->affinity_notify = notify;
256         raw_spin_unlock_irqrestore(&desc->lock, flags);
257
258         if (old_notify)
259                 kref_put(&old_notify->kref, old_notify->release);
260
261         return 0;
262 }
263 EXPORT_SYMBOL_GPL(irq_set_affinity_notifier);
264
265 #ifndef CONFIG_AUTO_IRQ_AFFINITY
266 /*
267  * Generic version of the affinity autoselector.
268  */
269 static int
270 setup_affinity(unsigned int irq, struct irq_desc *desc, struct cpumask *mask)
271 {
272         struct irq_chip *chip = irq_desc_get_chip(desc);
273         struct cpumask *set = irq_default_affinity;
274         int ret;
275
276         /* Excludes PER_CPU and NO_BALANCE interrupts */
277         if (!irq_can_set_affinity(irq))
278                 return 0;
279
280         /*
281          * Preserve an userspace affinity setup, but make sure that
282          * one of the targets is online.
283          */
284         if (irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) {
285                 if (cpumask_intersects(desc->irq_data.affinity,
286                                        cpu_online_mask))
287                         set = desc->irq_data.affinity;
288                 else {
289                         irq_compat_clr_affinity(desc);
290                         irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET);
291                 }
292         }
293
294         cpumask_and(mask, cpu_online_mask, set);
295         ret = chip->irq_set_affinity(&desc->irq_data, mask, false);
296         switch (ret) {
297         case IRQ_SET_MASK_OK:
298                 cpumask_copy(desc->irq_data.affinity, mask);
299         case IRQ_SET_MASK_OK_NOCOPY:
300                 irq_set_thread_affinity(desc);
301         }
302         return 0;
303 }
304 #else
305 static inline int
306 setup_affinity(unsigned int irq, struct irq_desc *d, struct cpumask *mask)
307 {
308         return irq_select_affinity(irq);
309 }
310 #endif
311
312 /*
313  * Called when affinity is set via /proc/irq
314  */
315 int irq_select_affinity_usr(unsigned int irq, struct cpumask *mask)
316 {
317         struct irq_desc *desc = irq_to_desc(irq);
318         unsigned long flags;
319         int ret;
320
321         raw_spin_lock_irqsave(&desc->lock, flags);
322         ret = setup_affinity(irq, desc, mask);
323         raw_spin_unlock_irqrestore(&desc->lock, flags);
324         return ret;
325 }
326
327 #else
328 static inline int
329 setup_affinity(unsigned int irq, struct irq_desc *desc, struct cpumask *mask)
330 {
331         return 0;
332 }
333 #endif
334
335 void __disable_irq(struct irq_desc *desc, unsigned int irq, bool suspend)
336 {
337         if (suspend) {
338                 if (!desc->action || (desc->action->flags & IRQF_NO_SUSPEND))
339                         return;
340                 desc->istate |= IRQS_SUSPENDED;
341         }
342
343         if (!desc->depth++)
344                 irq_disable(desc);
345 }
346
347 static int __disable_irq_nosync(unsigned int irq)
348 {
349         unsigned long flags;
350         struct irq_desc *desc = irq_get_desc_buslock(irq, &flags);
351
352         if (!desc)
353                 return -EINVAL;
354         __disable_irq(desc, irq, false);
355         irq_put_desc_busunlock(desc, flags);
356         return 0;
357 }
358
359 /**
360  *      disable_irq_nosync - disable an irq without waiting
361  *      @irq: Interrupt to disable
362  *
363  *      Disable the selected interrupt line.  Disables and Enables are
364  *      nested.
365  *      Unlike disable_irq(), this function does not ensure existing
366  *      instances of the IRQ handler have completed before returning.
367  *
368  *      This function may be called from IRQ context.
369  */
370 void disable_irq_nosync(unsigned int irq)
371 {
372         __disable_irq_nosync(irq);
373 }
374 EXPORT_SYMBOL(disable_irq_nosync);
375
376 /**
377  *      disable_irq - disable an irq and wait for completion
378  *      @irq: Interrupt to disable
379  *
380  *      Disable the selected interrupt line.  Enables and Disables are
381  *      nested.
382  *      This function waits for any pending IRQ handlers for this interrupt
383  *      to complete before returning. If you use this function while
384  *      holding a resource the IRQ handler may need you will deadlock.
385  *
386  *      This function may be called - with care - from IRQ context.
387  */
388 void disable_irq(unsigned int irq)
389 {
390         if (!__disable_irq_nosync(irq))
391                 synchronize_irq(irq);
392 }
393 EXPORT_SYMBOL(disable_irq);
394
395 void __enable_irq(struct irq_desc *desc, unsigned int irq, bool resume)
396 {
397         if (resume) {
398                 if (!(desc->istate & IRQS_SUSPENDED)) {
399                         if (!desc->action)
400                                 return;
401                         if (!(desc->action->flags & IRQF_FORCE_RESUME))
402                                 return;
403                         /* Pretend that it got disabled ! */
404                         desc->depth++;
405                 }
406                 desc->istate &= ~IRQS_SUSPENDED;
407         }
408
409         switch (desc->depth) {
410         case 0:
411  err_out:
412                 WARN(1, KERN_WARNING "Unbalanced enable for IRQ %d\n", irq);
413                 break;
414         case 1: {
415                 if (desc->istate & IRQS_SUSPENDED)
416                         goto err_out;
417                 /* Prevent probing on this irq: */
418                 irq_settings_set_noprobe(desc);
419                 irq_enable(desc);
420                 check_irq_resend(desc, irq);
421                 /* fall-through */
422         }
423         default:
424                 desc->depth--;
425         }
426 }
427
428 /**
429  *      enable_irq - enable handling of an irq
430  *      @irq: Interrupt to enable
431  *
432  *      Undoes the effect of one call to disable_irq().  If this
433  *      matches the last disable, processing of interrupts on this
434  *      IRQ line is re-enabled.
435  *
436  *      This function may be called from IRQ context only when
437  *      desc->irq_data.chip->bus_lock and desc->chip->bus_sync_unlock are NULL !
438  */
439 void enable_irq(unsigned int irq)
440 {
441         unsigned long flags;
442         struct irq_desc *desc = irq_get_desc_buslock(irq, &flags);
443
444         if (!desc)
445                 return;
446         if (WARN(!desc->irq_data.chip,
447                  KERN_ERR "enable_irq before setup/request_irq: irq %u\n", irq))
448                 goto out;
449
450         __enable_irq(desc, irq, false);
451 out:
452         irq_put_desc_busunlock(desc, flags);
453 }
454 EXPORT_SYMBOL(enable_irq);
455
456 static int set_irq_wake_real(unsigned int irq, unsigned int on)
457 {
458         struct irq_desc *desc = irq_to_desc(irq);
459         int ret = -ENXIO;
460
461         if (desc->irq_data.chip->irq_set_wake)
462                 ret = desc->irq_data.chip->irq_set_wake(&desc->irq_data, on);
463
464         return ret;
465 }
466
467 /**
468  *      irq_set_irq_wake - control irq power management wakeup
469  *      @irq:   interrupt to control
470  *      @on:    enable/disable power management wakeup
471  *
472  *      Enable/disable power management wakeup mode, which is
473  *      disabled by default.  Enables and disables must match,
474  *      just as they match for non-wakeup mode support.
475  *
476  *      Wakeup mode lets this IRQ wake the system from sleep
477  *      states like "suspend to RAM".
478  */
479 int irq_set_irq_wake(unsigned int irq, unsigned int on)
480 {
481         unsigned long flags;
482         struct irq_desc *desc = irq_get_desc_buslock(irq, &flags);
483         int ret = 0;
484
485         /* wakeup-capable irqs can be shared between drivers that
486          * don't need to have the same sleep mode behaviors.
487          */
488         if (on) {
489                 if (desc->wake_depth++ == 0) {
490                         ret = set_irq_wake_real(irq, on);
491                         if (ret)
492                                 desc->wake_depth = 0;
493                         else
494                                 irqd_set(&desc->irq_data, IRQD_WAKEUP_STATE);
495                 }
496         } else {
497                 if (desc->wake_depth == 0) {
498                         WARN(1, "Unbalanced IRQ %d wake disable\n", irq);
499                 } else if (--desc->wake_depth == 0) {
500                         ret = set_irq_wake_real(irq, on);
501                         if (ret)
502                                 desc->wake_depth = 1;
503                         else
504                                 irqd_clear(&desc->irq_data, IRQD_WAKEUP_STATE);
505                 }
506         }
507         irq_put_desc_busunlock(desc, flags);
508         return ret;
509 }
510 EXPORT_SYMBOL(irq_set_irq_wake);
511
512 /*
513  * Internal function that tells the architecture code whether a
514  * particular irq has been exclusively allocated or is available
515  * for driver use.
516  */
517 int can_request_irq(unsigned int irq, unsigned long irqflags)
518 {
519         unsigned long flags;
520         struct irq_desc *desc = irq_get_desc_lock(irq, &flags);
521         int canrequest = 0;
522
523         if (!desc)
524                 return 0;
525
526         if (irq_settings_can_request(desc)) {
527                 if (desc->action)
528                         if (irqflags & desc->action->flags & IRQF_SHARED)
529                                 canrequest =1;
530         }
531         irq_put_desc_unlock(desc, flags);
532         return canrequest;
533 }
534
535 int __irq_set_trigger(struct irq_desc *desc, unsigned int irq,
536                       unsigned long flags)
537 {
538         struct irq_chip *chip = desc->irq_data.chip;
539         int ret, unmask = 0;
540
541         if (!chip || !chip->irq_set_type) {
542                 /*
543                  * IRQF_TRIGGER_* but the PIC does not support multiple
544                  * flow-types?
545                  */
546                 pr_debug("No set_type function for IRQ %d (%s)\n", irq,
547                                 chip ? (chip->name ? : "unknown") : "unknown");
548                 return 0;
549         }
550
551         flags &= IRQ_TYPE_SENSE_MASK;
552
553         if (chip->flags & IRQCHIP_SET_TYPE_MASKED) {
554                 if (!(desc->istate & IRQS_MASKED))
555                         mask_irq(desc);
556                 if (!(desc->istate & IRQS_DISABLED))
557                         unmask = 1;
558         }
559
560         /* caller masked out all except trigger mode flags */
561         ret = chip->irq_set_type(&desc->irq_data, flags);
562
563         switch (ret) {
564         case IRQ_SET_MASK_OK:
565                 irqd_clear(&desc->irq_data, IRQD_TRIGGER_MASK);
566                 irqd_set(&desc->irq_data, flags);
567
568         case IRQ_SET_MASK_OK_NOCOPY:
569                 flags = irqd_get_trigger_type(&desc->irq_data);
570                 irq_settings_set_trigger_mask(desc, flags);
571                 irqd_clear(&desc->irq_data, IRQD_LEVEL);
572                 irq_settings_clr_level(desc);
573                 if (flags & IRQ_TYPE_LEVEL_MASK) {
574                         irq_settings_set_level(desc);
575                         irqd_set(&desc->irq_data, IRQD_LEVEL);
576                 }
577
578                 if (chip != desc->irq_data.chip)
579                         irq_chip_set_defaults(desc->irq_data.chip);
580                 ret = 0;
581                 break;
582         default:
583                 pr_err("setting trigger mode %lu for irq %u failed (%pF)\n",
584                        flags, irq, chip->irq_set_type);
585         }
586         if (unmask)
587                 unmask_irq(desc);
588         return ret;
589 }
590
591 /*
592  * Default primary interrupt handler for threaded interrupts. Is
593  * assigned as primary handler when request_threaded_irq is called
594  * with handler == NULL. Useful for oneshot interrupts.
595  */
596 static irqreturn_t irq_default_primary_handler(int irq, void *dev_id)
597 {
598         return IRQ_WAKE_THREAD;
599 }
600
601 /*
602  * Primary handler for nested threaded interrupts. Should never be
603  * called.
604  */
605 static irqreturn_t irq_nested_primary_handler(int irq, void *dev_id)
606 {
607         WARN(1, "Primary handler called for nested irq %d\n", irq);
608         return IRQ_NONE;
609 }
610
611 static int irq_wait_for_interrupt(struct irqaction *action)
612 {
613         while (!kthread_should_stop()) {
614                 set_current_state(TASK_INTERRUPTIBLE);
615
616                 if (test_and_clear_bit(IRQTF_RUNTHREAD,
617                                        &action->thread_flags)) {
618                         __set_current_state(TASK_RUNNING);
619                         return 0;
620                 }
621                 schedule();
622         }
623         return -1;
624 }
625
626 /*
627  * Oneshot interrupts keep the irq line masked until the threaded
628  * handler finished. unmask if the interrupt has not been disabled and
629  * is marked MASKED.
630  */
631 static void irq_finalize_oneshot(struct irq_desc *desc,
632                                  struct irqaction *action, bool force)
633 {
634         if (!(desc->istate & IRQS_ONESHOT))
635                 return;
636 again:
637         chip_bus_lock(desc);
638         raw_spin_lock_irq(&desc->lock);
639
640         /*
641          * Implausible though it may be we need to protect us against
642          * the following scenario:
643          *
644          * The thread is faster done than the hard interrupt handler
645          * on the other CPU. If we unmask the irq line then the
646          * interrupt can come in again and masks the line, leaves due
647          * to IRQS_INPROGRESS and the irq line is masked forever.
648          *
649          * This also serializes the state of shared oneshot handlers
650          * versus "desc->threads_onehsot |= action->thread_mask;" in
651          * irq_wake_thread(). See the comment there which explains the
652          * serialization.
653          */
654         if (unlikely(desc->istate & IRQS_INPROGRESS)) {
655                 raw_spin_unlock_irq(&desc->lock);
656                 chip_bus_sync_unlock(desc);
657                 cpu_relax();
658                 goto again;
659         }
660
661         /*
662          * Now check again, whether the thread should run. Otherwise
663          * we would clear the threads_oneshot bit of this thread which
664          * was just set.
665          */
666         if (!force && test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
667                 goto out_unlock;
668
669         desc->threads_oneshot &= ~action->thread_mask;
670
671         if (!desc->threads_oneshot && !(desc->istate & IRQS_DISABLED) &&
672             (desc->istate & IRQS_MASKED)) {
673                 irq_compat_clr_masked(desc);
674                 desc->istate &= ~IRQS_MASKED;
675                 desc->irq_data.chip->irq_unmask(&desc->irq_data);
676         }
677 out_unlock:
678         raw_spin_unlock_irq(&desc->lock);
679         chip_bus_sync_unlock(desc);
680 }
681
682 #ifdef CONFIG_SMP
683 /*
684  * Check whether we need to chasnge the affinity of the interrupt thread.
685  */
686 static void
687 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action)
688 {
689         cpumask_var_t mask;
690
691         if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags))
692                 return;
693
694         /*
695          * In case we are out of memory we set IRQTF_AFFINITY again and
696          * try again next time
697          */
698         if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
699                 set_bit(IRQTF_AFFINITY, &action->thread_flags);
700                 return;
701         }
702
703         raw_spin_lock_irq(&desc->lock);
704         cpumask_copy(mask, desc->irq_data.affinity);
705         raw_spin_unlock_irq(&desc->lock);
706
707         set_cpus_allowed_ptr(current, mask);
708         free_cpumask_var(mask);
709 }
710 #else
711 static inline void
712 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action) { }
713 #endif
714
715 /*
716  * Interrupts which are not explicitely requested as threaded
717  * interrupts rely on the implicit bh/preempt disable of the hard irq
718  * context. So we need to disable bh here to avoid deadlocks and other
719  * side effects.
720  */
721 static void
722 irq_forced_thread_fn(struct irq_desc *desc, struct irqaction *action)
723 {
724         local_bh_disable();
725         action->thread_fn(action->irq, action->dev_id);
726         irq_finalize_oneshot(desc, action, false);
727         local_bh_enable();
728 }
729
730 /*
731  * Interrupts explicitely requested as threaded interupts want to be
732  * preemtible - many of them need to sleep and wait for slow busses to
733  * complete.
734  */
735 static void irq_thread_fn(struct irq_desc *desc, struct irqaction *action)
736 {
737         action->thread_fn(action->irq, action->dev_id);
738         irq_finalize_oneshot(desc, action, false);
739 }
740
741 /*
742  * Interrupt handler thread
743  */
744 static int irq_thread(void *data)
745 {
746         static const struct sched_param param = {
747                 .sched_priority = MAX_USER_RT_PRIO/2,
748         };
749         struct irqaction *action = data;
750         struct irq_desc *desc = irq_to_desc(action->irq);
751         void (*handler_fn)(struct irq_desc *desc, struct irqaction *action);
752         int wake;
753
754         if (force_irqthreads & test_bit(IRQTF_FORCED_THREAD,
755                                         &action->thread_flags))
756                 handler_fn = irq_forced_thread_fn;
757         else
758                 handler_fn = irq_thread_fn;
759
760         sched_setscheduler(current, SCHED_FIFO, &param);
761         current->irqaction = action;
762
763         while (!irq_wait_for_interrupt(action)) {
764
765                 irq_thread_check_affinity(desc, action);
766
767                 atomic_inc(&desc->threads_active);
768
769                 raw_spin_lock_irq(&desc->lock);
770                 if (unlikely(desc->istate & IRQS_DISABLED)) {
771                         /*
772                          * CHECKME: We might need a dedicated
773                          * IRQ_THREAD_PENDING flag here, which
774                          * retriggers the thread in check_irq_resend()
775                          * but AFAICT IRQS_PENDING should be fine as it
776                          * retriggers the interrupt itself --- tglx
777                          */
778                         irq_compat_set_pending(desc);
779                         desc->istate |= IRQS_PENDING;
780                         raw_spin_unlock_irq(&desc->lock);
781                 } else {
782                         raw_spin_unlock_irq(&desc->lock);
783                         handler_fn(desc, action);
784                 }
785
786                 wake = atomic_dec_and_test(&desc->threads_active);
787
788                 if (wake && waitqueue_active(&desc->wait_for_threads))
789                         wake_up(&desc->wait_for_threads);
790         }
791
792         /* Prevent a stale desc->threads_oneshot */
793         irq_finalize_oneshot(desc, action, true);
794
795         /*
796          * Clear irqaction. Otherwise exit_irq_thread() would make
797          * fuzz about an active irq thread going into nirvana.
798          */
799         current->irqaction = NULL;
800         return 0;
801 }
802
803 /*
804  * Called from do_exit()
805  */
806 void exit_irq_thread(void)
807 {
808         struct task_struct *tsk = current;
809         struct irq_desc *desc;
810
811         if (!tsk->irqaction)
812                 return;
813
814         printk(KERN_ERR
815                "exiting task \"%s\" (%d) is an active IRQ thread (irq %d)\n",
816                tsk->comm ? tsk->comm : "", tsk->pid, tsk->irqaction->irq);
817
818         desc = irq_to_desc(tsk->irqaction->irq);
819
820         /*
821          * Prevent a stale desc->threads_oneshot. Must be called
822          * before setting the IRQTF_DIED flag.
823          */
824         irq_finalize_oneshot(desc, tsk->irqaction, true);
825
826         /*
827          * Set the THREAD DIED flag to prevent further wakeups of the
828          * soon to be gone threaded handler.
829          */
830         set_bit(IRQTF_DIED, &tsk->irqaction->flags);
831 }
832
833 static void irq_setup_forced_threading(struct irqaction *new)
834 {
835         if (!force_irqthreads)
836                 return;
837         if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))
838                 return;
839
840         new->flags |= IRQF_ONESHOT;
841
842         if (!new->thread_fn) {
843                 set_bit(IRQTF_FORCED_THREAD, &new->thread_flags);
844                 new->thread_fn = new->handler;
845                 new->handler = irq_default_primary_handler;
846         }
847 }
848
849 /*
850  * Internal function to register an irqaction - typically used to
851  * allocate special interrupts that are part of the architecture.
852  */
853 static int
854 __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
855 {
856         struct irqaction *old, **old_ptr;
857         const char *old_name = NULL;
858         unsigned long flags, thread_mask = 0;
859         int ret, nested, shared = 0;
860         cpumask_var_t mask;
861
862         if (!desc)
863                 return -EINVAL;
864
865         if (desc->irq_data.chip == &no_irq_chip)
866                 return -ENOSYS;
867         /*
868          * Some drivers like serial.c use request_irq() heavily,
869          * so we have to be careful not to interfere with a
870          * running system.
871          */
872         if (new->flags & IRQF_SAMPLE_RANDOM) {
873                 /*
874                  * This function might sleep, we want to call it first,
875                  * outside of the atomic block.
876                  * Yes, this might clear the entropy pool if the wrong
877                  * driver is attempted to be loaded, without actually
878                  * installing a new handler, but is this really a problem,
879                  * only the sysadmin is able to do this.
880                  */
881                 rand_initialize_irq(irq);
882         }
883
884         /*
885          * Check whether the interrupt nests into another interrupt
886          * thread.
887          */
888         nested = irq_settings_is_nested_thread(desc);
889         if (nested) {
890                 if (!new->thread_fn)
891                         return -EINVAL;
892                 /*
893                  * Replace the primary handler which was provided from
894                  * the driver for non nested interrupt handling by the
895                  * dummy function which warns when called.
896                  */
897                 new->handler = irq_nested_primary_handler;
898         } else {
899                 irq_setup_forced_threading(new);
900         }
901
902         /*
903          * Create a handler thread when a thread function is supplied
904          * and the interrupt does not nest into another interrupt
905          * thread.
906          */
907         if (new->thread_fn && !nested) {
908                 struct task_struct *t;
909
910                 t = kthread_create(irq_thread, new, "irq/%d-%s", irq,
911                                    new->name);
912                 if (IS_ERR(t))
913                         return PTR_ERR(t);
914                 /*
915                  * We keep the reference to the task struct even if
916                  * the thread dies to avoid that the interrupt code
917                  * references an already freed task_struct.
918                  */
919                 get_task_struct(t);
920                 new->thread = t;
921         }
922
923         if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
924                 ret = -ENOMEM;
925                 goto out_thread;
926         }
927
928         /*
929          * The following block of code has to be executed atomically
930          */
931         raw_spin_lock_irqsave(&desc->lock, flags);
932         old_ptr = &desc->action;
933         old = *old_ptr;
934         if (old) {
935                 /*
936                  * Can't share interrupts unless both agree to and are
937                  * the same type (level, edge, polarity). So both flag
938                  * fields must have IRQF_SHARED set and the bits which
939                  * set the trigger type must match. Also all must
940                  * agree on ONESHOT.
941                  */
942                 if (!((old->flags & new->flags) & IRQF_SHARED) ||
943                     ((old->flags ^ new->flags) & IRQF_TRIGGER_MASK) ||
944                     ((old->flags ^ new->flags) & IRQF_ONESHOT)) {
945                         old_name = old->name;
946                         goto mismatch;
947                 }
948
949                 /* All handlers must agree on per-cpuness */
950                 if ((old->flags & IRQF_PERCPU) !=
951                     (new->flags & IRQF_PERCPU))
952                         goto mismatch;
953
954                 /* add new interrupt at end of irq queue */
955                 do {
956                         thread_mask |= old->thread_mask;
957                         old_ptr = &old->next;
958                         old = *old_ptr;
959                 } while (old);
960                 shared = 1;
961         }
962
963         /*
964          * Setup the thread mask for this irqaction. Unlikely to have
965          * 32 resp 64 irqs sharing one line, but who knows.
966          */
967         if (new->flags & IRQF_ONESHOT && thread_mask == ~0UL) {
968                 ret = -EBUSY;
969                 goto out_mask;
970         }
971         new->thread_mask = 1 << ffz(thread_mask);
972
973         if (!shared) {
974                 irq_chip_set_defaults(desc->irq_data.chip);
975
976                 init_waitqueue_head(&desc->wait_for_threads);
977
978                 /* Setup the type (level, edge polarity) if configured: */
979                 if (new->flags & IRQF_TRIGGER_MASK) {
980                         ret = __irq_set_trigger(desc, irq,
981                                         new->flags & IRQF_TRIGGER_MASK);
982
983                         if (ret)
984                                 goto out_mask;
985                 }
986
987                 desc->istate &= ~(IRQS_AUTODETECT | IRQS_SPURIOUS_DISABLED | \
988                                   IRQS_INPROGRESS | IRQS_ONESHOT | \
989                                   IRQS_WAITING);
990
991                 if (new->flags & IRQF_PERCPU) {
992                         irqd_set(&desc->irq_data, IRQD_PER_CPU);
993                         irq_settings_set_per_cpu(desc);
994                 }
995
996                 if (new->flags & IRQF_ONESHOT)
997                         desc->istate |= IRQS_ONESHOT;
998
999                 if (irq_settings_can_autoenable(desc))
1000                         irq_startup(desc);
1001                 else
1002                         /* Undo nested disables: */
1003                         desc->depth = 1;
1004
1005                 /* Exclude IRQ from balancing if requested */
1006                 if (new->flags & IRQF_NOBALANCING) {
1007                         irq_settings_set_no_balancing(desc);
1008                         irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
1009                 }
1010
1011                 /* Set default affinity mask once everything is setup */
1012                 setup_affinity(irq, desc, mask);
1013
1014         } else if (new->flags & IRQF_TRIGGER_MASK) {
1015                 unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK;
1016                 unsigned int omsk = irq_settings_get_trigger_mask(desc);
1017
1018                 if (nmsk != omsk)
1019                         /* hope the handler works with current  trigger mode */
1020                         pr_warning("IRQ %d uses trigger mode %u; requested %u\n",
1021                                    irq, nmsk, omsk);
1022         }
1023
1024         new->irq = irq;
1025         *old_ptr = new;
1026
1027         /* Reset broken irq detection when installing new handler */
1028         desc->irq_count = 0;
1029         desc->irqs_unhandled = 0;
1030
1031         /*
1032          * Check whether we disabled the irq via the spurious handler
1033          * before. Reenable it and give it another chance.
1034          */
1035         if (shared && (desc->istate & IRQS_SPURIOUS_DISABLED)) {
1036                 desc->istate &= ~IRQS_SPURIOUS_DISABLED;
1037                 __enable_irq(desc, irq, false);
1038         }
1039
1040         raw_spin_unlock_irqrestore(&desc->lock, flags);
1041
1042         /*
1043          * Strictly no need to wake it up, but hung_task complains
1044          * when no hard interrupt wakes the thread up.
1045          */
1046         if (new->thread)
1047                 wake_up_process(new->thread);
1048
1049         register_irq_proc(irq, desc);
1050         new->dir = NULL;
1051         register_handler_proc(irq, new);
1052
1053         return 0;
1054
1055 mismatch:
1056 #ifdef CONFIG_DEBUG_SHIRQ
1057         if (!(new->flags & IRQF_PROBE_SHARED)) {
1058                 printk(KERN_ERR "IRQ handler type mismatch for IRQ %d\n", irq);
1059                 if (old_name)
1060                         printk(KERN_ERR "current handler: %s\n", old_name);
1061                 dump_stack();
1062         }
1063 #endif
1064         ret = -EBUSY;
1065
1066 out_mask:
1067         raw_spin_unlock_irqrestore(&desc->lock, flags);
1068         free_cpumask_var(mask);
1069
1070 out_thread:
1071         if (new->thread) {
1072                 struct task_struct *t = new->thread;
1073
1074                 new->thread = NULL;
1075                 if (likely(!test_bit(IRQTF_DIED, &new->thread_flags)))
1076                         kthread_stop(t);
1077                 put_task_struct(t);
1078         }
1079         return ret;
1080 }
1081
1082 /**
1083  *      setup_irq - setup an interrupt
1084  *      @irq: Interrupt line to setup
1085  *      @act: irqaction for the interrupt
1086  *
1087  * Used to statically setup interrupts in the early boot process.
1088  */
1089 int setup_irq(unsigned int irq, struct irqaction *act)
1090 {
1091         int retval;
1092         struct irq_desc *desc = irq_to_desc(irq);
1093
1094         chip_bus_lock(desc);
1095         retval = __setup_irq(irq, desc, act);
1096         chip_bus_sync_unlock(desc);
1097
1098         return retval;
1099 }
1100 EXPORT_SYMBOL_GPL(setup_irq);
1101
1102  /*
1103  * Internal function to unregister an irqaction - used to free
1104  * regular and special interrupts that are part of the architecture.
1105  */
1106 static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
1107 {
1108         struct irq_desc *desc = irq_to_desc(irq);
1109         struct irqaction *action, **action_ptr;
1110         unsigned long flags;
1111
1112         WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
1113
1114         if (!desc)
1115                 return NULL;
1116
1117         raw_spin_lock_irqsave(&desc->lock, flags);
1118
1119         /*
1120          * There can be multiple actions per IRQ descriptor, find the right
1121          * one based on the dev_id:
1122          */
1123         action_ptr = &desc->action;
1124         for (;;) {
1125                 action = *action_ptr;
1126
1127                 if (!action) {
1128                         WARN(1, "Trying to free already-free IRQ %d\n", irq);
1129                         raw_spin_unlock_irqrestore(&desc->lock, flags);
1130
1131                         return NULL;
1132                 }
1133
1134                 if (action->dev_id == dev_id)
1135                         break;
1136                 action_ptr = &action->next;
1137         }
1138
1139         /* Found it - now remove it from the list of entries: */
1140         *action_ptr = action->next;
1141
1142         /* Currently used only by UML, might disappear one day: */
1143 #ifdef CONFIG_IRQ_RELEASE_METHOD
1144         if (desc->irq_data.chip->release)
1145                 desc->irq_data.chip->release(irq, dev_id);
1146 #endif
1147
1148         /* If this was the last handler, shut down the IRQ line: */
1149         if (!desc->action)
1150                 irq_shutdown(desc);
1151
1152 #ifdef CONFIG_SMP
1153         /* make sure affinity_hint is cleaned up */
1154         if (WARN_ON_ONCE(desc->affinity_hint))
1155                 desc->affinity_hint = NULL;
1156 #endif
1157
1158         raw_spin_unlock_irqrestore(&desc->lock, flags);
1159
1160         unregister_handler_proc(irq, action);
1161
1162         /* Make sure it's not being used on another CPU: */
1163         synchronize_irq(irq);
1164
1165 #ifdef CONFIG_DEBUG_SHIRQ
1166         /*
1167          * It's a shared IRQ -- the driver ought to be prepared for an IRQ
1168          * event to happen even now it's being freed, so let's make sure that
1169          * is so by doing an extra call to the handler ....
1170          *
1171          * ( We do this after actually deregistering it, to make sure that a
1172          *   'real' IRQ doesn't run in * parallel with our fake. )
1173          */
1174         if (action->flags & IRQF_SHARED) {
1175                 local_irq_save(flags);
1176                 action->handler(irq, dev_id);
1177                 local_irq_restore(flags);
1178         }
1179 #endif
1180
1181         if (action->thread) {
1182                 if (!test_bit(IRQTF_DIED, &action->thread_flags))
1183                         kthread_stop(action->thread);
1184                 put_task_struct(action->thread);
1185         }
1186
1187         return action;
1188 }
1189
1190 /**
1191  *      remove_irq - free an interrupt
1192  *      @irq: Interrupt line to free
1193  *      @act: irqaction for the interrupt
1194  *
1195  * Used to remove interrupts statically setup by the early boot process.
1196  */
1197 void remove_irq(unsigned int irq, struct irqaction *act)
1198 {
1199         __free_irq(irq, act->dev_id);
1200 }
1201 EXPORT_SYMBOL_GPL(remove_irq);
1202
1203 /**
1204  *      free_irq - free an interrupt allocated with request_irq
1205  *      @irq: Interrupt line to free
1206  *      @dev_id: Device identity to free
1207  *
1208  *      Remove an interrupt handler. The handler is removed and if the
1209  *      interrupt line is no longer in use by any driver it is disabled.
1210  *      On a shared IRQ the caller must ensure the interrupt is disabled
1211  *      on the card it drives before calling this function. The function
1212  *      does not return until any executing interrupts for this IRQ
1213  *      have completed.
1214  *
1215  *      This function must not be called from interrupt context.
1216  */
1217 void free_irq(unsigned int irq, void *dev_id)
1218 {
1219         struct irq_desc *desc = irq_to_desc(irq);
1220
1221         if (!desc)
1222                 return;
1223
1224 #ifdef CONFIG_SMP
1225         if (WARN_ON(desc->affinity_notify))
1226                 desc->affinity_notify = NULL;
1227 #endif
1228
1229         chip_bus_lock(desc);
1230         kfree(__free_irq(irq, dev_id));
1231         chip_bus_sync_unlock(desc);
1232 }
1233 EXPORT_SYMBOL(free_irq);
1234
1235 /**
1236  *      request_threaded_irq - allocate an interrupt line
1237  *      @irq: Interrupt line to allocate
1238  *      @handler: Function to be called when the IRQ occurs.
1239  *                Primary handler for threaded interrupts
1240  *                If NULL and thread_fn != NULL the default
1241  *                primary handler is installed
1242  *      @thread_fn: Function called from the irq handler thread
1243  *                  If NULL, no irq thread is created
1244  *      @irqflags: Interrupt type flags
1245  *      @devname: An ascii name for the claiming device
1246  *      @dev_id: A cookie passed back to the handler function
1247  *
1248  *      This call allocates interrupt resources and enables the
1249  *      interrupt line and IRQ handling. From the point this
1250  *      call is made your handler function may be invoked. Since
1251  *      your handler function must clear any interrupt the board
1252  *      raises, you must take care both to initialise your hardware
1253  *      and to set up the interrupt handler in the right order.
1254  *
1255  *      If you want to set up a threaded irq handler for your device
1256  *      then you need to supply @handler and @thread_fn. @handler ist
1257  *      still called in hard interrupt context and has to check
1258  *      whether the interrupt originates from the device. If yes it
1259  *      needs to disable the interrupt on the device and return
1260  *      IRQ_WAKE_THREAD which will wake up the handler thread and run
1261  *      @thread_fn. This split handler design is necessary to support
1262  *      shared interrupts.
1263  *
1264  *      Dev_id must be globally unique. Normally the address of the
1265  *      device data structure is used as the cookie. Since the handler
1266  *      receives this value it makes sense to use it.
1267  *
1268  *      If your interrupt is shared you must pass a non NULL dev_id
1269  *      as this is required when freeing the interrupt.
1270  *
1271  *      Flags:
1272  *
1273  *      IRQF_SHARED             Interrupt is shared
1274  *      IRQF_SAMPLE_RANDOM      The interrupt can be used for entropy
1275  *      IRQF_TRIGGER_*          Specify active edge(s) or level
1276  *
1277  */
1278 int request_threaded_irq(unsigned int irq, irq_handler_t handler,
1279                          irq_handler_t thread_fn, unsigned long irqflags,
1280                          const char *devname, void *dev_id)
1281 {
1282         struct irqaction *action;
1283         struct irq_desc *desc;
1284         int retval;
1285
1286         /*
1287          * Sanity-check: shared interrupts must pass in a real dev-ID,
1288          * otherwise we'll have trouble later trying to figure out
1289          * which interrupt is which (messes up the interrupt freeing
1290          * logic etc).
1291          */
1292         if ((irqflags & IRQF_SHARED) && !dev_id)
1293                 return -EINVAL;
1294
1295         desc = irq_to_desc(irq);
1296         if (!desc)
1297                 return -EINVAL;
1298
1299         if (!irq_settings_can_request(desc))
1300                 return -EINVAL;
1301
1302         if (!handler) {
1303                 if (!thread_fn)
1304                         return -EINVAL;
1305                 handler = irq_default_primary_handler;
1306         }
1307
1308         action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
1309         if (!action)
1310                 return -ENOMEM;
1311
1312         action->handler = handler;
1313         action->thread_fn = thread_fn;
1314         action->flags = irqflags;
1315         action->name = devname;
1316         action->dev_id = dev_id;
1317
1318         chip_bus_lock(desc);
1319         retval = __setup_irq(irq, desc, action);
1320         chip_bus_sync_unlock(desc);
1321
1322         if (retval)
1323                 kfree(action);
1324
1325 #ifdef CONFIG_DEBUG_SHIRQ_FIXME
1326         if (!retval && (irqflags & IRQF_SHARED)) {
1327                 /*
1328                  * It's a shared IRQ -- the driver ought to be prepared for it
1329                  * to happen immediately, so let's make sure....
1330                  * We disable the irq to make sure that a 'real' IRQ doesn't
1331                  * run in parallel with our fake.
1332                  */
1333                 unsigned long flags;
1334
1335                 disable_irq(irq);
1336                 local_irq_save(flags);
1337
1338                 handler(irq, dev_id);
1339
1340                 local_irq_restore(flags);
1341                 enable_irq(irq);
1342         }
1343 #endif
1344         return retval;
1345 }
1346 EXPORT_SYMBOL(request_threaded_irq);
1347
1348 /**
1349  *      request_any_context_irq - allocate an interrupt line
1350  *      @irq: Interrupt line to allocate
1351  *      @handler: Function to be called when the IRQ occurs.
1352  *                Threaded handler for threaded interrupts.
1353  *      @flags: Interrupt type flags
1354  *      @name: An ascii name for the claiming device
1355  *      @dev_id: A cookie passed back to the handler function
1356  *
1357  *      This call allocates interrupt resources and enables the
1358  *      interrupt line and IRQ handling. It selects either a
1359  *      hardirq or threaded handling method depending on the
1360  *      context.
1361  *
1362  *      On failure, it returns a negative value. On success,
1363  *      it returns either IRQC_IS_HARDIRQ or IRQC_IS_NESTED.
1364  */
1365 int request_any_context_irq(unsigned int irq, irq_handler_t handler,
1366                             unsigned long flags, const char *name, void *dev_id)
1367 {
1368         struct irq_desc *desc = irq_to_desc(irq);
1369         int ret;
1370
1371         if (!desc)
1372                 return -EINVAL;
1373
1374         if (irq_settings_is_nested_thread(desc)) {
1375                 ret = request_threaded_irq(irq, NULL, handler,
1376                                            flags, name, dev_id);
1377                 return !ret ? IRQC_IS_NESTED : ret;
1378         }
1379
1380         ret = request_irq(irq, handler, flags, name, dev_id);
1381         return !ret ? IRQC_IS_HARDIRQ : ret;
1382 }
1383 EXPORT_SYMBOL_GPL(request_any_context_irq);