include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit...
[linux-3.10.git] / arch / blackfin / kernel / ipipe.c
1 /* -*- linux-c -*-
2  * linux/arch/blackfin/kernel/ipipe.c
3  *
4  * Copyright (C) 2005-2007 Philippe Gerum.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
9  * USA; either version 2 of the License, or (at your option) any later
10  * version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
20  *
21  * Architecture-dependent I-pipe support for the Blackfin.
22  */
23
24 #include <linux/kernel.h>
25 #include <linux/sched.h>
26 #include <linux/module.h>
27 #include <linux/interrupt.h>
28 #include <linux/percpu.h>
29 #include <linux/bitops.h>
30 #include <linux/errno.h>
31 #include <linux/kthread.h>
32 #include <linux/unistd.h>
33 #include <linux/io.h>
34 #include <asm/system.h>
35 #include <asm/atomic.h>
36
37 DEFINE_PER_CPU(struct pt_regs, __ipipe_tick_regs);
38
39 asmlinkage void asm_do_IRQ(unsigned int irq, struct pt_regs *regs);
40
41 static void __ipipe_no_irqtail(void);
42
43 unsigned long __ipipe_irq_tail_hook = (unsigned long)&__ipipe_no_irqtail;
44 EXPORT_SYMBOL(__ipipe_irq_tail_hook);
45
46 unsigned long __ipipe_core_clock;
47 EXPORT_SYMBOL(__ipipe_core_clock);
48
49 unsigned long __ipipe_freq_scale;
50 EXPORT_SYMBOL(__ipipe_freq_scale);
51
52 atomic_t __ipipe_irq_lvdepth[IVG15 + 1];
53
54 unsigned long __ipipe_irq_lvmask = bfin_no_irqs;
55 EXPORT_SYMBOL(__ipipe_irq_lvmask);
56
57 static void __ipipe_ack_irq(unsigned irq, struct irq_desc *desc)
58 {
59         desc->ipipe_ack(irq, desc);
60 }
61
62 /*
63  * __ipipe_enable_pipeline() -- We are running on the boot CPU, hw
64  * interrupts are off, and secondary CPUs are still lost in space.
65  */
66 void __ipipe_enable_pipeline(void)
67 {
68         unsigned irq;
69
70         __ipipe_core_clock = get_cclk(); /* Fetch this once. */
71         __ipipe_freq_scale = 1000000000UL / __ipipe_core_clock;
72
73         for (irq = 0; irq < NR_IRQS; ++irq)
74                 ipipe_virtualize_irq(ipipe_root_domain,
75                                      irq,
76                                      (ipipe_irq_handler_t)&asm_do_IRQ,
77                                      NULL,
78                                      &__ipipe_ack_irq,
79                                      IPIPE_HANDLE_MASK | IPIPE_PASS_MASK);
80 }
81
82 /*
83  * __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic
84  * interrupt protection log is maintained here for each domain. Hw
85  * interrupts are masked on entry.
86  */
87 void __ipipe_handle_irq(unsigned irq, struct pt_regs *regs)
88 {
89         struct ipipe_percpu_domain_data *p = ipipe_root_cpudom_ptr();
90         struct ipipe_domain *this_domain, *next_domain;
91         struct list_head *head, *pos;
92         struct ipipe_irqdesc *idesc;
93         int m_ack, s = -1;
94
95         /*
96          * Software-triggered IRQs do not need any ack.  The contents
97          * of the register frame should only be used when processing
98          * the timer interrupt, but not for handling any other
99          * interrupt.
100          */
101         m_ack = (regs == NULL || irq == IRQ_SYSTMR || irq == IRQ_CORETMR);
102         this_domain = __ipipe_current_domain;
103         idesc = &this_domain->irqs[irq];
104
105         if (unlikely(test_bit(IPIPE_STICKY_FLAG, &idesc->control)))
106                 head = &this_domain->p_link;
107         else {
108                 head = __ipipe_pipeline.next;
109                 next_domain = list_entry(head, struct ipipe_domain, p_link);
110                 idesc = &next_domain->irqs[irq];
111                 if (likely(test_bit(IPIPE_WIRED_FLAG, &idesc->control))) {
112                         if (!m_ack && idesc->acknowledge != NULL)
113                                 idesc->acknowledge(irq, irq_to_desc(irq));
114                         if (test_bit(IPIPE_SYNCDEFER_FLAG, &p->status))
115                                 s = __test_and_set_bit(IPIPE_STALL_FLAG,
116                                                        &p->status);
117                         __ipipe_dispatch_wired(next_domain, irq);
118                         goto out;
119                 }
120         }
121
122         /* Ack the interrupt. */
123
124         pos = head;
125         while (pos != &__ipipe_pipeline) {
126                 next_domain = list_entry(pos, struct ipipe_domain, p_link);
127                 idesc = &next_domain->irqs[irq];
128                 if (test_bit(IPIPE_HANDLE_FLAG, &idesc->control)) {
129                         __ipipe_set_irq_pending(next_domain, irq);
130                         if (!m_ack && idesc->acknowledge != NULL) {
131                                 idesc->acknowledge(irq, irq_to_desc(irq));
132                                 m_ack = 1;
133                         }
134                 }
135                 if (!test_bit(IPIPE_PASS_FLAG, &idesc->control))
136                         break;
137                 pos = next_domain->p_link.next;
138         }
139
140         /*
141          * Now walk the pipeline, yielding control to the highest
142          * priority domain that has pending interrupt(s) or
143          * immediately to the current domain if the interrupt has been
144          * marked as 'sticky'. This search does not go beyond the
145          * current domain in the pipeline. We also enforce the
146          * additional root stage lock (blackfin-specific).
147          */
148         if (test_bit(IPIPE_SYNCDEFER_FLAG, &p->status))
149                 s = __test_and_set_bit(IPIPE_STALL_FLAG, &p->status);
150
151         /*
152          * If the interrupt preempted the head domain, then do not
153          * even try to walk the pipeline, unless an interrupt is
154          * pending for it.
155          */
156         if (test_bit(IPIPE_AHEAD_FLAG, &this_domain->flags) &&
157             ipipe_head_cpudom_var(irqpend_himask) == 0)
158                 goto out;
159
160         __ipipe_walk_pipeline(head);
161 out:
162         if (!s)
163                 __clear_bit(IPIPE_STALL_FLAG, &p->status);
164 }
165
166 void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
167 {
168         struct irq_desc *desc = irq_to_desc(irq);
169         int prio = __ipipe_get_irq_priority(irq);
170
171         desc->depth = 0;
172         if (ipd != &ipipe_root &&
173             atomic_inc_return(&__ipipe_irq_lvdepth[prio]) == 1)
174                 __set_bit(prio, &__ipipe_irq_lvmask);
175 }
176 EXPORT_SYMBOL(__ipipe_enable_irqdesc);
177
178 void __ipipe_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
179 {
180         int prio = __ipipe_get_irq_priority(irq);
181
182         if (ipd != &ipipe_root &&
183             atomic_dec_and_test(&__ipipe_irq_lvdepth[prio]))
184                 __clear_bit(prio, &__ipipe_irq_lvmask);
185 }
186 EXPORT_SYMBOL(__ipipe_disable_irqdesc);
187
188 int __ipipe_syscall_root(struct pt_regs *regs)
189 {
190         struct ipipe_percpu_domain_data *p;
191         unsigned long flags;
192         int ret;
193
194         /*
195          * We need to run the IRQ tail hook whenever we don't
196          * propagate a syscall to higher domains, because we know that
197          * important operations might be pending there (e.g. Xenomai
198          * deferred rescheduling).
199          */
200
201         if (regs->orig_p0 < NR_syscalls) {
202                 void (*hook)(void) = (void (*)(void))__ipipe_irq_tail_hook;
203                 hook();
204                 if ((current->flags & PF_EVNOTIFY) == 0)
205                         return 0;
206         }
207
208         /*
209          * This routine either returns:
210          * 0 -- if the syscall is to be passed to Linux;
211          * >0 -- if the syscall should not be passed to Linux, and no
212          * tail work should be performed;
213          * <0 -- if the syscall should not be passed to Linux but the
214          * tail work has to be performed (for handling signals etc).
215          */
216
217         if (!__ipipe_event_monitored_p(IPIPE_EVENT_SYSCALL))
218                 return 0;
219
220         ret = __ipipe_dispatch_event(IPIPE_EVENT_SYSCALL, regs);
221
222         local_irq_save_hw(flags);
223
224         if (!__ipipe_root_domain_p) {
225                 local_irq_restore_hw(flags);
226                 return 1;
227         }
228
229         p = ipipe_root_cpudom_ptr();
230         if ((p->irqpend_himask & IPIPE_IRQMASK_VIRT) != 0)
231                 __ipipe_sync_pipeline(IPIPE_IRQMASK_VIRT);
232
233         local_irq_restore_hw(flags);
234
235         return -ret;
236 }
237
238 unsigned long ipipe_critical_enter(void (*syncfn) (void))
239 {
240         unsigned long flags;
241
242         local_irq_save_hw(flags);
243
244         return flags;
245 }
246
247 void ipipe_critical_exit(unsigned long flags)
248 {
249         local_irq_restore_hw(flags);
250 }
251
252 static void __ipipe_no_irqtail(void)
253 {
254 }
255
256 int ipipe_get_sysinfo(struct ipipe_sysinfo *info)
257 {
258         info->ncpus = num_online_cpus();
259         info->cpufreq = ipipe_cpu_freq();
260         info->archdep.tmirq = IPIPE_TIMER_IRQ;
261         info->archdep.tmfreq = info->cpufreq;
262
263         return 0;
264 }
265
266 /*
267  * ipipe_trigger_irq() -- Push the interrupt at front of the pipeline
268  * just like if it has been actually received from a hw source. Also
269  * works for virtual interrupts.
270  */
271 int ipipe_trigger_irq(unsigned irq)
272 {
273         unsigned long flags;
274
275 #ifdef CONFIG_IPIPE_DEBUG
276         if (irq >= IPIPE_NR_IRQS ||
277             (ipipe_virtual_irq_p(irq)
278              && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)))
279                 return -EINVAL;
280 #endif
281
282         local_irq_save_hw(flags);
283         __ipipe_handle_irq(irq, NULL);
284         local_irq_restore_hw(flags);
285
286         return 1;
287 }
288
289 asmlinkage void __ipipe_sync_root(void)
290 {
291         void (*irq_tail_hook)(void) = (void (*)(void))__ipipe_irq_tail_hook;
292         unsigned long flags;
293
294         BUG_ON(irqs_disabled());
295
296         local_irq_save_hw(flags);
297
298         if (irq_tail_hook)
299                 irq_tail_hook();
300
301         clear_thread_flag(TIF_IRQ_SYNC);
302
303         if (ipipe_root_cpudom_var(irqpend_himask) != 0)
304                 __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
305
306         local_irq_restore_hw(flags);
307 }
308
309 void ___ipipe_sync_pipeline(unsigned long syncmask)
310 {
311         if (__ipipe_root_domain_p &&
312             test_bit(IPIPE_SYNCDEFER_FLAG, &ipipe_root_cpudom_var(status)))
313                 return;
314
315         __ipipe_sync_stage(syncmask);
316 }
317
318 void __ipipe_disable_root_irqs_hw(void)
319 {
320         /*
321          * This code is called by the ins{bwl} routines (see
322          * arch/blackfin/lib/ins.S), which are heavily used by the
323          * network stack. It masks all interrupts but those handled by
324          * non-root domains, so that we keep decent network transfer
325          * rates for Linux without inducing pathological jitter for
326          * the real-time domain.
327          */
328         bfin_sti(__ipipe_irq_lvmask);
329         __set_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
330 }
331
332 void __ipipe_enable_root_irqs_hw(void)
333 {
334         __clear_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
335         bfin_sti(bfin_irq_flags);
336 }
337
338 /*
339  * We could use standard atomic bitops in the following root status
340  * manipulation routines, but let's prepare for SMP support in the
341  * same move, preventing CPU migration as required.
342  */
343 void __ipipe_stall_root(void)
344 {
345         unsigned long *p, flags;
346
347         local_irq_save_hw(flags);
348         p = &__ipipe_root_status;
349         __set_bit(IPIPE_STALL_FLAG, p);
350         local_irq_restore_hw(flags);
351 }
352 EXPORT_SYMBOL(__ipipe_stall_root);
353
354 unsigned long __ipipe_test_and_stall_root(void)
355 {
356         unsigned long *p, flags;
357         int x;
358
359         local_irq_save_hw(flags);
360         p = &__ipipe_root_status;
361         x = __test_and_set_bit(IPIPE_STALL_FLAG, p);
362         local_irq_restore_hw(flags);
363
364         return x;
365 }
366 EXPORT_SYMBOL(__ipipe_test_and_stall_root);
367
368 unsigned long __ipipe_test_root(void)
369 {
370         const unsigned long *p;
371         unsigned long flags;
372         int x;
373
374         local_irq_save_hw_smp(flags);
375         p = &__ipipe_root_status;
376         x = test_bit(IPIPE_STALL_FLAG, p);
377         local_irq_restore_hw_smp(flags);
378
379         return x;
380 }
381 EXPORT_SYMBOL(__ipipe_test_root);
382
383 void __ipipe_lock_root(void)
384 {
385         unsigned long *p, flags;
386
387         local_irq_save_hw(flags);
388         p = &__ipipe_root_status;
389         __set_bit(IPIPE_SYNCDEFER_FLAG, p);
390         local_irq_restore_hw(flags);
391 }
392 EXPORT_SYMBOL(__ipipe_lock_root);
393
394 void __ipipe_unlock_root(void)
395 {
396         unsigned long *p, flags;
397
398         local_irq_save_hw(flags);
399         p = &__ipipe_root_status;
400         __clear_bit(IPIPE_SYNCDEFER_FLAG, p);
401         local_irq_restore_hw(flags);
402 }
403 EXPORT_SYMBOL(__ipipe_unlock_root);