x86: optimise x86's do_page_fault (C entry point for the page fault path)
[linux-2.6.git] / arch / ia64 / sn / kernel / irq.c
1 /*
2  * Platform dependent support for SGI SN
3  *
4  * This file is subject to the terms and conditions of the GNU General Public
5  * License.  See the file "COPYING" in the main directory of this archive
6  * for more details.
7  *
8  * Copyright (c) 2000-2008 Silicon Graphics, Inc.  All Rights Reserved.
9  */
10
11 #include <linux/irq.h>
12 #include <linux/spinlock.h>
13 #include <linux/init.h>
14 #include <linux/rculist.h>
15 #include <asm/sn/addrs.h>
16 #include <asm/sn/arch.h>
17 #include <asm/sn/intr.h>
18 #include <asm/sn/pcibr_provider.h>
19 #include <asm/sn/pcibus_provider_defs.h>
20 #include <asm/sn/pcidev.h>
21 #include <asm/sn/shub_mmr.h>
22 #include <asm/sn/sn_sal.h>
23 #include <asm/sn/sn_feature_sets.h>
24
25 static void force_interrupt(int irq);
26 static void register_intr_pda(struct sn_irq_info *sn_irq_info);
27 static void unregister_intr_pda(struct sn_irq_info *sn_irq_info);
28
29 int sn_force_interrupt_flag = 1;
30 extern int sn_ioif_inited;
31 struct list_head **sn_irq_lh;
32 static DEFINE_SPINLOCK(sn_irq_info_lock); /* non-IRQ lock */
33
34 u64 sn_intr_alloc(nasid_t local_nasid, int local_widget,
35                                      struct sn_irq_info *sn_irq_info,
36                                      int req_irq, nasid_t req_nasid,
37                                      int req_slice)
38 {
39         struct ia64_sal_retval ret_stuff;
40         ret_stuff.status = 0;
41         ret_stuff.v0 = 0;
42
43         SAL_CALL_NOLOCK(ret_stuff, (u64) SN_SAL_IOIF_INTERRUPT,
44                         (u64) SAL_INTR_ALLOC, (u64) local_nasid,
45                         (u64) local_widget, __pa(sn_irq_info), (u64) req_irq,
46                         (u64) req_nasid, (u64) req_slice);
47
48         return ret_stuff.status;
49 }
50
51 void sn_intr_free(nasid_t local_nasid, int local_widget,
52                                 struct sn_irq_info *sn_irq_info)
53 {
54         struct ia64_sal_retval ret_stuff;
55         ret_stuff.status = 0;
56         ret_stuff.v0 = 0;
57
58         SAL_CALL_NOLOCK(ret_stuff, (u64) SN_SAL_IOIF_INTERRUPT,
59                         (u64) SAL_INTR_FREE, (u64) local_nasid,
60                         (u64) local_widget, (u64) sn_irq_info->irq_irq,
61                         (u64) sn_irq_info->irq_cookie, 0, 0);
62 }
63
64 u64 sn_intr_redirect(nasid_t local_nasid, int local_widget,
65                       struct sn_irq_info *sn_irq_info,
66                       nasid_t req_nasid, int req_slice)
67 {
68         struct ia64_sal_retval ret_stuff;
69         ret_stuff.status = 0;
70         ret_stuff.v0 = 0;
71
72         SAL_CALL_NOLOCK(ret_stuff, (u64) SN_SAL_IOIF_INTERRUPT,
73                         (u64) SAL_INTR_REDIRECT, (u64) local_nasid,
74                         (u64) local_widget, __pa(sn_irq_info),
75                         (u64) req_nasid, (u64) req_slice, 0);
76
77         return ret_stuff.status;
78 }
79
80 static unsigned int sn_startup_irq(unsigned int irq)
81 {
82         return 0;
83 }
84
85 static void sn_shutdown_irq(unsigned int irq)
86 {
87 }
88
89 extern void ia64_mca_register_cpev(int);
90
91 static void sn_disable_irq(unsigned int irq)
92 {
93         if (irq == local_vector_to_irq(IA64_CPE_VECTOR))
94                 ia64_mca_register_cpev(0);
95 }
96
97 static void sn_enable_irq(unsigned int irq)
98 {
99         if (irq == local_vector_to_irq(IA64_CPE_VECTOR))
100                 ia64_mca_register_cpev(irq);
101 }
102
103 static void sn_ack_irq(unsigned int irq)
104 {
105         u64 event_occurred, mask;
106
107         irq = irq & 0xff;
108         event_occurred = HUB_L((u64*)LOCAL_MMR_ADDR(SH_EVENT_OCCURRED));
109         mask = event_occurred & SH_ALL_INT_MASK;
110         HUB_S((u64*)LOCAL_MMR_ADDR(SH_EVENT_OCCURRED_ALIAS), mask);
111         __set_bit(irq, (volatile void *)pda->sn_in_service_ivecs);
112
113         move_native_irq(irq);
114 }
115
116 static void sn_end_irq(unsigned int irq)
117 {
118         int ivec;
119         u64 event_occurred;
120
121         ivec = irq & 0xff;
122         if (ivec == SGI_UART_VECTOR) {
123                 event_occurred = HUB_L((u64*)LOCAL_MMR_ADDR (SH_EVENT_OCCURRED));
124                 /* If the UART bit is set here, we may have received an
125                  * interrupt from the UART that the driver missed.  To
126                  * make sure, we IPI ourselves to force us to look again.
127                  */
128                 if (event_occurred & SH_EVENT_OCCURRED_UART_INT_MASK) {
129                         platform_send_ipi(smp_processor_id(), SGI_UART_VECTOR,
130                                           IA64_IPI_DM_INT, 0);
131                 }
132         }
133         __clear_bit(ivec, (volatile void *)pda->sn_in_service_ivecs);
134         if (sn_force_interrupt_flag)
135                 force_interrupt(irq);
136 }
137
138 static void sn_irq_info_free(struct rcu_head *head);
139
140 struct sn_irq_info *sn_retarget_vector(struct sn_irq_info *sn_irq_info,
141                                        nasid_t nasid, int slice)
142 {
143         int vector;
144         int cpuid;
145 #ifdef CONFIG_SMP
146         int cpuphys;
147 #endif
148         int64_t bridge;
149         int local_widget, status;
150         nasid_t local_nasid;
151         struct sn_irq_info *new_irq_info;
152         struct sn_pcibus_provider *pci_provider;
153
154         bridge = (u64) sn_irq_info->irq_bridge;
155         if (!bridge) {
156                 return NULL; /* irq is not a device interrupt */
157         }
158
159         local_nasid = NASID_GET(bridge);
160
161         if (local_nasid & 1)
162                 local_widget = TIO_SWIN_WIDGETNUM(bridge);
163         else
164                 local_widget = SWIN_WIDGETNUM(bridge);
165         vector = sn_irq_info->irq_irq;
166
167         /* Make use of SAL_INTR_REDIRECT if PROM supports it */
168         status = sn_intr_redirect(local_nasid, local_widget, sn_irq_info, nasid, slice);
169         if (!status) {
170                 new_irq_info = sn_irq_info;
171                 goto finish_up;
172         }
173
174         /*
175          * PROM does not support SAL_INTR_REDIRECT, or it failed.
176          * Revert to old method.
177          */
178         new_irq_info = kmalloc(sizeof(struct sn_irq_info), GFP_ATOMIC);
179         if (new_irq_info == NULL)
180                 return NULL;
181
182         memcpy(new_irq_info, sn_irq_info, sizeof(struct sn_irq_info));
183
184         /* Free the old PROM new_irq_info structure */
185         sn_intr_free(local_nasid, local_widget, new_irq_info);
186         unregister_intr_pda(new_irq_info);
187
188         /* allocate a new PROM new_irq_info struct */
189         status = sn_intr_alloc(local_nasid, local_widget,
190                                new_irq_info, vector,
191                                nasid, slice);
192
193         /* SAL call failed */
194         if (status) {
195                 kfree(new_irq_info);
196                 return NULL;
197         }
198
199         register_intr_pda(new_irq_info);
200         spin_lock(&sn_irq_info_lock);
201         list_replace_rcu(&sn_irq_info->list, &new_irq_info->list);
202         spin_unlock(&sn_irq_info_lock);
203         call_rcu(&sn_irq_info->rcu, sn_irq_info_free);
204
205
206 finish_up:
207         /* Update kernels new_irq_info with new target info */
208         cpuid = nasid_slice_to_cpuid(new_irq_info->irq_nasid,
209                                      new_irq_info->irq_slice);
210         new_irq_info->irq_cpuid = cpuid;
211
212         pci_provider = sn_pci_provider[new_irq_info->irq_bridge_type];
213
214         /*
215          * If this represents a line interrupt, target it.  If it's
216          * an msi (irq_int_bit < 0), it's already targeted.
217          */
218         if (new_irq_info->irq_int_bit >= 0 &&
219             pci_provider && pci_provider->target_interrupt)
220                 (pci_provider->target_interrupt)(new_irq_info);
221
222 #ifdef CONFIG_SMP
223         cpuphys = cpu_physical_id(cpuid);
224         set_irq_affinity_info((vector & 0xff), cpuphys, 0);
225 #endif
226
227         return new_irq_info;
228 }
229
230 static void sn_set_affinity_irq(unsigned int irq, const struct cpumask *mask)
231 {
232         struct sn_irq_info *sn_irq_info, *sn_irq_info_safe;
233         nasid_t nasid;
234         int slice;
235
236         nasid = cpuid_to_nasid(cpumask_first(mask));
237         slice = cpuid_to_slice(cpumask_first(mask));
238
239         list_for_each_entry_safe(sn_irq_info, sn_irq_info_safe,
240                                  sn_irq_lh[irq], list)
241                 (void)sn_retarget_vector(sn_irq_info, nasid, slice);
242 }
243
244 #ifdef CONFIG_SMP
245 void sn_set_err_irq_affinity(unsigned int irq)
246 {
247         /*
248          * On systems which support CPU disabling (SHub2), all error interrupts
249          * are targetted at the boot CPU.
250          */
251         if (is_shub2() && sn_prom_feature_available(PRF_CPU_DISABLE_SUPPORT))
252                 set_irq_affinity_info(irq, cpu_physical_id(0), 0);
253 }
254 #else
255 void sn_set_err_irq_affinity(unsigned int irq) { }
256 #endif
257
258 static void
259 sn_mask_irq(unsigned int irq)
260 {
261 }
262
263 static void
264 sn_unmask_irq(unsigned int irq)
265 {
266 }
267
268 struct irq_chip irq_type_sn = {
269         .name           = "SN hub",
270         .startup        = sn_startup_irq,
271         .shutdown       = sn_shutdown_irq,
272         .enable         = sn_enable_irq,
273         .disable        = sn_disable_irq,
274         .ack            = sn_ack_irq,
275         .end            = sn_end_irq,
276         .mask           = sn_mask_irq,
277         .unmask         = sn_unmask_irq,
278         .set_affinity   = sn_set_affinity_irq
279 };
280
281 ia64_vector sn_irq_to_vector(int irq)
282 {
283         if (irq >= IA64_NUM_VECTORS)
284                 return 0;
285         return (ia64_vector)irq;
286 }
287
288 unsigned int sn_local_vector_to_irq(u8 vector)
289 {
290         return (CPU_VECTOR_TO_IRQ(smp_processor_id(), vector));
291 }
292
293 void sn_irq_init(void)
294 {
295         int i;
296         irq_desc_t *base_desc = irq_desc;
297
298         ia64_first_device_vector = IA64_SN2_FIRST_DEVICE_VECTOR;
299         ia64_last_device_vector = IA64_SN2_LAST_DEVICE_VECTOR;
300
301         for (i = 0; i < NR_IRQS; i++) {
302                 if (base_desc[i].chip == &no_irq_type) {
303                         base_desc[i].chip = &irq_type_sn;
304                 }
305         }
306 }
307
308 static void register_intr_pda(struct sn_irq_info *sn_irq_info)
309 {
310         int irq = sn_irq_info->irq_irq;
311         int cpu = sn_irq_info->irq_cpuid;
312
313         if (pdacpu(cpu)->sn_last_irq < irq) {
314                 pdacpu(cpu)->sn_last_irq = irq;
315         }
316
317         if (pdacpu(cpu)->sn_first_irq == 0 || pdacpu(cpu)->sn_first_irq > irq)
318                 pdacpu(cpu)->sn_first_irq = irq;
319 }
320
321 static void unregister_intr_pda(struct sn_irq_info *sn_irq_info)
322 {
323         int irq = sn_irq_info->irq_irq;
324         int cpu = sn_irq_info->irq_cpuid;
325         struct sn_irq_info *tmp_irq_info;
326         int i, foundmatch;
327
328         rcu_read_lock();
329         if (pdacpu(cpu)->sn_last_irq == irq) {
330                 foundmatch = 0;
331                 for (i = pdacpu(cpu)->sn_last_irq - 1;
332                      i && !foundmatch; i--) {
333                         list_for_each_entry_rcu(tmp_irq_info,
334                                                 sn_irq_lh[i],
335                                                 list) {
336                                 if (tmp_irq_info->irq_cpuid == cpu) {
337                                         foundmatch = 1;
338                                         break;
339                                 }
340                         }
341                 }
342                 pdacpu(cpu)->sn_last_irq = i;
343         }
344
345         if (pdacpu(cpu)->sn_first_irq == irq) {
346                 foundmatch = 0;
347                 for (i = pdacpu(cpu)->sn_first_irq + 1;
348                      i < NR_IRQS && !foundmatch; i++) {
349                         list_for_each_entry_rcu(tmp_irq_info,
350                                                 sn_irq_lh[i],
351                                                 list) {
352                                 if (tmp_irq_info->irq_cpuid == cpu) {
353                                         foundmatch = 1;
354                                         break;
355                                 }
356                         }
357                 }
358                 pdacpu(cpu)->sn_first_irq = ((i == NR_IRQS) ? 0 : i);
359         }
360         rcu_read_unlock();
361 }
362
363 static void sn_irq_info_free(struct rcu_head *head)
364 {
365         struct sn_irq_info *sn_irq_info;
366
367         sn_irq_info = container_of(head, struct sn_irq_info, rcu);
368         kfree(sn_irq_info);
369 }
370
371 void sn_irq_fixup(struct pci_dev *pci_dev, struct sn_irq_info *sn_irq_info)
372 {
373         nasid_t nasid = sn_irq_info->irq_nasid;
374         int slice = sn_irq_info->irq_slice;
375         int cpu = nasid_slice_to_cpuid(nasid, slice);
376 #ifdef CONFIG_SMP
377         int cpuphys;
378         irq_desc_t *desc;
379 #endif
380
381         pci_dev_get(pci_dev);
382         sn_irq_info->irq_cpuid = cpu;
383         sn_irq_info->irq_pciioinfo = SN_PCIDEV_INFO(pci_dev);
384
385         /* link it into the sn_irq[irq] list */
386         spin_lock(&sn_irq_info_lock);
387         list_add_rcu(&sn_irq_info->list, sn_irq_lh[sn_irq_info->irq_irq]);
388         reserve_irq_vector(sn_irq_info->irq_irq);
389         spin_unlock(&sn_irq_info_lock);
390
391         register_intr_pda(sn_irq_info);
392 #ifdef CONFIG_SMP
393         cpuphys = cpu_physical_id(cpu);
394         set_irq_affinity_info(sn_irq_info->irq_irq, cpuphys, 0);
395         desc = irq_to_desc(sn_irq_info->irq_irq);
396         /*
397          * Affinity was set by the PROM, prevent it from
398          * being reset by the request_irq() path.
399          */
400         desc->status |= IRQ_AFFINITY_SET;
401 #endif
402 }
403
404 void sn_irq_unfixup(struct pci_dev *pci_dev)
405 {
406         struct sn_irq_info *sn_irq_info;
407
408         /* Only cleanup IRQ stuff if this device has a host bus context */
409         if (!SN_PCIDEV_BUSSOFT(pci_dev))
410                 return;
411
412         sn_irq_info = SN_PCIDEV_INFO(pci_dev)->pdi_sn_irq_info;
413         if (!sn_irq_info)
414                 return;
415         if (!sn_irq_info->irq_irq) {
416                 kfree(sn_irq_info);
417                 return;
418         }
419
420         unregister_intr_pda(sn_irq_info);
421         spin_lock(&sn_irq_info_lock);
422         list_del_rcu(&sn_irq_info->list);
423         spin_unlock(&sn_irq_info_lock);
424         if (list_empty(sn_irq_lh[sn_irq_info->irq_irq]))
425                 free_irq_vector(sn_irq_info->irq_irq);
426         call_rcu(&sn_irq_info->rcu, sn_irq_info_free);
427         pci_dev_put(pci_dev);
428
429 }
430
431 static inline void
432 sn_call_force_intr_provider(struct sn_irq_info *sn_irq_info)
433 {
434         struct sn_pcibus_provider *pci_provider;
435
436         pci_provider = sn_pci_provider[sn_irq_info->irq_bridge_type];
437
438         /* Don't force an interrupt if the irq has been disabled */
439         if (!(irq_desc[sn_irq_info->irq_irq].status & IRQ_DISABLED) &&
440             pci_provider && pci_provider->force_interrupt)
441                 (*pci_provider->force_interrupt)(sn_irq_info);
442 }
443
444 static void force_interrupt(int irq)
445 {
446         struct sn_irq_info *sn_irq_info;
447
448         if (!sn_ioif_inited)
449                 return;
450
451         rcu_read_lock();
452         list_for_each_entry_rcu(sn_irq_info, sn_irq_lh[irq], list)
453                 sn_call_force_intr_provider(sn_irq_info);
454
455         rcu_read_unlock();
456 }
457
458 /*
459  * Check for lost interrupts.  If the PIC int_status reg. says that
460  * an interrupt has been sent, but not handled, and the interrupt
461  * is not pending in either the cpu irr regs or in the soft irr regs,
462  * and the interrupt is not in service, then the interrupt may have
463  * been lost.  Force an interrupt on that pin.  It is possible that
464  * the interrupt is in flight, so we may generate a spurious interrupt,
465  * but we should never miss a real lost interrupt.
466  */
467 static void sn_check_intr(int irq, struct sn_irq_info *sn_irq_info)
468 {
469         u64 regval;
470         struct pcidev_info *pcidev_info;
471         struct pcibus_info *pcibus_info;
472
473         /*
474          * Bridge types attached to TIO (anything but PIC) do not need this WAR
475          * since they do not target Shub II interrupt registers.  If that
476          * ever changes, this check needs to accomodate.
477          */
478         if (sn_irq_info->irq_bridge_type != PCIIO_ASIC_TYPE_PIC)
479                 return;
480
481         pcidev_info = (struct pcidev_info *)sn_irq_info->irq_pciioinfo;
482         if (!pcidev_info)
483                 return;
484
485         pcibus_info =
486             (struct pcibus_info *)pcidev_info->pdi_host_pcidev_info->
487             pdi_pcibus_info;
488         regval = pcireg_intr_status_get(pcibus_info);
489
490         if (!ia64_get_irr(irq_to_vector(irq))) {
491                 if (!test_bit(irq, pda->sn_in_service_ivecs)) {
492                         regval &= 0xff;
493                         if (sn_irq_info->irq_int_bit & regval &
494                             sn_irq_info->irq_last_intr) {
495                                 regval &= ~(sn_irq_info->irq_int_bit & regval);
496                                 sn_call_force_intr_provider(sn_irq_info);
497                         }
498                 }
499         }
500         sn_irq_info->irq_last_intr = regval;
501 }
502
503 void sn_lb_int_war_check(void)
504 {
505         struct sn_irq_info *sn_irq_info;
506         int i;
507
508         if (!sn_ioif_inited || pda->sn_first_irq == 0)
509                 return;
510
511         rcu_read_lock();
512         for (i = pda->sn_first_irq; i <= pda->sn_last_irq; i++) {
513                 list_for_each_entry_rcu(sn_irq_info, sn_irq_lh[i], list) {
514                         sn_check_intr(i, sn_irq_info);
515                 }
516         }
517         rcu_read_unlock();
518 }
519
520 void __init sn_irq_lh_init(void)
521 {
522         int i;
523
524         sn_irq_lh = kmalloc(sizeof(struct list_head *) * NR_IRQS, GFP_KERNEL);
525         if (!sn_irq_lh)
526                 panic("SN PCI INIT: Failed to allocate memory for PCI init\n");
527
528         for (i = 0; i < NR_IRQS; i++) {
529                 sn_irq_lh[i] = kmalloc(sizeof(struct list_head), GFP_KERNEL);
530                 if (!sn_irq_lh[i])
531                         panic("SN PCI INIT: Failed IRQ memory allocation\n");
532
533                 INIT_LIST_HEAD(sn_irq_lh[i]);
534         }
535 }