x86, apicv: add virtual interrupt delivery support
[linux-3.10.git] / virt / kvm / irq_comm.c
1 /*
2  * irq_comm.c: Common API for in kernel interrupt controller
3  * Copyright (c) 2007, Intel Corporation.
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms and conditions of the GNU General Public License,
7  * version 2, as published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12  * more details.
13  *
14  * You should have received a copy of the GNU General Public License along with
15  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
16  * Place - Suite 330, Boston, MA 02111-1307 USA.
17  * Authors:
18  *   Yaozu (Eddie) Dong <Eddie.dong@intel.com>
19  *
20  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
21  */
22
23 #include <linux/kvm_host.h>
24 #include <linux/slab.h>
25 #include <linux/export.h>
26 #include <trace/events/kvm.h>
27
28 #include <asm/msidef.h>
29 #ifdef CONFIG_IA64
30 #include <asm/iosapic.h>
31 #endif
32
33 #include "irq.h"
34
35 #include "ioapic.h"
36
37 static int kvm_set_pic_irq(struct kvm_kernel_irq_routing_entry *e,
38                            struct kvm *kvm, int irq_source_id, int level)
39 {
40 #ifdef CONFIG_X86
41         struct kvm_pic *pic = pic_irqchip(kvm);
42         return kvm_pic_set_irq(pic, e->irqchip.pin, irq_source_id, level);
43 #else
44         return -1;
45 #endif
46 }
47
48 static int kvm_set_ioapic_irq(struct kvm_kernel_irq_routing_entry *e,
49                               struct kvm *kvm, int irq_source_id, int level)
50 {
51         struct kvm_ioapic *ioapic = kvm->arch.vioapic;
52         return kvm_ioapic_set_irq(ioapic, e->irqchip.pin, irq_source_id, level);
53 }
54
55 inline static bool kvm_is_dm_lowest_prio(struct kvm_lapic_irq *irq)
56 {
57 #ifdef CONFIG_IA64
58         return irq->delivery_mode ==
59                 (IOSAPIC_LOWEST_PRIORITY << IOSAPIC_DELIVERY_SHIFT);
60 #else
61         return irq->delivery_mode == APIC_DM_LOWEST;
62 #endif
63 }
64
65 int kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src,
66                 struct kvm_lapic_irq *irq)
67 {
68         int i, r = -1;
69         struct kvm_vcpu *vcpu, *lowest = NULL;
70
71         if (irq->dest_mode == 0 && irq->dest_id == 0xff &&
72                         kvm_is_dm_lowest_prio(irq)) {
73                 printk(KERN_INFO "kvm: apic: phys broadcast and lowest prio\n");
74                 irq->delivery_mode = APIC_DM_FIXED;
75         }
76
77         if (kvm_irq_delivery_to_apic_fast(kvm, src, irq, &r))
78                 return r;
79
80         kvm_for_each_vcpu(i, vcpu, kvm) {
81                 if (!kvm_apic_present(vcpu))
82                         continue;
83
84                 if (!kvm_apic_match_dest(vcpu, src, irq->shorthand,
85                                         irq->dest_id, irq->dest_mode))
86                         continue;
87
88                 if (!kvm_is_dm_lowest_prio(irq)) {
89                         if (r < 0)
90                                 r = 0;
91                         r += kvm_apic_set_irq(vcpu, irq);
92                 } else if (kvm_lapic_enabled(vcpu)) {
93                         if (!lowest)
94                                 lowest = vcpu;
95                         else if (kvm_apic_compare_prio(vcpu, lowest) < 0)
96                                 lowest = vcpu;
97                 }
98         }
99
100         if (lowest)
101                 r = kvm_apic_set_irq(lowest, irq);
102
103         return r;
104 }
105
106 static inline void kvm_set_msi_irq(struct kvm_kernel_irq_routing_entry *e,
107                                    struct kvm_lapic_irq *irq)
108 {
109         trace_kvm_msi_set_irq(e->msi.address_lo, e->msi.data);
110
111         irq->dest_id = (e->msi.address_lo &
112                         MSI_ADDR_DEST_ID_MASK) >> MSI_ADDR_DEST_ID_SHIFT;
113         irq->vector = (e->msi.data &
114                         MSI_DATA_VECTOR_MASK) >> MSI_DATA_VECTOR_SHIFT;
115         irq->dest_mode = (1 << MSI_ADDR_DEST_MODE_SHIFT) & e->msi.address_lo;
116         irq->trig_mode = (1 << MSI_DATA_TRIGGER_SHIFT) & e->msi.data;
117         irq->delivery_mode = e->msi.data & 0x700;
118         irq->level = 1;
119         irq->shorthand = 0;
120         /* TODO Deal with RH bit of MSI message address */
121 }
122
123 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e,
124                 struct kvm *kvm, int irq_source_id, int level)
125 {
126         struct kvm_lapic_irq irq;
127
128         if (!level)
129                 return -1;
130
131         kvm_set_msi_irq(e, &irq);
132
133         return kvm_irq_delivery_to_apic(kvm, NULL, &irq);
134 }
135
136
137 static int kvm_set_msi_inatomic(struct kvm_kernel_irq_routing_entry *e,
138                          struct kvm *kvm)
139 {
140         struct kvm_lapic_irq irq;
141         int r;
142
143         kvm_set_msi_irq(e, &irq);
144
145         if (kvm_irq_delivery_to_apic_fast(kvm, NULL, &irq, &r))
146                 return r;
147         else
148                 return -EWOULDBLOCK;
149 }
150
151 int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi)
152 {
153         struct kvm_kernel_irq_routing_entry route;
154
155         if (!irqchip_in_kernel(kvm) || msi->flags != 0)
156                 return -EINVAL;
157
158         route.msi.address_lo = msi->address_lo;
159         route.msi.address_hi = msi->address_hi;
160         route.msi.data = msi->data;
161
162         return kvm_set_msi(&route, kvm, KVM_USERSPACE_IRQ_SOURCE_ID, 1);
163 }
164
165 /*
166  * Return value:
167  *  < 0   Interrupt was ignored (masked or not delivered for other reasons)
168  *  = 0   Interrupt was coalesced (previous irq is still pending)
169  *  > 0   Number of CPUs interrupt was delivered to
170  */
171 int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level)
172 {
173         struct kvm_kernel_irq_routing_entry *e, irq_set[KVM_NR_IRQCHIPS];
174         int ret = -1, i = 0;
175         struct kvm_irq_routing_table *irq_rt;
176         struct hlist_node *n;
177
178         trace_kvm_set_irq(irq, level, irq_source_id);
179
180         /* Not possible to detect if the guest uses the PIC or the
181          * IOAPIC.  So set the bit in both. The guest will ignore
182          * writes to the unused one.
183          */
184         rcu_read_lock();
185         irq_rt = rcu_dereference(kvm->irq_routing);
186         if (irq < irq_rt->nr_rt_entries)
187                 hlist_for_each_entry(e, n, &irq_rt->map[irq], link)
188                         irq_set[i++] = *e;
189         rcu_read_unlock();
190
191         while(i--) {
192                 int r;
193                 r = irq_set[i].set(&irq_set[i], kvm, irq_source_id, level);
194                 if (r < 0)
195                         continue;
196
197                 ret = r + ((ret < 0) ? 0 : ret);
198         }
199
200         return ret;
201 }
202
203 /*
204  * Deliver an IRQ in an atomic context if we can, or return a failure,
205  * user can retry in a process context.
206  * Return value:
207  *  -EWOULDBLOCK - Can't deliver in atomic context: retry in a process context.
208  *  Other values - No need to retry.
209  */
210 int kvm_set_irq_inatomic(struct kvm *kvm, int irq_source_id, u32 irq, int level)
211 {
212         struct kvm_kernel_irq_routing_entry *e;
213         int ret = -EINVAL;
214         struct kvm_irq_routing_table *irq_rt;
215         struct hlist_node *n;
216
217         trace_kvm_set_irq(irq, level, irq_source_id);
218
219         /*
220          * Injection into either PIC or IOAPIC might need to scan all CPUs,
221          * which would need to be retried from thread context;  when same GSI
222          * is connected to both PIC and IOAPIC, we'd have to report a
223          * partial failure here.
224          * Since there's no easy way to do this, we only support injecting MSI
225          * which is limited to 1:1 GSI mapping.
226          */
227         rcu_read_lock();
228         irq_rt = rcu_dereference(kvm->irq_routing);
229         if (irq < irq_rt->nr_rt_entries)
230                 hlist_for_each_entry(e, n, &irq_rt->map[irq], link) {
231                         if (likely(e->type == KVM_IRQ_ROUTING_MSI))
232                                 ret = kvm_set_msi_inatomic(e, kvm);
233                         else
234                                 ret = -EWOULDBLOCK;
235                         break;
236                 }
237         rcu_read_unlock();
238         return ret;
239 }
240
241 bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin)
242 {
243         struct kvm_irq_ack_notifier *kian;
244         struct hlist_node *n;
245         int gsi;
246
247         rcu_read_lock();
248         gsi = rcu_dereference(kvm->irq_routing)->chip[irqchip][pin];
249         if (gsi != -1)
250                 hlist_for_each_entry_rcu(kian, n, &kvm->irq_ack_notifier_list,
251                                          link)
252                         if (kian->gsi == gsi) {
253                                 rcu_read_unlock();
254                                 return true;
255                         }
256
257         rcu_read_unlock();
258
259         return false;
260 }
261 EXPORT_SYMBOL_GPL(kvm_irq_has_notifier);
262
263 void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin)
264 {
265         struct kvm_irq_ack_notifier *kian;
266         struct hlist_node *n;
267         int gsi;
268
269         trace_kvm_ack_irq(irqchip, pin);
270
271         rcu_read_lock();
272         gsi = rcu_dereference(kvm->irq_routing)->chip[irqchip][pin];
273         if (gsi != -1)
274                 hlist_for_each_entry_rcu(kian, n, &kvm->irq_ack_notifier_list,
275                                          link)
276                         if (kian->gsi == gsi)
277                                 kian->irq_acked(kian);
278         rcu_read_unlock();
279 }
280
281 void kvm_register_irq_ack_notifier(struct kvm *kvm,
282                                    struct kvm_irq_ack_notifier *kian)
283 {
284         mutex_lock(&kvm->irq_lock);
285         hlist_add_head_rcu(&kian->link, &kvm->irq_ack_notifier_list);
286         mutex_unlock(&kvm->irq_lock);
287         kvm_ioapic_make_eoibitmap_request(kvm);
288 }
289
290 void kvm_unregister_irq_ack_notifier(struct kvm *kvm,
291                                     struct kvm_irq_ack_notifier *kian)
292 {
293         mutex_lock(&kvm->irq_lock);
294         hlist_del_init_rcu(&kian->link);
295         mutex_unlock(&kvm->irq_lock);
296         synchronize_rcu();
297         kvm_ioapic_make_eoibitmap_request(kvm);
298 }
299
300 int kvm_request_irq_source_id(struct kvm *kvm)
301 {
302         unsigned long *bitmap = &kvm->arch.irq_sources_bitmap;
303         int irq_source_id;
304
305         mutex_lock(&kvm->irq_lock);
306         irq_source_id = find_first_zero_bit(bitmap, BITS_PER_LONG);
307
308         if (irq_source_id >= BITS_PER_LONG) {
309                 printk(KERN_WARNING "kvm: exhaust allocatable IRQ sources!\n");
310                 irq_source_id = -EFAULT;
311                 goto unlock;
312         }
313
314         ASSERT(irq_source_id != KVM_USERSPACE_IRQ_SOURCE_ID);
315 #ifdef CONFIG_X86
316         ASSERT(irq_source_id != KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID);
317 #endif
318         set_bit(irq_source_id, bitmap);
319 unlock:
320         mutex_unlock(&kvm->irq_lock);
321
322         return irq_source_id;
323 }
324
325 void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id)
326 {
327         ASSERT(irq_source_id != KVM_USERSPACE_IRQ_SOURCE_ID);
328 #ifdef CONFIG_X86
329         ASSERT(irq_source_id != KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID);
330 #endif
331
332         mutex_lock(&kvm->irq_lock);
333         if (irq_source_id < 0 ||
334             irq_source_id >= BITS_PER_LONG) {
335                 printk(KERN_ERR "kvm: IRQ source ID out of range!\n");
336                 goto unlock;
337         }
338         clear_bit(irq_source_id, &kvm->arch.irq_sources_bitmap);
339         if (!irqchip_in_kernel(kvm))
340                 goto unlock;
341
342         kvm_ioapic_clear_all(kvm->arch.vioapic, irq_source_id);
343 #ifdef CONFIG_X86
344         kvm_pic_clear_all(pic_irqchip(kvm), irq_source_id);
345 #endif
346 unlock:
347         mutex_unlock(&kvm->irq_lock);
348 }
349
350 void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq,
351                                     struct kvm_irq_mask_notifier *kimn)
352 {
353         mutex_lock(&kvm->irq_lock);
354         kimn->irq = irq;
355         hlist_add_head_rcu(&kimn->link, &kvm->mask_notifier_list);
356         mutex_unlock(&kvm->irq_lock);
357 }
358
359 void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq,
360                                       struct kvm_irq_mask_notifier *kimn)
361 {
362         mutex_lock(&kvm->irq_lock);
363         hlist_del_rcu(&kimn->link);
364         mutex_unlock(&kvm->irq_lock);
365         synchronize_rcu();
366 }
367
368 void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin,
369                              bool mask)
370 {
371         struct kvm_irq_mask_notifier *kimn;
372         struct hlist_node *n;
373         int gsi;
374
375         rcu_read_lock();
376         gsi = rcu_dereference(kvm->irq_routing)->chip[irqchip][pin];
377         if (gsi != -1)
378                 hlist_for_each_entry_rcu(kimn, n, &kvm->mask_notifier_list, link)
379                         if (kimn->irq == gsi)
380                                 kimn->func(kimn, mask);
381         rcu_read_unlock();
382 }
383
384 void kvm_free_irq_routing(struct kvm *kvm)
385 {
386         /* Called only during vm destruction. Nobody can use the pointer
387            at this stage */
388         kfree(kvm->irq_routing);
389 }
390
391 static int setup_routing_entry(struct kvm_irq_routing_table *rt,
392                                struct kvm_kernel_irq_routing_entry *e,
393                                const struct kvm_irq_routing_entry *ue)
394 {
395         int r = -EINVAL;
396         int delta;
397         unsigned max_pin;
398         struct kvm_kernel_irq_routing_entry *ei;
399         struct hlist_node *n;
400
401         /*
402          * Do not allow GSI to be mapped to the same irqchip more than once.
403          * Allow only one to one mapping between GSI and MSI.
404          */
405         hlist_for_each_entry(ei, n, &rt->map[ue->gsi], link)
406                 if (ei->type == KVM_IRQ_ROUTING_MSI ||
407                     ue->type == KVM_IRQ_ROUTING_MSI ||
408                     ue->u.irqchip.irqchip == ei->irqchip.irqchip)
409                         return r;
410
411         e->gsi = ue->gsi;
412         e->type = ue->type;
413         switch (ue->type) {
414         case KVM_IRQ_ROUTING_IRQCHIP:
415                 delta = 0;
416                 switch (ue->u.irqchip.irqchip) {
417                 case KVM_IRQCHIP_PIC_MASTER:
418                         e->set = kvm_set_pic_irq;
419                         max_pin = PIC_NUM_PINS;
420                         break;
421                 case KVM_IRQCHIP_PIC_SLAVE:
422                         e->set = kvm_set_pic_irq;
423                         max_pin = PIC_NUM_PINS;
424                         delta = 8;
425                         break;
426                 case KVM_IRQCHIP_IOAPIC:
427                         max_pin = KVM_IOAPIC_NUM_PINS;
428                         e->set = kvm_set_ioapic_irq;
429                         break;
430                 default:
431                         goto out;
432                 }
433                 e->irqchip.irqchip = ue->u.irqchip.irqchip;
434                 e->irqchip.pin = ue->u.irqchip.pin + delta;
435                 if (e->irqchip.pin >= max_pin)
436                         goto out;
437                 rt->chip[ue->u.irqchip.irqchip][e->irqchip.pin] = ue->gsi;
438                 break;
439         case KVM_IRQ_ROUTING_MSI:
440                 e->set = kvm_set_msi;
441                 e->msi.address_lo = ue->u.msi.address_lo;
442                 e->msi.address_hi = ue->u.msi.address_hi;
443                 e->msi.data = ue->u.msi.data;
444                 break;
445         default:
446                 goto out;
447         }
448
449         hlist_add_head(&e->link, &rt->map[e->gsi]);
450         r = 0;
451 out:
452         return r;
453 }
454
455
456 int kvm_set_irq_routing(struct kvm *kvm,
457                         const struct kvm_irq_routing_entry *ue,
458                         unsigned nr,
459                         unsigned flags)
460 {
461         struct kvm_irq_routing_table *new, *old;
462         u32 i, j, nr_rt_entries = 0;
463         int r;
464
465         for (i = 0; i < nr; ++i) {
466                 if (ue[i].gsi >= KVM_MAX_IRQ_ROUTES)
467                         return -EINVAL;
468                 nr_rt_entries = max(nr_rt_entries, ue[i].gsi);
469         }
470
471         nr_rt_entries += 1;
472
473         new = kzalloc(sizeof(*new) + (nr_rt_entries * sizeof(struct hlist_head))
474                       + (nr * sizeof(struct kvm_kernel_irq_routing_entry)),
475                       GFP_KERNEL);
476
477         if (!new)
478                 return -ENOMEM;
479
480         new->rt_entries = (void *)&new->map[nr_rt_entries];
481
482         new->nr_rt_entries = nr_rt_entries;
483         for (i = 0; i < 3; i++)
484                 for (j = 0; j < KVM_IOAPIC_NUM_PINS; j++)
485                         new->chip[i][j] = -1;
486
487         for (i = 0; i < nr; ++i) {
488                 r = -EINVAL;
489                 if (ue->flags)
490                         goto out;
491                 r = setup_routing_entry(new, &new->rt_entries[i], ue);
492                 if (r)
493                         goto out;
494                 ++ue;
495         }
496
497         mutex_lock(&kvm->irq_lock);
498         old = kvm->irq_routing;
499         kvm_irq_routing_update(kvm, new);
500         mutex_unlock(&kvm->irq_lock);
501
502         synchronize_rcu();
503
504         new = old;
505         r = 0;
506
507 out:
508         kfree(new);
509         return r;
510 }
511
512 #define IOAPIC_ROUTING_ENTRY(irq) \
513         { .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP,  \
514           .u.irqchip.irqchip = KVM_IRQCHIP_IOAPIC, .u.irqchip.pin = (irq) }
515 #define ROUTING_ENTRY1(irq) IOAPIC_ROUTING_ENTRY(irq)
516
517 #ifdef CONFIG_X86
518 #  define PIC_ROUTING_ENTRY(irq) \
519         { .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP,  \
520           .u.irqchip.irqchip = SELECT_PIC(irq), .u.irqchip.pin = (irq) % 8 }
521 #  define ROUTING_ENTRY2(irq) \
522         IOAPIC_ROUTING_ENTRY(irq), PIC_ROUTING_ENTRY(irq)
523 #else
524 #  define ROUTING_ENTRY2(irq) \
525         IOAPIC_ROUTING_ENTRY(irq)
526 #endif
527
528 static const struct kvm_irq_routing_entry default_routing[] = {
529         ROUTING_ENTRY2(0), ROUTING_ENTRY2(1),
530         ROUTING_ENTRY2(2), ROUTING_ENTRY2(3),
531         ROUTING_ENTRY2(4), ROUTING_ENTRY2(5),
532         ROUTING_ENTRY2(6), ROUTING_ENTRY2(7),
533         ROUTING_ENTRY2(8), ROUTING_ENTRY2(9),
534         ROUTING_ENTRY2(10), ROUTING_ENTRY2(11),
535         ROUTING_ENTRY2(12), ROUTING_ENTRY2(13),
536         ROUTING_ENTRY2(14), ROUTING_ENTRY2(15),
537         ROUTING_ENTRY1(16), ROUTING_ENTRY1(17),
538         ROUTING_ENTRY1(18), ROUTING_ENTRY1(19),
539         ROUTING_ENTRY1(20), ROUTING_ENTRY1(21),
540         ROUTING_ENTRY1(22), ROUTING_ENTRY1(23),
541 #ifdef CONFIG_IA64
542         ROUTING_ENTRY1(24), ROUTING_ENTRY1(25),
543         ROUTING_ENTRY1(26), ROUTING_ENTRY1(27),
544         ROUTING_ENTRY1(28), ROUTING_ENTRY1(29),
545         ROUTING_ENTRY1(30), ROUTING_ENTRY1(31),
546         ROUTING_ENTRY1(32), ROUTING_ENTRY1(33),
547         ROUTING_ENTRY1(34), ROUTING_ENTRY1(35),
548         ROUTING_ENTRY1(36), ROUTING_ENTRY1(37),
549         ROUTING_ENTRY1(38), ROUTING_ENTRY1(39),
550         ROUTING_ENTRY1(40), ROUTING_ENTRY1(41),
551         ROUTING_ENTRY1(42), ROUTING_ENTRY1(43),
552         ROUTING_ENTRY1(44), ROUTING_ENTRY1(45),
553         ROUTING_ENTRY1(46), ROUTING_ENTRY1(47),
554 #endif
555 };
556
557 int kvm_setup_default_irq_routing(struct kvm *kvm)
558 {
559         return kvm_set_irq_routing(kvm, default_routing,
560                                    ARRAY_SIZE(default_routing), 0);
561 }