]> nv-tegra.nvidia Code Review - linux-3.10.git/blob - drivers/pci/dmar.c
intel-iommu: Avoid panic() for DRHD at address zero.
[linux-3.10.git] / drivers / pci / dmar.c
1 /*
2  * Copyright (c) 2006, Intel Corporation.
3  *
4  * This program is free software; you can redistribute it and/or modify it
5  * under the terms and conditions of the GNU General Public License,
6  * version 2, as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope it will be useful, but WITHOUT
9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
11  * more details.
12  *
13  * You should have received a copy of the GNU General Public License along with
14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15  * Place - Suite 330, Boston, MA 02111-1307 USA.
16  *
17  * Copyright (C) 2006-2008 Intel Corporation
18  * Author: Ashok Raj <ashok.raj@intel.com>
19  * Author: Shaohua Li <shaohua.li@intel.com>
20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21  *
22  * This file implements early detection/parsing of Remapping Devices
23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24  * tables.
25  *
26  * These routines are used by both DMA-remapping and Interrupt-remapping
27  */
28
29 #include <linux/pci.h>
30 #include <linux/dmar.h>
31 #include <linux/iova.h>
32 #include <linux/intel-iommu.h>
33 #include <linux/timer.h>
34 #include <linux/irq.h>
35 #include <linux/interrupt.h>
36
37 #undef PREFIX
38 #define PREFIX "DMAR:"
39
40 /* No locks are needed as DMA remapping hardware unit
41  * list is constructed at boot time and hotplug of
42  * these units are not supported by the architecture.
43  */
44 LIST_HEAD(dmar_drhd_units);
45
46 static struct acpi_table_header * __initdata dmar_tbl;
47 static acpi_size dmar_tbl_size;
48
49 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
50 {
51         /*
52          * add INCLUDE_ALL at the tail, so scan the list will find it at
53          * the very end.
54          */
55         if (drhd->include_all)
56                 list_add_tail(&drhd->list, &dmar_drhd_units);
57         else
58                 list_add(&drhd->list, &dmar_drhd_units);
59 }
60
61 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
62                                            struct pci_dev **dev, u16 segment)
63 {
64         struct pci_bus *bus;
65         struct pci_dev *pdev = NULL;
66         struct acpi_dmar_pci_path *path;
67         int count;
68
69         bus = pci_find_bus(segment, scope->bus);
70         path = (struct acpi_dmar_pci_path *)(scope + 1);
71         count = (scope->length - sizeof(struct acpi_dmar_device_scope))
72                 / sizeof(struct acpi_dmar_pci_path);
73
74         while (count) {
75                 if (pdev)
76                         pci_dev_put(pdev);
77                 /*
78                  * Some BIOSes list non-exist devices in DMAR table, just
79                  * ignore it
80                  */
81                 if (!bus) {
82                         printk(KERN_WARNING
83                         PREFIX "Device scope bus [%d] not found\n",
84                         scope->bus);
85                         break;
86                 }
87                 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
88                 if (!pdev) {
89                         printk(KERN_WARNING PREFIX
90                         "Device scope device [%04x:%02x:%02x.%02x] not found\n",
91                                 segment, bus->number, path->dev, path->fn);
92                         break;
93                 }
94                 path ++;
95                 count --;
96                 bus = pdev->subordinate;
97         }
98         if (!pdev) {
99                 printk(KERN_WARNING PREFIX
100                 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
101                 segment, scope->bus, path->dev, path->fn);
102                 *dev = NULL;
103                 return 0;
104         }
105         if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
106                         pdev->subordinate) || (scope->entry_type == \
107                         ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
108                 pci_dev_put(pdev);
109                 printk(KERN_WARNING PREFIX
110                         "Device scope type does not match for %s\n",
111                          pci_name(pdev));
112                 return -EINVAL;
113         }
114         *dev = pdev;
115         return 0;
116 }
117
118 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
119                                        struct pci_dev ***devices, u16 segment)
120 {
121         struct acpi_dmar_device_scope *scope;
122         void * tmp = start;
123         int index;
124         int ret;
125
126         *cnt = 0;
127         while (start < end) {
128                 scope = start;
129                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
130                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
131                         (*cnt)++;
132                 else
133                         printk(KERN_WARNING PREFIX
134                                 "Unsupported device scope\n");
135                 start += scope->length;
136         }
137         if (*cnt == 0)
138                 return 0;
139
140         *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
141         if (!*devices)
142                 return -ENOMEM;
143
144         start = tmp;
145         index = 0;
146         while (start < end) {
147                 scope = start;
148                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
149                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
150                         ret = dmar_parse_one_dev_scope(scope,
151                                 &(*devices)[index], segment);
152                         if (ret) {
153                                 kfree(*devices);
154                                 return ret;
155                         }
156                         index ++;
157                 }
158                 start += scope->length;
159         }
160
161         return 0;
162 }
163
164 /**
165  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
166  * structure which uniquely represent one DMA remapping hardware unit
167  * present in the platform
168  */
169 static int __init
170 dmar_parse_one_drhd(struct acpi_dmar_header *header)
171 {
172         struct acpi_dmar_hardware_unit *drhd;
173         struct dmar_drhd_unit *dmaru;
174         int ret = 0;
175
176         drhd = (struct acpi_dmar_hardware_unit *)header;
177         if (!drhd->address) {
178                 /* Promote an attitude of violence to a BIOS engineer today */
179                 WARN(1, "Your BIOS is broken; DMAR reported at address zero!\n"
180                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
181                      dmi_get_system_info(DMI_BIOS_VENDOR),
182                      dmi_get_system_info(DMI_BIOS_VERSION),
183                      dmi_get_system_info(DMI_PRODUCT_VERSION));
184                 return -ENODEV;
185         }
186         dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
187         if (!dmaru)
188                 return -ENOMEM;
189
190         dmaru->hdr = header;
191         dmaru->reg_base_addr = drhd->address;
192         dmaru->segment = drhd->segment;
193         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
194
195         ret = alloc_iommu(dmaru);
196         if (ret) {
197                 kfree(dmaru);
198                 return ret;
199         }
200         dmar_register_drhd_unit(dmaru);
201         return 0;
202 }
203
204 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
205 {
206         struct acpi_dmar_hardware_unit *drhd;
207         int ret = 0;
208
209         drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
210
211         if (dmaru->include_all)
212                 return 0;
213
214         ret = dmar_parse_dev_scope((void *)(drhd + 1),
215                                 ((void *)drhd) + drhd->header.length,
216                                 &dmaru->devices_cnt, &dmaru->devices,
217                                 drhd->segment);
218         if (ret) {
219                 list_del(&dmaru->list);
220                 kfree(dmaru);
221         }
222         return ret;
223 }
224
225 #ifdef CONFIG_DMAR
226 LIST_HEAD(dmar_rmrr_units);
227
228 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
229 {
230         list_add(&rmrr->list, &dmar_rmrr_units);
231 }
232
233
234 static int __init
235 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
236 {
237         struct acpi_dmar_reserved_memory *rmrr;
238         struct dmar_rmrr_unit *rmrru;
239
240         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
241         if (!rmrru)
242                 return -ENOMEM;
243
244         rmrru->hdr = header;
245         rmrr = (struct acpi_dmar_reserved_memory *)header;
246         rmrru->base_address = rmrr->base_address;
247         rmrru->end_address = rmrr->end_address;
248
249         dmar_register_rmrr_unit(rmrru);
250         return 0;
251 }
252
253 static int __init
254 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
255 {
256         struct acpi_dmar_reserved_memory *rmrr;
257         int ret;
258
259         rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
260         ret = dmar_parse_dev_scope((void *)(rmrr + 1),
261                 ((void *)rmrr) + rmrr->header.length,
262                 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
263
264         if (ret || (rmrru->devices_cnt == 0)) {
265                 list_del(&rmrru->list);
266                 kfree(rmrru);
267         }
268         return ret;
269 }
270 #endif
271
272 static void __init
273 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
274 {
275         struct acpi_dmar_hardware_unit *drhd;
276         struct acpi_dmar_reserved_memory *rmrr;
277
278         switch (header->type) {
279         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
280                 drhd = (struct acpi_dmar_hardware_unit *)header;
281                 printk (KERN_INFO PREFIX
282                         "DRHD (flags: 0x%08x)base: 0x%016Lx\n",
283                         drhd->flags, (unsigned long long)drhd->address);
284                 break;
285         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
286                 rmrr = (struct acpi_dmar_reserved_memory *)header;
287
288                 printk (KERN_INFO PREFIX
289                         "RMRR base: 0x%016Lx end: 0x%016Lx\n",
290                         (unsigned long long)rmrr->base_address,
291                         (unsigned long long)rmrr->end_address);
292                 break;
293         }
294 }
295
296 /**
297  * dmar_table_detect - checks to see if the platform supports DMAR devices
298  */
299 static int __init dmar_table_detect(void)
300 {
301         acpi_status status = AE_OK;
302
303         /* if we could find DMAR table, then there are DMAR devices */
304         status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
305                                 (struct acpi_table_header **)&dmar_tbl,
306                                 &dmar_tbl_size);
307
308         if (ACPI_SUCCESS(status) && !dmar_tbl) {
309                 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
310                 status = AE_NOT_FOUND;
311         }
312
313         return (ACPI_SUCCESS(status) ? 1 : 0);
314 }
315
316 /**
317  * parse_dmar_table - parses the DMA reporting table
318  */
319 static int __init
320 parse_dmar_table(void)
321 {
322         struct acpi_table_dmar *dmar;
323         struct acpi_dmar_header *entry_header;
324         int ret = 0;
325
326         /*
327          * Do it again, earlier dmar_tbl mapping could be mapped with
328          * fixed map.
329          */
330         dmar_table_detect();
331
332         dmar = (struct acpi_table_dmar *)dmar_tbl;
333         if (!dmar)
334                 return -ENODEV;
335
336         if (dmar->width < PAGE_SHIFT - 1) {
337                 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
338                 return -EINVAL;
339         }
340
341         printk (KERN_INFO PREFIX "Host address width %d\n",
342                 dmar->width + 1);
343
344         entry_header = (struct acpi_dmar_header *)(dmar + 1);
345         while (((unsigned long)entry_header) <
346                         (((unsigned long)dmar) + dmar_tbl->length)) {
347                 /* Avoid looping forever on bad ACPI tables */
348                 if (entry_header->length == 0) {
349                         printk(KERN_WARNING PREFIX
350                                 "Invalid 0-length structure\n");
351                         ret = -EINVAL;
352                         break;
353                 }
354
355                 dmar_table_print_dmar_entry(entry_header);
356
357                 switch (entry_header->type) {
358                 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
359                         ret = dmar_parse_one_drhd(entry_header);
360                         break;
361                 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
362 #ifdef CONFIG_DMAR
363                         ret = dmar_parse_one_rmrr(entry_header);
364 #endif
365                         break;
366                 default:
367                         printk(KERN_WARNING PREFIX
368                                 "Unknown DMAR structure type\n");
369                         ret = 0; /* for forward compatibility */
370                         break;
371                 }
372                 if (ret)
373                         break;
374
375                 entry_header = ((void *)entry_header + entry_header->length);
376         }
377         return ret;
378 }
379
380 int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
381                           struct pci_dev *dev)
382 {
383         int index;
384
385         while (dev) {
386                 for (index = 0; index < cnt; index++)
387                         if (dev == devices[index])
388                                 return 1;
389
390                 /* Check our parent */
391                 dev = dev->bus->self;
392         }
393
394         return 0;
395 }
396
397 struct dmar_drhd_unit *
398 dmar_find_matched_drhd_unit(struct pci_dev *dev)
399 {
400         struct dmar_drhd_unit *dmaru = NULL;
401         struct acpi_dmar_hardware_unit *drhd;
402
403         list_for_each_entry(dmaru, &dmar_drhd_units, list) {
404                 drhd = container_of(dmaru->hdr,
405                                     struct acpi_dmar_hardware_unit,
406                                     header);
407
408                 if (dmaru->include_all &&
409                     drhd->segment == pci_domain_nr(dev->bus))
410                         return dmaru;
411
412                 if (dmar_pci_device_match(dmaru->devices,
413                                           dmaru->devices_cnt, dev))
414                         return dmaru;
415         }
416
417         return NULL;
418 }
419
420 int __init dmar_dev_scope_init(void)
421 {
422         struct dmar_drhd_unit *drhd, *drhd_n;
423         int ret = -ENODEV;
424
425         list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
426                 ret = dmar_parse_dev(drhd);
427                 if (ret)
428                         return ret;
429         }
430
431 #ifdef CONFIG_DMAR
432         {
433                 struct dmar_rmrr_unit *rmrr, *rmrr_n;
434                 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
435                         ret = rmrr_parse_dev(rmrr);
436                         if (ret)
437                                 return ret;
438                 }
439         }
440 #endif
441
442         return ret;
443 }
444
445
446 int __init dmar_table_init(void)
447 {
448         static int dmar_table_initialized;
449         int ret;
450
451         if (dmar_table_initialized)
452                 return 0;
453
454         dmar_table_initialized = 1;
455
456         ret = parse_dmar_table();
457         if (ret) {
458                 if (ret != -ENODEV)
459                         printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
460                 return ret;
461         }
462
463         if (list_empty(&dmar_drhd_units)) {
464                 printk(KERN_INFO PREFIX "No DMAR devices found\n");
465                 return -ENODEV;
466         }
467
468 #ifdef CONFIG_DMAR
469         if (list_empty(&dmar_rmrr_units))
470                 printk(KERN_INFO PREFIX "No RMRR found\n");
471 #endif
472
473 #ifdef CONFIG_INTR_REMAP
474         parse_ioapics_under_ir();
475 #endif
476         return 0;
477 }
478
479 void __init detect_intel_iommu(void)
480 {
481         int ret;
482
483         ret = dmar_table_detect();
484
485         {
486 #ifdef CONFIG_INTR_REMAP
487                 struct acpi_table_dmar *dmar;
488                 /*
489                  * for now we will disable dma-remapping when interrupt
490                  * remapping is enabled.
491                  * When support for queued invalidation for IOTLB invalidation
492                  * is added, we will not need this any more.
493                  */
494                 dmar = (struct acpi_table_dmar *) dmar_tbl;
495                 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
496                         printk(KERN_INFO
497                                "Queued invalidation will be enabled to support "
498                                "x2apic and Intr-remapping.\n");
499 #endif
500 #ifdef CONFIG_DMAR
501                 if (ret && !no_iommu && !iommu_detected && !swiotlb &&
502                     !dmar_disabled)
503                         iommu_detected = 1;
504 #endif
505         }
506         early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
507         dmar_tbl = NULL;
508 }
509
510
511 int alloc_iommu(struct dmar_drhd_unit *drhd)
512 {
513         struct intel_iommu *iommu;
514         int map_size;
515         u32 ver;
516         static int iommu_allocated = 0;
517         int agaw = 0;
518
519         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
520         if (!iommu)
521                 return -ENOMEM;
522
523         iommu->seq_id = iommu_allocated++;
524         sprintf (iommu->name, "dmar%d", iommu->seq_id);
525
526         iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
527         if (!iommu->reg) {
528                 printk(KERN_ERR "IOMMU: can't map the region\n");
529                 goto error;
530         }
531         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
532         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
533
534 #ifdef CONFIG_DMAR
535         agaw = iommu_calculate_agaw(iommu);
536         if (agaw < 0) {
537                 printk(KERN_ERR
538                         "Cannot get a valid agaw for iommu (seq_id = %d)\n",
539                         iommu->seq_id);
540                 goto error;
541         }
542 #endif
543         iommu->agaw = agaw;
544
545         /* the registers might be more than one page */
546         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
547                 cap_max_fault_reg_offset(iommu->cap));
548         map_size = VTD_PAGE_ALIGN(map_size);
549         if (map_size > VTD_PAGE_SIZE) {
550                 iounmap(iommu->reg);
551                 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
552                 if (!iommu->reg) {
553                         printk(KERN_ERR "IOMMU: can't map the region\n");
554                         goto error;
555                 }
556         }
557
558         ver = readl(iommu->reg + DMAR_VER_REG);
559         pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
560                 (unsigned long long)drhd->reg_base_addr,
561                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
562                 (unsigned long long)iommu->cap,
563                 (unsigned long long)iommu->ecap);
564
565         spin_lock_init(&iommu->register_lock);
566
567         drhd->iommu = iommu;
568         return 0;
569 error:
570         kfree(iommu);
571         return -1;
572 }
573
574 void free_iommu(struct intel_iommu *iommu)
575 {
576         if (!iommu)
577                 return;
578
579 #ifdef CONFIG_DMAR
580         free_dmar_iommu(iommu);
581 #endif
582
583         if (iommu->reg)
584                 iounmap(iommu->reg);
585         kfree(iommu);
586 }
587
588 /*
589  * Reclaim all the submitted descriptors which have completed its work.
590  */
591 static inline void reclaim_free_desc(struct q_inval *qi)
592 {
593         while (qi->desc_status[qi->free_tail] == QI_DONE) {
594                 qi->desc_status[qi->free_tail] = QI_FREE;
595                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
596                 qi->free_cnt++;
597         }
598 }
599
600 static int qi_check_fault(struct intel_iommu *iommu, int index)
601 {
602         u32 fault;
603         int head;
604         struct q_inval *qi = iommu->qi;
605         int wait_index = (index + 1) % QI_LENGTH;
606
607         fault = readl(iommu->reg + DMAR_FSTS_REG);
608
609         /*
610          * If IQE happens, the head points to the descriptor associated
611          * with the error. No new descriptors are fetched until the IQE
612          * is cleared.
613          */
614         if (fault & DMA_FSTS_IQE) {
615                 head = readl(iommu->reg + DMAR_IQH_REG);
616                 if ((head >> 4) == index) {
617                         memcpy(&qi->desc[index], &qi->desc[wait_index],
618                                         sizeof(struct qi_desc));
619                         __iommu_flush_cache(iommu, &qi->desc[index],
620                                         sizeof(struct qi_desc));
621                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
622                         return -EINVAL;
623                 }
624         }
625
626         return 0;
627 }
628
629 /*
630  * Submit the queued invalidation descriptor to the remapping
631  * hardware unit and wait for its completion.
632  */
633 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
634 {
635         int rc = 0;
636         struct q_inval *qi = iommu->qi;
637         struct qi_desc *hw, wait_desc;
638         int wait_index, index;
639         unsigned long flags;
640
641         if (!qi)
642                 return 0;
643
644         hw = qi->desc;
645
646         spin_lock_irqsave(&qi->q_lock, flags);
647         while (qi->free_cnt < 3) {
648                 spin_unlock_irqrestore(&qi->q_lock, flags);
649                 cpu_relax();
650                 spin_lock_irqsave(&qi->q_lock, flags);
651         }
652
653         index = qi->free_head;
654         wait_index = (index + 1) % QI_LENGTH;
655
656         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
657
658         hw[index] = *desc;
659
660         wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
661                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
662         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
663
664         hw[wait_index] = wait_desc;
665
666         __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
667         __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
668
669         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
670         qi->free_cnt -= 2;
671
672         /*
673          * update the HW tail register indicating the presence of
674          * new descriptors.
675          */
676         writel(qi->free_head << 4, iommu->reg + DMAR_IQT_REG);
677
678         while (qi->desc_status[wait_index] != QI_DONE) {
679                 /*
680                  * We will leave the interrupts disabled, to prevent interrupt
681                  * context to queue another cmd while a cmd is already submitted
682                  * and waiting for completion on this cpu. This is to avoid
683                  * a deadlock where the interrupt context can wait indefinitely
684                  * for free slots in the queue.
685                  */
686                 rc = qi_check_fault(iommu, index);
687                 if (rc)
688                         goto out;
689
690                 spin_unlock(&qi->q_lock);
691                 cpu_relax();
692                 spin_lock(&qi->q_lock);
693         }
694 out:
695         qi->desc_status[index] = qi->desc_status[wait_index] = QI_DONE;
696
697         reclaim_free_desc(qi);
698         spin_unlock_irqrestore(&qi->q_lock, flags);
699
700         return rc;
701 }
702
703 /*
704  * Flush the global interrupt entry cache.
705  */
706 void qi_global_iec(struct intel_iommu *iommu)
707 {
708         struct qi_desc desc;
709
710         desc.low = QI_IEC_TYPE;
711         desc.high = 0;
712
713         /* should never fail */
714         qi_submit_sync(&desc, iommu);
715 }
716
717 int qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
718                      u64 type, int non_present_entry_flush)
719 {
720         struct qi_desc desc;
721
722         if (non_present_entry_flush) {
723                 if (!cap_caching_mode(iommu->cap))
724                         return 1;
725                 else
726                         did = 0;
727         }
728
729         desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
730                         | QI_CC_GRAN(type) | QI_CC_TYPE;
731         desc.high = 0;
732
733         return qi_submit_sync(&desc, iommu);
734 }
735
736 int qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
737                    unsigned int size_order, u64 type,
738                    int non_present_entry_flush)
739 {
740         u8 dw = 0, dr = 0;
741
742         struct qi_desc desc;
743         int ih = 0;
744
745         if (non_present_entry_flush) {
746                 if (!cap_caching_mode(iommu->cap))
747                         return 1;
748                 else
749                         did = 0;
750         }
751
752         if (cap_write_drain(iommu->cap))
753                 dw = 1;
754
755         if (cap_read_drain(iommu->cap))
756                 dr = 1;
757
758         desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
759                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
760         desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
761                 | QI_IOTLB_AM(size_order);
762
763         return qi_submit_sync(&desc, iommu);
764 }
765
766 /*
767  * Disable Queued Invalidation interface.
768  */
769 void dmar_disable_qi(struct intel_iommu *iommu)
770 {
771         unsigned long flags;
772         u32 sts;
773         cycles_t start_time = get_cycles();
774
775         if (!ecap_qis(iommu->ecap))
776                 return;
777
778         spin_lock_irqsave(&iommu->register_lock, flags);
779
780         sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
781         if (!(sts & DMA_GSTS_QIES))
782                 goto end;
783
784         /*
785          * Give a chance to HW to complete the pending invalidation requests.
786          */
787         while ((readl(iommu->reg + DMAR_IQT_REG) !=
788                 readl(iommu->reg + DMAR_IQH_REG)) &&
789                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
790                 cpu_relax();
791
792         iommu->gcmd &= ~DMA_GCMD_QIE;
793
794         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
795
796         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
797                       !(sts & DMA_GSTS_QIES), sts);
798 end:
799         spin_unlock_irqrestore(&iommu->register_lock, flags);
800 }
801
802 /*
803  * Enable queued invalidation.
804  */
805 static void __dmar_enable_qi(struct intel_iommu *iommu)
806 {
807         u32 cmd, sts;
808         unsigned long flags;
809         struct q_inval *qi = iommu->qi;
810
811         qi->free_head = qi->free_tail = 0;
812         qi->free_cnt = QI_LENGTH;
813
814         spin_lock_irqsave(&iommu->register_lock, flags);
815
816         /* write zero to the tail reg */
817         writel(0, iommu->reg + DMAR_IQT_REG);
818
819         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
820
821         cmd = iommu->gcmd | DMA_GCMD_QIE;
822         iommu->gcmd |= DMA_GCMD_QIE;
823         writel(cmd, iommu->reg + DMAR_GCMD_REG);
824
825         /* Make sure hardware complete it */
826         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
827
828         spin_unlock_irqrestore(&iommu->register_lock, flags);
829 }
830
831 /*
832  * Enable Queued Invalidation interface. This is a must to support
833  * interrupt-remapping. Also used by DMA-remapping, which replaces
834  * register based IOTLB invalidation.
835  */
836 int dmar_enable_qi(struct intel_iommu *iommu)
837 {
838         struct q_inval *qi;
839
840         if (!ecap_qis(iommu->ecap))
841                 return -ENOENT;
842
843         /*
844          * queued invalidation is already setup and enabled.
845          */
846         if (iommu->qi)
847                 return 0;
848
849         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
850         if (!iommu->qi)
851                 return -ENOMEM;
852
853         qi = iommu->qi;
854
855         qi->desc = (void *)(get_zeroed_page(GFP_ATOMIC));
856         if (!qi->desc) {
857                 kfree(qi);
858                 iommu->qi = 0;
859                 return -ENOMEM;
860         }
861
862         qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
863         if (!qi->desc_status) {
864                 free_page((unsigned long) qi->desc);
865                 kfree(qi);
866                 iommu->qi = 0;
867                 return -ENOMEM;
868         }
869
870         qi->free_head = qi->free_tail = 0;
871         qi->free_cnt = QI_LENGTH;
872
873         spin_lock_init(&qi->q_lock);
874
875         __dmar_enable_qi(iommu);
876
877         return 0;
878 }
879
880 /* iommu interrupt handling. Most stuff are MSI-like. */
881
882 enum faulttype {
883         DMA_REMAP,
884         INTR_REMAP,
885         UNKNOWN,
886 };
887
888 static const char *dma_remap_fault_reasons[] =
889 {
890         "Software",
891         "Present bit in root entry is clear",
892         "Present bit in context entry is clear",
893         "Invalid context entry",
894         "Access beyond MGAW",
895         "PTE Write access is not set",
896         "PTE Read access is not set",
897         "Next page table ptr is invalid",
898         "Root table address invalid",
899         "Context table ptr is invalid",
900         "non-zero reserved fields in RTP",
901         "non-zero reserved fields in CTP",
902         "non-zero reserved fields in PTE",
903 };
904
905 static const char *intr_remap_fault_reasons[] =
906 {
907         "Detected reserved fields in the decoded interrupt-remapped request",
908         "Interrupt index exceeded the interrupt-remapping table size",
909         "Present field in the IRTE entry is clear",
910         "Error accessing interrupt-remapping table pointed by IRTA_REG",
911         "Detected reserved fields in the IRTE entry",
912         "Blocked a compatibility format interrupt request",
913         "Blocked an interrupt request due to source-id verification failure",
914 };
915
916 #define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
917
918 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
919 {
920         if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
921                                      ARRAY_SIZE(intr_remap_fault_reasons))) {
922                 *fault_type = INTR_REMAP;
923                 return intr_remap_fault_reasons[fault_reason - 0x20];
924         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
925                 *fault_type = DMA_REMAP;
926                 return dma_remap_fault_reasons[fault_reason];
927         } else {
928                 *fault_type = UNKNOWN;
929                 return "Unknown";
930         }
931 }
932
933 void dmar_msi_unmask(unsigned int irq)
934 {
935         struct intel_iommu *iommu = get_irq_data(irq);
936         unsigned long flag;
937
938         /* unmask it */
939         spin_lock_irqsave(&iommu->register_lock, flag);
940         writel(0, iommu->reg + DMAR_FECTL_REG);
941         /* Read a reg to force flush the post write */
942         readl(iommu->reg + DMAR_FECTL_REG);
943         spin_unlock_irqrestore(&iommu->register_lock, flag);
944 }
945
946 void dmar_msi_mask(unsigned int irq)
947 {
948         unsigned long flag;
949         struct intel_iommu *iommu = get_irq_data(irq);
950
951         /* mask it */
952         spin_lock_irqsave(&iommu->register_lock, flag);
953         writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
954         /* Read a reg to force flush the post write */
955         readl(iommu->reg + DMAR_FECTL_REG);
956         spin_unlock_irqrestore(&iommu->register_lock, flag);
957 }
958
959 void dmar_msi_write(int irq, struct msi_msg *msg)
960 {
961         struct intel_iommu *iommu = get_irq_data(irq);
962         unsigned long flag;
963
964         spin_lock_irqsave(&iommu->register_lock, flag);
965         writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
966         writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
967         writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
968         spin_unlock_irqrestore(&iommu->register_lock, flag);
969 }
970
971 void dmar_msi_read(int irq, struct msi_msg *msg)
972 {
973         struct intel_iommu *iommu = get_irq_data(irq);
974         unsigned long flag;
975
976         spin_lock_irqsave(&iommu->register_lock, flag);
977         msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
978         msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
979         msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
980         spin_unlock_irqrestore(&iommu->register_lock, flag);
981 }
982
983 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
984                 u8 fault_reason, u16 source_id, unsigned long long addr)
985 {
986         const char *reason;
987         int fault_type;
988
989         reason = dmar_get_fault_reason(fault_reason, &fault_type);
990
991         if (fault_type == INTR_REMAP)
992                 printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
993                        "fault index %llx\n"
994                         "INTR-REMAP:[fault reason %02d] %s\n",
995                         (source_id >> 8), PCI_SLOT(source_id & 0xFF),
996                         PCI_FUNC(source_id & 0xFF), addr >> 48,
997                         fault_reason, reason);
998         else
999                 printk(KERN_ERR
1000                        "DMAR:[%s] Request device [%02x:%02x.%d] "
1001                        "fault addr %llx \n"
1002                        "DMAR:[fault reason %02d] %s\n",
1003                        (type ? "DMA Read" : "DMA Write"),
1004                        (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1005                        PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1006         return 0;
1007 }
1008
1009 #define PRIMARY_FAULT_REG_LEN (16)
1010 irqreturn_t dmar_fault(int irq, void *dev_id)
1011 {
1012         struct intel_iommu *iommu = dev_id;
1013         int reg, fault_index;
1014         u32 fault_status;
1015         unsigned long flag;
1016
1017         spin_lock_irqsave(&iommu->register_lock, flag);
1018         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1019         if (fault_status)
1020                 printk(KERN_ERR "DRHD: handling fault status reg %x\n",
1021                        fault_status);
1022
1023         /* TBD: ignore advanced fault log currently */
1024         if (!(fault_status & DMA_FSTS_PPF))
1025                 goto clear_rest;
1026
1027         fault_index = dma_fsts_fault_record_index(fault_status);
1028         reg = cap_fault_reg_offset(iommu->cap);
1029         while (1) {
1030                 u8 fault_reason;
1031                 u16 source_id;
1032                 u64 guest_addr;
1033                 int type;
1034                 u32 data;
1035
1036                 /* highest 32 bits */
1037                 data = readl(iommu->reg + reg +
1038                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1039                 if (!(data & DMA_FRCD_F))
1040                         break;
1041
1042                 fault_reason = dma_frcd_fault_reason(data);
1043                 type = dma_frcd_type(data);
1044
1045                 data = readl(iommu->reg + reg +
1046                                 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1047                 source_id = dma_frcd_source_id(data);
1048
1049                 guest_addr = dmar_readq(iommu->reg + reg +
1050                                 fault_index * PRIMARY_FAULT_REG_LEN);
1051                 guest_addr = dma_frcd_page_addr(guest_addr);
1052                 /* clear the fault */
1053                 writel(DMA_FRCD_F, iommu->reg + reg +
1054                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1055
1056                 spin_unlock_irqrestore(&iommu->register_lock, flag);
1057
1058                 dmar_fault_do_one(iommu, type, fault_reason,
1059                                 source_id, guest_addr);
1060
1061                 fault_index++;
1062                 if (fault_index > cap_num_fault_regs(iommu->cap))
1063                         fault_index = 0;
1064                 spin_lock_irqsave(&iommu->register_lock, flag);
1065         }
1066 clear_rest:
1067         /* clear all the other faults */
1068         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1069         writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1070
1071         spin_unlock_irqrestore(&iommu->register_lock, flag);
1072         return IRQ_HANDLED;
1073 }
1074
1075 int dmar_set_interrupt(struct intel_iommu *iommu)
1076 {
1077         int irq, ret;
1078
1079         /*
1080          * Check if the fault interrupt is already initialized.
1081          */
1082         if (iommu->irq)
1083                 return 0;
1084
1085         irq = create_irq();
1086         if (!irq) {
1087                 printk(KERN_ERR "IOMMU: no free vectors\n");
1088                 return -EINVAL;
1089         }
1090
1091         set_irq_data(irq, iommu);
1092         iommu->irq = irq;
1093
1094         ret = arch_setup_dmar_msi(irq);
1095         if (ret) {
1096                 set_irq_data(irq, NULL);
1097                 iommu->irq = 0;
1098                 destroy_irq(irq);
1099                 return 0;
1100         }
1101
1102         ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
1103         if (ret)
1104                 printk(KERN_ERR "IOMMU: can't request irq\n");
1105         return ret;
1106 }
1107
1108 int __init enable_drhd_fault_handling(void)
1109 {
1110         struct dmar_drhd_unit *drhd;
1111
1112         /*
1113          * Enable fault control interrupt.
1114          */
1115         for_each_drhd_unit(drhd) {
1116                 int ret;
1117                 struct intel_iommu *iommu = drhd->iommu;
1118                 ret = dmar_set_interrupt(iommu);
1119
1120                 if (ret) {
1121                         printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1122                                " interrupt, ret %d\n",
1123                                (unsigned long long)drhd->reg_base_addr, ret);
1124                         return -1;
1125                 }
1126         }
1127
1128         return 0;
1129 }
1130
1131 /*
1132  * Re-enable Queued Invalidation interface.
1133  */
1134 int dmar_reenable_qi(struct intel_iommu *iommu)
1135 {
1136         if (!ecap_qis(iommu->ecap))
1137                 return -ENOENT;
1138
1139         if (!iommu->qi)
1140                 return -ENOENT;
1141
1142         /*
1143          * First disable queued invalidation.
1144          */
1145         dmar_disable_qi(iommu);
1146         /*
1147          * Then enable queued invalidation again. Since there is no pending
1148          * invalidation requests now, it's safe to re-enable queued
1149          * invalidation.
1150          */
1151         __dmar_enable_qi(iommu);
1152
1153         return 0;
1154 }