SFI: remove unneeded includes
[linux-2.6.git] / drivers / pci / dmar.c
1 /*
2  * Copyright (c) 2006, Intel Corporation.
3  *
4  * This program is free software; you can redistribute it and/or modify it
5  * under the terms and conditions of the GNU General Public License,
6  * version 2, as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope it will be useful, but WITHOUT
9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
11  * more details.
12  *
13  * You should have received a copy of the GNU General Public License along with
14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15  * Place - Suite 330, Boston, MA 02111-1307 USA.
16  *
17  * Copyright (C) 2006-2008 Intel Corporation
18  * Author: Ashok Raj <ashok.raj@intel.com>
19  * Author: Shaohua Li <shaohua.li@intel.com>
20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21  *
22  * This file implements early detection/parsing of Remapping Devices
23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24  * tables.
25  *
26  * These routines are used by both DMA-remapping and Interrupt-remapping
27  */
28
29 #include <linux/pci.h>
30 #include <linux/dmar.h>
31 #include <linux/iova.h>
32 #include <linux/intel-iommu.h>
33 #include <linux/timer.h>
34 #include <linux/irq.h>
35 #include <linux/interrupt.h>
36
37 #define PREFIX "DMAR: "
38
39 /* No locks are needed as DMA remapping hardware unit
40  * list is constructed at boot time and hotplug of
41  * these units are not supported by the architecture.
42  */
43 LIST_HEAD(dmar_drhd_units);
44
45 static struct acpi_table_header * __initdata dmar_tbl;
46 static acpi_size dmar_tbl_size;
47
48 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
49 {
50         /*
51          * add INCLUDE_ALL at the tail, so scan the list will find it at
52          * the very end.
53          */
54         if (drhd->include_all)
55                 list_add_tail(&drhd->list, &dmar_drhd_units);
56         else
57                 list_add(&drhd->list, &dmar_drhd_units);
58 }
59
60 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
61                                            struct pci_dev **dev, u16 segment)
62 {
63         struct pci_bus *bus;
64         struct pci_dev *pdev = NULL;
65         struct acpi_dmar_pci_path *path;
66         int count;
67
68         bus = pci_find_bus(segment, scope->bus);
69         path = (struct acpi_dmar_pci_path *)(scope + 1);
70         count = (scope->length - sizeof(struct acpi_dmar_device_scope))
71                 / sizeof(struct acpi_dmar_pci_path);
72
73         while (count) {
74                 if (pdev)
75                         pci_dev_put(pdev);
76                 /*
77                  * Some BIOSes list non-exist devices in DMAR table, just
78                  * ignore it
79                  */
80                 if (!bus) {
81                         printk(KERN_WARNING
82                         PREFIX "Device scope bus [%d] not found\n",
83                         scope->bus);
84                         break;
85                 }
86                 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
87                 if (!pdev) {
88                         printk(KERN_WARNING PREFIX
89                         "Device scope device [%04x:%02x:%02x.%02x] not found\n",
90                                 segment, bus->number, path->dev, path->fn);
91                         break;
92                 }
93                 path ++;
94                 count --;
95                 bus = pdev->subordinate;
96         }
97         if (!pdev) {
98                 printk(KERN_WARNING PREFIX
99                 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
100                 segment, scope->bus, path->dev, path->fn);
101                 *dev = NULL;
102                 return 0;
103         }
104         if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
105                         pdev->subordinate) || (scope->entry_type == \
106                         ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
107                 pci_dev_put(pdev);
108                 printk(KERN_WARNING PREFIX
109                         "Device scope type does not match for %s\n",
110                          pci_name(pdev));
111                 return -EINVAL;
112         }
113         *dev = pdev;
114         return 0;
115 }
116
117 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
118                                        struct pci_dev ***devices, u16 segment)
119 {
120         struct acpi_dmar_device_scope *scope;
121         void * tmp = start;
122         int index;
123         int ret;
124
125         *cnt = 0;
126         while (start < end) {
127                 scope = start;
128                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
129                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
130                         (*cnt)++;
131                 else
132                         printk(KERN_WARNING PREFIX
133                                 "Unsupported device scope\n");
134                 start += scope->length;
135         }
136         if (*cnt == 0)
137                 return 0;
138
139         *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
140         if (!*devices)
141                 return -ENOMEM;
142
143         start = tmp;
144         index = 0;
145         while (start < end) {
146                 scope = start;
147                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
148                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
149                         ret = dmar_parse_one_dev_scope(scope,
150                                 &(*devices)[index], segment);
151                         if (ret) {
152                                 kfree(*devices);
153                                 return ret;
154                         }
155                         index ++;
156                 }
157                 start += scope->length;
158         }
159
160         return 0;
161 }
162
163 /**
164  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
165  * structure which uniquely represent one DMA remapping hardware unit
166  * present in the platform
167  */
168 static int __init
169 dmar_parse_one_drhd(struct acpi_dmar_header *header)
170 {
171         struct acpi_dmar_hardware_unit *drhd;
172         struct dmar_drhd_unit *dmaru;
173         int ret = 0;
174
175         drhd = (struct acpi_dmar_hardware_unit *)header;
176         if (!drhd->address) {
177                 /* Promote an attitude of violence to a BIOS engineer today */
178                 WARN(1, "Your BIOS is broken; DMAR reported at address zero!\n"
179                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
180                      dmi_get_system_info(DMI_BIOS_VENDOR),
181                      dmi_get_system_info(DMI_BIOS_VERSION),
182                      dmi_get_system_info(DMI_PRODUCT_VERSION));
183                 return -ENODEV;
184         }
185         dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
186         if (!dmaru)
187                 return -ENOMEM;
188
189         dmaru->hdr = header;
190         dmaru->reg_base_addr = drhd->address;
191         dmaru->segment = drhd->segment;
192         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
193
194         ret = alloc_iommu(dmaru);
195         if (ret) {
196                 kfree(dmaru);
197                 return ret;
198         }
199         dmar_register_drhd_unit(dmaru);
200         return 0;
201 }
202
203 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
204 {
205         struct acpi_dmar_hardware_unit *drhd;
206         int ret = 0;
207
208         drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
209
210         if (dmaru->include_all)
211                 return 0;
212
213         ret = dmar_parse_dev_scope((void *)(drhd + 1),
214                                 ((void *)drhd) + drhd->header.length,
215                                 &dmaru->devices_cnt, &dmaru->devices,
216                                 drhd->segment);
217         if (ret) {
218                 list_del(&dmaru->list);
219                 kfree(dmaru);
220         }
221         return ret;
222 }
223
224 #ifdef CONFIG_DMAR
225 LIST_HEAD(dmar_rmrr_units);
226
227 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
228 {
229         list_add(&rmrr->list, &dmar_rmrr_units);
230 }
231
232
233 static int __init
234 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
235 {
236         struct acpi_dmar_reserved_memory *rmrr;
237         struct dmar_rmrr_unit *rmrru;
238
239         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
240         if (!rmrru)
241                 return -ENOMEM;
242
243         rmrru->hdr = header;
244         rmrr = (struct acpi_dmar_reserved_memory *)header;
245         rmrru->base_address = rmrr->base_address;
246         rmrru->end_address = rmrr->end_address;
247
248         dmar_register_rmrr_unit(rmrru);
249         return 0;
250 }
251
252 static int __init
253 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
254 {
255         struct acpi_dmar_reserved_memory *rmrr;
256         int ret;
257
258         rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
259         ret = dmar_parse_dev_scope((void *)(rmrr + 1),
260                 ((void *)rmrr) + rmrr->header.length,
261                 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
262
263         if (ret || (rmrru->devices_cnt == 0)) {
264                 list_del(&rmrru->list);
265                 kfree(rmrru);
266         }
267         return ret;
268 }
269
270 static LIST_HEAD(dmar_atsr_units);
271
272 static int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
273 {
274         struct acpi_dmar_atsr *atsr;
275         struct dmar_atsr_unit *atsru;
276
277         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
278         atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
279         if (!atsru)
280                 return -ENOMEM;
281
282         atsru->hdr = hdr;
283         atsru->include_all = atsr->flags & 0x1;
284
285         list_add(&atsru->list, &dmar_atsr_units);
286
287         return 0;
288 }
289
290 static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
291 {
292         int rc;
293         struct acpi_dmar_atsr *atsr;
294
295         if (atsru->include_all)
296                 return 0;
297
298         atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
299         rc = dmar_parse_dev_scope((void *)(atsr + 1),
300                                 (void *)atsr + atsr->header.length,
301                                 &atsru->devices_cnt, &atsru->devices,
302                                 atsr->segment);
303         if (rc || !atsru->devices_cnt) {
304                 list_del(&atsru->list);
305                 kfree(atsru);
306         }
307
308         return rc;
309 }
310
311 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
312 {
313         int i;
314         struct pci_bus *bus;
315         struct acpi_dmar_atsr *atsr;
316         struct dmar_atsr_unit *atsru;
317
318         list_for_each_entry(atsru, &dmar_atsr_units, list) {
319                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
320                 if (atsr->segment == pci_domain_nr(dev->bus))
321                         goto found;
322         }
323
324         return 0;
325
326 found:
327         for (bus = dev->bus; bus; bus = bus->parent) {
328                 struct pci_dev *bridge = bus->self;
329
330                 if (!bridge || !bridge->is_pcie ||
331                     bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
332                         return 0;
333
334                 if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) {
335                         for (i = 0; i < atsru->devices_cnt; i++)
336                                 if (atsru->devices[i] == bridge)
337                                         return 1;
338                         break;
339                 }
340         }
341
342         if (atsru->include_all)
343                 return 1;
344
345         return 0;
346 }
347 #endif
348
349 static void __init
350 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
351 {
352         struct acpi_dmar_hardware_unit *drhd;
353         struct acpi_dmar_reserved_memory *rmrr;
354         struct acpi_dmar_atsr *atsr;
355
356         switch (header->type) {
357         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
358                 drhd = container_of(header, struct acpi_dmar_hardware_unit,
359                                     header);
360                 printk (KERN_INFO PREFIX
361                         "DRHD base: %#016Lx flags: %#x\n",
362                         (unsigned long long)drhd->address, drhd->flags);
363                 break;
364         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
365                 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
366                                     header);
367                 printk (KERN_INFO PREFIX
368                         "RMRR base: %#016Lx end: %#016Lx\n",
369                         (unsigned long long)rmrr->base_address,
370                         (unsigned long long)rmrr->end_address);
371                 break;
372         case ACPI_DMAR_TYPE_ATSR:
373                 atsr = container_of(header, struct acpi_dmar_atsr, header);
374                 printk(KERN_INFO PREFIX "ATSR flags: %#x\n", atsr->flags);
375                 break;
376         }
377 }
378
379 /**
380  * dmar_table_detect - checks to see if the platform supports DMAR devices
381  */
382 static int __init dmar_table_detect(void)
383 {
384         acpi_status status = AE_OK;
385
386         /* if we could find DMAR table, then there are DMAR devices */
387         status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
388                                 (struct acpi_table_header **)&dmar_tbl,
389                                 &dmar_tbl_size);
390
391         if (ACPI_SUCCESS(status) && !dmar_tbl) {
392                 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
393                 status = AE_NOT_FOUND;
394         }
395
396         return (ACPI_SUCCESS(status) ? 1 : 0);
397 }
398
399 /**
400  * parse_dmar_table - parses the DMA reporting table
401  */
402 static int __init
403 parse_dmar_table(void)
404 {
405         struct acpi_table_dmar *dmar;
406         struct acpi_dmar_header *entry_header;
407         int ret = 0;
408
409         /*
410          * Do it again, earlier dmar_tbl mapping could be mapped with
411          * fixed map.
412          */
413         dmar_table_detect();
414
415         dmar = (struct acpi_table_dmar *)dmar_tbl;
416         if (!dmar)
417                 return -ENODEV;
418
419         if (dmar->width < PAGE_SHIFT - 1) {
420                 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
421                 return -EINVAL;
422         }
423
424         printk (KERN_INFO PREFIX "Host address width %d\n",
425                 dmar->width + 1);
426
427         entry_header = (struct acpi_dmar_header *)(dmar + 1);
428         while (((unsigned long)entry_header) <
429                         (((unsigned long)dmar) + dmar_tbl->length)) {
430                 /* Avoid looping forever on bad ACPI tables */
431                 if (entry_header->length == 0) {
432                         printk(KERN_WARNING PREFIX
433                                 "Invalid 0-length structure\n");
434                         ret = -EINVAL;
435                         break;
436                 }
437
438                 dmar_table_print_dmar_entry(entry_header);
439
440                 switch (entry_header->type) {
441                 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
442                         ret = dmar_parse_one_drhd(entry_header);
443                         break;
444                 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
445 #ifdef CONFIG_DMAR
446                         ret = dmar_parse_one_rmrr(entry_header);
447 #endif
448                         break;
449                 case ACPI_DMAR_TYPE_ATSR:
450 #ifdef CONFIG_DMAR
451                         ret = dmar_parse_one_atsr(entry_header);
452 #endif
453                         break;
454                 default:
455                         printk(KERN_WARNING PREFIX
456                                 "Unknown DMAR structure type\n");
457                         ret = 0; /* for forward compatibility */
458                         break;
459                 }
460                 if (ret)
461                         break;
462
463                 entry_header = ((void *)entry_header + entry_header->length);
464         }
465         return ret;
466 }
467
468 int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
469                           struct pci_dev *dev)
470 {
471         int index;
472
473         while (dev) {
474                 for (index = 0; index < cnt; index++)
475                         if (dev == devices[index])
476                                 return 1;
477
478                 /* Check our parent */
479                 dev = dev->bus->self;
480         }
481
482         return 0;
483 }
484
485 struct dmar_drhd_unit *
486 dmar_find_matched_drhd_unit(struct pci_dev *dev)
487 {
488         struct dmar_drhd_unit *dmaru = NULL;
489         struct acpi_dmar_hardware_unit *drhd;
490
491         list_for_each_entry(dmaru, &dmar_drhd_units, list) {
492                 drhd = container_of(dmaru->hdr,
493                                     struct acpi_dmar_hardware_unit,
494                                     header);
495
496                 if (dmaru->include_all &&
497                     drhd->segment == pci_domain_nr(dev->bus))
498                         return dmaru;
499
500                 if (dmar_pci_device_match(dmaru->devices,
501                                           dmaru->devices_cnt, dev))
502                         return dmaru;
503         }
504
505         return NULL;
506 }
507
508 int __init dmar_dev_scope_init(void)
509 {
510         struct dmar_drhd_unit *drhd, *drhd_n;
511         int ret = -ENODEV;
512
513         list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
514                 ret = dmar_parse_dev(drhd);
515                 if (ret)
516                         return ret;
517         }
518
519 #ifdef CONFIG_DMAR
520         {
521                 struct dmar_rmrr_unit *rmrr, *rmrr_n;
522                 struct dmar_atsr_unit *atsr, *atsr_n;
523
524                 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
525                         ret = rmrr_parse_dev(rmrr);
526                         if (ret)
527                                 return ret;
528                 }
529
530                 list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
531                         ret = atsr_parse_dev(atsr);
532                         if (ret)
533                                 return ret;
534                 }
535         }
536 #endif
537
538         return ret;
539 }
540
541
542 int __init dmar_table_init(void)
543 {
544         static int dmar_table_initialized;
545         int ret;
546
547         if (dmar_table_initialized)
548                 return 0;
549
550         dmar_table_initialized = 1;
551
552         ret = parse_dmar_table();
553         if (ret) {
554                 if (ret != -ENODEV)
555                         printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
556                 return ret;
557         }
558
559         if (list_empty(&dmar_drhd_units)) {
560                 printk(KERN_INFO PREFIX "No DMAR devices found\n");
561                 return -ENODEV;
562         }
563
564 #ifdef CONFIG_DMAR
565         if (list_empty(&dmar_rmrr_units))
566                 printk(KERN_INFO PREFIX "No RMRR found\n");
567
568         if (list_empty(&dmar_atsr_units))
569                 printk(KERN_INFO PREFIX "No ATSR found\n");
570 #endif
571
572 #ifdef CONFIG_INTR_REMAP
573         parse_ioapics_under_ir();
574 #endif
575         return 0;
576 }
577
578 void __init detect_intel_iommu(void)
579 {
580         int ret;
581
582         ret = dmar_table_detect();
583
584         {
585 #ifdef CONFIG_INTR_REMAP
586                 struct acpi_table_dmar *dmar;
587                 /*
588                  * for now we will disable dma-remapping when interrupt
589                  * remapping is enabled.
590                  * When support for queued invalidation for IOTLB invalidation
591                  * is added, we will not need this any more.
592                  */
593                 dmar = (struct acpi_table_dmar *) dmar_tbl;
594                 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
595                         printk(KERN_INFO
596                                "Queued invalidation will be enabled to support "
597                                "x2apic and Intr-remapping.\n");
598 #endif
599 #ifdef CONFIG_DMAR
600                 if (ret && !no_iommu && !iommu_detected && !swiotlb &&
601                     !dmar_disabled)
602                         iommu_detected = 1;
603 #endif
604         }
605         early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
606         dmar_tbl = NULL;
607 }
608
609
610 int alloc_iommu(struct dmar_drhd_unit *drhd)
611 {
612         struct intel_iommu *iommu;
613         int map_size;
614         u32 ver;
615         static int iommu_allocated = 0;
616         int agaw = 0;
617         int msagaw = 0;
618
619         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
620         if (!iommu)
621                 return -ENOMEM;
622
623         iommu->seq_id = iommu_allocated++;
624         sprintf (iommu->name, "dmar%d", iommu->seq_id);
625
626         iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
627         if (!iommu->reg) {
628                 printk(KERN_ERR "IOMMU: can't map the region\n");
629                 goto error;
630         }
631         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
632         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
633
634 #ifdef CONFIG_DMAR
635         agaw = iommu_calculate_agaw(iommu);
636         if (agaw < 0) {
637                 printk(KERN_ERR
638                        "Cannot get a valid agaw for iommu (seq_id = %d)\n",
639                        iommu->seq_id);
640                 goto error;
641         }
642         msagaw = iommu_calculate_max_sagaw(iommu);
643         if (msagaw < 0) {
644                 printk(KERN_ERR
645                         "Cannot get a valid max agaw for iommu (seq_id = %d)\n",
646                         iommu->seq_id);
647                 goto error;
648         }
649 #endif
650         iommu->agaw = agaw;
651         iommu->msagaw = msagaw;
652
653         /* the registers might be more than one page */
654         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
655                 cap_max_fault_reg_offset(iommu->cap));
656         map_size = VTD_PAGE_ALIGN(map_size);
657         if (map_size > VTD_PAGE_SIZE) {
658                 iounmap(iommu->reg);
659                 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
660                 if (!iommu->reg) {
661                         printk(KERN_ERR "IOMMU: can't map the region\n");
662                         goto error;
663                 }
664         }
665
666         ver = readl(iommu->reg + DMAR_VER_REG);
667         pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
668                 (unsigned long long)drhd->reg_base_addr,
669                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
670                 (unsigned long long)iommu->cap,
671                 (unsigned long long)iommu->ecap);
672
673         spin_lock_init(&iommu->register_lock);
674
675         drhd->iommu = iommu;
676         return 0;
677 error:
678         kfree(iommu);
679         return -1;
680 }
681
682 void free_iommu(struct intel_iommu *iommu)
683 {
684         if (!iommu)
685                 return;
686
687 #ifdef CONFIG_DMAR
688         free_dmar_iommu(iommu);
689 #endif
690
691         if (iommu->reg)
692                 iounmap(iommu->reg);
693         kfree(iommu);
694 }
695
696 /*
697  * Reclaim all the submitted descriptors which have completed its work.
698  */
699 static inline void reclaim_free_desc(struct q_inval *qi)
700 {
701         while (qi->desc_status[qi->free_tail] == QI_DONE ||
702                qi->desc_status[qi->free_tail] == QI_ABORT) {
703                 qi->desc_status[qi->free_tail] = QI_FREE;
704                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
705                 qi->free_cnt++;
706         }
707 }
708
709 static int qi_check_fault(struct intel_iommu *iommu, int index)
710 {
711         u32 fault;
712         int head, tail;
713         struct q_inval *qi = iommu->qi;
714         int wait_index = (index + 1) % QI_LENGTH;
715
716         if (qi->desc_status[wait_index] == QI_ABORT)
717                 return -EAGAIN;
718
719         fault = readl(iommu->reg + DMAR_FSTS_REG);
720
721         /*
722          * If IQE happens, the head points to the descriptor associated
723          * with the error. No new descriptors are fetched until the IQE
724          * is cleared.
725          */
726         if (fault & DMA_FSTS_IQE) {
727                 head = readl(iommu->reg + DMAR_IQH_REG);
728                 if ((head >> DMAR_IQ_SHIFT) == index) {
729                         printk(KERN_ERR "VT-d detected invalid descriptor: "
730                                 "low=%llx, high=%llx\n",
731                                 (unsigned long long)qi->desc[index].low,
732                                 (unsigned long long)qi->desc[index].high);
733                         memcpy(&qi->desc[index], &qi->desc[wait_index],
734                                         sizeof(struct qi_desc));
735                         __iommu_flush_cache(iommu, &qi->desc[index],
736                                         sizeof(struct qi_desc));
737                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
738                         return -EINVAL;
739                 }
740         }
741
742         /*
743          * If ITE happens, all pending wait_desc commands are aborted.
744          * No new descriptors are fetched until the ITE is cleared.
745          */
746         if (fault & DMA_FSTS_ITE) {
747                 head = readl(iommu->reg + DMAR_IQH_REG);
748                 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
749                 head |= 1;
750                 tail = readl(iommu->reg + DMAR_IQT_REG);
751                 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
752
753                 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
754
755                 do {
756                         if (qi->desc_status[head] == QI_IN_USE)
757                                 qi->desc_status[head] = QI_ABORT;
758                         head = (head - 2 + QI_LENGTH) % QI_LENGTH;
759                 } while (head != tail);
760
761                 if (qi->desc_status[wait_index] == QI_ABORT)
762                         return -EAGAIN;
763         }
764
765         if (fault & DMA_FSTS_ICE)
766                 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
767
768         return 0;
769 }
770
771 /*
772  * Submit the queued invalidation descriptor to the remapping
773  * hardware unit and wait for its completion.
774  */
775 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
776 {
777         int rc;
778         struct q_inval *qi = iommu->qi;
779         struct qi_desc *hw, wait_desc;
780         int wait_index, index;
781         unsigned long flags;
782
783         if (!qi)
784                 return 0;
785
786         hw = qi->desc;
787
788 restart:
789         rc = 0;
790
791         spin_lock_irqsave(&qi->q_lock, flags);
792         while (qi->free_cnt < 3) {
793                 spin_unlock_irqrestore(&qi->q_lock, flags);
794                 cpu_relax();
795                 spin_lock_irqsave(&qi->q_lock, flags);
796         }
797
798         index = qi->free_head;
799         wait_index = (index + 1) % QI_LENGTH;
800
801         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
802
803         hw[index] = *desc;
804
805         wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
806                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
807         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
808
809         hw[wait_index] = wait_desc;
810
811         __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
812         __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
813
814         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
815         qi->free_cnt -= 2;
816
817         /*
818          * update the HW tail register indicating the presence of
819          * new descriptors.
820          */
821         writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
822
823         while (qi->desc_status[wait_index] != QI_DONE) {
824                 /*
825                  * We will leave the interrupts disabled, to prevent interrupt
826                  * context to queue another cmd while a cmd is already submitted
827                  * and waiting for completion on this cpu. This is to avoid
828                  * a deadlock where the interrupt context can wait indefinitely
829                  * for free slots in the queue.
830                  */
831                 rc = qi_check_fault(iommu, index);
832                 if (rc)
833                         break;
834
835                 spin_unlock(&qi->q_lock);
836                 cpu_relax();
837                 spin_lock(&qi->q_lock);
838         }
839
840         qi->desc_status[index] = QI_DONE;
841
842         reclaim_free_desc(qi);
843         spin_unlock_irqrestore(&qi->q_lock, flags);
844
845         if (rc == -EAGAIN)
846                 goto restart;
847
848         return rc;
849 }
850
851 /*
852  * Flush the global interrupt entry cache.
853  */
854 void qi_global_iec(struct intel_iommu *iommu)
855 {
856         struct qi_desc desc;
857
858         desc.low = QI_IEC_TYPE;
859         desc.high = 0;
860
861         /* should never fail */
862         qi_submit_sync(&desc, iommu);
863 }
864
865 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
866                       u64 type)
867 {
868         struct qi_desc desc;
869
870         desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
871                         | QI_CC_GRAN(type) | QI_CC_TYPE;
872         desc.high = 0;
873
874         qi_submit_sync(&desc, iommu);
875 }
876
877 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
878                     unsigned int size_order, u64 type)
879 {
880         u8 dw = 0, dr = 0;
881
882         struct qi_desc desc;
883         int ih = 0;
884
885         if (cap_write_drain(iommu->cap))
886                 dw = 1;
887
888         if (cap_read_drain(iommu->cap))
889                 dr = 1;
890
891         desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
892                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
893         desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
894                 | QI_IOTLB_AM(size_order);
895
896         qi_submit_sync(&desc, iommu);
897 }
898
899 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
900                         u64 addr, unsigned mask)
901 {
902         struct qi_desc desc;
903
904         if (mask) {
905                 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
906                 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
907                 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
908         } else
909                 desc.high = QI_DEV_IOTLB_ADDR(addr);
910
911         if (qdep >= QI_DEV_IOTLB_MAX_INVS)
912                 qdep = 0;
913
914         desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
915                    QI_DIOTLB_TYPE;
916
917         qi_submit_sync(&desc, iommu);
918 }
919
920 /*
921  * Disable Queued Invalidation interface.
922  */
923 void dmar_disable_qi(struct intel_iommu *iommu)
924 {
925         unsigned long flags;
926         u32 sts;
927         cycles_t start_time = get_cycles();
928
929         if (!ecap_qis(iommu->ecap))
930                 return;
931
932         spin_lock_irqsave(&iommu->register_lock, flags);
933
934         sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
935         if (!(sts & DMA_GSTS_QIES))
936                 goto end;
937
938         /*
939          * Give a chance to HW to complete the pending invalidation requests.
940          */
941         while ((readl(iommu->reg + DMAR_IQT_REG) !=
942                 readl(iommu->reg + DMAR_IQH_REG)) &&
943                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
944                 cpu_relax();
945
946         iommu->gcmd &= ~DMA_GCMD_QIE;
947         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
948
949         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
950                       !(sts & DMA_GSTS_QIES), sts);
951 end:
952         spin_unlock_irqrestore(&iommu->register_lock, flags);
953 }
954
955 /*
956  * Enable queued invalidation.
957  */
958 static void __dmar_enable_qi(struct intel_iommu *iommu)
959 {
960         u32 sts;
961         unsigned long flags;
962         struct q_inval *qi = iommu->qi;
963
964         qi->free_head = qi->free_tail = 0;
965         qi->free_cnt = QI_LENGTH;
966
967         spin_lock_irqsave(&iommu->register_lock, flags);
968
969         /* write zero to the tail reg */
970         writel(0, iommu->reg + DMAR_IQT_REG);
971
972         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
973
974         iommu->gcmd |= DMA_GCMD_QIE;
975         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
976
977         /* Make sure hardware complete it */
978         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
979
980         spin_unlock_irqrestore(&iommu->register_lock, flags);
981 }
982
983 /*
984  * Enable Queued Invalidation interface. This is a must to support
985  * interrupt-remapping. Also used by DMA-remapping, which replaces
986  * register based IOTLB invalidation.
987  */
988 int dmar_enable_qi(struct intel_iommu *iommu)
989 {
990         struct q_inval *qi;
991
992         if (!ecap_qis(iommu->ecap))
993                 return -ENOENT;
994
995         /*
996          * queued invalidation is already setup and enabled.
997          */
998         if (iommu->qi)
999                 return 0;
1000
1001         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1002         if (!iommu->qi)
1003                 return -ENOMEM;
1004
1005         qi = iommu->qi;
1006
1007         qi->desc = (void *)(get_zeroed_page(GFP_ATOMIC));
1008         if (!qi->desc) {
1009                 kfree(qi);
1010                 iommu->qi = 0;
1011                 return -ENOMEM;
1012         }
1013
1014         qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1015         if (!qi->desc_status) {
1016                 free_page((unsigned long) qi->desc);
1017                 kfree(qi);
1018                 iommu->qi = 0;
1019                 return -ENOMEM;
1020         }
1021
1022         qi->free_head = qi->free_tail = 0;
1023         qi->free_cnt = QI_LENGTH;
1024
1025         spin_lock_init(&qi->q_lock);
1026
1027         __dmar_enable_qi(iommu);
1028
1029         return 0;
1030 }
1031
1032 /* iommu interrupt handling. Most stuff are MSI-like. */
1033
1034 enum faulttype {
1035         DMA_REMAP,
1036         INTR_REMAP,
1037         UNKNOWN,
1038 };
1039
1040 static const char *dma_remap_fault_reasons[] =
1041 {
1042         "Software",
1043         "Present bit in root entry is clear",
1044         "Present bit in context entry is clear",
1045         "Invalid context entry",
1046         "Access beyond MGAW",
1047         "PTE Write access is not set",
1048         "PTE Read access is not set",
1049         "Next page table ptr is invalid",
1050         "Root table address invalid",
1051         "Context table ptr is invalid",
1052         "non-zero reserved fields in RTP",
1053         "non-zero reserved fields in CTP",
1054         "non-zero reserved fields in PTE",
1055 };
1056
1057 static const char *intr_remap_fault_reasons[] =
1058 {
1059         "Detected reserved fields in the decoded interrupt-remapped request",
1060         "Interrupt index exceeded the interrupt-remapping table size",
1061         "Present field in the IRTE entry is clear",
1062         "Error accessing interrupt-remapping table pointed by IRTA_REG",
1063         "Detected reserved fields in the IRTE entry",
1064         "Blocked a compatibility format interrupt request",
1065         "Blocked an interrupt request due to source-id verification failure",
1066 };
1067
1068 #define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
1069
1070 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1071 {
1072         if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
1073                                      ARRAY_SIZE(intr_remap_fault_reasons))) {
1074                 *fault_type = INTR_REMAP;
1075                 return intr_remap_fault_reasons[fault_reason - 0x20];
1076         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1077                 *fault_type = DMA_REMAP;
1078                 return dma_remap_fault_reasons[fault_reason];
1079         } else {
1080                 *fault_type = UNKNOWN;
1081                 return "Unknown";
1082         }
1083 }
1084
1085 void dmar_msi_unmask(unsigned int irq)
1086 {
1087         struct intel_iommu *iommu = get_irq_data(irq);
1088         unsigned long flag;
1089
1090         /* unmask it */
1091         spin_lock_irqsave(&iommu->register_lock, flag);
1092         writel(0, iommu->reg + DMAR_FECTL_REG);
1093         /* Read a reg to force flush the post write */
1094         readl(iommu->reg + DMAR_FECTL_REG);
1095         spin_unlock_irqrestore(&iommu->register_lock, flag);
1096 }
1097
1098 void dmar_msi_mask(unsigned int irq)
1099 {
1100         unsigned long flag;
1101         struct intel_iommu *iommu = get_irq_data(irq);
1102
1103         /* mask it */
1104         spin_lock_irqsave(&iommu->register_lock, flag);
1105         writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1106         /* Read a reg to force flush the post write */
1107         readl(iommu->reg + DMAR_FECTL_REG);
1108         spin_unlock_irqrestore(&iommu->register_lock, flag);
1109 }
1110
1111 void dmar_msi_write(int irq, struct msi_msg *msg)
1112 {
1113         struct intel_iommu *iommu = get_irq_data(irq);
1114         unsigned long flag;
1115
1116         spin_lock_irqsave(&iommu->register_lock, flag);
1117         writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1118         writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1119         writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1120         spin_unlock_irqrestore(&iommu->register_lock, flag);
1121 }
1122
1123 void dmar_msi_read(int irq, struct msi_msg *msg)
1124 {
1125         struct intel_iommu *iommu = get_irq_data(irq);
1126         unsigned long flag;
1127
1128         spin_lock_irqsave(&iommu->register_lock, flag);
1129         msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1130         msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1131         msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1132         spin_unlock_irqrestore(&iommu->register_lock, flag);
1133 }
1134
1135 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1136                 u8 fault_reason, u16 source_id, unsigned long long addr)
1137 {
1138         const char *reason;
1139         int fault_type;
1140
1141         reason = dmar_get_fault_reason(fault_reason, &fault_type);
1142
1143         if (fault_type == INTR_REMAP)
1144                 printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
1145                        "fault index %llx\n"
1146                         "INTR-REMAP:[fault reason %02d] %s\n",
1147                         (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1148                         PCI_FUNC(source_id & 0xFF), addr >> 48,
1149                         fault_reason, reason);
1150         else
1151                 printk(KERN_ERR
1152                        "DMAR:[%s] Request device [%02x:%02x.%d] "
1153                        "fault addr %llx \n"
1154                        "DMAR:[fault reason %02d] %s\n",
1155                        (type ? "DMA Read" : "DMA Write"),
1156                        (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1157                        PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1158         return 0;
1159 }
1160
1161 #define PRIMARY_FAULT_REG_LEN (16)
1162 irqreturn_t dmar_fault(int irq, void *dev_id)
1163 {
1164         struct intel_iommu *iommu = dev_id;
1165         int reg, fault_index;
1166         u32 fault_status;
1167         unsigned long flag;
1168
1169         spin_lock_irqsave(&iommu->register_lock, flag);
1170         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1171         if (fault_status)
1172                 printk(KERN_ERR "DRHD: handling fault status reg %x\n",
1173                        fault_status);
1174
1175         /* TBD: ignore advanced fault log currently */
1176         if (!(fault_status & DMA_FSTS_PPF))
1177                 goto clear_rest;
1178
1179         fault_index = dma_fsts_fault_record_index(fault_status);
1180         reg = cap_fault_reg_offset(iommu->cap);
1181         while (1) {
1182                 u8 fault_reason;
1183                 u16 source_id;
1184                 u64 guest_addr;
1185                 int type;
1186                 u32 data;
1187
1188                 /* highest 32 bits */
1189                 data = readl(iommu->reg + reg +
1190                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1191                 if (!(data & DMA_FRCD_F))
1192                         break;
1193
1194                 fault_reason = dma_frcd_fault_reason(data);
1195                 type = dma_frcd_type(data);
1196
1197                 data = readl(iommu->reg + reg +
1198                                 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1199                 source_id = dma_frcd_source_id(data);
1200
1201                 guest_addr = dmar_readq(iommu->reg + reg +
1202                                 fault_index * PRIMARY_FAULT_REG_LEN);
1203                 guest_addr = dma_frcd_page_addr(guest_addr);
1204                 /* clear the fault */
1205                 writel(DMA_FRCD_F, iommu->reg + reg +
1206                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1207
1208                 spin_unlock_irqrestore(&iommu->register_lock, flag);
1209
1210                 dmar_fault_do_one(iommu, type, fault_reason,
1211                                 source_id, guest_addr);
1212
1213                 fault_index++;
1214                 if (fault_index > cap_num_fault_regs(iommu->cap))
1215                         fault_index = 0;
1216                 spin_lock_irqsave(&iommu->register_lock, flag);
1217         }
1218 clear_rest:
1219         /* clear all the other faults */
1220         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1221         writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1222
1223         spin_unlock_irqrestore(&iommu->register_lock, flag);
1224         return IRQ_HANDLED;
1225 }
1226
1227 int dmar_set_interrupt(struct intel_iommu *iommu)
1228 {
1229         int irq, ret;
1230
1231         /*
1232          * Check if the fault interrupt is already initialized.
1233          */
1234         if (iommu->irq)
1235                 return 0;
1236
1237         irq = create_irq();
1238         if (!irq) {
1239                 printk(KERN_ERR "IOMMU: no free vectors\n");
1240                 return -EINVAL;
1241         }
1242
1243         set_irq_data(irq, iommu);
1244         iommu->irq = irq;
1245
1246         ret = arch_setup_dmar_msi(irq);
1247         if (ret) {
1248                 set_irq_data(irq, NULL);
1249                 iommu->irq = 0;
1250                 destroy_irq(irq);
1251                 return ret;
1252         }
1253
1254         ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
1255         if (ret)
1256                 printk(KERN_ERR "IOMMU: can't request irq\n");
1257         return ret;
1258 }
1259
1260 int __init enable_drhd_fault_handling(void)
1261 {
1262         struct dmar_drhd_unit *drhd;
1263
1264         /*
1265          * Enable fault control interrupt.
1266          */
1267         for_each_drhd_unit(drhd) {
1268                 int ret;
1269                 struct intel_iommu *iommu = drhd->iommu;
1270                 ret = dmar_set_interrupt(iommu);
1271
1272                 if (ret) {
1273                         printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1274                                " interrupt, ret %d\n",
1275                                (unsigned long long)drhd->reg_base_addr, ret);
1276                         return -1;
1277                 }
1278         }
1279
1280         return 0;
1281 }
1282
1283 /*
1284  * Re-enable Queued Invalidation interface.
1285  */
1286 int dmar_reenable_qi(struct intel_iommu *iommu)
1287 {
1288         if (!ecap_qis(iommu->ecap))
1289                 return -ENOENT;
1290
1291         if (!iommu->qi)
1292                 return -ENOENT;
1293
1294         /*
1295          * First disable queued invalidation.
1296          */
1297         dmar_disable_qi(iommu);
1298         /*
1299          * Then enable queued invalidation again. Since there is no pending
1300          * invalidation requests now, it's safe to re-enable queued
1301          * invalidation.
1302          */
1303         __dmar_enable_qi(iommu);
1304
1305         return 0;
1306 }