]> nv-tegra.nvidia Code Review - linux-3.10.git/blob - drivers/pci/dmar.c
25a00ce4f24d3f3a897c159f48d0b931200add54
[linux-3.10.git] / drivers / pci / dmar.c
1 /*
2  * Copyright (c) 2006, Intel Corporation.
3  *
4  * This program is free software; you can redistribute it and/or modify it
5  * under the terms and conditions of the GNU General Public License,
6  * version 2, as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope it will be useful, but WITHOUT
9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
11  * more details.
12  *
13  * You should have received a copy of the GNU General Public License along with
14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15  * Place - Suite 330, Boston, MA 02111-1307 USA.
16  *
17  * Copyright (C) 2006-2008 Intel Corporation
18  * Author: Ashok Raj <ashok.raj@intel.com>
19  * Author: Shaohua Li <shaohua.li@intel.com>
20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21  *
22  * This file implements early detection/parsing of Remapping Devices
23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24  * tables.
25  *
26  * These routines are used by both DMA-remapping and Interrupt-remapping
27  */
28
29 #include <linux/pci.h>
30 #include <linux/dmar.h>
31 #include <linux/iova.h>
32 #include <linux/intel-iommu.h>
33 #include <linux/timer.h>
34 #include <linux/irq.h>
35 #include <linux/interrupt.h>
36
37 #undef PREFIX
38 #define PREFIX "DMAR:"
39
40 /* No locks are needed as DMA remapping hardware unit
41  * list is constructed at boot time and hotplug of
42  * these units are not supported by the architecture.
43  */
44 LIST_HEAD(dmar_drhd_units);
45
46 static struct acpi_table_header * __initdata dmar_tbl;
47 static acpi_size dmar_tbl_size;
48
49 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
50 {
51         /*
52          * add INCLUDE_ALL at the tail, so scan the list will find it at
53          * the very end.
54          */
55         if (drhd->include_all)
56                 list_add_tail(&drhd->list, &dmar_drhd_units);
57         else
58                 list_add(&drhd->list, &dmar_drhd_units);
59 }
60
61 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
62                                            struct pci_dev **dev, u16 segment)
63 {
64         struct pci_bus *bus;
65         struct pci_dev *pdev = NULL;
66         struct acpi_dmar_pci_path *path;
67         int count;
68
69         bus = pci_find_bus(segment, scope->bus);
70         path = (struct acpi_dmar_pci_path *)(scope + 1);
71         count = (scope->length - sizeof(struct acpi_dmar_device_scope))
72                 / sizeof(struct acpi_dmar_pci_path);
73
74         while (count) {
75                 if (pdev)
76                         pci_dev_put(pdev);
77                 /*
78                  * Some BIOSes list non-exist devices in DMAR table, just
79                  * ignore it
80                  */
81                 if (!bus) {
82                         printk(KERN_WARNING
83                         PREFIX "Device scope bus [%d] not found\n",
84                         scope->bus);
85                         break;
86                 }
87                 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
88                 if (!pdev) {
89                         printk(KERN_WARNING PREFIX
90                         "Device scope device [%04x:%02x:%02x.%02x] not found\n",
91                                 segment, bus->number, path->dev, path->fn);
92                         break;
93                 }
94                 path ++;
95                 count --;
96                 bus = pdev->subordinate;
97         }
98         if (!pdev) {
99                 printk(KERN_WARNING PREFIX
100                 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
101                 segment, scope->bus, path->dev, path->fn);
102                 *dev = NULL;
103                 return 0;
104         }
105         if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
106                         pdev->subordinate) || (scope->entry_type == \
107                         ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
108                 pci_dev_put(pdev);
109                 printk(KERN_WARNING PREFIX
110                         "Device scope type does not match for %s\n",
111                          pci_name(pdev));
112                 return -EINVAL;
113         }
114         *dev = pdev;
115         return 0;
116 }
117
118 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
119                                        struct pci_dev ***devices, u16 segment)
120 {
121         struct acpi_dmar_device_scope *scope;
122         void * tmp = start;
123         int index;
124         int ret;
125
126         *cnt = 0;
127         while (start < end) {
128                 scope = start;
129                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
130                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
131                         (*cnt)++;
132                 else
133                         printk(KERN_WARNING PREFIX
134                                 "Unsupported device scope\n");
135                 start += scope->length;
136         }
137         if (*cnt == 0)
138                 return 0;
139
140         *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
141         if (!*devices)
142                 return -ENOMEM;
143
144         start = tmp;
145         index = 0;
146         while (start < end) {
147                 scope = start;
148                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
149                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
150                         ret = dmar_parse_one_dev_scope(scope,
151                                 &(*devices)[index], segment);
152                         if (ret) {
153                                 kfree(*devices);
154                                 return ret;
155                         }
156                         index ++;
157                 }
158                 start += scope->length;
159         }
160
161         return 0;
162 }
163
164 /**
165  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
166  * structure which uniquely represent one DMA remapping hardware unit
167  * present in the platform
168  */
169 static int __init
170 dmar_parse_one_drhd(struct acpi_dmar_header *header)
171 {
172         struct acpi_dmar_hardware_unit *drhd;
173         struct dmar_drhd_unit *dmaru;
174         int ret = 0;
175
176         dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
177         if (!dmaru)
178                 return -ENOMEM;
179
180         dmaru->hdr = header;
181         drhd = (struct acpi_dmar_hardware_unit *)header;
182         dmaru->reg_base_addr = drhd->address;
183         dmaru->segment = drhd->segment;
184         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
185
186         ret = alloc_iommu(dmaru);
187         if (ret) {
188                 kfree(dmaru);
189                 return ret;
190         }
191         dmar_register_drhd_unit(dmaru);
192         return 0;
193 }
194
195 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
196 {
197         struct acpi_dmar_hardware_unit *drhd;
198         int ret = 0;
199
200         drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
201
202         if (dmaru->include_all)
203                 return 0;
204
205         ret = dmar_parse_dev_scope((void *)(drhd + 1),
206                                 ((void *)drhd) + drhd->header.length,
207                                 &dmaru->devices_cnt, &dmaru->devices,
208                                 drhd->segment);
209         if (ret) {
210                 list_del(&dmaru->list);
211                 kfree(dmaru);
212         }
213         return ret;
214 }
215
216 #ifdef CONFIG_DMAR
217 LIST_HEAD(dmar_rmrr_units);
218
219 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
220 {
221         list_add(&rmrr->list, &dmar_rmrr_units);
222 }
223
224
225 static int __init
226 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
227 {
228         struct acpi_dmar_reserved_memory *rmrr;
229         struct dmar_rmrr_unit *rmrru;
230
231         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
232         if (!rmrru)
233                 return -ENOMEM;
234
235         rmrru->hdr = header;
236         rmrr = (struct acpi_dmar_reserved_memory *)header;
237         rmrru->base_address = rmrr->base_address;
238         rmrru->end_address = rmrr->end_address;
239
240         dmar_register_rmrr_unit(rmrru);
241         return 0;
242 }
243
244 static int __init
245 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
246 {
247         struct acpi_dmar_reserved_memory *rmrr;
248         int ret;
249
250         rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
251         ret = dmar_parse_dev_scope((void *)(rmrr + 1),
252                 ((void *)rmrr) + rmrr->header.length,
253                 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
254
255         if (ret || (rmrru->devices_cnt == 0)) {
256                 list_del(&rmrru->list);
257                 kfree(rmrru);
258         }
259         return ret;
260 }
261 #endif
262
263 static void __init
264 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
265 {
266         struct acpi_dmar_hardware_unit *drhd;
267         struct acpi_dmar_reserved_memory *rmrr;
268
269         switch (header->type) {
270         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
271                 drhd = (struct acpi_dmar_hardware_unit *)header;
272                 printk (KERN_INFO PREFIX
273                         "DRHD (flags: 0x%08x)base: 0x%016Lx\n",
274                         drhd->flags, (unsigned long long)drhd->address);
275                 break;
276         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
277                 rmrr = (struct acpi_dmar_reserved_memory *)header;
278
279                 printk (KERN_INFO PREFIX
280                         "RMRR base: 0x%016Lx end: 0x%016Lx\n",
281                         (unsigned long long)rmrr->base_address,
282                         (unsigned long long)rmrr->end_address);
283                 break;
284         }
285 }
286
287 /**
288  * dmar_table_detect - checks to see if the platform supports DMAR devices
289  */
290 static int __init dmar_table_detect(void)
291 {
292         acpi_status status = AE_OK;
293
294         /* if we could find DMAR table, then there are DMAR devices */
295         status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
296                                 (struct acpi_table_header **)&dmar_tbl,
297                                 &dmar_tbl_size);
298
299         if (ACPI_SUCCESS(status) && !dmar_tbl) {
300                 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
301                 status = AE_NOT_FOUND;
302         }
303
304         return (ACPI_SUCCESS(status) ? 1 : 0);
305 }
306
307 /**
308  * parse_dmar_table - parses the DMA reporting table
309  */
310 static int __init
311 parse_dmar_table(void)
312 {
313         struct acpi_table_dmar *dmar;
314         struct acpi_dmar_header *entry_header;
315         int ret = 0;
316
317         /*
318          * Do it again, earlier dmar_tbl mapping could be mapped with
319          * fixed map.
320          */
321         dmar_table_detect();
322
323         dmar = (struct acpi_table_dmar *)dmar_tbl;
324         if (!dmar)
325                 return -ENODEV;
326
327         if (dmar->width < PAGE_SHIFT - 1) {
328                 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
329                 return -EINVAL;
330         }
331
332         printk (KERN_INFO PREFIX "Host address width %d\n",
333                 dmar->width + 1);
334
335         entry_header = (struct acpi_dmar_header *)(dmar + 1);
336         while (((unsigned long)entry_header) <
337                         (((unsigned long)dmar) + dmar_tbl->length)) {
338                 /* Avoid looping forever on bad ACPI tables */
339                 if (entry_header->length == 0) {
340                         printk(KERN_WARNING PREFIX
341                                 "Invalid 0-length structure\n");
342                         ret = -EINVAL;
343                         break;
344                 }
345
346                 dmar_table_print_dmar_entry(entry_header);
347
348                 switch (entry_header->type) {
349                 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
350                         ret = dmar_parse_one_drhd(entry_header);
351                         break;
352                 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
353 #ifdef CONFIG_DMAR
354                         ret = dmar_parse_one_rmrr(entry_header);
355 #endif
356                         break;
357                 default:
358                         printk(KERN_WARNING PREFIX
359                                 "Unknown DMAR structure type\n");
360                         ret = 0; /* for forward compatibility */
361                         break;
362                 }
363                 if (ret)
364                         break;
365
366                 entry_header = ((void *)entry_header + entry_header->length);
367         }
368         return ret;
369 }
370
371 int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
372                           struct pci_dev *dev)
373 {
374         int index;
375
376         while (dev) {
377                 for (index = 0; index < cnt; index++)
378                         if (dev == devices[index])
379                                 return 1;
380
381                 /* Check our parent */
382                 dev = dev->bus->self;
383         }
384
385         return 0;
386 }
387
388 struct dmar_drhd_unit *
389 dmar_find_matched_drhd_unit(struct pci_dev *dev)
390 {
391         struct dmar_drhd_unit *dmaru = NULL;
392         struct acpi_dmar_hardware_unit *drhd;
393
394         list_for_each_entry(dmaru, &dmar_drhd_units, list) {
395                 drhd = container_of(dmaru->hdr,
396                                     struct acpi_dmar_hardware_unit,
397                                     header);
398
399                 if (dmaru->include_all &&
400                     drhd->segment == pci_domain_nr(dev->bus))
401                         return dmaru;
402
403                 if (dmar_pci_device_match(dmaru->devices,
404                                           dmaru->devices_cnt, dev))
405                         return dmaru;
406         }
407
408         return NULL;
409 }
410
411 int __init dmar_dev_scope_init(void)
412 {
413         struct dmar_drhd_unit *drhd, *drhd_n;
414         int ret = -ENODEV;
415
416         list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
417                 ret = dmar_parse_dev(drhd);
418                 if (ret)
419                         return ret;
420         }
421
422 #ifdef CONFIG_DMAR
423         {
424                 struct dmar_rmrr_unit *rmrr, *rmrr_n;
425                 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
426                         ret = rmrr_parse_dev(rmrr);
427                         if (ret)
428                                 return ret;
429                 }
430         }
431 #endif
432
433         return ret;
434 }
435
436
437 int __init dmar_table_init(void)
438 {
439         static int dmar_table_initialized;
440         int ret;
441
442         if (dmar_table_initialized)
443                 return 0;
444
445         dmar_table_initialized = 1;
446
447         ret = parse_dmar_table();
448         if (ret) {
449                 if (ret != -ENODEV)
450                         printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
451                 return ret;
452         }
453
454         if (list_empty(&dmar_drhd_units)) {
455                 printk(KERN_INFO PREFIX "No DMAR devices found\n");
456                 return -ENODEV;
457         }
458
459 #ifdef CONFIG_DMAR
460         if (list_empty(&dmar_rmrr_units))
461                 printk(KERN_INFO PREFIX "No RMRR found\n");
462 #endif
463
464 #ifdef CONFIG_INTR_REMAP
465         parse_ioapics_under_ir();
466 #endif
467         return 0;
468 }
469
470 void __init detect_intel_iommu(void)
471 {
472         int ret;
473
474         ret = dmar_table_detect();
475
476         {
477 #ifdef CONFIG_INTR_REMAP
478                 struct acpi_table_dmar *dmar;
479                 /*
480                  * for now we will disable dma-remapping when interrupt
481                  * remapping is enabled.
482                  * When support for queued invalidation for IOTLB invalidation
483                  * is added, we will not need this any more.
484                  */
485                 dmar = (struct acpi_table_dmar *) dmar_tbl;
486                 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
487                         printk(KERN_INFO
488                                "Queued invalidation will be enabled to support "
489                                "x2apic and Intr-remapping.\n");
490 #endif
491 #ifdef CONFIG_DMAR
492                 if (ret && !no_iommu && !iommu_detected && !swiotlb &&
493                     !dmar_disabled)
494                         iommu_detected = 1;
495 #endif
496         }
497         early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
498         dmar_tbl = NULL;
499 }
500
501
502 int alloc_iommu(struct dmar_drhd_unit *drhd)
503 {
504         struct intel_iommu *iommu;
505         int map_size;
506         u32 ver;
507         static int iommu_allocated = 0;
508         int agaw = 0;
509
510         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
511         if (!iommu)
512                 return -ENOMEM;
513
514         iommu->seq_id = iommu_allocated++;
515         sprintf (iommu->name, "dmar%d", iommu->seq_id);
516
517         iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
518         if (!iommu->reg) {
519                 printk(KERN_ERR "IOMMU: can't map the region\n");
520                 goto error;
521         }
522         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
523         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
524
525 #ifdef CONFIG_DMAR
526         agaw = iommu_calculate_agaw(iommu);
527         if (agaw < 0) {
528                 printk(KERN_ERR
529                         "Cannot get a valid agaw for iommu (seq_id = %d)\n",
530                         iommu->seq_id);
531                 goto error;
532         }
533 #endif
534         iommu->agaw = agaw;
535
536         /* the registers might be more than one page */
537         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
538                 cap_max_fault_reg_offset(iommu->cap));
539         map_size = VTD_PAGE_ALIGN(map_size);
540         if (map_size > VTD_PAGE_SIZE) {
541                 iounmap(iommu->reg);
542                 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
543                 if (!iommu->reg) {
544                         printk(KERN_ERR "IOMMU: can't map the region\n");
545                         goto error;
546                 }
547         }
548
549         ver = readl(iommu->reg + DMAR_VER_REG);
550         pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
551                 (unsigned long long)drhd->reg_base_addr,
552                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
553                 (unsigned long long)iommu->cap,
554                 (unsigned long long)iommu->ecap);
555
556         spin_lock_init(&iommu->register_lock);
557
558         drhd->iommu = iommu;
559         return 0;
560 error:
561         kfree(iommu);
562         return -1;
563 }
564
565 void free_iommu(struct intel_iommu *iommu)
566 {
567         if (!iommu)
568                 return;
569
570 #ifdef CONFIG_DMAR
571         free_dmar_iommu(iommu);
572 #endif
573
574         if (iommu->reg)
575                 iounmap(iommu->reg);
576         kfree(iommu);
577 }
578
579 /*
580  * Reclaim all the submitted descriptors which have completed its work.
581  */
582 static inline void reclaim_free_desc(struct q_inval *qi)
583 {
584         while (qi->desc_status[qi->free_tail] == QI_DONE) {
585                 qi->desc_status[qi->free_tail] = QI_FREE;
586                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
587                 qi->free_cnt++;
588         }
589 }
590
591 static int qi_check_fault(struct intel_iommu *iommu, int index)
592 {
593         u32 fault;
594         int head;
595         struct q_inval *qi = iommu->qi;
596         int wait_index = (index + 1) % QI_LENGTH;
597
598         fault = readl(iommu->reg + DMAR_FSTS_REG);
599
600         /*
601          * If IQE happens, the head points to the descriptor associated
602          * with the error. No new descriptors are fetched until the IQE
603          * is cleared.
604          */
605         if (fault & DMA_FSTS_IQE) {
606                 head = readl(iommu->reg + DMAR_IQH_REG);
607                 if ((head >> 4) == index) {
608                         memcpy(&qi->desc[index], &qi->desc[wait_index],
609                                         sizeof(struct qi_desc));
610                         __iommu_flush_cache(iommu, &qi->desc[index],
611                                         sizeof(struct qi_desc));
612                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
613                         return -EINVAL;
614                 }
615         }
616
617         return 0;
618 }
619
620 /*
621  * Submit the queued invalidation descriptor to the remapping
622  * hardware unit and wait for its completion.
623  */
624 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
625 {
626         int rc = 0;
627         struct q_inval *qi = iommu->qi;
628         struct qi_desc *hw, wait_desc;
629         int wait_index, index;
630         unsigned long flags;
631
632         if (!qi)
633                 return 0;
634
635         hw = qi->desc;
636
637         spin_lock_irqsave(&qi->q_lock, flags);
638         while (qi->free_cnt < 3) {
639                 spin_unlock_irqrestore(&qi->q_lock, flags);
640                 cpu_relax();
641                 spin_lock_irqsave(&qi->q_lock, flags);
642         }
643
644         index = qi->free_head;
645         wait_index = (index + 1) % QI_LENGTH;
646
647         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
648
649         hw[index] = *desc;
650
651         wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
652                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
653         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
654
655         hw[wait_index] = wait_desc;
656
657         __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
658         __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
659
660         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
661         qi->free_cnt -= 2;
662
663         /*
664          * update the HW tail register indicating the presence of
665          * new descriptors.
666          */
667         writel(qi->free_head << 4, iommu->reg + DMAR_IQT_REG);
668
669         while (qi->desc_status[wait_index] != QI_DONE) {
670                 /*
671                  * We will leave the interrupts disabled, to prevent interrupt
672                  * context to queue another cmd while a cmd is already submitted
673                  * and waiting for completion on this cpu. This is to avoid
674                  * a deadlock where the interrupt context can wait indefinitely
675                  * for free slots in the queue.
676                  */
677                 rc = qi_check_fault(iommu, index);
678                 if (rc)
679                         goto out;
680
681                 spin_unlock(&qi->q_lock);
682                 cpu_relax();
683                 spin_lock(&qi->q_lock);
684         }
685 out:
686         qi->desc_status[index] = qi->desc_status[wait_index] = QI_DONE;
687
688         reclaim_free_desc(qi);
689         spin_unlock_irqrestore(&qi->q_lock, flags);
690
691         return rc;
692 }
693
694 /*
695  * Flush the global interrupt entry cache.
696  */
697 void qi_global_iec(struct intel_iommu *iommu)
698 {
699         struct qi_desc desc;
700
701         desc.low = QI_IEC_TYPE;
702         desc.high = 0;
703
704         /* should never fail */
705         qi_submit_sync(&desc, iommu);
706 }
707
708 int qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
709                      u64 type, int non_present_entry_flush)
710 {
711         struct qi_desc desc;
712
713         if (non_present_entry_flush) {
714                 if (!cap_caching_mode(iommu->cap))
715                         return 1;
716                 else
717                         did = 0;
718         }
719
720         desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
721                         | QI_CC_GRAN(type) | QI_CC_TYPE;
722         desc.high = 0;
723
724         return qi_submit_sync(&desc, iommu);
725 }
726
727 int qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
728                    unsigned int size_order, u64 type,
729                    int non_present_entry_flush)
730 {
731         u8 dw = 0, dr = 0;
732
733         struct qi_desc desc;
734         int ih = 0;
735
736         if (non_present_entry_flush) {
737                 if (!cap_caching_mode(iommu->cap))
738                         return 1;
739                 else
740                         did = 0;
741         }
742
743         if (cap_write_drain(iommu->cap))
744                 dw = 1;
745
746         if (cap_read_drain(iommu->cap))
747                 dr = 1;
748
749         desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
750                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
751         desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
752                 | QI_IOTLB_AM(size_order);
753
754         return qi_submit_sync(&desc, iommu);
755 }
756
757 /*
758  * Disable Queued Invalidation interface.
759  */
760 void dmar_disable_qi(struct intel_iommu *iommu)
761 {
762         unsigned long flags;
763         u32 sts;
764         cycles_t start_time = get_cycles();
765
766         if (!ecap_qis(iommu->ecap))
767                 return;
768
769         spin_lock_irqsave(&iommu->register_lock, flags);
770
771         sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
772         if (!(sts & DMA_GSTS_QIES))
773                 goto end;
774
775         /*
776          * Give a chance to HW to complete the pending invalidation requests.
777          */
778         while ((readl(iommu->reg + DMAR_IQT_REG) !=
779                 readl(iommu->reg + DMAR_IQH_REG)) &&
780                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
781                 cpu_relax();
782
783         iommu->gcmd &= ~DMA_GCMD_QIE;
784
785         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
786
787         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
788                       !(sts & DMA_GSTS_QIES), sts);
789 end:
790         spin_unlock_irqrestore(&iommu->register_lock, flags);
791 }
792
793 /*
794  * Enable queued invalidation.
795  */
796 static void __dmar_enable_qi(struct intel_iommu *iommu)
797 {
798         u32 cmd, sts;
799         unsigned long flags;
800         struct q_inval *qi = iommu->qi;
801
802         qi->free_head = qi->free_tail = 0;
803         qi->free_cnt = QI_LENGTH;
804
805         spin_lock_irqsave(&iommu->register_lock, flags);
806
807         /* write zero to the tail reg */
808         writel(0, iommu->reg + DMAR_IQT_REG);
809
810         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
811
812         cmd = iommu->gcmd | DMA_GCMD_QIE;
813         iommu->gcmd |= DMA_GCMD_QIE;
814         writel(cmd, iommu->reg + DMAR_GCMD_REG);
815
816         /* Make sure hardware complete it */
817         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
818
819         spin_unlock_irqrestore(&iommu->register_lock, flags);
820 }
821
822 /*
823  * Enable Queued Invalidation interface. This is a must to support
824  * interrupt-remapping. Also used by DMA-remapping, which replaces
825  * register based IOTLB invalidation.
826  */
827 int dmar_enable_qi(struct intel_iommu *iommu)
828 {
829         struct q_inval *qi;
830
831         if (!ecap_qis(iommu->ecap))
832                 return -ENOENT;
833
834         /*
835          * queued invalidation is already setup and enabled.
836          */
837         if (iommu->qi)
838                 return 0;
839
840         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
841         if (!iommu->qi)
842                 return -ENOMEM;
843
844         qi = iommu->qi;
845
846         qi->desc = (void *)(get_zeroed_page(GFP_ATOMIC));
847         if (!qi->desc) {
848                 kfree(qi);
849                 iommu->qi = 0;
850                 return -ENOMEM;
851         }
852
853         qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
854         if (!qi->desc_status) {
855                 free_page((unsigned long) qi->desc);
856                 kfree(qi);
857                 iommu->qi = 0;
858                 return -ENOMEM;
859         }
860
861         qi->free_head = qi->free_tail = 0;
862         qi->free_cnt = QI_LENGTH;
863
864         spin_lock_init(&qi->q_lock);
865
866         __dmar_enable_qi(iommu);
867
868         return 0;
869 }
870
871 /* iommu interrupt handling. Most stuff are MSI-like. */
872
873 enum faulttype {
874         DMA_REMAP,
875         INTR_REMAP,
876         UNKNOWN,
877 };
878
879 static const char *dma_remap_fault_reasons[] =
880 {
881         "Software",
882         "Present bit in root entry is clear",
883         "Present bit in context entry is clear",
884         "Invalid context entry",
885         "Access beyond MGAW",
886         "PTE Write access is not set",
887         "PTE Read access is not set",
888         "Next page table ptr is invalid",
889         "Root table address invalid",
890         "Context table ptr is invalid",
891         "non-zero reserved fields in RTP",
892         "non-zero reserved fields in CTP",
893         "non-zero reserved fields in PTE",
894 };
895
896 static const char *intr_remap_fault_reasons[] =
897 {
898         "Detected reserved fields in the decoded interrupt-remapped request",
899         "Interrupt index exceeded the interrupt-remapping table size",
900         "Present field in the IRTE entry is clear",
901         "Error accessing interrupt-remapping table pointed by IRTA_REG",
902         "Detected reserved fields in the IRTE entry",
903         "Blocked a compatibility format interrupt request",
904         "Blocked an interrupt request due to source-id verification failure",
905 };
906
907 #define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
908
909 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
910 {
911         if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
912                                      ARRAY_SIZE(intr_remap_fault_reasons))) {
913                 *fault_type = INTR_REMAP;
914                 return intr_remap_fault_reasons[fault_reason - 0x20];
915         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
916                 *fault_type = DMA_REMAP;
917                 return dma_remap_fault_reasons[fault_reason];
918         } else {
919                 *fault_type = UNKNOWN;
920                 return "Unknown";
921         }
922 }
923
924 void dmar_msi_unmask(unsigned int irq)
925 {
926         struct intel_iommu *iommu = get_irq_data(irq);
927         unsigned long flag;
928
929         /* unmask it */
930         spin_lock_irqsave(&iommu->register_lock, flag);
931         writel(0, iommu->reg + DMAR_FECTL_REG);
932         /* Read a reg to force flush the post write */
933         readl(iommu->reg + DMAR_FECTL_REG);
934         spin_unlock_irqrestore(&iommu->register_lock, flag);
935 }
936
937 void dmar_msi_mask(unsigned int irq)
938 {
939         unsigned long flag;
940         struct intel_iommu *iommu = get_irq_data(irq);
941
942         /* mask it */
943         spin_lock_irqsave(&iommu->register_lock, flag);
944         writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
945         /* Read a reg to force flush the post write */
946         readl(iommu->reg + DMAR_FECTL_REG);
947         spin_unlock_irqrestore(&iommu->register_lock, flag);
948 }
949
950 void dmar_msi_write(int irq, struct msi_msg *msg)
951 {
952         struct intel_iommu *iommu = get_irq_data(irq);
953         unsigned long flag;
954
955         spin_lock_irqsave(&iommu->register_lock, flag);
956         writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
957         writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
958         writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
959         spin_unlock_irqrestore(&iommu->register_lock, flag);
960 }
961
962 void dmar_msi_read(int irq, struct msi_msg *msg)
963 {
964         struct intel_iommu *iommu = get_irq_data(irq);
965         unsigned long flag;
966
967         spin_lock_irqsave(&iommu->register_lock, flag);
968         msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
969         msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
970         msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
971         spin_unlock_irqrestore(&iommu->register_lock, flag);
972 }
973
974 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
975                 u8 fault_reason, u16 source_id, unsigned long long addr)
976 {
977         const char *reason;
978         int fault_type;
979
980         reason = dmar_get_fault_reason(fault_reason, &fault_type);
981
982         if (fault_type == INTR_REMAP)
983                 printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
984                        "fault index %llx\n"
985                         "INTR-REMAP:[fault reason %02d] %s\n",
986                         (source_id >> 8), PCI_SLOT(source_id & 0xFF),
987                         PCI_FUNC(source_id & 0xFF), addr >> 48,
988                         fault_reason, reason);
989         else
990                 printk(KERN_ERR
991                        "DMAR:[%s] Request device [%02x:%02x.%d] "
992                        "fault addr %llx \n"
993                        "DMAR:[fault reason %02d] %s\n",
994                        (type ? "DMA Read" : "DMA Write"),
995                        (source_id >> 8), PCI_SLOT(source_id & 0xFF),
996                        PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
997         return 0;
998 }
999
1000 #define PRIMARY_FAULT_REG_LEN (16)
1001 irqreturn_t dmar_fault(int irq, void *dev_id)
1002 {
1003         struct intel_iommu *iommu = dev_id;
1004         int reg, fault_index;
1005         u32 fault_status;
1006         unsigned long flag;
1007
1008         spin_lock_irqsave(&iommu->register_lock, flag);
1009         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1010         if (fault_status)
1011                 printk(KERN_ERR "DRHD: handling fault status reg %x\n",
1012                        fault_status);
1013
1014         /* TBD: ignore advanced fault log currently */
1015         if (!(fault_status & DMA_FSTS_PPF))
1016                 goto clear_rest;
1017
1018         fault_index = dma_fsts_fault_record_index(fault_status);
1019         reg = cap_fault_reg_offset(iommu->cap);
1020         while (1) {
1021                 u8 fault_reason;
1022                 u16 source_id;
1023                 u64 guest_addr;
1024                 int type;
1025                 u32 data;
1026
1027                 /* highest 32 bits */
1028                 data = readl(iommu->reg + reg +
1029                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1030                 if (!(data & DMA_FRCD_F))
1031                         break;
1032
1033                 fault_reason = dma_frcd_fault_reason(data);
1034                 type = dma_frcd_type(data);
1035
1036                 data = readl(iommu->reg + reg +
1037                                 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1038                 source_id = dma_frcd_source_id(data);
1039
1040                 guest_addr = dmar_readq(iommu->reg + reg +
1041                                 fault_index * PRIMARY_FAULT_REG_LEN);
1042                 guest_addr = dma_frcd_page_addr(guest_addr);
1043                 /* clear the fault */
1044                 writel(DMA_FRCD_F, iommu->reg + reg +
1045                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1046
1047                 spin_unlock_irqrestore(&iommu->register_lock, flag);
1048
1049                 dmar_fault_do_one(iommu, type, fault_reason,
1050                                 source_id, guest_addr);
1051
1052                 fault_index++;
1053                 if (fault_index > cap_num_fault_regs(iommu->cap))
1054                         fault_index = 0;
1055                 spin_lock_irqsave(&iommu->register_lock, flag);
1056         }
1057 clear_rest:
1058         /* clear all the other faults */
1059         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1060         writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1061
1062         spin_unlock_irqrestore(&iommu->register_lock, flag);
1063         return IRQ_HANDLED;
1064 }
1065
1066 int dmar_set_interrupt(struct intel_iommu *iommu)
1067 {
1068         int irq, ret;
1069
1070         /*
1071          * Check if the fault interrupt is already initialized.
1072          */
1073         if (iommu->irq)
1074                 return 0;
1075
1076         irq = create_irq();
1077         if (!irq) {
1078                 printk(KERN_ERR "IOMMU: no free vectors\n");
1079                 return -EINVAL;
1080         }
1081
1082         set_irq_data(irq, iommu);
1083         iommu->irq = irq;
1084
1085         ret = arch_setup_dmar_msi(irq);
1086         if (ret) {
1087                 set_irq_data(irq, NULL);
1088                 iommu->irq = 0;
1089                 destroy_irq(irq);
1090                 return 0;
1091         }
1092
1093         ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
1094         if (ret)
1095                 printk(KERN_ERR "IOMMU: can't request irq\n");
1096         return ret;
1097 }
1098
1099 int __init enable_drhd_fault_handling(void)
1100 {
1101         struct dmar_drhd_unit *drhd;
1102
1103         /*
1104          * Enable fault control interrupt.
1105          */
1106         for_each_drhd_unit(drhd) {
1107                 int ret;
1108                 struct intel_iommu *iommu = drhd->iommu;
1109                 ret = dmar_set_interrupt(iommu);
1110
1111                 if (ret) {
1112                         printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1113                                " interrupt, ret %d\n",
1114                                (unsigned long long)drhd->reg_base_addr, ret);
1115                         return -1;
1116                 }
1117         }
1118
1119         return 0;
1120 }
1121
1122 /*
1123  * Re-enable Queued Invalidation interface.
1124  */
1125 int dmar_reenable_qi(struct intel_iommu *iommu)
1126 {
1127         if (!ecap_qis(iommu->ecap))
1128                 return -ENOENT;
1129
1130         if (!iommu->qi)
1131                 return -ENOENT;
1132
1133         /*
1134          * First disable queued invalidation.
1135          */
1136         dmar_disable_qi(iommu);
1137         /*
1138          * Then enable queued invalidation again. Since there is no pending
1139          * invalidation requests now, it's safe to re-enable queued
1140          * invalidation.
1141          */
1142         __dmar_enable_qi(iommu);
1143
1144         return 0;
1145 }