drivers/pci/dmar.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  *
  22  * This file implements early detection/parsing of Remapping Devices
  23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24  * tables.
  25  *
  26  * These routines are used by both DMA-remapping and Interrupt-remapping
  27  */
  28
  29 #include <linux/pci.h>
  30 #include <linux/dmar.h>
  31 #include <linux/iova.h>
  32 #include <linux/intel-iommu.h>
  33 #include <linux/timer.h>
  34 #include <linux/irq.h>
  35 #include <linux/interrupt.h>
  36
  37 #undef PREFIX
  38 #define PREFIX "DMAR:"
  39
  40 /* No locks are needed as DMA remapping hardware unit
  41  * list is constructed at boot time and hotplug of
  42  * these units are not supported by the architecture.
  43  */
  44 LIST_HEAD(dmar_drhd_units);
  45
  46 static struct acpi_table_header * __initdata dmar_tbl;
  47 static acpi_size dmar_tbl_size;
  48
  49 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  50 {
  51         /*
  52          * add INCLUDE_ALL at the tail, so scan the list will find it at
  53          * the very end.
  54          */
  55         if (drhd->include_all)
  56                 list_add_tail(&drhd->list, &dmar_drhd_units);
  57         else
  58                 list_add(&drhd->list, &dmar_drhd_units);
  59 }
  60
  61 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
  62                                            struct pci_dev **dev, u16 segment)
  63 {
  64         struct pci_bus *bus;
  65         struct pci_dev *pdev = NULL;
  66         struct acpi_dmar_pci_path *path;
  67         int count;
  68
  69         bus = pci_find_bus(segment, scope->bus);
  70         path = (struct acpi_dmar_pci_path *)(scope + 1);
  71         count = (scope->length - sizeof(struct acpi_dmar_device_scope))
  72                 / sizeof(struct acpi_dmar_pci_path);
  73
  74         while (count) {
  75                 if (pdev)
  76                         pci_dev_put(pdev);
  77                 /*
  78                  * Some BIOSes list non-exist devices in DMAR table, just
  79                  * ignore it
  80                  */
  81                 if (!bus) {
  82                         printk(KERN_WARNING
  83                         PREFIX "Device scope bus [%d] not found\n",
  84                         scope->bus);
  85                         break;
  86                 }
  87                 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
  88                 if (!pdev) {
  89                         printk(KERN_WARNING PREFIX
  90                         "Device scope device [%04x:%02x:%02x.%02x] not found\n",
  91                                 segment, bus->number, path->dev, path->fn);
  92                         break;
  93                 }
  94                 path ++;
  95                 count --;
  96                 bus = pdev->subordinate;
  97         }
  98         if (!pdev) {
  99                 printk(KERN_WARNING PREFIX
 100                 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
 101                 segment, scope->bus, path->dev, path->fn);
 102                 *dev = NULL;
 103                 return 0;
 104         }
 105         if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
 106                         pdev->subordinate) || (scope->entry_type == \
 107                         ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
 108                 pci_dev_put(pdev);
 109                 printk(KERN_WARNING PREFIX
 110                         "Device scope type does not match for %s\n",
 111                          pci_name(pdev));
 112                 return -EINVAL;
 113         }
 114         *dev = pdev;
 115         return 0;
 116 }
 117
 118 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
 119                                        struct pci_dev ***devices, u16 segment)
 120 {
 121         struct acpi_dmar_device_scope *scope;
 122         void * tmp = start;
 123         int index;
 124         int ret;
 125
 126         *cnt = 0;
 127         while (start < end) {
 128                 scope = start;
 129                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 130                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 131                         (*cnt)++;
 132                 else
 133                         printk(KERN_WARNING PREFIX
 134                                 "Unsupported device scope\n");
 135                 start += scope->length;
 136         }
 137         if (*cnt == 0)
 138                 return 0;
 139
 140         *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
 141         if (!*devices)
 142                 return -ENOMEM;
 143
 144         start = tmp;
 145         index = 0;
 146         while (start < end) {
 147                 scope = start;
 148                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 149                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
 150                         ret = dmar_parse_one_dev_scope(scope,
 151                                 &(*devices)[index], segment);
 152                         if (ret) {
 153                                 kfree(*devices);
 154                                 return ret;
 155                         }
 156                         index ++;
 157                 }
 158                 start += scope->length;
 159         }
 160
 161         return 0;
 162 }
 163
 164 /**
 165  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 166  * structure which uniquely represent one DMA remapping hardware unit
 167  * present in the platform
 168  */
 169 static int __init
 170 dmar_parse_one_drhd(struct acpi_dmar_header *header)
 171 {
 172         struct acpi_dmar_hardware_unit *drhd;
 173         struct dmar_drhd_unit *dmaru;
 174         int ret = 0;
 175
 176         dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
 177         if (!dmaru)
 178                 return -ENOMEM;
 179
 180         dmaru->hdr = header;
 181         drhd = (struct acpi_dmar_hardware_unit *)header;
 182         dmaru->reg_base_addr = drhd->address;
 183         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 184
 185         ret = alloc_iommu(dmaru);
 186         if (ret) {
 187                 kfree(dmaru);
 188                 return ret;
 189         }
 190         dmar_register_drhd_unit(dmaru);
 191         return 0;
 192 }
 193
 194 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
 195 {
 196         struct acpi_dmar_hardware_unit *drhd;
 197         int ret = 0;
 198
 199         drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
 200
 201         if (dmaru->include_all)
 202                 return 0;
 203
 204         ret = dmar_parse_dev_scope((void *)(drhd + 1),
 205                                 ((void *)drhd) + drhd->header.length,
 206                                 &dmaru->devices_cnt, &dmaru->devices,
 207                                 drhd->segment);
 208         if (ret) {
 209                 list_del(&dmaru->list);
 210                 kfree(dmaru);
 211         }
 212         return ret;
 213 }
 214
 215 #ifdef CONFIG_DMAR
 216 LIST_HEAD(dmar_rmrr_units);
 217
 218 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
 219 {
 220         list_add(&rmrr->list, &dmar_rmrr_units);
 221 }
 222
 223
 224 static int __init
 225 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
 226 {
 227         struct acpi_dmar_reserved_memory *rmrr;
 228         struct dmar_rmrr_unit *rmrru;
 229
 230         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
 231         if (!rmrru)
 232                 return -ENOMEM;
 233
 234         rmrru->hdr = header;
 235         rmrr = (struct acpi_dmar_reserved_memory *)header;
 236         rmrru->base_address = rmrr->base_address;
 237         rmrru->end_address = rmrr->end_address;
 238
 239         dmar_register_rmrr_unit(rmrru);
 240         return 0;
 241 }
 242
 243 static int __init
 244 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
 245 {
 246         struct acpi_dmar_reserved_memory *rmrr;
 247         int ret;
 248
 249         rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
 250         ret = dmar_parse_dev_scope((void *)(rmrr + 1),
 251                 ((void *)rmrr) + rmrr->header.length,
 252                 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
 253
 254         if (ret || (rmrru->devices_cnt == 0)) {
 255                 list_del(&rmrru->list);
 256                 kfree(rmrru);
 257         }
 258         return ret;
 259 }
 260 #endif
 261
 262 static void __init
 263 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 264 {
 265         struct acpi_dmar_hardware_unit *drhd;
 266         struct acpi_dmar_reserved_memory *rmrr;
 267
 268         switch (header->type) {
 269         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 270                 drhd = (struct acpi_dmar_hardware_unit *)header;
 271                 printk (KERN_INFO PREFIX
 272                         "DRHD (flags: 0x%08x)base: 0x%016Lx\n",
 273                         drhd->flags, (unsigned long long)drhd->address);
 274                 break;
 275         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 276                 rmrr = (struct acpi_dmar_reserved_memory *)header;
 277
 278                 printk (KERN_INFO PREFIX
 279                         "RMRR base: 0x%016Lx end: 0x%016Lx\n",
 280                         (unsigned long long)rmrr->base_address,
 281                         (unsigned long long)rmrr->end_address);
 282                 break;
 283         }
 284 }
 285
 286 /**
 287  * dmar_table_detect - checks to see if the platform supports DMAR devices
 288  */
 289 static int __init dmar_table_detect(void)
 290 {
 291         acpi_status status = AE_OK;
 292
 293         /* if we could find DMAR table, then there are DMAR devices */
 294         status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
 295                                 (struct acpi_table_header **)&dmar_tbl,
 296                                 &dmar_tbl_size);
 297
 298         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 299                 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
 300                 status = AE_NOT_FOUND;
 301         }
 302
 303         return (ACPI_SUCCESS(status) ? 1 : 0);
 304 }
 305
 306 /**
 307  * parse_dmar_table - parses the DMA reporting table
 308  */
 309 static int __init
 310 parse_dmar_table(void)
 311 {
 312         struct acpi_table_dmar *dmar;
 313         struct acpi_dmar_header *entry_header;
 314         int ret = 0;
 315
 316         /*
 317          * Do it again, earlier dmar_tbl mapping could be mapped with
 318          * fixed map.
 319          */
 320         dmar_table_detect();
 321
 322         dmar = (struct acpi_table_dmar *)dmar_tbl;
 323         if (!dmar)
 324                 return -ENODEV;
 325
 326         if (dmar->width < PAGE_SHIFT - 1) {
 327                 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
 328                 return -EINVAL;
 329         }
 330
 331         printk (KERN_INFO PREFIX "Host address width %d\n",
 332                 dmar->width + 1);
 333
 334         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 335         while (((unsigned long)entry_header) <
 336                         (((unsigned long)dmar) + dmar_tbl->length)) {
 337                 /* Avoid looping forever on bad ACPI tables */
 338                 if (entry_header->length == 0) {
 339                         printk(KERN_WARNING PREFIX
 340                                 "Invalid 0-length structure\n");
 341                         ret = -EINVAL;
 342                         break;
 343                 }
 344
 345                 dmar_table_print_dmar_entry(entry_header);
 346
 347                 switch (entry_header->type) {
 348                 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 349                         ret = dmar_parse_one_drhd(entry_header);
 350                         break;
 351                 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 352 #ifdef CONFIG_DMAR
 353                         ret = dmar_parse_one_rmrr(entry_header);
 354 #endif
 355                         break;
 356                 default:
 357                         printk(KERN_WARNING PREFIX
 358                                 "Unknown DMAR structure type\n");
 359                         ret = 0; /* for forward compatibility */
 360                         break;
 361                 }
 362                 if (ret)
 363                         break;
 364
 365                 entry_header = ((void *)entry_header + entry_header->length);
 366         }
 367         return ret;
 368 }
 369
 370 int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
 371                           struct pci_dev *dev)
 372 {
 373         int index;
 374
 375         while (dev) {
 376                 for (index = 0; index < cnt; index++)
 377                         if (dev == devices[index])
 378                                 return 1;
 379
 380                 /* Check our parent */
 381                 dev = dev->bus->self;
 382         }
 383
 384         return 0;
 385 }
 386
 387 struct dmar_drhd_unit *
 388 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 389 {
 390         struct dmar_drhd_unit *dmaru = NULL;
 391         struct acpi_dmar_hardware_unit *drhd;
 392
 393         list_for_each_entry(dmaru, &dmar_drhd_units, list) {
 394                 drhd = container_of(dmaru->hdr,
 395                                     struct acpi_dmar_hardware_unit,
 396                                     header);
 397
 398                 if (dmaru->include_all &&
 399                     drhd->segment == pci_domain_nr(dev->bus))
 400                         return dmaru;
 401
 402                 if (dmar_pci_device_match(dmaru->devices,
 403                                           dmaru->devices_cnt, dev))
 404                         return dmaru;
 405         }
 406
 407         return NULL;
 408 }
 409
 410 int __init dmar_dev_scope_init(void)
 411 {
 412         struct dmar_drhd_unit *drhd, *drhd_n;
 413         int ret = -ENODEV;
 414
 415         list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
 416                 ret = dmar_parse_dev(drhd);
 417                 if (ret)
 418                         return ret;
 419         }
 420
 421 #ifdef CONFIG_DMAR
 422         {
 423                 struct dmar_rmrr_unit *rmrr, *rmrr_n;
 424                 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
 425                         ret = rmrr_parse_dev(rmrr);
 426                         if (ret)
 427                                 return ret;
 428                 }
 429         }
 430 #endif
 431
 432         return ret;
 433 }
 434
 435
 436 int __init dmar_table_init(void)
 437 {
 438         static int dmar_table_initialized;
 439         int ret;
 440
 441         if (dmar_table_initialized)
 442                 return 0;
 443
 444         dmar_table_initialized = 1;
 445
 446         ret = parse_dmar_table();
 447         if (ret) {
 448                 if (ret != -ENODEV)
 449                         printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
 450                 return ret;
 451         }
 452
 453         if (list_empty(&dmar_drhd_units)) {
 454                 printk(KERN_INFO PREFIX "No DMAR devices found\n");
 455                 return -ENODEV;
 456         }
 457
 458 #ifdef CONFIG_DMAR
 459         if (list_empty(&dmar_rmrr_units))
 460                 printk(KERN_INFO PREFIX "No RMRR found\n");
 461 #endif
 462
 463 #ifdef CONFIG_INTR_REMAP
 464         parse_ioapics_under_ir();
 465 #endif
 466         return 0;
 467 }
 468
 469 void __init detect_intel_iommu(void)
 470 {
 471         int ret;
 472
 473         ret = dmar_table_detect();
 474
 475         {
 476 #ifdef CONFIG_INTR_REMAP
 477                 struct acpi_table_dmar *dmar;
 478                 /*
 479                  * for now we will disable dma-remapping when interrupt
 480                  * remapping is enabled.
 481                  * When support for queued invalidation for IOTLB invalidation
 482                  * is added, we will not need this any more.
 483                  */
 484                 dmar = (struct acpi_table_dmar *) dmar_tbl;
 485                 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
 486                         printk(KERN_INFO
 487                                "Queued invalidation will be enabled to support "
 488                                "x2apic and Intr-remapping.\n");
 489 #endif
 490 #ifdef CONFIG_DMAR
 491                 if (ret && !no_iommu && !iommu_detected && !swiotlb &&
 492                     !dmar_disabled)
 493                         iommu_detected = 1;
 494 #endif
 495         }
 496         early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
 497         dmar_tbl = NULL;
 498 }
 499
 500
 501 int alloc_iommu(struct dmar_drhd_unit *drhd)
 502 {
 503         struct intel_iommu *iommu;
 504         int map_size;
 505         u32 ver;
 506         static int iommu_allocated = 0;
 507         int agaw = 0;
 508
 509         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 510         if (!iommu)
 511                 return -ENOMEM;
 512
 513         iommu->seq_id = iommu_allocated++;
 514         sprintf (iommu->name, "dmar%d", iommu->seq_id);
 515
 516         iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
 517         if (!iommu->reg) {
 518                 printk(KERN_ERR "IOMMU: can't map the region\n");
 519                 goto error;
 520         }
 521         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 522         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 523
 524 #ifdef CONFIG_DMAR
 525         agaw = iommu_calculate_agaw(iommu);
 526         if (agaw < 0) {
 527                 printk(KERN_ERR
 528                         "Cannot get a valid agaw for iommu (seq_id = %d)\n",
 529                         iommu->seq_id);
 530                 goto error;
 531         }
 532 #endif
 533         iommu->agaw = agaw;
 534
 535         /* the registers might be more than one page */
 536         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 537                 cap_max_fault_reg_offset(iommu->cap));
 538         map_size = VTD_PAGE_ALIGN(map_size);
 539         if (map_size > VTD_PAGE_SIZE) {
 540                 iounmap(iommu->reg);
 541                 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
 542                 if (!iommu->reg) {
 543                         printk(KERN_ERR "IOMMU: can't map the region\n");
 544                         goto error;
 545                 }
 546         }
 547
 548         ver = readl(iommu->reg + DMAR_VER_REG);
 549         pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
 550                 (unsigned long long)drhd->reg_base_addr,
 551                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
 552                 (unsigned long long)iommu->cap,
 553                 (unsigned long long)iommu->ecap);
 554
 555         spin_lock_init(&iommu->register_lock);
 556
 557         drhd->iommu = iommu;
 558         return 0;
 559 error:
 560         kfree(iommu);
 561         return -1;
 562 }
 563
 564 void free_iommu(struct intel_iommu *iommu)
 565 {
 566         if (!iommu)
 567                 return;
 568
 569 #ifdef CONFIG_DMAR
 570         free_dmar_iommu(iommu);
 571 #endif
 572
 573         if (iommu->reg)
 574                 iounmap(iommu->reg);
 575         kfree(iommu);
 576 }
 577
 578 /*
 579  * Reclaim all the submitted descriptors which have completed its work.
 580  */
 581 static inline void reclaim_free_desc(struct q_inval *qi)
 582 {
 583         while (qi->desc_status[qi->free_tail] == QI_DONE) {
 584                 qi->desc_status[qi->free_tail] = QI_FREE;
 585                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
 586                 qi->free_cnt++;
 587         }
 588 }
 589
 590 static int qi_check_fault(struct intel_iommu *iommu, int index)
 591 {
 592         u32 fault;
 593         int head;
 594         struct q_inval *qi = iommu->qi;
 595         int wait_index = (index + 1) % QI_LENGTH;
 596
 597         fault = readl(iommu->reg + DMAR_FSTS_REG);
 598
 599         /*
 600          * If IQE happens, the head points to the descriptor associated
 601          * with the error. No new descriptors are fetched until the IQE
 602          * is cleared.
 603          */
 604         if (fault & DMA_FSTS_IQE) {
 605                 head = readl(iommu->reg + DMAR_IQH_REG);
 606                 if ((head >> 4) == index) {
 607                         memcpy(&qi->desc[index], &qi->desc[wait_index],
 608                                         sizeof(struct qi_desc));
 609                         __iommu_flush_cache(iommu, &qi->desc[index],
 610                                         sizeof(struct qi_desc));
 611                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
 612                         return -EINVAL;
 613                 }
 614         }
 615
 616         return 0;
 617 }
 618
 619 /*
 620  * Submit the queued invalidation descriptor to the remapping
 621  * hardware unit and wait for its completion.
 622  */
 623 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
 624 {
 625         int rc = 0;
 626         struct q_inval *qi = iommu->qi;
 627         struct qi_desc *hw, wait_desc;
 628         int wait_index, index;
 629         unsigned long flags;
 630
 631         if (!qi)
 632                 return 0;
 633
 634         hw = qi->desc;
 635
 636         spin_lock_irqsave(&qi->q_lock, flags);
 637         while (qi->free_cnt < 3) {
 638                 spin_unlock_irqrestore(&qi->q_lock, flags);
 639                 cpu_relax();
 640                 spin_lock_irqsave(&qi->q_lock, flags);
 641         }
 642
 643         index = qi->free_head;
 644         wait_index = (index + 1) % QI_LENGTH;
 645
 646         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
 647
 648         hw[index] = *desc;
 649
 650         wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
 651                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
 652         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
 653
 654         hw[wait_index] = wait_desc;
 655
 656         __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
 657         __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
 658
 659         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
 660         qi->free_cnt -= 2;
 661
 662         /*
 663          * update the HW tail register indicating the presence of
 664          * new descriptors.
 665          */
 666         writel(qi->free_head << 4, iommu->reg + DMAR_IQT_REG);
 667
 668         while (qi->desc_status[wait_index] != QI_DONE) {
 669                 /*
 670                  * We will leave the interrupts disabled, to prevent interrupt
 671                  * context to queue another cmd while a cmd is already submitted
 672                  * and waiting for completion on this cpu. This is to avoid
 673                  * a deadlock where the interrupt context can wait indefinitely
 674                  * for free slots in the queue.
 675                  */
 676                 rc = qi_check_fault(iommu, index);
 677                 if (rc)
 678                         goto out;
 679
 680                 spin_unlock(&qi->q_lock);
 681                 cpu_relax();
 682                 spin_lock(&qi->q_lock);
 683         }
 684 out:
 685         qi->desc_status[index] = qi->desc_status[wait_index] = QI_DONE;
 686
 687         reclaim_free_desc(qi);
 688         spin_unlock_irqrestore(&qi->q_lock, flags);
 689
 690         return rc;
 691 }
 692
 693 /*
 694  * Flush the global interrupt entry cache.
 695  */
 696 void qi_global_iec(struct intel_iommu *iommu)
 697 {
 698         struct qi_desc desc;
 699
 700         desc.low = QI_IEC_TYPE;
 701         desc.high = 0;
 702
 703         /* should never fail */
 704         qi_submit_sync(&desc, iommu);
 705 }
 706
 707 int qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
 708                      u64 type, int non_present_entry_flush)
 709 {
 710         struct qi_desc desc;
 711
 712         if (non_present_entry_flush) {
 713                 if (!cap_caching_mode(iommu->cap))
 714                         return 1;
 715                 else
 716                         did = 0;
 717         }
 718
 719         desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
 720                         | QI_CC_GRAN(type) | QI_CC_TYPE;
 721         desc.high = 0;
 722
 723         return qi_submit_sync(&desc, iommu);
 724 }
 725
 726 int qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
 727                    unsigned int size_order, u64 type,
 728                    int non_present_entry_flush)
 729 {
 730         u8 dw = 0, dr = 0;
 731
 732         struct qi_desc desc;
 733         int ih = 0;
 734
 735         if (non_present_entry_flush) {
 736                 if (!cap_caching_mode(iommu->cap))
 737                         return 1;
 738                 else
 739                         did = 0;
 740         }
 741
 742         if (cap_write_drain(iommu->cap))
 743                 dw = 1;
 744
 745         if (cap_read_drain(iommu->cap))
 746                 dr = 1;
 747
 748         desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
 749                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
 750         desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
 751                 | QI_IOTLB_AM(size_order);
 752
 753         return qi_submit_sync(&desc, iommu);
 754 }
 755
 756 /*
 757  * Disable Queued Invalidation interface.
 758  */
 759 void dmar_disable_qi(struct intel_iommu *iommu)
 760 {
 761         unsigned long flags;
 762         u32 sts;
 763         cycles_t start_time = get_cycles();
 764
 765         if (!ecap_qis(iommu->ecap))
 766                 return;
 767
 768         spin_lock_irqsave(&iommu->register_lock, flags);
 769
 770         sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
 771         if (!(sts & DMA_GSTS_QIES))
 772                 goto end;
 773
 774         /*
 775          * Give a chance to HW to complete the pending invalidation requests.
 776          */
 777         while ((readl(iommu->reg + DMAR_IQT_REG) !=
 778                 readl(iommu->reg + DMAR_IQH_REG)) &&
 779                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
 780                 cpu_relax();
 781
 782         iommu->gcmd &= ~DMA_GCMD_QIE;
 783
 784         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
 785
 786         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
 787                       !(sts & DMA_GSTS_QIES), sts);
 788 end:
 789         spin_unlock_irqrestore(&iommu->register_lock, flags);
 790 }
 791
 792 /*
 793  * Enable Queued Invalidation interface. This is a must to support
 794  * interrupt-remapping. Also used by DMA-remapping, which replaces
 795  * register based IOTLB invalidation.
 796  */
 797 int dmar_enable_qi(struct intel_iommu *iommu)
 798 {
 799         u32 cmd, sts;
 800         unsigned long flags;
 801         struct q_inval *qi;
 802
 803         if (!ecap_qis(iommu->ecap))
 804                 return -ENOENT;
 805
 806         /*
 807          * queued invalidation is already setup and enabled.
 808          */
 809         if (iommu->qi)
 810                 return 0;
 811
 812         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
 813         if (!iommu->qi)
 814                 return -ENOMEM;
 815
 816         qi = iommu->qi;
 817
 818         qi->desc = (void *)(get_zeroed_page(GFP_ATOMIC));
 819         if (!qi->desc) {
 820                 kfree(qi);
 821                 iommu->qi = 0;
 822                 return -ENOMEM;
 823         }
 824
 825         qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
 826         if (!qi->desc_status) {
 827                 free_page((unsigned long) qi->desc);
 828                 kfree(qi);
 829                 iommu->qi = 0;
 830                 return -ENOMEM;
 831         }
 832
 833         qi->free_head = qi->free_tail = 0;
 834         qi->free_cnt = QI_LENGTH;
 835
 836         spin_lock_init(&qi->q_lock);
 837
 838         spin_lock_irqsave(&iommu->register_lock, flags);
 839         /* write zero to the tail reg */
 840         writel(0, iommu->reg + DMAR_IQT_REG);
 841
 842         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
 843
 844         cmd = iommu->gcmd | DMA_GCMD_QIE;
 845         iommu->gcmd |= DMA_GCMD_QIE;
 846         writel(cmd, iommu->reg + DMAR_GCMD_REG);
 847
 848         /* Make sure hardware complete it */
 849         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
 850         spin_unlock_irqrestore(&iommu->register_lock, flags);
 851
 852         return 0;
 853 }
 854
 855 /* iommu interrupt handling. Most stuff are MSI-like. */
 856
 857 enum faulttype {
 858         DMA_REMAP,
 859         INTR_REMAP,
 860         UNKNOWN,
 861 };
 862
 863 static const char *dma_remap_fault_reasons[] =
 864 {
 865         "Software",
 866         "Present bit in root entry is clear",
 867         "Present bit in context entry is clear",
 868         "Invalid context entry",
 869         "Access beyond MGAW",
 870         "PTE Write access is not set",
 871         "PTE Read access is not set",
 872         "Next page table ptr is invalid",
 873         "Root table address invalid",
 874         "Context table ptr is invalid",
 875         "non-zero reserved fields in RTP",
 876         "non-zero reserved fields in CTP",
 877         "non-zero reserved fields in PTE",
 878 };
 879
 880 static const char *intr_remap_fault_reasons[] =
 881 {
 882         "Detected reserved fields in the decoded interrupt-remapped request",
 883         "Interrupt index exceeded the interrupt-remapping table size",
 884         "Present field in the IRTE entry is clear",
 885         "Error accessing interrupt-remapping table pointed by IRTA_REG",
 886         "Detected reserved fields in the IRTE entry",
 887         "Blocked a compatibility format interrupt request",
 888         "Blocked an interrupt request due to source-id verification failure",
 889 };
 890
 891 #define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
 892
 893 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
 894 {
 895         if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
 896                                      ARRAY_SIZE(intr_remap_fault_reasons))) {
 897                 *fault_type = INTR_REMAP;
 898                 return intr_remap_fault_reasons[fault_reason - 0x20];
 899         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
 900                 *fault_type = DMA_REMAP;
 901                 return dma_remap_fault_reasons[fault_reason];
 902         } else {
 903                 *fault_type = UNKNOWN;
 904                 return "Unknown";
 905         }
 906 }
 907
 908 void dmar_msi_unmask(unsigned int irq)
 909 {
 910         struct intel_iommu *iommu = get_irq_data(irq);
 911         unsigned long flag;
 912
 913         /* unmask it */
 914         spin_lock_irqsave(&iommu->register_lock, flag);
 915         writel(0, iommu->reg + DMAR_FECTL_REG);
 916         /* Read a reg to force flush the post write */
 917         readl(iommu->reg + DMAR_FECTL_REG);
 918         spin_unlock_irqrestore(&iommu->register_lock, flag);
 919 }
 920
 921 void dmar_msi_mask(unsigned int irq)
 922 {
 923         unsigned long flag;
 924         struct intel_iommu *iommu = get_irq_data(irq);
 925
 926         /* mask it */
 927         spin_lock_irqsave(&iommu->register_lock, flag);
 928         writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
 929         /* Read a reg to force flush the post write */
 930         readl(iommu->reg + DMAR_FECTL_REG);
 931         spin_unlock_irqrestore(&iommu->register_lock, flag);
 932 }
 933
 934 void dmar_msi_write(int irq, struct msi_msg *msg)
 935 {
 936         struct intel_iommu *iommu = get_irq_data(irq);
 937         unsigned long flag;
 938
 939         spin_lock_irqsave(&iommu->register_lock, flag);
 940         writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
 941         writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
 942         writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
 943         spin_unlock_irqrestore(&iommu->register_lock, flag);
 944 }
 945
 946 void dmar_msi_read(int irq, struct msi_msg *msg)
 947 {
 948         struct intel_iommu *iommu = get_irq_data(irq);
 949         unsigned long flag;
 950
 951         spin_lock_irqsave(&iommu->register_lock, flag);
 952         msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
 953         msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
 954         msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
 955         spin_unlock_irqrestore(&iommu->register_lock, flag);
 956 }
 957
 958 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
 959                 u8 fault_reason, u16 source_id, unsigned long long addr)
 960 {
 961         const char *reason;
 962         int fault_type;
 963
 964         reason = dmar_get_fault_reason(fault_reason, &fault_type);
 965
 966         if (fault_type == INTR_REMAP)
 967                 printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
 968                        "fault index %llx\n"
 969                         "INTR-REMAP:[fault reason %02d] %s\n",
 970                         (source_id >> 8), PCI_SLOT(source_id & 0xFF),
 971                         PCI_FUNC(source_id & 0xFF), addr >> 48,
 972                         fault_reason, reason);
 973         else
 974                 printk(KERN_ERR
 975                        "DMAR:[%s] Request device [%02x:%02x.%d] "
 976                        "fault addr %llx \n"
 977                        "DMAR:[fault reason %02d] %s\n",
 978                        (type ? "DMA Read" : "DMA Write"),
 979                        (source_id >> 8), PCI_SLOT(source_id & 0xFF),
 980                        PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
 981         return 0;
 982 }
 983
 984 #define PRIMARY_FAULT_REG_LEN (16)
 985 irqreturn_t dmar_fault(int irq, void *dev_id)
 986 {
 987         struct intel_iommu *iommu = dev_id;
 988         int reg, fault_index;
 989         u32 fault_status;
 990         unsigned long flag;
 991
 992         spin_lock_irqsave(&iommu->register_lock, flag);
 993         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
 994         if (fault_status)
 995                 printk(KERN_ERR "DRHD: handling fault status reg %x\n",
 996                        fault_status);
 997
 998         /* TBD: ignore advanced fault log currently */
 999         if (!(fault_status & DMA_FSTS_PPF))
1000                 goto clear_rest;
1001
1002         fault_index = dma_fsts_fault_record_index(fault_status);
1003         reg = cap_fault_reg_offset(iommu->cap);
1004         while (1) {
1005                 u8 fault_reason;
1006                 u16 source_id;
1007                 u64 guest_addr;
1008                 int type;
1009                 u32 data;
1010
1011                 /* highest 32 bits */
1012                 data = readl(iommu->reg + reg +
1013                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1014                 if (!(data & DMA_FRCD_F))
1015                         break;
1016
1017                 fault_reason = dma_frcd_fault_reason(data);
1018                 type = dma_frcd_type(data);
1019
1020                 data = readl(iommu->reg + reg +
1021                                 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1022                 source_id = dma_frcd_source_id(data);
1023
1024                 guest_addr = dmar_readq(iommu->reg + reg +
1025                                 fault_index * PRIMARY_FAULT_REG_LEN);
1026                 guest_addr = dma_frcd_page_addr(guest_addr);
1027                 /* clear the fault */
1028                 writel(DMA_FRCD_F, iommu->reg + reg +
1029                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1030
1031                 spin_unlock_irqrestore(&iommu->register_lock, flag);
1032
1033                 dmar_fault_do_one(iommu, type, fault_reason,
1034                                 source_id, guest_addr);
1035
1036                 fault_index++;
1037                 if (fault_index > cap_num_fault_regs(iommu->cap))
1038                         fault_index = 0;
1039                 spin_lock_irqsave(&iommu->register_lock, flag);
1040         }
1041 clear_rest:
1042         /* clear all the other faults */
1043         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1044         writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1045
1046         spin_unlock_irqrestore(&iommu->register_lock, flag);
1047         return IRQ_HANDLED;
1048 }
1049
1050 int dmar_set_interrupt(struct intel_iommu *iommu)
1051 {
1052         int irq, ret;
1053
1054         /*
1055          * Check if the fault interrupt is already initialized.
1056          */
1057         if (iommu->irq)
1058                 return 0;
1059
1060         irq = create_irq();
1061         if (!irq) {
1062                 printk(KERN_ERR "IOMMU: no free vectors\n");
1063                 return -EINVAL;
1064         }
1065
1066         set_irq_data(irq, iommu);
1067         iommu->irq = irq;
1068
1069         ret = arch_setup_dmar_msi(irq);
1070         if (ret) {
1071                 set_irq_data(irq, NULL);
1072                 iommu->irq = 0;
1073                 destroy_irq(irq);
1074                 return 0;
1075         }
1076
1077         ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
1078         if (ret)
1079                 printk(KERN_ERR "IOMMU: can't request irq\n");
1080         return ret;
1081 }
1082
1083 int __init enable_drhd_fault_handling(void)
1084 {
1085         struct dmar_drhd_unit *drhd;
1086
1087         /*
1088          * Enable fault control interrupt.
1089          */
1090         for_each_drhd_unit(drhd) {
1091                 int ret;
1092                 struct intel_iommu *iommu = drhd->iommu;
1093                 ret = dmar_set_interrupt(iommu);
1094
1095                 if (ret) {
1096                         printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1097                                " interrupt, ret %d\n",
1098                                (unsigned long long)drhd->reg_base_addr, ret);
1099                         return -1;
1100                 }
1101         }
1102
1103         return 0;
1104 }