arch/x86/kernel/amd_iommu.c

   1 /*
   2  * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
   3  * Author: Joerg Roedel <joerg.roedel@amd.com>
   4  *         Leo Duran <leo.duran@amd.com>
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18  */
  19
  20 #include <linux/pci.h>
  21 #include <linux/gfp.h>
  22 #include <linux/bitops.h>
  23 #include <linux/scatterlist.h>
  24 #include <linux/iommu-helper.h>
  25 #include <asm/proto.h>
  26 #include <asm/gart.h>
  27 #include <asm/amd_iommu_types.h>
  28
  29 #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
  30
  31 #define to_pages(addr, size) \
  32          (round_up(((addr) & ~PAGE_MASK) + (size), PAGE_SIZE) >> PAGE_SHIFT)
  33
  34 static DEFINE_RWLOCK(amd_iommu_devtable_lock);
  35
  36 struct command {
  37         u32 data[4];
  38 };
  39
  40 static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
  41                              struct unity_map_entry *e);
  42
  43 static int __iommu_queue_command(struct amd_iommu *iommu, struct command *cmd)
  44 {
  45         u32 tail, head;
  46         u8 *target;
  47
  48         tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
  49         target = (iommu->cmd_buf + tail);
  50         memcpy_toio(target, cmd, sizeof(*cmd));
  51         tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
  52         head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
  53         if (tail == head)
  54                 return -ENOMEM;
  55         writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
  56
  57         return 0;
  58 }
  59
  60 static int iommu_queue_command(struct amd_iommu *iommu, struct command *cmd)
  61 {
  62         unsigned long flags;
  63         int ret;
  64
  65         spin_lock_irqsave(&iommu->lock, flags);
  66         ret = __iommu_queue_command(iommu, cmd);
  67         spin_unlock_irqrestore(&iommu->lock, flags);
  68
  69         return ret;
  70 }
  71
  72 static int iommu_completion_wait(struct amd_iommu *iommu)
  73 {
  74         int ret;
  75         struct command cmd;
  76         volatile u64 ready = 0;
  77         unsigned long ready_phys = virt_to_phys(&ready);
  78
  79         memset(&cmd, 0, sizeof(cmd));
  80         cmd.data[0] = LOW_U32(ready_phys) | CMD_COMPL_WAIT_STORE_MASK;
  81         cmd.data[1] = HIGH_U32(ready_phys);
  82         cmd.data[2] = 1; /* value written to 'ready' */
  83         CMD_SET_TYPE(&cmd, CMD_COMPL_WAIT);
  84
  85         iommu->need_sync = 0;
  86
  87         ret = iommu_queue_command(iommu, &cmd);
  88
  89         if (ret)
  90                 return ret;
  91
  92         while (!ready)
  93                 cpu_relax();
  94
  95         return 0;
  96 }
  97
  98 static int iommu_queue_inv_dev_entry(struct amd_iommu *iommu, u16 devid)
  99 {
 100         struct command cmd;
 101
 102         BUG_ON(iommu == NULL);
 103
 104         memset(&cmd, 0, sizeof(cmd));
 105         CMD_SET_TYPE(&cmd, CMD_INV_DEV_ENTRY);
 106         cmd.data[0] = devid;
 107
 108         iommu->need_sync = 1;
 109
 110         return iommu_queue_command(iommu, &cmd);
 111 }
 112
 113 static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu,
 114                 u64 address, u16 domid, int pde, int s)
 115 {
 116         struct command cmd;
 117
 118         memset(&cmd, 0, sizeof(cmd));
 119         address &= PAGE_MASK;
 120         CMD_SET_TYPE(&cmd, CMD_INV_IOMMU_PAGES);
 121         cmd.data[1] |= domid;
 122         cmd.data[2] = LOW_U32(address);
 123         cmd.data[3] = HIGH_U32(address);
 124         if (s)
 125                 cmd.data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
 126         if (pde)
 127                 cmd.data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
 128
 129         iommu->need_sync = 1;
 130
 131         return iommu_queue_command(iommu, &cmd);
 132 }
 133
 134 static int iommu_flush_pages(struct amd_iommu *iommu, u16 domid,
 135                 u64 address, size_t size)
 136 {
 137         int i;
 138         unsigned pages = to_pages(address, size);
 139
 140         address &= PAGE_MASK;
 141
 142         for (i = 0; i < pages; ++i) {
 143                 iommu_queue_inv_iommu_pages(iommu, address, domid, 0, 0);
 144                 address += PAGE_SIZE;
 145         }
 146
 147         return 0;
 148 }
 149
 150 static int iommu_map(struct protection_domain *dom,
 151                      unsigned long bus_addr,
 152                      unsigned long phys_addr,
 153                      int prot)
 154 {
 155         u64 __pte, *pte, *page;
 156
 157         bus_addr  = PAGE_ALIGN(bus_addr);
 158         phys_addr = PAGE_ALIGN(bus_addr);
 159
 160         /* only support 512GB address spaces for now */
 161         if (bus_addr > IOMMU_MAP_SIZE_L3 || !(prot & IOMMU_PROT_MASK))
 162                 return -EINVAL;
 163
 164         pte = &dom->pt_root[IOMMU_PTE_L2_INDEX(bus_addr)];
 165
 166         if (!IOMMU_PTE_PRESENT(*pte)) {
 167                 page = (u64 *)get_zeroed_page(GFP_KERNEL);
 168                 if (!page)
 169                         return -ENOMEM;
 170                 *pte = IOMMU_L2_PDE(virt_to_phys(page));
 171         }
 172
 173         pte = IOMMU_PTE_PAGE(*pte);
 174         pte = &pte[IOMMU_PTE_L1_INDEX(bus_addr)];
 175
 176         if (!IOMMU_PTE_PRESENT(*pte)) {
 177                 page = (u64 *)get_zeroed_page(GFP_KERNEL);
 178                 if (!page)
 179                         return -ENOMEM;
 180                 *pte = IOMMU_L1_PDE(virt_to_phys(page));
 181         }
 182
 183         pte = IOMMU_PTE_PAGE(*pte);
 184         pte = &pte[IOMMU_PTE_L0_INDEX(bus_addr)];
 185
 186         if (IOMMU_PTE_PRESENT(*pte))
 187                 return -EBUSY;
 188
 189         __pte = phys_addr | IOMMU_PTE_P;
 190         if (prot & IOMMU_PROT_IR)
 191                 __pte |= IOMMU_PTE_IR;
 192         if (prot & IOMMU_PROT_IW)
 193                 __pte |= IOMMU_PTE_IW;
 194
 195         *pte = __pte;
 196
 197         return 0;
 198 }
 199
 200 static int iommu_for_unity_map(struct amd_iommu *iommu,
 201                                struct unity_map_entry *entry)
 202 {
 203         u16 bdf, i;
 204
 205         for (i = entry->devid_start; i <= entry->devid_end; ++i) {
 206                 bdf = amd_iommu_alias_table[i];
 207                 if (amd_iommu_rlookup_table[bdf] == iommu)
 208                         return 1;
 209         }
 210
 211         return 0;
 212 }
 213
 214 static int iommu_init_unity_mappings(struct amd_iommu *iommu)
 215 {
 216         struct unity_map_entry *entry;
 217         int ret;
 218
 219         list_for_each_entry(entry, &amd_iommu_unity_map, list) {
 220                 if (!iommu_for_unity_map(iommu, entry))
 221                         continue;
 222                 ret = dma_ops_unity_map(iommu->default_dom, entry);
 223                 if (ret)
 224                         return ret;
 225         }
 226
 227         return 0;
 228 }
 229
 230 static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
 231                              struct unity_map_entry *e)
 232 {
 233         u64 addr;
 234         int ret;
 235
 236         for (addr = e->address_start; addr < e->address_end;
 237              addr += PAGE_SIZE) {
 238                 ret = iommu_map(&dma_dom->domain, addr, addr, e->prot);
 239                 if (ret)
 240                         return ret;
 241                 /*
 242                  * if unity mapping is in aperture range mark the page
 243                  * as allocated in the aperture
 244                  */
 245                 if (addr < dma_dom->aperture_size)
 246                         __set_bit(addr >> PAGE_SHIFT, dma_dom->bitmap);
 247         }
 248
 249         return 0;
 250 }
 251
 252 static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
 253                                           u16 devid)
 254 {
 255         struct unity_map_entry *e;
 256         int ret;
 257
 258         list_for_each_entry(e, &amd_iommu_unity_map, list) {
 259                 if (!(devid >= e->devid_start && devid <= e->devid_end))
 260                         continue;
 261                 ret = dma_ops_unity_map(dma_dom, e);
 262                 if (ret)
 263                         return ret;
 264         }
 265
 266         return 0;
 267 }
 268
 269 static unsigned long dma_mask_to_pages(unsigned long mask)
 270 {
 271         return (mask >> PAGE_SHIFT) +
 272                 (PAGE_ALIGN(mask & ~PAGE_MASK) >> PAGE_SHIFT);
 273 }
 274
 275 static unsigned long dma_ops_alloc_addresses(struct device *dev,
 276                                              struct dma_ops_domain *dom,
 277                                              unsigned int pages)
 278 {
 279         unsigned long limit = dma_mask_to_pages(*dev->dma_mask);
 280         unsigned long address;
 281         unsigned long size = dom->aperture_size >> PAGE_SHIFT;
 282         unsigned long boundary_size;
 283
 284         boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
 285                         PAGE_SIZE) >> PAGE_SHIFT;
 286         limit = limit < size ? limit : size;
 287
 288         if (dom->next_bit >= limit)
 289                 dom->next_bit = 0;
 290
 291         address = iommu_area_alloc(dom->bitmap, limit, dom->next_bit, pages,
 292                         0 , boundary_size, 0);
 293         if (address == -1)
 294                 address = iommu_area_alloc(dom->bitmap, limit, 0, pages,
 295                                 0, boundary_size, 0);
 296
 297         if (likely(address != -1)) {
 298                 set_bit_string(dom->bitmap, address, pages);
 299                 dom->next_bit = address + pages;
 300                 address <<= PAGE_SHIFT;
 301         } else
 302                 address = bad_dma_address;
 303
 304         WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
 305
 306         return address;
 307 }
 308
 309 static void dma_ops_free_addresses(struct dma_ops_domain *dom,
 310                                    unsigned long address,
 311                                    unsigned int pages)
 312 {
 313         address >>= PAGE_SHIFT;
 314         iommu_area_free(dom->bitmap, address, pages);
 315 }
 316
 317 static u16 domain_id_alloc(void)
 318 {
 319         unsigned long flags;
 320         int id;
 321
 322         write_lock_irqsave(&amd_iommu_devtable_lock, flags);
 323         id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
 324         BUG_ON(id == 0);
 325         if (id > 0 && id < MAX_DOMAIN_ID)
 326                 __set_bit(id, amd_iommu_pd_alloc_bitmap);
 327         else
 328                 id = 0;
 329         write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
 330
 331         return id;
 332 }
 333
 334 static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
 335                                       unsigned long start_page,
 336                                       unsigned int pages)
 337 {
 338         unsigned int last_page = dom->aperture_size >> PAGE_SHIFT;
 339
 340         if (start_page + pages > last_page)
 341                 pages = last_page - start_page;
 342
 343         set_bit_string(dom->bitmap, start_page, pages);
 344 }
 345
 346 static void dma_ops_free_pagetable(struct dma_ops_domain *dma_dom)
 347 {
 348         int i, j;
 349         u64 *p1, *p2, *p3;
 350
 351         p1 = dma_dom->domain.pt_root;
 352
 353         if (!p1)
 354                 return;
 355
 356         for (i = 0; i < 512; ++i) {
 357                 if (!IOMMU_PTE_PRESENT(p1[i]))
 358                         continue;
 359
 360                 p2 = IOMMU_PTE_PAGE(p1[i]);
 361                 for (j = 0; j < 512; ++i) {
 362                         if (!IOMMU_PTE_PRESENT(p2[j]))
 363                                 continue;
 364                         p3 = IOMMU_PTE_PAGE(p2[j]);
 365                         free_page((unsigned long)p3);
 366                 }
 367
 368                 free_page((unsigned long)p2);
 369         }
 370
 371         free_page((unsigned long)p1);
 372 }
 373
 374 static void dma_ops_domain_free(struct dma_ops_domain *dom)
 375 {
 376         if (!dom)
 377                 return;
 378
 379         dma_ops_free_pagetable(dom);
 380
 381         kfree(dom->pte_pages);
 382
 383         kfree(dom->bitmap);
 384
 385         kfree(dom);
 386 }
 387
 388 static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu,
 389                                                    unsigned order)
 390 {
 391         struct dma_ops_domain *dma_dom;
 392         unsigned i, num_pte_pages;
 393         u64 *l2_pde;
 394         u64 address;
 395
 396         /*
 397          * Currently the DMA aperture must be between 32 MB and 1GB in size
 398          */
 399         if ((order < 25) || (order > 30))
 400                 return NULL;
 401
 402         dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
 403         if (!dma_dom)
 404                 return NULL;
 405
 406         spin_lock_init(&dma_dom->domain.lock);
 407
 408         dma_dom->domain.id = domain_id_alloc();
 409         if (dma_dom->domain.id == 0)
 410                 goto free_dma_dom;
 411         dma_dom->domain.mode = PAGE_MODE_3_LEVEL;
 412         dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
 413         dma_dom->domain.priv = dma_dom;
 414         if (!dma_dom->domain.pt_root)
 415                 goto free_dma_dom;
 416         dma_dom->aperture_size = (1ULL << order);
 417         dma_dom->bitmap = kzalloc(dma_dom->aperture_size / (PAGE_SIZE * 8),
 418                                   GFP_KERNEL);
 419         if (!dma_dom->bitmap)
 420                 goto free_dma_dom;
 421         /*
 422          * mark the first page as allocated so we never return 0 as
 423          * a valid dma-address. So we can use 0 as error value
 424          */
 425         dma_dom->bitmap[0] = 1;
 426         dma_dom->next_bit = 0;
 427
 428         if (iommu->exclusion_start &&
 429             iommu->exclusion_start < dma_dom->aperture_size) {
 430                 unsigned long startpage = iommu->exclusion_start >> PAGE_SHIFT;
 431                 int pages = to_pages(iommu->exclusion_start,
 432                                 iommu->exclusion_length);
 433                 dma_ops_reserve_addresses(dma_dom, startpage, pages);
 434         }
 435
 436         num_pte_pages = dma_dom->aperture_size / (PAGE_SIZE * 512);
 437         dma_dom->pte_pages = kzalloc(num_pte_pages * sizeof(void *),
 438                         GFP_KERNEL);
 439         if (!dma_dom->pte_pages)
 440                 goto free_dma_dom;
 441
 442         l2_pde = (u64 *)get_zeroed_page(GFP_KERNEL);
 443         if (l2_pde == NULL)
 444                 goto free_dma_dom;
 445
 446         dma_dom->domain.pt_root[0] = IOMMU_L2_PDE(virt_to_phys(l2_pde));
 447
 448         for (i = 0; i < num_pte_pages; ++i) {
 449                 dma_dom->pte_pages[i] = (u64 *)get_zeroed_page(GFP_KERNEL);
 450                 if (!dma_dom->pte_pages[i])
 451                         goto free_dma_dom;
 452                 address = virt_to_phys(dma_dom->pte_pages[i]);
 453                 l2_pde[i] = IOMMU_L1_PDE(address);
 454         }
 455
 456         return dma_dom;
 457
 458 free_dma_dom:
 459         dma_ops_domain_free(dma_dom);
 460
 461         return NULL;
 462 }
 463
 464 static struct protection_domain *domain_for_device(u16 devid)
 465 {
 466         struct protection_domain *dom;
 467         unsigned long flags;
 468
 469         read_lock_irqsave(&amd_iommu_devtable_lock, flags);
 470         dom = amd_iommu_pd_table[devid];
 471         read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
 472
 473         return dom;
 474 }
 475
 476 static void set_device_domain(struct amd_iommu *iommu,
 477                               struct protection_domain *domain,
 478                               u16 devid)
 479 {
 480         unsigned long flags;
 481
 482         u64 pte_root = virt_to_phys(domain->pt_root);
 483
 484         pte_root |= (domain->mode & 0x07) << 9;
 485         pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | 2;
 486
 487         write_lock_irqsave(&amd_iommu_devtable_lock, flags);
 488         amd_iommu_dev_table[devid].data[0] = pte_root;
 489         amd_iommu_dev_table[devid].data[1] = pte_root >> 32;
 490         amd_iommu_dev_table[devid].data[2] = domain->id;
 491
 492         amd_iommu_pd_table[devid] = domain;
 493         write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
 494
 495         iommu_queue_inv_dev_entry(iommu, devid);
 496
 497         iommu->need_sync = 1;
 498 }
 499
 500 static int get_device_resources(struct device *dev,
 501                                 struct amd_iommu **iommu,
 502                                 struct protection_domain **domain,
 503                                 u16 *bdf)
 504 {
 505         struct dma_ops_domain *dma_dom;
 506         struct pci_dev *pcidev;
 507         u16 _bdf;
 508
 509         BUG_ON(!dev || dev->bus != &pci_bus_type || !dev->dma_mask);
 510
 511         pcidev = to_pci_dev(dev);
 512         _bdf = (pcidev->bus->number << 8) | pcidev->devfn;
 513
 514         if (_bdf >= amd_iommu_last_bdf) {
 515                 *iommu = NULL;
 516                 *domain = NULL;
 517                 *bdf = 0xffff;
 518                 return 0;
 519         }
 520
 521         *bdf = amd_iommu_alias_table[_bdf];
 522
 523         *iommu = amd_iommu_rlookup_table[*bdf];
 524         if (*iommu == NULL)
 525                 return 0;
 526         dma_dom = (*iommu)->default_dom;
 527         *domain = domain_for_device(*bdf);
 528         if (*domain == NULL) {
 529                 *domain = &dma_dom->domain;
 530                 set_device_domain(*iommu, *domain, *bdf);
 531                 printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "
 532                                 "device ", (*domain)->id);
 533                 print_devid(_bdf, 1);
 534         }
 535
 536         return 1;
 537 }
 538
 539 static dma_addr_t dma_ops_domain_map(struct amd_iommu *iommu,
 540                                      struct dma_ops_domain *dom,
 541                                      unsigned long address,
 542                                      phys_addr_t paddr,
 543                                      int direction)
 544 {
 545         u64 *pte, __pte;
 546
 547         WARN_ON(address > dom->aperture_size);
 548
 549         paddr &= PAGE_MASK;
 550
 551         pte  = dom->pte_pages[IOMMU_PTE_L1_INDEX(address)];
 552         pte += IOMMU_PTE_L0_INDEX(address);
 553
 554         __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
 555
 556         if (direction == DMA_TO_DEVICE)
 557                 __pte |= IOMMU_PTE_IR;
 558         else if (direction == DMA_FROM_DEVICE)
 559                 __pte |= IOMMU_PTE_IW;
 560         else if (direction == DMA_BIDIRECTIONAL)
 561                 __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
 562
 563         WARN_ON(*pte);
 564
 565         *pte = __pte;
 566
 567         return (dma_addr_t)address;
 568 }
 569
 570 static void dma_ops_domain_unmap(struct amd_iommu *iommu,
 571                                  struct dma_ops_domain *dom,
 572                                  unsigned long address)
 573 {
 574         u64 *pte;
 575
 576         if (address >= dom->aperture_size)
 577                 return;
 578
 579         WARN_ON(address & 0xfffULL || address > dom->aperture_size);
 580
 581         pte  = dom->pte_pages[IOMMU_PTE_L1_INDEX(address)];
 582         pte += IOMMU_PTE_L0_INDEX(address);
 583
 584         WARN_ON(!*pte);
 585
 586         *pte = 0ULL;
 587 }
 588
 589 static dma_addr_t __map_single(struct device *dev,
 590                                struct amd_iommu *iommu,
 591                                struct dma_ops_domain *dma_dom,
 592                                phys_addr_t paddr,
 593                                size_t size,
 594                                int dir)
 595 {
 596         dma_addr_t offset = paddr & ~PAGE_MASK;
 597         dma_addr_t address, start;
 598         unsigned int pages;
 599         int i;
 600
 601         pages = to_pages(paddr, size);
 602         paddr &= PAGE_MASK;
 603
 604         address = dma_ops_alloc_addresses(dev, dma_dom, pages);
 605         if (unlikely(address == bad_dma_address))
 606                 goto out;
 607
 608         start = address;
 609         for (i = 0; i < pages; ++i) {
 610                 dma_ops_domain_map(iommu, dma_dom, start, paddr, dir);
 611                 paddr += PAGE_SIZE;
 612                 start += PAGE_SIZE;
 613         }
 614         address += offset;
 615
 616 out:
 617         return address;
 618 }
 619
 620 static void __unmap_single(struct amd_iommu *iommu,
 621                            struct dma_ops_domain *dma_dom,
 622                            dma_addr_t dma_addr,
 623                            size_t size,
 624                            int dir)
 625 {
 626         dma_addr_t i, start;
 627         unsigned int pages;
 628
 629         if ((dma_addr == 0) || (dma_addr + size > dma_dom->aperture_size))
 630                 return;
 631
 632         pages = to_pages(dma_addr, size);
 633         dma_addr &= PAGE_MASK;
 634         start = dma_addr;
 635
 636         for (i = 0; i < pages; ++i) {
 637                 dma_ops_domain_unmap(iommu, dma_dom, start);
 638                 start += PAGE_SIZE;
 639         }
 640
 641         dma_ops_free_addresses(dma_dom, dma_addr, pages);
 642 }
 643