2 * IOMMU driver for SMMU on Tegra 3 series SoCs and later.
4 * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 * You should have received a copy of the GNU General Public License along with
16 * this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
20 #define pr_fmt(fmt) "%s(): " fmt, __func__
22 #include <linux/module.h>
23 #include <linux/platform_device.h>
24 #include <linux/spinlock.h>
25 #include <linux/slab.h>
26 #include <linux/vmalloc.h>
28 #include <linux/pagemap.h>
29 #include <linux/device.h>
30 #include <linux/sched.h>
31 #include <linux/iommu.h>
33 #include <linux/debugfs.h>
34 #include <linux/seq_file.h>
35 #include <linux/tegra-ahb.h>
37 #include <linux/of_iommu.h>
38 #include <linux/tegra-ahb.h>
39 #include <linux/dma-mapping.h>
40 #include <linux/bitops.h>
41 #include <linux/tegra-soc.h>
44 #include <asm/cacheflush.h>
45 #include <asm/dma-iommu.h>
47 #include <mach/tegra_smmu.h>
48 #include <mach/tegra-swgid.h>
50 /* HACK! This needs to come from device tree */
51 #include "../../arch/arm/mach-tegra/iomap.h"
53 /* bitmap of the page sizes currently supported */
54 #define SMMU_IOMMU_PGSIZES (SZ_4K | SZ_4M)
56 #define SMMU_CONFIG 0x10
57 #define SMMU_CONFIG_DISABLE 0
58 #define SMMU_CONFIG_ENABLE 1
65 #define SMMU_CACHE_CONFIG_BASE 0x14
66 #define __SMMU_CACHE_CONFIG(mc, cache) (SMMU_CACHE_CONFIG_BASE + 4 * cache)
67 #define SMMU_CACHE_CONFIG(cache) __SMMU_CACHE_CONFIG(_MC, cache)
69 #define SMMU_CACHE_CONFIG_STATS_SHIFT 31
70 #define SMMU_CACHE_CONFIG_STATS_ENABLE (1 << SMMU_CACHE_CONFIG_STATS_SHIFT)
71 #define SMMU_CACHE_CONFIG_STATS_TEST_SHIFT 30
72 #define SMMU_CACHE_CONFIG_STATS_TEST (1 << SMMU_CACHE_CONFIG_STATS_TEST_SHIFT)
74 #define SMMU_TLB_CONFIG_HIT_UNDER_MISS__ENABLE (1 << 29)
75 #define SMMU_TLB_CONFIG_ACTIVE_LINES__VALUE 0x10
76 #define SMMU_TLB_CONFIG_RESET_VAL 0x20000000
77 #define SMMU_TLB_RR_ARB (1 << 28)
79 #define SMMU_PTC_CONFIG_CACHE__ENABLE (1 << 29)
80 #define SMMU_PTC_CONFIG_INDEX_MAP__PATTERN 0x3f
81 #define SMMU_PTC_CONFIG_RESET_VAL 0x2000003f
82 #define SMMU_PTC_REQ_LIMIT (8 << 24)
84 #define SMMU_PTB_ASID 0x1c
85 #define SMMU_PTB_ASID_CURRENT_SHIFT 0
87 #define SMMU_PTB_DATA 0x20
88 #define SMMU_PTB_DATA_RESET_VAL 0
89 #define SMMU_PTB_DATA_ASID_NONSECURE_SHIFT 29
90 #define SMMU_PTB_DATA_ASID_WRITABLE_SHIFT 30
91 #define SMMU_PTB_DATA_ASID_READABLE_SHIFT 31
93 #define SMMU_TLB_FLUSH 0x30
94 #define SMMU_TLB_FLUSH_VA_MATCH_ALL 0
95 #define SMMU_TLB_FLUSH_VA_MATCH_SECTION 2
96 #define SMMU_TLB_FLUSH_VA_MATCH_GROUP 3
97 #define SMMU_TLB_FLUSH_ASID_SHIFT_BASE 31
98 #define SMMU_TLB_FLUSH_ASID_MATCH_DISABLE 0
99 #define SMMU_TLB_FLUSH_ASID_MATCH_ENABLE 1
100 #define SMMU_TLB_FLUSH_ASID_MATCH_SHIFT 31
101 #define SMMU_TLB_FLUSH_ASID_ENABLE \
102 (SMMU_TLB_FLUSH_ASID_MATCH_ENABLE << SMMU_TLB_FLUSH_ASID_MATCH_SHIFT)
104 #define SMMU_TLB_FLUSH_ASID_SHIFT(as) \
105 (SMMU_TLB_FLUSH_ASID_SHIFT_BASE - __ffs((as)->smmu->num_as))
107 #define SMMU_PTC_FLUSH 0x34
108 #define SMMU_PTC_FLUSH_TYPE_ALL 0
109 #define SMMU_PTC_FLUSH_TYPE_ADR 1
110 #define SMMU_PTC_FLUSH_ADR_SHIFT 4
112 #define SMMU_PTC_FLUSH_1 0x9b8
114 #define SMMU_ASID_SECURITY 0x38
115 #define SMMU_ASID_SECURITY_1 0x3c
116 #define SMMU_ASID_SECURITY_2 0x9e0
117 #define SMMU_ASID_SECURITY_3 0x9e4
118 #define SMMU_ASID_SECURITY_4 0x9e8
119 #define SMMU_ASID_SECURITY_5 0x9ec
120 #define SMMU_ASID_SECURITY_6 0x9f0
121 #define SMMU_ASID_SECURITY_7 0x9f4
123 #define SMMU_STATS_CACHE_COUNT_BASE 0x1f0
125 #define SMMU_STATS_CACHE_COUNT(mc, cache, hitmiss) \
126 (SMMU_STATS_CACHE_COUNT_BASE + 8 * cache + 4 * hitmiss)
128 #define SMMU_TRANSLATION_ENABLE_0 0x228
130 #define SMMU_AFI_ASID 0x238 /* PCIE */
132 #define SMMU_SWGRP_ASID_BASE SMMU_AFI_ASID
134 #define HWGRP_COUNT 64
136 #define SMMU_PDE_NEXT_SHIFT 28
138 /* AHB Arbiter Registers */
139 #define AHB_XBAR_CTRL 0xe0
140 #define AHB_XBAR_CTRL_SMMU_INIT_DONE_DONE 1
141 #define AHB_XBAR_CTRL_SMMU_INIT_DONE_SHIFT 17
143 #define SMMU_NUM_ASIDS 4
144 #define SMMU_NUM_ASIDS_TEGRA12 128
145 #define SMMU_TLB_FLUSH_VA_SECTION__MASK 0xffc00000
146 #define SMMU_TLB_FLUSH_VA_SECTION__SHIFT 12 /* right shift */
147 #define SMMU_TLB_FLUSH_VA_GROUP__MASK 0xffffc000
148 #define SMMU_TLB_FLUSH_VA_GROUP__SHIFT 12 /* right shift */
149 #define SMMU_TLB_FLUSH_VA(iova, which) \
150 ((((iova) & SMMU_TLB_FLUSH_VA_##which##__MASK) >> \
151 SMMU_TLB_FLUSH_VA_##which##__SHIFT) | \
152 SMMU_TLB_FLUSH_VA_MATCH_##which)
153 #define SMMU_PTB_ASID_CUR(n) \
154 ((n) << SMMU_PTB_ASID_CURRENT_SHIFT)
156 #define SMMU_TLB_FLUSH_ALL 0
158 #define SMMU_TLB_FLUSH_ASID_MATCH_disable \
159 (SMMU_TLB_FLUSH_ASID_MATCH_DISABLE << \
160 SMMU_TLB_FLUSH_ASID_MATCH_SHIFT)
161 #define SMMU_TLB_FLUSH_ASID_MATCH__ENABLE \
162 (SMMU_TLB_FLUSH_ASID_MATCH_ENABLE << \
163 SMMU_TLB_FLUSH_ASID_MATCH_SHIFT)
165 #define SMMU_PAGE_SHIFT 12
166 #define SMMU_PAGE_SIZE (1 << SMMU_PAGE_SHIFT)
168 #define SMMU_PDIR_COUNT 1024
169 #define SMMU_PDIR_SIZE (sizeof(u32) * SMMU_PDIR_COUNT)
170 #define SMMU_PTBL_COUNT 1024
171 #define SMMU_PTBL_SIZE (sizeof(u32) * SMMU_PTBL_COUNT)
172 #define SMMU_PDIR_SHIFT 12
173 #define SMMU_PDE_SHIFT 12
174 #define SMMU_PTE_SHIFT 12
175 #define SMMU_PFN_MASK 0x0fffffff
177 #define SMMU_ADDR_TO_PTN(addr) (((addr) >> 12) & (BIT(10) - 1))
178 #define SMMU_ADDR_TO_PDN(addr) ((addr) >> 22)
179 #define SMMU_PDN_TO_ADDR(pdn) ((pdn) << 22)
181 #define _READABLE (1 << SMMU_PTB_DATA_ASID_READABLE_SHIFT)
182 #define _WRITABLE (1 << SMMU_PTB_DATA_ASID_WRITABLE_SHIFT)
183 #define _NONSECURE (1 << SMMU_PTB_DATA_ASID_NONSECURE_SHIFT)
184 #define _PDE_NEXT (1 << SMMU_PDE_NEXT_SHIFT)
185 #define _MASK_ATTR (_READABLE | _WRITABLE | _NONSECURE)
187 #define _PDIR_ATTR (_READABLE | _WRITABLE | _NONSECURE)
189 #define _PDE_ATTR (_READABLE | _WRITABLE | _NONSECURE)
190 #define _PDE_ATTR_N (_PDE_ATTR | _PDE_NEXT)
191 #define _PDE_VACANT(pdn) (0)
193 #define _PTE_ATTR (_READABLE | _WRITABLE | _NONSECURE)
194 #define _PTE_VACANT(addr) (0)
196 #ifdef CONFIG_TEGRA_IOMMU_SMMU_LINEAR
199 #define _PDE_VACANT(pdn) (((pdn) << 10) | _PDE_ATTR)
200 #define _PTE_VACANT(addr) (((addr) >> SMMU_PAGE_SHIFT) | _PTE_ATTR)
203 #define SMMU_MK_PDIR(page, attr) \
204 ((page_to_phys(page) >> SMMU_PDIR_SHIFT) | (attr))
205 #define SMMU_MK_PDE(page, attr) \
206 (u32)((page_to_phys(page) >> SMMU_PDE_SHIFT) | (attr))
207 #define SMMU_EX_PTBL_PAGE(pde) \
208 pfn_to_page((u32)(pde) & SMMU_PFN_MASK)
209 #define SMMU_PFN_TO_PTE(pfn, attr) (u32)((pfn) | (attr))
211 #define SMMU_ASID_ENABLE(asid) ((asid) | (1 << 31))
212 #define SMMU_ASID_DISABLE 0
213 #define SMMU_ASID_ASID(n) ((n) & ~SMMU_ASID_ENABLE(0))
215 /* FIXME: client ID, only valid for T124 */
217 #define CSR_DISPLAY0A 1
218 #define CSR_DISPLAY0AB 2
219 #define CSR_DISPLAY0B 3
220 #define CSR_DISPLAY0BB 4
221 #define CSR_DISPLAY0C 5
222 #define CSR_DISPLAY0CB 6
224 #define CSR_AVPCARM7R 15
225 #define CSR_DISPLAYHC 16
226 #define CSR_DISPLAYHCB 17
228 #define CSR_HOST1XDMAR 22
229 #define CSR_HOST1XR 23
230 #define CSR_MSENCSRD 28
231 #define CSR_PPCSAHBDMAR 29
232 #define CSR_PPCSAHBSLVR 30
234 #define CSR_VDEBSEVR 34
235 #define CSR_VDEMBER 35
236 #define CSR_VDEMCER 36
237 #define CSR_VDETPER 37
238 #define CSR_MPCORELPR 38
239 #define CSR_MPCORER 39
240 #define CSW_MSENCSWR 43
242 #define CSW_AVPCARM7W 50
244 #define CSW_HOST1XW 54
245 #define CSW_MPCORELPW 56
246 #define CSW_MPCOREW 57
247 #define CSW_PPCSAHBDMAW 59
248 #define CSW_PPCSAHBSLVW 60
250 #define CSW_VDEBSEVW 62
251 #define CSW_VDEDBGW 63
252 #define CSW_VDEMBEW 64
253 #define CSW_VDETPMW 65
257 #define CSR_XUSB_HOSTR 74
258 #define CSW_XUSB_HOSTW 75
259 #define CSR_XUSB_DEVR 76
260 #define CSW_XUSB_DEVW 77
261 #define CSR_ISPRAB 78
262 #define CSW_ISPWAB 80
263 #define CSW_ISPWBB 81
264 #define CSR_TSECSRD 84
265 #define CSW_TSECSWR 85
266 #define CSR_A9AVPSCR 86
267 #define CSW_A9AVPSCW 87
268 #define CSR_GPUSRD 88
269 #define CSW_GPUSWR 89
270 #define CSR_DISPLAYT 90
271 #define CSR_SDMMCRA 96
272 #define CSR_SDMMCRAA 97
273 #define CSR_SDMMCR 98
274 #define CSR_SDMMCRAB 99
275 #define CSW_SDMMCWA 100
276 #define CSW_SDMMCWAA 101
277 #define CSW_SDMMCW 102
278 #define CSW_SDMMCWAB 103
279 #define CSR_VICSRD 108
280 #define CSW_VICSWR 109
282 #define CSR_DISPLAYD 115
284 #define SMMU_CLIENT_CONF0 0x40
286 #define smmu_client_enable_hwgrp(c, m) smmu_client_set_hwgrp(c, m, 1)
287 #define smmu_client_disable_hwgrp(c) smmu_client_set_hwgrp(c, 0, 0)
288 #define __smmu_client_enable_hwgrp(c, m) __smmu_client_set_hwgrp(c, m, 1)
289 #define __smmu_client_disable_hwgrp(c) __smmu_client_set_hwgrp(c, 0, 0)
291 static struct device *save_smmu_device;
293 static size_t smmu_flush_all_th_pages = SZ_512; /* number of threshold pages */
295 static const u32 smmu_asid_security_ofs[] = {
297 SMMU_ASID_SECURITY_1,
298 SMMU_ASID_SECURITY_2,
299 SMMU_ASID_SECURITY_3,
300 SMMU_ASID_SECURITY_4,
301 SMMU_ASID_SECURITY_5,
302 SMMU_ASID_SECURITY_6,
303 SMMU_ASID_SECURITY_7,
306 static size_t tegra_smmu_get_offset_base(int id)
309 return SMMU_SWGRP_ASID_BASE;
312 return 0xa88 - SWGID_DC12 * sizeof(u32);
314 return 0x490 - SWGID_DC14 * sizeof(u32);
318 * Per client for address space
322 struct list_head list;
331 struct smmu_device *smmu; /* back pointer to container */
333 spinlock_t lock; /* for pagetable */
334 struct page *pdir_page;
338 unsigned int *pte_count;
340 struct list_head client;
341 spinlock_t client_lock; /* for client list */
344 struct smmu_debugfs_info {
345 struct smmu_device *smmu;
351 * Per SMMU device - IOMMU device
354 void __iomem *regs, *regs_ahbarb;
355 unsigned long iovmm_base; /* remappable base address */
356 unsigned long page_count; /* total remappable size */
360 u64 swgids; /* memory client ID bitmap */
361 size_t ptc_cache_size;
362 struct page *avp_vector_page; /* dummy page shared by all AS's */
365 * Register image savers for suspend/resume
367 int num_translation_enable;
368 u32 translation_enable[4];
369 int num_asid_security;
370 u32 asid_security[8];
372 struct dentry *debugfs_root;
373 struct smmu_debugfs_info *debugfs_info;
376 struct smmu_as as[0]; /* Run-time allocated array */
379 static struct smmu_device *smmu_handle; /* unique for a system */
382 * SMMU/AHB register accessors
384 static inline u32 smmu_read(struct smmu_device *smmu, size_t offs)
386 return readl(smmu->regs + offs);
388 static inline void smmu_write(struct smmu_device *smmu, u32 val, size_t offs)
390 writel(val, smmu->regs + offs);
393 static inline u32 ahb_read(struct smmu_device *smmu, size_t offs)
395 return readl(smmu->regs_ahbarb + offs);
397 static inline void ahb_write(struct smmu_device *smmu, u32 val, size_t offs)
399 writel(val, smmu->regs_ahbarb + offs);
402 static void __smmu_client_ordered(struct smmu_device *smmu, int id)
407 offs = SMMU_CLIENT_CONF0;
408 offs += (id / BITS_PER_LONG) * sizeof(u32);
410 val = smmu_read(smmu, offs);
411 val |= BIT(id % BITS_PER_LONG);
412 smmu_write(smmu, val, offs);
415 static void smmu_client_ordered(struct smmu_device *smmu)
418 /* Add client ID here to be ordered */
421 for (i = 0; i < ARRAY_SIZE(id); i++)
422 __smmu_client_ordered(smmu, id[i]);
425 #define VA_PAGE_TO_PA(va, page) \
426 (page_to_phys(page) + ((unsigned long)(va) & ~PAGE_MASK))
428 #define VA_PAGE_TO_PA_HI(va, page) \
429 (u32)((u64)(page_to_phys(page)) >> 32)
431 #define FLUSH_CPU_DCACHE(va, page, size) \
433 unsigned long _pa_ = VA_PAGE_TO_PA(va, page); \
434 __cpuc_flush_dcache_area((void *)(va), (size_t)(size)); \
435 outer_flush_range(_pa_, _pa_+(size_t)(size)); \
439 * Any interaction between any block on PPSB and a block on APB or AHB
440 * must have these read-back barriers to ensure the APB/AHB bus
441 * transaction is complete before initiating activity on the PPSB
444 #define FLUSH_SMMU_REGS(smmu) smmu_read(smmu, SMMU_CONFIG)
446 static u64 tegra_smmu_of_get_swgids(struct device *dev)
449 const char *propname = "nvidia,memory-clients";
454 prop = of_get_property(dev->of_node, propname, &bytes);
458 for (i = 0; i < bytes / sizeof(u32); i++, prop++)
459 swgids |= 1ULL << be32_to_cpup(prop);
464 static int __smmu_client_set_hwgrp(struct smmu_client *c, u64 map, int on)
467 struct smmu_as *as = c->as;
468 u32 val, offs, mask = SMMU_ASID_ENABLE(as->asid);
469 struct smmu_device *smmu = as->smmu;
477 for_each_set_bit(i, (unsigned long *)&map, HWGRP_COUNT) {
479 /* FIXME: PCIe client hasn't been registered as IOMMU */
483 offs = i * sizeof(u32) + tegra_smmu_get_offset_base(i);
484 val = smmu_read(smmu, offs);
485 val &= ~3; /* always overwrite ASID */
489 else if (list_empty(&c->list))
490 val = 0; /* turn off if this is the last */
492 return 0; /* leave if off but not the last */
494 smmu_write(smmu, val, offs);
496 dev_dbg(c->dev, "swgid:%d asid:%d %s @%s\n",
497 i, val & 3, (val & BIT(31)) ? "Enabled" : "Disabled",
500 FLUSH_SMMU_REGS(smmu);
506 static int smmu_client_set_hwgrp(struct smmu_client *c, u64 map, int on)
510 struct smmu_as *as = c->as;
511 struct smmu_device *smmu = as->smmu;
513 spin_lock_irqsave(&smmu->lock, flags);
514 val = __smmu_client_set_hwgrp(c, map, on);
515 spin_unlock_irqrestore(&smmu->lock, flags);
520 * Flush all TLB entries and all PTC entries
521 * Caller must lock smmu
523 static void smmu_flush_regs(struct smmu_device *smmu, int enable)
527 smmu_write(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH);
528 FLUSH_SMMU_REGS(smmu);
529 val = SMMU_TLB_FLUSH_VA_MATCH_ALL |
530 SMMU_TLB_FLUSH_ASID_MATCH_disable;
531 smmu_write(smmu, val, SMMU_TLB_FLUSH);
534 smmu_write(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG);
535 FLUSH_SMMU_REGS(smmu);
538 static void smmu_setup_regs(struct smmu_device *smmu)
543 for (i = 0; i < smmu->num_as; i++) {
544 struct smmu_as *as = &smmu->as[i];
545 struct smmu_client *c;
547 smmu_write(smmu, SMMU_PTB_ASID_CUR(as->asid), SMMU_PTB_ASID);
548 val = as->pdir_page ?
549 SMMU_MK_PDIR(as->pdir_page, as->pdir_attr) :
550 SMMU_PTB_DATA_RESET_VAL;
551 smmu_write(smmu, val, SMMU_PTB_DATA);
553 list_for_each_entry(c, &as->client, list)
554 __smmu_client_set_hwgrp(c, c->swgids, 1);
557 for (i = 0; i < smmu->num_translation_enable; i++)
558 smmu_write(smmu, smmu->translation_enable[i],
559 SMMU_TRANSLATION_ENABLE_0 + i * sizeof(u32));
561 for (i = 0; i < smmu->num_asid_security; i++)
563 smmu->asid_security[i], smmu_asid_security_ofs[i]);
565 val = SMMU_PTC_CONFIG_RESET_VAL;
566 if (IS_ENABLED(CONFIG_ARCH_TEGRA_12x_SOC) &&
567 (tegra_get_chipid() == TEGRA_CHIPID_TEGRA12))
568 val |= SMMU_PTC_REQ_LIMIT;
570 smmu_write(smmu, val, SMMU_CACHE_CONFIG(_PTC));
572 val = SMMU_TLB_CONFIG_RESET_VAL;
573 if (IS_ENABLED(CONFIG_ARCH_TEGRA_12x_SOC) &&
574 (tegra_get_chipid() == TEGRA_CHIPID_TEGRA12)) {
575 val |= SMMU_TLB_RR_ARB;
576 val |= SMMU_TLB_CONFIG_ACTIVE_LINES__VALUE << 1;
578 val |= SMMU_TLB_CONFIG_ACTIVE_LINES__VALUE;
581 smmu_write(smmu, val, SMMU_CACHE_CONFIG(_TLB));
583 if (IS_ENABLED(CONFIG_ARCH_TEGRA_12x_SOC) &&
584 (tegra_get_chipid() == TEGRA_CHIPID_TEGRA12))
585 smmu_client_ordered(smmu);
587 smmu_flush_regs(smmu, 1);
589 if (tegra_get_chipid() == TEGRA_CHIPID_TEGRA3
590 || tegra_get_chipid() == TEGRA_CHIPID_TEGRA11
591 || tegra_get_chipid() == TEGRA_CHIPID_TEGRA14) {
592 val = ahb_read(smmu, AHB_XBAR_CTRL);
593 val |= AHB_XBAR_CTRL_SMMU_INIT_DONE_DONE <<
594 AHB_XBAR_CTRL_SMMU_INIT_DONE_SHIFT;
595 ahb_write(smmu, val, AHB_XBAR_CTRL);
600 static void __smmu_flush_ptc(struct smmu_device *smmu, u32 *pte,
606 smmu_write(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH);
610 if (IS_ENABLED(CONFIG_ARCH_TEGRA_12x_SOC) &&
611 (tegra_get_chipid() == TEGRA_CHIPID_TEGRA12)) {
612 val = VA_PAGE_TO_PA_HI(pte, page);
613 smmu_write(smmu, val, SMMU_PTC_FLUSH_1);
616 val = SMMU_PTC_FLUSH_TYPE_ADR | VA_PAGE_TO_PA(pte, page);
617 smmu_write(smmu, val, SMMU_PTC_FLUSH);
620 static void smmu_flush_ptc(struct smmu_device *smmu, u32 *pte,
623 __smmu_flush_ptc(smmu, pte, page);
624 FLUSH_SMMU_REGS(smmu);
627 static inline void __smmu_flush_ptc_all(struct smmu_device *smmu)
629 __smmu_flush_ptc(smmu, 0, NULL);
632 static void __smmu_flush_tlb(struct smmu_device *smmu, struct smmu_as *as,
633 dma_addr_t iova, int is_pde)
638 val = SMMU_TLB_FLUSH_VA(iova, SECTION);
640 val = SMMU_TLB_FLUSH_VA(iova, GROUP);
642 smmu_write(smmu, val, SMMU_TLB_FLUSH);
645 static inline void __smmu_flush_tlb_section(struct smmu_as *as, dma_addr_t iova)
647 __smmu_flush_tlb(as->smmu, as, iova, 1);
650 static void flush_ptc_and_tlb(struct smmu_device *smmu,
651 struct smmu_as *as, dma_addr_t iova,
652 u32 *pte, struct page *page, int is_pde)
654 __smmu_flush_ptc(smmu, pte, page);
655 __smmu_flush_tlb(smmu, as, iova, is_pde);
656 FLUSH_SMMU_REGS(smmu);
659 #ifdef CONFIG_TEGRA_ERRATA_1053704
660 /* Flush PTEs within the same L2 pagetable */
661 static void ____smmu_flush_tlb_range(struct smmu_device *smmu, dma_addr_t iova,
664 size_t unit = SZ_16K;
666 iova = round_down(iova, unit);
670 val = SMMU_TLB_FLUSH_VA(iova, GROUP);
671 smmu_write(smmu, val, SMMU_TLB_FLUSH);
677 static void flush_ptc_and_tlb_range(struct smmu_device *smmu,
678 struct smmu_as *as, dma_addr_t iova,
679 u32 *pte, struct page *page,
682 size_t unit = SZ_16K;
683 dma_addr_t end = iova + count * PAGE_SIZE;
685 iova = round_down(iova, unit);
689 __smmu_flush_ptc(smmu, pte, page);
690 pte += smmu->ptc_cache_size / PAGE_SIZE;
692 for (i = 0; i < smmu->ptc_cache_size / unit; i++) {
695 val = SMMU_TLB_FLUSH_VA(iova, GROUP);
696 smmu_write(smmu, val, SMMU_TLB_FLUSH);
701 FLUSH_SMMU_REGS(smmu);
704 static inline void flush_ptc_and_tlb_all(struct smmu_device *smmu,
707 flush_ptc_and_tlb(smmu, as, 0, 0, NULL, 1);
710 static void free_ptbl(struct smmu_as *as, dma_addr_t iova, bool flush)
712 int pdn = SMMU_ADDR_TO_PDN(iova);
713 u32 *pdir = (u32 *)page_address(as->pdir_page);
715 if (pdir[pdn] != _PDE_VACANT(pdn)) {
716 dev_dbg(as->smmu->dev, "pdn: %x\n", pdn);
718 ClearPageReserved(SMMU_EX_PTBL_PAGE(pdir[pdn]));
719 __free_page(SMMU_EX_PTBL_PAGE(pdir[pdn]));
720 pdir[pdn] = _PDE_VACANT(pdn);
721 FLUSH_CPU_DCACHE(&pdir[pdn], as->pdir_page, sizeof pdir[pdn]);
725 flush_ptc_and_tlb(as->smmu, as, iova, &pdir[pdn],
730 #ifdef CONFIG_TEGRA_ERRATA_1053704
731 static void __smmu_flush_tlb_range(struct smmu_as *as, dma_addr_t iova,
735 struct smmu_device *smmu = as->smmu;
737 if (!pfn_valid(page_to_pfn(as->pdir_page)))
740 pdir = page_address(as->pdir_page);
742 int pdn = SMMU_ADDR_TO_PDN(iova);
744 if (pdir[pdn] & _PDE_NEXT) {
745 struct page *page = SMMU_EX_PTBL_PAGE(pdir[pdn]);
746 dma_addr_t _end = min_t(dma_addr_t, end,
747 SMMU_PDN_TO_ADDR(pdn + 1));
749 if (pfn_valid(page_to_pfn(page)))
750 ____smmu_flush_tlb_range(smmu, iova, _end);
755 __smmu_flush_tlb_section(as, iova);
757 iova = SMMU_PDN_TO_ADDR(pdn + 1);
760 if (pdn == SMMU_PTBL_COUNT - 1)
765 static void __smmu_flush_tlb_as(struct smmu_as *as, dma_addr_t iova,
768 __smmu_flush_tlb_range(as, iova, end);
771 static void __smmu_flush_tlb_as(struct smmu_as *as, dma_addr_t iova,
775 struct smmu_device *smmu = as->smmu;
777 val = SMMU_TLB_FLUSH_ASID_ENABLE |
778 (as->asid << SMMU_TLB_FLUSH_ASID_SHIFT(as));
779 smmu_write(smmu, val, SMMU_TLB_FLUSH);
783 static void flush_ptc_and_tlb_as(struct smmu_as *as, dma_addr_t start,
786 struct smmu_device *smmu = as->smmu;
788 __smmu_flush_ptc_all(smmu);
789 __smmu_flush_tlb_as(as, start, end);
790 FLUSH_SMMU_REGS(smmu);
793 static void free_pdir(struct smmu_as *as)
797 struct device *dev = as->smmu->dev;
802 addr = as->smmu->iovmm_base;
803 count = as->smmu->page_count;
804 while (count-- > 0) {
805 free_ptbl(as, addr, 1);
806 addr += SMMU_PAGE_SIZE * SMMU_PTBL_COUNT;
808 ClearPageReserved(as->pdir_page);
809 __free_page(as->pdir_page);
810 as->pdir_page = NULL;
811 devm_kfree(dev, as->pte_count);
812 as->pte_count = NULL;
815 static struct page *alloc_ptbl(struct smmu_as *as, dma_addr_t iova, bool flush)
818 u32 *pdir = page_address(as->pdir_page);
819 int pdn = SMMU_ADDR_TO_PDN(iova);
820 unsigned long addr = SMMU_PDN_TO_ADDR(pdn);
823 gfp_t gfp = GFP_ATOMIC;
825 if (IS_ENABLED(CONFIG_PREEMPT) && !in_atomic())
828 /* Vacant - allocate a new page table */
829 dev_dbg(as->smmu->dev, "New PTBL pdn: %x\n", pdn);
831 page = alloc_page(gfp);
835 SetPageReserved(page);
836 ptbl = (u32 *)page_address(page);
837 for (i = 0; i < SMMU_PTBL_COUNT; i++) {
838 ptbl[i] = _PTE_VACANT(addr);
839 addr += SMMU_PAGE_SIZE;
842 FLUSH_CPU_DCACHE(ptbl, page, SMMU_PTBL_SIZE);
843 pdir[pdn] = SMMU_MK_PDE(page, as->pde_attr | _PDE_NEXT);
844 FLUSH_CPU_DCACHE(&pdir[pdn], as->pdir_page, sizeof pdir[pdn]);
846 flush_ptc_and_tlb(as->smmu, as, iova, &pdir[pdn],
852 * Maps PTBL for given iova and returns the PTE address
853 * Caller must unmap the mapped PTBL returned in *ptbl_page_p
855 static u32 *locate_pte(struct smmu_as *as,
856 dma_addr_t iova, bool allocate,
857 struct page **ptbl_page_p,
858 unsigned int **count)
860 int ptn = SMMU_ADDR_TO_PTN(iova);
861 int pdn = SMMU_ADDR_TO_PDN(iova);
862 u32 *pdir = page_address(as->pdir_page);
865 if (pdir[pdn] != _PDE_VACANT(pdn)) {
866 /* Mapped entry table already exists */
867 *ptbl_page_p = SMMU_EX_PTBL_PAGE(pdir[pdn]);
868 } else if (!allocate) {
871 *ptbl_page_p = alloc_ptbl(as, iova, 1);
876 ptbl = page_address(*ptbl_page_p);
877 *count = &as->pte_count[pdn];
881 #ifdef CONFIG_SMMU_SIG_DEBUG
882 static void put_signature(struct smmu_as *as,
883 dma_addr_t iova, unsigned long pfn)
888 page = pfn_to_page(pfn);
889 vaddr = page_address(page);
894 vaddr[1] = pfn << PAGE_SHIFT;
895 FLUSH_CPU_DCACHE(vaddr, page, sizeof(vaddr[0]) * 2);
898 static inline void put_signature(struct smmu_as *as,
899 unsigned long addr, unsigned long pfn)
905 * Caller must not hold as->lock
907 static int alloc_pdir(struct smmu_as *as)
913 struct smmu_device *smmu = as->smmu;
918 * do the allocation, then grab as->lock
920 cnt = devm_kzalloc(smmu->dev,
921 sizeof(cnt[0]) * SMMU_PDIR_COUNT,
923 page = alloc_page(GFP_KERNEL | __GFP_DMA);
925 spin_lock_irqsave(&as->lock, flags);
928 /* We raced, free the redundant */
934 dev_err(smmu->dev, "failed to allocate at %s\n", __func__);
939 as->pdir_page = page;
942 SetPageReserved(as->pdir_page);
943 pdir = page_address(as->pdir_page);
945 for (pdn = 0; pdn < SMMU_PDIR_COUNT; pdn++)
946 pdir[pdn] = _PDE_VACANT(pdn);
947 FLUSH_CPU_DCACHE(pdir, as->pdir_page, SMMU_PDIR_SIZE);
948 smmu_flush_ptc(smmu, pdir, as->pdir_page);
949 val = SMMU_TLB_FLUSH_VA_MATCH_ALL |
950 SMMU_TLB_FLUSH_ASID_MATCH__ENABLE |
951 (as->asid << SMMU_TLB_FLUSH_ASID_SHIFT(as));
952 smmu_write(smmu, val, SMMU_TLB_FLUSH);
953 FLUSH_SMMU_REGS(as->smmu);
955 spin_unlock_irqrestore(&as->lock, flags);
960 spin_unlock_irqrestore(&as->lock, flags);
965 devm_kfree(smmu->dev, cnt);
969 static size_t __smmu_iommu_unmap_pages(struct smmu_as *as, dma_addr_t iova,
972 int total = bytes >> PAGE_SHIFT;
973 u32 *pdir = page_address(as->pdir_page);
974 struct smmu_device *smmu = as->smmu;
975 unsigned long iova_base = iova;
976 bool flush_all = (total > smmu_flush_all_th_pages) ? true : false;
979 int ptn = SMMU_ADDR_TO_PTN(iova);
980 int pdn = SMMU_ADDR_TO_PDN(iova);
981 struct page *page = SMMU_EX_PTBL_PAGE(pdir[pdn]);
986 if (!pfn_valid(page_to_pfn(page))) {
987 total -= SMMU_PDN_TO_ADDR(pdn + 1) - iova;
988 iova = SMMU_PDN_TO_ADDR(pdn + 1);
992 ptbl = page_address(page);
994 count = min_t(int, SMMU_PTBL_COUNT - ptn, total);
996 dev_dbg(as->smmu->dev, "unmapping %d pages at once\n", count);
999 unsigned int *rest = &as->pte_count[pdn];
1000 size_t bytes = sizeof(*pte) * count;
1002 memset(pte, 0, bytes);
1003 FLUSH_CPU_DCACHE(pte, page, bytes);
1007 free_ptbl(as, iova, !flush_all);
1010 flush_ptc_and_tlb_range(smmu, as, iova, pte,
1014 iova += PAGE_SIZE * count;
1019 flush_ptc_and_tlb_as(as, iova_base,
1025 static size_t __smmu_iommu_unmap_largepage(struct smmu_as *as, dma_addr_t iova)
1027 int pdn = SMMU_ADDR_TO_PDN(iova);
1028 u32 *pdir = (u32 *)page_address(as->pdir_page);
1030 pdir[pdn] = _PDE_VACANT(pdn);
1031 FLUSH_CPU_DCACHE(&pdir[pdn], as->pdir_page, sizeof pdir[pdn]);
1032 flush_ptc_and_tlb(as->smmu, as, iova, &pdir[pdn], as->pdir_page, 1);
1036 static int __smmu_iommu_map_pfn(struct smmu_as *as, dma_addr_t iova,
1037 unsigned long pfn, int prot)
1039 struct smmu_device *smmu = as->smmu;
1041 unsigned int *count;
1043 int attrs = as->pte_attr;
1045 pte = locate_pte(as, iova, true, &page, &count);
1049 if (*pte == _PTE_VACANT(iova))
1052 if (dma_get_attr(DMA_ATTR_READ_ONLY, (struct dma_attrs *)prot))
1053 attrs &= ~_WRITABLE;
1054 else if (dma_get_attr(DMA_ATTR_WRITE_ONLY, (struct dma_attrs *)prot))
1055 attrs &= ~_READABLE;
1057 *pte = SMMU_PFN_TO_PTE(pfn, attrs);
1058 FLUSH_CPU_DCACHE(pte, page, sizeof(*pte));
1059 flush_ptc_and_tlb(smmu, as, iova, pte, page, 0);
1060 put_signature(as, iova, pfn);
1064 static int __smmu_iommu_map_page(struct smmu_as *as, dma_addr_t iova,
1065 phys_addr_t pa, int prot)
1067 unsigned long pfn = __phys_to_pfn(pa);
1069 return __smmu_iommu_map_pfn(as, iova, pfn, prot);
1072 static int __smmu_iommu_map_largepage(struct smmu_as *as, dma_addr_t iova,
1073 phys_addr_t pa, int prot)
1075 int pdn = SMMU_ADDR_TO_PDN(iova);
1076 u32 *pdir = (u32 *)page_address(as->pdir_page);
1077 int attrs = _PDE_ATTR;
1079 if (pdir[pdn] != _PDE_VACANT(pdn))
1082 if (dma_get_attr(DMA_ATTR_READ_ONLY, (struct dma_attrs *)prot))
1083 attrs &= ~_WRITABLE;
1084 else if (dma_get_attr(DMA_ATTR_WRITE_ONLY, (struct dma_attrs *)prot))
1085 attrs &= ~_READABLE;
1087 pdir[pdn] = SMMU_ADDR_TO_PDN(pa) << 10 | attrs;
1088 FLUSH_CPU_DCACHE(&pdir[pdn], as->pdir_page, sizeof pdir[pdn]);
1089 flush_ptc_and_tlb(as->smmu, as, iova, &pdir[pdn], as->pdir_page, 1);
1094 static int smmu_iommu_map(struct iommu_domain *domain, unsigned long iova,
1095 phys_addr_t pa, size_t bytes, int prot)
1097 struct smmu_as *as = domain->priv;
1098 unsigned long flags;
1100 int (*fn)(struct smmu_as *as, dma_addr_t iova, phys_addr_t pa,
1103 dev_dbg(as->smmu->dev, "[%d] %08lx:%llx\n", as->asid, iova, (u64)pa);
1107 fn = __smmu_iommu_map_page;
1110 fn = __smmu_iommu_map_largepage;
1113 WARN(1, "%d not supported\n", bytes);
1117 spin_lock_irqsave(&as->lock, flags);
1118 err = fn(as, iova, pa, prot);
1119 spin_unlock_irqrestore(&as->lock, flags);
1123 static int smmu_iommu_map_pages(struct iommu_domain *domain, unsigned long iova,
1124 struct page **pages, size_t total, int prot)
1126 struct smmu_as *as = domain->priv;
1127 struct smmu_device *smmu = as->smmu;
1128 unsigned long flags;
1129 u32 *pdir = page_address(as->pdir_page);
1131 unsigned long iova_base = iova;
1132 bool flush_all = (total > smmu_flush_all_th_pages) ? true : false;
1133 int attrs = as->pte_attr;
1135 if (dma_get_attr(DMA_ATTR_READ_ONLY, (struct dma_attrs *)prot))
1136 attrs &= ~_WRITABLE;
1137 else if (dma_get_attr(DMA_ATTR_WRITE_ONLY, (struct dma_attrs *)prot))
1138 attrs &= ~_READABLE;
1140 spin_lock_irqsave(&as->lock, flags);
1143 int pdn = SMMU_ADDR_TO_PDN(iova);
1144 int ptn = SMMU_ADDR_TO_PTN(iova);
1145 unsigned int *rest = &as->pte_count[pdn];
1146 int count = min_t(size_t, SMMU_PTBL_COUNT - ptn, total);
1147 struct page *tbl_page;
1152 if (pdir[pdn] == _PDE_VACANT(pdn)) {
1153 tbl_page = alloc_ptbl(as, iova, !flush_all);
1160 tbl_page = SMMU_EX_PTBL_PAGE(pdir[pdn]);
1163 if (WARN_ON(!pfn_valid(page_to_pfn(tbl_page))))
1166 ptbl = page_address(tbl_page);
1167 for (i = 0; i < count; i++) {
1168 pte = &ptbl[ptn + i];
1170 if (*pte == _PTE_VACANT(iova + i * PAGE_SIZE))
1173 *pte = SMMU_PFN_TO_PTE(page_to_pfn(pages[i]), attrs);
1177 FLUSH_CPU_DCACHE(pte, tbl_page, count * sizeof(u32 *));
1179 flush_ptc_and_tlb_range(smmu, as, iova, pte, tbl_page,
1182 iova += PAGE_SIZE * count;
1189 flush_ptc_and_tlb_as(as, iova_base,
1190 iova_base + total * PAGE_SIZE);
1192 spin_unlock_irqrestore(&as->lock, flags);
1196 static int smmu_iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
1197 struct scatterlist *sgl, int nents, int prot)
1199 unsigned long flags;
1201 struct scatterlist *s;
1203 unsigned long iova_base = iova;
1204 bool flush_all = (nents > smmu_flush_all_th_pages) ? true : false;
1205 struct smmu_as *as = domain->priv;
1206 struct smmu_device *smmu = as->smmu;
1207 int attrs = as->pte_attr;
1209 if (dma_get_attr(DMA_ATTR_READ_ONLY, (struct dma_attrs *)prot))
1210 attrs &= ~_WRITABLE;
1211 else if (dma_get_attr(DMA_ATTR_WRITE_ONLY, (struct dma_attrs *)prot))
1212 attrs &= ~_READABLE;
1214 spin_lock_irqsave(&as->lock, flags);
1215 for (count = 0, s = sgl; count < nents; s = sg_next(s)) {
1216 phys_addr_t phys = page_to_phys(sg_page(s));
1217 unsigned int len = PAGE_ALIGN(s->offset + s->length);
1220 int pfn = __phys_to_pfn(phys);
1222 u32 *pdir = page_address(as->pdir_page);
1223 int ptn = SMMU_ADDR_TO_PTN(iova);
1224 int pdn = SMMU_ADDR_TO_PDN(iova);
1226 struct page *tbl_page;
1227 size_t num = min_t(int, SMMU_PTBL_COUNT - ptn,
1231 if (pdir[pdn] != _PDE_VACANT(pdn)) {
1232 tbl_page = SMMU_EX_PTBL_PAGE(pdir[pdn]);
1234 tbl_page = alloc_ptbl(as, iova, !flush_all);
1241 if (WARN_ON(!pfn_valid(page_to_pfn(tbl_page))))
1244 ptbl = page_address(tbl_page);
1245 for (i = 0; i < num; i++) {
1246 pte = &ptbl[ptn + i];
1247 if (*pte == _PTE_VACANT(iova + i * PAGE_SIZE)) {
1250 rest = &as->pte_count[pdn];
1254 *pte = SMMU_PFN_TO_PTE(pfn + i, attrs);
1258 FLUSH_CPU_DCACHE(pte, tbl_page, num * sizeof(*pte));
1260 flush_ptc_and_tlb_range(smmu, as, iova, pte,
1264 iova += num * PAGE_SIZE;
1265 phys += num * PAGE_SIZE;
1266 len -= num * PAGE_SIZE;
1273 flush_ptc_and_tlb_as(as, iova_base,
1274 iova_base + nents * PAGE_SIZE);
1276 spin_unlock_irqrestore(&as->lock, flags);
1280 static int __smmu_iommu_unmap(struct smmu_as *as, dma_addr_t iova,
1283 int pdn = SMMU_ADDR_TO_PDN(iova);
1284 u32 *pdir = page_address(as->pdir_page);
1286 if (!(pdir[pdn] & _PDE_NEXT))
1287 return __smmu_iommu_unmap_largepage(as, iova);
1289 return __smmu_iommu_unmap_pages(as, iova, bytes);
1292 static size_t smmu_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
1295 struct smmu_as *as = domain->priv;
1296 unsigned long flags;
1299 dev_dbg(as->smmu->dev, "[%d] %08lx\n", as->asid, iova);
1301 spin_lock_irqsave(&as->lock, flags);
1302 unmapped = __smmu_iommu_unmap(as, iova, bytes);
1303 spin_unlock_irqrestore(&as->lock, flags);
1307 static phys_addr_t smmu_iommu_iova_to_phys(struct iommu_domain *domain,
1310 struct smmu_as *as = domain->priv;
1311 unsigned long flags;
1312 int pdn = SMMU_ADDR_TO_PDN(iova);
1313 u32 *pdir = page_address(as->pdir_page);
1316 spin_lock_irqsave(&as->lock, flags);
1318 if (pdir[pdn] & _PDE_NEXT) {
1320 unsigned int *count;
1323 pte = locate_pte(as, iova, false, &page, &count);
1325 unsigned long pfn = *pte & SMMU_PFN_MASK;
1329 pa = pdir[pdn] << SMMU_PDE_SHIFT;
1332 dev_dbg(as->smmu->dev,
1333 "iova:%08llx pfn:%08llx asid:%d\n", (unsigned long long)iova,
1336 spin_unlock_irqrestore(&as->lock, flags);
1340 static int smmu_iommu_domain_has_cap(struct iommu_domain *domain,
1346 static int smmu_iommu_attach_dev(struct iommu_domain *domain,
1349 struct smmu_as *as = domain->priv;
1350 struct smmu_device *smmu = as->smmu;
1351 struct smmu_client *client, *c;
1352 struct iommu_linear_map *area = NULL;
1356 map = tegra_smmu_of_get_swgids(dev);
1357 temp = tegra_smmu_fixup_swgids(dev, &area);
1362 if (map && temp && map != temp)
1363 dev_err(dev, "%llx %llx\n", map, temp);
1368 while (area && area->size) {
1369 DEFINE_DMA_ATTRS(attrs);
1370 size_t size = PAGE_ALIGN(area->size);
1372 dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs);
1373 err = dma_map_linear_attrs(dev, area->start, size, 0, &attrs);
1374 if (err == DMA_ERROR_CODE)
1375 dev_err(dev, "Failed to map %016llx(%x)\n",
1379 dev_info(dev, "map %016llx(%x)\n", (u64)area->start,
1385 map &= smmu->swgids;
1387 client = devm_kzalloc(smmu->dev, sizeof(*c), GFP_KERNEL);
1393 err = smmu_client_enable_hwgrp(client, map);
1397 spin_lock(&as->client_lock);
1398 list_for_each_entry(c, &as->client, list) {
1399 if (c->dev == dev) {
1401 "%s is already attached\n", dev_name(c->dev));
1406 list_add(&client->list, &as->client);
1407 spin_unlock(&as->client_lock);
1410 * Reserve "page zero" for AVP vectors using a common dummy
1413 if (map & SWGID(AVPC)) {
1416 page = as->smmu->avp_vector_page;
1417 __smmu_iommu_map_pfn(as, 0, page_to_pfn(page), 0);
1419 pr_debug("Reserve \"page zero\" \
1420 for AVP vectors using a common dummy\n");
1423 dev_dbg(smmu->dev, "%s is attached\n", dev_name(dev));
1427 smmu_client_disable_hwgrp(client);
1428 spin_unlock(&as->client_lock);
1430 devm_kfree(smmu->dev, client);
1434 static void smmu_iommu_detach_dev(struct iommu_domain *domain,
1437 struct smmu_as *as = domain->priv;
1438 struct smmu_device *smmu = as->smmu;
1439 struct smmu_client *c;
1441 spin_lock(&as->client_lock);
1443 list_for_each_entry(c, &as->client, list) {
1444 if (c->dev == dev) {
1446 smmu_client_disable_hwgrp(c);
1447 devm_kfree(smmu->dev, c);
1449 "%s is detached\n", dev_name(c->dev));
1453 dev_err(smmu->dev, "Couldn't find %s\n", dev_name(dev));
1455 spin_unlock(&as->client_lock);
1458 static int smmu_iommu_domain_init(struct iommu_domain *domain)
1460 int i, err = -EAGAIN;
1461 unsigned long flags;
1463 struct smmu_device *smmu = smmu_handle;
1465 /* Look for a free AS with lock held */
1466 for (i = 0; i < smmu->num_as; i++) {
1472 err = alloc_pdir(as);
1479 if (i == smmu->num_as)
1480 dev_err(smmu->dev, "no free AS\n");
1484 spin_lock_irqsave(&smmu->lock, flags);
1486 /* Update PDIR register */
1487 smmu_write(smmu, SMMU_PTB_ASID_CUR(as->asid), SMMU_PTB_ASID);
1489 SMMU_MK_PDIR(as->pdir_page, as->pdir_attr), SMMU_PTB_DATA);
1490 FLUSH_SMMU_REGS(smmu);
1492 spin_unlock_irqrestore(&smmu->lock, flags);
1496 domain->geometry.aperture_start = smmu->iovmm_base;
1497 domain->geometry.aperture_end = smmu->iovmm_base +
1498 smmu->page_count * SMMU_PAGE_SIZE - 1;
1499 domain->geometry.force_aperture = true;
1501 dev_dbg(smmu->dev, "smmu_as@%p\n", as);
1506 static void smmu_iommu_domain_destroy(struct iommu_domain *domain)
1508 struct smmu_as *as = domain->priv;
1509 struct smmu_device *smmu = as->smmu;
1510 unsigned long flags;
1512 spin_lock_irqsave(&as->lock, flags);
1514 if (as->pdir_page) {
1515 spin_lock(&smmu->lock);
1516 smmu_write(smmu, SMMU_PTB_ASID_CUR(as->asid), SMMU_PTB_ASID);
1517 smmu_write(smmu, SMMU_PTB_DATA_RESET_VAL, SMMU_PTB_DATA);
1518 FLUSH_SMMU_REGS(smmu);
1519 spin_unlock(&smmu->lock);
1524 if (!list_empty(&as->client)) {
1525 struct smmu_client *c;
1527 list_for_each_entry(c, &as->client, list)
1528 smmu_iommu_detach_dev(domain, c->dev);
1531 spin_unlock_irqrestore(&as->lock, flags);
1533 domain->priv = NULL;
1534 dev_dbg(smmu->dev, "smmu_as@%p\n", as);
1537 static struct iommu_ops smmu_iommu_ops = {
1538 .domain_init = smmu_iommu_domain_init,
1539 .domain_destroy = smmu_iommu_domain_destroy,
1540 .attach_dev = smmu_iommu_attach_dev,
1541 .detach_dev = smmu_iommu_detach_dev,
1542 .map = smmu_iommu_map,
1543 .map_pages = smmu_iommu_map_pages,
1544 .map_sg = smmu_iommu_map_sg,
1545 .unmap = smmu_iommu_unmap,
1546 .iova_to_phys = smmu_iommu_iova_to_phys,
1547 .domain_has_cap = smmu_iommu_domain_has_cap,
1548 .pgsize_bitmap = SMMU_IOMMU_PGSIZES,
1551 /* Should be in the order of enum */
1552 static const char * const smmu_debugfs_mc[] = { "mc", };
1553 static const char * const smmu_debugfs_cache[] = { "tlb", "ptc", };
1555 static ssize_t smmu_debugfs_stats_write(struct file *file,
1556 const char __user *buffer,
1557 size_t count, loff_t *pos)
1559 struct smmu_debugfs_info *info;
1560 struct smmu_device *smmu;
1567 const char * const command[] = {
1572 char str[] = "reset";
1576 count = min_t(size_t, count, sizeof(str));
1577 if (copy_from_user(str, buffer, count))
1580 for (i = 0; i < ARRAY_SIZE(command); i++)
1581 if (strncmp(str, command[i],
1582 strlen(command[i])) == 0)
1585 if (i == ARRAY_SIZE(command))
1588 info = file_inode(file)->i_private;
1591 offs = SMMU_CACHE_CONFIG(info->cache);
1592 val = smmu_read(smmu, offs);
1595 val &= ~SMMU_CACHE_CONFIG_STATS_ENABLE;
1596 val &= ~SMMU_CACHE_CONFIG_STATS_TEST;
1597 smmu_write(smmu, val, offs);
1600 val |= SMMU_CACHE_CONFIG_STATS_ENABLE;
1601 val &= ~SMMU_CACHE_CONFIG_STATS_TEST;
1602 smmu_write(smmu, val, offs);
1605 val |= SMMU_CACHE_CONFIG_STATS_TEST;
1606 smmu_write(smmu, val, offs);
1607 val &= ~SMMU_CACHE_CONFIG_STATS_TEST;
1608 smmu_write(smmu, val, offs);
1615 dev_dbg(smmu->dev, "%s() %08x, %08x @%08x\n", __func__,
1616 val, smmu_read(smmu, offs), offs);
1621 static int smmu_debugfs_stats_show(struct seq_file *s, void *v)
1623 struct smmu_debugfs_info *info = s->private;
1624 struct smmu_device *smmu = info->smmu;
1626 const char * const stats[] = { "hit", "miss", };
1629 for (i = 0; i < ARRAY_SIZE(stats); i++) {
1633 offs = SMMU_STATS_CACHE_COUNT(info->mc, info->cache, i);
1634 val = smmu_read(smmu, offs);
1635 seq_printf(s, "%s:%08x ", stats[i], val);
1637 dev_dbg(smmu->dev, "%s() %s %08x @%08x\n", __func__,
1638 stats[i], val, offs);
1640 seq_printf(s, "\n");
1644 static int smmu_debugfs_stats_open(struct inode *inode, struct file *file)
1646 return single_open(file, smmu_debugfs_stats_show, inode->i_private);
1649 static const struct file_operations smmu_debugfs_stats_fops = {
1650 .open = smmu_debugfs_stats_open,
1652 .llseek = seq_lseek,
1653 .release = single_release,
1654 .write = smmu_debugfs_stats_write,
1657 static void smmu_debugfs_delete(struct smmu_device *smmu)
1659 debugfs_remove_recursive(smmu->debugfs_root);
1660 kfree(smmu->debugfs_info);
1663 static void smmu_debugfs_create(struct smmu_device *smmu)
1667 struct dentry *root;
1669 bytes = ARRAY_SIZE(smmu_debugfs_mc) * ARRAY_SIZE(smmu_debugfs_cache) *
1670 sizeof(*smmu->debugfs_info);
1671 smmu->debugfs_info = kmalloc(bytes, GFP_KERNEL);
1672 if (!smmu->debugfs_info)
1675 root = debugfs_create_dir(dev_name(smmu->dev), NULL);
1678 smmu->debugfs_root = root;
1680 for (i = 0; i < ARRAY_SIZE(smmu_debugfs_mc); i++) {
1684 mc = debugfs_create_dir(smmu_debugfs_mc[i], root);
1688 for (j = 0; j < ARRAY_SIZE(smmu_debugfs_cache); j++) {
1689 struct dentry *cache;
1690 struct smmu_debugfs_info *info;
1692 info = smmu->debugfs_info;
1693 info += i * ARRAY_SIZE(smmu_debugfs_mc) + j;
1698 cache = debugfs_create_file(smmu_debugfs_cache[j],
1699 S_IWUSR | S_IRUSR, mc,
1701 &smmu_debugfs_stats_fops);
1707 debugfs_create_size_t("flush_all_threshold_pages", S_IWUSR | S_IRUSR,
1708 root, &smmu_flush_all_th_pages);
1712 smmu_debugfs_delete(smmu);
1715 int tegra_smmu_suspend(struct device *dev)
1718 struct smmu_device *smmu = dev_get_drvdata(dev);
1720 for (i = 0; i < smmu->num_translation_enable; i++)
1721 smmu->translation_enable[i] = smmu_read(smmu,
1722 SMMU_TRANSLATION_ENABLE_0 + i * sizeof(u32));
1724 for (i = 0; i < smmu->num_asid_security; i++)
1725 smmu->asid_security[i] =
1726 smmu_read(smmu, smmu_asid_security_ofs[i]);
1730 EXPORT_SYMBOL(tegra_smmu_suspend);
1732 struct device *get_smmu_device(void)
1734 return save_smmu_device;
1736 EXPORT_SYMBOL(get_smmu_device);
1738 int tegra_smmu_resume(struct device *dev)
1740 struct smmu_device *smmu = dev_get_drvdata(dev);
1741 unsigned long flags;
1743 spin_lock_irqsave(&smmu->lock, flags);
1744 smmu_setup_regs(smmu);
1745 spin_unlock_irqrestore(&smmu->lock, flags);
1748 EXPORT_SYMBOL(tegra_smmu_resume);
1750 static int tegra_smmu_probe(struct platform_device *pdev)
1752 struct smmu_device *smmu;
1753 struct resource *regs, *regs2, *window;
1754 struct device *dev = &pdev->dev;
1761 BUILD_BUG_ON(PAGE_SHIFT != SMMU_PAGE_SHIFT);
1763 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1764 regs2 = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1765 window = tegra_smmu_window(0);
1766 if (!regs || !regs2 || !window) {
1767 dev_err(dev, "No SMMU resources\n");
1771 num_as = SMMU_NUM_ASIDS;
1772 if (tegra_get_chipid() == TEGRA_CHIPID_TEGRA12)
1773 num_as = SMMU_NUM_ASIDS_TEGRA12;
1775 bytes = sizeof(*smmu) + num_as * sizeof(*smmu->as);
1776 smmu = devm_kzalloc(dev, bytes, GFP_KERNEL);
1778 dev_err(dev, "failed to allocate smmu_device\n");
1783 smmu->num_as = num_as;
1785 smmu->iovmm_base = (unsigned long)window->start;
1786 smmu->page_count = resource_size(window) >> SMMU_PAGE_SHIFT;
1787 smmu->regs = devm_ioremap(dev, regs->start, resource_size(regs));
1788 smmu->regs_ahbarb = devm_ioremap(dev, regs2->start,
1789 resource_size(regs2));
1790 if (!smmu->regs || !smmu->regs_ahbarb) {
1791 dev_err(dev, "failed to remap SMMU registers\n");
1795 if (IS_ENABLED(CONFIG_ARCH_TEGRA_3x_SOC) &&
1796 (tegra_get_chipid() == TEGRA_CHIPID_TEGRA3))
1797 smmu->swgids = 0x00000000000779ff;
1798 if (IS_ENABLED(CONFIG_ARCH_TEGRA_11x_SOC) &&
1799 (tegra_get_chipid() == TEGRA_CHIPID_TEGRA11))
1800 smmu->swgids = 0x0000000001b659fe;
1801 if (IS_ENABLED(CONFIG_ARCH_TEGRA_14x_SOC) &&
1802 (tegra_get_chipid() == TEGRA_CHIPID_TEGRA14))
1803 smmu->swgids = 0x00000020018659fe;
1804 if (IS_ENABLED(CONFIG_ARCH_TEGRA_12x_SOC) &&
1805 (tegra_get_chipid() == TEGRA_CHIPID_TEGRA12)) {
1806 smmu->swgids = 0x06f9000001ffc9cf;
1807 smmu->num_translation_enable = 4;
1808 smmu->num_asid_security = 8;
1809 smmu->ptc_cache_size = SZ_32K;
1811 smmu->num_translation_enable = 3;
1812 smmu->num_asid_security = 1;
1813 smmu->ptc_cache_size = SZ_16K;
1816 for (i = 0; i < smmu->num_translation_enable; i++)
1817 smmu->translation_enable[i] = ~0;
1819 for (i = 0; i < smmu->num_as; i++) {
1820 struct smmu_as *as = &smmu->as[i];
1824 as->pdir_attr = _PDIR_ATTR;
1825 as->pde_attr = _PDE_ATTR;
1826 as->pte_attr = _PTE_ATTR;
1828 spin_lock_init(&as->lock);
1829 spin_lock_init(&as->client_lock);
1830 INIT_LIST_HEAD(&as->client);
1832 spin_lock_init(&smmu->lock);
1833 smmu_setup_regs(smmu);
1834 platform_set_drvdata(pdev, smmu);
1836 smmu->avp_vector_page = alloc_page(GFP_KERNEL);
1837 if (!smmu->avp_vector_page)
1840 smmu_debugfs_create(smmu);
1842 bus_set_iommu(&platform_bus_type, &smmu_iommu_ops);
1844 dev_info(dev, "Loaded Tegra IOMMU driver\n");
1848 static int tegra_smmu_remove(struct platform_device *pdev)
1850 struct smmu_device *smmu = platform_get_drvdata(pdev);
1853 smmu_debugfs_delete(smmu);
1855 smmu_write(smmu, SMMU_CONFIG_DISABLE, SMMU_CONFIG);
1856 for (i = 0; i < smmu->num_as; i++)
1857 free_pdir(&smmu->as[i]);
1858 __free_page(smmu->avp_vector_page);
1863 const struct dev_pm_ops tegra_smmu_pm_ops = {
1864 .suspend = tegra_smmu_suspend,
1865 .resume = tegra_smmu_resume,
1868 static struct platform_driver tegra_smmu_driver = {
1869 .probe = tegra_smmu_probe,
1870 .remove = tegra_smmu_remove,
1872 .owner = THIS_MODULE,
1873 .name = "tegra_smmu",
1874 .pm = &tegra_smmu_pm_ops,
1878 static int tegra_smmu_device_notifier(struct notifier_block *nb,
1879 unsigned long event, void *_dev)
1881 struct dma_iommu_mapping *map;
1882 struct device *dev = _dev;
1885 case BUS_NOTIFY_BIND_DRIVER:
1886 if (get_dma_ops(dev) != &arm_dma_ops)
1889 case BUS_NOTIFY_ADD_DEVICE:
1890 if (strncmp(dev_name(dev), "tegra_smmu", 10) == 0)
1894 dev_warn(dev, "No map yet available\n");
1898 map = tegra_smmu_get_map(dev, tegra_smmu_of_get_swgids(dev));
1902 if (arm_iommu_attach_device(dev, map)) {
1903 arm_iommu_release_mapping(map);
1904 dev_err(dev, "Failed to attach %s\n", dev_name(dev));
1907 dev_dbg(dev, "Attached %s to map %p\n", dev_name(dev), map);
1909 case BUS_NOTIFY_DEL_DEVICE:
1913 case BUS_NOTIFY_UNBOUND_DRIVER:
1914 dev_dbg(dev, "Detaching %s from map %p\n", dev_name(dev),
1915 to_dma_iommu_mapping(dev));
1916 arm_iommu_detach_device(dev);
1924 static struct notifier_block tegra_smmu_device_nb = {
1925 .notifier_call = tegra_smmu_device_notifier,
1928 static int tegra_smmu_init(void)
1932 err = platform_driver_register(&tegra_smmu_driver);
1935 if (IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU))
1936 bus_register_notifier(&platform_bus_type,
1937 &tegra_smmu_device_nb);
1941 static int tegra_smmu_remove_map(struct device *dev, void *data)
1943 struct dma_iommu_mapping *map = to_dma_iommu_mapping(dev);
1945 arm_iommu_release_mapping(map);
1949 static void __exit tegra_smmu_exit(void)
1951 if (IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)) {
1952 bus_for_each_dev(&platform_bus_type, NULL, NULL,
1953 tegra_smmu_remove_map);
1954 bus_unregister_notifier(&platform_bus_type,
1955 &tegra_smmu_device_nb);
1957 platform_driver_unregister(&tegra_smmu_driver);
1960 core_initcall(tegra_smmu_init);
1961 module_exit(tegra_smmu_exit);
1963 MODULE_DESCRIPTION("IOMMU API for SMMU in Tegra SoC");
1964 MODULE_AUTHOR("Hiroshi DOYU <hdoyu@nvidia.com>");
1965 MODULE_LICENSE("GPL v2");