video: tegra: host: Separate kernel and user buffers
[linux-3.10.git] / drivers / video / tegra / host / gk20a / mm_gk20a.c
1 /*
2  * drivers/video/tegra/host/gk20a/mm_gk20a.c
3  *
4  * GK20A memory management
5  *
6  * Copyright (c) 2011-2013, NVIDIA CORPORATION.  All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms and conditions of the GNU General Public License,
10  * version 2, as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope it will be useful, but WITHOUT
13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15  * more details.
16  *
17  * You should have received a copy of the GNU General Public License along with
18  * this program; if not, write to the Free Software Foundation, Inc.,
19  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
20  */
21
22 #include <linux/delay.h>
23 #include <linux/highmem.h>
24 #include <linux/log2.h>
25 #include <linux/nvhost.h>
26 #include <linux/scatterlist.h>
27 #include <linux/nvmap.h>
28 #include <mach/hardware.h>
29 #include <asm/cacheflush.h>
30
31 #include "../../nvmap/nvmap_priv.h"
32 #include "../../nvmap/nvmap_ioctl.h"
33
34 #include "../dev.h"
35 #include "../nvhost_as.h"
36 #include "gk20a.h"
37 #include "mm_gk20a.h"
38 #include "hw_gmmu_gk20a.h"
39 #include "hw_fb_gk20a.h"
40 #include "hw_bus_gk20a.h"
41 #include "hw_ram_gk20a.h"
42 #include "hw_mc_gk20a.h"
43 #include "hw_flush_gk20a.h"
44 #include "hw_ltc_gk20a.h"
45
46 #include "kind_gk20a.h"
47
48 static inline int vm_aspace_id(struct vm_gk20a *vm)
49 {
50         /* -1 is bar1 or pmu, etc. */
51         return vm->as_share ? vm->as_share->id : -1;
52 }
53 static inline u32 hi32(u64 f)
54 {
55         return (u32)(f >> 32);
56 }
57 static inline u32 lo32(u64 f)
58 {
59         return (u32)(f & 0xffffffff);
60 }
61
62 #define FLUSH_CPU_DCACHE(va, pa, size)  \
63         do {    \
64                 __cpuc_flush_dcache_area((void *)(va), (size_t)(size)); \
65                 outer_flush_range(pa, pa + (size_t)(size));             \
66         } while (0)
67
68 static void gk20a_vm_unmap_locked(struct vm_gk20a *vm, u64 offset,
69                         struct mem_mgr **memmgr, struct mem_handle **r);
70 static struct mapped_buffer_node *find_mapped_buffer(struct rb_root *root,
71                                                      u64 addr);
72 static void gk20a_mm_tlb_invalidate(struct vm_gk20a *vm);
73 static int gk20a_vm_find_buffer(struct vm_gk20a *vm, u64 gpu_va,
74                 struct mem_mgr **memmgr, struct mem_handle **r, u64 *offset);
75 static int update_gmmu_ptes(struct vm_gk20a *vm,
76                             enum gmmu_pgsz_gk20a pgsz_idx, struct sg_table *sgt,
77                             u64 first_vaddr, u64 last_vaddr,
78                             u8 kind_v, u32 ctag_offset, bool cacheable);
79 static void update_gmmu_pde(struct vm_gk20a *vm, u32 i);
80
81
82 /* note: keep the page sizes sorted lowest to highest here */
83 static const u32 gmmu_page_sizes[gmmu_nr_page_sizes] = { SZ_4K, SZ_128K };
84 static const u32 gmmu_page_shifts[gmmu_nr_page_sizes] = { 12, 17 };
85 static const u64 gmmu_page_offset_masks[gmmu_nr_page_sizes] = { 0xfffLL,
86                                                                 0x1ffffLL };
87 static const u64 gmmu_page_masks[gmmu_nr_page_sizes] = { ~0xfffLL, ~0x1ffffLL };
88
89 static int gk20a_init_mm_reset_enable_hw(struct gk20a *g)
90 {
91         u32 pmc_enable;
92
93         pmc_enable = gk20a_readl(g, mc_enable_r());
94         pmc_enable &= ~mc_enable_pfb_enabled_f();
95         pmc_enable &= ~mc_enable_l2_enabled_f();
96         pmc_enable &= ~mc_enable_ce2_enabled_f();
97         pmc_enable &= ~mc_enable_xbar_enabled_f();
98         pmc_enable &= ~mc_enable_hub_enabled_f();
99         gk20a_writel(g, mc_enable_r(), pmc_enable);
100
101         pmc_enable = gk20a_readl(g, mc_enable_r());
102         pmc_enable |= mc_enable_pfb_enabled_f();
103         pmc_enable |= mc_enable_l2_enabled_f();
104         pmc_enable |= mc_enable_ce2_enabled_f();
105         pmc_enable |= mc_enable_xbar_enabled_f();
106         pmc_enable |= mc_enable_hub_enabled_f();
107         gk20a_writel(g, mc_enable_r(), pmc_enable);
108         gk20a_readl(g, mc_enable_r());
109
110         nvhost_dbg_fn("done");
111         return 0;
112 }
113
114 void gk20a_remove_mm_support(struct mm_gk20a *mm)
115 {
116         struct gk20a *g = mm->g;
117         struct vm_gk20a *vm = &mm->bar1.vm;
118         struct inst_desc *inst_block = &mm->bar1.inst_block;
119         struct mem_mgr *memmgr = mem_mgr_from_g(g);
120
121         nvhost_dbg_fn("");
122
123         nvhost_memmgr_free_sg_table(memmgr, inst_block->mem.ref,
124                         inst_block->mem.sgt);
125         nvhost_memmgr_put(memmgr, inst_block->mem.ref);
126
127         vm->remove_support(vm);
128 }
129
130 int gk20a_init_mm_setup_sw(struct gk20a *g)
131 {
132         struct mm_gk20a *mm = &g->mm;
133         int i;
134
135         nvhost_dbg_fn("");
136
137         if (mm->sw_ready) {
138                 nvhost_dbg_fn("skip init");
139                 return 0;
140         }
141
142         mm->g = g;
143         mm->big_page_size = gmmu_page_sizes[gmmu_page_size_big];
144         mm->pde_stride    = mm->big_page_size << 10;
145         mm->pde_stride_shift = ilog2(mm->pde_stride);
146         BUG_ON(mm->pde_stride_shift > 31); /* we have assumptions about this */
147
148         for (i = 0; i < ARRAY_SIZE(gmmu_page_sizes); i++) {
149
150                 u32 num_ptes, pte_space, num_pages;
151
152                 /* assuming "full" page tables */
153                 num_ptes = mm->pde_stride / gmmu_page_sizes[i];
154
155                 pte_space = num_ptes * gmmu_pte__size_v();
156                 /* allocate whole pages */
157                 pte_space = roundup(pte_space, PAGE_SIZE);
158
159                 num_pages = pte_space / PAGE_SIZE;
160                 /* make sure "order" is viable */
161                 BUG_ON(!is_power_of_2(num_pages));
162
163                 mm->page_table_sizing[i].num_ptes = num_ptes;
164                 mm->page_table_sizing[i].order = ilog2(num_pages);
165         }
166
167         /*TBD: make channel vm size configurable */
168         mm->channel.size = 1ULL << NV_GMMU_VA_RANGE;
169
170         nvhost_dbg_info("channel vm size: %dMB", (int)(mm->channel.size >> 20));
171
172         nvhost_dbg_info("small page-size (%dKB) pte array: %dKB",
173                         gmmu_page_sizes[gmmu_page_size_small] >> 10,
174                         (mm->page_table_sizing[gmmu_page_size_small].num_ptes *
175                          gmmu_pte__size_v()) >> 10);
176
177         nvhost_dbg_info("big page-size (%dKB) pte array: %dKB",
178                         gmmu_page_sizes[gmmu_page_size_big] >> 10,
179                         (mm->page_table_sizing[gmmu_page_size_big].num_ptes *
180                          gmmu_pte__size_v()) >> 10);
181
182
183         gk20a_init_bar1_vm(mm);
184
185         gk20a_init_uncompressed_kind_map();
186         gk20a_init_kind_attr();
187
188         mm->remove_support = gk20a_remove_mm_support;
189         mm->sw_ready = true;
190
191         nvhost_dbg_fn("done");
192         return 0;
193 }
194
195 /* make sure gk20a_init_mm_support is called before */
196 static int gk20a_init_mm_setup_hw(struct gk20a *g)
197 {
198         struct mm_gk20a *mm = &g->mm;
199         struct inst_desc *inst_block = &mm->bar1.inst_block;
200         phys_addr_t inst_pa = sg_phys(inst_block->mem.sgt->sgl);
201
202         nvhost_dbg_fn("");
203
204         /* set large page size in fb
205          * note this is very early on, can we defer it ? */
206         {
207                 u32 fb_mmu_ctrl = gk20a_readl(g, fb_mmu_ctrl_r());
208
209                 if (gmmu_page_sizes[gmmu_page_size_big] == SZ_128K)
210                         fb_mmu_ctrl = (fb_mmu_ctrl &
211                                        ~fb_mmu_ctrl_vm_pg_size_f(~0x0)) |
212                                 fb_mmu_ctrl_vm_pg_size_128kb_f();
213                 else
214                         BUG_ON(1); /* no support/testing for larger ones yet */
215
216                 gk20a_writel(g, fb_mmu_ctrl_r(), fb_mmu_ctrl);
217         }
218
219         inst_pa = (u32)(inst_pa >> bar1_instance_block_shift_gk20a());
220         nvhost_dbg_info("bar1 inst block ptr: 0x%08x",  (u32)inst_pa);
221
222         /* this is very early in init... can we defer this? */
223         {
224                 gk20a_writel(g, bus_bar1_block_r(),
225                              bus_bar1_block_target_vid_mem_f() |
226                              bus_bar1_block_mode_virtual_f() |
227                              bus_bar1_block_ptr_f(inst_pa));
228         }
229
230         nvhost_dbg_fn("done");
231         return 0;
232 }
233
234 int gk20a_init_mm_support(struct gk20a *g)
235 {
236         u32 err;
237
238         err = gk20a_init_mm_reset_enable_hw(g);
239         if (err)
240                 return err;
241
242         err = gk20a_init_mm_setup_sw(g);
243         if (err)
244                 return err;
245
246         err = gk20a_init_mm_setup_hw(g);
247         if (err)
248                 return err;
249
250         return err;
251 }
252
253 #ifdef CONFIG_TEGRA_IOMMU_SMMU
254 static int alloc_gmmu_pages(struct vm_gk20a *vm, u32 order,
255                             void **handle,
256                             struct sg_table **sgt)
257 {
258         u32 num_pages = 1 << order;
259         u32 len = num_pages * PAGE_SIZE;
260         int err;
261         struct page *pages;
262
263         nvhost_dbg_fn("");
264
265         pages = alloc_pages(GFP_KERNEL, order);
266         if (!pages) {
267                 nvhost_dbg(dbg_pte, "alloc_pages failed\n");
268                 goto err_out;
269         }
270         *sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
271         if (!sgt) {
272                 nvhost_dbg(dbg_pte, "cannot allocate sg table");
273                 goto err_alloced;
274         }
275         err = sg_alloc_table(*sgt, 1, GFP_KERNEL);
276         if (err) {
277                 nvhost_dbg(dbg_pte, "sg_alloc_table failed\n");
278                 goto err_sg_table;
279         }
280         sg_set_page((*sgt)->sgl, pages, len, 0);
281         *handle = page_address(pages);
282         memset(*handle, 0, len);
283         FLUSH_CPU_DCACHE(*handle, sg_phys((*sgt)->sgl), len);
284
285         return 0;
286
287 err_sg_table:
288         kfree(*sgt);
289 err_alloced:
290         __free_pages(pages, order);
291 err_out:
292         return -ENOMEM;
293 }
294
295 static void free_gmmu_pages(struct vm_gk20a *vm, void *handle,
296                             struct sg_table *sgt, u32 order)
297 {
298         nvhost_dbg_fn("");
299         BUG_ON(sgt == NULL);
300         free_pages((unsigned long)handle, order);
301         sg_free_table(sgt);
302         kfree(sgt);
303 }
304
305 static int map_gmmu_pages(void *handle, struct sg_table *sgt, void **va)
306 {
307         FLUSH_CPU_DCACHE(handle, sg_phys(sgt->sgl), sgt->sgl->length);
308         *va = handle;
309         return 0;
310 }
311
312 static void unmap_gmmu_pages(void *handle, struct sg_table *sgt, u32 *va)
313 {
314         FLUSH_CPU_DCACHE(handle, sg_phys(sgt->sgl), sgt->sgl->length);
315 }
316 #else
317 static int alloc_gmmu_pages(struct vm_gk20a *vm, u32 order,
318                             void **handle,
319                             struct sg_table **sgt)
320 {
321         struct mem_mgr *client = mem_mgr_from_vm(vm);
322         struct mem_handle *r;
323         u32 num_pages = 1 << order;
324         u32 len = num_pages * PAGE_SIZE;
325         void *va;
326
327         nvhost_dbg_fn("");
328
329         r = nvhost_memmgr_alloc(client, len,
330                                 DEFAULT_ALLOC_ALIGNMENT,
331                                 DEFAULT_ALLOC_FLAGS,
332                                 0);
333         if (IS_ERR(r)) {
334                 nvhost_dbg(dbg_pte, "nvmap_alloc failed\n");
335                 goto err_out;
336         }
337         va = nvhost_memmgr_mmap(r);
338         if (!va) {
339                 nvhost_dbg(dbg_pte, "nvmap_mmap failed\n");
340                 goto err_alloced;
341         }
342         *sgt = nvhost_memmgr_sg_table(client, r);
343         if (!*sgt) {
344                 nvhost_dbg(dbg_pte, "cannot allocate sg table");
345                 goto err_mmaped;
346         }
347         memset(va, 0, len);
348         nvhost_memmgr_munmap(r, va);
349         *handle = (void *)r;
350
351         return 0;
352
353 err_mmaped:
354         nvhost_memmgr_munmap(r, va);
355 err_alloced:
356         nvhost_memmgr_put(client, r);
357 err_out:
358         return -ENOMEM;
359 }
360
361 static void free_gmmu_pages(struct vm_gk20a *vm, void *handle,
362                             struct sg_table *sgt, u32 order)
363 {
364         struct mem_mgr *client = mem_mgr_from_vm(vm);
365         nvhost_dbg_fn("");
366         BUG_ON(sgt == NULL);
367         nvhost_memmgr_free_sg_table(client, handle, sgt);
368         nvhost_memmgr_put(client, handle);
369 }
370
371 static int map_gmmu_pages(void *handle, struct sg_table *sgt, void **va)
372 {
373         struct mem_handle *r = handle;
374         u32 *tmp_va;
375
376         nvhost_dbg_fn("");
377
378         tmp_va = nvhost_memmgr_mmap(r);
379         if (!tmp_va)
380                 goto err_out;
381
382         *va = tmp_va;
383         return 0;
384
385 err_out:
386         return -ENOMEM;
387 }
388
389 static void unmap_gmmu_pages(void *handle, struct sg_table *sgt, u32 *va)
390 {
391         struct mem_handle *r = handle;
392         nvhost_dbg_fn("");
393         nvhost_memmgr_munmap(r, va);
394 }
395 #endif
396
397 /* allocate a phys contig region big enough for a full
398  * sized gmmu page table for the given gmmu_page_size.
399  * the whole range is zeroed so it's "invalid"/will fault
400  */
401
402 static int zalloc_gmmu_page_table_gk20a(struct vm_gk20a *vm,
403                                         enum gmmu_pgsz_gk20a gmmu_pgsz_idx,
404                                         struct page_table_gk20a *pte)
405 {
406         int err;
407         u32 pte_order;
408         void *handle;
409         struct sg_table *sgt;
410
411         nvhost_dbg_fn("");
412
413         /* allocate enough pages for the table */
414         pte_order = vm->mm->page_table_sizing[gmmu_pgsz_idx].order;
415
416         err = alloc_gmmu_pages(vm, pte_order, &handle, &sgt);
417         if (err)
418                 return err;
419
420         nvhost_dbg(dbg_pte, "pte = 0x%p, addr=%08llx, size %d",
421                         pte, (u64)sg_phys(sgt->sgl), pte_order);
422
423         pte->ref = handle;
424         pte->sgt = sgt;
425
426         return 0;
427 }
428
429 /* given address range (inclusive) determine the pdes crossed */
430 static inline void pde_range_from_vaddr_range(struct vm_gk20a *vm,
431                                               u64 addr_lo, u64 addr_hi,
432                                               u32 *pde_lo, u32 *pde_hi)
433 {
434         *pde_lo = (u32)(addr_lo >> vm->mm->pde_stride_shift);
435         *pde_hi = (u32)(addr_hi >> vm->mm->pde_stride_shift);
436         nvhost_dbg(dbg_pte, "addr_lo=0x%llx addr_hi=0x%llx pde_ss=%d",
437                    addr_lo, addr_hi, vm->mm->pde_stride_shift);
438         nvhost_dbg(dbg_pte, "pde_lo=%d pde_hi=%d",
439                    *pde_lo, *pde_hi);
440 }
441
442 static inline u32 *pde_from_index(struct vm_gk20a *vm, u32 i)
443 {
444         return (u32 *) (((u8 *)vm->pdes.kv) + i*gmmu_pde__size_v());
445 }
446
447 static inline u32 pte_index_from_vaddr(struct vm_gk20a *vm,
448                                        u64 addr, enum gmmu_pgsz_gk20a pgsz_idx)
449 {
450         u32 ret;
451         /* mask off pde part */
452         addr = addr & ((((u64)1) << vm->mm->pde_stride_shift) - ((u64)1));
453         /* shift over to get pte index. note assumption that pte index
454          * doesn't leak over into the high 32b */
455         ret = (u32)(addr >> gmmu_page_shifts[pgsz_idx]);
456
457         nvhost_dbg(dbg_pte, "addr=0x%llx pte_i=0x%x", addr, ret);
458         return ret;
459 }
460
461 static inline void pte_space_page_offset_from_index(u32 i, u32 *pte_page,
462                                                     u32 *pte_offset)
463 {
464         /* ptes are 8B regardless of pagesize */
465         /* pte space pages are 4KB. so 512 ptes per 4KB page*/
466         *pte_page = i >> 9;
467
468         /* this offset is a pte offset, not a byte offset */
469         *pte_offset = i & ((1<<9)-1);
470
471         nvhost_dbg(dbg_pte, "i=0x%x pte_page=0x%x pte_offset=0x%x",
472                    i, *pte_page, *pte_offset);
473 }
474
475
476 /*
477  * given a pde index/page table number make sure it has
478  * backing store and if not go ahead allocate it and
479  * record it in the appropriate pde
480  */
481 static int validate_gmmu_page_table_gk20a(struct vm_gk20a *vm,
482                           u32 i,
483                           enum gmmu_pgsz_gk20a gmmu_pgsz_idx)
484 {
485         int err;
486         struct page_table_gk20a *pte =
487                 vm->pdes.ptes[gmmu_pgsz_idx] + i;
488
489         nvhost_dbg_fn("");
490
491         /* if it's already in place it's valid */
492         if (pte->ref)
493                 return 0;
494
495         nvhost_dbg(dbg_pte, "alloc %dKB ptes for pde %d",
496                    gmmu_page_sizes[gmmu_pgsz_idx]/1024, i);
497
498         err = zalloc_gmmu_page_table_gk20a(vm, gmmu_pgsz_idx, pte);
499         if (err)
500                 return err;
501
502         /* rewrite pde */
503         update_gmmu_pde(vm, i);
504
505         return 0;
506 }
507
508 static int gk20a_vm_get_buffers(struct vm_gk20a *vm,
509                                 struct mapped_buffer_node ***mapped_buffers,
510                                 int *num_buffers)
511 {
512         struct mapped_buffer_node *mapped_buffer;
513         struct mapped_buffer_node **buffer_list;
514         struct rb_node *node;
515         int i = 0;
516
517         mutex_lock(&vm->update_gmmu_lock);
518
519         buffer_list = kzalloc(sizeof(*buffer_list) *
520                               vm->num_user_mapped_buffers, GFP_KERNEL);
521         if (!buffer_list) {
522                 mutex_unlock(&vm->update_gmmu_lock);
523                 return -ENOMEM;
524         }
525
526         node = rb_first(&vm->mapped_buffers);
527         while (node) {
528                 mapped_buffer =
529                         container_of(node, struct mapped_buffer_node, node);
530                 if (mapped_buffer->user_mapped) {
531                         buffer_list[i] = mapped_buffer;
532                         kref_get(&mapped_buffer->ref);
533                         i++;
534                 }
535                 node = rb_next(&mapped_buffer->node);
536         }
537
538         BUG_ON(i != vm->num_user_mapped_buffers);
539
540         *num_buffers = vm->num_user_mapped_buffers;
541         *mapped_buffers = buffer_list;
542
543         mutex_unlock(&vm->update_gmmu_lock);
544
545         return 0;
546 }
547
548 static void gk20a_vm_unmap_buffer(struct kref *ref)
549 {
550         struct mapped_buffer_node *mapped_buffer =
551                 container_of(ref, struct mapped_buffer_node, ref);
552         struct vm_gk20a *vm = mapped_buffer->vm;
553         struct mem_mgr *memmgr = NULL;
554         struct mem_handle *r = NULL;
555
556         gk20a_vm_unmap_locked(vm, mapped_buffer->addr, &memmgr, &r);
557         nvhost_memmgr_put(memmgr, r);
558         nvhost_memmgr_put_mgr(memmgr);
559 }
560
561 static void gk20a_vm_put_buffers(struct vm_gk20a *vm,
562                                  struct mapped_buffer_node **mapped_buffers,
563                                  int num_buffers)
564 {
565         int i;
566
567         mutex_lock(&vm->update_gmmu_lock);
568
569         for (i = 0; i < num_buffers; ++i)
570                 kref_put(&mapped_buffers[i]->ref,
571                          gk20a_vm_unmap_buffer);
572
573         mutex_unlock(&vm->update_gmmu_lock);
574
575         kfree(mapped_buffers);
576 }
577
578 static void gk20a_vm_unmap_user(struct vm_gk20a *vm, u64 offset)
579 {
580         struct mapped_buffer_node *mapped_buffer;
581
582         mutex_lock(&vm->update_gmmu_lock);
583
584         mapped_buffer = find_mapped_buffer(&vm->mapped_buffers, offset);
585         if (!mapped_buffer) {
586                 mutex_unlock(&vm->update_gmmu_lock);
587                 nvhost_dbg(dbg_err, "invalid addr to unmap 0x%llx", offset);
588                 return;
589         }
590
591         mapped_buffer->user_mapped = false;
592         vm->num_user_mapped_buffers--;
593         list_add_tail(&mapped_buffer->unmap_list, &vm->deferred_unmaps);
594         kref_put(&mapped_buffer->ref, gk20a_vm_unmap_buffer);
595
596         mutex_unlock(&vm->update_gmmu_lock);
597 }
598
599 static u64 gk20a_vm_alloc_va(struct vm_gk20a *vm,
600                              u64 size,
601                              enum gmmu_pgsz_gk20a gmmu_pgsz_idx)
602
603 {
604         struct nvhost_allocator *vma = &vm->vma[gmmu_pgsz_idx];
605         int err;
606         u64 offset;
607         u32 start_page_nr = 0, num_pages;
608         u64 gmmu_page_size = gmmu_page_sizes[gmmu_pgsz_idx];
609
610         if (gmmu_pgsz_idx >= ARRAY_SIZE(gmmu_page_sizes)) {
611                 dev_warn(dev_from_vm(vm),
612                          "invalid page size requested in gk20a vm alloc");
613                 return -EINVAL;
614         }
615
616         if ((gmmu_pgsz_idx == gmmu_page_size_big) && !vm->big_pages) {
617                 dev_warn(dev_from_vm(vm),
618                          "unsupportd page size requested");
619                 return -EINVAL;
620
621         }
622
623         /* be certain we round up to gmmu_page_size if needed */
624         /* TBD: DIV_ROUND_UP -> undefined reference to __aeabi_uldivmod */
625         size = (size + ((u64)gmmu_page_size - 1)) & ~((u64)gmmu_page_size - 1);
626
627         nvhost_dbg_info("size=0x%llx @ pgsz=%dKB", size,
628                         gmmu_page_sizes[gmmu_pgsz_idx]>>10);
629
630         /* The vma allocator represents page accounting. */
631         num_pages = size >> gmmu_page_shifts[gmmu_pgsz_idx];
632
633         err = vma->alloc(vma, &start_page_nr, num_pages);
634
635         if (err) {
636                 nvhost_err(dev_from_vm(vm),
637                            "%s oom: sz=0x%llx", vma->name, size);
638                 return 0;
639         }
640
641         offset = (u64)start_page_nr << gmmu_page_shifts[gmmu_pgsz_idx];
642         nvhost_dbg_fn("%s found addr: 0x%llx", vma->name, offset);
643
644         return offset;
645 }
646
647 static void gk20a_vm_free_va(struct vm_gk20a *vm,
648                              u64 offset, u64 size,
649                              enum gmmu_pgsz_gk20a pgsz_idx)
650 {
651         struct nvhost_allocator *vma = &vm->vma[pgsz_idx];
652         u32 page_size = gmmu_page_sizes[pgsz_idx];
653         u32 page_shift = gmmu_page_shifts[pgsz_idx];
654         u32 start_page_nr, num_pages;
655         int err;
656
657         nvhost_dbg_info("%s free addr=0x%llx, size=0x%llx",
658                         vma->name, offset, size);
659
660         start_page_nr = (u32)(offset >> page_shift);
661         num_pages = (u32)((size + page_size - 1) >> page_shift);
662
663         err = vma->free(vma, start_page_nr, num_pages);
664         if (err) {
665                 nvhost_err(dev_from_vm(vm),
666                            "not found: offset=0x%llx, sz=0x%llx",
667                            offset, size);
668         }
669 }
670
671 static int insert_mapped_buffer(struct rb_root *root,
672                                 struct mapped_buffer_node *mapped_buffer)
673 {
674         struct rb_node **new_node = &(root->rb_node), *parent = NULL;
675
676         /* Figure out where to put new node */
677         while (*new_node) {
678                 struct mapped_buffer_node *cmp_with =
679                         container_of(*new_node, struct mapped_buffer_node,
680                                      node);
681
682                 parent = *new_node;
683
684                 if (cmp_with->addr > mapped_buffer->addr) /* u64 cmp */
685                         new_node = &((*new_node)->rb_left);
686                 else if (cmp_with->addr != mapped_buffer->addr) /* u64 cmp */
687                         new_node = &((*new_node)->rb_right);
688                 else
689                         return -EINVAL; /* no fair dup'ing */
690         }
691
692         /* Add new node and rebalance tree. */
693         rb_link_node(&mapped_buffer->node, parent, new_node);
694         rb_insert_color(&mapped_buffer->node, root);
695
696         return 0;
697 }
698
699 static struct mapped_buffer_node *find_deferred_unmap(struct vm_gk20a *vm,
700                                                       u64 paddr, u64 vaddr,
701                                                       u32 flags)
702 {
703         struct mapped_buffer_node *node;
704
705         list_for_each_entry(node, &vm->deferred_unmaps, unmap_list)
706                 if ((sg_phys(node->sgt->sgl) == paddr) &&
707                     (node->flags == flags) &&
708                     (!vaddr || (node->addr == vaddr)))
709                         return node;
710
711         return NULL;
712 }
713
714 static struct mapped_buffer_node *find_mapped_buffer(struct rb_root *root,
715                                                      u64 addr)
716 {
717
718         struct rb_node *node = root->rb_node;
719         while (node) {
720                 struct mapped_buffer_node *mapped_buffer =
721                         container_of(node, struct mapped_buffer_node, node);
722                 if (mapped_buffer->addr > addr) /* u64 cmp */
723                         node = node->rb_left;
724                 else if (mapped_buffer->addr != addr) /* u64 cmp */
725                         node = node->rb_right;
726                 else
727                         return mapped_buffer;
728         }
729         return 0;
730 }
731
732 static struct mapped_buffer_node *find_mapped_buffer_range(struct rb_root *root,
733                                                            u64 addr)
734 {
735         struct rb_node *node = root->rb_node;
736         while (node) {
737                 struct mapped_buffer_node *m =
738                         container_of(node, struct mapped_buffer_node, node);
739                 if (m->addr <= addr && m->addr + m->size > addr)
740                         return m;
741                 else if (m->addr > addr) /* u64 cmp */
742                         node = node->rb_left;
743                 else
744                         node = node->rb_right;
745         }
746         return 0;
747 }
748
749 /* convenience setup for nvmap buffer attr queries */
750 struct bfr_attr_query {
751         int err;
752         u64 v;
753 };
754 static u32 nvmap_bfr_param[] = {
755 #define BFR_SIZE   0
756         NVMAP_HANDLE_PARAM_SIZE,
757 #define BFR_ALIGN  1
758         NVMAP_HANDLE_PARAM_ALIGNMENT,
759 #define BFR_HEAP   2
760         NVMAP_HANDLE_PARAM_HEAP,
761 #define BFR_KIND   3
762         NVMAP_HANDLE_PARAM_KIND,
763 };
764 #define BFR_ATTRS (sizeof(nvmap_bfr_param)/sizeof(nvmap_bfr_param[0]))
765
766 struct buffer_attrs {
767         struct sg_table *sgt;
768         u64 size;
769         u64 align;
770         u32 ctag_offset;
771         u32 ctag_lines;
772         int pgsz_idx;
773         u8 kind_v;
774         u8 uc_kind_v;
775 };
776
777 static void gmmu_select_page_size(struct buffer_attrs *bfr)
778 {
779         int i;
780         /*  choose the biggest first (top->bottom) */
781         for (i = (gmmu_nr_page_sizes-1); i >= 0; i--)
782                 if (!(gmmu_page_offset_masks[i] & bfr->align)) {
783                         /* would like to add this too but nvmap returns the
784                          * original requested size not the allocated size.
785                          * (!(gmmu_page_offset_masks[i] & bfr->size)) */
786                         bfr->pgsz_idx = i;
787                         break;
788                 }
789 }
790
791 static int setup_buffer_size_and_align(struct device *d,
792                                        struct buffer_attrs *bfr,
793                                        struct bfr_attr_query *query,
794                                        u64 offset, u32 flags)
795 {
796         /* buffer allocation size and alignment must be a multiple
797            of one of the supported page sizes.*/
798         bfr->size = query[BFR_SIZE].v;
799         bfr->align = query[BFR_ALIGN].v;
800         bfr->pgsz_idx = -1;
801
802         /* If FIX_OFFSET is set, pgsz is determined. Otherwise, select
803          * page size according to memory alignment */
804         if (flags & NVHOST_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET) {
805                 bfr->pgsz_idx = NV_GMMU_VA_IS_UPPER(offset) ?
806                                 gmmu_page_size_big : gmmu_page_size_small;
807         } else {
808                 gmmu_select_page_size(bfr);
809         }
810
811         if (unlikely(bfr->pgsz_idx == -1)) {
812                 nvhost_warn(d, "unsupported buffer alignment: 0x%llx",
813                            bfr->align);
814                 return -EINVAL;
815         }
816
817         bfr->kind_v = query[BFR_KIND].v;
818
819         return 0;
820 }
821
822
823 static int setup_buffer_kind_and_compression(struct device *d,
824                                              u32 flags,
825                                              u32 kind,
826                                              struct buffer_attrs *bfr,
827                                              enum gmmu_pgsz_gk20a pgsz_idx)
828 {
829         bool kind_compressible;
830
831         /* This flag (which comes from map_buffer ioctl) is for override now.
832            It will be removed when all clients which use it have been
833            changed to specify kind in the nvmap buffer alloc. */
834         if (flags & NVHOST_MAP_BUFFER_FLAGS_KIND_SPECIFIED)
835                 bfr->kind_v = kind;
836
837         if (unlikely(bfr->kind_v == gmmu_pte_kind_invalid_v()))
838                 bfr->kind_v = gmmu_pte_kind_pitch_v();
839
840         if (unlikely(!gk20a_kind_is_supported(bfr->kind_v))) {
841                 nvhost_err(d, "kind 0x%x not supported", bfr->kind_v);
842                 return -EINVAL;
843         }
844
845         bfr->uc_kind_v = gmmu_pte_kind_invalid_v();
846         /* find a suitable uncompressed kind if it becomes necessary later */
847         kind_compressible = gk20a_kind_is_compressible(bfr->kind_v);
848         if (kind_compressible) {
849                 bfr->uc_kind_v = gk20a_get_uncompressed_kind(bfr->kind_v);
850                 if (unlikely(bfr->uc_kind_v == gmmu_pte_kind_invalid_v())) {
851                         /* shouldn't happen, but it is worth cross-checking */
852                         nvhost_err(d, "comptag kind 0x%x can't be"
853                                    " downgraded to uncompressed kind",
854                                    bfr->kind_v);
855                         return -EINVAL;
856                 }
857         }
858         /* comptags only supported for suitable kinds, 128KB pagesize */
859         if (unlikely(kind_compressible &&
860                      (gmmu_page_sizes[pgsz_idx] != 128*1024))) {
861                 /*
862                 nvhost_warn(d, "comptags specified"
863                 " but pagesize being used doesn't support it");*/
864                 /* it is safe to fall back to uncompressed as
865                    functionality is not harmed */
866                 bfr->kind_v = bfr->uc_kind_v;
867                 kind_compressible = false;
868         }
869         if (kind_compressible)
870                 bfr->ctag_lines = ALIGN(bfr->size, COMP_TAG_LINE_SIZE) >>
871                         COMP_TAG_LINE_SIZE_SHIFT;
872         else
873                 bfr->ctag_lines = 0;
874
875         return 0;
876 }
877
878 static u64 gk20a_vm_map(struct vm_gk20a *vm,
879                         struct mem_mgr *memmgr,
880                         struct mem_handle *r,
881                         u64 offset_align,
882                         u32 flags /*NVHOST_AS_MAP_BUFFER_FLAGS_*/,
883                         u32 kind,
884                         struct sg_table **sgt,
885                         bool user_mapped)
886 {
887         struct gk20a *g = gk20a_from_vm(vm);
888         struct nvhost_allocator *ctag_allocator = &g->gr.comp_tags;
889         struct device *d = dev_from_vm(vm);
890         struct mapped_buffer_node *mapped_buffer = 0;
891         bool inserted = false, va_allocated = false;
892         u32 gmmu_page_size = 0;
893         u64 map_offset = 0;
894         int attr, err = 0;
895         struct buffer_attrs bfr = {0};
896         struct bfr_attr_query query[BFR_ATTRS];
897         u32 i, pde_lo, pde_hi;
898         struct nvhost_comptags comptags;
899
900         mutex_lock(&vm->update_gmmu_lock);
901
902         /* pin buffer to get phys/iovmm addr */
903         bfr.sgt = nvhost_memmgr_pin(memmgr, r, d);
904         if (IS_ERR(bfr.sgt)) {
905                 /* Falling back to physical is actually possible
906                  * here in many cases if we use 4K phys pages in the
907                  * gmmu.  However we have some regions which require
908                  * contig regions to work properly (either phys-contig
909                  * or contig through smmu io_vaspace).  Until we can
910                  * track the difference between those two cases we have
911                  * to fail the mapping when we run out of SMMU space.
912                  */
913                 nvhost_warn(d, "oom allocating tracking buffer");
914                 goto clean_up;
915         }
916
917         mapped_buffer = find_deferred_unmap(vm, sg_phys(bfr.sgt->sgl),
918                                             offset_align, flags);
919         if (mapped_buffer) {
920                 nvhost_memmgr_unpin(memmgr, r, d, bfr.sgt);
921                 if (sgt)
922                         *sgt = mapped_buffer->sgt;
923
924                 /* mark the buffer as used */
925                 mapped_buffer->user_mapped = true;
926                 kref_get(&mapped_buffer->ref);
927                 vm->num_user_mapped_buffers++;
928                 list_del_init(&mapped_buffer->unmap_list);
929
930                 /* replace the old reference by the new reference */
931                 nvhost_memmgr_put(mapped_buffer->memmgr,
932                                   mapped_buffer->handle_ref);
933                 nvhost_memmgr_put_mgr(mapped_buffer->memmgr);
934                 mapped_buffer->memmgr = memmgr;
935                 mapped_buffer->handle_ref = r;
936
937                 mutex_unlock(&vm->update_gmmu_lock);
938                 return mapped_buffer->addr;
939         }
940
941         if (sgt)
942                 *sgt = bfr.sgt;
943
944         /* query bfr attributes: size, align, heap, kind */
945         for (attr = 0; attr < BFR_ATTRS; attr++) {
946                 query[attr].err =
947                         nvhost_memmgr_get_param(memmgr, r,
948                                                 nvmap_bfr_param[attr],
949                                                 &query[attr].v);
950                 if (unlikely(query[attr].err != 0)) {
951                         nvhost_err(d,
952                                    "failed to get nvmap buffer param %d: %d\n",
953                                    nvmap_bfr_param[attr],
954                                    query[attr].err);
955                         err = query[attr].err;
956                         goto clean_up;
957                 }
958         }
959
960         /* validate/adjust bfr attributes */
961         err = setup_buffer_size_and_align(d, &bfr, query, offset_align, flags);
962         if (unlikely(err))
963                 goto clean_up;
964         if (unlikely(bfr.pgsz_idx < gmmu_page_size_small ||
965                      bfr.pgsz_idx > gmmu_page_size_big)) {
966                 BUG_ON(1);
967                 err = -EINVAL;
968                 goto clean_up;
969         }
970         gmmu_page_size = gmmu_page_sizes[bfr.pgsz_idx];
971
972         /* if specified the map offset must be bfr page size aligned */
973         if (flags & NVHOST_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET) {
974                 map_offset = offset_align;
975                 if (map_offset & gmmu_page_offset_masks[bfr.pgsz_idx]) {
976                         nvhost_err(d,
977                            "map offset must be buffer page size aligned 0x%llx",
978                            map_offset);
979                         err = -EINVAL;
980                         goto clean_up;
981                 }
982         }
983
984         if (sgt)
985                 *sgt = bfr.sgt;
986
987         err = setup_buffer_kind_and_compression(d, flags, kind,
988                                                 &bfr, bfr.pgsz_idx);
989         if (unlikely(err)) {
990                 nvhost_err(d, "failure setting up kind and compression");
991                 goto clean_up;
992         }
993
994         /* bar1 and pmu vm don't need ctag */
995         if (!vm->enable_ctag)
996                 bfr.ctag_lines = 0;
997
998         nvhost_memmgr_get_comptags(r, &comptags);
999
1000         if (bfr.ctag_lines && !comptags.lines) {
1001                 /* allocate compression resources if needed */
1002                 err = nvhost_memmgr_alloc_comptags(r,
1003                                 ctag_allocator, bfr.ctag_lines);
1004                 if (err) {
1005                         /* ok to fall back here if we ran out */
1006                         /* TBD: we can partially alloc ctags as well... */
1007                         bfr.ctag_lines = bfr.ctag_offset = 0;
1008                         bfr.kind_v = bfr.uc_kind_v;
1009                 } else {
1010                         nvhost_memmgr_get_comptags(r, &comptags);
1011
1012                         /* init/clear the ctag buffer */
1013                         gk20a_gr_clear_comptags(g,
1014                                 comptags.offset,
1015                                 comptags.offset + comptags.lines - 1);
1016                 }
1017         }
1018
1019         /* store the comptag info */
1020         WARN_ON(bfr.ctag_lines != comptags.lines);
1021         bfr.ctag_offset = comptags.offset;
1022
1023         /* Allocate (or validate when map_offset != 0) the virtual address. */
1024         if (!map_offset) {
1025                 map_offset = vm->alloc_va(vm, bfr.size,
1026                                           bfr.pgsz_idx);
1027                 if (!map_offset) {
1028                         nvhost_err(d, "failed to allocate va space");
1029                         err = -ENOMEM;
1030                         goto clean_up;
1031                 }
1032                 va_allocated = true;
1033         } else {
1034                 /* TODO: allocate the offset to keep track? */
1035                 /* TODO: then we could warn on actual collisions... */
1036                 nvhost_warn(d, "fixed offset mapping isn't safe yet!");
1037                 nvhost_warn(d, "other mappings may collide!");
1038         }
1039
1040         pde_range_from_vaddr_range(vm,
1041                                    map_offset,
1042                                    map_offset + bfr.size - 1,
1043                                    &pde_lo, &pde_hi);
1044
1045         /* mark the addr range valid (but with 0 phys addr, which will fault) */
1046         for (i = pde_lo; i <= pde_hi; i++) {
1047                 err = validate_gmmu_page_table_gk20a(vm, i, bfr.pgsz_idx);
1048                 if (err) {
1049                         nvhost_err(dev_from_vm(vm),
1050                                    "failed to validate page table %d: %d",
1051                                    i, err);
1052                         goto clean_up;
1053                 }
1054         }
1055
1056         nvhost_dbg(dbg_map,
1057            "as=%d pgsz=%d "
1058            "kind=0x%x kind_uc=0x%x flags=0x%x "
1059            "ctags=%d start=%d gv=0x%x,%08x -> 0x%x,%08x -> 0x%x,%08x",
1060            vm_aspace_id(vm), gmmu_page_size,
1061            bfr.kind_v, bfr.uc_kind_v, flags,
1062            bfr.ctag_lines, bfr.ctag_offset,
1063            hi32(map_offset), lo32(map_offset),
1064            hi32((u64)sg_dma_address(bfr.sgt->sgl)),
1065            lo32((u64)sg_dma_address(bfr.sgt->sgl)),
1066            hi32((u64)sg_phys(bfr.sgt->sgl)),
1067            lo32((u64)sg_phys(bfr.sgt->sgl)));
1068
1069 #if defined(NVHOST_DEBUG)
1070         {
1071                 int i;
1072                 struct scatterlist *sg = NULL;
1073                 nvhost_dbg(dbg_pte, "for_each_sg(bfr.sgt->sgl, sg, bfr.sgt->nents, i)");
1074                 for_each_sg(bfr.sgt->sgl, sg, bfr.sgt->nents, i ) {
1075                         u64 da = sg_dma_address(sg);
1076                         u64 pa = sg_phys(sg);
1077                         u64 len = sg->length;
1078                         nvhost_dbg(dbg_pte, "i=%d pa=0x%x,%08x da=0x%x,%08x len=0x%x,%08x",
1079                                    i, hi32(pa), lo32(pa), hi32(da), lo32(da),
1080                                    hi32(len), lo32(len));
1081                 }
1082         }
1083 #endif
1084
1085         /* keep track of the buffer for unmapping */
1086         /* TBD: check for multiple mapping of same buffer */
1087         mapped_buffer = kzalloc(sizeof(*mapped_buffer), GFP_KERNEL);
1088         if (!mapped_buffer) {
1089                 nvhost_warn(d, "oom allocating tracking buffer");
1090                 goto clean_up;
1091         }
1092         mapped_buffer->memmgr      = memmgr;
1093         mapped_buffer->handle_ref  = r;
1094         mapped_buffer->sgt         = bfr.sgt;
1095         mapped_buffer->addr        = map_offset;
1096         mapped_buffer->size        = bfr.size;
1097         mapped_buffer->pgsz_idx    = bfr.pgsz_idx;
1098         mapped_buffer->ctag_offset = bfr.ctag_offset;
1099         mapped_buffer->ctag_lines  = bfr.ctag_lines;
1100         mapped_buffer->vm          = vm;
1101         mapped_buffer->flags       = flags;
1102         mapped_buffer->user_mapped = user_mapped;
1103         INIT_LIST_HEAD(&mapped_buffer->unmap_list);
1104         kref_init(&mapped_buffer->ref);
1105
1106         err = insert_mapped_buffer(&vm->mapped_buffers, mapped_buffer);
1107         if (err) {
1108                 nvhost_err(d, "failed to insert into mapped buffer tree");
1109                 goto clean_up;
1110         }
1111         inserted = true;
1112         if (user_mapped)
1113                 vm->num_user_mapped_buffers++;
1114
1115         nvhost_dbg_info("allocated va @ 0x%llx", map_offset);
1116
1117         err = update_gmmu_ptes(vm, bfr.pgsz_idx,
1118                                bfr.sgt,
1119                                map_offset, map_offset + bfr.size - 1,
1120                                bfr.kind_v,
1121                                bfr.ctag_offset,
1122                                flags & NVHOST_MAP_BUFFER_FLAGS_CACHEABLE_TRUE);
1123         if (err) {
1124                 nvhost_err(d, "failed to update ptes on map");
1125                 goto clean_up;
1126         }
1127
1128         mutex_unlock(&vm->update_gmmu_lock);
1129         return map_offset;
1130
1131 clean_up:
1132         if (inserted) {
1133                 rb_erase(&mapped_buffer->node, &vm->mapped_buffers);
1134                 if (user_mapped)
1135                         vm->num_user_mapped_buffers--;
1136         }
1137         kfree(mapped_buffer);
1138         if (va_allocated)
1139                 vm->free_va(vm, map_offset, bfr.size, bfr.pgsz_idx);
1140         if (bfr.ctag_lines)
1141                 ctag_allocator->free(ctag_allocator,
1142                                      bfr.ctag_offset,
1143                                      bfr.ctag_lines);
1144         if (bfr.sgt) {
1145                 nvhost_memmgr_unpin(memmgr, r, d, bfr.sgt);
1146         }
1147
1148         mutex_unlock(&vm->update_gmmu_lock);
1149         nvhost_dbg_info("err=%d\n", err);
1150         return 0;
1151 }
1152
1153 u64 gk20a_mm_iova_addr(struct scatterlist *sgl)
1154 {
1155         u64 result = sg_phys(sgl);
1156 #ifdef CONFIG_TEGRA_IOMMU_SMMU
1157         if (sg_dma_address(sgl) == DMA_ERROR_CODE)
1158                 result = 0;
1159         else if (sg_dma_address(sgl)) {
1160                 result = sg_dma_address(sgl) |
1161                         1ULL << NV_MC_SMMU_VADDR_TRANSLATION_BIT;
1162         }
1163 #endif
1164         return result;
1165 }
1166
1167 static int update_gmmu_ptes(struct vm_gk20a *vm,
1168                             enum gmmu_pgsz_gk20a pgsz_idx,
1169                             struct sg_table *sgt,
1170                             u64 first_vaddr, u64 last_vaddr,
1171                             u8 kind_v, u32 ctag_offset,
1172                             bool cacheable)
1173 {
1174         int err;
1175         u32 pde_lo, pde_hi, pde_i;
1176         struct scatterlist *cur_chunk;
1177         unsigned int cur_offset;
1178         u32 pte_w[2] = {0, 0}; /* invalid pte */
1179         u32 ctag = ctag_offset;
1180         u32 ctag_incr;
1181         u32 page_size  = gmmu_page_sizes[pgsz_idx];
1182         u64 addr = 0;
1183
1184         pde_range_from_vaddr_range(vm, first_vaddr, last_vaddr,
1185                                    &pde_lo, &pde_hi);
1186
1187         nvhost_dbg(dbg_pte, "size_idx=%d, pde_lo=%d, pde_hi=%d",
1188                    pgsz_idx, pde_lo, pde_hi);
1189
1190         /* If ctag_offset !=0 add 1 else add 0.  The idea is to avoid a branch
1191          * below (per-pte). Note: this doesn't work unless page size (when
1192          * comptags are active) is 128KB. We have checks elsewhere for that. */
1193         ctag_incr = !!ctag_offset;
1194
1195         if (sgt)
1196                 cur_chunk = sgt->sgl;
1197         else
1198                 cur_chunk = NULL;
1199
1200         cur_offset = 0;
1201
1202         for (pde_i = pde_lo; pde_i <= pde_hi; pde_i++) {
1203                 u32 pte_lo, pte_hi;
1204                 u32 pte_cur;
1205                 void *pte_kv_cur;
1206
1207                 struct page_table_gk20a *pte = vm->pdes.ptes[pgsz_idx] + pde_i;
1208
1209                 if (pde_i == pde_lo)
1210                         pte_lo = pte_index_from_vaddr(vm, first_vaddr,
1211                                                       pgsz_idx);
1212                 else
1213                         pte_lo = 0;
1214
1215                 if ((pde_i != pde_hi) && (pde_hi != pde_lo))
1216                         pte_hi = vm->mm->page_table_sizing[pgsz_idx].num_ptes-1;
1217                 else
1218                         pte_hi = pte_index_from_vaddr(vm, last_vaddr,
1219                                                       pgsz_idx);
1220
1221                 /* get cpu access to the ptes */
1222                 err = map_gmmu_pages(pte->ref, pte->sgt, &pte_kv_cur);
1223                 if (err) {
1224                         nvhost_err(dev_from_vm(vm),
1225                                    "couldn't map ptes for update as=%d pte_ref_cnt=%d",
1226                                    vm_aspace_id(vm), pte->ref_cnt);
1227                         goto clean_up;
1228                 }
1229
1230                 nvhost_dbg(dbg_pte, "pte_lo=%d, pte_hi=%d", pte_lo, pte_hi);
1231                 for (pte_cur = pte_lo; pte_cur <= pte_hi; pte_cur++) {
1232
1233                         if (likely(sgt)) {
1234                                 u64 new_addr = gk20a_mm_iova_addr(cur_chunk);
1235                                 if (new_addr) {
1236                                         addr = new_addr;
1237                                         addr += cur_offset;
1238                                 }
1239
1240                                 pte_w[0] = gmmu_pte_valid_true_f() |
1241                                         gmmu_pte_address_sys_f(addr
1242                                                 >> gmmu_pte_address_shift_v());
1243                                 pte_w[1] = gmmu_pte_aperture_video_memory_f() |
1244                                         gmmu_pte_kind_f(kind_v) |
1245                                         gmmu_pte_comptagline_f(ctag);
1246
1247                                 if (!cacheable)
1248                                         pte_w[1] |= gmmu_pte_vol_true_f();
1249
1250                                 pte->ref_cnt++;
1251
1252                                 nvhost_dbg(dbg_pte,
1253                                            "pte_cur=%d addr=0x%x,%08x kind=%d"
1254                                            " ctag=%d vol=%d refs=%d"
1255                                            " [0x%08x,0x%08x]",
1256                                            pte_cur, hi32(addr), lo32(addr),
1257                                            kind_v, ctag, !cacheable,
1258                                            pte->ref_cnt, pte_w[1], pte_w[0]);
1259
1260                                 ctag += ctag_incr;
1261                                 cur_offset += page_size;
1262                                 addr += page_size;
1263                                 while (cur_chunk &&
1264                                         cur_offset >= cur_chunk->length) {
1265                                         cur_offset -= cur_chunk->length;
1266                                         cur_chunk = sg_next(cur_chunk);
1267                                 }
1268
1269                         } else {
1270                                 pte->ref_cnt--;
1271                                 nvhost_dbg(dbg_pte,
1272                                            "pte_cur=%d ref=%d [0x0,0x0]",
1273                                            pte_cur, pte->ref_cnt);
1274                         }
1275
1276                         mem_wr32(pte_kv_cur + pte_cur*8, 0, pte_w[0]);
1277                         mem_wr32(pte_kv_cur + pte_cur*8, 1, pte_w[1]);
1278                 }
1279
1280                 unmap_gmmu_pages(pte->ref, pte->sgt, pte_kv_cur);
1281
1282                 if (pte->ref_cnt == 0) {
1283                         /* It can make sense to keep around one page table for
1284                          * each flavor (empty)... in case a new map is coming
1285                          * right back to alloc (and fill it in) again.
1286                          * But: deferring unmapping should help with pathologic
1287                          * unmap/map/unmap/map cases where we'd trigger pte
1288                          * free/alloc/free/alloc.
1289                          */
1290                         free_gmmu_pages(vm, pte->ref, pte->sgt,
1291                                 vm->mm->page_table_sizing[pgsz_idx].order);
1292                         pte->ref = NULL;
1293
1294                         /* rewrite pde */
1295                         update_gmmu_pde(vm, pde_i);
1296                 }
1297
1298         }
1299
1300         smp_mb();
1301         vm->tlb_dirty = true;
1302         nvhost_dbg_fn("set tlb dirty");
1303
1304         return 0;
1305
1306 clean_up:
1307         /*TBD: potentially rewrite above to pre-map everything it needs to
1308          * as that's the only way it can fail */
1309         return err;
1310
1311 }
1312
1313
1314 /* for gk20a the "video memory" apertures here are misnomers. */
1315 static inline u32 big_valid_pde0_bits(u64 pte_addr)
1316 {
1317         u32 pde0_bits =
1318                 gmmu_pde_aperture_big_video_memory_f() |
1319                 gmmu_pde_address_big_sys_f(
1320                            (u32)(pte_addr >> gmmu_pde_address_shift_v()));
1321         return  pde0_bits;
1322 }
1323 static inline u32 small_valid_pde1_bits(u64 pte_addr)
1324 {
1325         u32 pde1_bits =
1326                 gmmu_pde_aperture_small_video_memory_f() |
1327                 gmmu_pde_vol_small_true_f() | /* tbd: why? */
1328                 gmmu_pde_address_small_sys_f(
1329                            (u32)(pte_addr >> gmmu_pde_address_shift_v()));
1330         return pde1_bits;
1331 }
1332
1333 /* Given the current state of the ptes associated with a pde,
1334    determine value and write it out.  There's no checking
1335    here to determine whether or not a change was actually
1336    made.  So, superfluous updates will cause unnecessary
1337    pde invalidations.
1338 */
1339 static void update_gmmu_pde(struct vm_gk20a *vm, u32 i)
1340 {
1341         bool small_valid, big_valid;
1342         u64 pte_addr[2] = {0, 0};
1343         struct page_table_gk20a *small_pte =
1344                 vm->pdes.ptes[gmmu_page_size_small] + i;
1345         struct page_table_gk20a *big_pte =
1346                 vm->pdes.ptes[gmmu_page_size_big] + i;
1347         u32 pde_v[2] = {0, 0};
1348         u32 *pde;
1349
1350         small_valid = small_pte && small_pte->ref;
1351         big_valid   = big_pte && big_pte->ref;
1352
1353         if (small_valid)
1354                 pte_addr[gmmu_page_size_small] =
1355                         sg_phys(small_pte->sgt->sgl);
1356         if (big_valid)
1357                 pte_addr[gmmu_page_size_big] =
1358                         sg_phys(big_pte->sgt->sgl);
1359
1360         pde_v[0] = gmmu_pde_size_full_f();
1361         pde_v[0] |= big_valid ?
1362                 big_valid_pde0_bits(pte_addr[gmmu_page_size_big])
1363                 :
1364                 (gmmu_pde_aperture_big_invalid_f());
1365
1366         pde_v[1] |= (small_valid ?
1367                      small_valid_pde1_bits(pte_addr[gmmu_page_size_small])
1368                      :
1369                      (gmmu_pde_aperture_small_invalid_f() |
1370                       gmmu_pde_vol_small_false_f())
1371                      )
1372                 |
1373                 (big_valid ? (gmmu_pde_vol_big_true_f()) :
1374                  gmmu_pde_vol_big_false_f());
1375
1376         pde = pde_from_index(vm, i);
1377
1378         mem_wr32(pde, 0, pde_v[0]);
1379         mem_wr32(pde, 1, pde_v[1]);
1380
1381         smp_mb();
1382
1383         FLUSH_CPU_DCACHE(pde,
1384                          sg_phys(vm->pdes.sgt->sgl) + (i*gmmu_pde__size_v()),
1385                          sizeof(u32)*2);
1386
1387         gk20a_mm_l2_invalidate(vm->mm->g);
1388
1389         nvhost_dbg(dbg_pte, "pde:%d = 0x%x,0x%08x\n", i, pde_v[1], pde_v[0]);
1390
1391         vm->tlb_dirty  = true;
1392 }
1393
1394
1395 /* return mem_mgr and mem_handle to caller. If the mem_handle is a kernel dup
1396    from user space (as_ioctl), caller releases the kernel duplicated handle */
1397 /* NOTE! mapped_buffers lock must be held */
1398 static void gk20a_vm_unmap_locked(struct vm_gk20a *vm, u64 offset,
1399                         struct mem_mgr **memmgr, struct mem_handle **r)
1400 {
1401         struct mapped_buffer_node *mapped_buffer;
1402         struct gk20a *g = gk20a_from_vm(vm);
1403         struct nvhost_allocator *comp_tags = &g->gr.comp_tags;
1404         int err = 0;
1405
1406         BUG_ON(memmgr == NULL || r == NULL);
1407
1408         *memmgr = NULL;
1409         *r = NULL;
1410
1411         mapped_buffer = find_mapped_buffer(&vm->mapped_buffers, offset);
1412         if (!mapped_buffer) {
1413                 nvhost_dbg(dbg_err, "invalid addr to unmap 0x%llx", offset);
1414                 return;
1415         }
1416
1417         vm->free_va(vm, mapped_buffer->addr, mapped_buffer->size,
1418                     mapped_buffer->pgsz_idx);
1419
1420         if (mapped_buffer->ctag_offset)
1421                 comp_tags->free(comp_tags,
1422                         mapped_buffer->ctag_offset, mapped_buffer->ctag_lines);
1423
1424         nvhost_dbg(dbg_map, "as=%d pgsz=%d gv=0x%x,%08x",
1425                    vm_aspace_id(vm), gmmu_page_sizes[mapped_buffer->pgsz_idx],
1426                    hi32(offset), lo32(offset));
1427
1428         /* unmap here needs to know the page size we assigned at mapping */
1429         err = update_gmmu_ptes(vm,
1430                                mapped_buffer->pgsz_idx,
1431                                0, /* n/a for unmap */
1432                                mapped_buffer->addr,
1433                                mapped_buffer->addr + mapped_buffer->size - 1,
1434                                0, 0, false /* n/a for unmap */);
1435
1436         /* detect which if any pdes/ptes can now be released */
1437
1438         /* flush l2 so any dirty lines are written out *now*.
1439          *  also as we could potentially be switching this buffer
1440          * from nonvolatile (l2 cacheable) to volatile (l2 non-cacheable) at
1441          * some point in the future we need to invalidate l2.  e.g. switching
1442          * from a render buffer unmap (here) to later using the same memory
1443          * for gmmu ptes.  note the positioning of this relative to any smmu
1444          * unmapping (below). */
1445         gk20a_mm_l2_flush(g, true);
1446
1447         if (err)
1448                 dev_err(dev_from_vm(vm),
1449                         "failed to update gmmu ptes on unmap");
1450
1451         nvhost_memmgr_unpin(mapped_buffer->memmgr,
1452                             mapped_buffer->handle_ref,
1453                             dev_from_vm(vm),
1454                             mapped_buffer->sgt);
1455
1456         /* remove from mapped buffer tree and remove list, free */
1457         rb_erase(&mapped_buffer->node, &vm->mapped_buffers);
1458         list_del_init(&mapped_buffer->unmap_list);
1459
1460         /* keep track of mapped buffers */
1461         if (mapped_buffer->user_mapped)
1462                 vm->num_user_mapped_buffers--;
1463
1464         *memmgr = mapped_buffer->memmgr;
1465         *r = mapped_buffer->handle_ref;
1466         kfree(mapped_buffer);
1467
1468         return;
1469 }
1470
1471 /* called by kernel. mem_mgr and mem_handle are ignored */
1472 static void gk20a_vm_unmap(struct vm_gk20a *vm, u64 offset)
1473 {
1474         struct mem_mgr *memmgr;
1475         struct mem_handle *r;
1476
1477         mutex_lock(&vm->update_gmmu_lock);
1478
1479         gk20a_vm_unmap_locked(vm, offset, &memmgr, &r);
1480
1481         mutex_unlock(&vm->update_gmmu_lock);
1482 }
1483
1484 void gk20a_vm_remove_support(struct vm_gk20a *vm)
1485 {
1486         struct mapped_buffer_node *mapped_buffer;
1487         struct rb_node *node;
1488         struct mem_mgr *memmgr;
1489         struct mem_handle *r;
1490
1491         nvhost_dbg_fn("");
1492         mutex_lock(&vm->update_gmmu_lock);
1493
1494         /* TBD: add a flag here for the unmap code to recognize teardown
1495          * and short-circuit any otherwise expensive operations. */
1496
1497         node = rb_first(&vm->mapped_buffers);
1498         while (node) {
1499                 mapped_buffer =
1500                         container_of(node, struct mapped_buffer_node, node);
1501                 gk20a_vm_unmap_locked(vm, mapped_buffer->addr, &memmgr, &r);
1502                 if (memmgr != mem_mgr_from_vm(vm)) {
1503                         nvhost_memmgr_put(memmgr, r);
1504                         nvhost_memmgr_put_mgr(memmgr);
1505                 }
1506                 node = rb_first(&vm->mapped_buffers);
1507         }
1508
1509         /* TBD: unmapping all buffers above may not actually free
1510          * all vm ptes.  jettison them here for certain... */
1511
1512         unmap_gmmu_pages(vm->pdes.ref, vm->pdes.sgt, vm->pdes.kv);
1513         free_gmmu_pages(vm, vm->pdes.ref, vm->pdes.sgt, 0);
1514
1515         kfree(vm->pdes.ptes[gmmu_page_size_small]);
1516         kfree(vm->pdes.ptes[gmmu_page_size_big]);
1517         nvhost_allocator_destroy(&vm->vma[gmmu_page_size_small]);
1518         nvhost_allocator_destroy(&vm->vma[gmmu_page_size_big]);
1519
1520         mutex_unlock(&vm->update_gmmu_lock);
1521 }
1522
1523 /* address space interfaces for the gk20a module */
1524 static int gk20a_as_alloc_share(struct nvhost_as_share *as_share)
1525 {
1526         struct nvhost_as *as = as_share->as;
1527         struct gk20a *gk20a = get_gk20a(as->ch->dev);
1528         struct mm_gk20a *mm = &gk20a->mm;
1529         struct vm_gk20a *vm;
1530         u64 vma_size;
1531         u32 num_pages, low_hole_pages;
1532         char name[32];
1533         int err;
1534
1535         nvhost_dbg_fn("");
1536
1537         vm = kzalloc(sizeof(*vm), GFP_KERNEL);
1538         if (!vm)
1539                 return -ENOMEM;
1540
1541         as_share->priv = (void *)vm;
1542
1543         vm->mm = mm;
1544         vm->as_share = as_share;
1545
1546         vm->big_pages = true;
1547
1548         vm->va_start  = mm->pde_stride;   /* create a one pde hole */
1549         vm->va_limit  = mm->channel.size; /* note this means channel.size is
1550                                              really just the max */
1551         {
1552                 u32 pde_lo, pde_hi;
1553                 pde_range_from_vaddr_range(vm,
1554                                            0, vm->va_limit-1,
1555                                            &pde_lo, &pde_hi);
1556                 vm->pdes.num_pdes = pde_hi + 1;
1557         }
1558
1559         vm->pdes.ptes[gmmu_page_size_small] =
1560                 kzalloc(sizeof(struct page_table_gk20a) *
1561                         vm->pdes.num_pdes, GFP_KERNEL);
1562
1563         vm->pdes.ptes[gmmu_page_size_big] =
1564                 kzalloc(sizeof(struct page_table_gk20a) *
1565                         vm->pdes.num_pdes, GFP_KERNEL);
1566
1567         if (!(vm->pdes.ptes[gmmu_page_size_small] &&
1568               vm->pdes.ptes[gmmu_page_size_big]))
1569                 return -ENOMEM;
1570
1571         nvhost_dbg_info("init space for va_limit=0x%llx num_pdes=%d",
1572                    vm->va_limit, vm->pdes.num_pdes);
1573
1574         /* allocate the page table directory */
1575         err = alloc_gmmu_pages(vm, 0, &vm->pdes.ref,
1576                                &vm->pdes.sgt);
1577         if (err)
1578                 return -ENOMEM;
1579
1580         err = map_gmmu_pages(vm->pdes.ref, vm->pdes.sgt, &vm->pdes.kv);
1581         if (err) {
1582                 free_gmmu_pages(vm, vm->pdes.ref, vm->pdes.sgt, 0);
1583                 return -ENOMEM;
1584         }
1585         nvhost_dbg(dbg_pte, "pdes.kv = 0x%p, pdes.phys = 0x%llx",
1586                         vm->pdes.kv, (u64)sg_phys(vm->pdes.sgt->sgl));
1587         /* we could release vm->pdes.kv but it's only one page... */
1588
1589
1590         /* low-half: alloc small pages */
1591         /* high-half: alloc big pages */
1592         vma_size = mm->channel.size >> 1;
1593
1594         snprintf(name, sizeof(name), "gk20a_as_%d-%dKB", as_share->id,
1595                  gmmu_page_sizes[gmmu_page_size_small]>>10);
1596         num_pages = (u32)(vma_size >> gmmu_page_shifts[gmmu_page_size_small]);
1597
1598         /* num_pages above is without regard to the low-side hole. */
1599         low_hole_pages = (vm->va_start >>
1600                           gmmu_page_shifts[gmmu_page_size_small]);
1601
1602         nvhost_allocator_init(&vm->vma[gmmu_page_size_small], name,
1603               low_hole_pages,             /* start */
1604               num_pages - low_hole_pages, /* length */
1605               1);                         /* align */
1606
1607         snprintf(name, sizeof(name), "gk20a_as_%d-%dKB", as_share->id,
1608                  gmmu_page_sizes[gmmu_page_size_big]>>10);
1609
1610         num_pages = (u32)(vma_size >> gmmu_page_shifts[gmmu_page_size_big]);
1611         nvhost_allocator_init(&vm->vma[gmmu_page_size_big], name,
1612                               num_pages, /* start */
1613                               num_pages, /* length */
1614                               1); /* align */
1615
1616         vm->mapped_buffers = RB_ROOT;
1617
1618         mutex_init(&vm->update_gmmu_lock);
1619
1620         INIT_LIST_HEAD(&vm->deferred_unmaps);
1621
1622         vm->alloc_va       = gk20a_vm_alloc_va;
1623         vm->free_va        = gk20a_vm_free_va;
1624         vm->map            = gk20a_vm_map;
1625         vm->unmap          = gk20a_vm_unmap;
1626         vm->unmap_user     = gk20a_vm_unmap_user;
1627         vm->put_buffers    = gk20a_vm_put_buffers;
1628         vm->get_buffers    = gk20a_vm_get_buffers;
1629         vm->tlb_inval      = gk20a_mm_tlb_invalidate;
1630         vm->find_buffer    = gk20a_vm_find_buffer;
1631         vm->remove_support = gk20a_vm_remove_support;
1632
1633         vm->enable_ctag = true;
1634
1635         return 0;
1636 }
1637
1638
1639 static int gk20a_as_release_share(struct nvhost_as_share *as_share)
1640 {
1641         struct vm_gk20a *vm = (struct vm_gk20a *)as_share->priv;
1642
1643         nvhost_dbg_fn("");
1644
1645         gk20a_vm_remove_support(vm);
1646
1647         as_share->priv = NULL;
1648         kfree(vm);
1649
1650         return 0;
1651 }
1652
1653
1654 static int gk20a_as_alloc_space(struct nvhost_as_share *as_share,
1655                                 struct nvhost_as_alloc_space_args *args)
1656
1657 {       int err = -ENOMEM;
1658         int pgsz_idx;
1659         u32 start_page_nr;
1660         struct nvhost_allocator *vma;
1661         struct vm_gk20a *vm = (struct vm_gk20a *)as_share->priv;
1662
1663         nvhost_dbg_fn("flags=0x%x pgsz=0x%x nr_pages=0x%x o/a=0x%llx",
1664                         args->flags, args->page_size, args->pages,
1665                         args->o_a.offset);
1666
1667         /* determine pagesz idx */
1668         for (pgsz_idx = gmmu_page_size_small;
1669              pgsz_idx < gmmu_nr_page_sizes;
1670              pgsz_idx++) {
1671                 if (gmmu_page_sizes[pgsz_idx] == args->page_size)
1672                         break;
1673         }
1674
1675         if (pgsz_idx >= gmmu_nr_page_sizes) {
1676                 err = -EINVAL;
1677                 goto clean_up;
1678         }
1679
1680         start_page_nr = ~(u32)0;
1681         if (args->flags & NVHOST_AS_ALLOC_SPACE_FLAGS_FIXED_OFFSET)
1682                 start_page_nr = (u32)(args->o_a.offset >>
1683                                       gmmu_page_shifts[pgsz_idx]);
1684
1685         vma = &vm->vma[pgsz_idx];
1686         err = vma->alloc(vma, &start_page_nr, args->pages);
1687         args->o_a.offset = (u64)start_page_nr << gmmu_page_shifts[pgsz_idx];
1688
1689 clean_up:
1690         return err;
1691 }
1692
1693 static int gk20a_as_free_space(struct nvhost_as_share *as_share,
1694                                struct nvhost_as_free_space_args *args)
1695 {
1696         int err = -ENOMEM;
1697         int pgsz_idx;
1698         u32 start_page_nr;
1699         struct nvhost_allocator *vma;
1700         struct vm_gk20a *vm = (struct vm_gk20a *)as_share->priv;
1701
1702         nvhost_dbg_fn("pgsz=0x%x nr_pages=0x%x o/a=0x%llx", args->page_size,
1703                         args->pages, args->offset);
1704
1705         /* determine pagesz idx */
1706         for (pgsz_idx = gmmu_page_size_small;
1707              pgsz_idx < gmmu_nr_page_sizes;
1708              pgsz_idx++) {
1709                 if (gmmu_page_sizes[pgsz_idx] == args->page_size)
1710                         break;
1711         }
1712
1713         if (pgsz_idx >= gmmu_nr_page_sizes) {
1714                 err = -EINVAL;
1715                 goto clean_up;
1716         }
1717
1718         start_page_nr = (u32)(args->offset >>
1719                               gmmu_page_shifts[pgsz_idx]);
1720
1721         vma = &vm->vma[pgsz_idx];
1722         err = vma->free(vma, start_page_nr, args->pages);
1723
1724 clean_up:
1725         return err;
1726 }
1727
1728 static int gk20a_as_bind_hwctx(struct nvhost_as_share *as_share,
1729                                struct nvhost_hwctx *hwctx)
1730 {
1731         int err = 0;
1732         struct vm_gk20a *vm = (struct vm_gk20a *)as_share->priv;
1733         struct channel_gk20a *c = hwctx->priv;
1734
1735         nvhost_dbg_fn("");
1736
1737         c->vm = vm;
1738         err = channel_gk20a_commit_va(c);
1739         if (err)
1740                 c->vm = 0;
1741
1742         return err;
1743 }
1744
1745 static int gk20a_as_map_buffer(struct nvhost_as_share *as_share,
1746                                struct mem_mgr *nvmap,
1747                                struct mem_handle *r,
1748                                u64 *offset_align,
1749                                u32 flags /*NVHOST_AS_MAP_BUFFER_FLAGS_*/)
1750 {
1751         int err = 0;
1752         struct vm_gk20a *vm = (struct vm_gk20a *)as_share->priv;
1753         u64 ret_va;
1754
1755         nvhost_dbg_fn("");
1756
1757         ret_va = vm->map(vm, nvmap, r, *offset_align,
1758                         flags, 0/*no kind here, to be removed*/, NULL, true);
1759         *offset_align = ret_va;
1760         if (!ret_va)
1761                 err = -EINVAL;
1762
1763         return err;
1764
1765 }
1766
1767 static int gk20a_as_unmap_buffer(struct nvhost_as_share *as_share, u64 offset,
1768                                  struct mem_mgr **memmgr, struct mem_handle **r)
1769 {
1770         struct vm_gk20a *vm = (struct vm_gk20a *)as_share->priv;
1771
1772         nvhost_dbg_fn("");
1773
1774         vm->unmap_user(vm, offset);
1775
1776         /* these are not available */
1777         if (memmgr)
1778                 *memmgr = NULL;
1779         if (r)
1780                 *r = NULL;
1781         return 0;
1782 }
1783
1784
1785 const struct nvhost_as_moduleops gk20a_as_moduleops = {
1786         .alloc_share   = gk20a_as_alloc_share,
1787         .release_share = gk20a_as_release_share,
1788         .alloc_space   = gk20a_as_alloc_space,
1789         .free_space    = gk20a_as_free_space,
1790         .bind_hwctx    = gk20a_as_bind_hwctx,
1791         .map_buffer    = gk20a_as_map_buffer,
1792         .unmap_buffer  = gk20a_as_unmap_buffer,
1793 };
1794
1795 int gk20a_init_bar1_vm(struct mm_gk20a *mm)
1796 {
1797         int err;
1798         struct mem_mgr *nvmap = mem_mgr_from_mm(mm);
1799         phys_addr_t inst_pa;
1800         void *inst_ptr;
1801         struct vm_gk20a *vm = &mm->bar1.vm;
1802         struct inst_desc *inst_block = &mm->bar1.inst_block;
1803         phys_addr_t pde_addr;
1804         u32 pde_addr_lo;
1805         u32 pde_addr_hi;
1806
1807         vm->mm = mm;
1808
1809         mm->bar1.aperture_size = bar1_aperture_size_mb_gk20a() << 20;
1810
1811         nvhost_dbg_info("bar1 vm size = 0x%x", mm->bar1.aperture_size);
1812
1813         vm->va_start = mm->pde_stride * 1;
1814         vm->va_limit = mm->bar1.aperture_size;
1815
1816         {
1817                 u32 pde_lo, pde_hi;
1818                 pde_range_from_vaddr_range(vm,
1819                                            0, vm->va_limit-1,
1820                                            &pde_lo, &pde_hi);
1821                 vm->pdes.num_pdes = pde_hi + 1;
1822         }
1823
1824         /* bar1 is likely only to ever use/need small page sizes. */
1825         /* But just in case, for now... arrange for both.*/
1826         vm->pdes.ptes[gmmu_page_size_small] =
1827                 kzalloc(sizeof(struct page_table_gk20a) *
1828                         vm->pdes.num_pdes, GFP_KERNEL);
1829
1830         vm->pdes.ptes[gmmu_page_size_big] =
1831                 kzalloc(sizeof(struct page_table_gk20a) *
1832                         vm->pdes.num_pdes, GFP_KERNEL);
1833
1834         if (!(vm->pdes.ptes[gmmu_page_size_small] &&
1835               vm->pdes.ptes[gmmu_page_size_big]))
1836                 return -ENOMEM;
1837
1838         nvhost_dbg_info("init space for bar1 va_limit=0x%llx num_pdes=%d",
1839                    vm->va_limit, vm->pdes.num_pdes);
1840
1841
1842         /* allocate the page table directory */
1843         err = alloc_gmmu_pages(vm, 0, &vm->pdes.ref,
1844                                &vm->pdes.sgt);
1845         if (err)
1846                 goto clean_up;
1847
1848         err = map_gmmu_pages(vm->pdes.ref, vm->pdes.sgt, &vm->pdes.kv);
1849         if (err) {
1850                 free_gmmu_pages(vm, vm->pdes.ref, vm->pdes.sgt, 0);
1851                 goto clean_up;
1852         }
1853         nvhost_dbg(dbg_pte, "bar 1 pdes.kv = 0x%p, pdes.phys = 0x%llx",
1854                         vm->pdes.kv, (u64)sg_phys(vm->pdes.sgt->sgl));
1855         /* we could release vm->pdes.kv but it's only one page... */
1856
1857         pde_addr = sg_phys(vm->pdes.sgt->sgl);
1858         pde_addr_lo = u64_lo32(pde_addr) >> 12;
1859         pde_addr_hi = u64_hi32(pde_addr);
1860
1861         nvhost_dbg_info("pde pa=0x%llx pde_addr_lo=0x%x pde_addr_hi=0x%x",
1862                 (u64)sg_phys(vm->pdes.sgt->sgl), pde_addr_lo, pde_addr_hi);
1863
1864         /* allocate instance mem for bar1 */
1865         inst_block->mem.size = ram_in_alloc_size_v();
1866         inst_block->mem.ref =
1867                 nvhost_memmgr_alloc(nvmap, inst_block->mem.size,
1868                                     DEFAULT_ALLOC_ALIGNMENT,
1869                                     DEFAULT_ALLOC_FLAGS,
1870                                     0);
1871
1872         if (IS_ERR(inst_block->mem.ref)) {
1873                 inst_block->mem.ref = 0;
1874                 err = -ENOMEM;
1875                 goto clean_up;
1876         }
1877
1878         inst_block->mem.sgt = nvhost_memmgr_sg_table(nvmap,
1879                         inst_block->mem.ref);
1880         /* IS_ERR throws a warning here (expecting void *) */
1881         if (IS_ERR(inst_block->mem.sgt)) {
1882                 inst_pa = 0;
1883                 err = (int)inst_pa;
1884                 goto clean_up;
1885         }
1886         inst_pa = sg_phys(inst_block->mem.sgt->sgl);
1887
1888         inst_ptr = nvhost_memmgr_mmap(inst_block->mem.ref);
1889         if (IS_ERR(inst_ptr)) {
1890                 return -ENOMEM;
1891                 goto clean_up;
1892         }
1893
1894         nvhost_dbg_info("bar1 inst block physical phys = 0x%llx, kv = 0x%p",
1895                 (u64)inst_pa, inst_ptr);
1896
1897         memset(inst_ptr, 0, ram_fc_size_val_v());
1898
1899         mem_wr32(inst_ptr, ram_in_page_dir_base_lo_w(),
1900                 ram_in_page_dir_base_target_vid_mem_f() |
1901                 ram_in_page_dir_base_vol_true_f() |
1902                 ram_in_page_dir_base_lo_f(pde_addr_lo));
1903
1904         mem_wr32(inst_ptr, ram_in_page_dir_base_hi_w(),
1905                 ram_in_page_dir_base_hi_f(pde_addr_hi));
1906
1907         mem_wr32(inst_ptr, ram_in_adr_limit_lo_w(),
1908                  u64_lo32(vm->va_limit) | 0xFFF);
1909
1910         mem_wr32(inst_ptr, ram_in_adr_limit_hi_w(),
1911                 ram_in_adr_limit_hi_f(u64_hi32(vm->va_limit)));
1912
1913         nvhost_memmgr_munmap(inst_block->mem.ref, inst_ptr);
1914
1915         nvhost_dbg_info("bar1 inst block ptr: %08llx",  (u64)inst_pa);
1916         nvhost_allocator_init(&vm->vma[gmmu_page_size_small], "gk20a_bar1",
1917                               1,/*start*/
1918                               (vm->va_limit >> 12) - 1 /* length*/,
1919                               1); /* align */
1920         /* initialize just in case we try to use it anyway */
1921         nvhost_allocator_init(&vm->vma[gmmu_page_size_big], "gk20a_bar1-unused",
1922                               0x0badc0de, /* start */
1923                               1, /* length */
1924                               1); /* align */
1925
1926
1927         vm->mapped_buffers = RB_ROOT;
1928
1929         mutex_init(&vm->update_gmmu_lock);
1930
1931         INIT_LIST_HEAD(&vm->deferred_unmaps);
1932
1933         vm->alloc_va       = gk20a_vm_alloc_va;
1934         vm->free_va        = gk20a_vm_free_va;
1935         vm->map            = gk20a_vm_map;
1936         vm->unmap          = gk20a_vm_unmap;
1937         vm->unmap_user     = gk20a_vm_unmap_user;
1938         vm->put_buffers    = gk20a_vm_put_buffers;
1939         vm->get_buffers    = gk20a_vm_get_buffers;
1940         vm->tlb_inval      = gk20a_mm_tlb_invalidate;
1941         vm->remove_support = gk20a_vm_remove_support;
1942
1943         return 0;
1944
1945 clean_up:
1946         /* free, etc */
1947         return err;
1948 }
1949
1950 /* pmu vm, share channel_vm interfaces */
1951 int gk20a_init_pmu_vm(struct mm_gk20a *mm)
1952 {
1953         int err;
1954         struct mem_mgr *nvmap = mem_mgr_from_mm(mm);
1955         phys_addr_t inst_pa;
1956         void *inst_ptr;
1957         struct vm_gk20a *vm = &mm->pmu.vm;
1958         struct inst_desc *inst_block = &mm->pmu.inst_block;
1959         u64 pde_addr;
1960         u32 pde_addr_lo;
1961         u32 pde_addr_hi;
1962
1963         vm->mm = mm;
1964
1965         mm->pmu.aperture_size = GK20A_PMU_VA_SIZE;
1966
1967         nvhost_dbg_info("pmu vm size = 0x%x", mm->pmu.aperture_size);
1968
1969         vm->va_start  = GK20A_PMU_VA_START;
1970         vm->va_limit  = vm->va_start + mm->pmu.aperture_size;
1971
1972         {
1973                 u32 pde_lo, pde_hi;
1974                 pde_range_from_vaddr_range(vm,
1975                                            0, vm->va_limit-1,
1976                                            &pde_lo, &pde_hi);
1977                 vm->pdes.num_pdes = pde_hi + 1;
1978         }
1979
1980         /* The pmu is likely only to ever use/need small page sizes. */
1981         /* But just in case, for now... arrange for both.*/
1982         vm->pdes.ptes[gmmu_page_size_small] =
1983                 kzalloc(sizeof(struct page_table_gk20a) *
1984                         vm->pdes.num_pdes, GFP_KERNEL);
1985
1986         vm->pdes.ptes[gmmu_page_size_big] =
1987                 kzalloc(sizeof(struct page_table_gk20a) *
1988                         vm->pdes.num_pdes, GFP_KERNEL);
1989
1990         if (!(vm->pdes.ptes[gmmu_page_size_small] &&
1991               vm->pdes.ptes[gmmu_page_size_big]))
1992                 return -ENOMEM;
1993
1994         nvhost_dbg_info("init space for pmu va_limit=0x%llx num_pdes=%d",
1995                    vm->va_limit, vm->pdes.num_pdes);
1996
1997         /* allocate the page table directory */
1998         err = alloc_gmmu_pages(vm, 0, &vm->pdes.ref,
1999                                &vm->pdes.sgt);
2000         if (err)
2001                 goto clean_up;
2002
2003         err = map_gmmu_pages(vm->pdes.ref, vm->pdes.sgt, &vm->pdes.kv);
2004         if (err) {
2005                 free_gmmu_pages(vm, vm->pdes.ref, vm->pdes.sgt, 0);
2006                 goto clean_up;
2007         }
2008         nvhost_dbg_info("pmu pdes phys @ 0x%llx",
2009                         (u64)sg_phys(vm->pdes.sgt->sgl));
2010         /* we could release vm->pdes.kv but it's only one page... */
2011
2012         pde_addr = sg_phys(vm->pdes.sgt->sgl);
2013         pde_addr_lo = u64_lo32(pde_addr) >> 12;
2014         pde_addr_hi = u64_hi32(pde_addr);
2015
2016         nvhost_dbg_info("pde pa=0x%llx pde_addr_lo=0x%x pde_addr_hi=0x%x",
2017                         (u64)pde_addr, pde_addr_lo, pde_addr_hi);
2018
2019         /* allocate instance mem for pmu */
2020         inst_block->mem.size = GK20A_PMU_INST_SIZE;
2021         inst_block->mem.ref =
2022                 nvhost_memmgr_alloc(nvmap, inst_block->mem.size,
2023                                     DEFAULT_ALLOC_ALIGNMENT,
2024                                     DEFAULT_ALLOC_FLAGS,
2025                                     0);
2026
2027         if (IS_ERR(inst_block->mem.ref)) {
2028                 inst_block->mem.ref = 0;
2029                 err = -ENOMEM;
2030                 goto clean_up;
2031         }
2032
2033         inst_block->mem.sgt = nvhost_memmgr_sg_table(nvmap,
2034                         inst_block->mem.ref);
2035         /* IS_ERR throws a warning here (expecting void *) */
2036         if (IS_ERR(inst_block->mem.sgt)) {
2037                 inst_pa = 0;
2038                 err = (int)((uintptr_t)inst_block->mem.sgt);
2039                 goto clean_up;
2040         }
2041         inst_pa = sg_phys(inst_block->mem.sgt->sgl);
2042
2043         nvhost_dbg_info("pmu inst block physical addr: 0x%llx", (u64)inst_pa);
2044
2045         inst_ptr = nvhost_memmgr_mmap(inst_block->mem.ref);
2046         if (IS_ERR(inst_ptr)) {
2047                 return -ENOMEM;
2048                 goto clean_up;
2049         }
2050
2051         memset(inst_ptr, 0, GK20A_PMU_INST_SIZE);
2052
2053         mem_wr32(inst_ptr, ram_in_page_dir_base_lo_w(),
2054                 ram_in_page_dir_base_target_vid_mem_f() |
2055                 ram_in_page_dir_base_vol_true_f() |
2056                 ram_in_page_dir_base_lo_f(pde_addr_lo));
2057
2058         mem_wr32(inst_ptr, ram_in_page_dir_base_hi_w(),
2059                 ram_in_page_dir_base_hi_f(pde_addr_hi));
2060
2061         mem_wr32(inst_ptr, ram_in_adr_limit_lo_w(),
2062                  u64_lo32(vm->va_limit) | 0xFFF);
2063
2064         mem_wr32(inst_ptr, ram_in_adr_limit_hi_w(),
2065                 ram_in_adr_limit_hi_f(u64_hi32(vm->va_limit)));
2066
2067         nvhost_memmgr_munmap(inst_block->mem.ref, inst_ptr);
2068
2069         nvhost_allocator_init(&vm->vma[gmmu_page_size_small], "gk20a_pmu",
2070                               (vm->va_start >> 12), /* start */
2071                               (vm->va_limit - vm->va_start) >> 12, /*length*/
2072                               1); /* align */
2073         /* initialize just in case we try to use it anyway */
2074         nvhost_allocator_init(&vm->vma[gmmu_page_size_big], "gk20a_pmu-unused",
2075                               0x0badc0de, /* start */
2076                               1, /* length */
2077                               1); /* align */
2078
2079
2080         vm->mapped_buffers = RB_ROOT;
2081
2082         mutex_init(&vm->update_gmmu_lock);
2083
2084         INIT_LIST_HEAD(&vm->deferred_unmaps);
2085
2086         vm->alloc_va       = gk20a_vm_alloc_va;
2087         vm->free_va        = gk20a_vm_free_va;
2088         vm->map            = gk20a_vm_map;
2089         vm->unmap          = gk20a_vm_unmap;
2090         vm->unmap_user     = gk20a_vm_unmap_user;
2091         vm->put_buffers    = gk20a_vm_put_buffers;
2092         vm->get_buffers    = gk20a_vm_get_buffers;
2093         vm->tlb_inval      = gk20a_mm_tlb_invalidate;
2094         vm->remove_support = gk20a_vm_remove_support;
2095
2096         return 0;
2097
2098 clean_up:
2099         /* free, etc */
2100         return err;
2101 }
2102
2103 /* Flushes the compression bit cache as well as "data".
2104  * Note: the name here is a bit of a misnomer.  ELPG uses this
2105  * internally... but ELPG doesn't have to be on to do it manually.
2106  */
2107 static void gk20a_mm_g_elpg_flush(struct gk20a *g)
2108 {
2109         u32 data;
2110         s32 retry = 100;
2111
2112         nvhost_dbg_fn("");
2113
2114         /* Make sure all previous writes are committed to the L2. There's no
2115            guarantee that writes are to DRAM. This will be a sysmembar internal
2116            to the L2. */
2117         gk20a_writel(g, ltc_ltss_g_elpg_r(),
2118                      ltc_ltss_g_elpg_flush_pending_f());
2119         do {
2120                 data = gk20a_readl(g, ltc_ltss_g_elpg_r());
2121
2122                 if (ltc_ltss_g_elpg_flush_v(data) ==
2123                     ltc_ltss_g_elpg_flush_pending_v()) {
2124                         nvhost_dbg_info("g_elpg_flush 0x%x", data);
2125                         retry--;
2126                         usleep_range(20, 40);
2127                 } else
2128                         break;
2129         } while (retry >= 0);
2130
2131         if (retry < 0)
2132                 nvhost_warn(dev_from_gk20a(g),
2133                             "g_elpg_flush too many retries");
2134
2135 }
2136
2137 void gk20a_mm_fb_flush(struct gk20a *g)
2138 {
2139         u32 data;
2140         s32 retry = 100;
2141
2142         nvhost_dbg_fn("");
2143
2144         gk20a_mm_g_elpg_flush(g);
2145
2146         /* Make sure all previous writes are committed to the L2. There's no
2147            guarantee that writes are to DRAM. This will be a sysmembar internal
2148            to the L2. */
2149         gk20a_writel(g, flush_fb_flush_r(),
2150                 flush_fb_flush_pending_busy_f());
2151
2152         do {
2153                 data = gk20a_readl(g, flush_fb_flush_r());
2154
2155                 if (flush_fb_flush_outstanding_v(data) ==
2156                         flush_fb_flush_outstanding_true_v() ||
2157                     flush_fb_flush_pending_v(data) ==
2158                         flush_fb_flush_pending_busy_v()) {
2159                                 nvhost_dbg_info("fb_flush 0x%x", data);
2160                                 retry--;
2161                                 usleep_range(20, 40);
2162                 } else
2163                         break;
2164         } while (retry >= 0);
2165
2166         if (retry < 0)
2167                 nvhost_warn(dev_from_gk20a(g),
2168                         "fb_flush too many retries");
2169 }
2170
2171 void gk20a_mm_l2_flush(struct gk20a *g, bool invalidate)
2172 {
2173         u32 data;
2174         s32 retry = 200;
2175
2176         nvhost_dbg_fn("");
2177         /* Flush all dirty lines from the L2 to DRAM. Lines are left in the L2
2178            as clean, so subsequent reads might hit in the L2. */
2179         gk20a_writel(g, flush_l2_flush_dirty_r(),
2180                 flush_l2_flush_dirty_pending_busy_f());
2181
2182         do {
2183                 data = gk20a_readl(g, flush_l2_flush_dirty_r());
2184
2185                 if (flush_l2_flush_dirty_outstanding_v(data) ==
2186                         flush_l2_flush_dirty_outstanding_true_v() ||
2187                     flush_l2_flush_dirty_pending_v(data) ==
2188                         flush_l2_flush_dirty_pending_busy_v()) {
2189                                 nvhost_dbg_info("l2_flush_dirty 0x%x", data);
2190                                 retry--;
2191                                 usleep_range(20, 40);
2192                 } else
2193                         break;
2194         } while (retry >= 0);
2195
2196         if (retry < 0)
2197                 nvhost_warn(dev_from_gk20a(g),
2198                         "l2_flush_dirty too many retries");
2199
2200         if (!invalidate)
2201                 return;
2202
2203         gk20a_mm_l2_invalidate(g);
2204
2205         return;
2206 }
2207
2208 void gk20a_mm_l2_invalidate(struct gk20a *g)
2209 {
2210         u32 data;
2211         s32 retry = 200;
2212
2213         /* Invalidate any clean lines from the L2 so subsequent reads go to
2214            DRAM. Dirty lines are not affected by this operation. */
2215         gk20a_writel(g, flush_l2_system_invalidate_r(),
2216                 flush_l2_system_invalidate_pending_busy_f());
2217
2218         do {
2219                 data = gk20a_readl(g, flush_l2_system_invalidate_r());
2220
2221                 if (flush_l2_system_invalidate_outstanding_v(data) ==
2222                         flush_l2_system_invalidate_outstanding_true_v() ||
2223                     flush_l2_system_invalidate_pending_v(data) ==
2224                         flush_l2_system_invalidate_pending_busy_v()) {
2225                                 nvhost_dbg_info("l2_system_invalidate 0x%x", data);
2226                                 retry--;
2227                                 usleep_range(20, 40);
2228                 } else
2229                         break;
2230         } while (retry >= 0);
2231
2232         if (retry < 0)
2233                 nvhost_warn(dev_from_gk20a(g),
2234                         "l2_system_invalidate too many retries");
2235 }
2236
2237 static int gk20a_vm_find_buffer(struct vm_gk20a *vm, u64 gpu_va,
2238                 struct mem_mgr **mgr, struct mem_handle **r, u64 *offset)
2239 {
2240         struct mapped_buffer_node *mapped_buffer;
2241
2242         nvhost_dbg_fn("gpu_va=0x%llx", gpu_va);
2243         mapped_buffer = find_mapped_buffer_range(&vm->mapped_buffers, gpu_va);
2244         if (!mapped_buffer)
2245                 return -EINVAL;
2246
2247         *mgr = mapped_buffer->memmgr;
2248         *r = mapped_buffer->handle_ref;
2249         *offset = gpu_va - mapped_buffer->addr;
2250         return 0;
2251 }
2252
2253 static void gk20a_mm_tlb_invalidate(struct vm_gk20a *vm)
2254 {
2255         struct gk20a *g = gk20a_from_vm(vm);
2256         u32 addr_lo = u64_lo32(sg_phys(vm->pdes.sgt->sgl) >> 12);
2257         u32 data;
2258         s32 retry = 200;
2259
2260         /* pagetables are considered sw states which are preserved after
2261            prepare_poweroff. When gk20a deinit releases those pagetables,
2262            common code in vm unmap path calls tlb invalidate that touches
2263            hw. Use the power_on flag to skip tlb invalidation when gpu
2264            power is turned off */
2265         if (!g->power_on)
2266                 return;
2267
2268         nvhost_dbg_fn("");
2269
2270         do {
2271                 data = gk20a_readl(g, fb_mmu_ctrl_r());
2272                 if (fb_mmu_ctrl_pri_fifo_space_v(data) != 0)
2273                         break;
2274                 usleep_range(20, 40);
2275                 retry--;
2276         } while (retry >= 0);
2277
2278         if (retry < 0)
2279                 nvhost_warn(dev_from_gk20a(g),
2280                         "wait mmu fifo space too many retries");
2281
2282         gk20a_writel(g, fb_mmu_invalidate_pdb_r(),
2283                 fb_mmu_invalidate_pdb_addr_f(addr_lo) |
2284                 fb_mmu_invalidate_pdb_aperture_vid_mem_f());
2285
2286         /* this is a sledgehammer, it would seem */
2287         gk20a_writel(g, fb_mmu_invalidate_r(),
2288                 fb_mmu_invalidate_all_pdb_true_f() |
2289                 fb_mmu_invalidate_all_va_true_f() |
2290                 fb_mmu_invalidate_trigger_true_f());
2291
2292         do {
2293                 data = gk20a_readl(g, fb_mmu_ctrl_r());
2294                 if (fb_mmu_ctrl_pri_fifo_empty_v(data) !=
2295                         fb_mmu_ctrl_pri_fifo_empty_false_f())
2296                         break;
2297                 retry--;
2298                 usleep_range(20, 40);
2299         } while (retry >= 0);
2300
2301         if (retry < 0)
2302                 nvhost_warn(dev_from_gk20a(g),
2303                         "mmu invalidate too many retries");
2304 }
2305
2306 #if 0 /* VM DEBUG */
2307
2308 /* print pdes/ptes for a gpu virtual address range under a vm */
2309 void gk20a_mm_dump_vm(struct vm_gk20a *vm,
2310                 u64 va_begin, u64 va_end, char *label)
2311 {
2312         struct mem_mgr *client = mem_mgr_from_vm(vm);
2313         struct mm_gk20a *mm = vm->mm;
2314         struct page_table_gk20a *pte_s;
2315         u64 pde_va, pte_va;
2316         u32 pde_i, pde_lo, pde_hi;
2317         u32 pte_i, pte_lo, pte_hi;
2318         u32 pte_space_page_cur, pte_space_offset_cur;
2319         u32 pte_space_page_offset;
2320         u32 num_ptes, page_size;
2321         void *pde, *pte;
2322         phys_addr_t pte_addr;
2323         int err;
2324
2325         pde_range_from_vaddr_range(vm, va_begin, va_end,
2326                         &pde_lo, &pde_hi);
2327
2328         nvhost_err(dev_from_vm(vm),
2329                 "%s page table entries for gpu va 0x%016llx -> 0x%016llx\n",
2330                 label, va_begin, va_end);
2331
2332         for (pde_i = pde_lo; pde_i <= pde_hi; pde_i++) {
2333                 pde = pde_from_index(vm, pde_i);
2334                 pde_va = pde_i * mm->pde_stride;
2335                 nvhost_err(dev_from_vm(vm),
2336                         "\t[0x%016llx -> 0x%016llx] pde @ 0x%08x: 0x%08x, 0x%08x\n",
2337                         pde_va, pde_va + mm->pde_stride - 1,
2338                         sg_phys(vm->pdes.sgt->sgl) + pde_i * gmmu_pde__size_v(),
2339                         mem_rd32(pde, 0), mem_rd32(pde, 1));
2340
2341                 pte_s = vm->pdes.ptes[pte_s->pgsz_idx] + pde_i;
2342
2343                 num_ptes = mm->page_table_sizing[pte_s->pgsz_idx].num_ptes;
2344                 page_size = mm->pde_stride / num_ptes;
2345                 pte_lo = 0;
2346                 pte_hi = num_ptes - 1;
2347
2348                 pte_space_page_offset_from_index(pte_lo,
2349                                                 &pte_space_page_cur,
2350                                                 &pte_space_offset_cur);
2351
2352                 err = map_gmmu_pages(pte_s->ref, pte_s->sgt, &pte);
2353                 pte_s->sgt = nvhost_memmgr_sg_table(client, pte_s->ref);
2354                 if (WARN_ON(IS_ERR(pte_s->sgt)))
2355                         return;
2356                 pte_addr = sg_phys(pte_s->sgt->sgl);
2357
2358                 for (pte_i = pte_lo; pte_i <= pte_hi; pte_i++) {
2359
2360                         pte_va = pde_va + pte_i * page_size;
2361
2362                         if (pte_va < va_begin)
2363                                 continue;
2364                         if (pte_va > va_end)
2365                                 break;
2366
2367                         pte_space_page_offset = pte_i;
2368
2369                         nvhost_err(dev_from_vm(vm),
2370                                 "\t\t[0x%016llx -> 0x%016llx] pte @ 0x%08x : 0x%08x, 0x%08x\n",
2371                                 pte_va, pte_va + page_size - 1,
2372                                 pte_addr + pte_i * gmmu_pte__size_v(),
2373                                 mem_rd32(pte + pte_space_page_offset * 8, 0),
2374                                 mem_rd32(pte + pte_space_page_offset * 8, 1));
2375                 }
2376
2377                 unmap_gmmu_pages(pte_s->ref, pte_s->sgt, pte);
2378         }
2379 }
2380 #endif /* VM DEBUG */
2381
2382 int gk20a_mm_suspend(struct gk20a *g)
2383 {
2384         nvhost_dbg_fn("");
2385
2386         gk20a_mm_fb_flush(g);
2387         gk20a_mm_l2_flush(g, true);
2388
2389         nvhost_dbg_fn("done");
2390         return 0;
2391 }
2392
2393 void gk20a_mm_ltc_isr(struct gk20a *g)
2394 {
2395         u32 intr;
2396
2397         intr = gk20a_readl(g, ltc_ltc0_ltss_intr_r());
2398         nvhost_err(dev_from_gk20a(g), "ltc: %08x\n", intr);
2399         gk20a_writel(g, ltc_ltc0_ltss_intr_r(), intr);
2400 }