video: tegra: bitness sanitary cleanup
[linux-3.10.git] / drivers / video / tegra / host / gk20a / mm_gk20a.c
1 /*
2  * drivers/video/tegra/host/gk20a/mm_gk20a.c
3  *
4  * GK20A memory management
5  *
6  * Copyright (c) 2011, NVIDIA CORPORATION.  All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms and conditions of the GNU General Public License,
10  * version 2, as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope it will be useful, but WITHOUT
13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15  * more details.
16  *
17  * You should have received a copy of the GNU General Public License along with
18  * this program; if not, write to the Free Software Foundation, Inc.,
19  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
20  */
21
22 #include <linux/delay.h>
23 #include <linux/highmem.h>
24 #include <linux/log2.h>
25 #include <linux/nvhost.h>
26 #include <linux/scatterlist.h>
27 #include <linux/nvmap.h>
28 #include <mach/hardware.h>
29 #include <asm/cacheflush.h>
30
31 #include "../../nvmap/nvmap.h"
32 #include "../../nvmap/nvmap_ioctl.h"
33
34 #include "../dev.h"
35 #include "../nvhost_as.h"
36 #include "gk20a.h"
37 #include "mm_gk20a.h"
38 #include "hw_gmmu_gk20a.h"
39 #include "hw_fb_gk20a.h"
40 #include "hw_bus_gk20a.h"
41 #include "hw_ram_gk20a.h"
42 #include "hw_mc_gk20a.h"
43 #include "hw_flush_gk20a.h"
44
45 #include "kind_gk20a.h"
46
47 /* we support 2 page sizes, define indexing based upon that */
48 static inline int gmmu_page_size_idx(u32 ps)
49 {
50         return (ps) != 4096; /* 4K:=0, all else := 1 */
51 }
52 static const u32 gmmu_page_sizes[2] = { 0x1000, 0x20000 }; /* 4KB and 128KB */
53 static const u32 gmmu_page_shift[2] = { 12, 17 };
54 static const u64 gmmu_page_offset_mask[2] = { 0xfffLL, 0x1ffffLL };
55 static const u64 gmmu_page_mask[2] = { ~0xfffLL, ~0x1ffffLL };
56
57 static int gk20a_init_mm_reset_enable_hw(struct gk20a *g)
58 {
59         u32 pmc_enable;
60
61         pmc_enable = gk20a_readl(g, mc_enable_r());
62         pmc_enable &= ~mc_enable_pfb_enabled_f();
63         pmc_enable &= ~mc_enable_l2_enabled_f();
64         pmc_enable &= ~mc_enable_ce2_enabled_f();
65         pmc_enable &= ~mc_enable_xbar_enabled_f();
66         pmc_enable &= ~mc_enable_hub_enabled_f();
67         gk20a_writel(g, mc_enable_r(), pmc_enable);
68
69         pmc_enable = gk20a_readl(g, mc_enable_r());
70         pmc_enable |= mc_enable_pfb_enabled_f();
71         pmc_enable |= mc_enable_l2_enabled_f();
72         pmc_enable |= mc_enable_ce2_enabled_f();
73         pmc_enable |= mc_enable_xbar_enabled_f();
74         pmc_enable |= mc_enable_hub_enabled_f();
75         gk20a_writel(g, mc_enable_r(), pmc_enable);
76         gk20a_readl(g, mc_enable_r());
77
78         nvhost_dbg_fn("done");
79         return 0;
80 }
81
82 void gk20a_remove_mm_support(struct mm_gk20a *mm)
83 {
84         struct gk20a *g = mm->g;
85         struct vm_gk20a *vm = &mm->bar1.vm;
86         struct inst_desc *inst_block = &mm->bar1.inst_block;
87         struct mem_mgr *memmgr = mem_mgr_from_g(g);
88
89         nvhost_dbg_fn("");
90
91         mem_op().unpin(memmgr, inst_block->mem.ref,inst_block->mem.sgt);
92         mem_op().put(memmgr, inst_block->mem.ref);
93
94         vm->remove_support(vm);
95 }
96
97 int gk20a_init_mm_setup_sw(struct gk20a *g)
98 {
99         struct mm_gk20a *mm = &g->mm;
100         int i;
101
102         nvhost_dbg_fn("");
103
104         if (mm->sw_ready) {
105                 nvhost_dbg_fn("skip init");
106                 return 0;
107         }
108
109         mm->g = g;
110         mm->big_page_size = (128 << 10);
111         mm->pde_stride    = mm->big_page_size << 10;
112         mm->pde_stride_shift = ilog2(mm->pde_stride);
113         BUG_ON(mm->pde_stride_shift > 31); /* we have assumptions about this */
114
115         for (i = 0; i < ARRAY_SIZE(gmmu_page_sizes); i++) {
116
117                 u32 num_ptes, pte_space, num_pages;
118
119                 /* assuming "full" page tables */
120                 num_ptes = mm->pde_stride / gmmu_page_sizes[i];
121
122                 pte_space = num_ptes * gmmu_pte__size_v();
123                 /* allocate whole pages */
124                 pte_space = roundup(pte_space, PAGE_SIZE);
125
126                 num_pages = pte_space / PAGE_SIZE;
127                 /* make sure "order" is viable */
128                 BUG_ON(!is_power_of_2(num_pages));
129
130                 mm->page_table_sizing[i].num_ptes = num_ptes;
131                 mm->page_table_sizing[i].order = ilog2(num_pages);
132         }
133
134         /* we only have 256MB of sysmem to play with anyway*/
135         /* TBD: fixme */
136         mm->channel.size = mm->pde_stride * 2;
137         nvhost_dbg_info("channel vm size = 0x%llx", mm->channel.size);
138
139         gk20a_init_bar1_vm(mm);
140
141         gk20a_init_uncompressed_kind_map();
142         gk20a_init_kind_attr();
143
144         mm->remove_support = gk20a_remove_mm_support;
145         mm->sw_ready = true;
146
147         nvhost_dbg_fn("done");
148         return 0;
149 }
150
151 /* make sure gk20a_init_mm_support is called before */
152 static int gk20a_init_mm_setup_hw(struct gk20a *g)
153 {
154         struct mm_gk20a *mm = &g->mm;
155         struct inst_desc *inst_block = &mm->bar1.inst_block;
156         phys_addr_t inst_pa = inst_block->cpu_pa;
157
158         nvhost_dbg_fn("");
159
160         /* set large page size in fb
161          * note this is very early on, can we defer it ? */
162         {
163                 u32 fb_mmu_ctrl = gk20a_readl(g, fb_mmu_ctrl_r());
164                 fb_mmu_ctrl = (fb_mmu_ctrl & ~fb_mmu_ctrl_vm_pg_size_f(~0x0)) |
165                         fb_mmu_ctrl_vm_pg_size_128kb_f();
166                 gk20a_writel(g, fb_mmu_ctrl_r(), fb_mmu_ctrl);
167         }
168
169         inst_pa = (u32)(inst_pa >> bar1_instance_block_shift_gk20a());
170         nvhost_dbg_info("bar1 inst block ptr: 0x%08x",  (u32)inst_pa);
171
172         /* this is very early in init... can we defer this? */
173         {
174                 gk20a_writel(g, bus_bar1_block_r(),
175                              bus_bar1_block_target_vid_mem_f() |
176                              bus_bar1_block_mode_virtual_f() |
177                              bus_bar1_block_ptr_f(inst_pa));
178         }
179
180         nvhost_dbg_fn("done");
181         return 0;
182 }
183
184 int gk20a_init_mm_support(struct gk20a *g)
185 {
186         u32 err;
187
188         err = gk20a_init_mm_reset_enable_hw(g);
189         if (err)
190                 return err;
191
192         err = gk20a_init_mm_setup_sw(g);
193         if (err)
194                 return err;
195
196         err = gk20a_init_mm_setup_hw(g);
197         if (err)
198                 return err;
199
200         return err;
201 }
202
203 #ifdef CONFIG_TEGRA_SIMULATION_SPLIT_MEM
204 static int alloc_gmmu_nvmap_pages(struct vm_gk20a *vm, u32 order, phys_addr_t *pa, void **handle,
205                                   struct sg_table **sgt)
206 {
207         struct mem_mgr *client = mem_mgr_from_vm(vm);
208         struct mem_handle *r;
209         phys_addr_t phys;
210         u32 num_pages = 1 << order;
211         u32 len = num_pages * PAGE_SIZE;
212         u32 *va;
213
214         nvhost_dbg_fn("");
215
216         r = mem_op().alloc(client, len,
217                         DEFAULT_ALLOC_ALIGNMENT,
218                         DEFAULT_ALLOC_FLAGS,
219                         0);
220         if (IS_ERR_OR_NULL(r)) {
221                 nvhost_dbg(dbg_pte, "nvmap_alloc failed\n");
222                 goto err_out;
223         }
224         va = mem_op().mmap(r);
225         if (IS_ERR_OR_NULL(va)) {
226                 nvhost_dbg(dbg_pte, "nvmap_mmap failed\n");
227                 goto err_alloced;
228         }
229         *sgt =  mem_op().pin(client, r);
230         if (IS_ERR_OR_NULL(*sgt)) {
231                 nvhost_dbg(dbg_pte, "nvmap_pin failed\n");
232                 goto err_alloced;
233         }
234         phys = sg_dma_address((*sgt)->sgl);
235         memset(va, 0, len);
236         mem_op().munmap(r, va);
237         *pa = phys;
238         *handle = (void *)r;
239
240         return 0;
241
242 err_alloced:
243         mem_op().put(client, r);
244 err_out:
245         return -ENOMEM;
246 }
247 #endif
248
249 static int alloc_gmmu_sysmem_pages(struct vm_gk20a *vm, u32 order, phys_addr_t *pa,
250                                    void **handle, struct sg_table **sgt)
251 {
252         struct page *pte_page;
253
254         nvhost_dbg_fn("");
255
256         pte_page = alloc_pages(GFP_KERNEL | __GFP_ZERO, order);
257         if (!pte_page)
258                 return -ENOMEM;
259
260         *pa = page_to_phys(pte_page);
261         *sgt = 0;
262         *handle = pte_page;
263         return 0;
264 }
265
266 static int alloc_gmmu_pages(struct vm_gk20a *vm, u32 order, phys_addr_t *pa,
267                             void **handle, struct sg_table **sgt)
268 {
269
270         nvhost_dbg_fn("");
271
272 #ifdef CONFIG_TEGRA_SIMULATION_SPLIT_MEM
273         if (tegra_split_mem_active())
274                 return alloc_gmmu_nvmap_pages(vm, order, pa, handle, sgt);
275         else
276 #endif
277                 return alloc_gmmu_sysmem_pages(vm, order, pa, handle, sgt);
278 }
279
280 static void free_gmmu_pages(struct vm_gk20a *vm, void *handle,
281                             struct sg_table *sgt, u32 order)
282 {
283         nvhost_dbg_fn("");
284
285 #ifdef CONFIG_TEGRA_SIMULATION_SPLIT_MEM
286         if (tegra_split_mem_active()) {
287                 struct mem_mgr *client = mem_mgr_from_vm(vm);
288                 BUG_ON(sgt == NULL);
289                 mem_op().unpin(client, handle, sgt);
290                 mem_op().put(client, handle);
291         } else
292 #endif
293                 __free_pages((struct page *)handle, order);
294 }
295
296 static int map_gmmu_pages(void *handle, void **va)
297 {
298         u32 *tmp_va;
299
300         nvhost_dbg_fn("");
301
302 #ifdef CONFIG_TEGRA_SIMULATION_SPLIT_MEM
303         if (tegra_split_mem_active()) {
304                 struct mem_handle *r = handle;
305
306                 tmp_va = mem_op().mmap(r);
307                 if (IS_ERR_OR_NULL(tmp_va))
308                         goto err_out;
309         } else
310 #endif
311         {
312                 tmp_va = kmap((struct page *)handle);
313                 if (!tmp_va)
314                         goto err_out;
315         }
316         *va = tmp_va;
317         return 0;
318
319 err_out:
320         return -ENOMEM;
321 }
322
323 static void unmap_gmmu_pages(void *handle, u32 *va)
324 {
325
326         nvhost_dbg_fn("");
327
328 #ifdef CONFIG_TEGRA_SIMULATION_SPLIT_MEM
329         if (tegra_split_mem_active()) {
330                 struct mem_handle *r = handle;
331                 mem_op().munmap(r, va);
332         } else
333 #endif
334                 kunmap((struct page *)handle);
335 }
336
337 static int update_gmmu_ptes(struct vm_gk20a *vm, u32 page_size_idx,
338                         u64 bfr_addr, u64 first_vaddr, u64 last_vaddr,
339                         u8 kind_v, u32 ctag_offset, bool cachable);
340
341 /* allocate a phys contig region big enough for a full
342  * sized gmmu page table for the given gmmu_page_size.
343  * the whole range is zeroed so it's "invalid"/will fault
344  */
345
346 static int zalloc_gmmu_page_table_gk20a(struct vm_gk20a *vm,
347                                        u32 gmmu_page_size,
348                                        struct page_table_gk20a *pte,
349                                        phys_addr_t *pa)
350 {
351         int err, page_size_idx;
352         u32 pte_order;
353         phys_addr_t phys;
354         void *handle;
355         struct sg_table *sgt;
356
357         nvhost_dbg_fn("");
358
359         /* allocate enough pages for the table */
360         page_size_idx = gmmu_page_size_idx(gmmu_page_size);
361         pte_order = vm->mm->page_table_sizing[page_size_idx].order;
362
363         err = alloc_gmmu_pages(vm, pte_order, &phys, &handle, &sgt);
364         if (err)
365                 return err;
366
367         nvhost_dbg(dbg_pte, "pte = 0x%p, phys = 0x%x", pte, (unsigned int)phys);
368
369         *pa = phys;
370         pte->ref = handle;
371         pte->sgt = sgt;
372         pte->page_size_idx = page_size_idx;
373
374         return 0;
375 }
376
377 /* given address range (inclusive) determine the pdes crossed */
378 static inline void pde_range_from_vaddr_range(struct vm_gk20a *vm,
379                                               u64 addr_lo, u64 addr_hi,
380                                               u32 *pde_lo, u32 *pde_hi)
381 {
382         *pde_lo = (u32)(addr_lo >> vm->mm->pde_stride_shift);
383         *pde_hi = (u32)(addr_hi >> vm->mm->pde_stride_shift);
384         nvhost_dbg(dbg_pte, "addr_lo=0x%llx addr_hi=0x%llx pde_ss=%d",
385                    addr_lo, addr_hi, vm->mm->pde_stride_shift);
386         nvhost_dbg(dbg_pte, "pde_lo=%d pde_hi=%d",
387                    *pde_lo, *pde_hi);
388 }
389
390 static inline u32 *pde_from_index(struct vm_gk20a *vm, u32 i)
391 {
392         return (u32 *) (((u8 *)vm->pdes.kv) + i*gmmu_pde__size_v());
393 }
394
395 static inline u32 pte_index_from_vaddr(struct vm_gk20a *vm,
396                                        u64 addr, u32 page_size_idx)
397 {
398         static const u32 page_shift[2] = {12, 17};
399         u32 ret;
400         /* mask off pde part */
401         addr = addr & ((((u64)1) << vm->mm->pde_stride_shift) - ((u64)1));
402         /* shift over to get pte index. note assumption that pte index
403          * doesn't leak over into the high 32b */
404         ret = (u32)(addr >> page_shift[page_size_idx]);
405
406         nvhost_dbg(dbg_pte, "addr=0x%llx pte_i=0x%x", addr, ret);
407         return ret;
408 }
409
410 static inline void pte_space_page_offset_from_index(u32 i, u32 *pte_page,
411                                                     u32 *pte_offset)
412 {
413         /* ptes are 8B regardless of pagesize */
414         /* pte space pages are 4KB. so 512 ptes per 4KB page*/
415         *pte_page = i >> 9;
416
417         /* this offset is a pte offset, not a byte offset */
418         *pte_offset = i & ((1<<9)-1);
419
420         nvhost_dbg(dbg_pte, "i=0x%x pte_page=0x%x pte_offset=0x%x",
421                    i, *pte_page, *pte_offset);
422 }
423
424 /*
425  * given a pde index/page table number make sure it has
426  * backing store and if not go ahead allocate it and
427  * record it in the appropriate pde
428  */
429
430 static int validate_gmmu_page_table_gk20a(struct vm_gk20a *vm,
431                                           u32 i,
432                                           u32 gmmu_page_size)
433 {
434         u32 err;
435         phys_addr_t pte_addr;
436         phys_addr_t dbg_addr;
437         struct page_table_gk20a *pte = vm->pdes.ptes + i;
438         u32 *pde;
439
440         nvhost_dbg_fn("");
441         nvhost_dbg(dbg_pte, "i = %d", i);
442
443         if (unlikely(gmmu_page_size != 4096 &&
444                      gmmu_page_size != 128*1024))
445                 return -EINVAL;
446
447         /* if it's already in place it's valid */
448         /* tbd: unless it's set to the other pagesize... */
449         if (pte->ref)
450                 return 0;
451
452         err = zalloc_gmmu_page_table_gk20a(vm, gmmu_page_size, pte, &pte_addr);
453         if (err)
454                 return err;
455
456         dbg_addr = (u32)pte_addr;
457         pte_addr >>= gmmu_pde_address_shift_v();
458         pde = pde_from_index(vm, i);
459
460         if (gmmu_page_size == 4096) {
461                 /* "small" gmmu page size */
462                 nvhost_dbg(dbg_pte, "4KB ptes @ 0x%llx", (u64)dbg_addr);
463                 mem_wr32(pde, 0, (gmmu_pde_size_full_f() |
464                                   gmmu_pde_aperture_big_invalid_f()));
465                 mem_wr32(pde, 1, (gmmu_pde_aperture_small_video_memory_f() |
466                                   gmmu_pde_vol_small_true_f() |
467                                   gmmu_pde_address_small_sys_f((u32)pte_addr)));
468         } else {
469                 /* "large" gmmu page size */
470                 nvhost_dbg(dbg_pte, "128KB ptes @ 0x%llx", (u64)dbg_addr);
471                 mem_wr32(pde, 0, (gmmu_pde_size_full_f() |
472                                   gmmu_pde_aperture_big_video_memory_f() |
473                                   gmmu_pde_address_big_sys_f((u32)pte_addr)));
474                 mem_wr32(pde, 1, (gmmu_pde_aperture_small_invalid_f() |
475                                   gmmu_pde_vol_big_true_f()));
476         }
477
478         vm->pdes.dirty = true;
479
480         smp_mb();
481         __cpuc_flush_dcache_area(pde, sizeof(u32) * 2);
482         gk20a_mm_tlb_invalidate(vm->mm->g, vm);
483
484         return 0;
485 }
486
487 static u64 gk20a_vm_alloc_va(struct vm_gk20a *vm,
488                         u64 size,
489                         u32 gmmu_page_size)
490 {
491         struct nvhost_allocator *vma = &vm->vma;
492         int err;
493         u64 offset;
494         u32 gmmu_page_size_shift = ilog2(gmmu_page_size);
495         u32 start_page_nr = 0, num_pages;
496         u32 i, pde_lo, pde_hi;
497
498         /* be certain we round up to gmmu_page_size if needed */
499         size = (size + ((u64)gmmu_page_size - 1)) & ~((u64)gmmu_page_size - 1);
500
501         nvhost_dbg_info("size=0x%llx", size);
502
503         num_pages = size >> gmmu_page_size_shift;
504         err = vma->alloc(vma, &start_page_nr, num_pages);
505
506         offset = (u64)start_page_nr << gmmu_page_size_shift;
507         if (err) {
508                 nvhost_err(dev_from_vm(vm),
509                            "oom: sz=0x%llx", size);
510                 return 0;
511         }
512
513         pde_range_from_vaddr_range(vm,
514                                    offset, offset + size - 1,
515                                    &pde_lo, &pde_hi);
516
517         for (i = pde_lo; i <= pde_hi; i++) {
518
519                 err = validate_gmmu_page_table_gk20a(vm, i, gmmu_page_size);
520
521                 /* mark the pages valid, with correct phys addr */
522                 if (err) {
523                         nvhost_err(dev_from_vm(vm),
524                                    "failed to validate page table %d: %d",
525                                    i, err);
526                         return 0;
527                 }
528         }
529
530         nvhost_dbg_fn("ret=0x%llx", offset);
531
532         return offset;
533 }
534
535 static void gk20a_vm_free_va(struct vm_gk20a *vm, u64 offset,
536                                      u64 size, u32 gmmu_page_size)
537 {
538         struct nvhost_allocator *vma = &vm->vma;
539         u32 gmmu_page_size_shift = ilog2(gmmu_page_size);
540         u32 start_page_nr = 0, num_pages;
541         int err;
542
543         nvhost_dbg_info("offset=0x%llx, size=0x%llx", offset, size);
544
545         start_page_nr = (u32)(offset >> gmmu_page_size_shift);
546         num_pages = (u32)((size + gmmu_page_size - 1) >> gmmu_page_size_shift);
547
548         err = vma->free(vma, start_page_nr, num_pages);
549         if (err) {
550                 nvhost_err(dev_from_vm(vm),
551                            "oom: offset=0x%llx, sz=0x%llx",
552                            offset, size);
553         }
554 }
555
556 static int insert_mapped_buffer(struct rb_root *root,
557                                 struct mapped_buffer_node *mapped_buffer)
558 {
559         struct rb_node **new_node = &(root->rb_node), *parent = NULL;
560
561         /* Figure out where to put new node */
562         while (*new_node) {
563                 struct mapped_buffer_node *cmp_with =
564                         container_of(*new_node, struct mapped_buffer_node, node);
565
566                 parent = *new_node;
567
568                 if (cmp_with->addr > mapped_buffer->addr) /* u64 cmp */
569                         new_node = &((*new_node)->rb_left);
570                 else if (cmp_with->addr != mapped_buffer->addr) /* u64 cmp */
571                         new_node = &((*new_node)->rb_right);
572                 else
573                         return -EINVAL; /* no fair dup'ing */
574         }
575
576         /* Add new node and rebalance tree. */
577         rb_link_node(&mapped_buffer->node, parent, new_node);
578         rb_insert_color(&mapped_buffer->node, root);
579
580         return 0;
581 }
582
583 static struct mapped_buffer_node *find_mapped_buffer(struct rb_root *root,
584                                                      u64 addr)
585 {
586
587         struct rb_node *node = root->rb_node;
588         while (node) {
589                 struct mapped_buffer_node *mapped_buffer =
590                         container_of(node, struct mapped_buffer_node, node);
591                 if (mapped_buffer->addr > addr) /* u64 cmp */
592                         node = node->rb_left;
593                 else if (mapped_buffer->addr != addr) /* u64 cmp */
594                         node = node->rb_right;
595                 else
596                         return mapped_buffer;
597         }
598         return 0;
599 }
600
601 /* convenience setup for nvmap buffer attr queries */
602 struct bfr_attr_query {
603         int err;
604         u32 v;
605 };
606 static u32 nvmap_bfr_param[] = {
607 #define BFR_SIZE   0
608         NVMAP_HANDLE_PARAM_SIZE,
609 #define BFR_ALIGN  1
610         NVMAP_HANDLE_PARAM_ALIGNMENT,
611 #define BFR_HEAP   2
612         NVMAP_HANDLE_PARAM_HEAP,
613 #define BFR_KIND   3
614         NVMAP_HANDLE_PARAM_KIND,
615 };
616 #define BFR_ATTRS (sizeof(nvmap_bfr_param)/sizeof(nvmap_bfr_param[0]))
617
618 struct buffer_attrs {
619         phys_addr_t addr;
620         u64 size;
621         u64 align;
622         u32 ctag_offset;
623         u32 ctag_lines;
624         int contig;
625         int iovmm_mapped;
626         int page_size_idx; /* largest gmmu page size which fits the buffer's attributes */
627         u8 kind_v;
628         u8 uc_kind_v;
629 };
630
631 static int setup_buffer_size_and_align(struct mem_mgr *memmgr,
632                                        struct mem_handle *hdl,
633                                        struct buffer_attrs *bfr,
634                                        struct bfr_attr_query *query)
635 {
636         int i;
637         /* buffer allocation size and alignment must be a multiple
638            of one of the supported page sizes.*/
639         bfr->size = query[BFR_SIZE].v;
640         bfr->align = query[BFR_ALIGN].v;
641         bfr->page_size_idx = -1;
642
643         for (i = 1; i >= 0; i--) /*  choose the biggest first (top->bottom) */
644                 if (!(gmmu_page_offset_mask[i] & bfr->align)) {
645                         /* would like to add this too but nvmap returns the
646                          * original requested size not the allocated size.
647                          * (!(gmmu_page_offset_mask[i] & bfr->size)) */
648                         bfr->page_size_idx = i;
649                         break;
650                 }
651
652         if (unlikely(0 > bfr->page_size_idx)) {
653                 nvhost_err(0, "unsupported nvmap buffer allocation "
654                            "alignment, size: 0x%llx, 0x%llx\n",
655                            bfr->align, bfr->size);
656                 return -EINVAL;
657         }
658         switch (query[BFR_HEAP].v) {
659         case NVMAP_HEAP_SYSMEM:
660                 /* sysmem, contig
661                  * Fall through to carveout...
662                  * TBD: Need nvmap support for scattered sysmem allocs
663                  * w/o mapping through smmu.
664                  */
665
666         case NVMAP_HEAP_CARVEOUT_GENERIC:
667                 /* carveout sysmem, contig */
668                 bfr->contig = 1;
669                 bfr->iovmm_mapped = 0;
670                 break;
671
672         case NVMAP_HEAP_IOVMM:
673                 /* sysmem, iovmm/smmu mapped */
674                 bfr->contig = 1;
675                 bfr->iovmm_mapped = 1;
676                 break;
677         default:
678                 nvhost_err(0, "unsupported nvmap buffer heap: 0x%x\n",
679                            query[BFR_HEAP].v);
680                 return -EINVAL;
681         }
682
683         bfr->kind_v = query[BFR_KIND].v;
684
685         return 0;
686 }
687
688
689 static int setup_buffer_kind_and_compression(u32 flags,
690                                              u32 kind,
691                                              struct buffer_attrs *bfr,
692                                              u32 gmmu_page_size)
693 {
694         bool kind_compressible;
695
696         /* This flag (which comes from map_buffer ioctl) is for override now.
697            It will be removed when all clients which use it have been
698            changed to specify kind in the nvmap buffer alloc. */
699         if (flags & NVHOST_MAP_BUFFER_FLAGS_KIND_SPECIFIED)
700                 bfr->kind_v = kind;
701
702         if (unlikely(bfr->kind_v == gmmu_pte_kind_invalid_v()))
703                 bfr->kind_v = gmmu_pte_kind_pitch_v();
704
705         if (unlikely(!gk20a_kind_is_supported(bfr->kind_v))) {
706                 nvhost_err(0, "kind 0x%x not supported", bfr->kind_v);
707                 return EINVAL;
708         }
709
710         bfr->uc_kind_v = gmmu_pte_kind_invalid_v();
711         /* find a suitable uncompressed kind if it becomes necessary later */
712         kind_compressible = gk20a_kind_is_compressible(bfr->kind_v);
713         if (kind_compressible) {
714                 bfr->uc_kind_v = gk20a_get_uncompressed_kind(bfr->kind_v);
715                 if (unlikely(bfr->uc_kind_v == gmmu_pte_kind_invalid_v())) {
716                         /* shouldn't happen, but it is worth cross-checking */
717                         nvhost_err(0, "comptag kind 0x%x can't be"
718                                    " downgraded to uncompressed kind",
719                                    bfr->kind_v);
720                         return -EINVAL;
721                 }
722         }
723         /* comptags only supported for suitable kinds, 128KB pagesize */
724         if (unlikely(kind_compressible && (gmmu_page_size != 128*1024))) {
725                 nvhost_warn(0, "comptags specified"
726                             " but pagesize being used doesn't support it");
727                 /* it is safe to fall back to uncompressed as
728                    functionality is not harmed */
729                 bfr->kind_v = bfr->uc_kind_v;
730                 kind_compressible = false;
731         }
732         if (kind_compressible)
733                 bfr->ctag_lines = bfr->size >> COMP_TAG_LINE_SIZE_SHIFT;
734         else
735                 bfr->ctag_lines = 0;
736
737         return 0;
738 }
739
740
741 static u64 gk20a_vm_map(struct vm_gk20a *vm,
742                         struct mem_mgr *memmgr,
743                         struct mem_handle *r,
744                         u64 offset_align,
745                         u32 flags /*NVHOST_MAP_BUFFER_FLAGS_*/,
746                         u32 kind)
747 {
748         struct gk20a *g = gk20a_from_vm(vm);
749         struct nvhost_allocator *ctag_allocator = &g->gr.comp_tags;
750         struct device *d = &g->dev->dev;
751         struct mapped_buffer_node *mapped_buffer = 0;
752         struct sg_table *sgt;
753         bool inserted = false, va_allocated = false;
754         u32 gmmu_page_size = 0;
755         u64 map_offset = 0;
756         int attr, err = 0;
757         struct buffer_attrs bfr = {0};
758         struct bfr_attr_query query[BFR_ATTRS];
759
760         /* query bfr attributes: size, align, heap, kind */
761         for (attr = 0; attr < BFR_ATTRS; attr++) {
762                 query[attr].err =
763                         mem_op().get_param(memmgr, r,
764                                         nvmap_bfr_param[attr],
765                                         &query[attr].v);
766                 if (unlikely(query[attr].err != 0)) {
767                         nvhost_err(d,
768                                    "failed to get nvmap buffer param %d: %d\n",
769                                    nvmap_bfr_param[attr],
770                                    query[attr].err);
771                         return query[attr].err;
772                 }
773         }
774
775         /* validate/adjust bfr attributes */
776
777         err = setup_buffer_size_and_align(memmgr, r, &bfr, query);
778         if (unlikely(err))
779                 goto clean_up;
780
781         /* if specified the map offset needs to be gmmu page aligned */
782         if (flags & NVHOST_MAP_BUFFER_FLAGS_OFFSET) {
783                 map_offset = offset_align;
784                 if (map_offset & gmmu_page_offset_mask[bfr.page_size_idx]) {
785                         nvhost_err(d, "unsupported buffer map offset 0x%llx",
786                                    map_offset);
787                         err = -EINVAL;
788                         goto clean_up;
789                 }
790         }
791
792         gmmu_page_size = gmmu_page_sizes[bfr.page_size_idx];
793
794         err = setup_buffer_kind_and_compression(flags, kind,
795                                                 &bfr, gmmu_page_size);
796         if (unlikely(err)) {
797                 nvhost_err(d, "failure setting up kind and compression");
798                 goto clean_up;
799         }
800
801         /* bar1 and pmu vm don't need ctag */
802         if (!vm->enable_ctag)
803                 bfr.ctag_lines = 0;
804
805         /* allocate compression resources if needed */
806         if (bfr.ctag_lines) {
807                 err = ctag_allocator->alloc(ctag_allocator, &bfr.ctag_offset,
808                                             bfr.ctag_lines);
809                 /* ok to fall back here if we ran out */
810                 /* TBD: we can partially alloc ctags as well... */
811                 if (err) {
812                         bfr.ctag_lines = bfr.ctag_offset = 0;
813                         bfr.kind_v = bfr.uc_kind_v;
814                 }
815         }
816
817         /* init/clear the ctag buffer */
818         if (bfr.ctag_lines)
819                 gk20a_gr_clear_comptags(g,
820                                         bfr.ctag_offset,
821                                         bfr.ctag_offset + bfr.ctag_lines - 1);
822
823
824         /* TBD: need to get nvmap to assign the correct asid */
825         /* until then there's no point even trying */
826         if (bfr.iovmm_mapped) {
827                 nvhost_err(d, "iovmm remapping is unsupported at this time");
828                 err = -EINVAL;
829                 goto clean_up;
830         }
831
832         /* Allocate (or validate when map_offset != 0) the virtual address. */
833         if (!map_offset) {
834                 map_offset = vm->alloc_va(vm, bfr.size,
835                                           gmmu_page_size);
836                 if (!map_offset) {
837                         nvhost_err(d, "failed to allocate va space");
838                         err = -ENOMEM;
839                         goto clean_up;
840                 }
841                 va_allocated = true;
842         } else {
843                 /* TODO: validate that the offset has been previosuly allocated ||
844                  * allocate here to keep track */
845                 nvhost_warn(d, "fixed offset mapping isn't safe yet!");
846                 nvhost_warn(d, "other mappings may collide!");
847         }
848
849         /* pin buffer to get phys/iovmm addr */
850         sgt = mem_op().pin(memmgr, r);
851         if (IS_ERR_OR_NULL(sgt)) {
852                 nvhost_warn(d, "oom allocating tracking buffer");
853                 goto clean_up;
854         }
855         bfr.addr = sg_dma_address(sgt->sgl);
856
857         nvhost_dbg_info("nvmap pinned buffer @ 0x%llx", (u64)bfr.addr);
858         nvhost_dbg_fn("r=%p, map_offset=0x%llx, contig=%d "
859                       "iovmm_mapped=%d kind=0x%x kind_uc=0x%x flags=0x%x",
860                       r, map_offset, bfr.contig, bfr.iovmm_mapped,
861                       bfr.kind_v, bfr.uc_kind_v, flags);
862         nvhost_dbg_info("comptag size=%d start=%d for 0x%llx",
863                         bfr.ctag_lines, bfr.ctag_offset, (u64)bfr.addr);
864
865         /* keep track of the buffer for unmapping */
866         /* TBD: check for multiple mapping of same buffer */
867         mapped_buffer = kzalloc(sizeof(*mapped_buffer), GFP_KERNEL);
868         if (!mapped_buffer) {
869                 nvhost_warn(d, "oom allocating tracking buffer");
870                 goto clean_up;
871         }
872         mapped_buffer->memmgr     = memmgr;
873         mapped_buffer->handle_ref = r;
874         mapped_buffer->sgt        = sgt;
875         mapped_buffer->addr       = map_offset;
876         mapped_buffer->size       = bfr.size;
877         mapped_buffer->page_size  = gmmu_page_size;
878         mapped_buffer->ctag_offset = bfr.ctag_offset;
879         mapped_buffer->ctag_lines  = bfr.ctag_lines;
880
881         err = insert_mapped_buffer(&vm->mapped_buffers, mapped_buffer);
882         if (err) {
883                 nvhost_err(d, "failed to insert into mapped buffer tree");
884                 goto clean_up;
885         }
886         inserted = true;
887
888         nvhost_dbg_info("allocated va @ 0x%llx", map_offset);
889
890         err = update_gmmu_ptes(vm, bfr.page_size_idx,
891                                bfr.addr,
892                                map_offset, map_offset + bfr.size - 1,
893                                bfr.kind_v,
894                                bfr.ctag_offset,
895                                flags & NVHOST_MAP_BUFFER_FLAGS_CACHEABLE_TRUE);
896         if (err) {
897                 nvhost_err(d, "failed to update ptes on map");
898                 goto clean_up;
899         }
900
901         return map_offset;
902
903 clean_up:
904         if (inserted)
905                 rb_erase(&mapped_buffer->node, &vm->mapped_buffers);
906         kfree(mapped_buffer);
907         if (va_allocated)
908                 vm->free_va(vm, map_offset, bfr.size, gmmu_page_size);
909         if (bfr.ctag_lines)
910                 ctag_allocator->free(ctag_allocator,
911                                      bfr.ctag_offset,
912                                      bfr.ctag_lines);
913
914         nvhost_dbg_info("err=%d\n", err);
915         return 0;
916 }
917
918 static int update_gmmu_ptes(struct vm_gk20a *vm, u32 page_size_idx,
919                        u64 bfr_addr, u64 first_vaddr, u64 last_vaddr,
920                        u8 kind_v, u32 ctag_offset, bool cachable)
921 {
922         int err;
923         u32 pde_lo, pde_hi, pde_i, cur_page;
924         u32 pte_w[2] = {0, 0}; /* invalid pte */
925         u32 ctag = ctag_offset;
926         u32 ctag_ptes, ctag_pte_cnt;
927
928         pde_range_from_vaddr_range(vm, first_vaddr, last_vaddr,
929                                    &pde_lo, &pde_hi);
930
931         nvhost_dbg(dbg_pte, "pde_lo=%d, pde_hi=%d", pde_lo, pde_hi);
932
933         if (gmmu_page_sizes[page_size_idx] == 4096) {
934                 cur_page = (u32)(bfr_addr >> 12);
935                 ctag_ptes = COMP_TAG_LINE_SIZE >> 12;
936         } else {
937                 cur_page = (u32)(bfr_addr >> 17);
938                 ctag_ptes = COMP_TAG_LINE_SIZE >> 17;
939         }
940
941         ctag_pte_cnt = 0;
942
943         for (pde_i = pde_lo; pde_i <= pde_hi; pde_i++) {
944                 u32 pte_lo, pte_hi;
945                 u32 pte_cur;
946                 u32 pte_space_page_cur, pte_space_offset_cur;
947                 u32 pte_space_page, pte_space_page_offset;
948                 u32 *pde;
949                 void *pte_kv_cur;
950
951                 u32 pte_pfn0 = 0;               /* Keep compiler quiet */
952
953                 struct page_table_gk20a *pte = vm->pdes.ptes + pde_i;
954
955                 if (pde_i == pde_lo)
956                         pte_lo = pte_index_from_vaddr(vm, first_vaddr,
957                                                       page_size_idx);
958                 else
959                         pte_lo = 0;
960
961                 if ((pde_i != pde_hi) && (pde_hi != pde_lo))
962                         pte_hi = vm->mm->page_table_sizing[page_size_idx].num_ptes - 1;
963                 else
964                         pte_hi = pte_index_from_vaddr(vm, last_vaddr,
965                                                       page_size_idx);
966
967                 /* need to worry about crossing pages when accessing the ptes */
968                 pte_space_page_offset_from_index(pte_lo, &pte_space_page_cur,
969                                                  &pte_space_offset_cur);
970 #ifdef CONFIG_TEGRA_SIMULATION_SPLIT_MEM
971                 if (tegra_split_mem_active()) {
972                         err = map_gmmu_pages(pte->ref, &pte_kv_cur);
973                 } else
974 #endif
975                 {
976                         pte_pfn0 = page_to_pfn((struct page *)pte->ref);
977                         err = map_gmmu_pages((void *)pfn_to_page(pte_pfn0 +
978                                                                  pte_space_page_cur),
979                                                                  &pte_kv_cur);
980                 }
981
982                 if (err) {
983                         nvhost_err(dev_from_vm(vm),
984                                    "couldn't map ptes for update");
985                         goto clean_up;
986                 }
987
988                 nvhost_dbg(dbg_pte, "pte_lo=%d, pte_hi=%d", pte_lo, pte_hi);
989                 for (pte_cur = pte_lo; pte_cur <= pte_hi; pte_cur++) {
990
991 #ifdef CONFIG_TEGRA_SIMULATION_SPLIT_MEM
992                         if (tegra_split_mem_active()) {
993                                 pte_space_page_offset = pte_cur;
994                         } else
995 #endif
996                         {
997                                 pte_space_page_offset_from_index(pte_cur,
998                                                                 &pte_space_page,
999                                                                 &pte_space_page_offset);
1000                                 if (unlikely(pte_space_page != pte_space_page_cur)) {
1001                                         __cpuc_flush_dcache_area(pte_kv_cur,
1002                                                                  PAGE_SIZE);
1003                                         unmap_gmmu_pages(pfn_to_page(pte_pfn0 +
1004                                                          pte_space_page_cur),
1005                                                          pte_kv_cur);
1006                                         pte_space_page_cur = pte_space_page;
1007                                         err = map_gmmu_pages(pfn_to_page(pte_pfn0 +
1008                                                              pte_space_page_cur),
1009                                                              &pte_kv_cur);
1010                                         if (err) {
1011                                                 nvhost_err(dev_from_vm(vm),
1012                                                            "couldn't map ptes for update");
1013                                                 goto clean_up;
1014                                         }
1015                                 }
1016                         }
1017                         if (ctag) {
1018                                 ctag_pte_cnt++;
1019                                 if (ctag_pte_cnt > ctag_ptes) {
1020                                         ctag++;
1021                                         ctag_pte_cnt = 0;
1022                                 }
1023                         }
1024
1025                         nvhost_dbg(dbg_pte, "pte_cur=%d cur_page=0x%x kind=%d ctag=%d",
1026                                 pte_cur, cur_page, kind_v, ctag);
1027
1028                         if (likely(bfr_addr != 0)) {
1029                                 pte_w[0] = gmmu_pte_valid_true_f() |
1030                                         gmmu_pte_address_sys_f(cur_page);
1031                                 pte_w[1] = gmmu_pte_aperture_video_memory_f() |
1032                                         gmmu_pte_kind_f(kind_v) |
1033                                         gmmu_pte_comptagline_f(ctag);
1034                                 if (!cachable)
1035                                         pte_w[1] |= gmmu_pte_vol_true_f();
1036                                 cur_page++;
1037                                 pte->ref_cnt++;
1038                         } else
1039                                 pte->ref_cnt--;
1040
1041                         nvhost_dbg(dbg_pte, "pte[0]=0x%x pte[1]=0x%x", pte_w[0], pte_w[1]);
1042                         mem_wr32(pte_kv_cur + pte_space_page_offset*8, 0,
1043                                  pte_w[0]);
1044                         mem_wr32(pte_kv_cur + pte_space_page_offset*8, 1,
1045                                  pte_w[1]);
1046                 }
1047
1048                 __cpuc_flush_dcache_area(pte_kv_cur, PAGE_SIZE);
1049
1050 #ifdef CONFIG_TEGRA_SIMULATION_SPLIT_MEM
1051                 if (tegra_split_mem_active())
1052                         unmap_gmmu_pages(pte->ref, pte_kv_cur);
1053                 else
1054 #endif
1055                         unmap_gmmu_pages(pfn_to_page(pte_pfn0 + pte_space_page_cur),
1056                                          pte_kv_cur);
1057
1058                 if (pte->ref_cnt == 0) {
1059                         /* invalidate pde */
1060                         pde = pde_from_index(vm, pde_i);
1061                         mem_wr32(pde, 0, gmmu_pde_aperture_big_invalid_f());
1062                         mem_wr32(pde, 1, gmmu_pde_aperture_small_invalid_f());
1063                         vm->pdes.dirty = true;
1064                         /* free page table */
1065                         free_gmmu_pages(vm, pte->ref, pte->sgt,
1066                                         vm->mm->page_table_sizing[page_size_idx].order);
1067                         pte->ref = NULL;
1068                 }
1069         }
1070
1071         smp_mb();
1072         gk20a_mm_tlb_invalidate(vm->mm->g, vm);
1073
1074         return 0;
1075
1076 clean_up:
1077         /*TBD: potentially rewrite above to pre-map everything it needs to
1078          * as that's the only way it can fail */
1079         return err;
1080
1081 }
1082
1083 /* return mem_mgr and mem_handle to caller. If the mem_handle is a kernel dup
1084    from user space (as_ioctl), caller releases the kernel duplicated handle */
1085 static void gk20a_vm_unmap_user(struct vm_gk20a *vm, u64 offset,
1086                         struct mem_mgr **memmgr, struct mem_handle **r)
1087 {
1088         struct mapped_buffer_node *mapped_buffer;
1089         struct gk20a *g = gk20a_from_vm(vm);
1090         struct nvhost_allocator *comp_tags = &g->gr.comp_tags;
1091         int err = 0;
1092
1093         BUG_ON(memmgr == NULL || r == NULL);
1094
1095         *memmgr = NULL;
1096         *r = NULL;
1097
1098         nvhost_dbg_fn("offset=0x%llx", offset);
1099
1100         mapped_buffer = find_mapped_buffer(&vm->mapped_buffers, offset);
1101         if (!mapped_buffer) {
1102                 nvhost_dbg(dbg_err, "invalid addr to unmap 0x%llx", offset);
1103                 return;
1104         }
1105
1106         vm->free_va(vm, mapped_buffer->addr, mapped_buffer->size,
1107                     mapped_buffer->page_size);
1108
1109         if (mapped_buffer->ctag_offset)
1110                 comp_tags->free(comp_tags,
1111                         mapped_buffer->ctag_offset, mapped_buffer->ctag_lines);
1112
1113         err = update_gmmu_ptes(vm,
1114                                gmmu_page_size_idx(mapped_buffer->page_size),
1115                                0, /* n/a for unmap */
1116                                mapped_buffer->addr,
1117                                mapped_buffer->addr + mapped_buffer->size - 1,
1118                                0, 0, false /* n/a for unmap */);
1119         if (err) {
1120                 nvhost_dbg(dbg_err, "failed to update ptes on unmap");
1121         }
1122
1123         mem_op().unpin(mapped_buffer->memmgr,
1124                        mapped_buffer->handle_ref,
1125                        mapped_buffer->sgt);
1126
1127         /* remove from mapped buffer tree, free */
1128         rb_erase(&mapped_buffer->node, &vm->mapped_buffers);
1129
1130         *memmgr = mapped_buffer->memmgr;
1131         *r = mapped_buffer->handle_ref;
1132         kfree(mapped_buffer);
1133
1134         return;
1135 }
1136
1137 /* called by kernel, mem_mgr and mem_handle are ignored */
1138 static void gk20a_vm_unmap(struct vm_gk20a *vm, u64 offset)
1139 {
1140         struct mem_mgr *memmgr;
1141         struct mem_handle *r;
1142
1143         gk20a_vm_unmap_user(vm, offset, &memmgr, &r);
1144 }
1145
1146 void gk20a_vm_remove_support(struct vm_gk20a *vm)
1147 {
1148         struct mapped_buffer_node *mapped_buffer;
1149         struct rb_node *node;
1150         struct mem_mgr *memmgr;
1151         struct mem_handle *r;
1152
1153         nvhost_dbg_fn("");
1154
1155         node = rb_first(&vm->mapped_buffers);
1156         while (node) {
1157                 mapped_buffer =
1158                         container_of(node, struct mapped_buffer_node, node);
1159                 vm->unmap_user(vm, mapped_buffer->addr, &memmgr, &r);
1160                 if (memmgr != mem_mgr_from_vm(vm)) {
1161                         mem_op().put(memmgr, r);
1162                         mem_op().put_mgr(memmgr);
1163                 }
1164                 node = rb_first(&vm->mapped_buffers);
1165         }
1166
1167         unmap_gmmu_pages(vm->pdes.ref, vm->pdes.kv);
1168         free_gmmu_pages(vm, vm->pdes.ref, vm->pdes.sgt, 0);
1169         kfree(vm->pdes.ptes);
1170         nvhost_allocator_destroy(&vm->vma);
1171 }
1172
1173 /* address space interfaces for the gk20a module */
1174 static int gk20a_as_alloc_share(struct nvhost_as_share *as_share)
1175 {
1176         struct nvhost_as *as = as_share->as;
1177         struct gk20a *gk20a = get_gk20a(as->ch->dev);
1178         struct mm_gk20a *mm = &gk20a->mm;
1179         struct vm_gk20a *vm;
1180         char name[32];
1181         int err;
1182
1183         nvhost_dbg_fn("");
1184
1185         vm = kzalloc(sizeof(*vm), GFP_KERNEL);
1186         if (!vm)
1187                 return -ENOMEM;
1188
1189         as_share->priv = (void *)vm;
1190
1191         vm->mm = mm;
1192         vm->as_share = as_share;
1193
1194         vm->va_start  = 0; /* we have a one page hole though so zeros fault*/
1195         vm->va_limit  = mm->channel.size;
1196
1197         {
1198                 u32 pde_lo, pde_hi;
1199                 pde_range_from_vaddr_range(vm,
1200                                            0, vm->va_limit-1,
1201                                            &pde_lo, &pde_hi);
1202                 vm->pdes.num_pdes = pde_hi + 1;
1203         }
1204
1205         vm->pdes.ptes = kzalloc(sizeof(struct page_table_gk20a) *
1206                                 vm->pdes.num_pdes, GFP_KERNEL);
1207         if (!vm->pdes.ptes)
1208                 return -ENOMEM;
1209
1210         nvhost_dbg_info("init space for va_limit=0x%llx num_pdes=%d",
1211                    vm->va_limit, vm->pdes.num_pdes);
1212
1213         /* allocate the page table directory */
1214         err = alloc_gmmu_pages(vm, 0, &vm->pdes.phys, &vm->pdes.ref, &vm->pdes.sgt);
1215         if (err) {
1216                 return -ENOMEM;
1217         }
1218         err = map_gmmu_pages(vm->pdes.ref, &vm->pdes.kv);
1219         if (err) {
1220                 free_gmmu_pages(vm, vm->pdes.ref, vm->pdes.sgt, 0);
1221                 return -ENOMEM;
1222         }
1223         nvhost_dbg(dbg_pte, "pdes.kv = 0x%p, pdes.phys = 0x%llx",
1224                         vm->pdes.kv, (u64)vm->pdes.phys);
1225         /* we could release vm->pdes.kv but it's only one page... */
1226
1227         /* alloc in 4K granularity */
1228         snprintf(name, sizeof(name), "gk20a_as_%d", as_share->id);
1229         nvhost_allocator_init(&vm->vma, name,
1230                 1, mm->channel.size >> 12, 1);
1231
1232         vm->mapped_buffers = RB_ROOT;
1233
1234         vm->alloc_va = gk20a_vm_alloc_va;
1235         vm->free_va = gk20a_vm_free_va;
1236         vm->map = gk20a_vm_map;
1237         vm->unmap = gk20a_vm_unmap;
1238         vm->unmap_user = gk20a_vm_unmap_user;
1239         vm->remove_support = gk20a_vm_remove_support;
1240
1241         vm->enable_ctag = true;
1242
1243         return 0;
1244 }
1245
1246
1247 static int gk20a_as_release_share(struct nvhost_as_share *as_share)
1248 {
1249         struct vm_gk20a *vm = (struct vm_gk20a *)as_share->priv;
1250
1251         nvhost_dbg_fn("");
1252
1253         gk20a_vm_remove_support(vm);
1254
1255         as_share->priv = NULL;
1256         kfree(vm);
1257
1258         return 0;
1259 }
1260
1261 static int gk20a_as_alloc_space(struct nvhost_as_share *as_share,
1262                                 struct nvhost_as_alloc_space_args *args)
1263
1264 {       int err = -ENOMEM;
1265         /*struct vm_gk20a *vm = (struct vm_gk20a *)as_share->priv;*/
1266
1267         nvhost_dbg_fn("");
1268         return err;
1269 }
1270
1271 static int gk20a_as_free_space(struct nvhost_as_share *as_share,
1272                                struct nvhost_as_free_space_args *args)
1273 {
1274         int err = -ENOMEM;
1275         /*struct vm_gk20a *vm = (struct vm_gk20a *)as_share->priv;*/
1276         nvhost_dbg_fn("");
1277
1278         return err;
1279
1280 }
1281
1282 static int gk20a_as_bind_hwctx(struct nvhost_as_share *as_share,
1283                                struct nvhost_hwctx *hwctx)
1284 {
1285         int err = 0;
1286         struct vm_gk20a *vm = (struct vm_gk20a *)as_share->priv;
1287         struct channel_gk20a *c = hwctx->priv;
1288
1289         nvhost_dbg_fn("");
1290
1291         c->vm = vm;
1292         err = channel_gk20a_commit_va(c);
1293         if (err)
1294                 c->vm = 0;
1295
1296         return err;
1297 }
1298
1299 static int gk20a_as_map_buffer(struct nvhost_as_share *as_share,
1300                                struct mem_mgr *nvmap,
1301                                struct mem_handle *r,
1302                                u64 *offset_align,
1303                                u32 flags /*NVHOST_AS_MAP_BUFFER_FLAGS_*/)
1304 {
1305         int err = 0;
1306         struct vm_gk20a *vm = (struct vm_gk20a *)as_share->priv;
1307         u64 ret_va;
1308
1309         nvhost_dbg_fn("");
1310
1311         ret_va = vm->map(vm, nvmap, r, *offset_align, flags, 0/*no kind here, to be removed*/);
1312         *offset_align = ret_va;
1313         if (!ret_va)
1314                 err = -EINVAL;
1315
1316         return err;
1317
1318 }
1319
1320 static int gk20a_as_unmap_buffer(struct nvhost_as_share *as_share, u64 offset,
1321                                  struct mem_mgr **memmgr, struct mem_handle **r)
1322 {
1323         int err = 0;
1324         struct vm_gk20a *vm = (struct vm_gk20a *)as_share->priv;
1325
1326         nvhost_dbg_fn("");
1327
1328         vm->unmap_user(vm, offset, memmgr, r);
1329
1330         return err;
1331 }
1332
1333
1334 const struct nvhost_as_moduleops gk20a_as_moduleops = {
1335         .alloc_share   = gk20a_as_alloc_share,
1336         .release_share = gk20a_as_release_share,
1337         .alloc_space   = gk20a_as_alloc_space,
1338         .free_space    = gk20a_as_free_space,
1339         .bind_hwctx    = gk20a_as_bind_hwctx,
1340         .map_buffer    = gk20a_as_map_buffer,
1341         .unmap_buffer  = gk20a_as_unmap_buffer,
1342 };
1343
1344 int gk20a_init_bar1_vm(struct mm_gk20a *mm)
1345 {
1346         int err;
1347         struct mem_mgr *nvmap = mem_mgr_from_mm(mm);
1348         phys_addr_t inst_pa;
1349         void *inst_ptr;
1350         struct vm_gk20a *vm = &mm->bar1.vm;
1351         struct inst_desc *inst_block = &mm->bar1.inst_block;
1352         u32 pde_addr_lo;
1353         u32 pde_addr_hi;
1354
1355         vm->mm = mm;
1356
1357         mm->bar1.aperture_size = bar1_aperture_size_mb_gk20a() << 20;
1358
1359         nvhost_dbg_info("bar1 vm size = 0x%x", mm->bar1.aperture_size);
1360
1361         vm->va_start  = mm->pde_stride * 1;
1362         vm->va_limit  = mm->bar1.aperture_size;
1363
1364         {
1365                 u32 pde_lo, pde_hi;
1366                 pde_range_from_vaddr_range(vm,
1367                                            0, vm->va_limit-1,
1368                                            &pde_lo, &pde_hi);
1369                 vm->pdes.num_pdes = pde_hi + 1;
1370         }
1371
1372         vm->pdes.ptes = kzalloc(sizeof(struct page_table_gk20a) *
1373                                 vm->pdes.num_pdes, GFP_KERNEL);
1374         if (!vm->pdes.ptes)
1375                 return -ENOMEM;
1376
1377         nvhost_dbg_info("init space for bar1 va_limit=0x%llx num_pdes=%d",
1378                    vm->va_limit, vm->pdes.num_pdes);
1379
1380
1381         /* allocate the page table directory */
1382         err = alloc_gmmu_pages(vm, 0, &vm->pdes.phys, &vm->pdes.ref, &vm->pdes.sgt);
1383         if (err)
1384                 goto clean_up;
1385
1386         err = map_gmmu_pages(vm->pdes.ref, &vm->pdes.kv);
1387         if (err) {
1388                 free_gmmu_pages(vm, vm->pdes.ref, vm->pdes.sgt, 0);
1389                 goto clean_up;
1390         }
1391         nvhost_dbg(dbg_pte, "bar 1 pdes.kv = 0x%p, pdes.phys = 0x%llx",
1392                         vm->pdes.kv, (u64)vm->pdes.phys);
1393         /* we could release vm->pdes.kv but it's only one page... */
1394
1395         pde_addr_lo = u64_lo32(vm->pdes.phys) >> 12;
1396         pde_addr_hi = u64_hi32(vm->pdes.phys);
1397
1398         nvhost_dbg_info("pde pa=0x%llx pde_addr_lo=0x%x pde_addr_hi=0x%x",
1399                 (u64)vm->pdes.phys, pde_addr_lo, pde_addr_hi);
1400
1401         /* allocate instance mem for bar1 */
1402         inst_block->mem.size = ram_in_alloc_size_v();
1403         inst_block->mem.ref =
1404                 mem_op().alloc(nvmap, inst_block->mem.size,
1405                             DEFAULT_ALLOC_ALIGNMENT,
1406                             DEFAULT_ALLOC_FLAGS,
1407                             0);
1408
1409         if (IS_ERR(inst_block->mem.ref)) {
1410                 inst_block->mem.ref = 0;
1411                 err = -ENOMEM;
1412                 goto clean_up;
1413         }
1414
1415         inst_block->mem.sgt = mem_op().pin(nvmap, inst_block->mem.ref);
1416         /* IS_ERR throws a warning here (expecting void *) */
1417         if (IS_ERR_OR_NULL(inst_block->mem.sgt)) {
1418                 inst_pa = 0;
1419                 err = (int)inst_pa;
1420                 goto clean_up;
1421         }
1422         inst_block->cpu_pa = inst_pa = sg_dma_address(inst_block->mem.sgt->sgl);
1423
1424         inst_ptr = mem_op().mmap(inst_block->mem.ref);
1425         if (IS_ERR(inst_ptr)) {
1426                 return -ENOMEM;
1427                 goto clean_up;
1428         }
1429
1430         nvhost_dbg_info("bar1 inst block physical phys = 0x%llx, kv = 0x%p",
1431                 (u64)inst_pa, inst_ptr);
1432
1433         memset(inst_ptr, 0, ram_fc_size_val_v());
1434
1435         mem_wr32(inst_ptr, ram_in_page_dir_base_lo_w(),
1436                 ram_in_page_dir_base_target_vid_mem_f() |
1437                 ram_in_page_dir_base_vol_true_f() |
1438                 ram_in_page_dir_base_lo_f(pde_addr_lo));
1439
1440         mem_wr32(inst_ptr, ram_in_page_dir_base_hi_w(),
1441                 ram_in_page_dir_base_hi_f(pde_addr_hi));
1442
1443         mem_wr32(inst_ptr, ram_in_adr_limit_lo_w(),
1444                  u64_lo32(vm->va_limit) | 0xFFF);
1445
1446         mem_wr32(inst_ptr, ram_in_adr_limit_hi_w(),
1447                 ram_in_adr_limit_hi_f(u64_hi32(vm->va_limit)));
1448
1449         mem_op().munmap(inst_block->mem.ref, inst_ptr);
1450
1451         nvhost_dbg_info("bar1 inst block ptr: %08x",  (u32)inst_pa);
1452         nvhost_allocator_init(&vm->vma, "gk20a_bar1",
1453                 1, (vm->va_limit >> 12) - 1, 1);
1454
1455         vm->mapped_buffers = RB_ROOT;
1456
1457         vm->alloc_va = gk20a_vm_alloc_va;
1458         vm->free_va = gk20a_vm_free_va;
1459         vm->map = gk20a_vm_map;
1460         vm->unmap = gk20a_vm_unmap;
1461         vm->unmap_user = gk20a_vm_unmap_user;
1462         vm->remove_support = gk20a_vm_remove_support;
1463
1464         return 0;
1465
1466 clean_up:
1467         /* free, etc */
1468         return err;
1469 }
1470
1471 /* pmu vm, share channel_vm interfaces */
1472 int gk20a_init_pmu_vm(struct mm_gk20a *mm)
1473 {
1474         int err;
1475         struct mem_mgr *nvmap = mem_mgr_from_mm(mm);
1476         phys_addr_t inst_pa;
1477         void *inst_ptr;
1478         struct vm_gk20a *vm = &mm->pmu.vm;
1479         struct inst_desc *inst_block = &mm->pmu.inst_block;
1480         u32 pde_addr_lo;
1481         u32 pde_addr_hi;
1482
1483         vm->mm = mm;
1484
1485         mm->pmu.aperture_size = GK20A_PMU_VA_SIZE;
1486
1487         nvhost_dbg_info("pmu vm size = 0x%x", mm->pmu.aperture_size);
1488
1489         vm->va_start  = GK20A_PMU_VA_START;
1490         vm->va_limit  = vm->va_start + mm->pmu.aperture_size;
1491
1492         {
1493                 u32 pde_lo, pde_hi;
1494                 pde_range_from_vaddr_range(vm,
1495                                            0, vm->va_limit-1,
1496                                            &pde_lo, &pde_hi);
1497                 vm->pdes.num_pdes = pde_hi + 1;
1498         }
1499
1500         vm->pdes.ptes = kzalloc(sizeof(struct page_table_gk20a) *
1501                                 vm->pdes.num_pdes, GFP_KERNEL);
1502         if (!vm->pdes.ptes)
1503                 return -ENOMEM;
1504
1505         nvhost_dbg_info("init space for pmu va_limit=0x%llx num_pdes=%d",
1506                    vm->va_limit, vm->pdes.num_pdes);
1507
1508         /* allocate the page table directory */
1509         err = alloc_gmmu_pages(vm, 0, &vm->pdes.phys, &vm->pdes.ref, &vm->pdes.sgt);
1510         if (err)
1511                 goto clean_up;
1512
1513         err = map_gmmu_pages(vm->pdes.ref, &vm->pdes.kv);
1514         if (err) {
1515                 free_gmmu_pages(vm, vm->pdes.ref, vm->pdes.sgt, 0);
1516                 goto clean_up;
1517         }
1518         nvhost_dbg_info("pmu pdes phys @ 0x%llx", (u64)vm->pdes.phys);
1519         /* we could release vm->pdes.kv but it's only one page... */
1520
1521         pde_addr_lo = u64_lo32(vm->pdes.phys) >> 12;
1522         pde_addr_hi = u64_hi32(vm->pdes.phys);
1523
1524         nvhost_dbg_info("pde pa=0x%llx pde_addr_lo=0x%x pde_addr_hi=0x%x",
1525                         (u64)vm->pdes.phys, pde_addr_lo, pde_addr_hi);
1526
1527         /* allocate instance mem for pmu */
1528         inst_block->mem.size = GK20A_PMU_INST_SIZE;
1529         inst_block->mem.ref =
1530                 mem_op().alloc(nvmap, inst_block->mem.size,
1531                             DEFAULT_ALLOC_ALIGNMENT,
1532                             DEFAULT_ALLOC_FLAGS,
1533                             0);
1534
1535         if (IS_ERR(inst_block->mem.ref)) {
1536                 inst_block->mem.ref = 0;
1537                 err = -ENOMEM;
1538                 goto clean_up;
1539         }
1540
1541         inst_block->mem.sgt = mem_op().pin(nvmap, inst_block->mem.ref);
1542         /* IS_ERR throws a warning here (expecting void *) */
1543         if (IS_ERR_OR_NULL(inst_block->mem.sgt)) {
1544                 inst_pa = 0;
1545                 err = (int)((uintptr_t)inst_block->mem.sgt);
1546                 goto clean_up;
1547         }
1548         inst_block->cpu_pa = inst_pa =
1549                 sg_dma_address(inst_block->mem.sgt->sgl);
1550
1551         nvhost_dbg_info("pmu inst block physical addr: 0x%llx", (u64)inst_pa);
1552
1553         inst_ptr = mem_op().mmap(inst_block->mem.ref);
1554         if (IS_ERR(inst_ptr)) {
1555                 return -ENOMEM;
1556                 goto clean_up;
1557         }
1558
1559         memset(inst_ptr, 0, GK20A_PMU_INST_SIZE);
1560
1561         mem_wr32(inst_ptr, ram_in_page_dir_base_lo_w(),
1562                 ram_in_page_dir_base_target_vid_mem_f() |
1563                 ram_in_page_dir_base_vol_true_f() |
1564                 ram_in_page_dir_base_lo_f(pde_addr_lo));
1565
1566         mem_wr32(inst_ptr, ram_in_page_dir_base_hi_w(),
1567                 ram_in_page_dir_base_hi_f(pde_addr_hi));
1568
1569         mem_wr32(inst_ptr, ram_in_adr_limit_lo_w(),
1570                  u64_lo32(vm->va_limit) | 0xFFF);
1571
1572         mem_wr32(inst_ptr, ram_in_adr_limit_hi_w(),
1573                 ram_in_adr_limit_hi_f(u64_hi32(vm->va_limit)));
1574
1575         mem_op().munmap(inst_block->mem.ref, inst_ptr);
1576
1577         nvhost_allocator_init(&vm->vma, "gk20a_pmu",
1578                 (vm->va_start >> 12), (vm->va_limit >> 12) - 1, 1);
1579
1580         vm->mapped_buffers = RB_ROOT;
1581
1582         vm->alloc_va        = gk20a_vm_alloc_va;
1583         vm->free_va         = gk20a_vm_free_va;
1584         vm->map             = gk20a_vm_map;
1585         vm->unmap           = gk20a_vm_unmap;
1586         vm->unmap_user      = gk20a_vm_unmap_user;
1587         vm->remove_support  = gk20a_vm_remove_support;
1588
1589         return 0;
1590
1591 clean_up:
1592         /* free, etc */
1593         return err;
1594 }
1595
1596 void gk20a_mm_fb_flush(struct gk20a *g)
1597 {
1598         u32 data;
1599         s32 retry = 100;
1600
1601         /* Make sure all previous writes are committed to the L2. There's no
1602            guarantee that writes are to DRAM. This will be a sysmembar internal
1603            to the L2. */
1604         gk20a_writel(g, flush_fb_flush_r(),
1605                 flush_fb_flush_pending_busy_f());
1606
1607         do {
1608                 data = gk20a_readl(g, flush_fb_flush_r());
1609
1610                 if (flush_fb_flush_outstanding_v(data) ==
1611                         flush_fb_flush_outstanding_true_v() ||
1612                     flush_fb_flush_pending_v(data) ==
1613                         flush_fb_flush_pending_busy_v()) {
1614                                 nvhost_dbg_info("fb_flush 0x%x", data);
1615                                 retry--;
1616                                 udelay(20);
1617                 } else
1618                         break;
1619         }
1620         while (retry >= 0);
1621
1622         if (retry < 0)
1623                 nvhost_warn(dev_from_gk20a(g),
1624                         "fb_flush too many retries");
1625 }
1626
1627 void gk20a_mm_l2_flush(struct gk20a *g, bool invalidate)
1628 {
1629         u32 data;
1630         s32 retry = 200;
1631
1632         /* Flush all dirty lines from the L2 to DRAM. Lines are left in the L2
1633            as clean, so subsequent reads might hit in the L2. */
1634         gk20a_writel(g, flush_l2_flush_dirty_r(),
1635                 flush_l2_flush_dirty_pending_busy_f());
1636
1637         do {
1638                 data = gk20a_readl(g, flush_l2_flush_dirty_r());
1639
1640                 if (flush_l2_flush_dirty_outstanding_v(data) ==
1641                         flush_l2_flush_dirty_outstanding_true_v() ||
1642                     flush_l2_flush_dirty_pending_v(data) ==
1643                         flush_l2_flush_dirty_pending_busy_v()) {
1644                                 nvhost_dbg_info("l2_flush_dirty 0x%x", data);
1645                                 retry--;
1646                                 udelay(20);
1647                 } else
1648                         break;
1649         }
1650         while (retry >= 0);
1651
1652         if (retry < 0)
1653                 nvhost_warn(dev_from_gk20a(g),
1654                         "l2_flush_dirty too many retries");
1655
1656         if (!invalidate)
1657                 return;
1658
1659         /* Invalidate any clean lines from the L2 so subsequent reads go to
1660            DRAM. Dirty lines are not affected by this operation. */
1661         gk20a_writel(g, flush_l2_system_invalidate_r(),
1662                 flush_l2_system_invalidate_pending_busy_f());
1663
1664         do {
1665                 data = gk20a_readl(g, flush_l2_system_invalidate_r());
1666
1667                 if (flush_l2_system_invalidate_outstanding_v(data) ==
1668                         flush_l2_system_invalidate_outstanding_true_v() ||
1669                     flush_l2_system_invalidate_pending_v(data) ==
1670                         flush_l2_system_invalidate_pending_busy_v()) {
1671                                 nvhost_dbg_info("l2_system_invalidate 0x%x", data);
1672                                 retry--;
1673                                 udelay(20);
1674                 } else
1675                         break;
1676         }
1677         while (retry >= 0);
1678
1679         if (retry < 0)
1680                 nvhost_warn(dev_from_gk20a(g),
1681                         "l2_system_invalidate too many retries");
1682 }
1683
1684 void gk20a_mm_tlb_invalidate(struct gk20a *g, struct vm_gk20a *vm)
1685 {
1686         u32 addr_lo = u64_lo32(vm->pdes.phys) >> 12;
1687         u32 data;
1688         s32 retry = 200;
1689
1690         /* pagetables are considered sw states which are preserved after
1691            prepare_poweroff. When gk20a deinit releases those pagetables,
1692            common code in vm unmap path calls tlb invalidate that touches
1693            hw. Use the power_on flag to skip tlb invalidation when gpu
1694            power is turned off */
1695         if (!g->power_on)
1696                 return;
1697
1698         do {
1699                 data = gk20a_readl(g, fb_mmu_ctrl_r());
1700                 if (fb_mmu_ctrl_pri_fifo_space_v(data) != 0)
1701                         break;
1702                 udelay(20);
1703                 retry--;
1704         } while (retry >= 0);
1705
1706         if (retry < 0)
1707                 nvhost_warn(dev_from_gk20a(g),
1708                         "wait mmu fifo space too many retries");
1709
1710         gk20a_writel(g, fb_mmu_invalidate_pdb_r(),
1711                 fb_mmu_invalidate_pdb_addr_f(addr_lo) |
1712                 fb_mmu_invalidate_pdb_aperture_vid_mem_f());
1713
1714         gk20a_writel(g, fb_mmu_invalidate_r(),
1715                 fb_mmu_invalidate_all_pdb_true_f() |
1716                 fb_mmu_invalidate_all_va_true_f() |
1717                 fb_mmu_invalidate_trigger_true_f());
1718
1719         do {
1720                 data = gk20a_readl(g, fb_mmu_ctrl_r());
1721                 if (fb_mmu_ctrl_pri_fifo_empty_v(data) !=
1722                         fb_mmu_ctrl_pri_fifo_empty_false_f())
1723                         break;
1724                 retry--;
1725                 udelay(20);
1726         } while (retry >= 0);
1727
1728         if (retry < 0)
1729                 nvhost_warn(dev_from_gk20a(g),
1730                         "mmu invalidate too many retries");
1731 }
1732
1733 #if 0 /* VM DEBUG */
1734
1735 /* print pdes/ptes for a gpu virtual address range under a vm */
1736 void gk20a_mm_dump_vm(struct vm_gk20a *vm,
1737                 u64 va_begin, u64 va_end, char *label)
1738 {
1739         struct mem_mgr *client = mem_mgr_from_vm(vm);
1740         struct mm_gk20a *mm = vm->mm;
1741         struct page_table_gk20a *pte_s;
1742         u64 pde_va, pte_va;
1743         u32 pde_i, pde_lo, pde_hi;
1744         u32 pte_i, pte_lo, pte_hi, pte_pfn0;
1745         u32 pte_space_page_cur, pte_space_offset_cur;
1746         u32 pte_space_page, pte_space_page_offset;
1747         u32 num_ptes, page_size;
1748         void *pde, *pte;
1749         phys_addr_t pte_addr;
1750         int err;
1751
1752         pde_range_from_vaddr_range(vm, va_begin, va_end,
1753                         &pde_lo, &pde_hi);
1754
1755         nvhost_err(dev_from_vm(vm),
1756                 "%s page table entries for gpu va 0x%016llx -> 0x%016llx\n",
1757                 label, va_begin, va_end);
1758
1759         for (pde_i = pde_lo; pde_i <= pde_hi; pde_i++) {
1760                 pde = pde_from_index(vm, pde_i);
1761                 pde_va = pde_i * mm->pde_stride;
1762                 nvhost_err(dev_from_vm(vm),
1763                         "\t[0x%016llx -> 0x%016llx] pde @ 0x%08x: 0x%08x, 0x%08x\n",
1764                         pde_va, pde_va + mm->pde_stride - 1,
1765                         vm->pdes.phys + pde_i * gmmu_pde__size_v(),
1766                         mem_rd32(pde, 0), mem_rd32(pde, 1));
1767
1768                 pte_s = vm->pdes.ptes + pde_i;
1769
1770                 num_ptes = mm->page_table_sizing[pte_s->page_size_idx].num_ptes;
1771                 page_size = mm->pde_stride / num_ptes;
1772                 pte_lo = 0;
1773                 pte_hi = num_ptes - 1;
1774
1775                 pte_space_page_offset_from_index(pte_lo,
1776                                                 &pte_space_page_cur,
1777                                                 &pte_space_offset_cur);
1778
1779                 pte_pfn0 = 0;
1780 #ifdef CONFIG_TEGRA_SIMULATION_SPLIT_MEM
1781                 if (tegra_split_mem_active()) {
1782                         err = map_gmmu_pages(pte_s->ref, &pte);
1783                         pte_s->sgt = mem_op().pin(client, pte_s->ref);
1784                         pte_addr = sg_dma_address(pte_s->sgt->sgl);
1785                 } else
1786 #endif
1787                 {
1788                         pte_pfn0 = page_to_pfn((struct page *)pte_s->ref);
1789                         err = map_gmmu_pages((void *)pfn_to_page(pte_pfn0 +
1790                                         pte_space_page_cur),
1791                                         &pte);
1792                         pte_addr = __pfn_to_phys(pte_pfn0);
1793                 }
1794
1795                 for (pte_i = pte_lo; pte_i <= pte_hi; pte_i++) {
1796
1797                         pte_va = pde_va + pte_i * page_size;
1798
1799                         if (pte_va < va_begin)
1800                                 continue;
1801                         if (pte_va > va_end)
1802                                 break;
1803
1804 #ifdef CONFIG_TEGRA_SIMULATION_SPLIT_MEM
1805                         if (tegra_split_mem_active()) {
1806                                 pte_space_page_offset = pte_i;
1807                         } else
1808 #endif
1809                         {
1810                                 pte_space_page_offset_from_index(pte_i,
1811                                                 &pte_space_page,
1812                                                 &pte_space_page_offset);
1813                                 if (unlikely(pte_space_page !=
1814                                                 pte_space_page_cur)) {
1815                                         unmap_gmmu_pages(pfn_to_page(pte_pfn0 +
1816                                                          pte_space_page_cur),
1817                                                          pte);
1818                                         pte_space_page_cur = pte_space_page;
1819                                         err = map_gmmu_pages(
1820                                                         pfn_to_page(pte_pfn0 +
1821                                                         pte_space_page_cur),
1822                                                         &pte);
1823                                         if (err) {
1824                                                 nvhost_err(dev_from_vm(vm),
1825                                                 "couldn't map pde %d, pte %d",
1826                                                         pde_i, pte_i);
1827                                                 continue;
1828                                         }
1829                                 }
1830                         }
1831
1832                         nvhost_err(dev_from_vm(vm),
1833                                 "\t\t[0x%016llx -> 0x%016llx] pte @ 0x%08x : 0x%08x, 0x%08x\n",
1834                                 pte_va, pte_va + page_size - 1,
1835                                 pte_addr + pte_i * gmmu_pte__size_v(),
1836                                 mem_rd32(pte + pte_space_page_offset * 8, 0),
1837                                 mem_rd32(pte + pte_space_page_offset * 8, 1));
1838                 }
1839
1840 #ifdef CONFIG_TEGRA_SIMULATION_SPLIT_MEM
1841                 if (tegra_split_mem_active()) {
1842                         unmap_gmmu_pages(pte_s->ref, pte);
1843                         mem_op().unpin(client, pte_s->ref, pte_s->sgt);
1844                 } else
1845 #endif
1846                         unmap_gmmu_pages(
1847                                 pfn_to_page(pte_pfn0 + pte_space_page_cur),
1848                                 pte);
1849         }
1850 }
1851 #endif /* VM DEBUG */
1852
1853 int gk20a_mm_suspend(struct gk20a *g)
1854 {
1855         nvhost_dbg_fn("");
1856
1857         gk20a_mm_fb_flush(g);
1858         gk20a_mm_l2_flush(g, true);
1859
1860         nvhost_dbg_fn("done");
1861         return 0;
1862 }