9de589f9d6e2942ca5e9640f1b545475d93c5f77
[linux-3.10.git] / drivers / gpu / nvgpu / gk20a / mm_gk20a.c
1 /*
2  * drivers/video/tegra/host/gk20a/mm_gk20a.c
3  *
4  * GK20A memory management
5  *
6  * Copyright (c) 2011-2014, NVIDIA CORPORATION.  All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms and conditions of the GNU General Public License,
10  * version 2, as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope it will be useful, but WITHOUT
13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15  * more details.
16  *
17  * You should have received a copy of the GNU General Public License along with
18  * this program; if not, write to the Free Software Foundation, Inc.,
19  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
20  */
21
22 #include <linux/delay.h>
23 #include <linux/highmem.h>
24 #include <linux/log2.h>
25 #include <linux/nvhost.h>
26 #include <linux/pm_runtime.h>
27 #include <linux/scatterlist.h>
28 #include <linux/nvmap.h>
29 #include <linux/tegra-soc.h>
30 #include <linux/vmalloc.h>
31 #include <linux/dma-buf.h>
32 #include <asm/cacheflush.h>
33
34 #include "gk20a.h"
35 #include "mm_gk20a.h"
36 #include "hw_gmmu_gk20a.h"
37 #include "hw_fb_gk20a.h"
38 #include "hw_bus_gk20a.h"
39 #include "hw_ram_gk20a.h"
40 #include "hw_mc_gk20a.h"
41 #include "hw_flush_gk20a.h"
42 #include "hw_ltc_gk20a.h"
43
44 #include "kind_gk20a.h"
45
46 #ifdef CONFIG_ARM64
47 #define outer_flush_range(a, b)
48 #define __cpuc_flush_dcache_area __flush_dcache_area
49 #endif
50
51 /*
52  * GPU mapping life cycle
53  * ======================
54  *
55  * Kernel mappings
56  * ---------------
57  *
58  * Kernel mappings are created through vm.map(..., false):
59  *
60  *  - Mappings to the same allocations are reused and refcounted.
61  *  - This path does not support deferred unmapping (i.e. kernel must wait for
62  *    all hw operations on the buffer to complete before unmapping).
63  *  - References to dmabuf are owned and managed by the (kernel) clients of
64  *    the gk20a_vm layer.
65  *
66  *
67  * User space mappings
68  * -------------------
69  *
70  * User space mappings are created through as.map_buffer -> vm.map(..., true):
71  *
72  *  - Mappings to the same allocations are reused and refcounted.
73  *  - This path supports deferred unmapping (i.e. we delay the actual unmapping
74  *    until all hw operations have completed).
75  *  - References to dmabuf are owned and managed by the vm_gk20a
76  *    layer itself. vm.map acquires these refs, and sets
77  *    mapped_buffer->own_mem_ref to record that we must release the refs when we
78  *    actually unmap.
79  *
80  */
81
82 static inline int vm_aspace_id(struct vm_gk20a *vm)
83 {
84         /* -1 is bar1 or pmu, etc. */
85         return vm->as_share ? vm->as_share->id : -1;
86 }
87 static inline u32 hi32(u64 f)
88 {
89         return (u32)(f >> 32);
90 }
91 static inline u32 lo32(u64 f)
92 {
93         return (u32)(f & 0xffffffff);
94 }
95
96 #define FLUSH_CPU_DCACHE(va, pa, size)  \
97         do {    \
98                 __cpuc_flush_dcache_area((void *)(va), (size_t)(size)); \
99                 outer_flush_range(pa, pa + (size_t)(size));             \
100         } while (0)
101
102 static void gk20a_vm_unmap_locked(struct mapped_buffer_node *mapped_buffer);
103 static struct mapped_buffer_node *find_mapped_buffer_locked(
104                                         struct rb_root *root, u64 addr);
105 static struct mapped_buffer_node *find_mapped_buffer_reverse_locked(
106                                 struct rb_root *root, struct dma_buf *dmabuf,
107                                 u32 kind);
108 static int update_gmmu_ptes_locked(struct vm_gk20a *vm,
109                                    enum gmmu_pgsz_gk20a pgsz_idx,
110                                    struct sg_table *sgt,
111                                    u64 first_vaddr, u64 last_vaddr,
112                                    u8 kind_v, u32 ctag_offset, bool cacheable,
113                                    int rw_flag);
114 static void update_gmmu_pde_locked(struct vm_gk20a *vm, u32 i);
115 static void gk20a_vm_remove_support(struct vm_gk20a *vm);
116
117
118 /* note: keep the page sizes sorted lowest to highest here */
119 static const u32 gmmu_page_sizes[gmmu_nr_page_sizes] = { SZ_4K, SZ_128K };
120 static const u32 gmmu_page_shifts[gmmu_nr_page_sizes] = { 12, 17 };
121 static const u64 gmmu_page_offset_masks[gmmu_nr_page_sizes] = { 0xfffLL,
122                                                                 0x1ffffLL };
123 static const u64 gmmu_page_masks[gmmu_nr_page_sizes] = { ~0xfffLL, ~0x1ffffLL };
124
125 struct gk20a_comptags {
126         u32 offset;
127         u32 lines;
128 };
129
130 struct gk20a_dmabuf_priv {
131         struct mutex lock;
132
133         struct gk20a_allocator *comptag_allocator;
134         struct gk20a_comptags comptags;
135
136         struct dma_buf_attachment *attach;
137         struct sg_table *sgt;
138
139         int pin_count;
140 };
141
142 static void gk20a_mm_delete_priv(void *_priv)
143 {
144         struct gk20a_dmabuf_priv *priv = _priv;
145         if (!priv)
146                 return;
147
148         if (priv->comptags.lines) {
149                 BUG_ON(!priv->comptag_allocator);
150                 priv->comptag_allocator->free(priv->comptag_allocator,
151                                               priv->comptags.offset,
152                                               priv->comptags.lines);
153         }
154
155         kfree(priv);
156 }
157
158 struct sg_table *gk20a_mm_pin(struct device *dev, struct dma_buf *dmabuf)
159 {
160         struct gk20a_dmabuf_priv *priv;
161
162         priv = dma_buf_get_drvdata(dmabuf, dev);
163         if (WARN_ON(!priv))
164                 return ERR_PTR(-EINVAL);
165
166         mutex_lock(&priv->lock);
167
168         if (priv->pin_count == 0) {
169                 priv->attach = dma_buf_attach(dmabuf, dev);
170                 if (IS_ERR(priv->attach)) {
171                         mutex_unlock(&priv->lock);
172                         return (struct sg_table *)priv->attach;
173                 }
174
175                 priv->sgt = dma_buf_map_attachment(priv->attach,
176                                                    DMA_BIDIRECTIONAL);
177                 if (IS_ERR(priv->sgt)) {
178                         dma_buf_detach(dmabuf, priv->attach);
179                         mutex_unlock(&priv->lock);
180                         return priv->sgt;
181                 }
182         }
183
184         priv->pin_count++;
185         mutex_unlock(&priv->lock);
186         return priv->sgt;
187 }
188
189 void gk20a_mm_unpin(struct device *dev, struct dma_buf *dmabuf,
190                     struct sg_table *sgt)
191 {
192         struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev);
193         dma_addr_t dma_addr;
194
195         if (IS_ERR(priv) || !priv)
196                 return;
197
198         mutex_lock(&priv->lock);
199         WARN_ON(priv->sgt != sgt);
200         priv->pin_count--;
201         WARN_ON(priv->pin_count < 0);
202         dma_addr = sg_dma_address(priv->sgt->sgl);
203         if (priv->pin_count == 0) {
204                 dma_buf_unmap_attachment(priv->attach, priv->sgt,
205                                          DMA_BIDIRECTIONAL);
206                 dma_buf_detach(dmabuf, priv->attach);
207         }
208         mutex_unlock(&priv->lock);
209 }
210
211
212 static void gk20a_get_comptags(struct device *dev,
213                                struct dma_buf *dmabuf,
214                                struct gk20a_comptags *comptags)
215 {
216         struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev);
217
218         if (!comptags)
219                 return;
220
221         if (!priv) {
222                 comptags->lines = 0;
223                 comptags->offset = 0;
224                 return;
225         }
226
227         *comptags = priv->comptags;
228 }
229
230 static int gk20a_alloc_comptags(struct device *dev,
231                                 struct dma_buf *dmabuf,
232                                 struct gk20a_allocator *allocator,
233                                 int lines)
234 {
235         struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev);
236         u32 offset = 0;
237         int err;
238
239         if (!priv)
240                 return -ENOSYS;
241
242         if (!lines)
243                 return -EINVAL;
244
245         /* store the allocator so we can use it when we free the ctags */
246         priv->comptag_allocator = allocator;
247         err = allocator->alloc(allocator, &offset, lines);
248         if (!err) {
249                 priv->comptags.lines = lines;
250                 priv->comptags.offset = offset;
251         }
252         return err;
253 }
254
255
256
257
258 static int gk20a_init_mm_reset_enable_hw(struct gk20a *g)
259 {
260         gk20a_dbg_fn("");
261         if (g->ops.fb.reset)
262                 g->ops.fb.reset(g);
263
264         if (g->ops.fb.init_fs_state)
265                 g->ops.fb.init_fs_state(g);
266
267         return 0;
268 }
269
270 void gk20a_remove_mm_support(struct mm_gk20a *mm)
271 {
272         struct gk20a *g = mm->g;
273         struct device *d = dev_from_gk20a(g);
274         struct vm_gk20a *vm = &mm->bar1.vm;
275         struct inst_desc *inst_block = &mm->bar1.inst_block;
276
277         gk20a_dbg_fn("");
278
279         if (inst_block->cpuva)
280                 dma_free_coherent(d, inst_block->size,
281                         inst_block->cpuva, inst_block->iova);
282         inst_block->cpuva = NULL;
283         inst_block->iova = 0;
284
285         gk20a_vm_remove_support(vm);
286 }
287
288 int gk20a_init_mm_setup_sw(struct gk20a *g)
289 {
290         struct mm_gk20a *mm = &g->mm;
291         int i;
292
293         gk20a_dbg_fn("");
294
295         if (mm->sw_ready) {
296                 gk20a_dbg_fn("skip init");
297                 return 0;
298         }
299
300         mm->g = g;
301         mutex_init(&mm->l2_op_lock);
302         mm->big_page_size = gmmu_page_sizes[gmmu_page_size_big];
303         mm->compression_page_size = gmmu_page_sizes[gmmu_page_size_big];
304         mm->pde_stride    = mm->big_page_size << 10;
305         mm->pde_stride_shift = ilog2(mm->pde_stride);
306         BUG_ON(mm->pde_stride_shift > 31); /* we have assumptions about this */
307
308         for (i = 0; i < ARRAY_SIZE(gmmu_page_sizes); i++) {
309
310                 u32 num_ptes, pte_space, num_pages;
311
312                 /* assuming "full" page tables */
313                 num_ptes = mm->pde_stride / gmmu_page_sizes[i];
314
315                 pte_space = num_ptes * gmmu_pte__size_v();
316                 /* allocate whole pages */
317                 pte_space = roundup(pte_space, PAGE_SIZE);
318
319                 num_pages = pte_space / PAGE_SIZE;
320                 /* make sure "order" is viable */
321                 BUG_ON(!is_power_of_2(num_pages));
322
323                 mm->page_table_sizing[i].num_ptes = num_ptes;
324                 mm->page_table_sizing[i].order = ilog2(num_pages);
325         }
326
327         /*TBD: make channel vm size configurable */
328         mm->channel.size = 1ULL << NV_GMMU_VA_RANGE;
329
330         gk20a_dbg_info("channel vm size: %dMB", (int)(mm->channel.size >> 20));
331
332         gk20a_dbg_info("small page-size (%dKB) pte array: %dKB",
333                         gmmu_page_sizes[gmmu_page_size_small] >> 10,
334                         (mm->page_table_sizing[gmmu_page_size_small].num_ptes *
335                          gmmu_pte__size_v()) >> 10);
336
337         gk20a_dbg_info("big page-size (%dKB) pte array: %dKB",
338                         gmmu_page_sizes[gmmu_page_size_big] >> 10,
339                         (mm->page_table_sizing[gmmu_page_size_big].num_ptes *
340                          gmmu_pte__size_v()) >> 10);
341
342
343         gk20a_init_bar1_vm(mm);
344
345         mm->remove_support = gk20a_remove_mm_support;
346         mm->sw_ready = true;
347
348         gk20a_dbg_fn("done");
349         return 0;
350 }
351
352 /* make sure gk20a_init_mm_support is called before */
353 static int gk20a_init_mm_setup_hw(struct gk20a *g)
354 {
355         struct mm_gk20a *mm = &g->mm;
356         struct inst_desc *inst_block = &mm->bar1.inst_block;
357         phys_addr_t inst_pa = inst_block->cpu_pa;
358
359         gk20a_dbg_fn("");
360
361         /* set large page size in fb
362          * note this is very early on, can we defer it ? */
363         {
364                 u32 fb_mmu_ctrl = gk20a_readl(g, fb_mmu_ctrl_r());
365
366                 if (gmmu_page_sizes[gmmu_page_size_big] == SZ_128K)
367                         fb_mmu_ctrl = (fb_mmu_ctrl &
368                                        ~fb_mmu_ctrl_vm_pg_size_f(~0x0)) |
369                                 fb_mmu_ctrl_vm_pg_size_128kb_f();
370                 else
371                         BUG_ON(1); /* no support/testing for larger ones yet */
372
373                 gk20a_writel(g, fb_mmu_ctrl_r(), fb_mmu_ctrl);
374         }
375
376         inst_pa = (u32)(inst_pa >> bar1_instance_block_shift_gk20a());
377         gk20a_dbg_info("bar1 inst block ptr: 0x%08x",  (u32)inst_pa);
378
379         gk20a_writel(g, bus_bar1_block_r(),
380                      bus_bar1_block_target_vid_mem_f() |
381                      bus_bar1_block_mode_virtual_f() |
382                      bus_bar1_block_ptr_f(inst_pa));
383         if (gk20a_mm_fb_flush(g) || gk20a_mm_fb_flush(g))
384                 return -EBUSY;
385
386         gk20a_dbg_fn("done");
387         return 0;
388 }
389
390 int gk20a_init_mm_support(struct gk20a *g)
391 {
392         u32 err;
393
394         err = gk20a_init_mm_reset_enable_hw(g);
395         if (err)
396                 return err;
397
398         err = gk20a_init_mm_setup_sw(g);
399         if (err)
400                 return err;
401
402         err = gk20a_init_mm_setup_hw(g);
403         if (err)
404                 return err;
405
406         return err;
407 }
408
409 #ifdef CONFIG_GK20A_PHYS_PAGE_TABLES
410 static int alloc_gmmu_pages(struct vm_gk20a *vm, u32 order,
411                             void **handle,
412                             struct sg_table **sgt,
413                             size_t *size)
414 {
415         u32 num_pages = 1 << order;
416         u32 len = num_pages * PAGE_SIZE;
417         int err;
418         struct page *pages;
419
420         gk20a_dbg_fn("");
421
422         pages = alloc_pages(GFP_KERNEL, order);
423         if (!pages) {
424                 gk20a_dbg(gpu_dbg_pte, "alloc_pages failed\n");
425                 goto err_out;
426         }
427         *sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
428         if (!sgt) {
429                 gk20a_dbg(gpu_dbg_pte, "cannot allocate sg table");
430                 goto err_alloced;
431         }
432         err = sg_alloc_table(*sgt, 1, GFP_KERNEL);
433         if (err) {
434                 gk20a_dbg(gpu_dbg_pte, "sg_alloc_table failed\n");
435                 goto err_sg_table;
436         }
437         sg_set_page((*sgt)->sgl, pages, len, 0);
438         *handle = page_address(pages);
439         memset(*handle, 0, len);
440         *size = len;
441         FLUSH_CPU_DCACHE(*handle, sg_phys((*sgt)->sgl), len);
442
443         return 0;
444
445 err_sg_table:
446         kfree(*sgt);
447 err_alloced:
448         __free_pages(pages, order);
449 err_out:
450         return -ENOMEM;
451 }
452
453 static void free_gmmu_pages(struct vm_gk20a *vm, void *handle,
454                             struct sg_table *sgt, u32 order,
455                             size_t size)
456 {
457         gk20a_dbg_fn("");
458         BUG_ON(sgt == NULL);
459         free_pages((unsigned long)handle, order);
460         sg_free_table(sgt);
461         kfree(sgt);
462 }
463
464 static int map_gmmu_pages(void *handle, struct sg_table *sgt,
465                           void **va, size_t size)
466 {
467         FLUSH_CPU_DCACHE(handle, sg_phys(sgt->sgl), sgt->sgl->length);
468         *va = handle;
469         return 0;
470 }
471
472 static void unmap_gmmu_pages(void *handle, struct sg_table *sgt, void *va)
473 {
474         FLUSH_CPU_DCACHE(handle, sg_phys(sgt->sgl), sgt->sgl->length);
475 }
476 #else
477
478 static int alloc_gmmu_pages(struct vm_gk20a *vm, u32 order,
479                             void **handle,
480                             struct sg_table **sgt,
481                             size_t *size)
482 {
483         struct device *d = dev_from_vm(vm);
484         u32 num_pages = 1 << order;
485         u32 len = num_pages * PAGE_SIZE;
486         dma_addr_t iova;
487         DEFINE_DMA_ATTRS(attrs);
488         struct page **pages;
489         void *cpuva;
490         int err = 0;
491
492         gk20a_dbg_fn("");
493
494         *size = len;
495
496         if (IS_ENABLED(CONFIG_ARM64)) {
497                 cpuva = dma_zalloc_coherent(d, len, &iova, GFP_KERNEL);
498                 if (!cpuva) {
499                         gk20a_err(d, "memory allocation failed\n");
500                         goto err_out;
501                 }
502
503                 err = gk20a_get_sgtable(d, sgt, cpuva, iova, len);
504                 if (err) {
505                         gk20a_err(d, "sgt allocation failed\n");
506                         goto err_free;
507                 }
508
509                 *handle = cpuva;
510         } else {
511                 dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &attrs);
512                 pages = dma_alloc_attrs(d, len, &iova, GFP_KERNEL, &attrs);
513                 if (!pages) {
514                         gk20a_err(d, "memory allocation failed\n");
515                         goto err_out;
516                 }
517
518                 err = gk20a_get_sgtable_from_pages(d, sgt, pages,
519                                         iova, len);
520                 if (err) {
521                         gk20a_err(d, "sgt allocation failed\n");
522                         goto err_free;
523                 }
524
525                 *handle = (void *)pages;
526         }
527
528         return 0;
529
530 err_free:
531         if (IS_ENABLED(CONFIG_ARM64)) {
532                 dma_free_coherent(d, len, handle, iova);
533                 cpuva = NULL;
534         } else {
535                 dma_free_attrs(d, len, pages, iova, &attrs);
536                 pages = NULL;
537         }
538         iova = 0;
539 err_out:
540         return -ENOMEM;
541 }
542
543 static void free_gmmu_pages(struct vm_gk20a *vm, void *handle,
544                             struct sg_table *sgt, u32 order,
545                             size_t size)
546 {
547         struct device *d = dev_from_vm(vm);
548         u64 iova;
549         DEFINE_DMA_ATTRS(attrs);
550         struct page **pages;
551
552         gk20a_dbg_fn("");
553         BUG_ON(sgt == NULL);
554
555         iova = sg_dma_address(sgt->sgl);
556
557         gk20a_free_sgtable(&sgt);
558
559         if (IS_ENABLED(CONFIG_ARM64)) {
560                 dma_free_coherent(d, size, handle, iova);
561         } else {
562                 pages = (struct page **)handle;
563                 dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &attrs);
564                 dma_free_attrs(d, size, pages, iova, &attrs);
565                 pages = NULL;
566         }
567
568         handle = NULL;
569         iova = 0;
570 }
571
572 static int map_gmmu_pages(void *handle, struct sg_table *sgt,
573                           void **kva, size_t size)
574 {
575         int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
576         struct page **pages;
577         gk20a_dbg_fn("");
578
579         if (IS_ENABLED(CONFIG_ARM64)) {
580                 *kva = handle;
581         } else {
582                 pages = (struct page **)handle;
583                 *kva = vmap(pages, count, 0, pgprot_dmacoherent(PAGE_KERNEL));
584                 if (!(*kva))
585                         return -ENOMEM;
586         }
587
588         return 0;
589 }
590
591 static void unmap_gmmu_pages(void *handle, struct sg_table *sgt, void *va)
592 {
593         gk20a_dbg_fn("");
594
595         if (!IS_ENABLED(CONFIG_ARM64))
596                 vunmap(va);
597         va = NULL;
598 }
599 #endif
600
601 /* allocate a phys contig region big enough for a full
602  * sized gmmu page table for the given gmmu_page_size.
603  * the whole range is zeroed so it's "invalid"/will fault
604  */
605
606 static int zalloc_gmmu_page_table_gk20a(struct vm_gk20a *vm,
607                                         enum gmmu_pgsz_gk20a gmmu_pgsz_idx,
608                                         struct page_table_gk20a *pte)
609 {
610         int err;
611         u32 pte_order;
612         void *handle = NULL;
613         struct sg_table *sgt;
614         size_t size;
615
616         gk20a_dbg_fn("");
617
618         /* allocate enough pages for the table */
619         pte_order = vm->mm->page_table_sizing[gmmu_pgsz_idx].order;
620
621         err = alloc_gmmu_pages(vm, pte_order, &handle, &sgt, &size);
622         if (err)
623                 return err;
624
625         gk20a_dbg(gpu_dbg_pte, "pte = 0x%p, addr=%08llx, size %d",
626                         pte, gk20a_mm_iova_addr(sgt->sgl), pte_order);
627
628         pte->ref = handle;
629         pte->sgt = sgt;
630         pte->size = size;
631
632         return 0;
633 }
634
635 /* given address range (inclusive) determine the pdes crossed */
636 static inline void pde_range_from_vaddr_range(struct vm_gk20a *vm,
637                                               u64 addr_lo, u64 addr_hi,
638                                               u32 *pde_lo, u32 *pde_hi)
639 {
640         *pde_lo = (u32)(addr_lo >> vm->mm->pde_stride_shift);
641         *pde_hi = (u32)(addr_hi >> vm->mm->pde_stride_shift);
642         gk20a_dbg(gpu_dbg_pte, "addr_lo=0x%llx addr_hi=0x%llx pde_ss=%d",
643                    addr_lo, addr_hi, vm->mm->pde_stride_shift);
644         gk20a_dbg(gpu_dbg_pte, "pde_lo=%d pde_hi=%d",
645                    *pde_lo, *pde_hi);
646 }
647
648 static inline u32 *pde_from_index(struct vm_gk20a *vm, u32 i)
649 {
650         return (u32 *) (((u8 *)vm->pdes.kv) + i*gmmu_pde__size_v());
651 }
652
653 static inline u32 pte_index_from_vaddr(struct vm_gk20a *vm,
654                                        u64 addr, enum gmmu_pgsz_gk20a pgsz_idx)
655 {
656         u32 ret;
657         /* mask off pde part */
658         addr = addr & ((((u64)1) << vm->mm->pde_stride_shift) - ((u64)1));
659         /* shift over to get pte index. note assumption that pte index
660          * doesn't leak over into the high 32b */
661         ret = (u32)(addr >> gmmu_page_shifts[pgsz_idx]);
662
663         gk20a_dbg(gpu_dbg_pte, "addr=0x%llx pte_i=0x%x", addr, ret);
664         return ret;
665 }
666
667 static inline void pte_space_page_offset_from_index(u32 i, u32 *pte_page,
668                                                     u32 *pte_offset)
669 {
670         /* ptes are 8B regardless of pagesize */
671         /* pte space pages are 4KB. so 512 ptes per 4KB page*/
672         *pte_page = i >> 9;
673
674         /* this offset is a pte offset, not a byte offset */
675         *pte_offset = i & ((1<<9)-1);
676
677         gk20a_dbg(gpu_dbg_pte, "i=0x%x pte_page=0x%x pte_offset=0x%x",
678                    i, *pte_page, *pte_offset);
679 }
680
681
682 /*
683  * given a pde index/page table number make sure it has
684  * backing store and if not go ahead allocate it and
685  * record it in the appropriate pde
686  */
687 static int validate_gmmu_page_table_gk20a_locked(struct vm_gk20a *vm,
688                                 u32 i, enum gmmu_pgsz_gk20a gmmu_pgsz_idx)
689 {
690         int err;
691         struct page_table_gk20a *pte =
692                 vm->pdes.ptes[gmmu_pgsz_idx] + i;
693
694         gk20a_dbg_fn("");
695
696         /* if it's already in place it's valid */
697         if (pte->ref)
698                 return 0;
699
700         gk20a_dbg(gpu_dbg_pte, "alloc %dKB ptes for pde %d",
701                    gmmu_page_sizes[gmmu_pgsz_idx]/1024, i);
702
703         err = zalloc_gmmu_page_table_gk20a(vm, gmmu_pgsz_idx, pte);
704         if (err)
705                 return err;
706
707         /* rewrite pde */
708         update_gmmu_pde_locked(vm, i);
709
710         return 0;
711 }
712
713 static struct vm_reserved_va_node *addr_to_reservation(struct vm_gk20a *vm,
714                                                        u64 addr)
715 {
716         struct vm_reserved_va_node *va_node;
717         list_for_each_entry(va_node, &vm->reserved_va_list, reserved_va_list)
718                 if (addr >= va_node->vaddr_start &&
719                     addr < (u64)va_node->vaddr_start + (u64)va_node->size)
720                         return va_node;
721
722         return NULL;
723 }
724
725 int gk20a_vm_get_buffers(struct vm_gk20a *vm,
726                          struct mapped_buffer_node ***mapped_buffers,
727                          int *num_buffers)
728 {
729         struct mapped_buffer_node *mapped_buffer;
730         struct mapped_buffer_node **buffer_list;
731         struct rb_node *node;
732         int i = 0;
733
734         mutex_lock(&vm->update_gmmu_lock);
735
736         buffer_list = kzalloc(sizeof(*buffer_list) *
737                               vm->num_user_mapped_buffers, GFP_KERNEL);
738         if (!buffer_list) {
739                 mutex_unlock(&vm->update_gmmu_lock);
740                 return -ENOMEM;
741         }
742
743         node = rb_first(&vm->mapped_buffers);
744         while (node) {
745                 mapped_buffer =
746                         container_of(node, struct mapped_buffer_node, node);
747                 if (mapped_buffer->user_mapped) {
748                         buffer_list[i] = mapped_buffer;
749                         kref_get(&mapped_buffer->ref);
750                         i++;
751                 }
752                 node = rb_next(&mapped_buffer->node);
753         }
754
755         BUG_ON(i != vm->num_user_mapped_buffers);
756
757         *num_buffers = vm->num_user_mapped_buffers;
758         *mapped_buffers = buffer_list;
759
760         mutex_unlock(&vm->update_gmmu_lock);
761
762         return 0;
763 }
764
765 static void gk20a_vm_unmap_locked_kref(struct kref *ref)
766 {
767         struct mapped_buffer_node *mapped_buffer =
768                 container_of(ref, struct mapped_buffer_node, ref);
769         gk20a_vm_unmap_locked(mapped_buffer);
770 }
771
772 void gk20a_vm_put_buffers(struct vm_gk20a *vm,
773                                  struct mapped_buffer_node **mapped_buffers,
774                                  int num_buffers)
775 {
776         int i;
777
778         mutex_lock(&vm->update_gmmu_lock);
779
780         for (i = 0; i < num_buffers; ++i)
781                 kref_put(&mapped_buffers[i]->ref,
782                          gk20a_vm_unmap_locked_kref);
783
784         mutex_unlock(&vm->update_gmmu_lock);
785
786         kfree(mapped_buffers);
787 }
788
789 static void gk20a_vm_unmap_user(struct vm_gk20a *vm, u64 offset)
790 {
791         struct device *d = dev_from_vm(vm);
792         int retries;
793         struct mapped_buffer_node *mapped_buffer;
794
795         mutex_lock(&vm->update_gmmu_lock);
796
797         mapped_buffer = find_mapped_buffer_locked(&vm->mapped_buffers, offset);
798         if (!mapped_buffer) {
799                 mutex_unlock(&vm->update_gmmu_lock);
800                 gk20a_err(d, "invalid addr to unmap 0x%llx", offset);
801                 return;
802         }
803
804         if (mapped_buffer->flags & NVHOST_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET) {
805                 mutex_unlock(&vm->update_gmmu_lock);
806
807                 retries = 1000;
808                 while (retries) {
809                         if (atomic_read(&mapped_buffer->ref.refcount) == 1)
810                                 break;
811                         retries--;
812                         udelay(50);
813                 }
814                 if (!retries)
815                         gk20a_err(d, "sync-unmap failed on 0x%llx",
816                                                                 offset);
817                 mutex_lock(&vm->update_gmmu_lock);
818         }
819
820         mapped_buffer->user_mapped--;
821         if (mapped_buffer->user_mapped == 0)
822                 vm->num_user_mapped_buffers--;
823         kref_put(&mapped_buffer->ref, gk20a_vm_unmap_locked_kref);
824
825         mutex_unlock(&vm->update_gmmu_lock);
826 }
827
828 static u64 gk20a_vm_alloc_va(struct vm_gk20a *vm,
829                              u64 size,
830                              enum gmmu_pgsz_gk20a gmmu_pgsz_idx)
831
832 {
833         struct gk20a_allocator *vma = &vm->vma[gmmu_pgsz_idx];
834         int err;
835         u64 offset;
836         u32 start_page_nr = 0, num_pages;
837         u64 gmmu_page_size = gmmu_page_sizes[gmmu_pgsz_idx];
838
839         if (gmmu_pgsz_idx >= ARRAY_SIZE(gmmu_page_sizes)) {
840                 dev_warn(dev_from_vm(vm),
841                          "invalid page size requested in gk20a vm alloc");
842                 return -EINVAL;
843         }
844
845         if ((gmmu_pgsz_idx == gmmu_page_size_big) && !vm->big_pages) {
846                 dev_warn(dev_from_vm(vm),
847                          "unsupportd page size requested");
848                 return -EINVAL;
849
850         }
851
852         /* be certain we round up to gmmu_page_size if needed */
853         /* TBD: DIV_ROUND_UP -> undefined reference to __aeabi_uldivmod */
854         size = (size + ((u64)gmmu_page_size - 1)) & ~((u64)gmmu_page_size - 1);
855
856         gk20a_dbg_info("size=0x%llx @ pgsz=%dKB", size,
857                         gmmu_page_sizes[gmmu_pgsz_idx]>>10);
858
859         /* The vma allocator represents page accounting. */
860         num_pages = size >> gmmu_page_shifts[gmmu_pgsz_idx];
861
862         err = vma->alloc(vma, &start_page_nr, num_pages);
863
864         if (err) {
865                 gk20a_err(dev_from_vm(vm),
866                            "%s oom: sz=0x%llx", vma->name, size);
867                 return 0;
868         }
869
870         offset = (u64)start_page_nr << gmmu_page_shifts[gmmu_pgsz_idx];
871         gk20a_dbg_fn("%s found addr: 0x%llx", vma->name, offset);
872
873         return offset;
874 }
875
876 static int gk20a_vm_free_va(struct vm_gk20a *vm,
877                              u64 offset, u64 size,
878                              enum gmmu_pgsz_gk20a pgsz_idx)
879 {
880         struct gk20a_allocator *vma = &vm->vma[pgsz_idx];
881         u32 page_size = gmmu_page_sizes[pgsz_idx];
882         u32 page_shift = gmmu_page_shifts[pgsz_idx];
883         u32 start_page_nr, num_pages;
884         int err;
885
886         gk20a_dbg_info("%s free addr=0x%llx, size=0x%llx",
887                         vma->name, offset, size);
888
889         start_page_nr = (u32)(offset >> page_shift);
890         num_pages = (u32)((size + page_size - 1) >> page_shift);
891
892         err = vma->free(vma, start_page_nr, num_pages);
893         if (err) {
894                 gk20a_err(dev_from_vm(vm),
895                            "not found: offset=0x%llx, sz=0x%llx",
896                            offset, size);
897         }
898
899         return err;
900 }
901
902 static int insert_mapped_buffer(struct rb_root *root,
903                                 struct mapped_buffer_node *mapped_buffer)
904 {
905         struct rb_node **new_node = &(root->rb_node), *parent = NULL;
906
907         /* Figure out where to put new node */
908         while (*new_node) {
909                 struct mapped_buffer_node *cmp_with =
910                         container_of(*new_node, struct mapped_buffer_node,
911                                      node);
912
913                 parent = *new_node;
914
915                 if (cmp_with->addr > mapped_buffer->addr) /* u64 cmp */
916                         new_node = &((*new_node)->rb_left);
917                 else if (cmp_with->addr != mapped_buffer->addr) /* u64 cmp */
918                         new_node = &((*new_node)->rb_right);
919                 else
920                         return -EINVAL; /* no fair dup'ing */
921         }
922
923         /* Add new node and rebalance tree. */
924         rb_link_node(&mapped_buffer->node, parent, new_node);
925         rb_insert_color(&mapped_buffer->node, root);
926
927         return 0;
928 }
929
930 static struct mapped_buffer_node *find_mapped_buffer_reverse_locked(
931                                 struct rb_root *root, struct dma_buf *dmabuf,
932                                 u32 kind)
933 {
934         struct rb_node *node = rb_first(root);
935         while (node) {
936                 struct mapped_buffer_node *mapped_buffer =
937                         container_of(node, struct mapped_buffer_node, node);
938                 if (mapped_buffer->dmabuf == dmabuf &&
939                     kind == mapped_buffer->kind)
940                         return mapped_buffer;
941                 node = rb_next(&mapped_buffer->node);
942         }
943         return 0;
944 }
945
946 static struct mapped_buffer_node *find_mapped_buffer_locked(
947                                         struct rb_root *root, u64 addr)
948 {
949
950         struct rb_node *node = root->rb_node;
951         while (node) {
952                 struct mapped_buffer_node *mapped_buffer =
953                         container_of(node, struct mapped_buffer_node, node);
954                 if (mapped_buffer->addr > addr) /* u64 cmp */
955                         node = node->rb_left;
956                 else if (mapped_buffer->addr != addr) /* u64 cmp */
957                         node = node->rb_right;
958                 else
959                         return mapped_buffer;
960         }
961         return 0;
962 }
963
964 static struct mapped_buffer_node *find_mapped_buffer_range_locked(
965                                         struct rb_root *root, u64 addr)
966 {
967         struct rb_node *node = root->rb_node;
968         while (node) {
969                 struct mapped_buffer_node *m =
970                         container_of(node, struct mapped_buffer_node, node);
971                 if (m->addr <= addr && m->addr + m->size > addr)
972                         return m;
973                 else if (m->addr > addr) /* u64 cmp */
974                         node = node->rb_left;
975                 else
976                         node = node->rb_right;
977         }
978         return 0;
979 }
980
981 #define BFR_ATTRS (sizeof(nvmap_bfr_param)/sizeof(nvmap_bfr_param[0]))
982
983 struct buffer_attrs {
984         struct sg_table *sgt;
985         u64 size;
986         u64 align;
987         u32 ctag_offset;
988         u32 ctag_lines;
989         int pgsz_idx;
990         u8 kind_v;
991         u8 uc_kind_v;
992 };
993
994 static void gmmu_select_page_size(struct buffer_attrs *bfr)
995 {
996         int i;
997         /*  choose the biggest first (top->bottom) */
998         for (i = (gmmu_nr_page_sizes-1); i >= 0; i--)
999                 if (!(gmmu_page_offset_masks[i] & bfr->align)) {
1000                         /* would like to add this too but nvmap returns the
1001                          * original requested size not the allocated size.
1002                          * (!(gmmu_page_offset_masks[i] & bfr->size)) */
1003                         bfr->pgsz_idx = i;
1004                         break;
1005                 }
1006 }
1007
1008 static int setup_buffer_kind_and_compression(struct device *d,
1009                                              u32 flags,
1010                                              struct buffer_attrs *bfr,
1011                                              enum gmmu_pgsz_gk20a pgsz_idx)
1012 {
1013         bool kind_compressible;
1014
1015         if (unlikely(bfr->kind_v == gmmu_pte_kind_invalid_v()))
1016                 bfr->kind_v = gmmu_pte_kind_pitch_v();
1017
1018         if (unlikely(!gk20a_kind_is_supported(bfr->kind_v))) {
1019                 gk20a_err(d, "kind 0x%x not supported", bfr->kind_v);
1020                 return -EINVAL;
1021         }
1022
1023         bfr->uc_kind_v = gmmu_pte_kind_invalid_v();
1024         /* find a suitable uncompressed kind if it becomes necessary later */
1025         kind_compressible = gk20a_kind_is_compressible(bfr->kind_v);
1026         if (kind_compressible) {
1027                 bfr->uc_kind_v = gk20a_get_uncompressed_kind(bfr->kind_v);
1028                 if (unlikely(bfr->uc_kind_v == gmmu_pte_kind_invalid_v())) {
1029                         /* shouldn't happen, but it is worth cross-checking */
1030                         gk20a_err(d, "comptag kind 0x%x can't be"
1031                                    " downgraded to uncompressed kind",
1032                                    bfr->kind_v);
1033                         return -EINVAL;
1034                 }
1035         }
1036         /* comptags only supported for suitable kinds, 128KB pagesize */
1037         if (unlikely(kind_compressible &&
1038                      (gmmu_page_sizes[pgsz_idx] != 128*1024))) {
1039                 /*
1040                 gk20a_warn(d, "comptags specified"
1041                 " but pagesize being used doesn't support it");*/
1042                 /* it is safe to fall back to uncompressed as
1043                    functionality is not harmed */
1044                 bfr->kind_v = bfr->uc_kind_v;
1045                 kind_compressible = false;
1046         }
1047         if (kind_compressible)
1048                 bfr->ctag_lines = ALIGN(bfr->size, COMP_TAG_LINE_SIZE) >>
1049                         COMP_TAG_LINE_SIZE_SHIFT;
1050         else
1051                 bfr->ctag_lines = 0;
1052
1053         return 0;
1054 }
1055
1056 static int validate_fixed_buffer(struct vm_gk20a *vm,
1057                                  struct buffer_attrs *bfr,
1058                                  u64 map_offset)
1059 {
1060         struct device *dev = dev_from_vm(vm);
1061         struct vm_reserved_va_node *va_node;
1062         struct mapped_buffer_node *buffer;
1063
1064         if (map_offset & gmmu_page_offset_masks[bfr->pgsz_idx]) {
1065                 gk20a_err(dev, "map offset must be buffer page size aligned 0x%llx",
1066                            map_offset);
1067                 return -EINVAL;
1068         }
1069
1070         /* find the space reservation */
1071         va_node = addr_to_reservation(vm, map_offset);
1072         if (!va_node) {
1073                 gk20a_warn(dev, "fixed offset mapping without space allocation");
1074                 return -EINVAL;
1075         }
1076
1077         /* check that this mappings does not collide with existing
1078          * mappings by checking the overlapping area between the current
1079          * buffer and all other mapped buffers */
1080
1081         list_for_each_entry(buffer,
1082                 &va_node->va_buffers_list, va_buffers_list) {
1083                 s64 begin = max(buffer->addr, map_offset);
1084                 s64 end = min(buffer->addr +
1085                         buffer->size, map_offset + bfr->size);
1086                 if (end - begin > 0) {
1087                         gk20a_warn(dev, "overlapping buffer map requested");
1088                         return -EINVAL;
1089                 }
1090         }
1091
1092         return 0;
1093 }
1094
1095 static u64 __locked_gmmu_map(struct vm_gk20a *vm,
1096                                 u64 map_offset,
1097                                 struct sg_table *sgt,
1098                                 u64 size,
1099                                 int pgsz_idx,
1100                                 u8 kind_v,
1101                                 u32 ctag_offset,
1102                                 u32 flags,
1103                                 int rw_flag)
1104 {
1105         int err = 0, i = 0;
1106         u32 pde_lo, pde_hi;
1107         struct device *d = dev_from_vm(vm);
1108
1109         /* Allocate (or validate when map_offset != 0) the virtual address. */
1110         if (!map_offset) {
1111                 map_offset = gk20a_vm_alloc_va(vm, size,
1112                                           pgsz_idx);
1113                 if (!map_offset) {
1114                         gk20a_err(d, "failed to allocate va space");
1115                         err = -ENOMEM;
1116                         goto fail;
1117                 }
1118         }
1119
1120         pde_range_from_vaddr_range(vm,
1121                                    map_offset,
1122                                    map_offset + size - 1,
1123                                    &pde_lo, &pde_hi);
1124
1125         /* mark the addr range valid (but with 0 phys addr, which will fault) */
1126         for (i = pde_lo; i <= pde_hi; i++) {
1127                 err = validate_gmmu_page_table_gk20a_locked(vm, i,
1128                                                             pgsz_idx);
1129                 if (err) {
1130                         gk20a_err(d, "failed to validate page table %d: %d",
1131                                                            i, err);
1132                         goto fail;
1133                 }
1134         }
1135
1136         err = update_gmmu_ptes_locked(vm, pgsz_idx,
1137                                       sgt,
1138                                       map_offset, map_offset + size - 1,
1139                                       kind_v,
1140                                       ctag_offset,
1141                                       flags &
1142                                       NVHOST_MAP_BUFFER_FLAGS_CACHEABLE_TRUE,
1143                                       rw_flag);
1144         if (err) {
1145                 gk20a_err(d, "failed to update ptes on map");
1146                 goto fail;
1147         }
1148
1149         return map_offset;
1150  fail:
1151         gk20a_err(d, "%s: failed with err=%d\n", __func__, err);
1152         return 0;
1153 }
1154
1155 static void __locked_gmmu_unmap(struct vm_gk20a *vm,
1156                                 u64 vaddr,
1157                                 u64 size,
1158                                 int pgsz_idx,
1159                                 bool va_allocated,
1160                                 int rw_flag)
1161 {
1162         int err = 0;
1163         struct gk20a *g = gk20a_from_vm(vm);
1164
1165         if (va_allocated) {
1166                 err = gk20a_vm_free_va(vm, vaddr, size, pgsz_idx);
1167                 if (err) {
1168                         dev_err(dev_from_vm(vm),
1169                                 "failed to free va");
1170                         return;
1171                 }
1172         }
1173
1174         /* unmap here needs to know the page size we assigned at mapping */
1175         err = update_gmmu_ptes_locked(vm,
1176                                 pgsz_idx,
1177                                 0, /* n/a for unmap */
1178                                 vaddr,
1179                                 vaddr + size - 1,
1180                                 0, 0, false /* n/a for unmap */,
1181                                 rw_flag);
1182         if (err)
1183                 dev_err(dev_from_vm(vm),
1184                         "failed to update gmmu ptes on unmap");
1185
1186         /* detect which if any pdes/ptes can now be released */
1187
1188         /* flush l2 so any dirty lines are written out *now*.
1189          *  also as we could potentially be switching this buffer
1190          * from nonvolatile (l2 cacheable) to volatile (l2 non-cacheable) at
1191          * some point in the future we need to invalidate l2.  e.g. switching
1192          * from a render buffer unmap (here) to later using the same memory
1193          * for gmmu ptes.  note the positioning of this relative to any smmu
1194          * unmapping (below). */
1195
1196         gk20a_mm_l2_flush(g, true);
1197 }
1198
1199 static u64 gk20a_vm_map_duplicate_locked(struct vm_gk20a *vm,
1200                                          struct dma_buf *dmabuf,
1201                                          u64 offset_align,
1202                                          u32 flags,
1203                                          int kind,
1204                                          struct sg_table **sgt,
1205                                          bool user_mapped,
1206                                          int rw_flag)
1207 {
1208         struct mapped_buffer_node *mapped_buffer = 0;
1209
1210         mapped_buffer =
1211                 find_mapped_buffer_reverse_locked(&vm->mapped_buffers,
1212                                                   dmabuf, kind);
1213         if (!mapped_buffer)
1214                 return 0;
1215
1216         if (mapped_buffer->flags != flags)
1217                 return 0;
1218
1219         if (flags & NVHOST_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET &&
1220             mapped_buffer->addr != offset_align)
1221                 return 0;
1222
1223         BUG_ON(mapped_buffer->vm != vm);
1224
1225         /* mark the buffer as used */
1226         if (user_mapped) {
1227                 if (mapped_buffer->user_mapped == 0)
1228                         vm->num_user_mapped_buffers++;
1229                 mapped_buffer->user_mapped++;
1230
1231                 /* If the mapping comes from user space, we own
1232                  * the handle ref. Since we reuse an
1233                  * existing mapping here, we need to give back those
1234                  * refs once in order not to leak.
1235                  */
1236                 if (mapped_buffer->own_mem_ref)
1237                         dma_buf_put(mapped_buffer->dmabuf);
1238                 else
1239                         mapped_buffer->own_mem_ref = true;
1240         }
1241         kref_get(&mapped_buffer->ref);
1242
1243         gk20a_dbg(gpu_dbg_map,
1244                    "reusing as=%d pgsz=%d flags=0x%x ctags=%d "
1245                    "start=%d gv=0x%x,%08x -> 0x%x,%08x -> 0x%x,%08x "
1246                    "own_mem_ref=%d user_mapped=%d",
1247                    vm_aspace_id(vm), mapped_buffer->pgsz_idx,
1248                    mapped_buffer->flags,
1249                    mapped_buffer->ctag_lines,
1250                    mapped_buffer->ctag_offset,
1251                    hi32(mapped_buffer->addr), lo32(mapped_buffer->addr),
1252                    hi32((u64)sg_dma_address(mapped_buffer->sgt->sgl)),
1253                    lo32((u64)sg_dma_address(mapped_buffer->sgt->sgl)),
1254                    hi32((u64)sg_phys(mapped_buffer->sgt->sgl)),
1255                    lo32((u64)sg_phys(mapped_buffer->sgt->sgl)),
1256                    mapped_buffer->own_mem_ref, user_mapped);
1257
1258         if (sgt)
1259                 *sgt = mapped_buffer->sgt;
1260         return mapped_buffer->addr;
1261 }
1262
1263 u64 gk20a_vm_map(struct vm_gk20a *vm,
1264                         struct dma_buf *dmabuf,
1265                         u64 offset_align,
1266                         u32 flags /*NVHOST_AS_MAP_BUFFER_FLAGS_*/,
1267                         int kind,
1268                         struct sg_table **sgt,
1269                         bool user_mapped,
1270                         int rw_flag)
1271 {
1272         struct gk20a *g = gk20a_from_vm(vm);
1273         struct gk20a_allocator *ctag_allocator = &g->gr.comp_tags;
1274         struct device *d = dev_from_vm(vm);
1275         struct mapped_buffer_node *mapped_buffer = 0;
1276         bool inserted = false, va_allocated = false;
1277         u32 gmmu_page_size = 0;
1278         u64 map_offset = 0;
1279         int err = 0;
1280         struct buffer_attrs bfr = {0};
1281         struct gk20a_comptags comptags;
1282
1283         mutex_lock(&vm->update_gmmu_lock);
1284
1285         /* check if this buffer is already mapped */
1286         map_offset = gk20a_vm_map_duplicate_locked(vm, dmabuf, offset_align,
1287                                                    flags, kind, sgt,
1288                                                    user_mapped, rw_flag);
1289         if (map_offset) {
1290                 mutex_unlock(&vm->update_gmmu_lock);
1291                 return map_offset;
1292         }
1293
1294         /* pin buffer to get phys/iovmm addr */
1295         bfr.sgt = gk20a_mm_pin(d, dmabuf);
1296         if (IS_ERR(bfr.sgt)) {
1297                 /* Falling back to physical is actually possible
1298                  * here in many cases if we use 4K phys pages in the
1299                  * gmmu.  However we have some regions which require
1300                  * contig regions to work properly (either phys-contig
1301                  * or contig through smmu io_vaspace).  Until we can
1302                  * track the difference between those two cases we have
1303                  * to fail the mapping when we run out of SMMU space.
1304                  */
1305                 gk20a_warn(d, "oom allocating tracking buffer");
1306                 goto clean_up;
1307         }
1308
1309         if (sgt)
1310                 *sgt = bfr.sgt;
1311
1312         bfr.kind_v = kind;
1313         bfr.size = dmabuf->size;
1314         bfr.align = 1 << __ffs((u64)sg_dma_address(bfr.sgt->sgl));
1315         bfr.pgsz_idx = -1;
1316
1317         /* If FIX_OFFSET is set, pgsz is determined. Otherwise, select
1318          * page size according to memory alignment */
1319         if (flags & NVHOST_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET) {
1320                 bfr.pgsz_idx = NV_GMMU_VA_IS_UPPER(offset_align) ?
1321                                 gmmu_page_size_big : gmmu_page_size_small;
1322         } else {
1323                 if (vm->big_pages)
1324                         gmmu_select_page_size(&bfr);
1325                 else
1326                         bfr.pgsz_idx = gmmu_page_size_small;
1327         }
1328
1329         /* validate/adjust bfr attributes */
1330         if (unlikely(bfr.pgsz_idx == -1)) {
1331                 gk20a_err(d, "unsupported page size detected");
1332                 goto clean_up;
1333         }
1334
1335         if (unlikely(bfr.pgsz_idx < gmmu_page_size_small ||
1336                      bfr.pgsz_idx > gmmu_page_size_big)) {
1337                 BUG_ON(1);
1338                 err = -EINVAL;
1339                 goto clean_up;
1340         }
1341         gmmu_page_size = gmmu_page_sizes[bfr.pgsz_idx];
1342
1343         /* Check if we should use a fixed offset for mapping this buffer */
1344         if (flags & NVHOST_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET)  {
1345                 err = validate_fixed_buffer(vm, &bfr, offset_align);
1346                 if (err)
1347                         goto clean_up;
1348
1349                 map_offset = offset_align;
1350                 va_allocated = false;
1351         } else
1352                 va_allocated = true;
1353
1354         if (sgt)
1355                 *sgt = bfr.sgt;
1356
1357         err = setup_buffer_kind_and_compression(d, flags, &bfr, bfr.pgsz_idx);
1358         if (unlikely(err)) {
1359                 gk20a_err(d, "failure setting up kind and compression");
1360                 goto clean_up;
1361         }
1362
1363         /* bar1 and pmu vm don't need ctag */
1364         if (!vm->enable_ctag)
1365                 bfr.ctag_lines = 0;
1366
1367         gk20a_get_comptags(d, dmabuf, &comptags);
1368
1369         if (bfr.ctag_lines && !comptags.lines) {
1370                 /* allocate compression resources if needed */
1371                 err = gk20a_alloc_comptags(d, dmabuf, ctag_allocator,
1372                                            bfr.ctag_lines);
1373                 if (err) {
1374                         /* ok to fall back here if we ran out */
1375                         /* TBD: we can partially alloc ctags as well... */
1376                         bfr.ctag_lines = bfr.ctag_offset = 0;
1377                         bfr.kind_v = bfr.uc_kind_v;
1378                 } else {
1379                         gk20a_get_comptags(d, dmabuf, &comptags);
1380
1381                         /* init/clear the ctag buffer */
1382                         g->ops.ltc.cbc_ctrl(g, gk20a_cbc_op_clear,
1383                                           comptags.offset,
1384                                           comptags.offset + comptags.lines - 1);
1385                 }
1386         }
1387
1388         /* store the comptag info */
1389         bfr.ctag_offset = comptags.offset;
1390
1391         /* update gmmu ptes */
1392         map_offset = __locked_gmmu_map(vm, map_offset,
1393                                         bfr.sgt,
1394                                         bfr.size,
1395                                         bfr.pgsz_idx,
1396                                         bfr.kind_v,
1397                                         bfr.ctag_offset,
1398                                         flags, rw_flag);
1399         if (!map_offset)
1400                 goto clean_up;
1401
1402         gk20a_dbg(gpu_dbg_map,
1403            "as=%d pgsz=%d "
1404            "kind=0x%x kind_uc=0x%x flags=0x%x "
1405            "ctags=%d start=%d gv=0x%x,%08x -> 0x%x,%08x -> 0x%x,%08x",
1406            vm_aspace_id(vm), gmmu_page_size,
1407            bfr.kind_v, bfr.uc_kind_v, flags,
1408            bfr.ctag_lines, bfr.ctag_offset,
1409            hi32(map_offset), lo32(map_offset),
1410            hi32((u64)sg_dma_address(bfr.sgt->sgl)),
1411            lo32((u64)sg_dma_address(bfr.sgt->sgl)),
1412            hi32((u64)sg_phys(bfr.sgt->sgl)),
1413            lo32((u64)sg_phys(bfr.sgt->sgl)));
1414
1415 #if defined(NVHOST_DEBUG)
1416         {
1417                 int i;
1418                 struct scatterlist *sg = NULL;
1419                 gk20a_dbg(gpu_dbg_pte, "for_each_sg(bfr.sgt->sgl, sg, bfr.sgt->nents, i)");
1420                 for_each_sg(bfr.sgt->sgl, sg, bfr.sgt->nents, i ) {
1421                         u64 da = sg_dma_address(sg);
1422                         u64 pa = sg_phys(sg);
1423                         u64 len = sg->length;
1424                         gk20a_dbg(gpu_dbg_pte, "i=%d pa=0x%x,%08x da=0x%x,%08x len=0x%x,%08x",
1425                                    i, hi32(pa), lo32(pa), hi32(da), lo32(da),
1426                                    hi32(len), lo32(len));
1427                 }
1428         }
1429 #endif
1430
1431         /* keep track of the buffer for unmapping */
1432         /* TBD: check for multiple mapping of same buffer */
1433         mapped_buffer = kzalloc(sizeof(*mapped_buffer), GFP_KERNEL);
1434         if (!mapped_buffer) {
1435                 gk20a_warn(d, "oom allocating tracking buffer");
1436                 goto clean_up;
1437         }
1438         mapped_buffer->dmabuf      = dmabuf;
1439         mapped_buffer->sgt         = bfr.sgt;
1440         mapped_buffer->addr        = map_offset;
1441         mapped_buffer->size        = bfr.size;
1442         mapped_buffer->pgsz_idx    = bfr.pgsz_idx;
1443         mapped_buffer->ctag_offset = bfr.ctag_offset;
1444         mapped_buffer->ctag_lines  = bfr.ctag_lines;
1445         mapped_buffer->vm          = vm;
1446         mapped_buffer->flags       = flags;
1447         mapped_buffer->kind        = kind;
1448         mapped_buffer->va_allocated = va_allocated;
1449         mapped_buffer->user_mapped = user_mapped ? 1 : 0;
1450         mapped_buffer->own_mem_ref = user_mapped;
1451         INIT_LIST_HEAD(&mapped_buffer->unmap_list);
1452         INIT_LIST_HEAD(&mapped_buffer->va_buffers_list);
1453         kref_init(&mapped_buffer->ref);
1454
1455         err = insert_mapped_buffer(&vm->mapped_buffers, mapped_buffer);
1456         if (err) {
1457                 gk20a_err(d, "failed to insert into mapped buffer tree");
1458                 goto clean_up;
1459         }
1460         inserted = true;
1461         if (user_mapped)
1462                 vm->num_user_mapped_buffers++;
1463
1464         gk20a_dbg_info("allocated va @ 0x%llx", map_offset);
1465
1466         if (!va_allocated) {
1467                 struct vm_reserved_va_node *va_node;
1468
1469                 /* find the space reservation */
1470                 va_node = addr_to_reservation(vm, map_offset);
1471                 list_add_tail(&mapped_buffer->va_buffers_list,
1472                               &va_node->va_buffers_list);
1473                 mapped_buffer->va_node = va_node;
1474         }
1475
1476         mutex_unlock(&vm->update_gmmu_lock);
1477
1478         /* Invalidate kernel mappings immediately */
1479         if (vm_aspace_id(vm) == -1)
1480                 gk20a_mm_tlb_invalidate(vm);
1481
1482         return map_offset;
1483
1484 clean_up:
1485         if (inserted) {
1486                 rb_erase(&mapped_buffer->node, &vm->mapped_buffers);
1487                 if (user_mapped)
1488                         vm->num_user_mapped_buffers--;
1489         }
1490         kfree(mapped_buffer);
1491         if (va_allocated)
1492                 gk20a_vm_free_va(vm, map_offset, bfr.size, bfr.pgsz_idx);
1493         if (!IS_ERR(bfr.sgt))
1494                 gk20a_mm_unpin(d, dmabuf, bfr.sgt);
1495
1496         mutex_unlock(&vm->update_gmmu_lock);
1497         gk20a_dbg_info("err=%d\n", err);
1498         return 0;
1499 }
1500
1501 u64 gk20a_gmmu_map(struct vm_gk20a *vm,
1502                 struct sg_table **sgt,
1503                 u64 size,
1504                 u32 flags,
1505                 int rw_flag)
1506 {
1507         u64 vaddr;
1508
1509         mutex_lock(&vm->update_gmmu_lock);
1510         vaddr = __locked_gmmu_map(vm, 0, /* already mapped? - No */
1511                                 *sgt, /* sg table */
1512                                 size,
1513                                 0, /* page size index = 0 i.e. SZ_4K */
1514                                 0, /* kind */
1515                                 0, /* ctag_offset */
1516                                 flags, rw_flag);
1517         mutex_unlock(&vm->update_gmmu_lock);
1518         if (!vaddr) {
1519                 gk20a_err(dev_from_vm(vm), "failed to allocate va space");
1520                 return 0;
1521         }
1522
1523         /* Invalidate kernel mappings immediately */
1524         gk20a_mm_tlb_invalidate(vm);
1525
1526         return vaddr;
1527 }
1528
1529 void gk20a_gmmu_unmap(struct vm_gk20a *vm,
1530                 u64 vaddr,
1531                 u64 size,
1532                 int rw_flag)
1533 {
1534         mutex_lock(&vm->update_gmmu_lock);
1535         __locked_gmmu_unmap(vm,
1536                         vaddr,
1537                         size,
1538                         0, /* page size 4K */
1539                         true, /*va_allocated */
1540                         rw_flag);
1541         mutex_unlock(&vm->update_gmmu_lock);
1542 }
1543
1544 phys_addr_t gk20a_get_phys_from_iova(struct device *d,
1545                                 u64 dma_addr)
1546 {
1547         phys_addr_t phys;
1548         u64 iova;
1549
1550         struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(d);
1551         if (!mapping)
1552                 return dma_addr;
1553
1554         iova = dma_addr & PAGE_MASK;
1555         phys = iommu_iova_to_phys(mapping->domain, iova);
1556         return phys;
1557 }
1558
1559 /* get sg_table from already allocated buffer */
1560 int gk20a_get_sgtable(struct device *d, struct sg_table **sgt,
1561                         void *cpuva, u64 iova,
1562                         size_t size)
1563 {
1564         int err = 0;
1565         *sgt = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
1566         if (!(*sgt)) {
1567                 dev_err(d, "failed to allocate memory\n");
1568                 err = -ENOMEM;
1569                 goto fail;
1570         }
1571         err = dma_get_sgtable(d, *sgt,
1572                         cpuva, iova,
1573                         size);
1574         if (err) {
1575                 dev_err(d, "failed to create sg table\n");
1576                 goto fail;
1577         }
1578         sg_dma_address((*sgt)->sgl) = iova;
1579
1580         return 0;
1581  fail:
1582         if (*sgt) {
1583                 kfree(*sgt);
1584                 *sgt = NULL;
1585         }
1586         return err;
1587 }
1588
1589 int gk20a_get_sgtable_from_pages(struct device *d, struct sg_table **sgt,
1590                         struct page **pages, u64 iova,
1591                         size_t size)
1592 {
1593         int err = 0;
1594         *sgt = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
1595         if (!(*sgt)) {
1596                 dev_err(d, "failed to allocate memory\n");
1597                 err = -ENOMEM;
1598                 goto fail;
1599         }
1600         err = sg_alloc_table(*sgt, 1, GFP_KERNEL);
1601         if (err) {
1602                 dev_err(d, "failed to allocate sg_table\n");
1603                 goto fail;
1604         }
1605         sg_set_page((*sgt)->sgl, *pages, size, 0);
1606         sg_dma_address((*sgt)->sgl) = iova;
1607
1608         return 0;
1609  fail:
1610         if (*sgt) {
1611                 kfree(*sgt);
1612                 *sgt = NULL;
1613         }
1614         return err;
1615 }
1616
1617 void gk20a_free_sgtable(struct sg_table **sgt)
1618 {
1619         sg_free_table(*sgt);
1620         kfree(*sgt);
1621         *sgt = NULL;
1622 }
1623
1624 u64 gk20a_mm_iova_addr(struct scatterlist *sgl)
1625 {
1626         u64 result = sg_phys(sgl);
1627 #ifdef CONFIG_TEGRA_IOMMU_SMMU
1628         if (sg_dma_address(sgl) == DMA_ERROR_CODE)
1629                 result = 0;
1630         else if (sg_dma_address(sgl)) {
1631                 result = sg_dma_address(sgl) |
1632                         1ULL << NV_MC_SMMU_VADDR_TRANSLATION_BIT;
1633         }
1634 #endif
1635         return result;
1636 }
1637
1638 static int update_gmmu_ptes_locked(struct vm_gk20a *vm,
1639                                    enum gmmu_pgsz_gk20a pgsz_idx,
1640                                    struct sg_table *sgt,
1641                                    u64 first_vaddr, u64 last_vaddr,
1642                                    u8 kind_v, u32 ctag_offset,
1643                                    bool cacheable,
1644                                    int rw_flag)
1645 {
1646         int err;
1647         u32 pde_lo, pde_hi, pde_i;
1648         struct scatterlist *cur_chunk;
1649         unsigned int cur_offset;
1650         u32 pte_w[2] = {0, 0}; /* invalid pte */
1651         u32 ctag = ctag_offset;
1652         u32 ctag_incr;
1653         u32 page_size  = gmmu_page_sizes[pgsz_idx];
1654         u64 addr = 0;
1655
1656         pde_range_from_vaddr_range(vm, first_vaddr, last_vaddr,
1657                                    &pde_lo, &pde_hi);
1658
1659         gk20a_dbg(gpu_dbg_pte, "size_idx=%d, pde_lo=%d, pde_hi=%d",
1660                    pgsz_idx, pde_lo, pde_hi);
1661
1662         /* If ctag_offset !=0 add 1 else add 0.  The idea is to avoid a branch
1663          * below (per-pte). Note: this doesn't work unless page size (when
1664          * comptags are active) is 128KB. We have checks elsewhere for that. */
1665         ctag_incr = !!ctag_offset;
1666
1667         if (sgt)
1668                 cur_chunk = sgt->sgl;
1669         else
1670                 cur_chunk = NULL;
1671
1672         cur_offset = 0;
1673
1674         for (pde_i = pde_lo; pde_i <= pde_hi; pde_i++) {
1675                 u32 pte_lo, pte_hi;
1676                 u32 pte_cur;
1677                 void *pte_kv_cur;
1678
1679                 struct page_table_gk20a *pte = vm->pdes.ptes[pgsz_idx] + pde_i;
1680
1681                 if (pde_i == pde_lo)
1682                         pte_lo = pte_index_from_vaddr(vm, first_vaddr,
1683                                                       pgsz_idx);
1684                 else
1685                         pte_lo = 0;
1686
1687                 if ((pde_i != pde_hi) && (pde_hi != pde_lo))
1688                         pte_hi = vm->mm->page_table_sizing[pgsz_idx].num_ptes-1;
1689                 else
1690                         pte_hi = pte_index_from_vaddr(vm, last_vaddr,
1691                                                       pgsz_idx);
1692
1693                 /* get cpu access to the ptes */
1694                 err = map_gmmu_pages(pte->ref, pte->sgt, &pte_kv_cur,
1695                                      pte->size);
1696                 if (err) {
1697                         gk20a_err(dev_from_vm(vm),
1698                                    "couldn't map ptes for update as=%d pte_ref_cnt=%d",
1699                                    vm_aspace_id(vm), pte->ref_cnt);
1700                         goto clean_up;
1701                 }
1702
1703                 gk20a_dbg(gpu_dbg_pte, "pte_lo=%d, pte_hi=%d", pte_lo, pte_hi);
1704                 for (pte_cur = pte_lo; pte_cur <= pte_hi; pte_cur++) {
1705
1706                         if (likely(sgt)) {
1707                                 u64 new_addr = gk20a_mm_iova_addr(cur_chunk);
1708                                 if (new_addr) {
1709                                         addr = new_addr;
1710                                         addr += cur_offset;
1711                                 }
1712
1713                                 pte_w[0] = gmmu_pte_valid_true_f() |
1714                                         gmmu_pte_address_sys_f(addr
1715                                                 >> gmmu_pte_address_shift_v());
1716                                 pte_w[1] = gmmu_pte_aperture_video_memory_f() |
1717                                         gmmu_pte_kind_f(kind_v) |
1718                                         gmmu_pte_comptagline_f(ctag);
1719
1720                                 if (rw_flag == gk20a_mem_flag_read_only) {
1721                                         pte_w[0] |= gmmu_pte_read_only_true_f();
1722                                         pte_w[1] |=
1723                                                 gmmu_pte_write_disable_true_f();
1724                                 } else if (rw_flag ==
1725                                            gk20a_mem_flag_write_only) {
1726                                         pte_w[1] |=
1727                                                 gmmu_pte_read_disable_true_f();
1728                                 }
1729
1730                                 if (!cacheable)
1731                                         pte_w[1] |= gmmu_pte_vol_true_f();
1732
1733                                 pte->ref_cnt++;
1734
1735                                 gk20a_dbg(gpu_dbg_pte,
1736                                            "pte_cur=%d addr=0x%x,%08x kind=%d"
1737                                            " ctag=%d vol=%d refs=%d"
1738                                            " [0x%08x,0x%08x]",
1739                                            pte_cur, hi32(addr), lo32(addr),
1740                                            kind_v, ctag, !cacheable,
1741                                            pte->ref_cnt, pte_w[1], pte_w[0]);
1742
1743                                 ctag += ctag_incr;
1744                                 cur_offset += page_size;
1745                                 addr += page_size;
1746                                 while (cur_chunk &&
1747                                         cur_offset >= cur_chunk->length) {
1748                                         cur_offset -= cur_chunk->length;
1749                                         cur_chunk = sg_next(cur_chunk);
1750                                 }
1751
1752                         } else {
1753                                 pte->ref_cnt--;
1754                                 gk20a_dbg(gpu_dbg_pte,
1755                                            "pte_cur=%d ref=%d [0x0,0x0]",
1756                                            pte_cur, pte->ref_cnt);
1757                         }
1758
1759                         gk20a_mem_wr32(pte_kv_cur + pte_cur*8, 0, pte_w[0]);
1760                         gk20a_mem_wr32(pte_kv_cur + pte_cur*8, 1, pte_w[1]);
1761                 }
1762
1763                 unmap_gmmu_pages(pte->ref, pte->sgt, pte_kv_cur);
1764
1765                 if (pte->ref_cnt == 0) {
1766                         /* It can make sense to keep around one page table for
1767                          * each flavor (empty)... in case a new map is coming
1768                          * right back to alloc (and fill it in) again.
1769                          * But: deferring unmapping should help with pathologic
1770                          * unmap/map/unmap/map cases where we'd trigger pte
1771                          * free/alloc/free/alloc.
1772                          */
1773                         free_gmmu_pages(vm, pte->ref, pte->sgt,
1774                                 vm->mm->page_table_sizing[pgsz_idx].order,
1775                                 pte->size);
1776                         pte->ref = NULL;
1777
1778                         /* rewrite pde */
1779                         update_gmmu_pde_locked(vm, pde_i);
1780                 }
1781
1782         }
1783
1784         smp_mb();
1785         vm->tlb_dirty = true;
1786         gk20a_dbg_fn("set tlb dirty");
1787
1788         return 0;
1789
1790 clean_up:
1791         /*TBD: potentially rewrite above to pre-map everything it needs to
1792          * as that's the only way it can fail */
1793         return err;
1794
1795 }
1796
1797
1798 /* for gk20a the "video memory" apertures here are misnomers. */
1799 static inline u32 big_valid_pde0_bits(u64 pte_addr)
1800 {
1801         u32 pde0_bits =
1802                 gmmu_pde_aperture_big_video_memory_f() |
1803                 gmmu_pde_address_big_sys_f(
1804                            (u32)(pte_addr >> gmmu_pde_address_shift_v()));
1805         return  pde0_bits;
1806 }
1807 static inline u32 small_valid_pde1_bits(u64 pte_addr)
1808 {
1809         u32 pde1_bits =
1810                 gmmu_pde_aperture_small_video_memory_f() |
1811                 gmmu_pde_vol_small_true_f() | /* tbd: why? */
1812                 gmmu_pde_address_small_sys_f(
1813                            (u32)(pte_addr >> gmmu_pde_address_shift_v()));
1814         return pde1_bits;
1815 }
1816
1817 /* Given the current state of the ptes associated with a pde,
1818    determine value and write it out.  There's no checking
1819    here to determine whether or not a change was actually
1820    made.  So, superfluous updates will cause unnecessary
1821    pde invalidations.
1822 */
1823 static void update_gmmu_pde_locked(struct vm_gk20a *vm, u32 i)
1824 {
1825         bool small_valid, big_valid;
1826         u64 pte_addr[2] = {0, 0};
1827         struct page_table_gk20a *small_pte =
1828                 vm->pdes.ptes[gmmu_page_size_small] + i;
1829         struct page_table_gk20a *big_pte =
1830                 vm->pdes.ptes[gmmu_page_size_big] + i;
1831         u32 pde_v[2] = {0, 0};
1832         u32 *pde;
1833
1834         small_valid = small_pte && small_pte->ref;
1835         big_valid   = big_pte && big_pte->ref;
1836
1837         if (small_valid)
1838                 pte_addr[gmmu_page_size_small] =
1839                         gk20a_mm_iova_addr(small_pte->sgt->sgl);
1840         if (big_valid)
1841                 pte_addr[gmmu_page_size_big] =
1842                         gk20a_mm_iova_addr(big_pte->sgt->sgl);
1843
1844         pde_v[0] = gmmu_pde_size_full_f();
1845         pde_v[0] |= big_valid ?
1846                 big_valid_pde0_bits(pte_addr[gmmu_page_size_big])
1847                 :
1848                 (gmmu_pde_aperture_big_invalid_f());
1849
1850         pde_v[1] |= (small_valid ?
1851                      small_valid_pde1_bits(pte_addr[gmmu_page_size_small])
1852                      :
1853                      (gmmu_pde_aperture_small_invalid_f() |
1854                       gmmu_pde_vol_small_false_f())
1855                      )
1856                 |
1857                 (big_valid ? (gmmu_pde_vol_big_true_f()) :
1858                  gmmu_pde_vol_big_false_f());
1859
1860         pde = pde_from_index(vm, i);
1861
1862         gk20a_mem_wr32(pde, 0, pde_v[0]);
1863         gk20a_mem_wr32(pde, 1, pde_v[1]);
1864
1865         smp_mb();
1866
1867         FLUSH_CPU_DCACHE(pde,
1868                          sg_phys(vm->pdes.sgt->sgl) + (i*gmmu_pde__size_v()),
1869                          sizeof(u32)*2);
1870
1871         gk20a_mm_l2_invalidate(vm->mm->g);
1872
1873         gk20a_dbg(gpu_dbg_pte, "pde:%d = 0x%x,0x%08x\n", i, pde_v[1], pde_v[0]);
1874
1875         vm->tlb_dirty  = true;
1876 }
1877
1878
1879 static int gk20a_vm_put_empty(struct vm_gk20a *vm, u64 vaddr,
1880                                u32 num_pages, u32 pgsz_idx)
1881 {
1882         struct mm_gk20a *mm = vm->mm;
1883         struct gk20a *g = mm->g;
1884         u32 pgsz = gmmu_page_sizes[pgsz_idx];
1885         u32 i;
1886         dma_addr_t iova;
1887
1888         /* allocate the zero page if the va does not already have one */
1889         if (!vm->zero_page_cpuva) {
1890                 int err = 0;
1891                 vm->zero_page_cpuva = dma_alloc_coherent(&g->dev->dev,
1892                                                          mm->big_page_size,
1893                                                          &iova,
1894                                                          GFP_KERNEL);
1895                 if (!vm->zero_page_cpuva) {
1896                         dev_err(&g->dev->dev, "failed to allocate zero page\n");
1897                         return -ENOMEM;
1898                 }
1899
1900                 vm->zero_page_iova = iova;
1901                 err = gk20a_get_sgtable(&g->dev->dev, &vm->zero_page_sgt,
1902                                         vm->zero_page_cpuva, vm->zero_page_iova,
1903                                         mm->big_page_size);
1904                 if (err) {
1905                         dma_free_coherent(&g->dev->dev, mm->big_page_size,
1906                                           vm->zero_page_cpuva,
1907                                           vm->zero_page_iova);
1908                         vm->zero_page_iova = 0;
1909                         vm->zero_page_cpuva = NULL;
1910
1911                         dev_err(&g->dev->dev, "failed to create sg table for zero page\n");
1912                         return -ENOMEM;
1913                 }
1914         }
1915
1916         for (i = 0; i < num_pages; i++) {
1917                 u64 page_vaddr = __locked_gmmu_map(vm, vaddr,
1918                         vm->zero_page_sgt, pgsz, pgsz_idx, 0, 0,
1919                         NVHOST_AS_ALLOC_SPACE_FLAGS_FIXED_OFFSET,
1920                         gk20a_mem_flag_none);
1921
1922                 if (!page_vaddr) {
1923                         gk20a_err(dev_from_vm(vm), "failed to remap clean buffers!");
1924                         goto err_unmap;
1925                 }
1926                 vaddr += pgsz;
1927         }
1928
1929         gk20a_mm_l2_flush(mm->g, true);
1930
1931         return 0;
1932
1933 err_unmap:
1934
1935         WARN_ON(1);
1936         /* something went wrong. unmap pages */
1937         while (i--) {
1938                 vaddr -= pgsz;
1939                 __locked_gmmu_unmap(vm, vaddr, pgsz, pgsz_idx, 0,
1940                                     gk20a_mem_flag_none);
1941         }
1942
1943         return -EINVAL;
1944 }
1945
1946 /* NOTE! mapped_buffers lock must be held */
1947 static void gk20a_vm_unmap_locked(struct mapped_buffer_node *mapped_buffer)
1948 {
1949         struct vm_gk20a *vm = mapped_buffer->vm;
1950
1951         if (mapped_buffer->va_node &&
1952             mapped_buffer->va_node->sparse) {
1953                 u64 vaddr = mapped_buffer->addr;
1954                 u32 pgsz_idx = mapped_buffer->pgsz_idx;
1955                 u32 num_pages = mapped_buffer->size >>
1956                         gmmu_page_shifts[pgsz_idx];
1957
1958                 /* there is little we can do if this fails... */
1959                 gk20a_vm_put_empty(vm, vaddr, num_pages, pgsz_idx);
1960
1961         } else
1962                 __locked_gmmu_unmap(vm,
1963                                 mapped_buffer->addr,
1964                                 mapped_buffer->size,
1965                                 mapped_buffer->pgsz_idx,
1966                                 mapped_buffer->va_allocated,
1967                                 gk20a_mem_flag_none);
1968
1969         gk20a_dbg(gpu_dbg_map, "as=%d pgsz=%d gv=0x%x,%08x own_mem_ref=%d",
1970                    vm_aspace_id(vm), gmmu_page_sizes[mapped_buffer->pgsz_idx],
1971                    hi32(mapped_buffer->addr), lo32(mapped_buffer->addr),
1972                    mapped_buffer->own_mem_ref);
1973
1974         gk20a_mm_unpin(dev_from_vm(vm), mapped_buffer->dmabuf,
1975                        mapped_buffer->sgt);
1976
1977         /* remove from mapped buffer tree and remove list, free */
1978         rb_erase(&mapped_buffer->node, &vm->mapped_buffers);
1979         if (!list_empty(&mapped_buffer->va_buffers_list))
1980                 list_del(&mapped_buffer->va_buffers_list);
1981
1982         /* keep track of mapped buffers */
1983         if (mapped_buffer->user_mapped)
1984                 vm->num_user_mapped_buffers--;
1985
1986         if (mapped_buffer->own_mem_ref)
1987                 dma_buf_put(mapped_buffer->dmabuf);
1988
1989         kfree(mapped_buffer);
1990
1991         return;
1992 }
1993
1994 void gk20a_vm_unmap(struct vm_gk20a *vm, u64 offset)
1995 {
1996         struct device *d = dev_from_vm(vm);
1997         struct mapped_buffer_node *mapped_buffer;
1998
1999         mutex_lock(&vm->update_gmmu_lock);
2000         mapped_buffer = find_mapped_buffer_locked(&vm->mapped_buffers, offset);
2001         if (!mapped_buffer) {
2002                 mutex_unlock(&vm->update_gmmu_lock);
2003                 gk20a_err(d, "invalid addr to unmap 0x%llx", offset);
2004                 return;
2005         }
2006         kref_put(&mapped_buffer->ref, gk20a_vm_unmap_locked_kref);
2007         mutex_unlock(&vm->update_gmmu_lock);
2008 }
2009
2010 static void gk20a_vm_remove_support(struct vm_gk20a *vm)
2011 {
2012         struct gk20a *g = vm->mm->g;
2013         struct mapped_buffer_node *mapped_buffer;
2014         struct vm_reserved_va_node *va_node, *va_node_tmp;
2015         struct rb_node *node;
2016         int i;
2017
2018         gk20a_dbg_fn("");
2019         mutex_lock(&vm->update_gmmu_lock);
2020
2021         /* TBD: add a flag here for the unmap code to recognize teardown
2022          * and short-circuit any otherwise expensive operations. */
2023
2024         node = rb_first(&vm->mapped_buffers);
2025         while (node) {
2026                 mapped_buffer =
2027                         container_of(node, struct mapped_buffer_node, node);
2028                 gk20a_vm_unmap_locked(mapped_buffer);
2029                 node = rb_first(&vm->mapped_buffers);
2030         }
2031
2032         /* destroy remaining reserved memory areas */
2033         list_for_each_entry_safe(va_node, va_node_tmp, &vm->reserved_va_list,
2034                 reserved_va_list) {
2035                 list_del(&va_node->reserved_va_list);
2036                 kfree(va_node);
2037         }
2038
2039         /* unmapping all buffers above may not actually free
2040          * all vm ptes.  jettison them here for certain... */
2041         for (i = 0; i < vm->pdes.num_pdes; i++) {
2042                 struct page_table_gk20a *pte =
2043                         &vm->pdes.ptes[gmmu_page_size_small][i];
2044                 if (pte->ref) {
2045                         free_gmmu_pages(vm, pte->ref, pte->sgt,
2046                                 vm->mm->page_table_sizing[gmmu_page_size_small].order,
2047                                 pte->size);
2048                         pte->ref = NULL;
2049                 }
2050                 pte = &vm->pdes.ptes[gmmu_page_size_big][i];
2051                 if (pte->ref) {
2052                         free_gmmu_pages(vm, pte->ref, pte->sgt,
2053                                 vm->mm->page_table_sizing[gmmu_page_size_big].order,
2054                                 pte->size);
2055                         pte->ref = NULL;
2056                 }
2057         }
2058
2059         unmap_gmmu_pages(vm->pdes.ref, vm->pdes.sgt, vm->pdes.kv);
2060         free_gmmu_pages(vm, vm->pdes.ref, vm->pdes.sgt, 0, vm->pdes.size);
2061
2062         kfree(vm->pdes.ptes[gmmu_page_size_small]);
2063         kfree(vm->pdes.ptes[gmmu_page_size_big]);
2064         gk20a_allocator_destroy(&vm->vma[gmmu_page_size_small]);
2065         gk20a_allocator_destroy(&vm->vma[gmmu_page_size_big]);
2066
2067         mutex_unlock(&vm->update_gmmu_lock);
2068
2069         /* release zero page if used */
2070         if (vm->zero_page_cpuva)
2071                 dma_free_coherent(&g->dev->dev, vm->mm->big_page_size,
2072                                   vm->zero_page_cpuva, vm->zero_page_iova);
2073
2074         /* vm is not used anymore. release it. */
2075         kfree(vm);
2076 }
2077
2078 static void gk20a_vm_remove_support_kref(struct kref *ref)
2079 {
2080         struct vm_gk20a *vm = container_of(ref, struct vm_gk20a, ref);
2081         gk20a_vm_remove_support(vm);
2082 }
2083
2084 void gk20a_vm_get(struct vm_gk20a *vm)
2085 {
2086         kref_get(&vm->ref);
2087 }
2088
2089 void gk20a_vm_put(struct vm_gk20a *vm)
2090 {
2091         kref_put(&vm->ref, gk20a_vm_remove_support_kref);
2092 }
2093
2094 /* address space interfaces for the gk20a module */
2095 int gk20a_vm_alloc_share(struct gk20a_as_share *as_share)
2096 {
2097         struct gk20a_as *as = as_share->as;
2098         struct gk20a *g = gk20a_from_as(as);
2099         struct mm_gk20a *mm = &g->mm;
2100         struct vm_gk20a *vm;
2101         u64 vma_size;
2102         u32 num_pages, low_hole_pages;
2103         char name[32];
2104         int err;
2105
2106         gk20a_dbg_fn("");
2107
2108         vm = kzalloc(sizeof(*vm), GFP_KERNEL);
2109         if (!vm)
2110                 return -ENOMEM;
2111
2112         as_share->vm = vm;
2113
2114         vm->mm = mm;
2115         vm->as_share = as_share;
2116
2117         vm->big_pages = true;
2118
2119         vm->va_start  = mm->pde_stride;   /* create a one pde hole */
2120         vm->va_limit  = mm->channel.size; /* note this means channel.size is
2121                                              really just the max */
2122         {
2123                 u32 pde_lo, pde_hi;
2124                 pde_range_from_vaddr_range(vm,
2125                                            0, vm->va_limit-1,
2126                                            &pde_lo, &pde_hi);
2127                 vm->pdes.num_pdes = pde_hi + 1;
2128         }
2129
2130         vm->pdes.ptes[gmmu_page_size_small] =
2131                 kzalloc(sizeof(struct page_table_gk20a) *
2132                         vm->pdes.num_pdes, GFP_KERNEL);
2133
2134         vm->pdes.ptes[gmmu_page_size_big] =
2135                 kzalloc(sizeof(struct page_table_gk20a) *
2136                         vm->pdes.num_pdes, GFP_KERNEL);
2137
2138         if (!(vm->pdes.ptes[gmmu_page_size_small] &&
2139               vm->pdes.ptes[gmmu_page_size_big]))
2140                 return -ENOMEM;
2141
2142         gk20a_dbg_info("init space for va_limit=0x%llx num_pdes=%d",
2143                    vm->va_limit, vm->pdes.num_pdes);
2144
2145         /* allocate the page table directory */
2146         err = alloc_gmmu_pages(vm, 0, &vm->pdes.ref,
2147                                &vm->pdes.sgt, &vm->pdes.size);
2148         if (err)
2149                 return -ENOMEM;
2150
2151         err = map_gmmu_pages(vm->pdes.ref, vm->pdes.sgt, &vm->pdes.kv,
2152                              vm->pdes.size);
2153         if (err) {
2154                 free_gmmu_pages(vm, vm->pdes.ref, vm->pdes.sgt, 0,
2155                                         vm->pdes.size);
2156                 return -ENOMEM;
2157         }
2158         gk20a_dbg(gpu_dbg_pte, "pdes.kv = 0x%p, pdes.phys = 0x%llx",
2159                         vm->pdes.kv,
2160                         gk20a_mm_iova_addr(vm->pdes.sgt->sgl));
2161         /* we could release vm->pdes.kv but it's only one page... */
2162
2163
2164         /* low-half: alloc small pages */
2165         /* high-half: alloc big pages */
2166         vma_size = mm->channel.size >> 1;
2167
2168         snprintf(name, sizeof(name), "gk20a_as_%d-%dKB", as_share->id,
2169                  gmmu_page_sizes[gmmu_page_size_small]>>10);
2170         num_pages = (u32)(vma_size >> gmmu_page_shifts[gmmu_page_size_small]);
2171
2172         /* num_pages above is without regard to the low-side hole. */
2173         low_hole_pages = (vm->va_start >>
2174                           gmmu_page_shifts[gmmu_page_size_small]);
2175
2176         gk20a_allocator_init(&vm->vma[gmmu_page_size_small], name,
2177               low_hole_pages,             /* start */
2178               num_pages - low_hole_pages, /* length */
2179               1);                         /* align */
2180
2181         snprintf(name, sizeof(name), "gk20a_as_%d-%dKB", as_share->id,
2182                  gmmu_page_sizes[gmmu_page_size_big]>>10);
2183
2184         num_pages = (u32)(vma_size >> gmmu_page_shifts[gmmu_page_size_big]);
2185         gk20a_allocator_init(&vm->vma[gmmu_page_size_big], name,
2186                               num_pages, /* start */
2187                               num_pages, /* length */
2188                               1); /* align */
2189
2190         vm->mapped_buffers = RB_ROOT;
2191
2192         mutex_init(&vm->update_gmmu_lock);
2193         kref_init(&vm->ref);
2194         INIT_LIST_HEAD(&vm->reserved_va_list);
2195
2196         vm->enable_ctag = true;
2197
2198         return 0;
2199 }
2200
2201
2202 int gk20a_vm_release_share(struct gk20a_as_share *as_share)
2203 {
2204         struct vm_gk20a *vm = as_share->vm;
2205
2206         gk20a_dbg_fn("");
2207
2208         vm->as_share = NULL;
2209
2210         /* put as reference to vm */
2211         gk20a_vm_put(vm);
2212
2213         as_share->vm = NULL;
2214
2215         return 0;
2216 }
2217
2218
2219 int gk20a_vm_alloc_space(struct gk20a_as_share *as_share,
2220                          struct nvhost_as_alloc_space_args *args)
2221
2222 {       int err = -ENOMEM;
2223         int pgsz_idx;
2224         u32 start_page_nr;
2225         struct gk20a_allocator *vma;
2226         struct vm_gk20a *vm = as_share->vm;
2227         struct vm_reserved_va_node *va_node;
2228         u64 vaddr_start = 0;
2229
2230         gk20a_dbg_fn("flags=0x%x pgsz=0x%x nr_pages=0x%x o/a=0x%llx",
2231                         args->flags, args->page_size, args->pages,
2232                         args->o_a.offset);
2233
2234         /* determine pagesz idx */
2235         for (pgsz_idx = gmmu_page_size_small;
2236              pgsz_idx < gmmu_nr_page_sizes;
2237              pgsz_idx++) {
2238                 if (gmmu_page_sizes[pgsz_idx] == args->page_size)
2239                         break;
2240         }
2241
2242         if (pgsz_idx >= gmmu_nr_page_sizes) {
2243                 err = -EINVAL;
2244                 goto clean_up;
2245         }
2246
2247         va_node = kzalloc(sizeof(*va_node), GFP_KERNEL);
2248         if (!va_node) {
2249                 err = -ENOMEM;
2250                 goto clean_up;
2251         }
2252
2253         if (args->flags & NVHOST_AS_ALLOC_SPACE_FLAGS_SPARSE &&
2254             pgsz_idx != gmmu_page_size_big) {
2255                 err = -ENOSYS;
2256                 kfree(va_node);
2257                 goto clean_up;
2258         }
2259
2260         start_page_nr = 0;
2261         if (args->flags & NVHOST_AS_ALLOC_SPACE_FLAGS_FIXED_OFFSET)
2262                 start_page_nr = (u32)(args->o_a.offset >>
2263                                       gmmu_page_shifts[pgsz_idx]);
2264
2265         vma = &vm->vma[pgsz_idx];
2266         err = vma->alloc(vma, &start_page_nr, args->pages);
2267         if (err) {
2268                 kfree(va_node);
2269                 goto clean_up;
2270         }
2271
2272         vaddr_start = (u64)start_page_nr << gmmu_page_shifts[pgsz_idx];
2273
2274         va_node->vaddr_start = vaddr_start;
2275         va_node->size = (u64)args->page_size * (u64)args->pages;
2276         va_node->pgsz_idx = args->page_size;
2277         INIT_LIST_HEAD(&va_node->va_buffers_list);
2278         INIT_LIST_HEAD(&va_node->reserved_va_list);
2279
2280         mutex_lock(&vm->update_gmmu_lock);
2281
2282         /* mark that we need to use sparse mappings here */
2283         if (args->flags & NVHOST_AS_ALLOC_SPACE_FLAGS_SPARSE) {
2284                 err = gk20a_vm_put_empty(vm, vaddr_start, args->pages,
2285                                          pgsz_idx);
2286                 if (err) {
2287                         mutex_unlock(&vm->update_gmmu_lock);
2288                         vma->free(vma, start_page_nr, args->pages);
2289                         kfree(va_node);
2290                         goto clean_up;
2291                 }
2292
2293                 va_node->sparse = true;
2294         }
2295
2296         list_add_tail(&va_node->reserved_va_list, &vm->reserved_va_list);
2297
2298         mutex_unlock(&vm->update_gmmu_lock);
2299
2300         args->o_a.offset = vaddr_start;
2301
2302 clean_up:
2303         return err;
2304 }
2305
2306 int gk20a_vm_free_space(struct gk20a_as_share *as_share,
2307                         struct nvhost_as_free_space_args *args)
2308 {
2309         int err = -ENOMEM;
2310         int pgsz_idx;
2311         u32 start_page_nr;
2312         struct gk20a_allocator *vma;
2313         struct vm_gk20a *vm = as_share->vm;
2314         struct vm_reserved_va_node *va_node;
2315
2316         gk20a_dbg_fn("pgsz=0x%x nr_pages=0x%x o/a=0x%llx", args->page_size,
2317                         args->pages, args->offset);
2318
2319         /* determine pagesz idx */
2320         for (pgsz_idx = gmmu_page_size_small;
2321              pgsz_idx < gmmu_nr_page_sizes;
2322              pgsz_idx++) {
2323                 if (gmmu_page_sizes[pgsz_idx] == args->page_size)
2324                         break;
2325         }
2326
2327         if (pgsz_idx >= gmmu_nr_page_sizes) {
2328                 err = -EINVAL;
2329                 goto clean_up;
2330         }
2331
2332         start_page_nr = (u32)(args->offset >>
2333                               gmmu_page_shifts[pgsz_idx]);
2334
2335         vma = &vm->vma[pgsz_idx];
2336         err = vma->free(vma, start_page_nr, args->pages);
2337
2338         if (err)
2339                 goto clean_up;
2340
2341         mutex_lock(&vm->update_gmmu_lock);
2342         va_node = addr_to_reservation(vm, args->offset);
2343         if (va_node) {
2344                 struct mapped_buffer_node *buffer;
2345
2346                 /* there is no need to unallocate the buffers in va. Just
2347                  * convert them into normal buffers */
2348
2349                 list_for_each_entry(buffer,
2350                         &va_node->va_buffers_list, va_buffers_list)
2351                         list_del_init(&buffer->va_buffers_list);
2352
2353                 list_del(&va_node->reserved_va_list);
2354
2355                 /* if this was a sparse mapping, free the va */
2356                 if (va_node->sparse)
2357                         __locked_gmmu_unmap(vm,
2358                                 va_node->vaddr_start,
2359                                 va_node->size,
2360                                 va_node->pgsz_idx,
2361                                 false,
2362                                 gk20a_mem_flag_none);
2363                 kfree(va_node);
2364         }
2365         mutex_unlock(&vm->update_gmmu_lock);
2366
2367 clean_up:
2368         return err;
2369 }
2370
2371 int gk20a_vm_bind_channel(struct gk20a_as_share *as_share,
2372                           struct channel_gk20a *ch)
2373 {
2374         int err = 0;
2375         struct vm_gk20a *vm = as_share->vm;
2376
2377         gk20a_dbg_fn("");
2378
2379         ch->vm = vm;
2380         err = channel_gk20a_commit_va(ch);
2381         if (err)
2382                 ch->vm = 0;
2383
2384         return err;
2385 }
2386
2387 int gk20a_dmabuf_alloc_drvdata(struct dma_buf *dmabuf, struct device *dev)
2388 {
2389         struct gk20a_dmabuf_priv *priv;
2390         static DEFINE_MUTEX(priv_lock);
2391
2392         priv = dma_buf_get_drvdata(dmabuf, dev);
2393         if (likely(priv))
2394                 return 0;
2395
2396         mutex_lock(&priv_lock);
2397         priv = dma_buf_get_drvdata(dmabuf, dev);
2398         if (priv)
2399                 goto priv_exist_or_err;
2400         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
2401         if (!priv) {
2402                 priv = ERR_PTR(-ENOMEM);
2403                 goto priv_exist_or_err;
2404         }
2405         mutex_init(&priv->lock);
2406         dma_buf_set_drvdata(dmabuf, dev, priv, gk20a_mm_delete_priv);
2407 priv_exist_or_err:
2408         mutex_unlock(&priv_lock);
2409         if (IS_ERR(priv))
2410                 return -ENOMEM;
2411
2412         return 0;
2413 }
2414
2415
2416 static int gk20a_dmabuf_get_kind(struct dma_buf *dmabuf)
2417 {
2418         int kind = 0;
2419 #ifdef CONFIG_TEGRA_NVMAP
2420         int err;
2421         u64 nvmap_param;
2422
2423         err = nvmap_get_dmabuf_param(dmabuf, NVMAP_HANDLE_PARAM_KIND,
2424                                      &nvmap_param);
2425         kind = err ? kind : nvmap_param;
2426 #endif
2427         return kind;
2428 }
2429
2430 int gk20a_vm_map_buffer(struct gk20a_as_share *as_share,
2431                         int dmabuf_fd,
2432                         u64 *offset_align,
2433                         u32 flags, /*NVHOST_AS_MAP_BUFFER_FLAGS_*/
2434                         int kind)
2435 {
2436         int err = 0;
2437         struct vm_gk20a *vm = as_share->vm;
2438         struct dma_buf *dmabuf;
2439         u64 ret_va;
2440
2441         gk20a_dbg_fn("");
2442
2443         /* get ref to the mem handle (released on unmap_locked) */
2444         dmabuf = dma_buf_get(dmabuf_fd);
2445         if (!dmabuf)
2446                 return 0;
2447
2448         err = gk20a_dmabuf_alloc_drvdata(dmabuf, dev_from_vm(vm));
2449         if (err) {
2450                 dma_buf_put(dmabuf);
2451                 return err;
2452         }
2453
2454         if (kind == -1)
2455                 kind = gk20a_dmabuf_get_kind(dmabuf);
2456
2457         ret_va = gk20a_vm_map(vm, dmabuf, *offset_align,
2458                         flags, kind, NULL, true,
2459                         gk20a_mem_flag_none);
2460         *offset_align = ret_va;
2461         if (!ret_va) {
2462                 dma_buf_put(dmabuf);
2463                 err = -EINVAL;
2464         }
2465
2466         return err;
2467 }
2468
2469 int gk20a_vm_unmap_buffer(struct gk20a_as_share *as_share, u64 offset)
2470 {
2471         struct vm_gk20a *vm = as_share->vm;
2472
2473         gk20a_dbg_fn("");
2474
2475         gk20a_vm_unmap_user(vm, offset);
2476         return 0;
2477 }
2478
2479 int gk20a_init_bar1_vm(struct mm_gk20a *mm)
2480 {
2481         int err;
2482         phys_addr_t inst_pa;
2483         void *inst_ptr;
2484         struct vm_gk20a *vm = &mm->bar1.vm;
2485         struct gk20a *g = gk20a_from_mm(mm);
2486         struct device *d = dev_from_gk20a(g);
2487         struct inst_desc *inst_block = &mm->bar1.inst_block;
2488         u64 pde_addr;
2489         u32 pde_addr_lo;
2490         u32 pde_addr_hi;
2491         dma_addr_t iova;
2492
2493         vm->mm = mm;
2494
2495         mm->bar1.aperture_size = bar1_aperture_size_mb_gk20a() << 20;
2496
2497         gk20a_dbg_info("bar1 vm size = 0x%x", mm->bar1.aperture_size);
2498
2499         vm->va_start = mm->pde_stride * 1;
2500         vm->va_limit = mm->bar1.aperture_size;
2501
2502         {
2503                 u32 pde_lo, pde_hi;
2504                 pde_range_from_vaddr_range(vm,
2505                                            0, vm->va_limit-1,
2506                                            &pde_lo, &pde_hi);
2507                 vm->pdes.num_pdes = pde_hi + 1;
2508         }
2509
2510         /* bar1 is likely only to ever use/need small page sizes. */
2511         /* But just in case, for now... arrange for both.*/
2512         vm->pdes.ptes[gmmu_page_size_small] =
2513                 kzalloc(sizeof(struct page_table_gk20a) *
2514                         vm->pdes.num_pdes, GFP_KERNEL);
2515
2516         vm->pdes.ptes[gmmu_page_size_big] =
2517                 kzalloc(sizeof(struct page_table_gk20a) *
2518                         vm->pdes.num_pdes, GFP_KERNEL);
2519
2520         if (!(vm->pdes.ptes[gmmu_page_size_small] &&
2521               vm->pdes.ptes[gmmu_page_size_big]))
2522                 return -ENOMEM;
2523
2524         gk20a_dbg_info("init space for bar1 va_limit=0x%llx num_pdes=%d",
2525                    vm->va_limit, vm->pdes.num_pdes);
2526
2527
2528         /* allocate the page table directory */
2529         err = alloc_gmmu_pages(vm, 0, &vm->pdes.ref,
2530                                &vm->pdes.sgt, &vm->pdes.size);
2531         if (err)
2532                 goto clean_up;
2533
2534         err = map_gmmu_pages(vm->pdes.ref, vm->pdes.sgt, &vm->pdes.kv,
2535                              vm->pdes.size);
2536         if (err) {
2537                 free_gmmu_pages(vm, vm->pdes.ref, vm->pdes.sgt, 0,
2538                                         vm->pdes.size);
2539                 goto clean_up;
2540         }
2541         gk20a_dbg(gpu_dbg_pte, "bar 1 pdes.kv = 0x%p, pdes.phys = 0x%llx",
2542                         vm->pdes.kv, gk20a_mm_iova_addr(vm->pdes.sgt->sgl));
2543         /* we could release vm->pdes.kv but it's only one page... */
2544
2545         pde_addr = gk20a_mm_iova_addr(vm->pdes.sgt->sgl);
2546         pde_addr_lo = u64_lo32(pde_addr >> 12);
2547         pde_addr_hi = u64_hi32(pde_addr);
2548
2549         gk20a_dbg_info("pde pa=0x%llx pde_addr_lo=0x%x pde_addr_hi=0x%x",
2550                 (u64)gk20a_mm_iova_addr(vm->pdes.sgt->sgl),
2551                 pde_addr_lo, pde_addr_hi);
2552
2553         /* allocate instance mem for bar1 */
2554         inst_block->size = ram_in_alloc_size_v();
2555         inst_block->cpuva = dma_alloc_coherent(d, inst_block->size,
2556                                 &iova, GFP_KERNEL);
2557         if (!inst_block->cpuva) {
2558                 gk20a_err(d, "%s: memory allocation failed\n", __func__);
2559                 err = -ENOMEM;
2560                 goto clean_up;
2561         }
2562
2563         inst_block->iova = iova;
2564         inst_block->cpu_pa = gk20a_get_phys_from_iova(d, inst_block->iova);
2565         if (!inst_block->cpu_pa) {
2566                 gk20a_err(d, "%s: failed to get phys address\n", __func__);
2567                 err = -ENOMEM;
2568                 goto clean_up;
2569         }
2570
2571         inst_pa = inst_block->cpu_pa;
2572         inst_ptr = inst_block->cpuva;
2573
2574         gk20a_dbg_info("bar1 inst block physical phys = 0x%llx, kv = 0x%p",
2575                 (u64)inst_pa, inst_ptr);
2576
2577         memset(inst_ptr, 0, ram_fc_size_val_v());
2578
2579         gk20a_mem_wr32(inst_ptr, ram_in_page_dir_base_lo_w(),
2580                 ram_in_page_dir_base_target_vid_mem_f() |
2581                 ram_in_page_dir_base_vol_true_f() |
2582                 ram_in_page_dir_base_lo_f(pde_addr_lo));
2583
2584         gk20a_mem_wr32(inst_ptr, ram_in_page_dir_base_hi_w(),
2585                 ram_in_page_dir_base_hi_f(pde_addr_hi));
2586
2587         gk20a_mem_wr32(inst_ptr, ram_in_adr_limit_lo_w(),
2588                  u64_lo32(vm->va_limit) | 0xFFF);
2589
2590         gk20a_mem_wr32(inst_ptr, ram_in_adr_limit_hi_w(),
2591                 ram_in_adr_limit_hi_f(u64_hi32(vm->va_limit)));
2592
2593         gk20a_dbg_info("bar1 inst block ptr: %08llx",  (u64)inst_pa);
2594         gk20a_allocator_init(&vm->vma[gmmu_page_size_small], "gk20a_bar1",
2595                               1,/*start*/
2596                               (vm->va_limit >> 12) - 1 /* length*/,
2597                               1); /* align */
2598         /* initialize just in case we try to use it anyway */
2599         gk20a_allocator_init(&vm->vma[gmmu_page_size_big], "gk20a_bar1-unused",
2600                               0x0badc0de, /* start */
2601                               1, /* length */
2602                               1); /* align */
2603
2604         vm->mapped_buffers = RB_ROOT;
2605
2606         mutex_init(&vm->update_gmmu_lock);
2607         kref_init(&vm->ref);
2608         INIT_LIST_HEAD(&vm->reserved_va_list);
2609
2610         return 0;
2611
2612 clean_up:
2613         /* free, etc */
2614         if (inst_block->cpuva)
2615                 dma_free_coherent(d, inst_block->size,
2616                         inst_block->cpuva, inst_block->iova);
2617         inst_block->cpuva = NULL;
2618         inst_block->iova = 0;
2619         return err;
2620 }
2621
2622 /* pmu vm, share channel_vm interfaces */
2623 int gk20a_init_pmu_vm(struct mm_gk20a *mm)
2624 {
2625         int err;
2626         phys_addr_t inst_pa;
2627         void *inst_ptr;
2628         struct vm_gk20a *vm = &mm->pmu.vm;
2629         struct gk20a *g = gk20a_from_mm(mm);
2630         struct device *d = dev_from_gk20a(g);
2631         struct inst_desc *inst_block = &mm->pmu.inst_block;
2632         u64 pde_addr;
2633         u32 pde_addr_lo;
2634         u32 pde_addr_hi;
2635         dma_addr_t iova;
2636
2637         vm->mm = mm;
2638
2639         mm->pmu.aperture_size = GK20A_PMU_VA_SIZE;
2640
2641         gk20a_dbg_info("pmu vm size = 0x%x", mm->pmu.aperture_size);
2642
2643         vm->va_start  = GK20A_PMU_VA_START;
2644         vm->va_limit  = vm->va_start + mm->pmu.aperture_size;
2645
2646         {
2647                 u32 pde_lo, pde_hi;
2648                 pde_range_from_vaddr_range(vm,
2649                                            0, vm->va_limit-1,
2650                                            &pde_lo, &pde_hi);
2651                 vm->pdes.num_pdes = pde_hi + 1;
2652         }
2653
2654         /* The pmu is likely only to ever use/need small page sizes. */
2655         /* But just in case, for now... arrange for both.*/
2656         vm->pdes.ptes[gmmu_page_size_small] =
2657                 kzalloc(sizeof(struct page_table_gk20a) *
2658                         vm->pdes.num_pdes, GFP_KERNEL);
2659
2660         vm->pdes.ptes[gmmu_page_size_big] =
2661                 kzalloc(sizeof(struct page_table_gk20a) *
2662                         vm->pdes.num_pdes, GFP_KERNEL);
2663
2664         if (!(vm->pdes.ptes[gmmu_page_size_small] &&
2665               vm->pdes.ptes[gmmu_page_size_big]))
2666                 return -ENOMEM;
2667
2668         gk20a_dbg_info("init space for pmu va_limit=0x%llx num_pdes=%d",
2669                    vm->va_limit, vm->pdes.num_pdes);
2670
2671         /* allocate the page table directory */
2672         err = alloc_gmmu_pages(vm, 0, &vm->pdes.ref,
2673                                &vm->pdes.sgt, &vm->pdes.size);
2674         if (err)
2675                 goto clean_up;
2676
2677         err = map_gmmu_pages(vm->pdes.ref, vm->pdes.sgt, &vm->pdes.kv,
2678                              vm->pdes.size);
2679         if (err) {
2680                 free_gmmu_pages(vm, vm->pdes.ref, vm->pdes.sgt, 0,
2681                                         vm->pdes.size);
2682                 goto clean_up;
2683         }
2684         gk20a_dbg_info("pmu pdes phys @ 0x%llx",
2685                         (u64)gk20a_mm_iova_addr(vm->pdes.sgt->sgl));
2686         /* we could release vm->pdes.kv but it's only one page... */
2687
2688         pde_addr = gk20a_mm_iova_addr(vm->pdes.sgt->sgl);
2689         pde_addr_lo = u64_lo32(pde_addr >> 12);
2690         pde_addr_hi = u64_hi32(pde_addr);
2691
2692         gk20a_dbg_info("pde pa=0x%llx pde_addr_lo=0x%x pde_addr_hi=0x%x",
2693                         (u64)pde_addr, pde_addr_lo, pde_addr_hi);
2694
2695         /* allocate instance mem for pmu */
2696         inst_block->size = GK20A_PMU_INST_SIZE;
2697         inst_block->cpuva = dma_alloc_coherent(d, inst_block->size,
2698                                 &iova, GFP_KERNEL);
2699         if (!inst_block->cpuva) {
2700                 gk20a_err(d, "%s: memory allocation failed\n", __func__);
2701                 err = -ENOMEM;
2702                 goto clean_up;
2703         }
2704
2705         inst_block->iova = iova;
2706         inst_block->cpu_pa = gk20a_get_phys_from_iova(d, inst_block->iova);
2707         if (!inst_block->cpu_pa) {
2708                 gk20a_err(d, "%s: failed to get phys address\n", __func__);
2709                 err = -ENOMEM;
2710                 goto clean_up;
2711         }
2712
2713         inst_pa = inst_block->cpu_pa;
2714         inst_ptr = inst_block->cpuva;
2715
2716         gk20a_dbg_info("pmu inst block physical addr: 0x%llx", (u64)inst_pa);
2717
2718         memset(inst_ptr, 0, GK20A_PMU_INST_SIZE);
2719
2720         gk20a_mem_wr32(inst_ptr, ram_in_page_dir_base_lo_w(),
2721                 ram_in_page_dir_base_target_vid_mem_f() |
2722                 ram_in_page_dir_base_vol_true_f() |
2723                 ram_in_page_dir_base_lo_f(pde_addr_lo));
2724
2725         gk20a_mem_wr32(inst_ptr, ram_in_page_dir_base_hi_w(),
2726                 ram_in_page_dir_base_hi_f(pde_addr_hi));
2727
2728         gk20a_mem_wr32(inst_ptr, ram_in_adr_limit_lo_w(),
2729                  u64_lo32(vm->va_limit) | 0xFFF);
2730
2731         gk20a_mem_wr32(inst_ptr, ram_in_adr_limit_hi_w(),
2732                 ram_in_adr_limit_hi_f(u64_hi32(vm->va_limit)));
2733
2734         gk20a_allocator_init(&vm->vma[gmmu_page_size_small], "gk20a_pmu",
2735                               (vm->va_start >> 12), /* start */
2736                               (vm->va_limit - vm->va_start) >> 12, /*length*/
2737                               1); /* align */
2738         /* initialize just in case we try to use it anyway */
2739         gk20a_allocator_init(&vm->vma[gmmu_page_size_big], "gk20a_pmu-unused",
2740                               0x0badc0de, /* start */
2741                               1, /* length */
2742                               1); /* align */
2743
2744
2745         vm->mapped_buffers = RB_ROOT;
2746
2747         mutex_init(&vm->update_gmmu_lock);
2748         kref_init(&vm->ref);
2749         INIT_LIST_HEAD(&vm->reserved_va_list);
2750
2751         return 0;
2752
2753 clean_up:
2754         /* free, etc */
2755         if (inst_block->cpuva)
2756                 dma_free_coherent(d, inst_block->size,
2757                         inst_block->cpuva, inst_block->iova);
2758         inst_block->cpuva = NULL;
2759         inst_block->iova = 0;
2760         return err;
2761 }
2762
2763 int gk20a_mm_fb_flush(struct gk20a *g)
2764 {
2765         struct mm_gk20a *mm = &g->mm;
2766         u32 data;
2767         s32 retry = 100;
2768         int ret = 0;
2769
2770         gk20a_dbg_fn("");
2771
2772         mutex_lock(&mm->l2_op_lock);
2773
2774         g->ops.ltc.elpg_flush(g);
2775
2776         /* Make sure all previous writes are committed to the L2. There's no
2777            guarantee that writes are to DRAM. This will be a sysmembar internal
2778            to the L2. */
2779         gk20a_writel(g, flush_fb_flush_r(),
2780                 flush_fb_flush_pending_busy_f());
2781
2782         do {
2783                 data = gk20a_readl(g, flush_fb_flush_r());
2784
2785                 if (flush_fb_flush_outstanding_v(data) ==
2786                         flush_fb_flush_outstanding_true_v() ||
2787                     flush_fb_flush_pending_v(data) ==
2788                         flush_fb_flush_pending_busy_v()) {
2789                                 gk20a_dbg_info("fb_flush 0x%x", data);
2790                                 retry--;
2791                                 usleep_range(20, 40);
2792                 } else
2793                         break;
2794         } while (retry >= 0 || !tegra_platform_is_silicon());
2795
2796         if (retry < 0) {
2797                 gk20a_warn(dev_from_gk20a(g),
2798                         "fb_flush too many retries");
2799                 ret = -EBUSY;
2800         }
2801
2802         mutex_unlock(&mm->l2_op_lock);
2803
2804         return ret;
2805 }
2806
2807 static void gk20a_mm_l2_invalidate_locked(struct gk20a *g)
2808 {
2809         u32 data;
2810         s32 retry = 200;
2811
2812         /* Invalidate any clean lines from the L2 so subsequent reads go to
2813            DRAM. Dirty lines are not affected by this operation. */
2814         gk20a_writel(g, flush_l2_system_invalidate_r(),
2815                 flush_l2_system_invalidate_pending_busy_f());
2816
2817         do {
2818                 data = gk20a_readl(g, flush_l2_system_invalidate_r());
2819
2820                 if (flush_l2_system_invalidate_outstanding_v(data) ==
2821                         flush_l2_system_invalidate_outstanding_true_v() ||
2822                     flush_l2_system_invalidate_pending_v(data) ==
2823                         flush_l2_system_invalidate_pending_busy_v()) {
2824                                 gk20a_dbg_info("l2_system_invalidate 0x%x",
2825                                                 data);
2826                                 retry--;
2827                                 usleep_range(20, 40);
2828                 } else
2829                         break;
2830         } while (retry >= 0 || !tegra_platform_is_silicon());
2831
2832         if (retry < 0)
2833                 gk20a_warn(dev_from_gk20a(g),
2834                         "l2_system_invalidate too many retries");
2835 }
2836
2837 void gk20a_mm_l2_invalidate(struct gk20a *g)
2838 {
2839         struct mm_gk20a *mm = &g->mm;
2840         mutex_lock(&mm->l2_op_lock);
2841         gk20a_mm_l2_invalidate_locked(g);
2842         mutex_unlock(&mm->l2_op_lock);
2843 }
2844
2845 void gk20a_mm_l2_flush(struct gk20a *g, bool invalidate)
2846 {
2847         struct mm_gk20a *mm = &g->mm;
2848         u32 data;
2849         s32 retry = 200;
2850
2851         gk20a_dbg_fn("");
2852
2853         mutex_lock(&mm->l2_op_lock);
2854
2855         /* Flush all dirty lines from the L2 to DRAM. Lines are left in the L2
2856            as clean, so subsequent reads might hit in the L2. */
2857         gk20a_writel(g, flush_l2_flush_dirty_r(),
2858                 flush_l2_flush_dirty_pending_busy_f());
2859
2860         do {
2861                 data = gk20a_readl(g, flush_l2_flush_dirty_r());
2862
2863                 if (flush_l2_flush_dirty_outstanding_v(data) ==
2864                         flush_l2_flush_dirty_outstanding_true_v() ||
2865                     flush_l2_flush_dirty_pending_v(data) ==
2866                         flush_l2_flush_dirty_pending_busy_v()) {
2867                                 gk20a_dbg_info("l2_flush_dirty 0x%x", data);
2868                                 retry--;
2869                                 usleep_range(20, 40);
2870                 } else
2871                         break;
2872         } while (retry >= 0 || !tegra_platform_is_silicon());
2873
2874         if (retry < 0)
2875                 gk20a_warn(dev_from_gk20a(g),
2876                         "l2_flush_dirty too many retries");
2877
2878         if (invalidate)
2879                 gk20a_mm_l2_invalidate_locked(g);
2880
2881         mutex_unlock(&mm->l2_op_lock);
2882 }
2883
2884
2885 int gk20a_vm_find_buffer(struct vm_gk20a *vm, u64 gpu_va,
2886                          struct dma_buf **dmabuf,
2887                          u64 *offset)
2888 {
2889         struct mapped_buffer_node *mapped_buffer;
2890
2891         gk20a_dbg_fn("gpu_va=0x%llx", gpu_va);
2892
2893         mutex_lock(&vm->update_gmmu_lock);
2894
2895         mapped_buffer = find_mapped_buffer_range_locked(&vm->mapped_buffers,
2896                                                         gpu_va);
2897         if (!mapped_buffer) {
2898                 mutex_unlock(&vm->update_gmmu_lock);
2899                 return -EINVAL;
2900         }
2901
2902         *dmabuf = mapped_buffer->dmabuf;
2903         *offset = gpu_va - mapped_buffer->addr;
2904
2905         mutex_unlock(&vm->update_gmmu_lock);
2906
2907         return 0;
2908 }
2909
2910 void gk20a_mm_tlb_invalidate(struct vm_gk20a *vm)
2911 {
2912         struct gk20a *g = gk20a_from_vm(vm);
2913         u32 addr_lo = u64_lo32(gk20a_mm_iova_addr(vm->pdes.sgt->sgl) >> 12);
2914         u32 data;
2915         s32 retry = 200;
2916         static DEFINE_MUTEX(tlb_lock);
2917
2918         gk20a_dbg_fn("");
2919
2920         /* pagetables are considered sw states which are preserved after
2921            prepare_poweroff. When gk20a deinit releases those pagetables,
2922            common code in vm unmap path calls tlb invalidate that touches
2923            hw. Use the power_on flag to skip tlb invalidation when gpu
2924            power is turned off */
2925
2926         if (!g->power_on)
2927                 return;
2928
2929         /* No need to invalidate if tlb is clean */
2930         mutex_lock(&vm->update_gmmu_lock);
2931         if (!vm->tlb_dirty) {
2932                 mutex_unlock(&vm->update_gmmu_lock);
2933                 return;
2934         }
2935
2936         mutex_lock(&tlb_lock);
2937         do {
2938                 data = gk20a_readl(g, fb_mmu_ctrl_r());
2939                 if (fb_mmu_ctrl_pri_fifo_space_v(data) != 0)
2940                         break;
2941                 usleep_range(20, 40);
2942                 retry--;
2943         } while (retry >= 0 || !tegra_platform_is_silicon());
2944
2945         if (retry < 0) {
2946                 gk20a_warn(dev_from_gk20a(g),
2947                         "wait mmu fifo space too many retries");
2948                 goto out;
2949         }
2950
2951         gk20a_writel(g, fb_mmu_invalidate_pdb_r(),
2952                 fb_mmu_invalidate_pdb_addr_f(addr_lo) |
2953                 fb_mmu_invalidate_pdb_aperture_vid_mem_f());
2954
2955         gk20a_writel(g, fb_mmu_invalidate_r(),
2956                 fb_mmu_invalidate_all_va_true_f() |
2957                 fb_mmu_invalidate_trigger_true_f());
2958
2959         do {
2960                 data = gk20a_readl(g, fb_mmu_ctrl_r());
2961                 if (fb_mmu_ctrl_pri_fifo_empty_v(data) !=
2962                         fb_mmu_ctrl_pri_fifo_empty_false_f())
2963                         break;
2964                 retry--;
2965                 usleep_range(20, 40);
2966         } while (retry >= 0 || !tegra_platform_is_silicon());
2967
2968         if (retry < 0)
2969                 gk20a_warn(dev_from_gk20a(g),
2970                         "mmu invalidate too many retries");
2971
2972 out:
2973         mutex_unlock(&tlb_lock);
2974         vm->tlb_dirty = false;
2975         mutex_unlock(&vm->update_gmmu_lock);
2976 }
2977
2978 int gk20a_mm_suspend(struct gk20a *g)
2979 {
2980         gk20a_dbg_fn("");
2981
2982         gk20a_mm_fb_flush(g);
2983         gk20a_mm_l2_flush(g, true);
2984
2985         gk20a_dbg_fn("done");
2986         return 0;
2987 }
2988
2989 void gk20a_mm_ltc_isr(struct gk20a *g)
2990 {
2991         u32 intr;
2992
2993         intr = gk20a_readl(g, ltc_ltc0_ltss_intr_r());
2994         gk20a_err(dev_from_gk20a(g), "ltc: %08x\n", intr);
2995         gk20a_writel(g, ltc_ltc0_ltss_intr_r(), intr);
2996 }
2997
2998 bool gk20a_mm_mmu_debug_mode_enabled(struct gk20a *g)
2999 {
3000         u32 debug_ctrl = gk20a_readl(g, fb_mmu_debug_ctrl_r());
3001         return fb_mmu_debug_ctrl_debug_v(debug_ctrl) ==
3002                 fb_mmu_debug_ctrl_debug_enabled_v();
3003 }
3004
3005 static int gk20a_mm_mmu_vpr_info_fetch_wait(struct gk20a *g,
3006                                             const unsigned int msec)
3007 {
3008         unsigned long timeout;
3009
3010         timeout = jiffies + msecs_to_jiffies(msec);
3011         while (1) {
3012                 u32 val;
3013
3014                 val = gk20a_readl(g, fb_mmu_vpr_info_r());
3015                 if (fb_mmu_vpr_info_fetch_v(val) ==
3016                     fb_mmu_vpr_info_fetch_false_v())
3017                         break;
3018
3019                 if (tegra_platform_is_silicon() &&
3020                                 WARN_ON(time_after(jiffies, timeout)))
3021                         return -ETIME;
3022         }
3023
3024         return 0;
3025 }
3026
3027 int gk20a_mm_mmu_vpr_info_fetch(struct gk20a *g)
3028 {
3029         int ret = 0;
3030
3031         gk20a_busy_noresume(g->dev);
3032         if (!pm_runtime_active(&g->dev->dev))
3033                 goto fail;
3034
3035         if (gk20a_mm_mmu_vpr_info_fetch_wait(g, 5)) {
3036                 ret = -ETIME;
3037                 goto fail;
3038         }
3039
3040         gk20a_writel(g, fb_mmu_vpr_info_r(),
3041                      fb_mmu_vpr_info_fetch_true_v());
3042
3043         ret = gk20a_mm_mmu_vpr_info_fetch_wait(g, 5);
3044
3045  fail:
3046         gk20a_idle(g->dev);
3047         return ret;
3048 }