gpu: nvgpu: mm: free allocations on validate error
[linux-3.10.git] / drivers / gpu / nvgpu / gk20a / mm_gk20a.c
1 /*
2  * drivers/video/tegra/host/gk20a/mm_gk20a.c
3  *
4  * GK20A memory management
5  *
6  * Copyright (c) 2011-2014, NVIDIA CORPORATION.  All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms and conditions of the GNU General Public License,
10  * version 2, as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope it will be useful, but WITHOUT
13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15  * more details.
16  *
17  * You should have received a copy of the GNU General Public License along with
18  * this program; if not, write to the Free Software Foundation, Inc.,
19  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
20  */
21
22 #include <linux/delay.h>
23 #include <linux/highmem.h>
24 #include <linux/log2.h>
25 #include <linux/nvhost.h>
26 #include <linux/pm_runtime.h>
27 #include <linux/scatterlist.h>
28 #include <linux/nvmap.h>
29 #include <linux/tegra-soc.h>
30 #include <linux/vmalloc.h>
31 #include <linux/dma-buf.h>
32 #include <asm/cacheflush.h>
33
34 #include "gk20a.h"
35 #include "mm_gk20a.h"
36 #include "hw_gmmu_gk20a.h"
37 #include "hw_fb_gk20a.h"
38 #include "hw_bus_gk20a.h"
39 #include "hw_ram_gk20a.h"
40 #include "hw_mc_gk20a.h"
41 #include "hw_flush_gk20a.h"
42 #include "hw_ltc_gk20a.h"
43
44 #include "kind_gk20a.h"
45
46 #ifdef CONFIG_ARM64
47 #define outer_flush_range(a, b)
48 #define __cpuc_flush_dcache_area __flush_dcache_area
49 #endif
50
51 /*
52  * GPU mapping life cycle
53  * ======================
54  *
55  * Kernel mappings
56  * ---------------
57  *
58  * Kernel mappings are created through vm.map(..., false):
59  *
60  *  - Mappings to the same allocations are reused and refcounted.
61  *  - This path does not support deferred unmapping (i.e. kernel must wait for
62  *    all hw operations on the buffer to complete before unmapping).
63  *  - References to dmabuf are owned and managed by the (kernel) clients of
64  *    the gk20a_vm layer.
65  *
66  *
67  * User space mappings
68  * -------------------
69  *
70  * User space mappings are created through as.map_buffer -> vm.map(..., true):
71  *
72  *  - Mappings to the same allocations are reused and refcounted.
73  *  - This path supports deferred unmapping (i.e. we delay the actual unmapping
74  *    until all hw operations have completed).
75  *  - References to dmabuf are owned and managed by the vm_gk20a
76  *    layer itself. vm.map acquires these refs, and sets
77  *    mapped_buffer->own_mem_ref to record that we must release the refs when we
78  *    actually unmap.
79  *
80  */
81
82 static inline int vm_aspace_id(struct vm_gk20a *vm)
83 {
84         /* -1 is bar1 or pmu, etc. */
85         return vm->as_share ? vm->as_share->id : -1;
86 }
87 static inline u32 hi32(u64 f)
88 {
89         return (u32)(f >> 32);
90 }
91 static inline u32 lo32(u64 f)
92 {
93         return (u32)(f & 0xffffffff);
94 }
95
96 #define FLUSH_CPU_DCACHE(va, pa, size)  \
97         do {    \
98                 __cpuc_flush_dcache_area((void *)(va), (size_t)(size)); \
99                 outer_flush_range(pa, pa + (size_t)(size));             \
100         } while (0)
101
102 static void gk20a_vm_unmap_locked(struct mapped_buffer_node *mapped_buffer);
103 static struct mapped_buffer_node *find_mapped_buffer_locked(
104                                         struct rb_root *root, u64 addr);
105 static struct mapped_buffer_node *find_mapped_buffer_reverse_locked(
106                                 struct rb_root *root, struct dma_buf *dmabuf,
107                                 u32 kind);
108 static int update_gmmu_ptes_locked(struct vm_gk20a *vm,
109                                    enum gmmu_pgsz_gk20a pgsz_idx,
110                                    struct sg_table *sgt,
111                                    u64 first_vaddr, u64 last_vaddr,
112                                    u8 kind_v, u32 ctag_offset, bool cacheable,
113                                    int rw_flag);
114 static void update_gmmu_pde_locked(struct vm_gk20a *vm, u32 i);
115 static void gk20a_vm_remove_support(struct vm_gk20a *vm);
116
117
118 /* note: keep the page sizes sorted lowest to highest here */
119 static const u32 gmmu_page_sizes[gmmu_nr_page_sizes] = { SZ_4K, SZ_128K };
120 static const u32 gmmu_page_shifts[gmmu_nr_page_sizes] = { 12, 17 };
121 static const u64 gmmu_page_offset_masks[gmmu_nr_page_sizes] = { 0xfffLL,
122                                                                 0x1ffffLL };
123 static const u64 gmmu_page_masks[gmmu_nr_page_sizes] = { ~0xfffLL, ~0x1ffffLL };
124
125 struct gk20a_comptags {
126         u32 offset;
127         u32 lines;
128 };
129
130 struct gk20a_dmabuf_priv {
131         struct mutex lock;
132
133         struct gk20a_allocator *comptag_allocator;
134         struct gk20a_comptags comptags;
135
136         struct dma_buf_attachment *attach;
137         struct sg_table *sgt;
138
139         int pin_count;
140 };
141
142 static void gk20a_mm_delete_priv(void *_priv)
143 {
144         struct gk20a_dmabuf_priv *priv = _priv;
145         if (!priv)
146                 return;
147
148         if (priv->comptags.lines) {
149                 BUG_ON(!priv->comptag_allocator);
150                 priv->comptag_allocator->free(priv->comptag_allocator,
151                                               priv->comptags.offset,
152                                               priv->comptags.lines);
153         }
154
155         kfree(priv);
156 }
157
158 struct sg_table *gk20a_mm_pin(struct device *dev, struct dma_buf *dmabuf)
159 {
160         struct gk20a_dmabuf_priv *priv;
161
162         priv = dma_buf_get_drvdata(dmabuf, dev);
163         if (WARN_ON(!priv))
164                 return ERR_PTR(-EINVAL);
165
166         mutex_lock(&priv->lock);
167
168         if (priv->pin_count == 0) {
169                 priv->attach = dma_buf_attach(dmabuf, dev);
170                 if (IS_ERR(priv->attach)) {
171                         mutex_unlock(&priv->lock);
172                         return (struct sg_table *)priv->attach;
173                 }
174
175                 priv->sgt = dma_buf_map_attachment(priv->attach,
176                                                    DMA_BIDIRECTIONAL);
177                 if (IS_ERR(priv->sgt)) {
178                         dma_buf_detach(dmabuf, priv->attach);
179                         mutex_unlock(&priv->lock);
180                         return priv->sgt;
181                 }
182         }
183
184         priv->pin_count++;
185         mutex_unlock(&priv->lock);
186         return priv->sgt;
187 }
188
189 void gk20a_mm_unpin(struct device *dev, struct dma_buf *dmabuf,
190                     struct sg_table *sgt)
191 {
192         struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev);
193         dma_addr_t dma_addr;
194
195         if (IS_ERR(priv) || !priv)
196                 return;
197
198         mutex_lock(&priv->lock);
199         WARN_ON(priv->sgt != sgt);
200         priv->pin_count--;
201         WARN_ON(priv->pin_count < 0);
202         dma_addr = sg_dma_address(priv->sgt->sgl);
203         if (priv->pin_count == 0) {
204                 dma_buf_unmap_attachment(priv->attach, priv->sgt,
205                                          DMA_BIDIRECTIONAL);
206                 dma_buf_detach(dmabuf, priv->attach);
207         }
208         mutex_unlock(&priv->lock);
209 }
210
211
212 static void gk20a_get_comptags(struct device *dev,
213                                struct dma_buf *dmabuf,
214                                struct gk20a_comptags *comptags)
215 {
216         struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev);
217
218         if (!comptags)
219                 return;
220
221         if (!priv) {
222                 comptags->lines = 0;
223                 comptags->offset = 0;
224                 return;
225         }
226
227         *comptags = priv->comptags;
228 }
229
230 static int gk20a_alloc_comptags(struct device *dev,
231                                 struct dma_buf *dmabuf,
232                                 struct gk20a_allocator *allocator,
233                                 int lines)
234 {
235         struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev);
236         u32 offset = 0;
237         int err;
238
239         if (!priv)
240                 return -ENOSYS;
241
242         if (!lines)
243                 return -EINVAL;
244
245         /* store the allocator so we can use it when we free the ctags */
246         priv->comptag_allocator = allocator;
247         err = allocator->alloc(allocator, &offset, lines);
248         if (!err) {
249                 priv->comptags.lines = lines;
250                 priv->comptags.offset = offset;
251         }
252         return err;
253 }
254
255
256
257
258 static int gk20a_init_mm_reset_enable_hw(struct gk20a *g)
259 {
260         gk20a_dbg_fn("");
261         if (g->ops.fb.reset)
262                 g->ops.fb.reset(g);
263
264         if (g->ops.fb.init_fs_state)
265                 g->ops.fb.init_fs_state(g);
266
267         return 0;
268 }
269
270 void gk20a_remove_mm_support(struct mm_gk20a *mm)
271 {
272         struct gk20a *g = mm->g;
273         struct device *d = dev_from_gk20a(g);
274         struct vm_gk20a *vm = &mm->bar1.vm;
275         struct inst_desc *inst_block = &mm->bar1.inst_block;
276
277         gk20a_dbg_fn("");
278
279         if (inst_block->cpuva)
280                 dma_free_coherent(d, inst_block->size,
281                         inst_block->cpuva, inst_block->iova);
282         inst_block->cpuva = NULL;
283         inst_block->iova = 0;
284
285         gk20a_vm_remove_support(vm);
286 }
287
288 int gk20a_init_mm_setup_sw(struct gk20a *g)
289 {
290         struct mm_gk20a *mm = &g->mm;
291         int i;
292
293         gk20a_dbg_fn("");
294
295         if (mm->sw_ready) {
296                 gk20a_dbg_fn("skip init");
297                 return 0;
298         }
299
300         mm->g = g;
301         mutex_init(&mm->l2_op_lock);
302         mm->big_page_size = gmmu_page_sizes[gmmu_page_size_big];
303         mm->compression_page_size = gmmu_page_sizes[gmmu_page_size_big];
304         mm->pde_stride    = mm->big_page_size << 10;
305         mm->pde_stride_shift = ilog2(mm->pde_stride);
306         BUG_ON(mm->pde_stride_shift > 31); /* we have assumptions about this */
307
308         for (i = 0; i < ARRAY_SIZE(gmmu_page_sizes); i++) {
309
310                 u32 num_ptes, pte_space, num_pages;
311
312                 /* assuming "full" page tables */
313                 num_ptes = mm->pde_stride / gmmu_page_sizes[i];
314
315                 pte_space = num_ptes * gmmu_pte__size_v();
316                 /* allocate whole pages */
317                 pte_space = roundup(pte_space, PAGE_SIZE);
318
319                 num_pages = pte_space / PAGE_SIZE;
320                 /* make sure "order" is viable */
321                 BUG_ON(!is_power_of_2(num_pages));
322
323                 mm->page_table_sizing[i].num_ptes = num_ptes;
324                 mm->page_table_sizing[i].order = ilog2(num_pages);
325         }
326
327         /*TBD: make channel vm size configurable */
328         mm->channel.size = 1ULL << NV_GMMU_VA_RANGE;
329
330         gk20a_dbg_info("channel vm size: %dMB", (int)(mm->channel.size >> 20));
331
332         gk20a_dbg_info("small page-size (%dKB) pte array: %dKB",
333                         gmmu_page_sizes[gmmu_page_size_small] >> 10,
334                         (mm->page_table_sizing[gmmu_page_size_small].num_ptes *
335                          gmmu_pte__size_v()) >> 10);
336
337         gk20a_dbg_info("big page-size (%dKB) pte array: %dKB",
338                         gmmu_page_sizes[gmmu_page_size_big] >> 10,
339                         (mm->page_table_sizing[gmmu_page_size_big].num_ptes *
340                          gmmu_pte__size_v()) >> 10);
341
342
343         gk20a_init_bar1_vm(mm);
344
345         mm->remove_support = gk20a_remove_mm_support;
346         mm->sw_ready = true;
347
348         gk20a_dbg_fn("done");
349         return 0;
350 }
351
352 /* make sure gk20a_init_mm_support is called before */
353 static int gk20a_init_mm_setup_hw(struct gk20a *g)
354 {
355         struct mm_gk20a *mm = &g->mm;
356         struct inst_desc *inst_block = &mm->bar1.inst_block;
357         phys_addr_t inst_pa = inst_block->cpu_pa;
358
359         gk20a_dbg_fn("");
360
361         /* set large page size in fb
362          * note this is very early on, can we defer it ? */
363         {
364                 u32 fb_mmu_ctrl = gk20a_readl(g, fb_mmu_ctrl_r());
365
366                 if (gmmu_page_sizes[gmmu_page_size_big] == SZ_128K)
367                         fb_mmu_ctrl = (fb_mmu_ctrl &
368                                        ~fb_mmu_ctrl_vm_pg_size_f(~0x0)) |
369                                 fb_mmu_ctrl_vm_pg_size_128kb_f();
370                 else
371                         BUG_ON(1); /* no support/testing for larger ones yet */
372
373                 gk20a_writel(g, fb_mmu_ctrl_r(), fb_mmu_ctrl);
374         }
375
376         inst_pa = (u32)(inst_pa >> bar1_instance_block_shift_gk20a());
377         gk20a_dbg_info("bar1 inst block ptr: 0x%08x",  (u32)inst_pa);
378
379         gk20a_writel(g, bus_bar1_block_r(),
380                      bus_bar1_block_target_vid_mem_f() |
381                      bus_bar1_block_mode_virtual_f() |
382                      bus_bar1_block_ptr_f(inst_pa));
383         if (gk20a_mm_fb_flush(g) || gk20a_mm_fb_flush(g))
384                 return -EBUSY;
385
386         gk20a_dbg_fn("done");
387         return 0;
388 }
389
390 int gk20a_init_mm_support(struct gk20a *g)
391 {
392         u32 err;
393
394         err = gk20a_init_mm_reset_enable_hw(g);
395         if (err)
396                 return err;
397
398         err = gk20a_init_mm_setup_sw(g);
399         if (err)
400                 return err;
401
402         err = gk20a_init_mm_setup_hw(g);
403         if (err)
404                 return err;
405
406         return err;
407 }
408
409 #ifdef CONFIG_GK20A_PHYS_PAGE_TABLES
410 static int alloc_gmmu_pages(struct vm_gk20a *vm, u32 order,
411                             void **handle,
412                             struct sg_table **sgt,
413                             size_t *size)
414 {
415         u32 num_pages = 1 << order;
416         u32 len = num_pages * PAGE_SIZE;
417         int err;
418         struct page *pages;
419
420         gk20a_dbg_fn("");
421
422         pages = alloc_pages(GFP_KERNEL, order);
423         if (!pages) {
424                 gk20a_dbg(gpu_dbg_pte, "alloc_pages failed\n");
425                 goto err_out;
426         }
427         *sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
428         if (!sgt) {
429                 gk20a_dbg(gpu_dbg_pte, "cannot allocate sg table");
430                 goto err_alloced;
431         }
432         err = sg_alloc_table(*sgt, 1, GFP_KERNEL);
433         if (err) {
434                 gk20a_dbg(gpu_dbg_pte, "sg_alloc_table failed\n");
435                 goto err_sg_table;
436         }
437         sg_set_page((*sgt)->sgl, pages, len, 0);
438         *handle = page_address(pages);
439         memset(*handle, 0, len);
440         *size = len;
441         FLUSH_CPU_DCACHE(*handle, sg_phys((*sgt)->sgl), len);
442
443         return 0;
444
445 err_sg_table:
446         kfree(*sgt);
447 err_alloced:
448         __free_pages(pages, order);
449 err_out:
450         return -ENOMEM;
451 }
452
453 static void free_gmmu_pages(struct vm_gk20a *vm, void *handle,
454                             struct sg_table *sgt, u32 order,
455                             size_t size)
456 {
457         gk20a_dbg_fn("");
458         BUG_ON(sgt == NULL);
459         free_pages((unsigned long)handle, order);
460         sg_free_table(sgt);
461         kfree(sgt);
462 }
463
464 static int map_gmmu_pages(void *handle, struct sg_table *sgt,
465                           void **va, size_t size)
466 {
467         FLUSH_CPU_DCACHE(handle, sg_phys(sgt->sgl), sgt->sgl->length);
468         *va = handle;
469         return 0;
470 }
471
472 static void unmap_gmmu_pages(void *handle, struct sg_table *sgt, void *va)
473 {
474         FLUSH_CPU_DCACHE(handle, sg_phys(sgt->sgl), sgt->sgl->length);
475 }
476 #else
477
478 static int alloc_gmmu_pages(struct vm_gk20a *vm, u32 order,
479                             void **handle,
480                             struct sg_table **sgt,
481                             size_t *size)
482 {
483         struct device *d = dev_from_vm(vm);
484         u32 num_pages = 1 << order;
485         u32 len = num_pages * PAGE_SIZE;
486         dma_addr_t iova;
487         DEFINE_DMA_ATTRS(attrs);
488         struct page **pages;
489         void *cpuva;
490         int err = 0;
491
492         gk20a_dbg_fn("");
493
494         *size = len;
495
496         if (IS_ENABLED(CONFIG_ARM64)) {
497                 cpuva = dma_zalloc_coherent(d, len, &iova, GFP_KERNEL);
498                 if (!cpuva) {
499                         gk20a_err(d, "memory allocation failed\n");
500                         goto err_out;
501                 }
502
503                 err = gk20a_get_sgtable(d, sgt, cpuva, iova, len);
504                 if (err) {
505                         gk20a_err(d, "sgt allocation failed\n");
506                         goto err_free;
507                 }
508
509                 *handle = cpuva;
510         } else {
511                 dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &attrs);
512                 pages = dma_alloc_attrs(d, len, &iova, GFP_KERNEL, &attrs);
513                 if (!pages) {
514                         gk20a_err(d, "memory allocation failed\n");
515                         goto err_out;
516                 }
517
518                 err = gk20a_get_sgtable_from_pages(d, sgt, pages,
519                                         iova, len);
520                 if (err) {
521                         gk20a_err(d, "sgt allocation failed\n");
522                         goto err_free;
523                 }
524
525                 *handle = (void *)pages;
526         }
527
528         return 0;
529
530 err_free:
531         if (IS_ENABLED(CONFIG_ARM64)) {
532                 dma_free_coherent(d, len, handle, iova);
533                 cpuva = NULL;
534         } else {
535                 dma_free_attrs(d, len, pages, iova, &attrs);
536                 pages = NULL;
537         }
538         iova = 0;
539 err_out:
540         return -ENOMEM;
541 }
542
543 static void free_gmmu_pages(struct vm_gk20a *vm, void *handle,
544                             struct sg_table *sgt, u32 order,
545                             size_t size)
546 {
547         struct device *d = dev_from_vm(vm);
548         u64 iova;
549         DEFINE_DMA_ATTRS(attrs);
550         struct page **pages;
551
552         gk20a_dbg_fn("");
553         BUG_ON(sgt == NULL);
554
555         iova = sg_dma_address(sgt->sgl);
556
557         gk20a_free_sgtable(&sgt);
558
559         if (IS_ENABLED(CONFIG_ARM64)) {
560                 dma_free_coherent(d, size, handle, iova);
561         } else {
562                 pages = (struct page **)handle;
563                 dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &attrs);
564                 dma_free_attrs(d, size, pages, iova, &attrs);
565                 pages = NULL;
566         }
567
568         handle = NULL;
569         iova = 0;
570 }
571
572 static int map_gmmu_pages(void *handle, struct sg_table *sgt,
573                           void **kva, size_t size)
574 {
575         int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
576         struct page **pages;
577         gk20a_dbg_fn("");
578
579         if (IS_ENABLED(CONFIG_ARM64)) {
580                 *kva = handle;
581         } else {
582                 pages = (struct page **)handle;
583                 *kva = vmap(pages, count, 0, pgprot_dmacoherent(PAGE_KERNEL));
584                 if (!(*kva))
585                         return -ENOMEM;
586         }
587
588         return 0;
589 }
590
591 static void unmap_gmmu_pages(void *handle, struct sg_table *sgt, void *va)
592 {
593         gk20a_dbg_fn("");
594
595         if (!IS_ENABLED(CONFIG_ARM64))
596                 vunmap(va);
597         va = NULL;
598 }
599 #endif
600
601 /* allocate a phys contig region big enough for a full
602  * sized gmmu page table for the given gmmu_page_size.
603  * the whole range is zeroed so it's "invalid"/will fault
604  */
605
606 static int zalloc_gmmu_page_table_gk20a(struct vm_gk20a *vm,
607                                         enum gmmu_pgsz_gk20a gmmu_pgsz_idx,
608                                         struct page_table_gk20a *pte)
609 {
610         int err;
611         u32 pte_order;
612         void *handle = NULL;
613         struct sg_table *sgt;
614         size_t size;
615
616         gk20a_dbg_fn("");
617
618         /* allocate enough pages for the table */
619         pte_order = vm->mm->page_table_sizing[gmmu_pgsz_idx].order;
620
621         err = alloc_gmmu_pages(vm, pte_order, &handle, &sgt, &size);
622         if (err)
623                 return err;
624
625         gk20a_dbg(gpu_dbg_pte, "pte = 0x%p, addr=%08llx, size %d",
626                         pte, gk20a_mm_iova_addr(sgt->sgl), pte_order);
627
628         pte->ref = handle;
629         pte->sgt = sgt;
630         pte->size = size;
631
632         return 0;
633 }
634
635 /* given address range (inclusive) determine the pdes crossed */
636 static inline void pde_range_from_vaddr_range(struct vm_gk20a *vm,
637                                               u64 addr_lo, u64 addr_hi,
638                                               u32 *pde_lo, u32 *pde_hi)
639 {
640         *pde_lo = (u32)(addr_lo >> vm->mm->pde_stride_shift);
641         *pde_hi = (u32)(addr_hi >> vm->mm->pde_stride_shift);
642         gk20a_dbg(gpu_dbg_pte, "addr_lo=0x%llx addr_hi=0x%llx pde_ss=%d",
643                    addr_lo, addr_hi, vm->mm->pde_stride_shift);
644         gk20a_dbg(gpu_dbg_pte, "pde_lo=%d pde_hi=%d",
645                    *pde_lo, *pde_hi);
646 }
647
648 static inline u32 *pde_from_index(struct vm_gk20a *vm, u32 i)
649 {
650         return (u32 *) (((u8 *)vm->pdes.kv) + i*gmmu_pde__size_v());
651 }
652
653 static inline u32 pte_index_from_vaddr(struct vm_gk20a *vm,
654                                        u64 addr, enum gmmu_pgsz_gk20a pgsz_idx)
655 {
656         u32 ret;
657         /* mask off pde part */
658         addr = addr & ((((u64)1) << vm->mm->pde_stride_shift) - ((u64)1));
659         /* shift over to get pte index. note assumption that pte index
660          * doesn't leak over into the high 32b */
661         ret = (u32)(addr >> gmmu_page_shifts[pgsz_idx]);
662
663         gk20a_dbg(gpu_dbg_pte, "addr=0x%llx pte_i=0x%x", addr, ret);
664         return ret;
665 }
666
667 static inline void pte_space_page_offset_from_index(u32 i, u32 *pte_page,
668                                                     u32 *pte_offset)
669 {
670         /* ptes are 8B regardless of pagesize */
671         /* pte space pages are 4KB. so 512 ptes per 4KB page*/
672         *pte_page = i >> 9;
673
674         /* this offset is a pte offset, not a byte offset */
675         *pte_offset = i & ((1<<9)-1);
676
677         gk20a_dbg(gpu_dbg_pte, "i=0x%x pte_page=0x%x pte_offset=0x%x",
678                    i, *pte_page, *pte_offset);
679 }
680
681
682 /*
683  * given a pde index/page table number make sure it has
684  * backing store and if not go ahead allocate it and
685  * record it in the appropriate pde
686  */
687 static int validate_gmmu_page_table_gk20a_locked(struct vm_gk20a *vm,
688                                 u32 i, enum gmmu_pgsz_gk20a gmmu_pgsz_idx)
689 {
690         int err;
691         struct page_table_gk20a *pte =
692                 vm->pdes.ptes[gmmu_pgsz_idx] + i;
693
694         gk20a_dbg_fn("");
695
696         /* if it's already in place it's valid */
697         if (pte->ref)
698                 return 0;
699
700         gk20a_dbg(gpu_dbg_pte, "alloc %dKB ptes for pde %d",
701                    gmmu_page_sizes[gmmu_pgsz_idx]/1024, i);
702
703         err = zalloc_gmmu_page_table_gk20a(vm, gmmu_pgsz_idx, pte);
704         if (err)
705                 return err;
706
707         /* rewrite pde */
708         update_gmmu_pde_locked(vm, i);
709
710         return 0;
711 }
712
713 static struct vm_reserved_va_node *addr_to_reservation(struct vm_gk20a *vm,
714                                                        u64 addr)
715 {
716         struct vm_reserved_va_node *va_node;
717         list_for_each_entry(va_node, &vm->reserved_va_list, reserved_va_list)
718                 if (addr >= va_node->vaddr_start &&
719                     addr < (u64)va_node->vaddr_start + (u64)va_node->size)
720                         return va_node;
721
722         return NULL;
723 }
724
725 int gk20a_vm_get_buffers(struct vm_gk20a *vm,
726                          struct mapped_buffer_node ***mapped_buffers,
727                          int *num_buffers)
728 {
729         struct mapped_buffer_node *mapped_buffer;
730         struct mapped_buffer_node **buffer_list;
731         struct rb_node *node;
732         int i = 0;
733
734         mutex_lock(&vm->update_gmmu_lock);
735
736         buffer_list = kzalloc(sizeof(*buffer_list) *
737                               vm->num_user_mapped_buffers, GFP_KERNEL);
738         if (!buffer_list) {
739                 mutex_unlock(&vm->update_gmmu_lock);
740                 return -ENOMEM;
741         }
742
743         node = rb_first(&vm->mapped_buffers);
744         while (node) {
745                 mapped_buffer =
746                         container_of(node, struct mapped_buffer_node, node);
747                 if (mapped_buffer->user_mapped) {
748                         buffer_list[i] = mapped_buffer;
749                         kref_get(&mapped_buffer->ref);
750                         i++;
751                 }
752                 node = rb_next(&mapped_buffer->node);
753         }
754
755         BUG_ON(i != vm->num_user_mapped_buffers);
756
757         *num_buffers = vm->num_user_mapped_buffers;
758         *mapped_buffers = buffer_list;
759
760         mutex_unlock(&vm->update_gmmu_lock);
761
762         return 0;
763 }
764
765 static void gk20a_vm_unmap_locked_kref(struct kref *ref)
766 {
767         struct mapped_buffer_node *mapped_buffer =
768                 container_of(ref, struct mapped_buffer_node, ref);
769         gk20a_vm_unmap_locked(mapped_buffer);
770 }
771
772 void gk20a_vm_put_buffers(struct vm_gk20a *vm,
773                                  struct mapped_buffer_node **mapped_buffers,
774                                  int num_buffers)
775 {
776         int i;
777
778         mutex_lock(&vm->update_gmmu_lock);
779
780         for (i = 0; i < num_buffers; ++i)
781                 kref_put(&mapped_buffers[i]->ref,
782                          gk20a_vm_unmap_locked_kref);
783
784         mutex_unlock(&vm->update_gmmu_lock);
785
786         kfree(mapped_buffers);
787 }
788
789 static void gk20a_vm_unmap_user(struct vm_gk20a *vm, u64 offset)
790 {
791         struct device *d = dev_from_vm(vm);
792         int retries;
793         struct mapped_buffer_node *mapped_buffer;
794
795         mutex_lock(&vm->update_gmmu_lock);
796
797         mapped_buffer = find_mapped_buffer_locked(&vm->mapped_buffers, offset);
798         if (!mapped_buffer) {
799                 mutex_unlock(&vm->update_gmmu_lock);
800                 gk20a_err(d, "invalid addr to unmap 0x%llx", offset);
801                 return;
802         }
803
804         if (mapped_buffer->flags & NVHOST_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET) {
805                 mutex_unlock(&vm->update_gmmu_lock);
806
807                 retries = 1000;
808                 while (retries) {
809                         if (atomic_read(&mapped_buffer->ref.refcount) == 1)
810                                 break;
811                         retries--;
812                         udelay(50);
813                 }
814                 if (!retries)
815                         gk20a_err(d, "sync-unmap failed on 0x%llx",
816                                                                 offset);
817                 mutex_lock(&vm->update_gmmu_lock);
818         }
819
820         mapped_buffer->user_mapped--;
821         if (mapped_buffer->user_mapped == 0)
822                 vm->num_user_mapped_buffers--;
823         kref_put(&mapped_buffer->ref, gk20a_vm_unmap_locked_kref);
824
825         mutex_unlock(&vm->update_gmmu_lock);
826 }
827
828 static u64 gk20a_vm_alloc_va(struct vm_gk20a *vm,
829                              u64 size,
830                              enum gmmu_pgsz_gk20a gmmu_pgsz_idx)
831
832 {
833         struct gk20a_allocator *vma = &vm->vma[gmmu_pgsz_idx];
834         int err;
835         u64 offset;
836         u32 start_page_nr = 0, num_pages;
837         u64 gmmu_page_size = gmmu_page_sizes[gmmu_pgsz_idx];
838
839         if (gmmu_pgsz_idx >= ARRAY_SIZE(gmmu_page_sizes)) {
840                 dev_warn(dev_from_vm(vm),
841                          "invalid page size requested in gk20a vm alloc");
842                 return -EINVAL;
843         }
844
845         if ((gmmu_pgsz_idx == gmmu_page_size_big) && !vm->big_pages) {
846                 dev_warn(dev_from_vm(vm),
847                          "unsupportd page size requested");
848                 return -EINVAL;
849
850         }
851
852         /* be certain we round up to gmmu_page_size if needed */
853         /* TBD: DIV_ROUND_UP -> undefined reference to __aeabi_uldivmod */
854         size = (size + ((u64)gmmu_page_size - 1)) & ~((u64)gmmu_page_size - 1);
855
856         gk20a_dbg_info("size=0x%llx @ pgsz=%dKB", size,
857                         gmmu_page_sizes[gmmu_pgsz_idx]>>10);
858
859         /* The vma allocator represents page accounting. */
860         num_pages = size >> gmmu_page_shifts[gmmu_pgsz_idx];
861
862         err = vma->alloc(vma, &start_page_nr, num_pages);
863
864         if (err) {
865                 gk20a_err(dev_from_vm(vm),
866                            "%s oom: sz=0x%llx", vma->name, size);
867                 return 0;
868         }
869
870         offset = (u64)start_page_nr << gmmu_page_shifts[gmmu_pgsz_idx];
871         gk20a_dbg_fn("%s found addr: 0x%llx", vma->name, offset);
872
873         return offset;
874 }
875
876 static int gk20a_vm_free_va(struct vm_gk20a *vm,
877                              u64 offset, u64 size,
878                              enum gmmu_pgsz_gk20a pgsz_idx)
879 {
880         struct gk20a_allocator *vma = &vm->vma[pgsz_idx];
881         u32 page_size = gmmu_page_sizes[pgsz_idx];
882         u32 page_shift = gmmu_page_shifts[pgsz_idx];
883         u32 start_page_nr, num_pages;
884         int err;
885
886         gk20a_dbg_info("%s free addr=0x%llx, size=0x%llx",
887                         vma->name, offset, size);
888
889         start_page_nr = (u32)(offset >> page_shift);
890         num_pages = (u32)((size + page_size - 1) >> page_shift);
891
892         err = vma->free(vma, start_page_nr, num_pages);
893         if (err) {
894                 gk20a_err(dev_from_vm(vm),
895                            "not found: offset=0x%llx, sz=0x%llx",
896                            offset, size);
897         }
898
899         return err;
900 }
901
902 static int insert_mapped_buffer(struct rb_root *root,
903                                 struct mapped_buffer_node *mapped_buffer)
904 {
905         struct rb_node **new_node = &(root->rb_node), *parent = NULL;
906
907         /* Figure out where to put new node */
908         while (*new_node) {
909                 struct mapped_buffer_node *cmp_with =
910                         container_of(*new_node, struct mapped_buffer_node,
911                                      node);
912
913                 parent = *new_node;
914
915                 if (cmp_with->addr > mapped_buffer->addr) /* u64 cmp */
916                         new_node = &((*new_node)->rb_left);
917                 else if (cmp_with->addr != mapped_buffer->addr) /* u64 cmp */
918                         new_node = &((*new_node)->rb_right);
919                 else
920                         return -EINVAL; /* no fair dup'ing */
921         }
922
923         /* Add new node and rebalance tree. */
924         rb_link_node(&mapped_buffer->node, parent, new_node);
925         rb_insert_color(&mapped_buffer->node, root);
926
927         return 0;
928 }
929
930 static struct mapped_buffer_node *find_mapped_buffer_reverse_locked(
931                                 struct rb_root *root, struct dma_buf *dmabuf,
932                                 u32 kind)
933 {
934         struct rb_node *node = rb_first(root);
935         while (node) {
936                 struct mapped_buffer_node *mapped_buffer =
937                         container_of(node, struct mapped_buffer_node, node);
938                 if (mapped_buffer->dmabuf == dmabuf &&
939                     kind == mapped_buffer->kind)
940                         return mapped_buffer;
941                 node = rb_next(&mapped_buffer->node);
942         }
943         return 0;
944 }
945
946 static struct mapped_buffer_node *find_mapped_buffer_locked(
947                                         struct rb_root *root, u64 addr)
948 {
949
950         struct rb_node *node = root->rb_node;
951         while (node) {
952                 struct mapped_buffer_node *mapped_buffer =
953                         container_of(node, struct mapped_buffer_node, node);
954                 if (mapped_buffer->addr > addr) /* u64 cmp */
955                         node = node->rb_left;
956                 else if (mapped_buffer->addr != addr) /* u64 cmp */
957                         node = node->rb_right;
958                 else
959                         return mapped_buffer;
960         }
961         return 0;
962 }
963
964 static struct mapped_buffer_node *find_mapped_buffer_range_locked(
965                                         struct rb_root *root, u64 addr)
966 {
967         struct rb_node *node = root->rb_node;
968         while (node) {
969                 struct mapped_buffer_node *m =
970                         container_of(node, struct mapped_buffer_node, node);
971                 if (m->addr <= addr && m->addr + m->size > addr)
972                         return m;
973                 else if (m->addr > addr) /* u64 cmp */
974                         node = node->rb_left;
975                 else
976                         node = node->rb_right;
977         }
978         return 0;
979 }
980
981 #define BFR_ATTRS (sizeof(nvmap_bfr_param)/sizeof(nvmap_bfr_param[0]))
982
983 struct buffer_attrs {
984         struct sg_table *sgt;
985         u64 size;
986         u64 align;
987         u32 ctag_offset;
988         u32 ctag_lines;
989         int pgsz_idx;
990         u8 kind_v;
991         u8 uc_kind_v;
992 };
993
994 static void gmmu_select_page_size(struct buffer_attrs *bfr)
995 {
996         int i;
997         /*  choose the biggest first (top->bottom) */
998         for (i = (gmmu_nr_page_sizes-1); i >= 0; i--)
999                 if (!(gmmu_page_offset_masks[i] & bfr->align)) {
1000                         /* would like to add this too but nvmap returns the
1001                          * original requested size not the allocated size.
1002                          * (!(gmmu_page_offset_masks[i] & bfr->size)) */
1003                         bfr->pgsz_idx = i;
1004                         break;
1005                 }
1006 }
1007
1008 static int setup_buffer_kind_and_compression(struct device *d,
1009                                              u32 flags,
1010                                              struct buffer_attrs *bfr,
1011                                              enum gmmu_pgsz_gk20a pgsz_idx)
1012 {
1013         bool kind_compressible;
1014
1015         if (unlikely(bfr->kind_v == gmmu_pte_kind_invalid_v()))
1016                 bfr->kind_v = gmmu_pte_kind_pitch_v();
1017
1018         if (unlikely(!gk20a_kind_is_supported(bfr->kind_v))) {
1019                 gk20a_err(d, "kind 0x%x not supported", bfr->kind_v);
1020                 return -EINVAL;
1021         }
1022
1023         bfr->uc_kind_v = gmmu_pte_kind_invalid_v();
1024         /* find a suitable uncompressed kind if it becomes necessary later */
1025         kind_compressible = gk20a_kind_is_compressible(bfr->kind_v);
1026         if (kind_compressible) {
1027                 bfr->uc_kind_v = gk20a_get_uncompressed_kind(bfr->kind_v);
1028                 if (unlikely(bfr->uc_kind_v == gmmu_pte_kind_invalid_v())) {
1029                         /* shouldn't happen, but it is worth cross-checking */
1030                         gk20a_err(d, "comptag kind 0x%x can't be"
1031                                    " downgraded to uncompressed kind",
1032                                    bfr->kind_v);
1033                         return -EINVAL;
1034                 }
1035         }
1036         /* comptags only supported for suitable kinds, 128KB pagesize */
1037         if (unlikely(kind_compressible &&
1038                      (gmmu_page_sizes[pgsz_idx] != 128*1024))) {
1039                 /*
1040                 gk20a_warn(d, "comptags specified"
1041                 " but pagesize being used doesn't support it");*/
1042                 /* it is safe to fall back to uncompressed as
1043                    functionality is not harmed */
1044                 bfr->kind_v = bfr->uc_kind_v;
1045                 kind_compressible = false;
1046         }
1047         if (kind_compressible)
1048                 bfr->ctag_lines = ALIGN(bfr->size, COMP_TAG_LINE_SIZE) >>
1049                         COMP_TAG_LINE_SIZE_SHIFT;
1050         else
1051                 bfr->ctag_lines = 0;
1052
1053         return 0;
1054 }
1055
1056 static int validate_fixed_buffer(struct vm_gk20a *vm,
1057                                  struct buffer_attrs *bfr,
1058                                  u64 map_offset)
1059 {
1060         struct device *dev = dev_from_vm(vm);
1061         struct vm_reserved_va_node *va_node;
1062         struct mapped_buffer_node *buffer;
1063
1064         if (map_offset & gmmu_page_offset_masks[bfr->pgsz_idx]) {
1065                 gk20a_err(dev, "map offset must be buffer page size aligned 0x%llx",
1066                            map_offset);
1067                 return -EINVAL;
1068         }
1069
1070         /* find the space reservation */
1071         va_node = addr_to_reservation(vm, map_offset);
1072         if (!va_node) {
1073                 gk20a_warn(dev, "fixed offset mapping without space allocation");
1074                 return -EINVAL;
1075         }
1076
1077         /* check that this mappings does not collide with existing
1078          * mappings by checking the overlapping area between the current
1079          * buffer and all other mapped buffers */
1080
1081         list_for_each_entry(buffer,
1082                 &va_node->va_buffers_list, va_buffers_list) {
1083                 s64 begin = max(buffer->addr, map_offset);
1084                 s64 end = min(buffer->addr +
1085                         buffer->size, map_offset + bfr->size);
1086                 if (end - begin > 0) {
1087                         gk20a_warn(dev, "overlapping buffer map requested");
1088                         return -EINVAL;
1089                 }
1090         }
1091
1092         return 0;
1093 }
1094
1095 static u64 __locked_gmmu_map(struct vm_gk20a *vm,
1096                                 u64 map_offset,
1097                                 struct sg_table *sgt,
1098                                 u64 size,
1099                                 int pgsz_idx,
1100                                 u8 kind_v,
1101                                 u32 ctag_offset,
1102                                 u32 flags,
1103                                 int rw_flag)
1104 {
1105         int err = 0, i = 0;
1106         bool allocated = false;
1107         u32 pde_lo, pde_hi;
1108         struct device *d = dev_from_vm(vm);
1109
1110         /* Allocate (or validate when map_offset != 0) the virtual address. */
1111         if (!map_offset) {
1112                 map_offset = gk20a_vm_alloc_va(vm, size,
1113                                           pgsz_idx);
1114                 if (!map_offset) {
1115                         gk20a_err(d, "failed to allocate va space");
1116                         err = -ENOMEM;
1117                         goto fail_alloc;
1118                 }
1119                 allocated = true;
1120         }
1121
1122         pde_range_from_vaddr_range(vm,
1123                                    map_offset,
1124                                    map_offset + size - 1,
1125                                    &pde_lo, &pde_hi);
1126
1127         /* mark the addr range valid (but with 0 phys addr, which will fault) */
1128         for (i = pde_lo; i <= pde_hi; i++) {
1129                 err = validate_gmmu_page_table_gk20a_locked(vm, i,
1130                                                             pgsz_idx);
1131                 if (err) {
1132                         gk20a_err(d, "failed to validate page table %d: %d",
1133                                                            i, err);
1134                         goto fail_validate;
1135                 }
1136         }
1137
1138         err = update_gmmu_ptes_locked(vm, pgsz_idx,
1139                                       sgt,
1140                                       map_offset, map_offset + size - 1,
1141                                       kind_v,
1142                                       ctag_offset,
1143                                       flags &
1144                                       NVHOST_MAP_BUFFER_FLAGS_CACHEABLE_TRUE,
1145                                       rw_flag);
1146         if (err) {
1147                 gk20a_err(d, "failed to update ptes on map");
1148                 goto fail_validate;
1149         }
1150
1151         return map_offset;
1152 fail_validate:
1153         if (allocated)
1154                 gk20a_vm_free_va(vm, map_offset, size, pgsz_idx);
1155 fail_alloc:
1156         gk20a_err(d, "%s: failed with err=%d\n", __func__, err);
1157         return 0;
1158 }
1159
1160 static void __locked_gmmu_unmap(struct vm_gk20a *vm,
1161                                 u64 vaddr,
1162                                 u64 size,
1163                                 int pgsz_idx,
1164                                 bool va_allocated,
1165                                 int rw_flag)
1166 {
1167         int err = 0;
1168         struct gk20a *g = gk20a_from_vm(vm);
1169
1170         if (va_allocated) {
1171                 err = gk20a_vm_free_va(vm, vaddr, size, pgsz_idx);
1172                 if (err) {
1173                         dev_err(dev_from_vm(vm),
1174                                 "failed to free va");
1175                         return;
1176                 }
1177         }
1178
1179         /* unmap here needs to know the page size we assigned at mapping */
1180         err = update_gmmu_ptes_locked(vm,
1181                                 pgsz_idx,
1182                                 0, /* n/a for unmap */
1183                                 vaddr,
1184                                 vaddr + size - 1,
1185                                 0, 0, false /* n/a for unmap */,
1186                                 rw_flag);
1187         if (err)
1188                 dev_err(dev_from_vm(vm),
1189                         "failed to update gmmu ptes on unmap");
1190
1191         /* detect which if any pdes/ptes can now be released */
1192
1193         /* flush l2 so any dirty lines are written out *now*.
1194          *  also as we could potentially be switching this buffer
1195          * from nonvolatile (l2 cacheable) to volatile (l2 non-cacheable) at
1196          * some point in the future we need to invalidate l2.  e.g. switching
1197          * from a render buffer unmap (here) to later using the same memory
1198          * for gmmu ptes.  note the positioning of this relative to any smmu
1199          * unmapping (below). */
1200
1201         gk20a_mm_l2_flush(g, true);
1202 }
1203
1204 static u64 gk20a_vm_map_duplicate_locked(struct vm_gk20a *vm,
1205                                          struct dma_buf *dmabuf,
1206                                          u64 offset_align,
1207                                          u32 flags,
1208                                          int kind,
1209                                          struct sg_table **sgt,
1210                                          bool user_mapped,
1211                                          int rw_flag)
1212 {
1213         struct mapped_buffer_node *mapped_buffer = 0;
1214
1215         mapped_buffer =
1216                 find_mapped_buffer_reverse_locked(&vm->mapped_buffers,
1217                                                   dmabuf, kind);
1218         if (!mapped_buffer)
1219                 return 0;
1220
1221         if (mapped_buffer->flags != flags)
1222                 return 0;
1223
1224         if (flags & NVHOST_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET &&
1225             mapped_buffer->addr != offset_align)
1226                 return 0;
1227
1228         BUG_ON(mapped_buffer->vm != vm);
1229
1230         /* mark the buffer as used */
1231         if (user_mapped) {
1232                 if (mapped_buffer->user_mapped == 0)
1233                         vm->num_user_mapped_buffers++;
1234                 mapped_buffer->user_mapped++;
1235
1236                 /* If the mapping comes from user space, we own
1237                  * the handle ref. Since we reuse an
1238                  * existing mapping here, we need to give back those
1239                  * refs once in order not to leak.
1240                  */
1241                 if (mapped_buffer->own_mem_ref)
1242                         dma_buf_put(mapped_buffer->dmabuf);
1243                 else
1244                         mapped_buffer->own_mem_ref = true;
1245         }
1246         kref_get(&mapped_buffer->ref);
1247
1248         gk20a_dbg(gpu_dbg_map,
1249                    "reusing as=%d pgsz=%d flags=0x%x ctags=%d "
1250                    "start=%d gv=0x%x,%08x -> 0x%x,%08x -> 0x%x,%08x "
1251                    "own_mem_ref=%d user_mapped=%d",
1252                    vm_aspace_id(vm), mapped_buffer->pgsz_idx,
1253                    mapped_buffer->flags,
1254                    mapped_buffer->ctag_lines,
1255                    mapped_buffer->ctag_offset,
1256                    hi32(mapped_buffer->addr), lo32(mapped_buffer->addr),
1257                    hi32((u64)sg_dma_address(mapped_buffer->sgt->sgl)),
1258                    lo32((u64)sg_dma_address(mapped_buffer->sgt->sgl)),
1259                    hi32((u64)sg_phys(mapped_buffer->sgt->sgl)),
1260                    lo32((u64)sg_phys(mapped_buffer->sgt->sgl)),
1261                    mapped_buffer->own_mem_ref, user_mapped);
1262
1263         if (sgt)
1264                 *sgt = mapped_buffer->sgt;
1265         return mapped_buffer->addr;
1266 }
1267
1268 u64 gk20a_vm_map(struct vm_gk20a *vm,
1269                         struct dma_buf *dmabuf,
1270                         u64 offset_align,
1271                         u32 flags /*NVHOST_AS_MAP_BUFFER_FLAGS_*/,
1272                         int kind,
1273                         struct sg_table **sgt,
1274                         bool user_mapped,
1275                         int rw_flag)
1276 {
1277         struct gk20a *g = gk20a_from_vm(vm);
1278         struct gk20a_allocator *ctag_allocator = &g->gr.comp_tags;
1279         struct device *d = dev_from_vm(vm);
1280         struct mapped_buffer_node *mapped_buffer = 0;
1281         bool inserted = false, va_allocated = false;
1282         u32 gmmu_page_size = 0;
1283         u64 map_offset = 0;
1284         int err = 0;
1285         struct buffer_attrs bfr = {0};
1286         struct gk20a_comptags comptags;
1287
1288         mutex_lock(&vm->update_gmmu_lock);
1289
1290         /* check if this buffer is already mapped */
1291         map_offset = gk20a_vm_map_duplicate_locked(vm, dmabuf, offset_align,
1292                                                    flags, kind, sgt,
1293                                                    user_mapped, rw_flag);
1294         if (map_offset) {
1295                 mutex_unlock(&vm->update_gmmu_lock);
1296                 return map_offset;
1297         }
1298
1299         /* pin buffer to get phys/iovmm addr */
1300         bfr.sgt = gk20a_mm_pin(d, dmabuf);
1301         if (IS_ERR(bfr.sgt)) {
1302                 /* Falling back to physical is actually possible
1303                  * here in many cases if we use 4K phys pages in the
1304                  * gmmu.  However we have some regions which require
1305                  * contig regions to work properly (either phys-contig
1306                  * or contig through smmu io_vaspace).  Until we can
1307                  * track the difference between those two cases we have
1308                  * to fail the mapping when we run out of SMMU space.
1309                  */
1310                 gk20a_warn(d, "oom allocating tracking buffer");
1311                 goto clean_up;
1312         }
1313
1314         if (sgt)
1315                 *sgt = bfr.sgt;
1316
1317         bfr.kind_v = kind;
1318         bfr.size = dmabuf->size;
1319         bfr.align = 1 << __ffs((u64)sg_dma_address(bfr.sgt->sgl));
1320         bfr.pgsz_idx = -1;
1321
1322         /* If FIX_OFFSET is set, pgsz is determined. Otherwise, select
1323          * page size according to memory alignment */
1324         if (flags & NVHOST_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET) {
1325                 bfr.pgsz_idx = NV_GMMU_VA_IS_UPPER(offset_align) ?
1326                                 gmmu_page_size_big : gmmu_page_size_small;
1327         } else {
1328                 if (vm->big_pages)
1329                         gmmu_select_page_size(&bfr);
1330                 else
1331                         bfr.pgsz_idx = gmmu_page_size_small;
1332         }
1333
1334         /* validate/adjust bfr attributes */
1335         if (unlikely(bfr.pgsz_idx == -1)) {
1336                 gk20a_err(d, "unsupported page size detected");
1337                 goto clean_up;
1338         }
1339
1340         if (unlikely(bfr.pgsz_idx < gmmu_page_size_small ||
1341                      bfr.pgsz_idx > gmmu_page_size_big)) {
1342                 BUG_ON(1);
1343                 err = -EINVAL;
1344                 goto clean_up;
1345         }
1346         gmmu_page_size = gmmu_page_sizes[bfr.pgsz_idx];
1347
1348         /* Check if we should use a fixed offset for mapping this buffer */
1349         if (flags & NVHOST_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET)  {
1350                 err = validate_fixed_buffer(vm, &bfr, offset_align);
1351                 if (err)
1352                         goto clean_up;
1353
1354                 map_offset = offset_align;
1355                 va_allocated = false;
1356         } else
1357                 va_allocated = true;
1358
1359         if (sgt)
1360                 *sgt = bfr.sgt;
1361
1362         err = setup_buffer_kind_and_compression(d, flags, &bfr, bfr.pgsz_idx);
1363         if (unlikely(err)) {
1364                 gk20a_err(d, "failure setting up kind and compression");
1365                 goto clean_up;
1366         }
1367
1368         /* bar1 and pmu vm don't need ctag */
1369         if (!vm->enable_ctag)
1370                 bfr.ctag_lines = 0;
1371
1372         gk20a_get_comptags(d, dmabuf, &comptags);
1373
1374         if (bfr.ctag_lines && !comptags.lines) {
1375                 /* allocate compression resources if needed */
1376                 err = gk20a_alloc_comptags(d, dmabuf, ctag_allocator,
1377                                            bfr.ctag_lines);
1378                 if (err) {
1379                         /* ok to fall back here if we ran out */
1380                         /* TBD: we can partially alloc ctags as well... */
1381                         bfr.ctag_lines = bfr.ctag_offset = 0;
1382                         bfr.kind_v = bfr.uc_kind_v;
1383                 } else {
1384                         gk20a_get_comptags(d, dmabuf, &comptags);
1385
1386                         /* init/clear the ctag buffer */
1387                         g->ops.ltc.cbc_ctrl(g, gk20a_cbc_op_clear,
1388                                           comptags.offset,
1389                                           comptags.offset + comptags.lines - 1);
1390                 }
1391         }
1392
1393         /* store the comptag info */
1394         bfr.ctag_offset = comptags.offset;
1395
1396         /* update gmmu ptes */
1397         map_offset = __locked_gmmu_map(vm, map_offset,
1398                                         bfr.sgt,
1399                                         bfr.size,
1400                                         bfr.pgsz_idx,
1401                                         bfr.kind_v,
1402                                         bfr.ctag_offset,
1403                                         flags, rw_flag);
1404         if (!map_offset)
1405                 goto clean_up;
1406
1407         gk20a_dbg(gpu_dbg_map,
1408            "as=%d pgsz=%d "
1409            "kind=0x%x kind_uc=0x%x flags=0x%x "
1410            "ctags=%d start=%d gv=0x%x,%08x -> 0x%x,%08x -> 0x%x,%08x",
1411            vm_aspace_id(vm), gmmu_page_size,
1412            bfr.kind_v, bfr.uc_kind_v, flags,
1413            bfr.ctag_lines, bfr.ctag_offset,
1414            hi32(map_offset), lo32(map_offset),
1415            hi32((u64)sg_dma_address(bfr.sgt->sgl)),
1416            lo32((u64)sg_dma_address(bfr.sgt->sgl)),
1417            hi32((u64)sg_phys(bfr.sgt->sgl)),
1418            lo32((u64)sg_phys(bfr.sgt->sgl)));
1419
1420 #if defined(NVHOST_DEBUG)
1421         {
1422                 int i;
1423                 struct scatterlist *sg = NULL;
1424                 gk20a_dbg(gpu_dbg_pte, "for_each_sg(bfr.sgt->sgl, sg, bfr.sgt->nents, i)");
1425                 for_each_sg(bfr.sgt->sgl, sg, bfr.sgt->nents, i ) {
1426                         u64 da = sg_dma_address(sg);
1427                         u64 pa = sg_phys(sg);
1428                         u64 len = sg->length;
1429                         gk20a_dbg(gpu_dbg_pte, "i=%d pa=0x%x,%08x da=0x%x,%08x len=0x%x,%08x",
1430                                    i, hi32(pa), lo32(pa), hi32(da), lo32(da),
1431                                    hi32(len), lo32(len));
1432                 }
1433         }
1434 #endif
1435
1436         /* keep track of the buffer for unmapping */
1437         /* TBD: check for multiple mapping of same buffer */
1438         mapped_buffer = kzalloc(sizeof(*mapped_buffer), GFP_KERNEL);
1439         if (!mapped_buffer) {
1440                 gk20a_warn(d, "oom allocating tracking buffer");
1441                 goto clean_up;
1442         }
1443         mapped_buffer->dmabuf      = dmabuf;
1444         mapped_buffer->sgt         = bfr.sgt;
1445         mapped_buffer->addr        = map_offset;
1446         mapped_buffer->size        = bfr.size;
1447         mapped_buffer->pgsz_idx    = bfr.pgsz_idx;
1448         mapped_buffer->ctag_offset = bfr.ctag_offset;
1449         mapped_buffer->ctag_lines  = bfr.ctag_lines;
1450         mapped_buffer->vm          = vm;
1451         mapped_buffer->flags       = flags;
1452         mapped_buffer->kind        = kind;
1453         mapped_buffer->va_allocated = va_allocated;
1454         mapped_buffer->user_mapped = user_mapped ? 1 : 0;
1455         mapped_buffer->own_mem_ref = user_mapped;
1456         INIT_LIST_HEAD(&mapped_buffer->unmap_list);
1457         INIT_LIST_HEAD(&mapped_buffer->va_buffers_list);
1458         kref_init(&mapped_buffer->ref);
1459
1460         err = insert_mapped_buffer(&vm->mapped_buffers, mapped_buffer);
1461         if (err) {
1462                 gk20a_err(d, "failed to insert into mapped buffer tree");
1463                 goto clean_up;
1464         }
1465         inserted = true;
1466         if (user_mapped)
1467                 vm->num_user_mapped_buffers++;
1468
1469         gk20a_dbg_info("allocated va @ 0x%llx", map_offset);
1470
1471         if (!va_allocated) {
1472                 struct vm_reserved_va_node *va_node;
1473
1474                 /* find the space reservation */
1475                 va_node = addr_to_reservation(vm, map_offset);
1476                 list_add_tail(&mapped_buffer->va_buffers_list,
1477                               &va_node->va_buffers_list);
1478                 mapped_buffer->va_node = va_node;
1479         }
1480
1481         mutex_unlock(&vm->update_gmmu_lock);
1482
1483         /* Invalidate kernel mappings immediately */
1484         if (vm_aspace_id(vm) == -1)
1485                 gk20a_mm_tlb_invalidate(vm);
1486
1487         return map_offset;
1488
1489 clean_up:
1490         if (inserted) {
1491                 rb_erase(&mapped_buffer->node, &vm->mapped_buffers);
1492                 if (user_mapped)
1493                         vm->num_user_mapped_buffers--;
1494         }
1495         kfree(mapped_buffer);
1496         if (va_allocated)
1497                 gk20a_vm_free_va(vm, map_offset, bfr.size, bfr.pgsz_idx);
1498         if (!IS_ERR(bfr.sgt))
1499                 gk20a_mm_unpin(d, dmabuf, bfr.sgt);
1500
1501         mutex_unlock(&vm->update_gmmu_lock);
1502         gk20a_dbg_info("err=%d\n", err);
1503         return 0;
1504 }
1505
1506 u64 gk20a_gmmu_map(struct vm_gk20a *vm,
1507                 struct sg_table **sgt,
1508                 u64 size,
1509                 u32 flags,
1510                 int rw_flag)
1511 {
1512         u64 vaddr;
1513
1514         mutex_lock(&vm->update_gmmu_lock);
1515         vaddr = __locked_gmmu_map(vm, 0, /* already mapped? - No */
1516                                 *sgt, /* sg table */
1517                                 size,
1518                                 0, /* page size index = 0 i.e. SZ_4K */
1519                                 0, /* kind */
1520                                 0, /* ctag_offset */
1521                                 flags, rw_flag);
1522         mutex_unlock(&vm->update_gmmu_lock);
1523         if (!vaddr) {
1524                 gk20a_err(dev_from_vm(vm), "failed to allocate va space");
1525                 return 0;
1526         }
1527
1528         /* Invalidate kernel mappings immediately */
1529         gk20a_mm_tlb_invalidate(vm);
1530
1531         return vaddr;
1532 }
1533
1534 void gk20a_gmmu_unmap(struct vm_gk20a *vm,
1535                 u64 vaddr,
1536                 u64 size,
1537                 int rw_flag)
1538 {
1539         mutex_lock(&vm->update_gmmu_lock);
1540         __locked_gmmu_unmap(vm,
1541                         vaddr,
1542                         size,
1543                         0, /* page size 4K */
1544                         true, /*va_allocated */
1545                         rw_flag);
1546         mutex_unlock(&vm->update_gmmu_lock);
1547 }
1548
1549 phys_addr_t gk20a_get_phys_from_iova(struct device *d,
1550                                 u64 dma_addr)
1551 {
1552         phys_addr_t phys;
1553         u64 iova;
1554
1555         struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(d);
1556         if (!mapping)
1557                 return dma_addr;
1558
1559         iova = dma_addr & PAGE_MASK;
1560         phys = iommu_iova_to_phys(mapping->domain, iova);
1561         return phys;
1562 }
1563
1564 /* get sg_table from already allocated buffer */
1565 int gk20a_get_sgtable(struct device *d, struct sg_table **sgt,
1566                         void *cpuva, u64 iova,
1567                         size_t size)
1568 {
1569         int err = 0;
1570         *sgt = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
1571         if (!(*sgt)) {
1572                 dev_err(d, "failed to allocate memory\n");
1573                 err = -ENOMEM;
1574                 goto fail;
1575         }
1576         err = dma_get_sgtable(d, *sgt,
1577                         cpuva, iova,
1578                         size);
1579         if (err) {
1580                 dev_err(d, "failed to create sg table\n");
1581                 goto fail;
1582         }
1583         sg_dma_address((*sgt)->sgl) = iova;
1584
1585         return 0;
1586  fail:
1587         if (*sgt) {
1588                 kfree(*sgt);
1589                 *sgt = NULL;
1590         }
1591         return err;
1592 }
1593
1594 int gk20a_get_sgtable_from_pages(struct device *d, struct sg_table **sgt,
1595                         struct page **pages, u64 iova,
1596                         size_t size)
1597 {
1598         int err = 0;
1599         *sgt = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
1600         if (!(*sgt)) {
1601                 dev_err(d, "failed to allocate memory\n");
1602                 err = -ENOMEM;
1603                 goto fail;
1604         }
1605         err = sg_alloc_table(*sgt, 1, GFP_KERNEL);
1606         if (err) {
1607                 dev_err(d, "failed to allocate sg_table\n");
1608                 goto fail;
1609         }
1610         sg_set_page((*sgt)->sgl, *pages, size, 0);
1611         sg_dma_address((*sgt)->sgl) = iova;
1612
1613         return 0;
1614  fail:
1615         if (*sgt) {
1616                 kfree(*sgt);
1617                 *sgt = NULL;
1618         }
1619         return err;
1620 }
1621
1622 void gk20a_free_sgtable(struct sg_table **sgt)
1623 {
1624         sg_free_table(*sgt);
1625         kfree(*sgt);
1626         *sgt = NULL;
1627 }
1628
1629 u64 gk20a_mm_iova_addr(struct scatterlist *sgl)
1630 {
1631         u64 result = sg_phys(sgl);
1632 #ifdef CONFIG_TEGRA_IOMMU_SMMU
1633         if (sg_dma_address(sgl) == DMA_ERROR_CODE)
1634                 result = 0;
1635         else if (sg_dma_address(sgl)) {
1636                 result = sg_dma_address(sgl) |
1637                         1ULL << NV_MC_SMMU_VADDR_TRANSLATION_BIT;
1638         }
1639 #endif
1640         return result;
1641 }
1642
1643 static int update_gmmu_ptes_locked(struct vm_gk20a *vm,
1644                                    enum gmmu_pgsz_gk20a pgsz_idx,
1645                                    struct sg_table *sgt,
1646                                    u64 first_vaddr, u64 last_vaddr,
1647                                    u8 kind_v, u32 ctag_offset,
1648                                    bool cacheable,
1649                                    int rw_flag)
1650 {
1651         int err;
1652         u32 pde_lo, pde_hi, pde_i;
1653         struct scatterlist *cur_chunk;
1654         unsigned int cur_offset;
1655         u32 pte_w[2] = {0, 0}; /* invalid pte */
1656         u32 ctag = ctag_offset;
1657         u32 ctag_incr;
1658         u32 page_size  = gmmu_page_sizes[pgsz_idx];
1659         u64 addr = 0;
1660
1661         pde_range_from_vaddr_range(vm, first_vaddr, last_vaddr,
1662                                    &pde_lo, &pde_hi);
1663
1664         gk20a_dbg(gpu_dbg_pte, "size_idx=%d, pde_lo=%d, pde_hi=%d",
1665                    pgsz_idx, pde_lo, pde_hi);
1666
1667         /* If ctag_offset !=0 add 1 else add 0.  The idea is to avoid a branch
1668          * below (per-pte). Note: this doesn't work unless page size (when
1669          * comptags are active) is 128KB. We have checks elsewhere for that. */
1670         ctag_incr = !!ctag_offset;
1671
1672         if (sgt)
1673                 cur_chunk = sgt->sgl;
1674         else
1675                 cur_chunk = NULL;
1676
1677         cur_offset = 0;
1678
1679         for (pde_i = pde_lo; pde_i <= pde_hi; pde_i++) {
1680                 u32 pte_lo, pte_hi;
1681                 u32 pte_cur;
1682                 void *pte_kv_cur;
1683
1684                 struct page_table_gk20a *pte = vm->pdes.ptes[pgsz_idx] + pde_i;
1685
1686                 if (pde_i == pde_lo)
1687                         pte_lo = pte_index_from_vaddr(vm, first_vaddr,
1688                                                       pgsz_idx);
1689                 else
1690                         pte_lo = 0;
1691
1692                 if ((pde_i != pde_hi) && (pde_hi != pde_lo))
1693                         pte_hi = vm->mm->page_table_sizing[pgsz_idx].num_ptes-1;
1694                 else
1695                         pte_hi = pte_index_from_vaddr(vm, last_vaddr,
1696                                                       pgsz_idx);
1697
1698                 /* get cpu access to the ptes */
1699                 err = map_gmmu_pages(pte->ref, pte->sgt, &pte_kv_cur,
1700                                      pte->size);
1701                 if (err) {
1702                         gk20a_err(dev_from_vm(vm),
1703                                    "couldn't map ptes for update as=%d pte_ref_cnt=%d",
1704                                    vm_aspace_id(vm), pte->ref_cnt);
1705                         goto clean_up;
1706                 }
1707
1708                 gk20a_dbg(gpu_dbg_pte, "pte_lo=%d, pte_hi=%d", pte_lo, pte_hi);
1709                 for (pte_cur = pte_lo; pte_cur <= pte_hi; pte_cur++) {
1710
1711                         if (likely(sgt)) {
1712                                 u64 new_addr = gk20a_mm_iova_addr(cur_chunk);
1713                                 if (new_addr) {
1714                                         addr = new_addr;
1715                                         addr += cur_offset;
1716                                 }
1717
1718                                 pte_w[0] = gmmu_pte_valid_true_f() |
1719                                         gmmu_pte_address_sys_f(addr
1720                                                 >> gmmu_pte_address_shift_v());
1721                                 pte_w[1] = gmmu_pte_aperture_video_memory_f() |
1722                                         gmmu_pte_kind_f(kind_v) |
1723                                         gmmu_pte_comptagline_f(ctag);
1724
1725                                 if (rw_flag == gk20a_mem_flag_read_only) {
1726                                         pte_w[0] |= gmmu_pte_read_only_true_f();
1727                                         pte_w[1] |=
1728                                                 gmmu_pte_write_disable_true_f();
1729                                 } else if (rw_flag ==
1730                                            gk20a_mem_flag_write_only) {
1731                                         pte_w[1] |=
1732                                                 gmmu_pte_read_disable_true_f();
1733                                 }
1734
1735                                 if (!cacheable)
1736                                         pte_w[1] |= gmmu_pte_vol_true_f();
1737
1738                                 pte->ref_cnt++;
1739
1740                                 gk20a_dbg(gpu_dbg_pte,
1741                                            "pte_cur=%d addr=0x%x,%08x kind=%d"
1742                                            " ctag=%d vol=%d refs=%d"
1743                                            " [0x%08x,0x%08x]",
1744                                            pte_cur, hi32(addr), lo32(addr),
1745                                            kind_v, ctag, !cacheable,
1746                                            pte->ref_cnt, pte_w[1], pte_w[0]);
1747
1748                                 ctag += ctag_incr;
1749                                 cur_offset += page_size;
1750                                 addr += page_size;
1751                                 while (cur_chunk &&
1752                                         cur_offset >= cur_chunk->length) {
1753                                         cur_offset -= cur_chunk->length;
1754                                         cur_chunk = sg_next(cur_chunk);
1755                                 }
1756
1757                         } else {
1758                                 pte->ref_cnt--;
1759                                 gk20a_dbg(gpu_dbg_pte,
1760                                            "pte_cur=%d ref=%d [0x0,0x0]",
1761                                            pte_cur, pte->ref_cnt);
1762                         }
1763
1764                         gk20a_mem_wr32(pte_kv_cur + pte_cur*8, 0, pte_w[0]);
1765                         gk20a_mem_wr32(pte_kv_cur + pte_cur*8, 1, pte_w[1]);
1766                 }
1767
1768                 unmap_gmmu_pages(pte->ref, pte->sgt, pte_kv_cur);
1769
1770                 if (pte->ref_cnt == 0) {
1771                         /* It can make sense to keep around one page table for
1772                          * each flavor (empty)... in case a new map is coming
1773                          * right back to alloc (and fill it in) again.
1774                          * But: deferring unmapping should help with pathologic
1775                          * unmap/map/unmap/map cases where we'd trigger pte
1776                          * free/alloc/free/alloc.
1777                          */
1778                         free_gmmu_pages(vm, pte->ref, pte->sgt,
1779                                 vm->mm->page_table_sizing[pgsz_idx].order,
1780                                 pte->size);
1781                         pte->ref = NULL;
1782
1783                         /* rewrite pde */
1784                         update_gmmu_pde_locked(vm, pde_i);
1785                 }
1786
1787         }
1788
1789         smp_mb();
1790         vm->tlb_dirty = true;
1791         gk20a_dbg_fn("set tlb dirty");
1792
1793         return 0;
1794
1795 clean_up:
1796         /*TBD: potentially rewrite above to pre-map everything it needs to
1797          * as that's the only way it can fail */
1798         return err;
1799
1800 }
1801
1802
1803 /* for gk20a the "video memory" apertures here are misnomers. */
1804 static inline u32 big_valid_pde0_bits(u64 pte_addr)
1805 {
1806         u32 pde0_bits =
1807                 gmmu_pde_aperture_big_video_memory_f() |
1808                 gmmu_pde_address_big_sys_f(
1809                            (u32)(pte_addr >> gmmu_pde_address_shift_v()));
1810         return  pde0_bits;
1811 }
1812 static inline u32 small_valid_pde1_bits(u64 pte_addr)
1813 {
1814         u32 pde1_bits =
1815                 gmmu_pde_aperture_small_video_memory_f() |
1816                 gmmu_pde_vol_small_true_f() | /* tbd: why? */
1817                 gmmu_pde_address_small_sys_f(
1818                            (u32)(pte_addr >> gmmu_pde_address_shift_v()));
1819         return pde1_bits;
1820 }
1821
1822 /* Given the current state of the ptes associated with a pde,
1823    determine value and write it out.  There's no checking
1824    here to determine whether or not a change was actually
1825    made.  So, superfluous updates will cause unnecessary
1826    pde invalidations.
1827 */
1828 static void update_gmmu_pde_locked(struct vm_gk20a *vm, u32 i)
1829 {
1830         bool small_valid, big_valid;
1831         u64 pte_addr[2] = {0, 0};
1832         struct page_table_gk20a *small_pte =
1833                 vm->pdes.ptes[gmmu_page_size_small] + i;
1834         struct page_table_gk20a *big_pte =
1835                 vm->pdes.ptes[gmmu_page_size_big] + i;
1836         u32 pde_v[2] = {0, 0};
1837         u32 *pde;
1838
1839         small_valid = small_pte && small_pte->ref;
1840         big_valid   = big_pte && big_pte->ref;
1841
1842         if (small_valid)
1843                 pte_addr[gmmu_page_size_small] =
1844                         gk20a_mm_iova_addr(small_pte->sgt->sgl);
1845         if (big_valid)
1846                 pte_addr[gmmu_page_size_big] =
1847                         gk20a_mm_iova_addr(big_pte->sgt->sgl);
1848
1849         pde_v[0] = gmmu_pde_size_full_f();
1850         pde_v[0] |= big_valid ?
1851                 big_valid_pde0_bits(pte_addr[gmmu_page_size_big])
1852                 :
1853                 (gmmu_pde_aperture_big_invalid_f());
1854
1855         pde_v[1] |= (small_valid ?
1856                      small_valid_pde1_bits(pte_addr[gmmu_page_size_small])
1857                      :
1858                      (gmmu_pde_aperture_small_invalid_f() |
1859                       gmmu_pde_vol_small_false_f())
1860                      )
1861                 |
1862                 (big_valid ? (gmmu_pde_vol_big_true_f()) :
1863                  gmmu_pde_vol_big_false_f());
1864
1865         pde = pde_from_index(vm, i);
1866
1867         gk20a_mem_wr32(pde, 0, pde_v[0]);
1868         gk20a_mem_wr32(pde, 1, pde_v[1]);
1869
1870         smp_mb();
1871
1872         FLUSH_CPU_DCACHE(pde,
1873                          sg_phys(vm->pdes.sgt->sgl) + (i*gmmu_pde__size_v()),
1874                          sizeof(u32)*2);
1875
1876         gk20a_mm_l2_invalidate(vm->mm->g);
1877
1878         gk20a_dbg(gpu_dbg_pte, "pde:%d = 0x%x,0x%08x\n", i, pde_v[1], pde_v[0]);
1879
1880         vm->tlb_dirty  = true;
1881 }
1882
1883
1884 static int gk20a_vm_put_empty(struct vm_gk20a *vm, u64 vaddr,
1885                                u32 num_pages, u32 pgsz_idx)
1886 {
1887         struct mm_gk20a *mm = vm->mm;
1888         struct gk20a *g = mm->g;
1889         u32 pgsz = gmmu_page_sizes[pgsz_idx];
1890         u32 i;
1891         dma_addr_t iova;
1892
1893         /* allocate the zero page if the va does not already have one */
1894         if (!vm->zero_page_cpuva) {
1895                 int err = 0;
1896                 vm->zero_page_cpuva = dma_alloc_coherent(&g->dev->dev,
1897                                                          mm->big_page_size,
1898                                                          &iova,
1899                                                          GFP_KERNEL);
1900                 if (!vm->zero_page_cpuva) {
1901                         dev_err(&g->dev->dev, "failed to allocate zero page\n");
1902                         return -ENOMEM;
1903                 }
1904
1905                 vm->zero_page_iova = iova;
1906                 err = gk20a_get_sgtable(&g->dev->dev, &vm->zero_page_sgt,
1907                                         vm->zero_page_cpuva, vm->zero_page_iova,
1908                                         mm->big_page_size);
1909                 if (err) {
1910                         dma_free_coherent(&g->dev->dev, mm->big_page_size,
1911                                           vm->zero_page_cpuva,
1912                                           vm->zero_page_iova);
1913                         vm->zero_page_iova = 0;
1914                         vm->zero_page_cpuva = NULL;
1915
1916                         dev_err(&g->dev->dev, "failed to create sg table for zero page\n");
1917                         return -ENOMEM;
1918                 }
1919         }
1920
1921         for (i = 0; i < num_pages; i++) {
1922                 u64 page_vaddr = __locked_gmmu_map(vm, vaddr,
1923                         vm->zero_page_sgt, pgsz, pgsz_idx, 0, 0,
1924                         NVHOST_AS_ALLOC_SPACE_FLAGS_FIXED_OFFSET,
1925                         gk20a_mem_flag_none);
1926
1927                 if (!page_vaddr) {
1928                         gk20a_err(dev_from_vm(vm), "failed to remap clean buffers!");
1929                         goto err_unmap;
1930                 }
1931                 vaddr += pgsz;
1932         }
1933
1934         gk20a_mm_l2_flush(mm->g, true);
1935
1936         return 0;
1937
1938 err_unmap:
1939
1940         WARN_ON(1);
1941         /* something went wrong. unmap pages */
1942         while (i--) {
1943                 vaddr -= pgsz;
1944                 __locked_gmmu_unmap(vm, vaddr, pgsz, pgsz_idx, 0,
1945                                     gk20a_mem_flag_none);
1946         }
1947
1948         return -EINVAL;
1949 }
1950
1951 /* NOTE! mapped_buffers lock must be held */
1952 static void gk20a_vm_unmap_locked(struct mapped_buffer_node *mapped_buffer)
1953 {
1954         struct vm_gk20a *vm = mapped_buffer->vm;
1955
1956         if (mapped_buffer->va_node &&
1957             mapped_buffer->va_node->sparse) {
1958                 u64 vaddr = mapped_buffer->addr;
1959                 u32 pgsz_idx = mapped_buffer->pgsz_idx;
1960                 u32 num_pages = mapped_buffer->size >>
1961                         gmmu_page_shifts[pgsz_idx];
1962
1963                 /* there is little we can do if this fails... */
1964                 gk20a_vm_put_empty(vm, vaddr, num_pages, pgsz_idx);
1965
1966         } else
1967                 __locked_gmmu_unmap(vm,
1968                                 mapped_buffer->addr,
1969                                 mapped_buffer->size,
1970                                 mapped_buffer->pgsz_idx,
1971                                 mapped_buffer->va_allocated,
1972                                 gk20a_mem_flag_none);
1973
1974         gk20a_dbg(gpu_dbg_map, "as=%d pgsz=%d gv=0x%x,%08x own_mem_ref=%d",
1975                    vm_aspace_id(vm), gmmu_page_sizes[mapped_buffer->pgsz_idx],
1976                    hi32(mapped_buffer->addr), lo32(mapped_buffer->addr),
1977                    mapped_buffer->own_mem_ref);
1978
1979         gk20a_mm_unpin(dev_from_vm(vm), mapped_buffer->dmabuf,
1980                        mapped_buffer->sgt);
1981
1982         /* remove from mapped buffer tree and remove list, free */
1983         rb_erase(&mapped_buffer->node, &vm->mapped_buffers);
1984         if (!list_empty(&mapped_buffer->va_buffers_list))
1985                 list_del(&mapped_buffer->va_buffers_list);
1986
1987         /* keep track of mapped buffers */
1988         if (mapped_buffer->user_mapped)
1989                 vm->num_user_mapped_buffers--;
1990
1991         if (mapped_buffer->own_mem_ref)
1992                 dma_buf_put(mapped_buffer->dmabuf);
1993
1994         kfree(mapped_buffer);
1995
1996         return;
1997 }
1998
1999 void gk20a_vm_unmap(struct vm_gk20a *vm, u64 offset)
2000 {
2001         struct device *d = dev_from_vm(vm);
2002         struct mapped_buffer_node *mapped_buffer;
2003
2004         mutex_lock(&vm->update_gmmu_lock);
2005         mapped_buffer = find_mapped_buffer_locked(&vm->mapped_buffers, offset);
2006         if (!mapped_buffer) {
2007                 mutex_unlock(&vm->update_gmmu_lock);
2008                 gk20a_err(d, "invalid addr to unmap 0x%llx", offset);
2009                 return;
2010         }
2011         kref_put(&mapped_buffer->ref, gk20a_vm_unmap_locked_kref);
2012         mutex_unlock(&vm->update_gmmu_lock);
2013 }
2014
2015 static void gk20a_vm_remove_support(struct vm_gk20a *vm)
2016 {
2017         struct gk20a *g = vm->mm->g;
2018         struct mapped_buffer_node *mapped_buffer;
2019         struct vm_reserved_va_node *va_node, *va_node_tmp;
2020         struct rb_node *node;
2021         int i;
2022
2023         gk20a_dbg_fn("");
2024         mutex_lock(&vm->update_gmmu_lock);
2025
2026         /* TBD: add a flag here for the unmap code to recognize teardown
2027          * and short-circuit any otherwise expensive operations. */
2028
2029         node = rb_first(&vm->mapped_buffers);
2030         while (node) {
2031                 mapped_buffer =
2032                         container_of(node, struct mapped_buffer_node, node);
2033                 gk20a_vm_unmap_locked(mapped_buffer);
2034                 node = rb_first(&vm->mapped_buffers);
2035         }
2036
2037         /* destroy remaining reserved memory areas */
2038         list_for_each_entry_safe(va_node, va_node_tmp, &vm->reserved_va_list,
2039                 reserved_va_list) {
2040                 list_del(&va_node->reserved_va_list);
2041                 kfree(va_node);
2042         }
2043
2044         /* unmapping all buffers above may not actually free
2045          * all vm ptes.  jettison them here for certain... */
2046         for (i = 0; i < vm->pdes.num_pdes; i++) {
2047                 struct page_table_gk20a *pte =
2048                         &vm->pdes.ptes[gmmu_page_size_small][i];
2049                 if (pte->ref) {
2050                         free_gmmu_pages(vm, pte->ref, pte->sgt,
2051                                 vm->mm->page_table_sizing[gmmu_page_size_small].order,
2052                                 pte->size);
2053                         pte->ref = NULL;
2054                 }
2055                 pte = &vm->pdes.ptes[gmmu_page_size_big][i];
2056                 if (pte->ref) {
2057                         free_gmmu_pages(vm, pte->ref, pte->sgt,
2058                                 vm->mm->page_table_sizing[gmmu_page_size_big].order,
2059                                 pte->size);
2060                         pte->ref = NULL;
2061                 }
2062         }
2063
2064         unmap_gmmu_pages(vm->pdes.ref, vm->pdes.sgt, vm->pdes.kv);
2065         free_gmmu_pages(vm, vm->pdes.ref, vm->pdes.sgt, 0, vm->pdes.size);
2066
2067         kfree(vm->pdes.ptes[gmmu_page_size_small]);
2068         kfree(vm->pdes.ptes[gmmu_page_size_big]);
2069         gk20a_allocator_destroy(&vm->vma[gmmu_page_size_small]);
2070         gk20a_allocator_destroy(&vm->vma[gmmu_page_size_big]);
2071
2072         mutex_unlock(&vm->update_gmmu_lock);
2073
2074         /* release zero page if used */
2075         if (vm->zero_page_cpuva)
2076                 dma_free_coherent(&g->dev->dev, vm->mm->big_page_size,
2077                                   vm->zero_page_cpuva, vm->zero_page_iova);
2078
2079         /* vm is not used anymore. release it. */
2080         kfree(vm);
2081 }
2082
2083 static void gk20a_vm_remove_support_kref(struct kref *ref)
2084 {
2085         struct vm_gk20a *vm = container_of(ref, struct vm_gk20a, ref);
2086         gk20a_vm_remove_support(vm);
2087 }
2088
2089 void gk20a_vm_get(struct vm_gk20a *vm)
2090 {
2091         kref_get(&vm->ref);
2092 }
2093
2094 void gk20a_vm_put(struct vm_gk20a *vm)
2095 {
2096         kref_put(&vm->ref, gk20a_vm_remove_support_kref);
2097 }
2098
2099 /* address space interfaces for the gk20a module */
2100 int gk20a_vm_alloc_share(struct gk20a_as_share *as_share)
2101 {
2102         struct gk20a_as *as = as_share->as;
2103         struct gk20a *g = gk20a_from_as(as);
2104         struct mm_gk20a *mm = &g->mm;
2105         struct vm_gk20a *vm;
2106         u64 vma_size;
2107         u32 num_pages, low_hole_pages;
2108         char name[32];
2109         int err;
2110
2111         gk20a_dbg_fn("");
2112
2113         vm = kzalloc(sizeof(*vm), GFP_KERNEL);
2114         if (!vm)
2115                 return -ENOMEM;
2116
2117         as_share->vm = vm;
2118
2119         vm->mm = mm;
2120         vm->as_share = as_share;
2121
2122         vm->big_pages = true;
2123
2124         vm->va_start  = mm->pde_stride;   /* create a one pde hole */
2125         vm->va_limit  = mm->channel.size; /* note this means channel.size is
2126                                              really just the max */
2127         {
2128                 u32 pde_lo, pde_hi;
2129                 pde_range_from_vaddr_range(vm,
2130                                            0, vm->va_limit-1,
2131                                            &pde_lo, &pde_hi);
2132                 vm->pdes.num_pdes = pde_hi + 1;
2133         }
2134
2135         vm->pdes.ptes[gmmu_page_size_small] =
2136                 kzalloc(sizeof(struct page_table_gk20a) *
2137                         vm->pdes.num_pdes, GFP_KERNEL);
2138
2139         vm->pdes.ptes[gmmu_page_size_big] =
2140                 kzalloc(sizeof(struct page_table_gk20a) *
2141                         vm->pdes.num_pdes, GFP_KERNEL);
2142
2143         if (!(vm->pdes.ptes[gmmu_page_size_small] &&
2144               vm->pdes.ptes[gmmu_page_size_big]))
2145                 return -ENOMEM;
2146
2147         gk20a_dbg_info("init space for va_limit=0x%llx num_pdes=%d",
2148                    vm->va_limit, vm->pdes.num_pdes);
2149
2150         /* allocate the page table directory */
2151         err = alloc_gmmu_pages(vm, 0, &vm->pdes.ref,
2152                                &vm->pdes.sgt, &vm->pdes.size);
2153         if (err)
2154                 return -ENOMEM;
2155
2156         err = map_gmmu_pages(vm->pdes.ref, vm->pdes.sgt, &vm->pdes.kv,
2157                              vm->pdes.size);
2158         if (err) {
2159                 free_gmmu_pages(vm, vm->pdes.ref, vm->pdes.sgt, 0,
2160                                         vm->pdes.size);
2161                 return -ENOMEM;
2162         }
2163         gk20a_dbg(gpu_dbg_pte, "pdes.kv = 0x%p, pdes.phys = 0x%llx",
2164                         vm->pdes.kv,
2165                         gk20a_mm_iova_addr(vm->pdes.sgt->sgl));
2166         /* we could release vm->pdes.kv but it's only one page... */
2167
2168
2169         /* low-half: alloc small pages */
2170         /* high-half: alloc big pages */
2171         vma_size = mm->channel.size >> 1;
2172
2173         snprintf(name, sizeof(name), "gk20a_as_%d-%dKB", as_share->id,
2174                  gmmu_page_sizes[gmmu_page_size_small]>>10);
2175         num_pages = (u32)(vma_size >> gmmu_page_shifts[gmmu_page_size_small]);
2176
2177         /* num_pages above is without regard to the low-side hole. */
2178         low_hole_pages = (vm->va_start >>
2179                           gmmu_page_shifts[gmmu_page_size_small]);
2180
2181         gk20a_allocator_init(&vm->vma[gmmu_page_size_small], name,
2182               low_hole_pages,             /* start */
2183               num_pages - low_hole_pages, /* length */
2184               1);                         /* align */
2185
2186         snprintf(name, sizeof(name), "gk20a_as_%d-%dKB", as_share->id,
2187                  gmmu_page_sizes[gmmu_page_size_big]>>10);
2188
2189         num_pages = (u32)(vma_size >> gmmu_page_shifts[gmmu_page_size_big]);
2190         gk20a_allocator_init(&vm->vma[gmmu_page_size_big], name,
2191                               num_pages, /* start */
2192                               num_pages, /* length */
2193                               1); /* align */
2194
2195         vm->mapped_buffers = RB_ROOT;
2196
2197         mutex_init(&vm->update_gmmu_lock);
2198         kref_init(&vm->ref);
2199         INIT_LIST_HEAD(&vm->reserved_va_list);
2200
2201         vm->enable_ctag = true;
2202
2203         return 0;
2204 }
2205
2206
2207 int gk20a_vm_release_share(struct gk20a_as_share *as_share)
2208 {
2209         struct vm_gk20a *vm = as_share->vm;
2210
2211         gk20a_dbg_fn("");
2212
2213         vm->as_share = NULL;
2214
2215         /* put as reference to vm */
2216         gk20a_vm_put(vm);
2217
2218         as_share->vm = NULL;
2219
2220         return 0;
2221 }
2222
2223
2224 int gk20a_vm_alloc_space(struct gk20a_as_share *as_share,
2225                          struct nvhost_as_alloc_space_args *args)
2226
2227 {       int err = -ENOMEM;
2228         int pgsz_idx;
2229         u32 start_page_nr;
2230         struct gk20a_allocator *vma;
2231         struct vm_gk20a *vm = as_share->vm;
2232         struct vm_reserved_va_node *va_node;
2233         u64 vaddr_start = 0;
2234
2235         gk20a_dbg_fn("flags=0x%x pgsz=0x%x nr_pages=0x%x o/a=0x%llx",
2236                         args->flags, args->page_size, args->pages,
2237                         args->o_a.offset);
2238
2239         /* determine pagesz idx */
2240         for (pgsz_idx = gmmu_page_size_small;
2241              pgsz_idx < gmmu_nr_page_sizes;
2242              pgsz_idx++) {
2243                 if (gmmu_page_sizes[pgsz_idx] == args->page_size)
2244                         break;
2245         }
2246
2247         if (pgsz_idx >= gmmu_nr_page_sizes) {
2248                 err = -EINVAL;
2249                 goto clean_up;
2250         }
2251
2252         va_node = kzalloc(sizeof(*va_node), GFP_KERNEL);
2253         if (!va_node) {
2254                 err = -ENOMEM;
2255                 goto clean_up;
2256         }
2257
2258         if (args->flags & NVHOST_AS_ALLOC_SPACE_FLAGS_SPARSE &&
2259             pgsz_idx != gmmu_page_size_big) {
2260                 err = -ENOSYS;
2261                 kfree(va_node);
2262                 goto clean_up;
2263         }
2264
2265         start_page_nr = 0;
2266         if (args->flags & NVHOST_AS_ALLOC_SPACE_FLAGS_FIXED_OFFSET)
2267                 start_page_nr = (u32)(args->o_a.offset >>
2268                                       gmmu_page_shifts[pgsz_idx]);
2269
2270         vma = &vm->vma[pgsz_idx];
2271         err = vma->alloc(vma, &start_page_nr, args->pages);
2272         if (err) {
2273                 kfree(va_node);
2274                 goto clean_up;
2275         }
2276
2277         vaddr_start = (u64)start_page_nr << gmmu_page_shifts[pgsz_idx];
2278
2279         va_node->vaddr_start = vaddr_start;
2280         va_node->size = (u64)args->page_size * (u64)args->pages;
2281         va_node->pgsz_idx = args->page_size;
2282         INIT_LIST_HEAD(&va_node->va_buffers_list);
2283         INIT_LIST_HEAD(&va_node->reserved_va_list);
2284
2285         mutex_lock(&vm->update_gmmu_lock);
2286
2287         /* mark that we need to use sparse mappings here */
2288         if (args->flags & NVHOST_AS_ALLOC_SPACE_FLAGS_SPARSE) {
2289                 err = gk20a_vm_put_empty(vm, vaddr_start, args->pages,
2290                                          pgsz_idx);
2291                 if (err) {
2292                         mutex_unlock(&vm->update_gmmu_lock);
2293                         vma->free(vma, start_page_nr, args->pages);
2294                         kfree(va_node);
2295                         goto clean_up;
2296                 }
2297
2298                 va_node->sparse = true;
2299         }
2300
2301         list_add_tail(&va_node->reserved_va_list, &vm->reserved_va_list);
2302
2303         mutex_unlock(&vm->update_gmmu_lock);
2304
2305         args->o_a.offset = vaddr_start;
2306
2307 clean_up:
2308         return err;
2309 }
2310
2311 int gk20a_vm_free_space(struct gk20a_as_share *as_share,
2312                         struct nvhost_as_free_space_args *args)
2313 {
2314         int err = -ENOMEM;
2315         int pgsz_idx;
2316         u32 start_page_nr;
2317         struct gk20a_allocator *vma;
2318         struct vm_gk20a *vm = as_share->vm;
2319         struct vm_reserved_va_node *va_node;
2320
2321         gk20a_dbg_fn("pgsz=0x%x nr_pages=0x%x o/a=0x%llx", args->page_size,
2322                         args->pages, args->offset);
2323
2324         /* determine pagesz idx */
2325         for (pgsz_idx = gmmu_page_size_small;
2326              pgsz_idx < gmmu_nr_page_sizes;
2327              pgsz_idx++) {
2328                 if (gmmu_page_sizes[pgsz_idx] == args->page_size)
2329                         break;
2330         }
2331
2332         if (pgsz_idx >= gmmu_nr_page_sizes) {
2333                 err = -EINVAL;
2334                 goto clean_up;
2335         }
2336
2337         start_page_nr = (u32)(args->offset >>
2338                               gmmu_page_shifts[pgsz_idx]);
2339
2340         vma = &vm->vma[pgsz_idx];
2341         err = vma->free(vma, start_page_nr, args->pages);
2342
2343         if (err)
2344                 goto clean_up;
2345
2346         mutex_lock(&vm->update_gmmu_lock);
2347         va_node = addr_to_reservation(vm, args->offset);
2348         if (va_node) {
2349                 struct mapped_buffer_node *buffer;
2350
2351                 /* there is no need to unallocate the buffers in va. Just
2352                  * convert them into normal buffers */
2353
2354                 list_for_each_entry(buffer,
2355                         &va_node->va_buffers_list, va_buffers_list)
2356                         list_del_init(&buffer->va_buffers_list);
2357
2358                 list_del(&va_node->reserved_va_list);
2359
2360                 /* if this was a sparse mapping, free the va */
2361                 if (va_node->sparse)
2362                         __locked_gmmu_unmap(vm,
2363                                 va_node->vaddr_start,
2364                                 va_node->size,
2365                                 va_node->pgsz_idx,
2366                                 false,
2367                                 gk20a_mem_flag_none);
2368                 kfree(va_node);
2369         }
2370         mutex_unlock(&vm->update_gmmu_lock);
2371
2372 clean_up:
2373         return err;
2374 }
2375
2376 int gk20a_vm_bind_channel(struct gk20a_as_share *as_share,
2377                           struct channel_gk20a *ch)
2378 {
2379         int err = 0;
2380         struct vm_gk20a *vm = as_share->vm;
2381
2382         gk20a_dbg_fn("");
2383
2384         ch->vm = vm;
2385         err = channel_gk20a_commit_va(ch);
2386         if (err)
2387                 ch->vm = 0;
2388
2389         return err;
2390 }
2391
2392 int gk20a_dmabuf_alloc_drvdata(struct dma_buf *dmabuf, struct device *dev)
2393 {
2394         struct gk20a_dmabuf_priv *priv;
2395         static DEFINE_MUTEX(priv_lock);
2396
2397         priv = dma_buf_get_drvdata(dmabuf, dev);
2398         if (likely(priv))
2399                 return 0;
2400
2401         mutex_lock(&priv_lock);
2402         priv = dma_buf_get_drvdata(dmabuf, dev);
2403         if (priv)
2404                 goto priv_exist_or_err;
2405         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
2406         if (!priv) {
2407                 priv = ERR_PTR(-ENOMEM);
2408                 goto priv_exist_or_err;
2409         }
2410         mutex_init(&priv->lock);
2411         dma_buf_set_drvdata(dmabuf, dev, priv, gk20a_mm_delete_priv);
2412 priv_exist_or_err:
2413         mutex_unlock(&priv_lock);
2414         if (IS_ERR(priv))
2415                 return -ENOMEM;
2416
2417         return 0;
2418 }
2419
2420
2421 static int gk20a_dmabuf_get_kind(struct dma_buf *dmabuf)
2422 {
2423         int kind = 0;
2424 #ifdef CONFIG_TEGRA_NVMAP
2425         int err;
2426         u64 nvmap_param;
2427
2428         err = nvmap_get_dmabuf_param(dmabuf, NVMAP_HANDLE_PARAM_KIND,
2429                                      &nvmap_param);
2430         kind = err ? kind : nvmap_param;
2431 #endif
2432         return kind;
2433 }
2434
2435 int gk20a_vm_map_buffer(struct gk20a_as_share *as_share,
2436                         int dmabuf_fd,
2437                         u64 *offset_align,
2438                         u32 flags, /*NVHOST_AS_MAP_BUFFER_FLAGS_*/
2439                         int kind)
2440 {
2441         int err = 0;
2442         struct vm_gk20a *vm = as_share->vm;
2443         struct dma_buf *dmabuf;
2444         u64 ret_va;
2445
2446         gk20a_dbg_fn("");
2447
2448         /* get ref to the mem handle (released on unmap_locked) */
2449         dmabuf = dma_buf_get(dmabuf_fd);
2450         if (!dmabuf)
2451                 return 0;
2452
2453         err = gk20a_dmabuf_alloc_drvdata(dmabuf, dev_from_vm(vm));
2454         if (err) {
2455                 dma_buf_put(dmabuf);
2456                 return err;
2457         }
2458
2459         if (kind == -1)
2460                 kind = gk20a_dmabuf_get_kind(dmabuf);
2461
2462         ret_va = gk20a_vm_map(vm, dmabuf, *offset_align,
2463                         flags, kind, NULL, true,
2464                         gk20a_mem_flag_none);
2465         *offset_align = ret_va;
2466         if (!ret_va) {
2467                 dma_buf_put(dmabuf);
2468                 err = -EINVAL;
2469         }
2470
2471         return err;
2472 }
2473
2474 int gk20a_vm_unmap_buffer(struct gk20a_as_share *as_share, u64 offset)
2475 {
2476         struct vm_gk20a *vm = as_share->vm;
2477
2478         gk20a_dbg_fn("");
2479
2480         gk20a_vm_unmap_user(vm, offset);
2481         return 0;
2482 }
2483
2484 int gk20a_init_bar1_vm(struct mm_gk20a *mm)
2485 {
2486         int err;
2487         phys_addr_t inst_pa;
2488         void *inst_ptr;
2489         struct vm_gk20a *vm = &mm->bar1.vm;
2490         struct gk20a *g = gk20a_from_mm(mm);
2491         struct device *d = dev_from_gk20a(g);
2492         struct inst_desc *inst_block = &mm->bar1.inst_block;
2493         u64 pde_addr;
2494         u32 pde_addr_lo;
2495         u32 pde_addr_hi;
2496         dma_addr_t iova;
2497
2498         vm->mm = mm;
2499
2500         mm->bar1.aperture_size = bar1_aperture_size_mb_gk20a() << 20;
2501
2502         gk20a_dbg_info("bar1 vm size = 0x%x", mm->bar1.aperture_size);
2503
2504         vm->va_start = mm->pde_stride * 1;
2505         vm->va_limit = mm->bar1.aperture_size;
2506
2507         {
2508                 u32 pde_lo, pde_hi;
2509                 pde_range_from_vaddr_range(vm,
2510                                            0, vm->va_limit-1,
2511                                            &pde_lo, &pde_hi);
2512                 vm->pdes.num_pdes = pde_hi + 1;
2513         }
2514
2515         /* bar1 is likely only to ever use/need small page sizes. */
2516         /* But just in case, for now... arrange for both.*/
2517         vm->pdes.ptes[gmmu_page_size_small] =
2518                 kzalloc(sizeof(struct page_table_gk20a) *
2519                         vm->pdes.num_pdes, GFP_KERNEL);
2520
2521         vm->pdes.ptes[gmmu_page_size_big] =
2522                 kzalloc(sizeof(struct page_table_gk20a) *
2523                         vm->pdes.num_pdes, GFP_KERNEL);
2524
2525         if (!(vm->pdes.ptes[gmmu_page_size_small] &&
2526               vm->pdes.ptes[gmmu_page_size_big]))
2527                 return -ENOMEM;
2528
2529         gk20a_dbg_info("init space for bar1 va_limit=0x%llx num_pdes=%d",
2530                    vm->va_limit, vm->pdes.num_pdes);
2531
2532
2533         /* allocate the page table directory */
2534         err = alloc_gmmu_pages(vm, 0, &vm->pdes.ref,
2535                                &vm->pdes.sgt, &vm->pdes.size);
2536         if (err)
2537                 goto clean_up;
2538
2539         err = map_gmmu_pages(vm->pdes.ref, vm->pdes.sgt, &vm->pdes.kv,
2540                              vm->pdes.size);
2541         if (err) {
2542                 free_gmmu_pages(vm, vm->pdes.ref, vm->pdes.sgt, 0,
2543                                         vm->pdes.size);
2544                 goto clean_up;
2545         }
2546         gk20a_dbg(gpu_dbg_pte, "bar 1 pdes.kv = 0x%p, pdes.phys = 0x%llx",
2547                         vm->pdes.kv, gk20a_mm_iova_addr(vm->pdes.sgt->sgl));
2548         /* we could release vm->pdes.kv but it's only one page... */
2549
2550         pde_addr = gk20a_mm_iova_addr(vm->pdes.sgt->sgl);
2551         pde_addr_lo = u64_lo32(pde_addr >> 12);
2552         pde_addr_hi = u64_hi32(pde_addr);
2553
2554         gk20a_dbg_info("pde pa=0x%llx pde_addr_lo=0x%x pde_addr_hi=0x%x",
2555                 (u64)gk20a_mm_iova_addr(vm->pdes.sgt->sgl),
2556                 pde_addr_lo, pde_addr_hi);
2557
2558         /* allocate instance mem for bar1 */
2559         inst_block->size = ram_in_alloc_size_v();
2560         inst_block->cpuva = dma_alloc_coherent(d, inst_block->size,
2561                                 &iova, GFP_KERNEL);
2562         if (!inst_block->cpuva) {
2563                 gk20a_err(d, "%s: memory allocation failed\n", __func__);
2564                 err = -ENOMEM;
2565                 goto clean_up;
2566         }
2567
2568         inst_block->iova = iova;
2569         inst_block->cpu_pa = gk20a_get_phys_from_iova(d, inst_block->iova);
2570         if (!inst_block->cpu_pa) {
2571                 gk20a_err(d, "%s: failed to get phys address\n", __func__);
2572                 err = -ENOMEM;
2573                 goto clean_up;
2574         }
2575
2576         inst_pa = inst_block->cpu_pa;
2577         inst_ptr = inst_block->cpuva;
2578
2579         gk20a_dbg_info("bar1 inst block physical phys = 0x%llx, kv = 0x%p",
2580                 (u64)inst_pa, inst_ptr);
2581
2582         memset(inst_ptr, 0, ram_fc_size_val_v());
2583
2584         gk20a_mem_wr32(inst_ptr, ram_in_page_dir_base_lo_w(),
2585                 ram_in_page_dir_base_target_vid_mem_f() |
2586                 ram_in_page_dir_base_vol_true_f() |
2587                 ram_in_page_dir_base_lo_f(pde_addr_lo));
2588
2589         gk20a_mem_wr32(inst_ptr, ram_in_page_dir_base_hi_w(),
2590                 ram_in_page_dir_base_hi_f(pde_addr_hi));
2591
2592         gk20a_mem_wr32(inst_ptr, ram_in_adr_limit_lo_w(),
2593                  u64_lo32(vm->va_limit) | 0xFFF);
2594
2595         gk20a_mem_wr32(inst_ptr, ram_in_adr_limit_hi_w(),
2596                 ram_in_adr_limit_hi_f(u64_hi32(vm->va_limit)));
2597
2598         gk20a_dbg_info("bar1 inst block ptr: %08llx",  (u64)inst_pa);
2599         gk20a_allocator_init(&vm->vma[gmmu_page_size_small], "gk20a_bar1",
2600                               1,/*start*/
2601                               (vm->va_limit >> 12) - 1 /* length*/,
2602                               1); /* align */
2603         /* initialize just in case we try to use it anyway */
2604         gk20a_allocator_init(&vm->vma[gmmu_page_size_big], "gk20a_bar1-unused",
2605                               0x0badc0de, /* start */
2606                               1, /* length */
2607                               1); /* align */
2608
2609         vm->mapped_buffers = RB_ROOT;
2610
2611         mutex_init(&vm->update_gmmu_lock);
2612         kref_init(&vm->ref);
2613         INIT_LIST_HEAD(&vm->reserved_va_list);
2614
2615         return 0;
2616
2617 clean_up:
2618         /* free, etc */
2619         if (inst_block->cpuva)
2620                 dma_free_coherent(d, inst_block->size,
2621                         inst_block->cpuva, inst_block->iova);
2622         inst_block->cpuva = NULL;
2623         inst_block->iova = 0;
2624         return err;
2625 }
2626
2627 /* pmu vm, share channel_vm interfaces */
2628 int gk20a_init_pmu_vm(struct mm_gk20a *mm)
2629 {
2630         int err;
2631         phys_addr_t inst_pa;
2632         void *inst_ptr;
2633         struct vm_gk20a *vm = &mm->pmu.vm;
2634         struct gk20a *g = gk20a_from_mm(mm);
2635         struct device *d = dev_from_gk20a(g);
2636         struct inst_desc *inst_block = &mm->pmu.inst_block;
2637         u64 pde_addr;
2638         u32 pde_addr_lo;
2639         u32 pde_addr_hi;
2640         dma_addr_t iova;
2641
2642         vm->mm = mm;
2643
2644         mm->pmu.aperture_size = GK20A_PMU_VA_SIZE;
2645
2646         gk20a_dbg_info("pmu vm size = 0x%x", mm->pmu.aperture_size);
2647
2648         vm->va_start  = GK20A_PMU_VA_START;
2649         vm->va_limit  = vm->va_start + mm->pmu.aperture_size;
2650
2651         {
2652                 u32 pde_lo, pde_hi;
2653                 pde_range_from_vaddr_range(vm,
2654                                            0, vm->va_limit-1,
2655                                            &pde_lo, &pde_hi);
2656                 vm->pdes.num_pdes = pde_hi + 1;
2657         }
2658
2659         /* The pmu is likely only to ever use/need small page sizes. */
2660         /* But just in case, for now... arrange for both.*/
2661         vm->pdes.ptes[gmmu_page_size_small] =
2662                 kzalloc(sizeof(struct page_table_gk20a) *
2663                         vm->pdes.num_pdes, GFP_KERNEL);
2664
2665         vm->pdes.ptes[gmmu_page_size_big] =
2666                 kzalloc(sizeof(struct page_table_gk20a) *
2667                         vm->pdes.num_pdes, GFP_KERNEL);
2668
2669         if (!(vm->pdes.ptes[gmmu_page_size_small] &&
2670               vm->pdes.ptes[gmmu_page_size_big]))
2671                 return -ENOMEM;
2672
2673         gk20a_dbg_info("init space for pmu va_limit=0x%llx num_pdes=%d",
2674                    vm->va_limit, vm->pdes.num_pdes);
2675
2676         /* allocate the page table directory */
2677         err = alloc_gmmu_pages(vm, 0, &vm->pdes.ref,
2678                                &vm->pdes.sgt, &vm->pdes.size);
2679         if (err)
2680                 goto clean_up;
2681
2682         err = map_gmmu_pages(vm->pdes.ref, vm->pdes.sgt, &vm->pdes.kv,
2683                              vm->pdes.size);
2684         if (err) {
2685                 free_gmmu_pages(vm, vm->pdes.ref, vm->pdes.sgt, 0,
2686                                         vm->pdes.size);
2687                 goto clean_up;
2688         }
2689         gk20a_dbg_info("pmu pdes phys @ 0x%llx",
2690                         (u64)gk20a_mm_iova_addr(vm->pdes.sgt->sgl));
2691         /* we could release vm->pdes.kv but it's only one page... */
2692
2693         pde_addr = gk20a_mm_iova_addr(vm->pdes.sgt->sgl);
2694         pde_addr_lo = u64_lo32(pde_addr >> 12);
2695         pde_addr_hi = u64_hi32(pde_addr);
2696
2697         gk20a_dbg_info("pde pa=0x%llx pde_addr_lo=0x%x pde_addr_hi=0x%x",
2698                         (u64)pde_addr, pde_addr_lo, pde_addr_hi);
2699
2700         /* allocate instance mem for pmu */
2701         inst_block->size = GK20A_PMU_INST_SIZE;
2702         inst_block->cpuva = dma_alloc_coherent(d, inst_block->size,
2703                                 &iova, GFP_KERNEL);
2704         if (!inst_block->cpuva) {
2705                 gk20a_err(d, "%s: memory allocation failed\n", __func__);
2706                 err = -ENOMEM;
2707                 goto clean_up;
2708         }
2709
2710         inst_block->iova = iova;
2711         inst_block->cpu_pa = gk20a_get_phys_from_iova(d, inst_block->iova);
2712         if (!inst_block->cpu_pa) {
2713                 gk20a_err(d, "%s: failed to get phys address\n", __func__);
2714                 err = -ENOMEM;
2715                 goto clean_up;
2716         }
2717
2718         inst_pa = inst_block->cpu_pa;
2719         inst_ptr = inst_block->cpuva;
2720
2721         gk20a_dbg_info("pmu inst block physical addr: 0x%llx", (u64)inst_pa);
2722
2723         memset(inst_ptr, 0, GK20A_PMU_INST_SIZE);
2724
2725         gk20a_mem_wr32(inst_ptr, ram_in_page_dir_base_lo_w(),
2726                 ram_in_page_dir_base_target_vid_mem_f() |
2727                 ram_in_page_dir_base_vol_true_f() |
2728                 ram_in_page_dir_base_lo_f(pde_addr_lo));
2729
2730         gk20a_mem_wr32(inst_ptr, ram_in_page_dir_base_hi_w(),
2731                 ram_in_page_dir_base_hi_f(pde_addr_hi));
2732
2733         gk20a_mem_wr32(inst_ptr, ram_in_adr_limit_lo_w(),
2734                  u64_lo32(vm->va_limit) | 0xFFF);
2735
2736         gk20a_mem_wr32(inst_ptr, ram_in_adr_limit_hi_w(),
2737                 ram_in_adr_limit_hi_f(u64_hi32(vm->va_limit)));
2738
2739         gk20a_allocator_init(&vm->vma[gmmu_page_size_small], "gk20a_pmu",
2740                               (vm->va_start >> 12), /* start */
2741                               (vm->va_limit - vm->va_start) >> 12, /*length*/
2742                               1); /* align */
2743         /* initialize just in case we try to use it anyway */
2744         gk20a_allocator_init(&vm->vma[gmmu_page_size_big], "gk20a_pmu-unused",
2745                               0x0badc0de, /* start */
2746                               1, /* length */
2747                               1); /* align */
2748
2749
2750         vm->mapped_buffers = RB_ROOT;
2751
2752         mutex_init(&vm->update_gmmu_lock);
2753         kref_init(&vm->ref);
2754         INIT_LIST_HEAD(&vm->reserved_va_list);
2755
2756         return 0;
2757
2758 clean_up:
2759         /* free, etc */
2760         if (inst_block->cpuva)
2761                 dma_free_coherent(d, inst_block->size,
2762                         inst_block->cpuva, inst_block->iova);
2763         inst_block->cpuva = NULL;
2764         inst_block->iova = 0;
2765         return err;
2766 }
2767
2768 int gk20a_mm_fb_flush(struct gk20a *g)
2769 {
2770         struct mm_gk20a *mm = &g->mm;
2771         u32 data;
2772         s32 retry = 100;
2773         int ret = 0;
2774
2775         gk20a_dbg_fn("");
2776
2777         mutex_lock(&mm->l2_op_lock);
2778
2779         g->ops.ltc.elpg_flush(g);
2780
2781         /* Make sure all previous writes are committed to the L2. There's no
2782            guarantee that writes are to DRAM. This will be a sysmembar internal
2783            to the L2. */
2784         gk20a_writel(g, flush_fb_flush_r(),
2785                 flush_fb_flush_pending_busy_f());
2786
2787         do {
2788                 data = gk20a_readl(g, flush_fb_flush_r());
2789
2790                 if (flush_fb_flush_outstanding_v(data) ==
2791                         flush_fb_flush_outstanding_true_v() ||
2792                     flush_fb_flush_pending_v(data) ==
2793                         flush_fb_flush_pending_busy_v()) {
2794                                 gk20a_dbg_info("fb_flush 0x%x", data);
2795                                 retry--;
2796                                 usleep_range(20, 40);
2797                 } else
2798                         break;
2799         } while (retry >= 0 || !tegra_platform_is_silicon());
2800
2801         if (retry < 0) {
2802                 gk20a_warn(dev_from_gk20a(g),
2803                         "fb_flush too many retries");
2804                 ret = -EBUSY;
2805         }
2806
2807         mutex_unlock(&mm->l2_op_lock);
2808
2809         return ret;
2810 }
2811
2812 static void gk20a_mm_l2_invalidate_locked(struct gk20a *g)
2813 {
2814         u32 data;
2815         s32 retry = 200;
2816
2817         /* Invalidate any clean lines from the L2 so subsequent reads go to
2818            DRAM. Dirty lines are not affected by this operation. */
2819         gk20a_writel(g, flush_l2_system_invalidate_r(),
2820                 flush_l2_system_invalidate_pending_busy_f());
2821
2822         do {
2823                 data = gk20a_readl(g, flush_l2_system_invalidate_r());
2824
2825                 if (flush_l2_system_invalidate_outstanding_v(data) ==
2826                         flush_l2_system_invalidate_outstanding_true_v() ||
2827                     flush_l2_system_invalidate_pending_v(data) ==
2828                         flush_l2_system_invalidate_pending_busy_v()) {
2829                                 gk20a_dbg_info("l2_system_invalidate 0x%x",
2830                                                 data);
2831                                 retry--;
2832                                 usleep_range(20, 40);
2833                 } else
2834                         break;
2835         } while (retry >= 0 || !tegra_platform_is_silicon());
2836
2837         if (retry < 0)
2838                 gk20a_warn(dev_from_gk20a(g),
2839                         "l2_system_invalidate too many retries");
2840 }
2841
2842 void gk20a_mm_l2_invalidate(struct gk20a *g)
2843 {
2844         struct mm_gk20a *mm = &g->mm;
2845         mutex_lock(&mm->l2_op_lock);
2846         gk20a_mm_l2_invalidate_locked(g);
2847         mutex_unlock(&mm->l2_op_lock);
2848 }
2849
2850 void gk20a_mm_l2_flush(struct gk20a *g, bool invalidate)
2851 {
2852         struct mm_gk20a *mm = &g->mm;
2853         u32 data;
2854         s32 retry = 200;
2855
2856         gk20a_dbg_fn("");
2857
2858         mutex_lock(&mm->l2_op_lock);
2859
2860         /* Flush all dirty lines from the L2 to DRAM. Lines are left in the L2
2861            as clean, so subsequent reads might hit in the L2. */
2862         gk20a_writel(g, flush_l2_flush_dirty_r(),
2863                 flush_l2_flush_dirty_pending_busy_f());
2864
2865         do {
2866                 data = gk20a_readl(g, flush_l2_flush_dirty_r());
2867
2868                 if (flush_l2_flush_dirty_outstanding_v(data) ==
2869                         flush_l2_flush_dirty_outstanding_true_v() ||
2870                     flush_l2_flush_dirty_pending_v(data) ==
2871                         flush_l2_flush_dirty_pending_busy_v()) {
2872                                 gk20a_dbg_info("l2_flush_dirty 0x%x", data);
2873                                 retry--;
2874                                 usleep_range(20, 40);
2875                 } else
2876                         break;
2877         } while (retry >= 0 || !tegra_platform_is_silicon());
2878
2879         if (retry < 0)
2880                 gk20a_warn(dev_from_gk20a(g),
2881                         "l2_flush_dirty too many retries");
2882
2883         if (invalidate)
2884                 gk20a_mm_l2_invalidate_locked(g);
2885
2886         mutex_unlock(&mm->l2_op_lock);
2887 }
2888
2889
2890 int gk20a_vm_find_buffer(struct vm_gk20a *vm, u64 gpu_va,
2891                          struct dma_buf **dmabuf,
2892                          u64 *offset)
2893 {
2894         struct mapped_buffer_node *mapped_buffer;
2895
2896         gk20a_dbg_fn("gpu_va=0x%llx", gpu_va);
2897
2898         mutex_lock(&vm->update_gmmu_lock);
2899
2900         mapped_buffer = find_mapped_buffer_range_locked(&vm->mapped_buffers,
2901                                                         gpu_va);
2902         if (!mapped_buffer) {
2903                 mutex_unlock(&vm->update_gmmu_lock);
2904                 return -EINVAL;
2905         }
2906
2907         *dmabuf = mapped_buffer->dmabuf;
2908         *offset = gpu_va - mapped_buffer->addr;
2909
2910         mutex_unlock(&vm->update_gmmu_lock);
2911
2912         return 0;
2913 }
2914
2915 void gk20a_mm_tlb_invalidate(struct vm_gk20a *vm)
2916 {
2917         struct gk20a *g = gk20a_from_vm(vm);
2918         u32 addr_lo = u64_lo32(gk20a_mm_iova_addr(vm->pdes.sgt->sgl) >> 12);
2919         u32 data;
2920         s32 retry = 200;
2921         static DEFINE_MUTEX(tlb_lock);
2922
2923         gk20a_dbg_fn("");
2924
2925         /* pagetables are considered sw states which are preserved after
2926            prepare_poweroff. When gk20a deinit releases those pagetables,
2927            common code in vm unmap path calls tlb invalidate that touches
2928            hw. Use the power_on flag to skip tlb invalidation when gpu
2929            power is turned off */
2930
2931         if (!g->power_on)
2932                 return;
2933
2934         /* No need to invalidate if tlb is clean */
2935         mutex_lock(&vm->update_gmmu_lock);
2936         if (!vm->tlb_dirty) {
2937                 mutex_unlock(&vm->update_gmmu_lock);
2938                 return;
2939         }
2940
2941         mutex_lock(&tlb_lock);
2942         do {
2943                 data = gk20a_readl(g, fb_mmu_ctrl_r());
2944                 if (fb_mmu_ctrl_pri_fifo_space_v(data) != 0)
2945                         break;
2946                 usleep_range(20, 40);
2947                 retry--;
2948         } while (retry >= 0 || !tegra_platform_is_silicon());
2949
2950         if (retry < 0) {
2951                 gk20a_warn(dev_from_gk20a(g),
2952                         "wait mmu fifo space too many retries");
2953                 goto out;
2954         }
2955
2956         gk20a_writel(g, fb_mmu_invalidate_pdb_r(),
2957                 fb_mmu_invalidate_pdb_addr_f(addr_lo) |
2958                 fb_mmu_invalidate_pdb_aperture_vid_mem_f());
2959
2960         gk20a_writel(g, fb_mmu_invalidate_r(),
2961                 fb_mmu_invalidate_all_va_true_f() |
2962                 fb_mmu_invalidate_trigger_true_f());
2963
2964         do {
2965                 data = gk20a_readl(g, fb_mmu_ctrl_r());
2966                 if (fb_mmu_ctrl_pri_fifo_empty_v(data) !=
2967                         fb_mmu_ctrl_pri_fifo_empty_false_f())
2968                         break;
2969                 retry--;
2970                 usleep_range(20, 40);
2971         } while (retry >= 0 || !tegra_platform_is_silicon());
2972
2973         if (retry < 0)
2974                 gk20a_warn(dev_from_gk20a(g),
2975                         "mmu invalidate too many retries");
2976
2977 out:
2978         mutex_unlock(&tlb_lock);
2979         vm->tlb_dirty = false;
2980         mutex_unlock(&vm->update_gmmu_lock);
2981 }
2982
2983 int gk20a_mm_suspend(struct gk20a *g)
2984 {
2985         gk20a_dbg_fn("");
2986
2987         gk20a_mm_fb_flush(g);
2988         gk20a_mm_l2_flush(g, true);
2989
2990         gk20a_dbg_fn("done");
2991         return 0;
2992 }
2993
2994 void gk20a_mm_ltc_isr(struct gk20a *g)
2995 {
2996         u32 intr;
2997
2998         intr = gk20a_readl(g, ltc_ltc0_ltss_intr_r());
2999         gk20a_err(dev_from_gk20a(g), "ltc: %08x\n", intr);
3000         gk20a_writel(g, ltc_ltc0_ltss_intr_r(), intr);
3001 }
3002
3003 bool gk20a_mm_mmu_debug_mode_enabled(struct gk20a *g)
3004 {
3005         u32 debug_ctrl = gk20a_readl(g, fb_mmu_debug_ctrl_r());
3006         return fb_mmu_debug_ctrl_debug_v(debug_ctrl) ==
3007                 fb_mmu_debug_ctrl_debug_enabled_v();
3008 }
3009
3010 static int gk20a_mm_mmu_vpr_info_fetch_wait(struct gk20a *g,
3011                                             const unsigned int msec)
3012 {
3013         unsigned long timeout;
3014
3015         timeout = jiffies + msecs_to_jiffies(msec);
3016         while (1) {
3017                 u32 val;
3018
3019                 val = gk20a_readl(g, fb_mmu_vpr_info_r());
3020                 if (fb_mmu_vpr_info_fetch_v(val) ==
3021                     fb_mmu_vpr_info_fetch_false_v())
3022                         break;
3023
3024                 if (tegra_platform_is_silicon() &&
3025                                 WARN_ON(time_after(jiffies, timeout)))
3026                         return -ETIME;
3027         }
3028
3029         return 0;
3030 }
3031
3032 int gk20a_mm_mmu_vpr_info_fetch(struct gk20a *g)
3033 {
3034         int ret = 0;
3035
3036         gk20a_busy_noresume(g->dev);
3037         if (!pm_runtime_active(&g->dev->dev))
3038                 goto fail;
3039
3040         if (gk20a_mm_mmu_vpr_info_fetch_wait(g, 5)) {
3041                 ret = -ETIME;
3042                 goto fail;
3043         }
3044
3045         gk20a_writel(g, fb_mmu_vpr_info_r(),
3046                      fb_mmu_vpr_info_fetch_true_v());
3047
3048         ret = gk20a_mm_mmu_vpr_info_fetch_wait(g, 5);
3049
3050  fail:
3051         gk20a_idle(g->dev);
3052         return ret;
3053 }