gpu: nvgpu: Fix CDE bind channel usage
[linux-3.10.git] / drivers / gpu / nvgpu / gk20a / mm_gk20a.c
1 /*
2  * GK20A memory management
3  *
4  * Copyright (c) 2011-2016, NVIDIA CORPORATION.  All rights reserved.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms and conditions of the GNU General Public License,
8  * version 2, as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope it will be useful, but WITHOUT
11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13  * more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
17  */
18
19 #include <linux/delay.h>
20 #include <linux/highmem.h>
21 #include <linux/log2.h>
22 #include <linux/nvhost.h>
23 #include <linux/pm_runtime.h>
24 #include <linux/scatterlist.h>
25 #include <linux/nvmap.h>
26 #include <linux/tegra-soc.h>
27 #include <linux/vmalloc.h>
28 #include <linux/dma-buf.h>
29 #include <linux/lcm.h>
30 #include <uapi/linux/nvgpu.h>
31 #include <trace/events/gk20a.h>
32
33 #include "gk20a.h"
34 #include "mm_gk20a.h"
35 #include "fence_gk20a.h"
36 #include "hw_gmmu_gk20a.h"
37 #include "hw_fb_gk20a.h"
38 #include "hw_bus_gk20a.h"
39 #include "hw_ram_gk20a.h"
40 #include "hw_mc_gk20a.h"
41 #include "hw_flush_gk20a.h"
42 #include "hw_ltc_gk20a.h"
43
44 #include "kind_gk20a.h"
45 #include "semaphore_gk20a.h"
46
47 /*
48  * GPU mapping life cycle
49  * ======================
50  *
51  * Kernel mappings
52  * ---------------
53  *
54  * Kernel mappings are created through vm.map(..., false):
55  *
56  *  - Mappings to the same allocations are reused and refcounted.
57  *  - This path does not support deferred unmapping (i.e. kernel must wait for
58  *    all hw operations on the buffer to complete before unmapping).
59  *  - References to dmabuf are owned and managed by the (kernel) clients of
60  *    the gk20a_vm layer.
61  *
62  *
63  * User space mappings
64  * -------------------
65  *
66  * User space mappings are created through as.map_buffer -> vm.map(..., true):
67  *
68  *  - Mappings to the same allocations are reused and refcounted.
69  *  - This path supports deferred unmapping (i.e. we delay the actual unmapping
70  *    until all hw operations have completed).
71  *  - References to dmabuf are owned and managed by the vm_gk20a
72  *    layer itself. vm.map acquires these refs, and sets
73  *    mapped_buffer->own_mem_ref to record that we must release the refs when we
74  *    actually unmap.
75  *
76  */
77
78 static inline int vm_aspace_id(struct vm_gk20a *vm)
79 {
80         /* -1 is bar1 or pmu, etc. */
81         return vm->as_share ? vm->as_share->id : -1;
82 }
83 static inline u32 hi32(u64 f)
84 {
85         return (u32)(f >> 32);
86 }
87 static inline u32 lo32(u64 f)
88 {
89         return (u32)(f & 0xffffffff);
90 }
91
92 static struct mapped_buffer_node *find_mapped_buffer_locked(
93                                         struct rb_root *root, u64 addr);
94 static struct mapped_buffer_node *find_mapped_buffer_reverse_locked(
95                                 struct rb_root *root, struct dma_buf *dmabuf,
96                                 u32 kind);
97 static int update_gmmu_ptes_locked(struct vm_gk20a *vm,
98                                    enum gmmu_pgsz_gk20a pgsz_idx,
99                                    struct sg_table *sgt, u64 buffer_offset,
100                                    u64 first_vaddr, u64 last_vaddr,
101                                    u8 kind_v, u32 ctag_offset, bool cacheable,
102                                    bool umapped_pte, int rw_flag,
103                                    bool sparse,
104                                    bool priv);
105 static int __must_check gk20a_init_system_vm(struct mm_gk20a *mm);
106 static int __must_check gk20a_init_bar1_vm(struct mm_gk20a *mm);
107 static int __must_check gk20a_init_hwpm(struct mm_gk20a *mm);
108
109
110 struct gk20a_dmabuf_priv {
111         struct mutex lock;
112
113         struct gk20a_comptag_allocator *comptag_allocator;
114         struct gk20a_comptags comptags;
115
116         struct dma_buf_attachment *attach;
117         struct sg_table *sgt;
118
119         int pin_count;
120
121         struct list_head states;
122 };
123
124 static void gk20a_vm_remove_support_nofree(struct vm_gk20a *vm);
125
126 static int gk20a_comptaglines_alloc(struct gk20a_comptag_allocator *allocator,
127                 u32 *offset, u32 len)
128 {
129         unsigned long addr;
130         int err = 0;
131
132         mutex_lock(&allocator->lock);
133         addr = bitmap_find_next_zero_area(allocator->bitmap, allocator->size,
134                         0, len, 0);
135         if (addr < allocator->size) {
136                 /* number zero is reserved; bitmap base is 1 */
137                 *offset = 1 + addr;
138                 bitmap_set(allocator->bitmap, addr, len);
139         } else {
140                 err = -ENOMEM;
141         }
142         mutex_unlock(&allocator->lock);
143
144         return err;
145 }
146
147 static void gk20a_comptaglines_free(struct gk20a_comptag_allocator *allocator,
148                 u32 offset, u32 len)
149 {
150         /* number zero is reserved; bitmap base is 1 */
151         u32 addr = offset - 1;
152         WARN_ON(offset == 0);
153         WARN_ON(addr > allocator->size);
154         WARN_ON(addr + len > allocator->size);
155
156         mutex_lock(&allocator->lock);
157         bitmap_clear(allocator->bitmap, addr, len);
158         mutex_unlock(&allocator->lock);
159 }
160
161 static void gk20a_mm_delete_priv(void *_priv)
162 {
163         struct gk20a_buffer_state *s, *s_tmp;
164         struct gk20a_dmabuf_priv *priv = _priv;
165         if (!priv)
166                 return;
167
168         if (priv->comptags.lines) {
169                 BUG_ON(!priv->comptag_allocator);
170                 gk20a_comptaglines_free(priv->comptag_allocator,
171                                 priv->comptags.offset,
172                                 priv->comptags.allocated_lines);
173         }
174
175         /* Free buffer states */
176         list_for_each_entry_safe(s, s_tmp, &priv->states, list) {
177                 gk20a_fence_put(s->fence);
178                 list_del(&s->list);
179                 kfree(s);
180         }
181
182         kfree(priv);
183 }
184
185 struct sg_table *gk20a_mm_pin(struct device *dev, struct dma_buf *dmabuf)
186 {
187         struct gk20a_dmabuf_priv *priv;
188
189         priv = dma_buf_get_drvdata(dmabuf, dev);
190         if (WARN_ON(!priv))
191                 return ERR_PTR(-EINVAL);
192
193         mutex_lock(&priv->lock);
194
195         if (priv->pin_count == 0) {
196                 priv->attach = dma_buf_attach(dmabuf, dev);
197                 if (IS_ERR(priv->attach)) {
198                         mutex_unlock(&priv->lock);
199                         return (struct sg_table *)priv->attach;
200                 }
201
202                 priv->sgt = dma_buf_map_attachment(priv->attach,
203                                                    DMA_BIDIRECTIONAL);
204                 if (IS_ERR(priv->sgt)) {
205                         dma_buf_detach(dmabuf, priv->attach);
206                         mutex_unlock(&priv->lock);
207                         return priv->sgt;
208                 }
209         }
210
211         priv->pin_count++;
212         mutex_unlock(&priv->lock);
213         return priv->sgt;
214 }
215
216 void gk20a_mm_unpin(struct device *dev, struct dma_buf *dmabuf,
217                     struct sg_table *sgt)
218 {
219         struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev);
220         dma_addr_t dma_addr;
221
222         if (IS_ERR(priv) || !priv)
223                 return;
224
225         mutex_lock(&priv->lock);
226         WARN_ON(priv->sgt != sgt);
227         priv->pin_count--;
228         WARN_ON(priv->pin_count < 0);
229         dma_addr = sg_dma_address(priv->sgt->sgl);
230         if (priv->pin_count == 0) {
231                 dma_buf_unmap_attachment(priv->attach, priv->sgt,
232                                          DMA_BIDIRECTIONAL);
233                 dma_buf_detach(dmabuf, priv->attach);
234         }
235         mutex_unlock(&priv->lock);
236 }
237
238 void gk20a_get_comptags(struct device *dev, struct dma_buf *dmabuf,
239                         struct gk20a_comptags *comptags)
240 {
241         struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev);
242
243         if (!comptags)
244                 return;
245
246         if (!priv) {
247                 memset(comptags, 0, sizeof(*comptags));
248                 return;
249         }
250
251         *comptags = priv->comptags;
252 }
253
254 static int gk20a_alloc_comptags(struct gk20a *g,
255                                 struct device *dev,
256                                 struct dma_buf *dmabuf,
257                                 struct gk20a_comptag_allocator *allocator,
258                                 u32 lines, bool user_mappable,
259                                 u64 *ctag_map_win_size,
260                                 u32 *ctag_map_win_ctagline)
261 {
262         struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev);
263         u32 ctaglines_allocsize;
264         u32 ctagline_align;
265         u32 offset;
266         u32 alignment_lines;
267         const u32 aggregate_cacheline_sz =
268                 g->gr.cacheline_size * g->gr.slices_per_ltc *
269                 g->ltc_count;
270         const u32 small_pgsz = 4096;
271         int err;
272
273         if (!priv)
274                 return -ENOSYS;
275
276         if (!lines)
277                 return -EINVAL;
278
279         if (!user_mappable) {
280                 ctaglines_allocsize = lines;
281                 ctagline_align = 1;
282         } else {
283                 /*
284                  * For security, align the allocation on a page, and reserve
285                  * whole pages. Unfortunately, we cannot ask the allocator to
286                  * align here, since compbits per cacheline is not always a
287                  * power of two. So, we just have to allocate enough extra that
288                  * we're guaranteed to find a ctagline inside the allocation so
289                  * that: 1) it is the first ctagline in a cacheline that starts
290                  * at a page boundary, and 2) we can add enough overallocation
291                  * that the ctaglines of the succeeding allocation are on
292                  * different page than ours.
293                  */
294
295                 ctagline_align =
296                         (lcm(aggregate_cacheline_sz, small_pgsz) /
297                          aggregate_cacheline_sz) *
298                         g->gr.comptags_per_cacheline;
299
300                 ctaglines_allocsize =
301                         /* for alignment */
302                         ctagline_align +
303
304                         /* lines rounded up to cachelines */
305                         DIV_ROUND_UP(lines, g->gr.comptags_per_cacheline) *
306                         g->gr.comptags_per_cacheline +
307
308                         /* trail-padding */
309                         DIV_ROUND_UP(aggregate_cacheline_sz, small_pgsz) *
310                         g->gr.comptags_per_cacheline;
311
312                 if (ctaglines_allocsize < lines)
313                         return -EINVAL; /* integer overflow */
314         }
315
316         /* store the allocator so we can use it when we free the ctags */
317         priv->comptag_allocator = allocator;
318         err = gk20a_comptaglines_alloc(allocator, &offset,
319                                ctaglines_allocsize);
320         if (err)
321                 return err;
322
323         /*
324          * offset needs to be at the start of a page/cacheline boundary;
325          * prune the preceding ctaglines that were allocated for alignment.
326          */
327         alignment_lines =
328                 DIV_ROUND_UP(offset, ctagline_align) * ctagline_align - offset;
329         if (alignment_lines) {
330                 gk20a_comptaglines_free(allocator, offset, alignment_lines);
331                 offset += alignment_lines;
332                 ctaglines_allocsize -= alignment_lines;
333         }
334
335         /*
336          * check if we can prune the trailing, too; we just need to reserve
337          * whole pages and ctagcachelines.
338          */
339         if (user_mappable) {
340                 u32 needed_cachelines =
341                         DIV_ROUND_UP(lines, g->gr.comptags_per_cacheline);
342                 u32 needed_bytes = round_up(needed_cachelines *
343                                             aggregate_cacheline_sz,
344                                             small_pgsz);
345                 u32 first_unneeded_cacheline =
346                         DIV_ROUND_UP(needed_bytes, aggregate_cacheline_sz);
347                 u32 needed_ctaglines = first_unneeded_cacheline *
348                         g->gr.comptags_per_cacheline;
349                 u64 win_size;
350
351                 if (needed_ctaglines < ctaglines_allocsize) {
352                         gk20a_comptaglines_free(allocator,
353                                 offset + needed_ctaglines,
354                                 ctaglines_allocsize - needed_ctaglines);
355                         ctaglines_allocsize = needed_ctaglines;
356                 }
357
358                 *ctag_map_win_ctagline = offset;
359                 win_size =
360                         DIV_ROUND_UP(lines, g->gr.comptags_per_cacheline) *
361                         aggregate_cacheline_sz;
362
363                 *ctag_map_win_size = round_up(win_size, small_pgsz);
364         }
365
366         priv->comptags.offset = offset;
367         priv->comptags.lines = lines;
368         priv->comptags.allocated_lines = ctaglines_allocsize;
369         priv->comptags.user_mappable = user_mappable;
370
371         return 0;
372 }
373
374
375
376
377 static int gk20a_init_mm_reset_enable_hw(struct gk20a *g)
378 {
379         gk20a_dbg_fn("");
380
381         if (g->ops.clock_gating.slcg_fb_load_gating_prod)
382                 g->ops.clock_gating.slcg_fb_load_gating_prod(g,
383                                 g->slcg_enabled);
384         if (g->ops.clock_gating.slcg_ltc_load_gating_prod)
385                 g->ops.clock_gating.slcg_ltc_load_gating_prod(g,
386                                 g->slcg_enabled);
387         if (g->ops.clock_gating.blcg_fb_load_gating_prod)
388                 g->ops.clock_gating.blcg_fb_load_gating_prod(g,
389                                 g->blcg_enabled);
390         if (g->ops.clock_gating.blcg_ltc_load_gating_prod)
391                 g->ops.clock_gating.blcg_ltc_load_gating_prod(g,
392                                 g->blcg_enabled);
393
394         if (g->ops.fb.init_fs_state)
395                 g->ops.fb.init_fs_state(g);
396
397         return 0;
398 }
399
400 static void gk20a_remove_vm(struct vm_gk20a *vm, struct mem_desc *inst_block)
401 {
402         struct gk20a *g = vm->mm->g;
403
404         gk20a_dbg_fn("");
405
406         gk20a_free_inst_block(g, inst_block);
407         gk20a_vm_remove_support_nofree(vm);
408 }
409
410 static void gk20a_remove_mm_support(struct mm_gk20a *mm)
411 {
412         gk20a_remove_vm(&mm->bar1.vm, &mm->bar1.inst_block);
413         gk20a_remove_vm(&mm->pmu.vm, &mm->pmu.inst_block);
414         gk20a_free_inst_block(gk20a_from_mm(mm), &mm->hwpm.inst_block);
415 }
416
417 int gk20a_init_mm_setup_sw(struct gk20a *g)
418 {
419         struct mm_gk20a *mm = &g->mm;
420         int err;
421
422         gk20a_dbg_fn("");
423
424         if (mm->sw_ready) {
425                 gk20a_dbg_fn("skip init");
426                 return 0;
427         }
428
429         mm->g = g;
430         mutex_init(&mm->l2_op_lock);
431
432         /*TBD: make channel vm size configurable */
433         mm->channel.user_size = NV_MM_DEFAULT_USER_SIZE;
434         mm->channel.kernel_size = NV_MM_DEFAULT_KERNEL_SIZE;
435
436         gk20a_dbg_info("channel vm size: user %dMB  kernel %dMB",
437                        (int)(mm->channel.user_size >> 20),
438                        (int)(mm->channel.kernel_size >> 20));
439
440         err = gk20a_init_bar1_vm(mm);
441         if (err)
442                 return err;
443
444         if (g->ops.mm.init_bar2_vm) {
445                 err = g->ops.mm.init_bar2_vm(g);
446                 if (err)
447                         return err;
448         }
449         err = gk20a_init_system_vm(mm);
450         if (err)
451                 return err;
452
453         err = gk20a_init_hwpm(mm);
454         if (err)
455                 return err;
456
457         /* set vm_alloc_share op here as gk20a_as_alloc_share needs it */
458         g->ops.mm.vm_alloc_share = gk20a_vm_alloc_share;
459         mm->remove_support = gk20a_remove_mm_support;
460         mm->sw_ready = true;
461
462         gk20a_dbg_fn("done");
463         return 0;
464 }
465
466 /* make sure gk20a_init_mm_support is called before */
467 int gk20a_init_mm_setup_hw(struct gk20a *g)
468 {
469         struct mm_gk20a *mm = &g->mm;
470         struct mem_desc *inst_block = &mm->bar1.inst_block;
471         phys_addr_t inst_pa = gk20a_mem_phys(inst_block);
472         int err;
473
474         gk20a_dbg_fn("");
475
476         g->ops.fb.set_mmu_page_size(g);
477         if (g->ops.fb.set_use_full_comp_tag_line)
478                 mm->use_full_comp_tag_line =
479                         g->ops.fb.set_use_full_comp_tag_line(g);
480
481
482         inst_pa = (u32)(inst_pa >> bar1_instance_block_shift_gk20a());
483         gk20a_dbg_info("bar1 inst block ptr: 0x%08x",  (u32)inst_pa);
484
485         gk20a_writel(g, bus_bar1_block_r(),
486                      bus_bar1_block_target_vid_mem_f() |
487                      bus_bar1_block_mode_virtual_f() |
488                      bus_bar1_block_ptr_f(inst_pa));
489
490         if (g->ops.mm.init_bar2_mm_hw_setup) {
491                 err = g->ops.mm.init_bar2_mm_hw_setup(g);
492                 if (err)
493                         return err;
494         }
495
496         if (gk20a_mm_fb_flush(g) || gk20a_mm_fb_flush(g))
497                 return -EBUSY;
498
499         gk20a_dbg_fn("done");
500         return 0;
501 }
502
503 int gk20a_init_mm_support(struct gk20a *g)
504 {
505         u32 err;
506
507         err = gk20a_init_mm_reset_enable_hw(g);
508         if (err)
509                 return err;
510
511         err = gk20a_init_mm_setup_sw(g);
512         if (err)
513                 return err;
514
515         if (g->ops.mm.init_mm_setup_hw)
516                 err = g->ops.mm.init_mm_setup_hw(g);
517
518         return err;
519 }
520
521 static int alloc_gmmu_phys_pages(struct vm_gk20a *vm, u32 order,
522                                  struct gk20a_mm_entry *entry)
523 {
524         u32 num_pages = 1 << order;
525         u32 len = num_pages * PAGE_SIZE;
526         int err;
527         struct page *pages;
528
529         gk20a_dbg_fn("");
530
531         pages = alloc_pages(GFP_KERNEL, order);
532         if (!pages) {
533                 gk20a_dbg(gpu_dbg_pte, "alloc_pages failed\n");
534                 goto err_out;
535         }
536         entry->sgt = kzalloc(sizeof(*entry->sgt), GFP_KERNEL);
537         if (!entry->sgt) {
538                 gk20a_dbg(gpu_dbg_pte, "cannot allocate sg table");
539                 goto err_alloced;
540         }
541         err = sg_alloc_table(entry->sgt, 1, GFP_KERNEL);
542         if (err) {
543                 gk20a_dbg(gpu_dbg_pte, "sg_alloc_table failed\n");
544                 goto err_sg_table;
545         }
546         sg_set_page(entry->sgt->sgl, pages, len, 0);
547         entry->cpu_va = page_address(pages);
548         memset(entry->cpu_va, 0, len);
549         entry->size = len;
550         FLUSH_CPU_DCACHE(entry->cpu_va, sg_phys(entry->sgt->sgl), len);
551
552         return 0;
553
554 err_sg_table:
555         kfree(entry->sgt);
556 err_alloced:
557         __free_pages(pages, order);
558 err_out:
559         return -ENOMEM;
560 }
561
562 static void free_gmmu_phys_pages(struct vm_gk20a *vm,
563                             struct gk20a_mm_entry *entry)
564 {
565         gk20a_dbg_fn("");
566         free_pages((unsigned long)entry->cpu_va, get_order(entry->size));
567         entry->cpu_va = NULL;
568
569         sg_free_table(entry->sgt);
570         kfree(entry->sgt);
571         entry->sgt = NULL;
572 }
573
574 static int map_gmmu_phys_pages(struct gk20a_mm_entry *entry)
575 {
576         FLUSH_CPU_DCACHE(entry->cpu_va,
577                          sg_phys(entry->sgt->sgl),
578                          entry->sgt->sgl->length);
579         return 0;
580 }
581
582 static void unmap_gmmu_phys_pages(struct gk20a_mm_entry *entry)
583 {
584         FLUSH_CPU_DCACHE(entry->cpu_va,
585                          sg_phys(entry->sgt->sgl),
586                          entry->sgt->sgl->length);
587 }
588
589 static int alloc_gmmu_pages(struct vm_gk20a *vm, u32 order,
590                             struct gk20a_mm_entry *entry)
591 {
592         struct device *d = dev_from_vm(vm);
593         u32 num_pages = 1 << order;
594         u32 len = num_pages * PAGE_SIZE;
595         dma_addr_t iova;
596         DEFINE_DMA_ATTRS(attrs);
597         void *cpuva;
598         int err = 0;
599
600         gk20a_dbg_fn("");
601
602         if (tegra_platform_is_linsim())
603                 return alloc_gmmu_phys_pages(vm, order, entry);
604
605         entry->size = len;
606
607         /*
608          * On arm32 we're limited by vmalloc space, so we do not map pages by
609          * default.
610          */
611         if (IS_ENABLED(CONFIG_ARM64)) {
612                 cpuva = dma_zalloc_coherent(d, len, &iova, GFP_KERNEL);
613                 if (!cpuva) {
614                         gk20a_err(d, "memory allocation failed\n");
615                         goto err_out;
616                 }
617
618                 err = gk20a_get_sgtable(d, &entry->sgt, cpuva, iova, len);
619                 if (err) {
620                         gk20a_err(d, "sgt allocation failed\n");
621                         goto err_free;
622                 }
623
624                 entry->cpu_va = cpuva;
625         } else {
626                 struct page **pages;
627
628                 dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &attrs);
629                 pages = dma_alloc_attrs(d, len, &iova, GFP_KERNEL, &attrs);
630                 if (!pages) {
631                         gk20a_err(d, "memory allocation failed\n");
632                         goto err_out;
633                 }
634
635                 err = gk20a_get_sgtable_from_pages(d, &entry->sgt, pages,
636                                         iova, len);
637                 if (err) {
638                         gk20a_err(d, "sgt allocation failed\n");
639                         goto err_free;
640                 }
641
642                 entry->pages = pages;
643         }
644
645         return 0;
646
647 err_free:
648         if (IS_ENABLED(CONFIG_ARM64)) {
649                 dma_free_coherent(d, len, entry->cpu_va, iova);
650                 cpuva = NULL;
651         } else {
652                 dma_free_attrs(d, len, entry->pages, iova, &attrs);
653                 entry->pages = NULL;
654         }
655         iova = 0;
656 err_out:
657         return -ENOMEM;
658 }
659
660 void free_gmmu_pages(struct vm_gk20a *vm,
661                      struct gk20a_mm_entry *entry)
662 {
663         struct device *d = dev_from_vm(vm);
664         u64 iova;
665         DEFINE_DMA_ATTRS(attrs);
666
667         gk20a_dbg_fn("");
668
669         if (!entry->sgt)
670                 return;
671
672         if (tegra_platform_is_linsim()) {
673                 free_gmmu_phys_pages(vm, entry);
674                 return;
675         }
676
677         iova = sg_dma_address(entry->sgt->sgl);
678
679         gk20a_free_sgtable(&entry->sgt);
680
681         /*
682          * On arm32 we're limited by vmalloc space, so we do not map pages by
683          * default.
684          */
685         if (IS_ENABLED(CONFIG_ARM64)) {
686                 dma_free_coherent(d, entry->size, entry->cpu_va, iova);
687                 entry->cpu_va = NULL;
688         } else {
689                 dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &attrs);
690                 dma_free_attrs(d, entry->size, entry->pages, iova, &attrs);
691                 entry->pages = NULL;
692         }
693         entry->size = 0;
694         entry->sgt = NULL;
695 }
696
697 int map_gmmu_pages(struct gk20a_mm_entry *entry)
698 {
699         int count = PAGE_ALIGN(entry->size) >> PAGE_SHIFT;
700         struct page **pages;
701         gk20a_dbg_fn("");
702
703         if (tegra_platform_is_linsim())
704                 return map_gmmu_phys_pages(entry);
705
706         if (IS_ENABLED(CONFIG_ARM64)) {
707                 FLUSH_CPU_DCACHE(entry->cpu_va,
708                                  sg_phys(entry->sgt->sgl),
709                                  entry->size);
710         } else {
711                 pages = entry->pages;
712                 entry->cpu_va = vmap(pages, count, 0,
713                                      pgprot_dmacoherent(PAGE_KERNEL));
714                 if (!entry->cpu_va)
715                         return -ENOMEM;
716         }
717
718         return 0;
719 }
720
721 void unmap_gmmu_pages(struct gk20a_mm_entry *entry)
722 {
723         gk20a_dbg_fn("");
724
725         if (tegra_platform_is_linsim()) {
726                 unmap_gmmu_phys_pages(entry);
727                 return;
728         }
729
730         if (IS_ENABLED(CONFIG_ARM64)) {
731                 FLUSH_CPU_DCACHE(entry->cpu_va,
732                                  sg_phys(entry->sgt->sgl),
733                                  entry->size);
734         } else {
735                 vunmap(entry->cpu_va);
736                 entry->cpu_va = NULL;
737         }
738 }
739
740 /* allocate a phys contig region big enough for a full
741  * sized gmmu page table for the given gmmu_page_size.
742  * the whole range is zeroed so it's "invalid"/will fault
743  */
744
745 static int gk20a_zalloc_gmmu_page_table(struct vm_gk20a *vm,
746                                  enum gmmu_pgsz_gk20a pgsz_idx,
747                                  const struct gk20a_mmu_level *l,
748                                  struct gk20a_mm_entry *entry)
749 {
750         int err;
751         int order;
752         struct gk20a *g = gk20a_from_vm(vm);
753
754         gk20a_dbg_fn("");
755
756         /* allocate enough pages for the table */
757         order = l->hi_bit[pgsz_idx] - l->lo_bit[pgsz_idx] + 1;
758         order += ilog2(l->entry_size);
759         order -= PAGE_SHIFT;
760         order = max(0, order);
761
762         err = alloc_gmmu_pages(vm, order, entry);
763         gk20a_dbg(gpu_dbg_pte, "entry = 0x%p, addr=%08llx, size %d",
764                   entry, g->ops.mm.get_iova_addr(g, entry->sgt->sgl, 0),
765                   order);
766         if (err)
767                 return err;
768         entry->pgsz = pgsz_idx;
769
770         return err;
771 }
772
773 int gk20a_mm_pde_coverage_bit_count(struct vm_gk20a *vm)
774 {
775         return vm->mmu_levels[0].lo_bit[0];
776 }
777
778 /* given address range (inclusive) determine the pdes crossed */
779 void pde_range_from_vaddr_range(struct vm_gk20a *vm,
780                                               u64 addr_lo, u64 addr_hi,
781                                               u32 *pde_lo, u32 *pde_hi)
782 {
783         int pde_shift = gk20a_mm_pde_coverage_bit_count(vm);
784
785         *pde_lo = (u32)(addr_lo >> pde_shift);
786         *pde_hi = (u32)(addr_hi >> pde_shift);
787         gk20a_dbg(gpu_dbg_pte, "addr_lo=0x%llx addr_hi=0x%llx pde_ss=%d",
788                    addr_lo, addr_hi, pde_shift);
789         gk20a_dbg(gpu_dbg_pte, "pde_lo=%d pde_hi=%d",
790                    *pde_lo, *pde_hi);
791 }
792
793 u32 *pde_from_index(struct vm_gk20a *vm, u32 i)
794 {
795         return (u32 *) (((u8 *)vm->pdb.cpu_va) + i*gmmu_pde__size_v());
796 }
797
798 u32 pte_index_from_vaddr(struct vm_gk20a *vm,
799                                        u64 addr, enum gmmu_pgsz_gk20a pgsz_idx)
800 {
801         u32 ret;
802         /* mask off pde part */
803         addr = addr & ((1ULL << gk20a_mm_pde_coverage_bit_count(vm)) - 1ULL);
804
805         /* shift over to get pte index. note assumption that pte index
806          * doesn't leak over into the high 32b */
807         ret = (u32)(addr >> ilog2(vm->gmmu_page_sizes[pgsz_idx]));
808
809         gk20a_dbg(gpu_dbg_pte, "addr=0x%llx pte_i=0x%x", addr, ret);
810         return ret;
811 }
812
813 static struct vm_reserved_va_node *addr_to_reservation(struct vm_gk20a *vm,
814                                                        u64 addr)
815 {
816         struct vm_reserved_va_node *va_node;
817         list_for_each_entry(va_node, &vm->reserved_va_list, reserved_va_list)
818                 if (addr >= va_node->vaddr_start &&
819                     addr < (u64)va_node->vaddr_start + (u64)va_node->size)
820                         return va_node;
821
822         return NULL;
823 }
824
825 int gk20a_vm_get_buffers(struct vm_gk20a *vm,
826                          struct mapped_buffer_node ***mapped_buffers,
827                          int *num_buffers)
828 {
829         struct mapped_buffer_node *mapped_buffer;
830         struct mapped_buffer_node **buffer_list;
831         struct rb_node *node;
832         int i = 0;
833
834         if (vm->userspace_managed) {
835                 *mapped_buffers = NULL;
836                 *num_buffers = 0;
837                 return 0;
838         }
839
840         mutex_lock(&vm->update_gmmu_lock);
841
842         buffer_list = nvgpu_alloc(sizeof(*buffer_list) *
843                               vm->num_user_mapped_buffers, true);
844         if (!buffer_list) {
845                 mutex_unlock(&vm->update_gmmu_lock);
846                 return -ENOMEM;
847         }
848
849         node = rb_first(&vm->mapped_buffers);
850         while (node) {
851                 mapped_buffer =
852                         container_of(node, struct mapped_buffer_node, node);
853                 if (mapped_buffer->user_mapped) {
854                         buffer_list[i] = mapped_buffer;
855                         kref_get(&mapped_buffer->ref);
856                         i++;
857                 }
858                 node = rb_next(&mapped_buffer->node);
859         }
860
861         BUG_ON(i != vm->num_user_mapped_buffers);
862
863         *num_buffers = vm->num_user_mapped_buffers;
864         *mapped_buffers = buffer_list;
865
866         mutex_unlock(&vm->update_gmmu_lock);
867
868         return 0;
869 }
870
871 static void gk20a_vm_unmap_locked_kref(struct kref *ref)
872 {
873         struct mapped_buffer_node *mapped_buffer =
874                 container_of(ref, struct mapped_buffer_node, ref);
875         gk20a_vm_unmap_locked(mapped_buffer, mapped_buffer->vm->kref_put_batch);
876 }
877
878 void gk20a_vm_mapping_batch_start(struct vm_gk20a_mapping_batch *mapping_batch)
879 {
880         memset(mapping_batch, 0, sizeof(*mapping_batch));
881         mapping_batch->gpu_l2_flushed = false;
882         mapping_batch->need_tlb_invalidate = false;
883 }
884
885 void gk20a_vm_mapping_batch_finish_locked(
886         struct vm_gk20a *vm, struct vm_gk20a_mapping_batch *mapping_batch)
887 {
888          /* hanging kref_put batch pointer? */
889         WARN_ON(vm->kref_put_batch == mapping_batch);
890
891         if (mapping_batch->need_tlb_invalidate) {
892                 struct gk20a *g = gk20a_from_vm(vm);
893                 g->ops.mm.tlb_invalidate(vm);
894         }
895 }
896
897 void gk20a_vm_mapping_batch_finish(struct vm_gk20a *vm,
898                                    struct vm_gk20a_mapping_batch *mapping_batch)
899 {
900         mutex_lock(&vm->update_gmmu_lock);
901         gk20a_vm_mapping_batch_finish_locked(vm, mapping_batch);
902         mutex_unlock(&vm->update_gmmu_lock);
903 }
904
905 void gk20a_vm_put_buffers(struct vm_gk20a *vm,
906                                  struct mapped_buffer_node **mapped_buffers,
907                                  int num_buffers)
908 {
909         int i;
910         struct vm_gk20a_mapping_batch batch;
911
912         mutex_lock(&vm->update_gmmu_lock);
913         gk20a_vm_mapping_batch_start(&batch);
914         vm->kref_put_batch = &batch;
915
916         for (i = 0; i < num_buffers; ++i)
917                 kref_put(&mapped_buffers[i]->ref,
918                          gk20a_vm_unmap_locked_kref);
919
920         vm->kref_put_batch = NULL;
921         gk20a_vm_mapping_batch_finish_locked(vm, &batch);
922         mutex_unlock(&vm->update_gmmu_lock);
923
924         nvgpu_free(mapped_buffers);
925 }
926
927 static void gk20a_vm_unmap_user(struct vm_gk20a *vm, u64 offset,
928                                 struct vm_gk20a_mapping_batch *batch)
929 {
930         struct device *d = dev_from_vm(vm);
931         int retries = 10000; /* 50 ms */
932         struct mapped_buffer_node *mapped_buffer;
933
934         mutex_lock(&vm->update_gmmu_lock);
935
936         mapped_buffer = find_mapped_buffer_locked(&vm->mapped_buffers, offset);
937         if (!mapped_buffer) {
938                 mutex_unlock(&vm->update_gmmu_lock);
939                 gk20a_err(d, "invalid addr to unmap 0x%llx", offset);
940                 return;
941         }
942
943         if (mapped_buffer->flags & NVGPU_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET) {
944                 mutex_unlock(&vm->update_gmmu_lock);
945
946                 while (retries >= 0 || !tegra_platform_is_silicon()) {
947                         if (atomic_read(&mapped_buffer->ref.refcount) == 1)
948                                 break;
949                         retries--;
950                         udelay(5);
951                 }
952                 if (retries < 0 && tegra_platform_is_silicon())
953                         gk20a_err(d, "sync-unmap failed on 0x%llx",
954                                                                 offset);
955                 mutex_lock(&vm->update_gmmu_lock);
956         }
957
958         mapped_buffer->user_mapped--;
959         if (mapped_buffer->user_mapped == 0)
960                 vm->num_user_mapped_buffers--;
961
962         vm->kref_put_batch = batch;
963         kref_put(&mapped_buffer->ref, gk20a_vm_unmap_locked_kref);
964         vm->kref_put_batch = NULL;
965
966         mutex_unlock(&vm->update_gmmu_lock);
967 }
968
969 u64 gk20a_vm_alloc_va(struct vm_gk20a *vm,
970                       u64 size,
971                       enum gmmu_pgsz_gk20a gmmu_pgsz_idx)
972
973 {
974         struct gk20a_allocator *vma = &vm->vma[gmmu_pgsz_idx];
975         u64 offset;
976         u64 gmmu_page_size = vm->gmmu_page_sizes[gmmu_pgsz_idx];
977
978         if (gmmu_pgsz_idx >= gmmu_nr_page_sizes) {
979                 dev_warn(dev_from_vm(vm),
980                          "invalid page size requested in gk20a vm alloc");
981                 return 0;
982         }
983
984         if ((gmmu_pgsz_idx == gmmu_page_size_big) && !vm->big_pages) {
985                 dev_warn(dev_from_vm(vm),
986                          "unsupportd page size requested");
987                 return 0;
988
989         }
990
991         /* Be certain we round up to gmmu_page_size if needed */
992         size = (size + ((u64)gmmu_page_size - 1)) & ~((u64)gmmu_page_size - 1);
993         gk20a_dbg_info("size=0x%llx @ pgsz=%dKB", size,
994                         vm->gmmu_page_sizes[gmmu_pgsz_idx]>>10);
995
996         offset = gk20a_balloc(vma, size);
997         if (!offset) {
998                 gk20a_err(dev_from_vm(vm),
999                           "%s oom: sz=0x%llx", vma->name, size);
1000                 return 0;
1001         }
1002
1003         gk20a_dbg_fn("%s found addr: 0x%llx", vma->name, offset);
1004         return offset;
1005 }
1006
1007 int gk20a_vm_free_va(struct vm_gk20a *vm,
1008                      u64 offset, u64 size,
1009                      enum gmmu_pgsz_gk20a pgsz_idx)
1010 {
1011         struct gk20a_allocator *vma = &vm->vma[pgsz_idx];
1012
1013         gk20a_dbg_info("%s free addr=0x%llx, size=0x%llx",
1014                         vma->name, offset, size);
1015         gk20a_bfree(vma, offset);
1016
1017         return 0;
1018 }
1019
1020 static int insert_mapped_buffer(struct rb_root *root,
1021                                 struct mapped_buffer_node *mapped_buffer)
1022 {
1023         struct rb_node **new_node = &(root->rb_node), *parent = NULL;
1024
1025         /* Figure out where to put new node */
1026         while (*new_node) {
1027                 struct mapped_buffer_node *cmp_with =
1028                         container_of(*new_node, struct mapped_buffer_node,
1029                                      node);
1030
1031                 parent = *new_node;
1032
1033                 if (cmp_with->addr > mapped_buffer->addr) /* u64 cmp */
1034                         new_node = &((*new_node)->rb_left);
1035                 else if (cmp_with->addr != mapped_buffer->addr) /* u64 cmp */
1036                         new_node = &((*new_node)->rb_right);
1037                 else
1038                         return -EINVAL; /* no fair dup'ing */
1039         }
1040
1041         /* Add new node and rebalance tree. */
1042         rb_link_node(&mapped_buffer->node, parent, new_node);
1043         rb_insert_color(&mapped_buffer->node, root);
1044
1045         return 0;
1046 }
1047
1048 static struct mapped_buffer_node *find_mapped_buffer_reverse_locked(
1049                                 struct rb_root *root, struct dma_buf *dmabuf,
1050                                 u32 kind)
1051 {
1052         struct rb_node *node = rb_first(root);
1053         while (node) {
1054                 struct mapped_buffer_node *mapped_buffer =
1055                         container_of(node, struct mapped_buffer_node, node);
1056                 if (mapped_buffer->dmabuf == dmabuf &&
1057                     kind == mapped_buffer->kind)
1058                         return mapped_buffer;
1059                 node = rb_next(&mapped_buffer->node);
1060         }
1061         return NULL;
1062 }
1063
1064 static struct mapped_buffer_node *find_mapped_buffer_locked(
1065                                         struct rb_root *root, u64 addr)
1066 {
1067
1068         struct rb_node *node = root->rb_node;
1069         while (node) {
1070                 struct mapped_buffer_node *mapped_buffer =
1071                         container_of(node, struct mapped_buffer_node, node);
1072                 if (mapped_buffer->addr > addr) /* u64 cmp */
1073                         node = node->rb_left;
1074                 else if (mapped_buffer->addr != addr) /* u64 cmp */
1075                         node = node->rb_right;
1076                 else
1077                         return mapped_buffer;
1078         }
1079         return NULL;
1080 }
1081
1082 static struct mapped_buffer_node *find_mapped_buffer_range_locked(
1083                                         struct rb_root *root, u64 addr)
1084 {
1085         struct rb_node *node = root->rb_node;
1086         while (node) {
1087                 struct mapped_buffer_node *m =
1088                         container_of(node, struct mapped_buffer_node, node);
1089                 if (m->addr <= addr && m->addr + m->size > addr)
1090                         return m;
1091                 else if (m->addr > addr) /* u64 cmp */
1092                         node = node->rb_left;
1093                 else
1094                         node = node->rb_right;
1095         }
1096         return NULL;
1097 }
1098
1099 /* find the first mapped buffer with GPU VA less than addr */
1100 static struct mapped_buffer_node *find_mapped_buffer_less_than_locked(
1101         struct rb_root *root, u64 addr)
1102 {
1103         struct rb_node *node = root->rb_node;
1104         struct mapped_buffer_node *ret = NULL;
1105
1106         while (node) {
1107                 struct mapped_buffer_node *mapped_buffer =
1108                         container_of(node, struct mapped_buffer_node, node);
1109                 if (mapped_buffer->addr >= addr)
1110                         node = node->rb_left;
1111                 else {
1112                         ret = mapped_buffer;
1113                         node = node->rb_right;
1114                 }
1115         }
1116
1117         return ret;
1118 }
1119
1120 #define BFR_ATTRS (sizeof(nvmap_bfr_param)/sizeof(nvmap_bfr_param[0]))
1121
1122 struct buffer_attrs {
1123         struct sg_table *sgt;
1124         u64 size;
1125         u64 align;
1126         u32 ctag_offset;
1127         u32 ctag_lines;
1128         u32 ctag_allocated_lines;
1129         int pgsz_idx;
1130         u8 kind_v;
1131         u8 uc_kind_v;
1132         bool ctag_user_mappable;
1133 };
1134
1135 static void gmmu_select_page_size(struct vm_gk20a *vm,
1136                                   struct buffer_attrs *bfr)
1137 {
1138         int i;
1139         /*  choose the biggest first (top->bottom) */
1140         for (i = gmmu_page_size_kernel - 1; i >= 0; i--)
1141                 if (!((vm->gmmu_page_sizes[i] - 1) & bfr->align)) {
1142                         bfr->pgsz_idx = i;
1143                         break;
1144                 }
1145 }
1146
1147 static int setup_buffer_kind_and_compression(struct vm_gk20a *vm,
1148                                              u32 flags,
1149                                              struct buffer_attrs *bfr,
1150                                              enum gmmu_pgsz_gk20a pgsz_idx)
1151 {
1152         bool kind_compressible;
1153         struct gk20a *g = gk20a_from_vm(vm);
1154         struct device *d = dev_from_gk20a(g);
1155         int ctag_granularity = g->ops.fb.compression_page_size(g);
1156
1157         if (unlikely(bfr->kind_v == gmmu_pte_kind_invalid_v()))
1158                 bfr->kind_v = gmmu_pte_kind_pitch_v();
1159
1160         if (unlikely(!gk20a_kind_is_supported(bfr->kind_v))) {
1161                 gk20a_err(d, "kind 0x%x not supported", bfr->kind_v);
1162                 return -EINVAL;
1163         }
1164
1165         bfr->uc_kind_v = gmmu_pte_kind_invalid_v();
1166         /* find a suitable uncompressed kind if it becomes necessary later */
1167         kind_compressible = gk20a_kind_is_compressible(bfr->kind_v);
1168         if (kind_compressible) {
1169                 bfr->uc_kind_v = gk20a_get_uncompressed_kind(bfr->kind_v);
1170                 if (unlikely(bfr->uc_kind_v == gmmu_pte_kind_invalid_v())) {
1171                         /* shouldn't happen, but it is worth cross-checking */
1172                         gk20a_err(d, "comptag kind 0x%x can't be"
1173                                    " downgraded to uncompressed kind",
1174                                    bfr->kind_v);
1175                         return -EINVAL;
1176                 }
1177         }
1178         /* comptags only supported for suitable kinds, 128KB pagesize */
1179         if (kind_compressible &&
1180             vm->gmmu_page_sizes[pgsz_idx] < g->ops.fb.compressible_page_size(g)) {
1181                 /*
1182                 gk20a_warn(d, "comptags specified"
1183                 " but pagesize being used doesn't support it");*/
1184                 /* it is safe to fall back to uncompressed as
1185                    functionality is not harmed */
1186                 bfr->kind_v = bfr->uc_kind_v;
1187                 kind_compressible = false;
1188         }
1189         if (kind_compressible)
1190                 bfr->ctag_lines = DIV_ROUND_UP_ULL(bfr->size, ctag_granularity);
1191         else
1192                 bfr->ctag_lines = 0;
1193
1194         return 0;
1195 }
1196
1197 static int validate_fixed_buffer(struct vm_gk20a *vm,
1198                                  struct buffer_attrs *bfr,
1199                                  u64 map_offset, u64 map_size,
1200                                  struct vm_reserved_va_node **pva_node)
1201 {
1202         struct device *dev = dev_from_vm(vm);
1203         struct vm_reserved_va_node *va_node;
1204         struct mapped_buffer_node *buffer;
1205         u64 map_end = map_offset + map_size;
1206
1207         /* can wrap around with insane map_size; zero is disallowed too */
1208         if (map_end <= map_offset) {
1209                 gk20a_warn(dev, "fixed offset mapping with invalid map_size");
1210                 return -EINVAL;
1211         }
1212
1213         if (map_offset & (vm->gmmu_page_sizes[bfr->pgsz_idx] - 1)) {
1214                 gk20a_err(dev, "map offset must be buffer page size aligned 0x%llx",
1215                           map_offset);
1216                 return -EINVAL;
1217         }
1218
1219         /* Find the space reservation, but it's ok to have none for
1220          * userspace-managed address spaces */
1221         va_node = addr_to_reservation(vm, map_offset);
1222         if (!va_node && !vm->userspace_managed) {
1223                 gk20a_warn(dev, "fixed offset mapping without space allocation");
1224                 return -EINVAL;
1225         }
1226
1227         /* Mapped area should fit inside va, if there's one */
1228         if (va_node && map_end > va_node->vaddr_start + va_node->size) {
1229                 gk20a_warn(dev, "fixed offset mapping size overflows va node");
1230                 return -EINVAL;
1231         }
1232
1233         /* check that this mapping does not collide with existing
1234          * mappings by checking the buffer with the highest GPU VA
1235          * that is less than our buffer end */
1236         buffer = find_mapped_buffer_less_than_locked(
1237                 &vm->mapped_buffers, map_offset + map_size);
1238         if (buffer && buffer->addr + buffer->size > map_offset) {
1239                 gk20a_warn(dev, "overlapping buffer map requested");
1240                 return -EINVAL;
1241         }
1242
1243         *pva_node = va_node;
1244
1245         return 0;
1246 }
1247
1248 u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm,
1249                         u64 map_offset,
1250                         struct sg_table *sgt,
1251                         u64 buffer_offset,
1252                         u64 size,
1253                         int pgsz_idx,
1254                         u8 kind_v,
1255                         u32 ctag_offset,
1256                         u32 flags,
1257                         int rw_flag,
1258                         bool clear_ctags,
1259                         bool sparse,
1260                         bool priv,
1261                         struct vm_gk20a_mapping_batch *batch)
1262 {
1263         int err = 0;
1264         bool allocated = false;
1265         struct device *d = dev_from_vm(vm);
1266         struct gk20a *g = gk20a_from_vm(vm);
1267
1268         /* Allocate (or validate when map_offset != 0) the virtual address. */
1269         if (!map_offset) {
1270                 map_offset = gk20a_vm_alloc_va(vm, size,
1271                                           pgsz_idx);
1272                 if (!map_offset) {
1273                         gk20a_err(d, "failed to allocate va space");
1274                         err = -ENOMEM;
1275                         goto fail_alloc;
1276                 }
1277                 allocated = true;
1278         }
1279
1280         err = update_gmmu_ptes_locked(vm, pgsz_idx,
1281                                       sgt,
1282                                       buffer_offset,
1283                                       map_offset, map_offset + size,
1284                                       kind_v,
1285                                       ctag_offset,
1286                                       flags &
1287                                       NVGPU_MAP_BUFFER_FLAGS_CACHEABLE_TRUE,
1288                                       flags &
1289                                       NVGPU_AS_MAP_BUFFER_FLAGS_UNMAPPED_PTE,
1290                                       rw_flag,
1291                                       sparse,
1292                                       priv);
1293         if (err) {
1294                 gk20a_err(d, "failed to update ptes on map");
1295                 goto fail_validate;
1296         }
1297
1298         if (!batch)
1299                 g->ops.mm.tlb_invalidate(vm);
1300         else
1301                 batch->need_tlb_invalidate = true;
1302
1303         return map_offset;
1304 fail_validate:
1305         if (allocated)
1306                 gk20a_vm_free_va(vm, map_offset, size, pgsz_idx);
1307 fail_alloc:
1308         gk20a_err(d, "%s: failed with err=%d\n", __func__, err);
1309         return 0;
1310 }
1311
1312 void gk20a_locked_gmmu_unmap(struct vm_gk20a *vm,
1313                         u64 vaddr,
1314                         u64 size,
1315                         int pgsz_idx,
1316                         bool va_allocated,
1317                         int rw_flag,
1318                         bool sparse,
1319                         struct vm_gk20a_mapping_batch *batch)
1320 {
1321         int err = 0;
1322         struct gk20a *g = gk20a_from_vm(vm);
1323
1324         if (va_allocated) {
1325                 err = gk20a_vm_free_va(vm, vaddr, size, pgsz_idx);
1326                 if (err) {
1327                         dev_err(dev_from_vm(vm),
1328                                 "failed to free va");
1329                         return;
1330                 }
1331         }
1332
1333         /* unmap here needs to know the page size we assigned at mapping */
1334         err = update_gmmu_ptes_locked(vm,
1335                                 pgsz_idx,
1336                                 NULL, /* n/a for unmap */
1337                                 0,
1338                                 vaddr,
1339                                 vaddr + size,
1340                                 0, 0, false /* n/a for unmap */,
1341                                 false, rw_flag,
1342                                 sparse, 0);
1343         if (err)
1344                 dev_err(dev_from_vm(vm),
1345                         "failed to update gmmu ptes on unmap");
1346
1347         /* flush l2 so any dirty lines are written out *now*.
1348          *  also as we could potentially be switching this buffer
1349          * from nonvolatile (l2 cacheable) to volatile (l2 non-cacheable) at
1350          * some point in the future we need to invalidate l2.  e.g. switching
1351          * from a render buffer unmap (here) to later using the same memory
1352          * for gmmu ptes.  note the positioning of this relative to any smmu
1353          * unmapping (below). */
1354
1355         if (!batch) {
1356                 gk20a_mm_l2_flush(g, true);
1357                 g->ops.mm.tlb_invalidate(vm);
1358         } else {
1359                 if (!batch->gpu_l2_flushed) {
1360                         gk20a_mm_l2_flush(g, true);
1361                         batch->gpu_l2_flushed = true;
1362                 }
1363                 batch->need_tlb_invalidate = true;
1364         }
1365 }
1366
1367 static u64 gk20a_vm_map_duplicate_locked(struct vm_gk20a *vm,
1368                                          struct dma_buf *dmabuf,
1369                                          u64 offset_align,
1370                                          u32 flags,
1371                                          int kind,
1372                                          struct sg_table **sgt,
1373                                          bool user_mapped,
1374                                          int rw_flag)
1375 {
1376         struct mapped_buffer_node *mapped_buffer = NULL;
1377
1378         mapped_buffer =
1379                 find_mapped_buffer_reverse_locked(&vm->mapped_buffers,
1380                                                   dmabuf, kind);
1381         if (!mapped_buffer)
1382                 return 0;
1383
1384         if (mapped_buffer->flags != flags)
1385                 return 0;
1386
1387         if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET &&
1388             mapped_buffer->addr != offset_align)
1389                 return 0;
1390
1391         BUG_ON(mapped_buffer->vm != vm);
1392
1393         /* mark the buffer as used */
1394         if (user_mapped) {
1395                 if (mapped_buffer->user_mapped == 0)
1396                         vm->num_user_mapped_buffers++;
1397                 mapped_buffer->user_mapped++;
1398
1399                 /* If the mapping comes from user space, we own
1400                  * the handle ref. Since we reuse an
1401                  * existing mapping here, we need to give back those
1402                  * refs once in order not to leak.
1403                  */
1404                 if (mapped_buffer->own_mem_ref)
1405                         dma_buf_put(mapped_buffer->dmabuf);
1406                 else
1407                         mapped_buffer->own_mem_ref = true;
1408         }
1409         kref_get(&mapped_buffer->ref);
1410
1411         gk20a_dbg(gpu_dbg_map,
1412                    "reusing as=%d pgsz=%d flags=0x%x ctags=%d "
1413                    "start=%d gv=0x%x,%08x -> 0x%x,%08x -> 0x%x,%08x "
1414                    "own_mem_ref=%d user_mapped=%d",
1415                    vm_aspace_id(vm), mapped_buffer->pgsz_idx,
1416                    mapped_buffer->flags,
1417                    mapped_buffer->ctag_lines,
1418                    mapped_buffer->ctag_offset,
1419                    hi32(mapped_buffer->addr), lo32(mapped_buffer->addr),
1420                    hi32((u64)sg_dma_address(mapped_buffer->sgt->sgl)),
1421                    lo32((u64)sg_dma_address(mapped_buffer->sgt->sgl)),
1422                    hi32((u64)sg_phys(mapped_buffer->sgt->sgl)),
1423                    lo32((u64)sg_phys(mapped_buffer->sgt->sgl)),
1424                    mapped_buffer->own_mem_ref, user_mapped);
1425
1426         if (sgt)
1427                 *sgt = mapped_buffer->sgt;
1428         return mapped_buffer->addr;
1429 }
1430
1431 u64 gk20a_vm_map(struct vm_gk20a *vm,
1432                         struct dma_buf *dmabuf,
1433                         u64 offset_align,
1434                         u32 flags /*NVGPU_AS_MAP_BUFFER_FLAGS_*/,
1435                         int kind,
1436                         struct sg_table **sgt,
1437                         bool user_mapped,
1438                         int rw_flag,
1439                         u64 buffer_offset,
1440                         u64 mapping_size,
1441                         struct vm_gk20a_mapping_batch *batch)
1442 {
1443         struct gk20a *g = gk20a_from_vm(vm);
1444         struct gk20a_comptag_allocator *ctag_allocator = &g->gr.comp_tags;
1445         struct device *d = dev_from_vm(vm);
1446         struct mapped_buffer_node *mapped_buffer = NULL;
1447         bool inserted = false, va_allocated = false;
1448         u32 gmmu_page_size = 0;
1449         u64 map_offset = 0;
1450         int err = 0;
1451         struct buffer_attrs bfr = {NULL};
1452         struct gk20a_comptags comptags;
1453         bool clear_ctags = false;
1454         struct scatterlist *sgl;
1455         u64 buf_addr;
1456         u64 ctag_map_win_size = 0;
1457         u32 ctag_map_win_ctagline = 0;
1458         struct vm_reserved_va_node *va_node = NULL;
1459         u32 ctag_offset;
1460
1461         if (user_mapped && vm->userspace_managed &&
1462             !(flags & NVGPU_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET)) {
1463                 gk20a_err(d,
1464                           "%s: non-fixed-offset mapping not available on userspace managed address spaces",
1465                           __func__);
1466                 return -EFAULT;
1467         }
1468
1469         mutex_lock(&vm->update_gmmu_lock);
1470
1471         /* check if this buffer is already mapped */
1472         if (!vm->userspace_managed) {
1473                 map_offset = gk20a_vm_map_duplicate_locked(
1474                         vm, dmabuf, offset_align,
1475                         flags, kind, sgt,
1476                         user_mapped, rw_flag);
1477                 if (map_offset) {
1478                         mutex_unlock(&vm->update_gmmu_lock);
1479                         return map_offset;
1480                 }
1481         }
1482
1483         /* pin buffer to get phys/iovmm addr */
1484         bfr.sgt = gk20a_mm_pin(d, dmabuf);
1485         if (IS_ERR(bfr.sgt)) {
1486                 /* Falling back to physical is actually possible
1487                  * here in many cases if we use 4K phys pages in the
1488                  * gmmu.  However we have some regions which require
1489                  * contig regions to work properly (either phys-contig
1490                  * or contig through smmu io_vaspace).  Until we can
1491                  * track the difference between those two cases we have
1492                  * to fail the mapping when we run out of SMMU space.
1493                  */
1494                 gk20a_warn(d, "oom allocating tracking buffer");
1495                 goto clean_up;
1496         }
1497
1498         if (sgt)
1499                 *sgt = bfr.sgt;
1500
1501         bfr.kind_v = kind;
1502         bfr.size = dmabuf->size;
1503         sgl = bfr.sgt->sgl;
1504         buf_addr = (u64)sg_dma_address(bfr.sgt->sgl);
1505         if (g->mm.bypass_smmu || buf_addr == DMA_ERROR_CODE || !buf_addr) {
1506                 while (sgl) {
1507                         u64 align;
1508
1509                         buf_addr = (u64)sg_phys(sgl);
1510
1511                         align = 1ULL << __ffs(buf_addr | (u64)sgl->length);
1512                         if (bfr.align)
1513                                 bfr.align = min_t(u64, align, bfr.align);
1514                         else
1515                                 bfr.align = align;
1516                         sgl = sg_next(sgl);
1517                 }
1518         } else
1519                 bfr.align = 1ULL << __ffs(buf_addr);
1520
1521         bfr.pgsz_idx = -1;
1522         mapping_size = mapping_size ? mapping_size : bfr.size;
1523
1524         if (vm->big_pages)
1525                 gmmu_select_page_size(vm, &bfr);
1526         else
1527                 bfr.pgsz_idx = gmmu_page_size_small;
1528
1529         /* If FIX_OFFSET is set, pgsz is determined at address allocation
1530          * time. The alignment at address alloc time must be the same as
1531          * the alignment determined by gmmu_select_page_size().
1532          */
1533         if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET) {
1534                 int pgsz_idx =
1535                         __nv_gmmu_va_is_big_page_region(vm, offset_align) ?
1536                         gmmu_page_size_big : gmmu_page_size_small;
1537                 if (pgsz_idx > bfr.pgsz_idx) {
1538                         gk20a_err(d, "%llx buffer pgsz %d, VA pgsz %d",
1539                                   offset_align, bfr.pgsz_idx, pgsz_idx);
1540                         err = -EINVAL;
1541                         goto clean_up;
1542                 }
1543                 bfr.pgsz_idx = min(bfr.pgsz_idx, pgsz_idx);
1544         }
1545
1546         /* validate/adjust bfr attributes */
1547         if (unlikely(bfr.pgsz_idx == -1)) {
1548                 gk20a_err(d, "unsupported page size detected");
1549                 goto clean_up;
1550         }
1551
1552         if (unlikely(bfr.pgsz_idx < gmmu_page_size_small ||
1553                      bfr.pgsz_idx > gmmu_page_size_big)) {
1554                 BUG_ON(1);
1555                 err = -EINVAL;
1556                 goto clean_up;
1557         }
1558         gmmu_page_size = vm->gmmu_page_sizes[bfr.pgsz_idx];
1559
1560         /* Check if we should use a fixed offset for mapping this buffer */
1561
1562         if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET)  {
1563                 err = validate_fixed_buffer(vm, &bfr,
1564                                             offset_align, mapping_size,
1565                                             &va_node);
1566                 if (err)
1567                         goto clean_up;
1568
1569                 map_offset = offset_align;
1570                 va_allocated = false;
1571         } else
1572                 va_allocated = true;
1573
1574         if (sgt)
1575                 *sgt = bfr.sgt;
1576
1577         err = setup_buffer_kind_and_compression(vm, flags, &bfr, bfr.pgsz_idx);
1578         if (unlikely(err)) {
1579                 gk20a_err(d, "failure setting up kind and compression");
1580                 goto clean_up;
1581         }
1582
1583         /* bar1 and pmu vm don't need ctag */
1584         if (!vm->enable_ctag)
1585                 bfr.ctag_lines = 0;
1586
1587         gk20a_get_comptags(d, dmabuf, &comptags);
1588
1589         /* ensure alignment to compression page size if compression enabled */
1590         if (bfr.ctag_offset)
1591                 mapping_size = ALIGN(mapping_size,
1592                                      g->ops.fb.compression_page_size(g));
1593
1594         if (bfr.ctag_lines && !comptags.lines) {
1595                 const bool user_mappable =
1596                         !!(flags & NVGPU_AS_MAP_BUFFER_FLAGS_MAPPABLE_COMPBITS);
1597
1598                 /* allocate compression resources if needed */
1599                 err = gk20a_alloc_comptags(g, d, dmabuf, ctag_allocator,
1600                                            bfr.ctag_lines, user_mappable,
1601                                            &ctag_map_win_size,
1602                                            &ctag_map_win_ctagline);
1603                 if (err) {
1604                         /* ok to fall back here if we ran out */
1605                         /* TBD: we can partially alloc ctags as well... */
1606                         bfr.kind_v = bfr.uc_kind_v;
1607                 } else {
1608                         gk20a_get_comptags(d, dmabuf, &comptags);
1609
1610                         if (g->ops.ltc.cbc_ctrl)
1611                                 g->ops.ltc.cbc_ctrl(g, gk20a_cbc_op_clear,
1612                                                     comptags.offset,
1613                                                     comptags.offset +
1614                                                         comptags.allocated_lines - 1);
1615                         else
1616                                 clear_ctags = true;
1617                 }
1618         }
1619
1620         /* store the comptag info */
1621         bfr.ctag_offset = comptags.offset;
1622         bfr.ctag_lines = comptags.lines;
1623         bfr.ctag_allocated_lines = comptags.allocated_lines;
1624         bfr.ctag_user_mappable = comptags.user_mappable;
1625
1626         /*
1627          * Calculate comptag index for this mapping. Differs in
1628          * case of partial mapping.
1629          */
1630         ctag_offset = comptags.offset;
1631         if (ctag_offset)
1632                 ctag_offset += buffer_offset >>
1633                                ilog2(g->ops.fb.compression_page_size(g));
1634
1635         /* update gmmu ptes */
1636         map_offset = g->ops.mm.gmmu_map(vm, map_offset,
1637                                         bfr.sgt,
1638                                         buffer_offset, /* sg offset */
1639                                         mapping_size,
1640                                         bfr.pgsz_idx,
1641                                         bfr.kind_v,
1642                                         ctag_offset,
1643                                         flags, rw_flag,
1644                                         clear_ctags,
1645                                         false,
1646                                         false,
1647                                         batch);
1648         if (!map_offset)
1649                 goto clean_up;
1650
1651         gk20a_dbg(gpu_dbg_map,
1652            "as=%d pgsz=%d "
1653            "kind=0x%x kind_uc=0x%x flags=0x%x "
1654            "ctags=%d start=%d ctags_allocated=%d ctags_mappable=%d gv=0x%x,%08x -> 0x%x,%08x -> 0x%x,%08x",
1655            vm_aspace_id(vm), gmmu_page_size,
1656            bfr.kind_v, bfr.uc_kind_v, flags,
1657            bfr.ctag_lines, bfr.ctag_offset,
1658            bfr.ctag_allocated_lines, bfr.ctag_user_mappable,
1659            hi32(map_offset), lo32(map_offset),
1660            hi32((u64)sg_dma_address(bfr.sgt->sgl)),
1661            lo32((u64)sg_dma_address(bfr.sgt->sgl)),
1662            hi32((u64)sg_phys(bfr.sgt->sgl)),
1663            lo32((u64)sg_phys(bfr.sgt->sgl)));
1664
1665 #if defined(NVHOST_DEBUG)
1666         {
1667                 int i;
1668                 struct scatterlist *sg = NULL;
1669                 gk20a_dbg(gpu_dbg_pte, "for_each_sg(bfr.sgt->sgl, sg, bfr.sgt->nents, i)");
1670                 for_each_sg(bfr.sgt->sgl, sg, bfr.sgt->nents, i ) {
1671                         u64 da = sg_dma_address(sg);
1672                         u64 pa = sg_phys(sg);
1673                         u64 len = sg->length;
1674                         gk20a_dbg(gpu_dbg_pte, "i=%d pa=0x%x,%08x da=0x%x,%08x len=0x%x,%08x",
1675                                    i, hi32(pa), lo32(pa), hi32(da), lo32(da),
1676                                    hi32(len), lo32(len));
1677                 }
1678         }
1679 #endif
1680
1681         /* keep track of the buffer for unmapping */
1682         /* TBD: check for multiple mapping of same buffer */
1683         mapped_buffer = kzalloc(sizeof(*mapped_buffer), GFP_KERNEL);
1684         if (!mapped_buffer) {
1685                 gk20a_warn(d, "oom allocating tracking buffer");
1686                 goto clean_up;
1687         }
1688         mapped_buffer->dmabuf      = dmabuf;
1689         mapped_buffer->sgt         = bfr.sgt;
1690         mapped_buffer->addr        = map_offset;
1691         mapped_buffer->size        = mapping_size;
1692         mapped_buffer->pgsz_idx    = bfr.pgsz_idx;
1693         mapped_buffer->ctag_offset = bfr.ctag_offset;
1694         mapped_buffer->ctag_lines  = bfr.ctag_lines;
1695         mapped_buffer->ctag_allocated_lines = bfr.ctag_allocated_lines;
1696         mapped_buffer->ctags_mappable = bfr.ctag_user_mappable;
1697         mapped_buffer->ctag_map_win_size = ctag_map_win_size;
1698         mapped_buffer->ctag_map_win_ctagline = ctag_map_win_ctagline;
1699         mapped_buffer->vm          = vm;
1700         mapped_buffer->flags       = flags;
1701         mapped_buffer->kind        = kind;
1702         mapped_buffer->va_allocated = va_allocated;
1703         mapped_buffer->user_mapped = user_mapped ? 1 : 0;
1704         mapped_buffer->own_mem_ref = user_mapped;
1705         INIT_LIST_HEAD(&mapped_buffer->unmap_list);
1706         INIT_LIST_HEAD(&mapped_buffer->va_buffers_list);
1707         kref_init(&mapped_buffer->ref);
1708
1709         err = insert_mapped_buffer(&vm->mapped_buffers, mapped_buffer);
1710         if (err) {
1711                 gk20a_err(d, "failed to insert into mapped buffer tree");
1712                 goto clean_up;
1713         }
1714         inserted = true;
1715         if (user_mapped)
1716                 vm->num_user_mapped_buffers++;
1717
1718         gk20a_dbg_info("allocated va @ 0x%llx", map_offset);
1719
1720         if (va_node) {
1721                 list_add_tail(&mapped_buffer->va_buffers_list,
1722                               &va_node->va_buffers_list);
1723                 mapped_buffer->va_node = va_node;
1724         }
1725
1726         mutex_unlock(&vm->update_gmmu_lock);
1727
1728         return map_offset;
1729
1730 clean_up:
1731         if (inserted) {
1732                 rb_erase(&mapped_buffer->node, &vm->mapped_buffers);
1733                 if (user_mapped)
1734                         vm->num_user_mapped_buffers--;
1735         }
1736         kfree(mapped_buffer);
1737         if (va_allocated)
1738                 gk20a_vm_free_va(vm, map_offset, bfr.size, bfr.pgsz_idx);
1739         if (!IS_ERR(bfr.sgt))
1740                 gk20a_mm_unpin(d, dmabuf, bfr.sgt);
1741
1742         mutex_unlock(&vm->update_gmmu_lock);
1743         gk20a_dbg_info("err=%d\n", err);
1744         return 0;
1745 }
1746
1747 int gk20a_vm_get_compbits_info(struct vm_gk20a *vm,
1748                                u64 mapping_gva,
1749                                u64 *compbits_win_size,
1750                                u32 *compbits_win_ctagline,
1751                                u32 *mapping_ctagline,
1752                                u32 *flags)
1753 {
1754         struct mapped_buffer_node *mapped_buffer;
1755         struct device *d = dev_from_vm(vm);
1756
1757         mutex_lock(&vm->update_gmmu_lock);
1758
1759         mapped_buffer = find_mapped_buffer_locked(&vm->mapped_buffers, mapping_gva);
1760
1761         if (!mapped_buffer | !mapped_buffer->user_mapped)
1762         {
1763                 mutex_unlock(&vm->update_gmmu_lock);
1764                 gk20a_err(d, "%s: bad offset 0x%llx", __func__, mapping_gva);
1765                 return -EFAULT;
1766         }
1767
1768         *compbits_win_size = 0;
1769         *compbits_win_ctagline = 0;
1770         *mapping_ctagline = 0;
1771         *flags = 0;
1772
1773         if (mapped_buffer->ctag_offset)
1774                 *flags |= NVGPU_AS_GET_BUFFER_COMPBITS_INFO_FLAGS_HAS_COMPBITS;
1775
1776         if (mapped_buffer->ctags_mappable)
1777         {
1778                 *flags |= NVGPU_AS_GET_BUFFER_COMPBITS_INFO_FLAGS_MAPPABLE;
1779                 *compbits_win_size = mapped_buffer->ctag_map_win_size;
1780                 *compbits_win_ctagline = mapped_buffer->ctag_map_win_ctagline;
1781                 *mapping_ctagline = mapped_buffer->ctag_offset;
1782         }
1783
1784         mutex_unlock(&vm->update_gmmu_lock);
1785         return 0;
1786 }
1787
1788
1789 int gk20a_vm_map_compbits(struct vm_gk20a *vm,
1790                           u64 mapping_gva,
1791                           u64 *compbits_win_gva,
1792                           u64 *mapping_iova,
1793                           u32 flags)
1794 {
1795         struct mapped_buffer_node *mapped_buffer;
1796         struct gk20a *g = gk20a_from_vm(vm);
1797         struct device *d = dev_from_vm(vm);
1798         const bool fixed_mapping =
1799                 (flags & NVGPU_AS_MAP_BUFFER_COMPBITS_FLAGS_FIXED_OFFSET) != 0;
1800
1801         if (vm->userspace_managed && !fixed_mapping) {
1802                 gk20a_err(d,
1803                           "%s: non-fixed-offset mapping is not available on userspace managed address spaces",
1804                           __func__);
1805                 return -EFAULT;
1806         }
1807
1808         if (fixed_mapping && !vm->userspace_managed) {
1809                 gk20a_err(d,
1810                           "%s: fixed-offset mapping is available only on userspace managed address spaces",
1811                           __func__);
1812                 return -EFAULT;
1813         }
1814
1815         mutex_lock(&vm->update_gmmu_lock);
1816
1817         mapped_buffer =
1818                 find_mapped_buffer_locked(&vm->mapped_buffers, mapping_gva);
1819
1820         if (!mapped_buffer || !mapped_buffer->user_mapped) {
1821                 mutex_unlock(&vm->update_gmmu_lock);
1822                 gk20a_err(d, "%s: bad offset 0x%llx", __func__, mapping_gva);
1823                 return -EFAULT;
1824         }
1825
1826         if (!mapped_buffer->ctags_mappable) {
1827                 mutex_unlock(&vm->update_gmmu_lock);
1828                 gk20a_err(d, "%s: comptags not mappable, offset 0x%llx",
1829                           __func__, mapping_gva);
1830                 return -EFAULT;
1831         }
1832
1833         if (!mapped_buffer->ctag_map_win_addr) {
1834                 const u32 small_pgsz_index = 0; /* small pages, 4K */
1835                 const u32 aggregate_cacheline_sz =
1836                         g->gr.cacheline_size * g->gr.slices_per_ltc *
1837                         g->ltc_count;
1838
1839                 /* first aggregate cacheline to map */
1840                 u32 cacheline_start; /* inclusive */
1841
1842                 /* offset of the start cacheline (will be page aligned) */
1843                 u64 cacheline_offset_start;
1844
1845                 if (!mapped_buffer->ctag_map_win_size) {
1846                         mutex_unlock(&vm->update_gmmu_lock);
1847                         gk20a_err(d,
1848                                   "%s: mapping 0x%llx does not have "
1849                                   "mappable comptags",
1850                                   __func__, mapping_gva);
1851                         return -EFAULT;
1852                 }
1853
1854                 cacheline_start = mapped_buffer->ctag_offset /
1855                         g->gr.comptags_per_cacheline;
1856                 cacheline_offset_start =
1857                         cacheline_start * aggregate_cacheline_sz;
1858
1859                 if (fixed_mapping) {
1860                         struct buffer_attrs bfr;
1861                         int err;
1862                         struct vm_reserved_va_node *va_node = NULL;
1863
1864                         memset(&bfr, 0, sizeof(bfr));
1865
1866                         bfr.pgsz_idx = small_pgsz_index;
1867
1868                         err = validate_fixed_buffer(
1869                                 vm, &bfr, *compbits_win_gva,
1870                                 mapped_buffer->ctag_map_win_size, &va_node);
1871
1872                         if (err) {
1873                                 mutex_unlock(&vm->update_gmmu_lock);
1874                                 return err;
1875                         }
1876
1877                         if (va_node) {
1878                                 /* this would create a dangling GPU VA
1879                                  * pointer if the space is freed
1880                                  * before before the buffer is
1881                                  * unmapped */
1882                                 mutex_unlock(&vm->update_gmmu_lock);
1883                                 gk20a_err(d,
1884                                           "%s: comptags cannot be mapped into allocated space",
1885                                           __func__);
1886                                 return -EINVAL;
1887                         }
1888                 }
1889
1890                 mapped_buffer->ctag_map_win_addr =
1891                         g->ops.mm.gmmu_map(
1892                                 vm,
1893                                 !fixed_mapping ? 0 : *compbits_win_gva, /* va */
1894                                 g->gr.compbit_store.mem.sgt,
1895                                 cacheline_offset_start, /* sg offset */
1896                                 mapped_buffer->ctag_map_win_size, /* size */
1897                                 small_pgsz_index,
1898                                 0, /* kind */
1899                                 0, /* ctag_offset */
1900                                 NVGPU_MAP_BUFFER_FLAGS_CACHEABLE_TRUE,
1901                                 gk20a_mem_flag_read_only,
1902                                 false, /* clear_ctags */
1903                                 false, /* sparse */
1904                                 false, /* priv */
1905                                 NULL); /* mapping_batch handle */
1906
1907                 if (!mapped_buffer->ctag_map_win_addr) {
1908                         mutex_unlock(&vm->update_gmmu_lock);
1909                         gk20a_err(d,
1910                                   "%s: failed to map comptags for mapping 0x%llx",
1911                                   __func__, mapping_gva);
1912                         return -ENOMEM;
1913                 }
1914         } else if (fixed_mapping && *compbits_win_gva &&
1915                    mapped_buffer->ctag_map_win_addr != *compbits_win_gva) {
1916                 mutex_unlock(&vm->update_gmmu_lock);
1917                 gk20a_err(d,
1918                           "%s: re-requesting comptags map into mismatching address. buffer offset 0x"
1919                           "%llx, existing comptag map at 0x%llx, requested remap 0x%llx",
1920                           __func__, mapping_gva,
1921                           mapped_buffer->ctag_map_win_addr, *compbits_win_gva);
1922                 return -EINVAL;
1923         }
1924
1925         *mapping_iova = gk20a_mm_iova_addr(g, mapped_buffer->sgt->sgl, 0);
1926         *compbits_win_gva = mapped_buffer->ctag_map_win_addr;
1927
1928         mutex_unlock(&vm->update_gmmu_lock);
1929
1930         return 0;
1931 }
1932
1933 u64 gk20a_gmmu_map(struct vm_gk20a *vm,
1934                 struct sg_table **sgt,
1935                 u64 size,
1936                 u32 flags,
1937                 int rw_flag,
1938                 bool priv)
1939 {
1940         struct gk20a *g = gk20a_from_vm(vm);
1941         u64 vaddr;
1942
1943         mutex_lock(&vm->update_gmmu_lock);
1944         vaddr = g->ops.mm.gmmu_map(vm, 0, /* already mapped? - No */
1945                                 *sgt, /* sg table */
1946                                 0, /* sg offset */
1947                                 size,
1948                                 gmmu_page_size_kernel,
1949                                 0, /* kind */
1950                                 0, /* ctag_offset */
1951                                 flags, rw_flag,
1952                                 false, /* clear_ctags */
1953                                 false, /* sparse */
1954                                 priv, /* priv */
1955                                 NULL); /* mapping_batch handle */
1956         mutex_unlock(&vm->update_gmmu_lock);
1957         if (!vaddr) {
1958                 gk20a_err(dev_from_vm(vm), "failed to allocate va space");
1959                 return 0;
1960         }
1961
1962         return vaddr;
1963 }
1964
1965 int gk20a_gmmu_alloc(struct gk20a *g, size_t size, struct mem_desc *mem)
1966 {
1967         return gk20a_gmmu_alloc_attr(g, 0, size, mem);
1968 }
1969
1970 int gk20a_gmmu_alloc_attr(struct gk20a *g, enum dma_attr attr, size_t size, struct mem_desc *mem)
1971 {
1972         struct device *d = dev_from_gk20a(g);
1973         int err;
1974         dma_addr_t iova;
1975
1976         gk20a_dbg_fn("");
1977
1978         if (attr) {
1979                 DEFINE_DMA_ATTRS(attrs);
1980                 dma_set_attr(attr, &attrs);
1981                 if (attr == DMA_ATTR_NO_KERNEL_MAPPING) {
1982                         mem->pages = dma_alloc_attrs(d,
1983                                         size, &iova, GFP_KERNEL, &attrs);
1984                         if (!mem->pages)
1985                                 return -ENOMEM;
1986                 } else {
1987                         mem->cpu_va = dma_alloc_attrs(d,
1988                                         size, &iova, GFP_KERNEL, &attrs);
1989                         if (!mem->cpu_va)
1990                                 return -ENOMEM;
1991                 }
1992         } else {
1993                 mem->cpu_va = dma_alloc_coherent(d, size, &iova, GFP_KERNEL);
1994                 if (!mem->cpu_va)
1995                         return -ENOMEM;
1996         }
1997
1998         if (attr == DMA_ATTR_NO_KERNEL_MAPPING)
1999                 err = gk20a_get_sgtable_from_pages(d, &mem->sgt, mem->pages,
2000                                                    iova, size);
2001         else {
2002                 err = gk20a_get_sgtable(d, &mem->sgt, mem->cpu_va, iova, size);
2003                 memset(mem->cpu_va, 0, size);
2004         }
2005         if (err)
2006                 goto fail_free;
2007
2008         mem->size = size;
2009
2010         gk20a_dbg_fn("done");
2011
2012         return 0;
2013
2014 fail_free:
2015         dma_free_coherent(d, size, mem->cpu_va, iova);
2016         mem->cpu_va = NULL;
2017         mem->sgt = NULL;
2018         return err;
2019 }
2020
2021 void gk20a_gmmu_free_attr(struct gk20a *g, enum dma_attr attr,
2022                           struct mem_desc *mem)
2023 {
2024         struct device *d = dev_from_gk20a(g);
2025
2026         if (mem->cpu_va || mem->pages) {
2027                 if (attr) {
2028                         DEFINE_DMA_ATTRS(attrs);
2029                         dma_set_attr(attr, &attrs);
2030                         if (attr == DMA_ATTR_NO_KERNEL_MAPPING)
2031                                 dma_free_attrs(d, mem->size, mem->pages,
2032                                                sg_dma_address(mem->sgt->sgl),
2033                                                &attrs);
2034                         else
2035                                 dma_free_attrs(d, mem->size, mem->cpu_va,
2036                                                sg_dma_address(mem->sgt->sgl),
2037                                                &attrs);
2038                 } else {
2039                         dma_free_coherent(d, mem->size, mem->cpu_va,
2040                                   sg_dma_address(mem->sgt->sgl));
2041                 }
2042
2043                 mem->cpu_va = NULL;
2044                 mem->pages = NULL;
2045         }
2046
2047         if (mem->sgt)
2048                 gk20a_free_sgtable(&mem->sgt);
2049 }
2050
2051 void gk20a_gmmu_free(struct gk20a *g, struct mem_desc *mem)
2052 {
2053         return gk20a_gmmu_free_attr(g, 0, mem);
2054 }
2055
2056 int gk20a_gmmu_alloc_map(struct vm_gk20a *vm, size_t size, struct mem_desc *mem)
2057 {
2058         return gk20a_gmmu_alloc_map_attr(vm, 0, size, mem);
2059 }
2060
2061 int gk20a_gmmu_alloc_map_attr(struct vm_gk20a *vm,
2062                          enum dma_attr attr, size_t size, struct mem_desc *mem)
2063 {
2064         int err = gk20a_gmmu_alloc_attr(vm->mm->g, attr, size, mem);
2065
2066         if (err)
2067                 return err;
2068
2069         mem->gpu_va = gk20a_gmmu_map(vm, &mem->sgt, size, 0,
2070                                      gk20a_mem_flag_none, false);
2071         if (!mem->gpu_va) {
2072                 err = -ENOMEM;
2073                 goto fail_free;
2074         }
2075
2076         return 0;
2077
2078 fail_free:
2079         gk20a_gmmu_free(vm->mm->g, mem);
2080         return err;
2081 }
2082
2083 void gk20a_gmmu_unmap_free(struct vm_gk20a *vm, struct mem_desc *mem)
2084 {
2085         if (mem->gpu_va)
2086                 gk20a_gmmu_unmap(vm, mem->gpu_va, mem->size, gk20a_mem_flag_none);
2087         mem->gpu_va = 0;
2088
2089         gk20a_gmmu_free(vm->mm->g, mem);
2090 }
2091
2092 dma_addr_t gk20a_mm_gpuva_to_iova_base(struct vm_gk20a *vm, u64 gpu_vaddr)
2093 {
2094         struct mapped_buffer_node *buffer;
2095         dma_addr_t addr = 0;
2096         struct gk20a *g = gk20a_from_vm(vm);
2097
2098         mutex_lock(&vm->update_gmmu_lock);
2099         buffer = find_mapped_buffer_locked(&vm->mapped_buffers, gpu_vaddr);
2100         if (buffer)
2101                 addr = g->ops.mm.get_iova_addr(g, buffer->sgt->sgl,
2102                                 buffer->flags);
2103         mutex_unlock(&vm->update_gmmu_lock);
2104
2105         return addr;
2106 }
2107
2108 void gk20a_gmmu_unmap(struct vm_gk20a *vm,
2109                 u64 vaddr,
2110                 u64 size,
2111                 int rw_flag)
2112 {
2113         struct gk20a *g = gk20a_from_vm(vm);
2114
2115         mutex_lock(&vm->update_gmmu_lock);
2116         g->ops.mm.gmmu_unmap(vm,
2117                         vaddr,
2118                         size,
2119                         gmmu_page_size_kernel,
2120                         true, /*va_allocated */
2121                         rw_flag,
2122                         false,
2123                         NULL);
2124         mutex_unlock(&vm->update_gmmu_lock);
2125 }
2126
2127 phys_addr_t gk20a_get_phys_from_iova(struct device *d,
2128                                 u64 dma_addr)
2129 {
2130         phys_addr_t phys;
2131         u64 iova;
2132
2133         struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(d);
2134         if (!mapping)
2135                 return dma_addr;
2136
2137         iova = dma_addr & PAGE_MASK;
2138         phys = iommu_iova_to_phys(mapping->domain, iova);
2139         return phys;
2140 }
2141
2142 /* get sg_table from already allocated buffer */
2143 int gk20a_get_sgtable(struct device *d, struct sg_table **sgt,
2144                         void *cpuva, u64 iova,
2145                         size_t size)
2146 {
2147         int err = 0;
2148         *sgt = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
2149         if (!(*sgt)) {
2150                 dev_err(d, "failed to allocate memory\n");
2151                 err = -ENOMEM;
2152                 goto fail;
2153         }
2154         err = dma_get_sgtable(d, *sgt,
2155                         cpuva, iova,
2156                         size);
2157         if (err) {
2158                 dev_err(d, "failed to create sg table\n");
2159                 goto fail;
2160         }
2161         sg_dma_address((*sgt)->sgl) = iova;
2162
2163         return 0;
2164  fail:
2165         if (*sgt) {
2166                 kfree(*sgt);
2167                 *sgt = NULL;
2168         }
2169         return err;
2170 }
2171
2172 int gk20a_get_sgtable_from_pages(struct device *d, struct sg_table **sgt,
2173                         struct page **pages, u64 iova,
2174                         size_t size)
2175 {
2176         int err = 0;
2177         *sgt = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
2178         if (!(*sgt)) {
2179                 dev_err(d, "failed to allocate memory\n");
2180                 err = -ENOMEM;
2181                 goto fail;
2182         }
2183         err = sg_alloc_table_from_pages(*sgt, pages,
2184                         DIV_ROUND_UP(size, 4096), 0, size, GFP_KERNEL);
2185         if (err) {
2186                 dev_err(d, "failed to allocate sg_table\n");
2187                 goto fail;
2188         }
2189         sg_dma_address((*sgt)->sgl) = iova;
2190
2191         return 0;
2192  fail:
2193         if (*sgt) {
2194                 kfree(*sgt);
2195                 *sgt = NULL;
2196         }
2197         return err;
2198 }
2199
2200 void gk20a_free_sgtable(struct sg_table **sgt)
2201 {
2202         sg_free_table(*sgt);
2203         kfree(*sgt);
2204         *sgt = NULL;
2205 }
2206
2207 u64 gk20a_mm_smmu_vaddr_translate(struct gk20a *g, dma_addr_t iova)
2208 {
2209         if (!device_is_iommuable(dev_from_gk20a(g)))
2210                 return iova;
2211         else
2212                 return iova | 1ULL << g->ops.mm.get_physical_addr_bits(g);
2213 }
2214
2215 u64 gk20a_mm_iova_addr(struct gk20a *g, struct scatterlist *sgl,
2216                 u32 flags)
2217 {
2218         if (!device_is_iommuable(dev_from_gk20a(g)))
2219                 return sg_phys(sgl);
2220
2221         if (sg_dma_address(sgl) == 0)
2222                 return sg_phys(sgl);
2223
2224         if (sg_dma_address(sgl) == DMA_ERROR_CODE)
2225                 return 0;
2226
2227         return gk20a_mm_smmu_vaddr_translate(g, sg_dma_address(sgl));
2228 }
2229
2230 /* for gk20a the "video memory" apertures here are misnomers. */
2231 static inline u32 big_valid_pde0_bits(u64 pte_addr)
2232 {
2233         u32 pde0_bits =
2234                 gmmu_pde_aperture_big_video_memory_f() |
2235                 gmmu_pde_address_big_sys_f(
2236                            (u32)(pte_addr >> gmmu_pde_address_shift_v()));
2237         return  pde0_bits;
2238 }
2239
2240 static inline u32 small_valid_pde1_bits(u64 pte_addr)
2241 {
2242         u32 pde1_bits =
2243                 gmmu_pde_aperture_small_video_memory_f() |
2244                 gmmu_pde_vol_small_true_f() | /* tbd: why? */
2245                 gmmu_pde_address_small_sys_f(
2246                            (u32)(pte_addr >> gmmu_pde_address_shift_v()));
2247         return pde1_bits;
2248 }
2249
2250 /* Given the current state of the ptes associated with a pde,
2251    determine value and write it out.  There's no checking
2252    here to determine whether or not a change was actually
2253    made.  So, superfluous updates will cause unnecessary
2254    pde invalidations.
2255 */
2256 static int update_gmmu_pde_locked(struct vm_gk20a *vm,
2257                            struct gk20a_mm_entry *pte,
2258                            u32 i, u32 gmmu_pgsz_idx,
2259                            struct scatterlist **sgl,
2260                            u64 *offset,
2261                            u64 *iova,
2262                            u32 kind_v, u32 *ctag,
2263                            bool cacheable, bool unammped_pte,
2264                            int rw_flag, bool sparse, bool priv)
2265 {
2266         struct gk20a *g = gk20a_from_vm(vm);
2267         bool small_valid, big_valid;
2268         u64 pte_addr_small = 0, pte_addr_big = 0;
2269         struct gk20a_mm_entry *entry = vm->pdb.entries + i;
2270         u32 pde_v[2] = {0, 0};
2271         u32 *pde;
2272
2273         gk20a_dbg_fn("");
2274
2275         small_valid = entry->size && entry->pgsz == gmmu_page_size_small;
2276         big_valid   = entry->size && entry->pgsz == gmmu_page_size_big;
2277
2278         if (small_valid)
2279                 pte_addr_small = g->ops.mm.get_iova_addr(g, entry->sgt->sgl, 0);
2280
2281         if (big_valid)
2282                 pte_addr_big = g->ops.mm.get_iova_addr(g, entry->sgt->sgl, 0);
2283
2284         pde_v[0] = gmmu_pde_size_full_f();
2285         pde_v[0] |= big_valid ? big_valid_pde0_bits(pte_addr_big) :
2286                 (gmmu_pde_aperture_big_invalid_f());
2287
2288         pde_v[1] |= (small_valid ?
2289                      small_valid_pde1_bits(pte_addr_small) :
2290                      (gmmu_pde_aperture_small_invalid_f() |
2291                       gmmu_pde_vol_small_false_f()))
2292                     |
2293                     (big_valid ? (gmmu_pde_vol_big_true_f()) :
2294                      gmmu_pde_vol_big_false_f());
2295
2296         pde = pde_from_index(vm, i);
2297
2298         gk20a_mem_wr32(pde, 0, pde_v[0]);
2299         gk20a_mem_wr32(pde, 1, pde_v[1]);
2300
2301         gk20a_dbg(gpu_dbg_pte, "pde:%d,sz=%d = 0x%x,0x%08x",
2302                   i, gmmu_pgsz_idx, pde_v[1], pde_v[0]);
2303         return 0;
2304 }
2305
2306 static int update_gmmu_pte_locked(struct vm_gk20a *vm,
2307                            struct gk20a_mm_entry *pte,
2308                            u32 i, u32 gmmu_pgsz_idx,
2309                            struct scatterlist **sgl,
2310                            u64 *offset,
2311                            u64 *iova,
2312                            u32 kind_v, u32 *ctag,
2313                            bool cacheable, bool unmapped_pte,
2314                            int rw_flag, bool sparse, bool priv)
2315 {
2316         struct gk20a *g = gk20a_from_vm(vm);
2317         u32 ctag_granularity = g->ops.fb.compression_page_size(g);
2318         u32 page_size  = vm->gmmu_page_sizes[gmmu_pgsz_idx];
2319         u32 pte_w[2] = {0, 0}; /* invalid pte */
2320
2321         if (*iova) {
2322                 if (unmapped_pte)
2323                         pte_w[0] = gmmu_pte_valid_false_f() |
2324                                 gmmu_pte_address_sys_f(*iova
2325                                 >> gmmu_pte_address_shift_v());
2326                 else
2327                         pte_w[0] = gmmu_pte_valid_true_f() |
2328                                 gmmu_pte_address_sys_f(*iova
2329                                 >> gmmu_pte_address_shift_v());
2330
2331                 if (priv)
2332                         pte_w[0] |= gmmu_pte_privilege_true_f();
2333
2334                 pte_w[1] = gmmu_pte_aperture_video_memory_f() |
2335                         gmmu_pte_kind_f(kind_v) |
2336                         gmmu_pte_comptagline_f(*ctag / ctag_granularity);
2337
2338                 if (vm->mm->use_full_comp_tag_line && *iova & 0x10000) {
2339                         pte_w[1] |= gmmu_pte_comptagline_f(
2340                                         1 << (gmmu_pte_comptagline_s() - 1));
2341                 }
2342
2343                 if (rw_flag == gk20a_mem_flag_read_only) {
2344                         pte_w[0] |= gmmu_pte_read_only_true_f();
2345                         pte_w[1] |=
2346                                 gmmu_pte_write_disable_true_f();
2347                 } else if (rw_flag ==
2348                            gk20a_mem_flag_write_only) {
2349                         pte_w[1] |=
2350                                 gmmu_pte_read_disable_true_f();
2351                 }
2352                 if (!unmapped_pte) {
2353                         if (!cacheable)
2354                                 pte_w[1] |=
2355                                         gmmu_pte_vol_true_f();
2356                         else {
2357                         /* Store cachable value behind
2358                          * gmmu_pte_write_disable_true_f */
2359                                 if (!cacheable)
2360                                         pte_w[1] |=
2361                                         gmmu_pte_write_disable_true_f();
2362                         }
2363                 }
2364
2365                 gk20a_dbg(gpu_dbg_pte,
2366                         "pte=%d iova=0x%llx kind=%d ctag=%d vol=%d [0x%08x, 0x%08x]",
2367                            i, *iova,
2368                            kind_v, *ctag / ctag_granularity, !cacheable,
2369                            pte_w[1], pte_w[0]);
2370
2371                 if (*ctag)
2372                         *ctag += page_size;
2373         } else if (sparse) {
2374                 pte_w[0] = gmmu_pte_valid_false_f();
2375                 pte_w[1] |= gmmu_pte_vol_true_f();
2376         } else {
2377                 gk20a_dbg(gpu_dbg_pte, "pte_cur=%d [0x0,0x0]", i);
2378         }
2379
2380         gk20a_mem_wr32(pte->cpu_va + i*8, 0, pte_w[0]);
2381         gk20a_mem_wr32(pte->cpu_va + i*8, 1, pte_w[1]);
2382
2383         if (*iova) {
2384                 *iova += page_size;
2385                 *offset += page_size;
2386                 if (*sgl && *offset + page_size > (*sgl)->length) {
2387                         u64 new_iova;
2388                         *sgl = sg_next(*sgl);
2389                         if (*sgl) {
2390                                 new_iova = sg_phys(*sgl);
2391                                 gk20a_dbg(gpu_dbg_pte, "chunk address %llx, size %d",
2392                                           new_iova, (*sgl)->length);
2393                                 if (new_iova) {
2394                                         *offset = 0;
2395                                         *iova = new_iova;
2396                                 }
2397                         }
2398                 }
2399         }
2400
2401         return 0;
2402 }
2403
2404 static int update_gmmu_level_locked(struct vm_gk20a *vm,
2405                                     struct gk20a_mm_entry *pte,
2406                                     enum gmmu_pgsz_gk20a pgsz_idx,
2407                                     struct scatterlist **sgl,
2408                                     u64 *offset,
2409                                     u64 *iova,
2410                                     u64 gpu_va, u64 gpu_end,
2411                                     u8 kind_v, u32 *ctag,
2412                                     bool cacheable, bool unmapped_pte,
2413                                     int rw_flag,
2414                                     bool sparse,
2415                                     int lvl,
2416                                     bool priv)
2417 {
2418         const struct gk20a_mmu_level *l = &vm->mmu_levels[lvl];
2419         const struct gk20a_mmu_level *next_l = &vm->mmu_levels[lvl+1];
2420         int err = 0;
2421         u32 pde_i;
2422         u64 pde_size = 1ULL << (u64)l->lo_bit[pgsz_idx];
2423
2424         gk20a_dbg_fn("");
2425
2426         pde_i = (gpu_va & ((1ULL << ((u64)l->hi_bit[pgsz_idx]+1)) - 1ULL))
2427                 >> (u64)l->lo_bit[pgsz_idx];
2428
2429         gk20a_dbg(gpu_dbg_pte, "size_idx=%d, l: %d, [%llx,%llx], iova=%llx",
2430                   pgsz_idx, lvl, gpu_va, gpu_end-1, *iova);
2431
2432         while (gpu_va < gpu_end) {
2433                 struct gk20a_mm_entry *next_pte = NULL;
2434                 u64 next = min((gpu_va + pde_size) & ~(pde_size-1), gpu_end);
2435
2436                 /* Allocate next level */
2437                 if (next_l->update_entry) {
2438                         if (!pte->entries) {
2439                                 int num_entries =
2440                                         1 <<
2441                                          (l->hi_bit[pgsz_idx]
2442                                           - l->lo_bit[pgsz_idx] + 1);
2443                                 pte->entries =
2444                                         vzalloc(sizeof(struct gk20a_mm_entry) *
2445                                                 num_entries);
2446                                 if (!pte->entries)
2447                                         return -ENOMEM;
2448                                 pte->pgsz = pgsz_idx;
2449                                 pte->num_entries = num_entries;
2450                         }
2451                         next_pte = pte->entries + pde_i;
2452
2453                         if (!next_pte->size) {
2454                                 err = gk20a_zalloc_gmmu_page_table(vm,
2455                                         pgsz_idx, next_l, next_pte);
2456                                 if (err)
2457                                         return err;
2458                         }
2459                 }
2460
2461                 err = l->update_entry(vm, pte, pde_i, pgsz_idx,
2462                                 sgl, offset, iova,
2463                                 kind_v, ctag, cacheable, unmapped_pte,
2464                                 rw_flag, sparse, priv);
2465                 if (err)
2466                         return err;
2467
2468                 if (next_l->update_entry) {
2469                         /* get cpu access to the ptes */
2470                         err = map_gmmu_pages(next_pte);
2471                         if (err) {
2472                                 gk20a_err(dev_from_vm(vm),
2473                                            "couldn't map ptes for update as=%d",
2474                                            vm_aspace_id(vm));
2475                                 return err;
2476                         }
2477                         err = update_gmmu_level_locked(vm, next_pte,
2478                                 pgsz_idx,
2479                                 sgl,
2480                                 offset,
2481                                 iova,
2482                                 gpu_va,
2483                                 next,
2484                                 kind_v, ctag, cacheable, unmapped_pte,
2485                                 rw_flag, sparse, lvl+1, priv);
2486                         unmap_gmmu_pages(next_pte);
2487
2488                         if (err)
2489                                 return err;
2490                 }
2491
2492                 pde_i++;
2493                 gpu_va = next;
2494         }
2495
2496         gk20a_dbg_fn("done");
2497
2498         return 0;
2499 }
2500
2501 static int update_gmmu_ptes_locked(struct vm_gk20a *vm,
2502                                    enum gmmu_pgsz_gk20a pgsz_idx,
2503                                    struct sg_table *sgt,
2504                                    u64 buffer_offset,
2505                                    u64 gpu_va, u64 gpu_end,
2506                                    u8 kind_v, u32 ctag_offset,
2507                                    bool cacheable, bool unmapped_pte,
2508                                    int rw_flag,
2509                                    bool sparse,
2510                                    bool priv)
2511 {
2512         struct gk20a *g = gk20a_from_vm(vm);
2513         int ctag_granularity = g->ops.fb.compression_page_size(g);
2514         u32 ctag = ctag_offset * ctag_granularity;
2515         u64 iova = 0;
2516         u64 space_to_skip = buffer_offset;
2517         u32 page_size  = vm->gmmu_page_sizes[pgsz_idx];
2518         int err;
2519         struct scatterlist *sgl = NULL;
2520
2521         gk20a_dbg(gpu_dbg_pte, "size_idx=%d, iova=%llx, buffer offset %lld, nents %d",
2522                    pgsz_idx,
2523                    sgt ? g->ops.mm.get_iova_addr(vm->mm->g, sgt->sgl, 0)
2524                        : 0ULL,
2525                    buffer_offset,
2526                    sgt ? sgt->nents : 0);
2527
2528         /* note: here we need to map kernel to small, since the
2529          * low-level mmu code assumes 0 is small and 1 is big pages */
2530         if (pgsz_idx == gmmu_page_size_kernel)
2531                 pgsz_idx = gmmu_page_size_small;
2532
2533         if (space_to_skip & (page_size - 1))
2534                 return -EINVAL;
2535
2536         if (sgt) {
2537                 iova = g->ops.mm.get_iova_addr(vm->mm->g, sgt->sgl, 0);
2538                 if (!vm->mm->bypass_smmu && iova) {
2539                         iova += space_to_skip;
2540                 } else {
2541                         sgl = sgt->sgl;
2542
2543                         gk20a_dbg(gpu_dbg_pte, "chunk address %llx, size %d",
2544                                         (u64)sg_phys(sgl),
2545                                         sgl->length);
2546                         while (space_to_skip && sgl &&
2547                                space_to_skip + page_size > sgl->length) {
2548                                 space_to_skip -= sgl->length;
2549                                 sgl = sg_next(sgl);
2550                                 gk20a_dbg(gpu_dbg_pte, "chunk address %llx, size %d",
2551                                                 (u64)sg_phys(sgl),
2552                                                 sgl->length);
2553                         }
2554                         iova = sg_phys(sgl) + space_to_skip;
2555                 }
2556         }
2557
2558         gk20a_dbg(gpu_dbg_map, "size_idx=%d, gpu_va=[%llx,%llx], iova=%llx",
2559                         pgsz_idx, gpu_va, gpu_end-1, iova);
2560         err = map_gmmu_pages(&vm->pdb);
2561         if (err) {
2562                 gk20a_err(dev_from_vm(vm),
2563                            "couldn't map ptes for update as=%d",
2564                            vm_aspace_id(vm));
2565                 return err;
2566         }
2567         err = update_gmmu_level_locked(vm, &vm->pdb, pgsz_idx,
2568                         &sgl,
2569                         &space_to_skip,
2570                         &iova,
2571                         gpu_va, gpu_end,
2572                         kind_v, &ctag,
2573                         cacheable, unmapped_pte, rw_flag, sparse, 0, priv);
2574         unmap_gmmu_pages(&vm->pdb);
2575
2576         smp_mb();
2577
2578         gk20a_dbg_fn("done");
2579
2580         return err;
2581 }
2582
2583 /* NOTE! mapped_buffers lock must be held */
2584 void gk20a_vm_unmap_locked(struct mapped_buffer_node *mapped_buffer,
2585                            struct vm_gk20a_mapping_batch *batch)
2586 {
2587         struct vm_gk20a *vm = mapped_buffer->vm;
2588         struct gk20a *g = vm->mm->g;
2589
2590         if (mapped_buffer->ctag_map_win_addr) {
2591                 /* unmap compbits */
2592
2593                 g->ops.mm.gmmu_unmap(vm,
2594                                      mapped_buffer->ctag_map_win_addr,
2595                                      mapped_buffer->ctag_map_win_size,
2596                                      0,       /* page size 4k */
2597                                      true,    /* va allocated */
2598                                      gk20a_mem_flag_none,
2599                                      false,   /* not sparse */
2600                                      batch);  /* batch handle */
2601         }
2602
2603         g->ops.mm.gmmu_unmap(vm,
2604                 mapped_buffer->addr,
2605                 mapped_buffer->size,
2606                 mapped_buffer->pgsz_idx,
2607                 mapped_buffer->va_allocated,
2608                 gk20a_mem_flag_none,
2609                 mapped_buffer->va_node ?
2610                   mapped_buffer->va_node->sparse : false,
2611                 batch);
2612
2613         gk20a_dbg(gpu_dbg_map, "as=%d pgsz=%d gv=0x%x,%08x own_mem_ref=%d",
2614                    vm_aspace_id(vm),
2615                    vm->gmmu_page_sizes[mapped_buffer->pgsz_idx],
2616                    hi32(mapped_buffer->addr), lo32(mapped_buffer->addr),
2617                    mapped_buffer->own_mem_ref);
2618
2619         gk20a_mm_unpin(dev_from_vm(vm), mapped_buffer->dmabuf,
2620                        mapped_buffer->sgt);
2621
2622         /* remove from mapped buffer tree and remove list, free */
2623         rb_erase(&mapped_buffer->node, &vm->mapped_buffers);
2624         if (!list_empty(&mapped_buffer->va_buffers_list))
2625                 list_del(&mapped_buffer->va_buffers_list);
2626
2627         /* keep track of mapped buffers */
2628         if (mapped_buffer->user_mapped)
2629                 vm->num_user_mapped_buffers--;
2630
2631         if (mapped_buffer->own_mem_ref)
2632                 dma_buf_put(mapped_buffer->dmabuf);
2633
2634         kfree(mapped_buffer);
2635
2636         return;
2637 }
2638
2639 void gk20a_vm_unmap(struct vm_gk20a *vm, u64 offset)
2640 {
2641         struct device *d = dev_from_vm(vm);
2642         struct mapped_buffer_node *mapped_buffer;
2643
2644         mutex_lock(&vm->update_gmmu_lock);
2645         mapped_buffer = find_mapped_buffer_locked(&vm->mapped_buffers, offset);
2646         if (!mapped_buffer) {
2647                 mutex_unlock(&vm->update_gmmu_lock);
2648                 gk20a_err(d, "invalid addr to unmap 0x%llx", offset);
2649                 return;
2650         }
2651
2652         kref_put(&mapped_buffer->ref, gk20a_vm_unmap_locked_kref);
2653         mutex_unlock(&vm->update_gmmu_lock);
2654 }
2655
2656 static void gk20a_vm_free_entries(struct vm_gk20a *vm,
2657                                   struct gk20a_mm_entry *parent,
2658                                   int level)
2659 {
2660         int i;
2661
2662         if (parent->entries)
2663                 for (i = 0; i < parent->num_entries; i++)
2664                         gk20a_vm_free_entries(vm, &parent->entries[i], level+1);
2665
2666         if (parent->size)
2667                 free_gmmu_pages(vm, parent);
2668         vfree(parent->entries);
2669         parent->entries = NULL;
2670 }
2671
2672 static void gk20a_vm_remove_support_nofree(struct vm_gk20a *vm)
2673 {
2674         struct mapped_buffer_node *mapped_buffer;
2675         struct vm_reserved_va_node *va_node, *va_node_tmp;
2676         struct rb_node *node;
2677
2678         gk20a_dbg_fn("");
2679         mutex_lock(&vm->update_gmmu_lock);
2680
2681         /* TBD: add a flag here for the unmap code to recognize teardown
2682          * and short-circuit any otherwise expensive operations. */
2683
2684         node = rb_first(&vm->mapped_buffers);
2685         while (node) {
2686                 mapped_buffer =
2687                         container_of(node, struct mapped_buffer_node, node);
2688                 gk20a_vm_unmap_locked(mapped_buffer, NULL);
2689                 node = rb_first(&vm->mapped_buffers);
2690         }
2691
2692         /* destroy remaining reserved memory areas */
2693         list_for_each_entry_safe(va_node, va_node_tmp, &vm->reserved_va_list,
2694                 reserved_va_list) {
2695                 list_del(&va_node->reserved_va_list);
2696                 kfree(va_node);
2697         }
2698
2699         gk20a_deinit_vm(vm);
2700
2701         mutex_unlock(&vm->update_gmmu_lock);
2702 }
2703
2704 void gk20a_vm_remove_support(struct vm_gk20a *vm)
2705 {
2706         gk20a_vm_remove_support_nofree(vm);
2707         /* vm is not used anymore. release it. */
2708         kfree(vm);
2709 }
2710
2711 static void gk20a_vm_remove_support_kref(struct kref *ref)
2712 {
2713         struct vm_gk20a *vm = container_of(ref, struct vm_gk20a, ref);
2714         struct gk20a *g = gk20a_from_vm(vm);
2715         g->ops.mm.vm_remove(vm);
2716 }
2717
2718 void gk20a_vm_get(struct vm_gk20a *vm)
2719 {
2720         kref_get(&vm->ref);
2721 }
2722
2723 void gk20a_vm_put(struct vm_gk20a *vm)
2724 {
2725         kref_put(&vm->ref, gk20a_vm_remove_support_kref);
2726 }
2727
2728 const struct gk20a_mmu_level gk20a_mm_levels_64k[] = {
2729         {.hi_bit = {NV_GMMU_VA_RANGE-1, NV_GMMU_VA_RANGE-1},
2730          .lo_bit = {26, 26},
2731          .update_entry = update_gmmu_pde_locked,
2732          .entry_size = 8},
2733         {.hi_bit = {25, 25},
2734          .lo_bit = {12, 16},
2735          .update_entry = update_gmmu_pte_locked,
2736          .entry_size = 8},
2737         {.update_entry = NULL}
2738 };
2739
2740 const struct gk20a_mmu_level gk20a_mm_levels_128k[] = {
2741         {.hi_bit = {NV_GMMU_VA_RANGE-1, NV_GMMU_VA_RANGE-1},
2742          .lo_bit = {27, 27},
2743          .update_entry = update_gmmu_pde_locked,
2744          .entry_size = 8},
2745         {.hi_bit = {26, 26},
2746          .lo_bit = {12, 17},
2747          .update_entry = update_gmmu_pte_locked,
2748          .entry_size = 8},
2749         {.update_entry = NULL}
2750 };
2751
2752 int gk20a_init_vm(struct mm_gk20a *mm,
2753                 struct vm_gk20a *vm,
2754                 u32 big_page_size,
2755                 u64 low_hole,
2756                 u64 kernel_reserved,
2757                 u64 aperture_size,
2758                 bool big_pages,
2759                 bool userspace_managed,
2760                 char *name)
2761 {
2762         int err, i;
2763         char alloc_name[32];
2764         u64 small_vma_start, small_vma_limit, large_vma_start, large_vma_limit,
2765                 kernel_vma_start, kernel_vma_limit;
2766         u32 pde_lo, pde_hi;
2767
2768         /* note: this must match gmmu_pgsz_gk20a enum */
2769         u32 gmmu_page_sizes[gmmu_nr_page_sizes] = { SZ_4K, big_page_size, SZ_4K };
2770
2771         WARN_ON(kernel_reserved + low_hole > aperture_size);
2772         if (kernel_reserved > aperture_size)
2773                 return -ENOMEM;
2774
2775         vm->mm = mm;
2776
2777         vm->va_start = low_hole;
2778         vm->va_limit = aperture_size;
2779         vm->big_pages = big_pages;
2780
2781         vm->big_page_size = gmmu_page_sizes[gmmu_page_size_big];
2782
2783         vm->userspace_managed = userspace_managed;
2784
2785         vm->mmu_levels = vm->mm->g->ops.mm.get_mmu_levels(vm->mm->g,
2786                         vm->big_page_size);
2787
2788         for (i = 0; i < gmmu_nr_page_sizes; i++)
2789                 vm->gmmu_page_sizes[i] = gmmu_page_sizes[i];
2790
2791         gk20a_dbg_info("small page-size (%dKB)",
2792                         vm->gmmu_page_sizes[gmmu_page_size_small] >> 10);
2793
2794         gk20a_dbg_info("big page-size (%dKB)",
2795                         vm->gmmu_page_sizes[gmmu_page_size_big] >> 10);
2796
2797         gk20a_dbg_info("kernel page-size (%dKB)",
2798                         vm->gmmu_page_sizes[gmmu_page_size_kernel] >> 10);
2799
2800         pde_range_from_vaddr_range(vm,
2801                                    0, vm->va_limit-1,
2802                                    &pde_lo, &pde_hi);
2803         vm->pdb.entries = vzalloc(sizeof(struct gk20a_mm_entry) *
2804                         (pde_hi + 1));
2805         vm->pdb.num_entries = pde_hi + 1;
2806
2807         if (!vm->pdb.entries)
2808                 return -ENOMEM;
2809
2810         gk20a_dbg_info("init space for %s va_limit=0x%llx num_pdes=%d",
2811                    name, vm->va_limit, pde_hi + 1);
2812
2813         /* allocate the page table directory */
2814         err = gk20a_zalloc_gmmu_page_table(vm, 0, &vm->mmu_levels[0], &vm->pdb);
2815         if (err)
2816                 goto clean_up_pdes;
2817
2818         /* setup vma limits */
2819         small_vma_start = low_hole;
2820
2821         if (big_pages) {
2822                 /* First 16GB of the address space goes towards small
2823                  * pages. What ever remains is allocated to large
2824                  * pages. */
2825                 small_vma_limit = __nv_gmmu_va_small_page_limit();
2826                 large_vma_start = small_vma_limit;
2827                 large_vma_limit = vm->va_limit - kernel_reserved;
2828         } else {
2829                 small_vma_limit = vm->va_limit - kernel_reserved;
2830                 large_vma_start = 0;
2831                 large_vma_limit = 0;
2832         }
2833
2834         kernel_vma_start = vm->va_limit - kernel_reserved;
2835         kernel_vma_limit = vm->va_limit;
2836
2837         gk20a_dbg_info(
2838                 "small_vma=[0x%llx,0x%llx) large_vma=[0x%llx,0x%llx) kernel_vma=[0x%llx,0x%llx)\n",
2839                 small_vma_start, small_vma_limit,
2840                 large_vma_start, large_vma_limit,
2841                 kernel_vma_start, kernel_vma_limit);
2842
2843         /* check that starts do not exceed limits */
2844         WARN_ON(small_vma_start > small_vma_limit);
2845         WARN_ON(large_vma_start > large_vma_limit);
2846         /* kernel_vma must also be non-zero */
2847         WARN_ON(kernel_vma_start >= kernel_vma_limit);
2848
2849         if (small_vma_start > small_vma_limit ||
2850             large_vma_start > large_vma_limit ||
2851             kernel_vma_start >= kernel_vma_limit) {
2852                 err = -EINVAL;
2853                 goto clean_up_pdes;
2854         }
2855
2856         if (small_vma_start < small_vma_limit) {
2857                 snprintf(alloc_name, sizeof(alloc_name), "gk20a_%s-%dKB", name,
2858                          vm->gmmu_page_sizes[gmmu_page_size_small] >> 10);
2859                 err = __gk20a_allocator_init(&vm->vma[gmmu_page_size_small],
2860                                              vm, alloc_name,
2861                                              small_vma_start,
2862                                              small_vma_limit - small_vma_start,
2863                                              SZ_4K,
2864                                              GPU_BALLOC_MAX_ORDER,
2865                                              GPU_BALLOC_GVA_SPACE);
2866                 if (err)
2867                         goto clean_up_ptes;
2868         }
2869
2870         if (large_vma_start < large_vma_limit) {
2871                 snprintf(alloc_name, sizeof(alloc_name), "gk20a_%s-%dKB",
2872                          name, vm->gmmu_page_sizes[gmmu_page_size_big] >> 10);
2873                 err = __gk20a_allocator_init(&vm->vma[gmmu_page_size_big],
2874                                              vm, alloc_name,
2875                                              large_vma_start,
2876                                              large_vma_limit - large_vma_start,
2877                                              big_page_size,
2878                                              GPU_BALLOC_MAX_ORDER,
2879                                              GPU_BALLOC_GVA_SPACE);
2880                 if (err)
2881                         goto clean_up_small_allocator;
2882         }
2883
2884         snprintf(alloc_name, sizeof(alloc_name), "gk20a_%s-%dKB-sys",
2885                  name, vm->gmmu_page_sizes[gmmu_page_size_kernel] >> 10);
2886         /*
2887          * kernel reserved VMA is at the end of the aperture
2888          */
2889         err = __gk20a_allocator_init(&vm->vma[gmmu_page_size_kernel],
2890                                      vm, alloc_name,
2891                                      kernel_vma_start,
2892                                      kernel_vma_limit - kernel_vma_start,
2893                                      SZ_4K,
2894                                      GPU_BALLOC_MAX_ORDER,
2895                                      GPU_BALLOC_GVA_SPACE);
2896         if (err)
2897                 goto clean_up_big_allocator;
2898
2899         vm->mapped_buffers = RB_ROOT;
2900
2901         mutex_init(&vm->update_gmmu_lock);
2902         kref_init(&vm->ref);
2903         INIT_LIST_HEAD(&vm->reserved_va_list);
2904
2905         return 0;
2906
2907 clean_up_big_allocator:
2908         if (large_vma_start < large_vma_limit)
2909                 gk20a_allocator_destroy(&vm->vma[gmmu_page_size_big]);
2910 clean_up_small_allocator:
2911         if (small_vma_start < small_vma_limit)
2912                 gk20a_allocator_destroy(&vm->vma[gmmu_page_size_small]);
2913 clean_up_ptes:
2914         free_gmmu_pages(vm, &vm->pdb);
2915 clean_up_pdes:
2916         vfree(vm->pdb.entries);
2917         return err;
2918 }
2919
2920 /* address space interfaces for the gk20a module */
2921 int gk20a_vm_alloc_share(struct gk20a_as_share *as_share, u32 big_page_size,
2922                          u32 flags)
2923 {
2924         struct gk20a_as *as = as_share->as;
2925         struct gk20a *g = gk20a_from_as(as);
2926         struct mm_gk20a *mm = &g->mm;
2927         struct vm_gk20a *vm;
2928         char name[32];
2929         int err;
2930         const bool userspace_managed =
2931                 (flags & NVGPU_GPU_IOCTL_ALLOC_AS_FLAGS_USERSPACE_MANAGED) != 0;
2932
2933         gk20a_dbg_fn("");
2934
2935         if (big_page_size == 0) {
2936                 big_page_size =
2937                         gk20a_get_platform(g->dev)->default_big_page_size;
2938         } else {
2939                 if (!is_power_of_2(big_page_size))
2940                         return -EINVAL;
2941
2942                 if (!(big_page_size & g->gpu_characteristics.available_big_page_sizes))
2943                         return -EINVAL;
2944         }
2945
2946         vm = kzalloc(sizeof(*vm), GFP_KERNEL);
2947         if (!vm)
2948                 return -ENOMEM;
2949
2950         as_share->vm = vm;
2951         vm->as_share = as_share;
2952         vm->enable_ctag = true;
2953
2954         snprintf(name, sizeof(name), "gk20a_as_%d", as_share->id);
2955
2956         err = gk20a_init_vm(mm, vm, big_page_size, big_page_size << 10,
2957                             mm->channel.kernel_size,
2958                             mm->channel.user_size + mm->channel.kernel_size,
2959                             !mm->disable_bigpage, userspace_managed, name);
2960
2961         return err;
2962 }
2963
2964 int gk20a_vm_release_share(struct gk20a_as_share *as_share)
2965 {
2966         struct vm_gk20a *vm = as_share->vm;
2967
2968         gk20a_dbg_fn("");
2969
2970         vm->as_share = NULL;
2971
2972         /* put as reference to vm */
2973         gk20a_vm_put(vm);
2974
2975         as_share->vm = NULL;
2976
2977         return 0;
2978 }
2979
2980
2981 int gk20a_vm_alloc_space(struct gk20a_as_share *as_share,
2982                          struct nvgpu_as_alloc_space_args *args)
2983
2984 {
2985         int err = -ENOMEM;
2986         int pgsz_idx = gmmu_page_size_small;
2987         struct gk20a_allocator *vma;
2988         struct vm_gk20a *vm = as_share->vm;
2989         struct gk20a *g = vm->mm->g;
2990         struct vm_reserved_va_node *va_node;
2991         u64 vaddr_start = 0;
2992         int page_sizes = gmmu_nr_page_sizes;
2993
2994         gk20a_dbg_fn("flags=0x%x pgsz=0x%x nr_pages=0x%x o/a=0x%llx",
2995                         args->flags, args->page_size, args->pages,
2996                         args->o_a.offset);
2997
2998         if (!vm->big_pages)
2999                 page_sizes--;
3000
3001         for (; pgsz_idx < page_sizes; pgsz_idx++) {
3002                 if (vm->gmmu_page_sizes[pgsz_idx] == args->page_size)
3003                         break;
3004         }
3005
3006         if (pgsz_idx >= page_sizes) {
3007                 err = -EINVAL;
3008                 goto clean_up;
3009         }
3010
3011         va_node = kzalloc(sizeof(*va_node), GFP_KERNEL);
3012         if (!va_node) {
3013                 err = -ENOMEM;
3014                 goto clean_up;
3015         }
3016
3017         vma = &vm->vma[pgsz_idx];
3018         if (args->flags & NVGPU_AS_ALLOC_SPACE_FLAGS_FIXED_OFFSET)
3019                 vaddr_start = gk20a_balloc_fixed(vma, args->o_a.offset,
3020                                                  (u64)args->pages *
3021                                                  (u64)args->page_size);
3022         else
3023                 vaddr_start = gk20a_balloc(vma,
3024                                            (u64)args->pages *
3025                                            (u64)args->page_size);
3026
3027         if (!vaddr_start) {
3028                 kfree(va_node);
3029                 goto clean_up;
3030         }
3031
3032         va_node->vaddr_start = vaddr_start;
3033         va_node->size = (u64)args->page_size * (u64)args->pages;
3034         va_node->pgsz_idx = pgsz_idx;
3035         INIT_LIST_HEAD(&va_node->va_buffers_list);
3036         INIT_LIST_HEAD(&va_node->reserved_va_list);
3037
3038         mutex_lock(&vm->update_gmmu_lock);
3039
3040         /* mark that we need to use sparse mappings here */
3041         if (args->flags & NVGPU_AS_ALLOC_SPACE_FLAGS_SPARSE) {
3042                 u64 map_offset = g->ops.mm.gmmu_map(vm, vaddr_start,
3043                                          NULL,
3044                                          0,
3045                                          va_node->size,
3046                                          pgsz_idx,
3047                                          0,
3048                                          0,
3049                                          args->flags,
3050                                          gk20a_mem_flag_none,
3051                                          false,
3052                                          true,
3053                                          false,
3054                                          NULL);
3055                 if (!map_offset) {
3056                         mutex_unlock(&vm->update_gmmu_lock);
3057                         gk20a_bfree(vma, vaddr_start);
3058                         kfree(va_node);
3059                         goto clean_up;
3060                 }
3061
3062                 va_node->sparse = true;
3063         }
3064         list_add_tail(&va_node->reserved_va_list, &vm->reserved_va_list);
3065
3066         mutex_unlock(&vm->update_gmmu_lock);
3067
3068         args->o_a.offset = vaddr_start;
3069         err = 0;
3070
3071 clean_up:
3072         return err;
3073 }
3074
3075 int gk20a_vm_free_space(struct gk20a_as_share *as_share,
3076                         struct nvgpu_as_free_space_args *args)
3077 {
3078         int err = -ENOMEM;
3079         int pgsz_idx;
3080         struct gk20a_allocator *vma;
3081         struct vm_gk20a *vm = as_share->vm;
3082         struct vm_reserved_va_node *va_node;
3083         struct gk20a *g = gk20a_from_vm(vm);
3084
3085         gk20a_dbg_fn("pgsz=0x%x nr_pages=0x%x o/a=0x%llx", args->page_size,
3086                         args->pages, args->offset);
3087
3088         /* determine pagesz idx */
3089         pgsz_idx = __nv_gmmu_va_is_big_page_region(vm, args->offset) ?
3090                         gmmu_page_size_big : gmmu_page_size_small;
3091
3092         vma = &vm->vma[pgsz_idx];
3093         gk20a_bfree(vma, args->offset);
3094
3095         mutex_lock(&vm->update_gmmu_lock);
3096         va_node = addr_to_reservation(vm, args->offset);
3097         if (va_node) {
3098                 struct mapped_buffer_node *buffer, *n;
3099
3100                 /* Decrement the ref count on all buffers in this va_node. This
3101                  * allows userspace to let the kernel free mappings that are
3102                  * only used by this va_node. */
3103                 list_for_each_entry_safe(buffer, n,
3104                         &va_node->va_buffers_list, va_buffers_list) {
3105                         list_del_init(&buffer->va_buffers_list);
3106                         kref_put(&buffer->ref, gk20a_vm_unmap_locked_kref);
3107                 }
3108
3109                 list_del(&va_node->reserved_va_list);
3110
3111                 /* if this was a sparse mapping, free the va */
3112                 if (va_node->sparse)
3113                         g->ops.mm.gmmu_unmap(vm,
3114                                         va_node->vaddr_start,
3115                                         va_node->size,
3116                                         va_node->pgsz_idx,
3117                                         true,
3118                                         gk20a_mem_flag_none,
3119                                         true,
3120                                         NULL);
3121                 kfree(va_node);
3122         }
3123         mutex_unlock(&vm->update_gmmu_lock);
3124         err = 0;
3125
3126         return err;
3127 }
3128
3129 int __gk20a_vm_bind_channel(struct vm_gk20a *vm, struct channel_gk20a *ch)
3130 {
3131         int err = 0;
3132
3133         gk20a_dbg_fn("");
3134
3135         gk20a_vm_get(vm);
3136         ch->vm = vm;
3137         err = channel_gk20a_commit_va(ch);
3138         if (err)
3139                 ch->vm = NULL;
3140
3141         return err;
3142 }
3143
3144 int gk20a_vm_bind_channel(struct gk20a_as_share *as_share,
3145                           struct channel_gk20a *ch)
3146 {
3147         return __gk20a_vm_bind_channel(as_share->vm, ch);
3148 }
3149
3150 int gk20a_dmabuf_alloc_drvdata(struct dma_buf *dmabuf, struct device *dev)
3151 {
3152         struct gk20a_dmabuf_priv *priv;
3153         static DEFINE_MUTEX(priv_lock);
3154
3155         priv = dma_buf_get_drvdata(dmabuf, dev);
3156         if (likely(priv))
3157                 return 0;
3158
3159         mutex_lock(&priv_lock);
3160         priv = dma_buf_get_drvdata(dmabuf, dev);
3161         if (priv)
3162                 goto priv_exist_or_err;
3163         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
3164         if (!priv) {
3165                 priv = ERR_PTR(-ENOMEM);
3166                 goto priv_exist_or_err;
3167         }
3168         mutex_init(&priv->lock);
3169         INIT_LIST_HEAD(&priv->states);
3170         dma_buf_set_drvdata(dmabuf, dev, priv, gk20a_mm_delete_priv);
3171 priv_exist_or_err:
3172         mutex_unlock(&priv_lock);
3173         if (IS_ERR(priv))
3174                 return -ENOMEM;
3175
3176         return 0;
3177 }
3178
3179 int gk20a_dmabuf_get_state(struct dma_buf *dmabuf, struct device *dev,
3180                            u64 offset, struct gk20a_buffer_state **state)
3181 {
3182         int err = 0;
3183         struct gk20a_dmabuf_priv *priv;
3184         struct gk20a_buffer_state *s;
3185
3186         if (WARN_ON(offset >= (u64)dmabuf->size))
3187                 return -EINVAL;
3188
3189         err = gk20a_dmabuf_alloc_drvdata(dmabuf, dev);
3190         if (err)
3191                 return err;
3192
3193         priv = dma_buf_get_drvdata(dmabuf, dev);
3194         if (WARN_ON(!priv))
3195                 return -ENOSYS;
3196
3197         mutex_lock(&priv->lock);
3198
3199         list_for_each_entry(s, &priv->states, list)
3200                 if (s->offset == offset)
3201                         goto out;
3202
3203         /* State not found, create state. */
3204         s = kzalloc(sizeof(*s), GFP_KERNEL);
3205         if (!s) {
3206                 err = -ENOMEM;
3207                 goto out;
3208         }
3209
3210         s->offset = offset;
3211         INIT_LIST_HEAD(&s->list);
3212         mutex_init(&s->lock);
3213         list_add_tail(&s->list, &priv->states);
3214
3215 out:
3216         mutex_unlock(&priv->lock);
3217         if (!err)
3218                 *state = s;
3219         return err;
3220
3221
3222 }
3223
3224 static int gk20a_dmabuf_get_kind(struct dma_buf *dmabuf)
3225 {
3226         int kind = 0;
3227 #ifdef CONFIG_TEGRA_NVMAP
3228         int err;
3229         u64 nvmap_param;
3230
3231         err = nvmap_get_dmabuf_param(dmabuf, NVMAP_HANDLE_PARAM_KIND,
3232                                      &nvmap_param);
3233         kind = err ? kind : nvmap_param;
3234 #endif
3235         return kind;
3236 }
3237
3238 int gk20a_vm_map_buffer(struct vm_gk20a *vm,
3239                         int dmabuf_fd,
3240                         u64 *offset_align,
3241                         u32 flags, /*NVGPU_AS_MAP_BUFFER_FLAGS_*/
3242                         int kind,
3243                         u64 buffer_offset,
3244                         u64 mapping_size,
3245                         struct vm_gk20a_mapping_batch *batch)
3246 {
3247         int err = 0;
3248         struct dma_buf *dmabuf;
3249         u64 ret_va;
3250
3251         gk20a_dbg_fn("");
3252
3253         /* get ref to the mem handle (released on unmap_locked) */
3254         dmabuf = dma_buf_get(dmabuf_fd);
3255         if (IS_ERR(dmabuf))
3256                 return PTR_ERR(dmabuf);
3257
3258         err = gk20a_dmabuf_alloc_drvdata(dmabuf, dev_from_vm(vm));
3259         if (err) {
3260                 dma_buf_put(dmabuf);
3261                 return err;
3262         }
3263
3264         if (kind == -1)
3265                 kind = gk20a_dmabuf_get_kind(dmabuf);
3266
3267         ret_va = gk20a_vm_map(vm, dmabuf, *offset_align,
3268                         flags, kind, NULL, true,
3269                         gk20a_mem_flag_none,
3270                         buffer_offset,
3271                         mapping_size,
3272                         batch);
3273
3274         *offset_align = ret_va;
3275         if (!ret_va) {
3276                 dma_buf_put(dmabuf);
3277                 err = -EINVAL;
3278         }
3279
3280         return err;
3281 }
3282
3283 int gk20a_vm_unmap_buffer(struct vm_gk20a *vm, u64 offset,
3284                           struct vm_gk20a_mapping_batch *batch)
3285 {
3286         gk20a_dbg_fn("");
3287
3288         gk20a_vm_unmap_user(vm, offset, batch);
3289         return 0;
3290 }
3291
3292 void gk20a_deinit_vm(struct vm_gk20a *vm)
3293 {
3294         gk20a_allocator_destroy(&vm->vma[gmmu_page_size_kernel]);
3295         if (vm->vma[gmmu_page_size_big].init)
3296                 gk20a_allocator_destroy(&vm->vma[gmmu_page_size_big]);
3297         if (vm->vma[gmmu_page_size_small].init)
3298                 gk20a_allocator_destroy(&vm->vma[gmmu_page_size_small]);
3299
3300         gk20a_vm_free_entries(vm, &vm->pdb, 0);
3301 }
3302
3303 int gk20a_alloc_inst_block(struct gk20a *g, struct mem_desc *inst_block)
3304 {
3305         struct device *dev = dev_from_gk20a(g);
3306         int err;
3307
3308         gk20a_dbg_fn("");
3309
3310         err = gk20a_gmmu_alloc(g, ram_in_alloc_size_v(), inst_block);
3311         if (err) {
3312                 gk20a_err(dev, "%s: memory allocation failed\n", __func__);
3313                 return err;
3314         }
3315
3316         gk20a_dbg_fn("done");
3317         return 0;
3318 }
3319
3320 void gk20a_free_inst_block(struct gk20a *g, struct mem_desc *inst_block)
3321 {
3322         if (inst_block->cpu_va)
3323                 gk20a_gmmu_free(g, inst_block);
3324 }
3325
3326 static int gk20a_init_bar1_vm(struct mm_gk20a *mm)
3327 {
3328         int err;
3329         struct vm_gk20a *vm = &mm->bar1.vm;
3330         struct gk20a *g = gk20a_from_mm(mm);
3331         struct mem_desc *inst_block = &mm->bar1.inst_block;
3332         u32 big_page_size = gk20a_get_platform(g->dev)->default_big_page_size;
3333
3334         mm->bar1.aperture_size = bar1_aperture_size_mb_gk20a() << 20;
3335         gk20a_dbg_info("bar1 vm size = 0x%x", mm->bar1.aperture_size);
3336         gk20a_init_vm(mm, vm, big_page_size, SZ_4K,
3337                       mm->bar1.aperture_size - SZ_4K,
3338                       mm->bar1.aperture_size, false, false, "bar1");
3339
3340         err = gk20a_alloc_inst_block(g, inst_block);
3341         if (err)
3342                 goto clean_up_va;
3343         gk20a_init_inst_block(inst_block, vm, big_page_size);
3344
3345         return 0;
3346
3347 clean_up_va:
3348         gk20a_deinit_vm(vm);
3349         return err;
3350 }
3351
3352 /* pmu vm, share channel_vm interfaces */
3353 static int gk20a_init_system_vm(struct mm_gk20a *mm)
3354 {
3355         int err;
3356         struct vm_gk20a *vm = &mm->pmu.vm;
3357         struct gk20a *g = gk20a_from_mm(mm);
3358         struct mem_desc *inst_block = &mm->pmu.inst_block;
3359         u32 big_page_size = gk20a_get_platform(g->dev)->default_big_page_size;
3360
3361         mm->pmu.aperture_size = GK20A_PMU_VA_SIZE;
3362         gk20a_dbg_info("pmu vm size = 0x%x", mm->pmu.aperture_size);
3363
3364         gk20a_init_vm(mm, vm, big_page_size,
3365                       SZ_4K * 16, GK20A_PMU_VA_SIZE,
3366                       GK20A_PMU_VA_SIZE * 2, false, false,
3367                       "system");
3368
3369         err = gk20a_alloc_inst_block(g, inst_block);
3370         if (err)
3371                 goto clean_up_va;
3372         gk20a_init_inst_block(inst_block, vm, big_page_size);
3373
3374         return 0;
3375
3376 clean_up_va:
3377         gk20a_deinit_vm(vm);
3378         return err;
3379 }
3380
3381 static int gk20a_init_hwpm(struct mm_gk20a *mm)
3382 {
3383         int err;
3384         struct vm_gk20a *vm = &mm->pmu.vm;
3385         struct gk20a *g = gk20a_from_mm(mm);
3386         struct mem_desc *inst_block = &mm->hwpm.inst_block;
3387
3388         err = gk20a_alloc_inst_block(g, inst_block);
3389         if (err)
3390                 return err;
3391         gk20a_init_inst_block(inst_block, vm, 0);
3392
3393         return 0;
3394 }
3395
3396 void gk20a_mm_init_pdb(struct gk20a *g, void *inst_ptr, u64 pdb_addr)
3397 {
3398         u32 pdb_addr_lo = u64_lo32(pdb_addr >> ram_in_base_shift_v());
3399         u32 pdb_addr_hi = u64_hi32(pdb_addr);
3400
3401         gk20a_mem_wr32(inst_ptr, ram_in_page_dir_base_lo_w(),
3402                 ram_in_page_dir_base_target_vid_mem_f() |
3403                 ram_in_page_dir_base_vol_true_f() |
3404                 ram_in_page_dir_base_lo_f(pdb_addr_lo));
3405
3406         gk20a_mem_wr32(inst_ptr, ram_in_page_dir_base_hi_w(),
3407                 ram_in_page_dir_base_hi_f(pdb_addr_hi));
3408 }
3409
3410 void gk20a_init_inst_block(struct mem_desc *inst_block, struct vm_gk20a *vm,
3411                 u32 big_page_size)
3412 {
3413         struct gk20a *g = gk20a_from_vm(vm);
3414         u64 pde_addr = g->ops.mm.get_iova_addr(g, vm->pdb.sgt->sgl, 0);
3415         phys_addr_t inst_pa = gk20a_mem_phys(inst_block);
3416         void *inst_ptr = inst_block->cpu_va;
3417
3418         gk20a_dbg_info("inst block phys = 0x%llx, kv = 0x%p",
3419                 (u64)inst_pa, inst_ptr);
3420
3421         gk20a_dbg_info("pde pa=0x%llx", (u64)pde_addr);
3422
3423         g->ops.mm.init_pdb(g, inst_ptr, pde_addr);
3424
3425         gk20a_mem_wr32(inst_ptr, ram_in_adr_limit_lo_w(),
3426                 u64_lo32(vm->va_limit - 1) & ~0xfff);
3427
3428         gk20a_mem_wr32(inst_ptr, ram_in_adr_limit_hi_w(),
3429                 ram_in_adr_limit_hi_f(u64_hi32(vm->va_limit - 1)));
3430
3431         if (big_page_size && g->ops.mm.set_big_page_size)
3432                 g->ops.mm.set_big_page_size(g, inst_ptr, big_page_size);
3433 }
3434
3435 int gk20a_mm_fb_flush(struct gk20a *g)
3436 {
3437         struct mm_gk20a *mm = &g->mm;
3438         u32 data;
3439         s32 retry = 100;
3440         int ret = 0;
3441
3442         gk20a_dbg_fn("");
3443
3444         gk20a_busy_noresume(g->dev);
3445         if (!g->power_on) {
3446                 pm_runtime_put_noidle(&g->dev->dev);
3447                 return 0;
3448         }
3449
3450         mutex_lock(&mm->l2_op_lock);
3451
3452         /* Make sure all previous writes are committed to the L2. There's no
3453            guarantee that writes are to DRAM. This will be a sysmembar internal
3454            to the L2. */
3455
3456         trace_gk20a_mm_fb_flush(g->dev->name);
3457
3458         gk20a_writel(g, flush_fb_flush_r(),
3459                 flush_fb_flush_pending_busy_f());
3460
3461         do {
3462                 data = gk20a_readl(g, flush_fb_flush_r());
3463
3464                 if (flush_fb_flush_outstanding_v(data) ==
3465                         flush_fb_flush_outstanding_true_v() ||
3466                     flush_fb_flush_pending_v(data) ==
3467                         flush_fb_flush_pending_busy_v()) {
3468                                 gk20a_dbg_info("fb_flush 0x%x", data);
3469                                 retry--;
3470                                 udelay(5);
3471                 } else
3472                         break;
3473         } while (retry >= 0 || !tegra_platform_is_silicon());
3474
3475         if (tegra_platform_is_silicon() && retry < 0) {
3476                 gk20a_warn(dev_from_gk20a(g),
3477                         "fb_flush too many retries");
3478                 if (g->ops.fb.dump_vpr_wpr_info)
3479                         g->ops.fb.dump_vpr_wpr_info(g);
3480                 ret = -EBUSY;
3481         }
3482
3483         trace_gk20a_mm_fb_flush_done(g->dev->name);
3484
3485         mutex_unlock(&mm->l2_op_lock);
3486
3487         pm_runtime_put_noidle(&g->dev->dev);
3488
3489         return ret;
3490 }
3491
3492 static void gk20a_mm_l2_invalidate_locked(struct gk20a *g)
3493 {
3494         u32 data;
3495         s32 retry = 200;
3496
3497         trace_gk20a_mm_l2_invalidate(g->dev->name);
3498
3499         /* Invalidate any clean lines from the L2 so subsequent reads go to
3500            DRAM. Dirty lines are not affected by this operation. */
3501         gk20a_writel(g, flush_l2_system_invalidate_r(),
3502                 flush_l2_system_invalidate_pending_busy_f());
3503
3504         do {
3505                 data = gk20a_readl(g, flush_l2_system_invalidate_r());
3506
3507                 if (flush_l2_system_invalidate_outstanding_v(data) ==
3508                         flush_l2_system_invalidate_outstanding_true_v() ||
3509                     flush_l2_system_invalidate_pending_v(data) ==
3510                         flush_l2_system_invalidate_pending_busy_v()) {
3511                                 gk20a_dbg_info("l2_system_invalidate 0x%x",
3512                                                 data);
3513                                 retry--;
3514                                 udelay(5);
3515                 } else
3516                         break;
3517         } while (retry >= 0 || !tegra_platform_is_silicon());
3518
3519         if (tegra_platform_is_silicon() && retry < 0)
3520                 gk20a_warn(dev_from_gk20a(g),
3521                         "l2_system_invalidate too many retries");
3522
3523         trace_gk20a_mm_l2_invalidate_done(g->dev->name);
3524 }
3525
3526 void gk20a_mm_l2_invalidate(struct gk20a *g)
3527 {
3528         struct mm_gk20a *mm = &g->mm;
3529         gk20a_busy_noresume(g->dev);
3530         if (g->power_on) {
3531                 mutex_lock(&mm->l2_op_lock);
3532                 gk20a_mm_l2_invalidate_locked(g);
3533                 mutex_unlock(&mm->l2_op_lock);
3534         }
3535         pm_runtime_put_noidle(&g->dev->dev);
3536 }
3537
3538 void gk20a_mm_l2_flush(struct gk20a *g, bool invalidate)
3539 {
3540         struct mm_gk20a *mm = &g->mm;
3541         u32 data;
3542         s32 retry = 200;
3543
3544         gk20a_dbg_fn("");
3545
3546         gk20a_busy_noresume(g->dev);
3547         if (!g->power_on)
3548                 goto hw_was_off;
3549
3550         mutex_lock(&mm->l2_op_lock);
3551
3552         trace_gk20a_mm_l2_flush(g->dev->name);
3553
3554         /* Flush all dirty lines from the L2 to DRAM. Lines are left in the L2
3555            as clean, so subsequent reads might hit in the L2. */
3556         gk20a_writel(g, flush_l2_flush_dirty_r(),
3557                 flush_l2_flush_dirty_pending_busy_f());
3558
3559         do {
3560                 data = gk20a_readl(g, flush_l2_flush_dirty_r());
3561
3562                 if (flush_l2_flush_dirty_outstanding_v(data) ==
3563                         flush_l2_flush_dirty_outstanding_true_v() ||
3564                     flush_l2_flush_dirty_pending_v(data) ==
3565                         flush_l2_flush_dirty_pending_busy_v()) {
3566                                 gk20a_dbg_info("l2_flush_dirty 0x%x", data);
3567                                 retry--;
3568                                 udelay(5);
3569                 } else
3570                         break;
3571         } while (retry >= 0 || !tegra_platform_is_silicon());
3572
3573         if (tegra_platform_is_silicon() && retry < 0)
3574                 gk20a_warn(dev_from_gk20a(g),
3575                         "l2_flush_dirty too many retries");
3576
3577         trace_gk20a_mm_l2_flush_done(g->dev->name);
3578
3579         if (invalidate)
3580                 gk20a_mm_l2_invalidate_locked(g);
3581
3582         mutex_unlock(&mm->l2_op_lock);
3583
3584 hw_was_off:
3585         pm_runtime_put_noidle(&g->dev->dev);
3586 }
3587
3588
3589 int gk20a_vm_find_buffer(struct vm_gk20a *vm, u64 gpu_va,
3590                          struct dma_buf **dmabuf,
3591                          u64 *offset)
3592 {
3593         struct mapped_buffer_node *mapped_buffer;
3594
3595         gk20a_dbg_fn("gpu_va=0x%llx", gpu_va);
3596
3597         mutex_lock(&vm->update_gmmu_lock);
3598
3599         mapped_buffer = find_mapped_buffer_range_locked(&vm->mapped_buffers,
3600                                                         gpu_va);
3601         if (!mapped_buffer) {
3602                 mutex_unlock(&vm->update_gmmu_lock);
3603                 return -EINVAL;
3604         }
3605
3606         *dmabuf = mapped_buffer->dmabuf;
3607         *offset = gpu_va - mapped_buffer->addr;
3608
3609         mutex_unlock(&vm->update_gmmu_lock);
3610
3611         return 0;
3612 }
3613
3614 void gk20a_mm_tlb_invalidate(struct vm_gk20a *vm)
3615 {
3616         struct gk20a *g = gk20a_from_vm(vm);
3617         u32 addr_lo = u64_lo32(g->ops.mm.get_iova_addr(vm->mm->g,
3618                                                   vm->pdb.sgt->sgl, 0) >> 12);
3619         u32 data;
3620         s32 retry = 2000;
3621         static DEFINE_MUTEX(tlb_lock);
3622
3623         gk20a_dbg_fn("");
3624
3625         /* pagetables are considered sw states which are preserved after
3626            prepare_poweroff. When gk20a deinit releases those pagetables,
3627            common code in vm unmap path calls tlb invalidate that touches
3628            hw. Use the power_on flag to skip tlb invalidation when gpu
3629            power is turned off */
3630
3631         if (!g->power_on)
3632                 return;
3633
3634         mutex_lock(&tlb_lock);
3635
3636         trace_gk20a_mm_tlb_invalidate(g->dev->name);
3637
3638         do {
3639                 data = gk20a_readl(g, fb_mmu_ctrl_r());
3640                 if (fb_mmu_ctrl_pri_fifo_space_v(data) != 0)
3641                         break;
3642                 udelay(2);
3643                 retry--;
3644         } while (retry >= 0 || !tegra_platform_is_silicon());
3645
3646         if (tegra_platform_is_silicon() && retry < 0) {
3647                 gk20a_warn(dev_from_gk20a(g),
3648                         "wait mmu fifo space too many retries");
3649                 goto out;
3650         }
3651
3652         gk20a_writel(g, fb_mmu_invalidate_pdb_r(),
3653                 fb_mmu_invalidate_pdb_addr_f(addr_lo) |
3654                 fb_mmu_invalidate_pdb_aperture_vid_mem_f());
3655
3656         gk20a_writel(g, fb_mmu_invalidate_r(),
3657                 fb_mmu_invalidate_all_va_true_f() |
3658                 fb_mmu_invalidate_trigger_true_f());
3659
3660         do {
3661                 data = gk20a_readl(g, fb_mmu_ctrl_r());
3662                 if (fb_mmu_ctrl_pri_fifo_empty_v(data) !=
3663                         fb_mmu_ctrl_pri_fifo_empty_false_f())
3664                         break;
3665                 retry--;
3666                 udelay(2);
3667         } while (retry >= 0 || !tegra_platform_is_silicon());
3668
3669         if (tegra_platform_is_silicon() && retry < 0)
3670                 gk20a_warn(dev_from_gk20a(g),
3671                         "mmu invalidate too many retries");
3672
3673         trace_gk20a_mm_tlb_invalidate_done(g->dev->name);
3674
3675 out:
3676         mutex_unlock(&tlb_lock);
3677 }
3678
3679 int gk20a_mm_suspend(struct gk20a *g)
3680 {
3681         gk20a_dbg_fn("");
3682
3683         g->ops.ltc.elpg_flush(g);
3684
3685         gk20a_dbg_fn("done");
3686         return 0;
3687 }
3688
3689 bool gk20a_mm_mmu_debug_mode_enabled(struct gk20a *g)
3690 {
3691         u32 debug_ctrl = gk20a_readl(g, fb_mmu_debug_ctrl_r());
3692         return fb_mmu_debug_ctrl_debug_v(debug_ctrl) ==
3693                 fb_mmu_debug_ctrl_debug_enabled_v();
3694 }
3695
3696 u32 gk20a_mm_get_physical_addr_bits(struct gk20a *g)
3697 {
3698         return 34;
3699 }
3700
3701 const struct gk20a_mmu_level *gk20a_mm_get_mmu_levels(struct gk20a *g,
3702                                                       u32 big_page_size)
3703 {
3704         return (big_page_size == SZ_64K) ?
3705                  gk20a_mm_levels_64k : gk20a_mm_levels_128k;
3706 }
3707
3708 void gk20a_init_mm(struct gpu_ops *gops)
3709 {
3710         gops->mm.is_debug_mode_enabled = gk20a_mm_mmu_debug_mode_enabled;
3711         gops->mm.gmmu_map = gk20a_locked_gmmu_map;
3712         gops->mm.gmmu_unmap = gk20a_locked_gmmu_unmap;
3713         gops->mm.vm_remove = gk20a_vm_remove_support;
3714         gops->mm.vm_alloc_share = gk20a_vm_alloc_share;
3715         gops->mm.vm_bind_channel = gk20a_vm_bind_channel;
3716         gops->mm.fb_flush = gk20a_mm_fb_flush;
3717         gops->mm.l2_invalidate = gk20a_mm_l2_invalidate;
3718         gops->mm.l2_flush = gk20a_mm_l2_flush;
3719         gops->mm.tlb_invalidate = gk20a_mm_tlb_invalidate;
3720         gops->mm.get_iova_addr = gk20a_mm_iova_addr;
3721         gops->mm.get_physical_addr_bits = gk20a_mm_get_physical_addr_bits;
3722         gops->mm.get_mmu_levels = gk20a_mm_get_mmu_levels;
3723         gops->mm.init_pdb = gk20a_mm_init_pdb;
3724         gops->mm.init_mm_setup_hw = gk20a_init_mm_setup_hw;
3725 }