video: tegra: nvmap: support for carveout resize
[linux-3.10.git] / drivers / video / tegra / nvmap / nvmap_dev.c
1 /*
2  * drivers/video/tegra/nvmap/nvmap_dev.c
3  *
4  * User-space interface to nvmap
5  *
6  * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/backing-dev.h>
24 #include <linux/bitmap.h>
25 #include <linux/debugfs.h>
26 #include <linux/delay.h>
27 #include <linux/kernel.h>
28 #include <linux/device.h>
29 #include <linux/miscdevice.h>
30 #include <linux/mm.h>
31 #include <linux/oom.h>
32 #include <linux/platform_device.h>
33 #include <linux/seq_file.h>
34 #include <linux/slab.h>
35 #include <linux/spinlock.h>
36 #include <linux/uaccess.h>
37 #include <linux/vmalloc.h>
38 #include <linux/nvmap.h>
39 #include <linux/module.h>
40 #include <linux/resource.h>
41 #include <linux/security.h>
42 #include <linux/stat.h>
43
44 #include <asm/cputype.h>
45 #include <asm/cacheflush.h>
46 #include <asm/tlbflush.h>
47
48 #define CREATE_TRACE_POINTS
49 #include <trace/events/nvmap.h>
50
51 #include "nvmap_priv.h"
52 #include "nvmap_ioctl.h"
53
54 #define NVMAP_NUM_PTES          64
55 #define NVMAP_CARVEOUT_KILLER_RETRY_TIME 100 /* msecs */
56
57 #ifdef CONFIG_NVMAP_CACHE_MAINT_BY_SET_WAYS
58 size_t cache_maint_inner_threshold = SZ_2M;
59 #endif
60 #ifdef CONFIG_NVMAP_OUTER_CACHE_MAINT_BY_SET_WAYS
61 size_t cache_maint_outer_threshold = SZ_1M;
62 #endif
63
64 struct nvmap_carveout_node {
65         unsigned int            heap_bit;
66         struct nvmap_heap       *carveout;
67         int                     index;
68         struct list_head        clients;
69         spinlock_t              clients_lock;
70         phys_addr_t                     base;
71         size_t                  size;
72 };
73
74 struct nvmap_device {
75         struct vm_struct *vm_rgn;
76         pte_t           *ptes[NVMAP_NUM_PTES];
77         unsigned long   ptebits[NVMAP_NUM_PTES / BITS_PER_LONG];
78         unsigned int    lastpte;
79         spinlock_t      ptelock;
80
81         struct rb_root  handles;
82         spinlock_t      handle_lock;
83         wait_queue_head_t pte_wait;
84         struct miscdevice dev_super;
85         struct miscdevice dev_user;
86         struct nvmap_carveout_node *heaps;
87         int nr_carveouts;
88         struct nvmap_share iovmm_master;
89         struct list_head clients;
90         spinlock_t      clients_lock;
91         struct nvmap_deferred_ops deferred_ops;
92 };
93
94 struct platform_device *nvmap_pdev;
95 EXPORT_SYMBOL(nvmap_pdev);
96 struct nvmap_device *nvmap_dev;
97 EXPORT_SYMBOL(nvmap_dev);
98 struct nvmap_share *nvmap_share;
99 EXPORT_SYMBOL(nvmap_share);
100
101 static struct backing_dev_info nvmap_bdi = {
102         .ra_pages       = 0,
103         .capabilities   = (BDI_CAP_NO_ACCT_AND_WRITEBACK |
104                            BDI_CAP_READ_MAP | BDI_CAP_WRITE_MAP),
105 };
106
107 static struct device_dma_parameters nvmap_dma_parameters = {
108         .max_segment_size = UINT_MAX,
109 };
110
111 static int nvmap_open(struct inode *inode, struct file *filp);
112 static int nvmap_release(struct inode *inode, struct file *filp);
113 static long nvmap_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
114 static int nvmap_map(struct file *filp, struct vm_area_struct *vma);
115 static void nvmap_vma_open(struct vm_area_struct *vma);
116 static void nvmap_vma_close(struct vm_area_struct *vma);
117 static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
118
119 static const struct file_operations nvmap_user_fops = {
120         .owner          = THIS_MODULE,
121         .open           = nvmap_open,
122         .release        = nvmap_release,
123         .unlocked_ioctl = nvmap_ioctl,
124         .mmap           = nvmap_map,
125 };
126
127 static const struct file_operations nvmap_super_fops = {
128         .owner          = THIS_MODULE,
129         .open           = nvmap_open,
130         .release        = nvmap_release,
131         .unlocked_ioctl = nvmap_ioctl,
132         .mmap           = nvmap_map,
133 };
134
135 static struct vm_operations_struct nvmap_vma_ops = {
136         .open           = nvmap_vma_open,
137         .close          = nvmap_vma_close,
138         .fault          = nvmap_vma_fault,
139 };
140
141 int is_nvmap_vma(struct vm_area_struct *vma)
142 {
143         return vma->vm_ops == &nvmap_vma_ops;
144 }
145
146 struct device *nvmap_client_to_device(struct nvmap_client *client)
147 {
148         if (!client)
149                 return 0;
150         if (client->super)
151                 return nvmap_dev->dev_super.this_device;
152         else
153                 return nvmap_dev->dev_user.this_device;
154 }
155
156 struct nvmap_share *nvmap_get_share_from_dev(struct nvmap_device *dev)
157 {
158         return &dev->iovmm_master;
159 }
160
161 struct nvmap_deferred_ops *nvmap_get_deferred_ops_from_dev(
162                 struct nvmap_device *dev)
163 {
164         return &dev->deferred_ops;
165 }
166
167 /* allocates a PTE for the caller's use; returns the PTE pointer or
168  * a negative errno. not safe from IRQs */
169 pte_t **nvmap_alloc_pte_irq(struct nvmap_device *dev, void **vaddr)
170 {
171         unsigned long bit;
172
173         spin_lock(&dev->ptelock);
174         bit = find_next_zero_bit(dev->ptebits, NVMAP_NUM_PTES, dev->lastpte);
175         if (bit == NVMAP_NUM_PTES) {
176                 bit = find_first_zero_bit(dev->ptebits, dev->lastpte);
177                 if (bit == dev->lastpte)
178                         bit = NVMAP_NUM_PTES;
179         }
180
181         if (bit == NVMAP_NUM_PTES) {
182                 spin_unlock(&dev->ptelock);
183                 return ERR_PTR(-ENOMEM);
184         }
185
186         dev->lastpte = bit;
187         set_bit(bit, dev->ptebits);
188         spin_unlock(&dev->ptelock);
189
190         *vaddr = dev->vm_rgn->addr + bit * PAGE_SIZE;
191         return &(dev->ptes[bit]);
192 }
193
194 /* allocates a PTE for the caller's use; returns the PTE pointer or
195  * a negative errno. must be called from sleepable contexts */
196 pte_t **nvmap_alloc_pte(struct nvmap_device *dev, void **vaddr)
197 {
198         int ret;
199         pte_t **pte;
200         ret = wait_event_interruptible(dev->pte_wait,
201                         !IS_ERR(pte = nvmap_alloc_pte_irq(dev, vaddr)));
202
203         if (ret == -ERESTARTSYS)
204                 return ERR_PTR(-EINTR);
205
206         return pte;
207 }
208
209 /* frees a PTE */
210 void nvmap_free_pte(struct nvmap_device *dev, pte_t **pte)
211 {
212         unsigned long addr;
213         unsigned int bit = pte - dev->ptes;
214
215         if (WARN_ON(bit >= NVMAP_NUM_PTES))
216                 return;
217
218         addr = (unsigned long)dev->vm_rgn->addr + bit * PAGE_SIZE;
219         set_pte_at(&init_mm, addr, *pte, 0);
220
221         spin_lock(&dev->ptelock);
222         clear_bit(bit, dev->ptebits);
223         spin_unlock(&dev->ptelock);
224         wake_up(&dev->pte_wait);
225 }
226
227 /* get pte for the virtual address */
228 pte_t **nvmap_vaddr_to_pte(struct nvmap_device *dev, unsigned long vaddr)
229 {
230         unsigned int bit;
231
232         BUG_ON(vaddr < (unsigned long)dev->vm_rgn->addr);
233         bit = (vaddr - (unsigned long)dev->vm_rgn->addr) >> PAGE_SHIFT;
234         BUG_ON(bit >= NVMAP_NUM_PTES);
235         return &(dev->ptes[bit]);
236 }
237
238 /*
239  * Verifies that the passed ID is a valid handle ID. Then the passed client's
240  * reference to the handle is returned.
241  *
242  * Note: to call this function make sure you own the client ref lock.
243  */
244 struct nvmap_handle_ref *__nvmap_validate_id_locked(struct nvmap_client *c,
245                                                     unsigned long id)
246 {
247         struct rb_node *n = c->handle_refs.rb_node;
248
249         while (n) {
250                 struct nvmap_handle_ref *ref;
251                 ref = rb_entry(n, struct nvmap_handle_ref, node);
252                 if ((unsigned long)ref->handle == id)
253                         return ref;
254                 else if (id > (unsigned long)ref->handle)
255                         n = n->rb_right;
256                 else
257                         n = n->rb_left;
258         }
259
260         return NULL;
261 }
262
263 struct nvmap_handle *nvmap_get_handle_id(struct nvmap_client *client,
264                                          unsigned long id)
265 {
266 #ifdef CONFIG_NVMAP_USE_FD_FOR_HANDLE
267         return nvmap_handle_get((struct nvmap_handle *)id);
268 #else
269         struct nvmap_handle_ref *ref;
270         struct nvmap_handle *h = NULL;
271
272         nvmap_ref_lock(client);
273         ref = __nvmap_validate_id_locked(client, id);
274         if (ref)
275                 h = ref->handle;
276         if (h)
277                 h = nvmap_handle_get(h);
278         nvmap_ref_unlock(client);
279         return h;
280 #endif
281 }
282
283 unsigned long nvmap_carveout_usage(struct nvmap_client *c,
284                                    struct nvmap_heap_block *b)
285 {
286         struct nvmap_heap *h = nvmap_block_to_heap(b);
287         struct nvmap_carveout_node *n;
288         int i;
289
290         for (i = 0; i < nvmap_dev->nr_carveouts; i++) {
291                 n = &nvmap_dev->heaps[i];
292                 if (n->carveout == h)
293                         return n->heap_bit;
294         }
295         return 0;
296 }
297
298 /*
299  * This routine is used to flush the carveout memory from cache.
300  * Why cache flush is needed for carveout? Consider the case, where a piece of
301  * carveout is allocated as cached and released. After this, if the same memory is
302  * allocated for uncached request and the memory is not flushed out from cache.
303  * In this case, the client might pass this to H/W engine and it could start modify
304  * the memory. As this was cached earlier, it might have some portion of it in cache.
305  * During cpu request to read/write other memory, the cached portion of this memory
306  * might get flushed back to main memory and would cause corruptions, if it happens
307  * after H/W writes data to memory.
308  *
309  * But flushing out the memory blindly on each carveout allocation is redundant.
310  *
311  * In order to optimize the carveout buffer cache flushes, the following
312  * strategy is used.
313  *
314  * The whole Carveout is flushed out from cache during its initialization.
315  * During allocation, carveout buffers are not flused from cache.
316  * During deallocation, carveout buffers are flushed, if they were allocated as cached.
317  * if they were allocated as uncached/writecombined, no cache flush is needed.
318  * Just draining store buffers is enough.
319  */
320 int nvmap_flush_heap_block(struct nvmap_client *client,
321         struct nvmap_heap_block *block, size_t len, unsigned int prot)
322 {
323         pte_t **pte;
324         void *addr;
325         uintptr_t kaddr;
326         phys_addr_t phys = block->base;
327         phys_addr_t end = block->base + len;
328
329         if (prot == NVMAP_HANDLE_UNCACHEABLE || prot == NVMAP_HANDLE_WRITE_COMBINE)
330                 goto out;
331
332 #ifdef CONFIG_NVMAP_CACHE_MAINT_BY_SET_WAYS
333         if (len >= cache_maint_inner_threshold) {
334                 inner_flush_cache_all();
335                 if (prot != NVMAP_HANDLE_INNER_CACHEABLE)
336                         outer_flush_range(block->base, block->base + len);
337                 goto out;
338         }
339 #endif
340
341         pte = nvmap_alloc_pte(nvmap_dev, &addr);
342         if (IS_ERR(pte))
343                 return PTR_ERR(pte);
344
345         kaddr = (uintptr_t)addr;
346
347         while (phys < end) {
348                 phys_addr_t next = (phys + PAGE_SIZE) & PAGE_MASK;
349                 unsigned long pfn = __phys_to_pfn(phys);
350                 void *base = (void *)kaddr + (phys & ~PAGE_MASK);
351
352                 next = min(next, end);
353                 set_pte_at(&init_mm, kaddr, *pte, pfn_pte(pfn, pgprot_kernel));
354                 nvmap_flush_tlb_kernel_page(kaddr);
355                 __cpuc_flush_dcache_area(base, next - phys);
356                 phys = next;
357         }
358
359         if (prot != NVMAP_HANDLE_INNER_CACHEABLE)
360                 outer_flush_range(block->base, block->base + len);
361
362         nvmap_free_pte(nvmap_dev, pte);
363 out:
364         wmb();
365         return 0;
366 }
367
368 void nvmap_carveout_commit_add(struct nvmap_client *client,
369                                struct nvmap_carveout_node *node,
370                                size_t len)
371 {
372         spin_lock(&node->clients_lock);
373         BUG_ON(list_empty(&client->carveout_commit[node->index].list) &&
374                client->carveout_commit[node->index].commit != 0);
375
376         client->carveout_commit[node->index].commit += len;
377         /* if this client isn't already on the list of nodes for this heap,
378            add it */
379         if (list_empty(&client->carveout_commit[node->index].list)) {
380                 list_add(&client->carveout_commit[node->index].list,
381                          &node->clients);
382         }
383         spin_unlock(&node->clients_lock);
384 }
385
386 void nvmap_carveout_commit_subtract(struct nvmap_client *client,
387                                     struct nvmap_carveout_node *node,
388                                     size_t len)
389 {
390         if (!client)
391                 return;
392
393         spin_lock(&node->clients_lock);
394         BUG_ON(client->carveout_commit[node->index].commit < len);
395         client->carveout_commit[node->index].commit -= len;
396         /* if no more allocation in this carveout for this node, delete it */
397         if (!client->carveout_commit[node->index].commit)
398                 list_del_init(&client->carveout_commit[node->index].list);
399         spin_unlock(&node->clients_lock);
400 }
401
402 static struct nvmap_client *get_client_from_carveout_commit(
403         struct nvmap_carveout_node *node, struct nvmap_carveout_commit *commit)
404 {
405         struct nvmap_carveout_commit *first_commit = commit - node->index;
406         return (void *)first_commit - offsetof(struct nvmap_client,
407                                                carveout_commit);
408 }
409
410 static
411 struct nvmap_heap_block *do_nvmap_carveout_alloc(struct nvmap_client *client,
412                                               struct nvmap_handle *handle,
413                                               unsigned long type)
414 {
415         struct nvmap_carveout_node *co_heap;
416         struct nvmap_device *dev = nvmap_dev;
417         int i;
418
419         for (i = 0; i < dev->nr_carveouts; i++) {
420                 struct nvmap_heap_block *block;
421                 co_heap = &dev->heaps[i];
422
423                 if (!(co_heap->heap_bit & type))
424                         continue;
425
426                 block = nvmap_heap_alloc(co_heap->carveout, handle);
427                 if (block)
428                         return block;
429         }
430         return NULL;
431 }
432
433 struct nvmap_heap_block *nvmap_carveout_alloc(struct nvmap_client *client,
434                                               struct nvmap_handle *handle,
435                                               unsigned long type)
436 {
437         return do_nvmap_carveout_alloc(client, handle, type);
438 }
439
440 /* remove a handle from the device's tree of all handles; called
441  * when freeing handles. */
442 int nvmap_handle_remove(struct nvmap_device *dev, struct nvmap_handle *h)
443 {
444         spin_lock(&dev->handle_lock);
445
446         /* re-test inside the spinlock if the handle really has no clients;
447          * only remove the handle if it is unreferenced */
448         if (atomic_add_return(0, &h->ref) > 0) {
449                 spin_unlock(&dev->handle_lock);
450                 return -EBUSY;
451         }
452         smp_rmb();
453         BUG_ON(atomic_read(&h->ref) < 0);
454         BUG_ON(atomic_read(&h->pin) != 0);
455
456         rb_erase(&h->node, &dev->handles);
457
458         spin_unlock(&dev->handle_lock);
459         return 0;
460 }
461
462 /* adds a newly-created handle to the device master tree */
463 void nvmap_handle_add(struct nvmap_device *dev, struct nvmap_handle *h)
464 {
465         struct rb_node **p;
466         struct rb_node *parent = NULL;
467
468         spin_lock(&dev->handle_lock);
469         p = &dev->handles.rb_node;
470         while (*p) {
471                 struct nvmap_handle *b;
472
473                 parent = *p;
474                 b = rb_entry(parent, struct nvmap_handle, node);
475                 if (h > b)
476                         p = &parent->rb_right;
477                 else
478                         p = &parent->rb_left;
479         }
480         rb_link_node(&h->node, parent, p);
481         rb_insert_color(&h->node, &dev->handles);
482         spin_unlock(&dev->handle_lock);
483 }
484
485 /* validates that a handle is in the device master tree, and that the
486  * client has permission to access it */
487 struct nvmap_handle *nvmap_validate_get(struct nvmap_client *client,
488                                         unsigned long id, bool skip_val)
489 {
490 #ifdef CONFIG_NVMAP_USE_FD_FOR_HANDLE
491         return nvmap_handle_get((struct nvmap_handle *)id);
492 #else
493         struct nvmap_handle *h = NULL;
494         struct rb_node *n;
495
496         spin_lock(&nvmap_dev->handle_lock);
497
498         n = nvmap_dev->handles.rb_node;
499
500         while (n) {
501                 h = rb_entry(n, struct nvmap_handle, node);
502                 if ((unsigned long)h == id) {
503                         if (client->super || h->global ||
504                             (h->owner == client) || skip_val)
505                                 h = nvmap_handle_get(h);
506                         else
507                                 h = nvmap_get_handle_id(client, id);
508                         spin_unlock(&nvmap_dev->handle_lock);
509                         return h;
510                 }
511                 if (id > (unsigned long)h)
512                         n = n->rb_right;
513                 else
514                         n = n->rb_left;
515         }
516         spin_unlock(&nvmap_dev->handle_lock);
517         return NULL;
518 #endif
519 }
520
521 struct nvmap_client *__nvmap_create_client(struct nvmap_device *dev,
522                                            const char *name)
523 {
524         struct nvmap_client *client;
525         struct task_struct *task;
526         int i;
527
528         if (WARN_ON(!dev))
529                 return NULL;
530
531         client = kzalloc(sizeof(*client) + (sizeof(struct nvmap_carveout_commit)
532                          * dev->nr_carveouts), GFP_KERNEL);
533         if (!client)
534                 return NULL;
535
536         client->name = name;
537         client->super = true;
538         client->kernel_client = true;
539         client->handle_refs = RB_ROOT;
540
541         atomic_set(&client->iovm_commit, 0);
542
543         for (i = 0; i < dev->nr_carveouts; i++) {
544                 INIT_LIST_HEAD(&client->carveout_commit[i].list);
545                 client->carveout_commit[i].commit = 0;
546         }
547
548         get_task_struct(current->group_leader);
549         task_lock(current->group_leader);
550         /* don't bother to store task struct for kernel threads,
551            they can't be killed anyway */
552         if (current->flags & PF_KTHREAD) {
553                 put_task_struct(current->group_leader);
554                 task = NULL;
555         } else {
556                 task = current->group_leader;
557         }
558         task_unlock(current->group_leader);
559         client->task = task;
560
561         mutex_init(&client->ref_lock);
562         atomic_set(&client->count, 1);
563
564         spin_lock(&dev->clients_lock);
565         list_add(&client->list, &dev->clients);
566         spin_unlock(&dev->clients_lock);
567         return client;
568 }
569
570 static void destroy_client(struct nvmap_client *client)
571 {
572         struct rb_node *n;
573         int i;
574
575         if (!client)
576                 return;
577
578         spin_lock(&nvmap_dev->clients_lock);
579         list_del(&client->list);
580         spin_unlock(&nvmap_dev->clients_lock);
581
582         while ((n = rb_first(&client->handle_refs))) {
583                 struct nvmap_handle_ref *ref;
584                 int pins, dupes;
585
586                 ref = rb_entry(n, struct nvmap_handle_ref, node);
587
588                 smp_rmb();
589                 pins = atomic_read(&ref->pin);
590
591                 while (pins--)
592                         __nvmap_unpin(ref);
593
594                 if (ref->handle->owner == client) {
595                         ref->handle->owner = NULL;
596                         ref->handle->owner_ref = NULL;
597                 }
598
599                 dma_buf_put(ref->handle->dmabuf);
600                 rb_erase(&ref->node, &client->handle_refs);
601
602                 dupes = atomic_read(&ref->dupes);
603                 while (dupes--)
604                         nvmap_handle_put(ref->handle);
605
606                 kfree(ref);
607         }
608
609         for (i = 0; i < nvmap_dev->nr_carveouts; i++)
610                 list_del(&client->carveout_commit[i].list);
611
612         if (client->task)
613                 put_task_struct(client->task);
614
615         kfree(client);
616 }
617
618 struct nvmap_client *nvmap_client_get(struct nvmap_client *client)
619 {
620         if (!virt_addr_valid(client))
621                 return NULL;
622
623         if (!atomic_add_unless(&client->count, 1, 0))
624                 return NULL;
625
626         return client;
627 }
628
629 struct nvmap_client *nvmap_client_get_file(int fd)
630 {
631         struct nvmap_client *client = ERR_PTR(-EFAULT);
632         struct file *f = fget(fd);
633         if (!f)
634                 return ERR_PTR(-EINVAL);
635
636         if ((f->f_op == &nvmap_user_fops) || (f->f_op == &nvmap_super_fops)) {
637                 client = f->private_data;
638                 atomic_inc(&client->count);
639         }
640
641         fput(f);
642         return client;
643 }
644
645 void nvmap_client_put(struct nvmap_client *client)
646 {
647         if (!client)
648                 return;
649
650         if (!atomic_dec_return(&client->count))
651                 destroy_client(client);
652 }
653 EXPORT_SYMBOL(nvmap_client_put);
654
655 static int nvmap_open(struct inode *inode, struct file *filp)
656 {
657         struct miscdevice *miscdev = filp->private_data;
658         struct nvmap_device *dev = dev_get_drvdata(miscdev->parent);
659         struct nvmap_client *priv;
660         int ret;
661         __attribute__((unused)) struct rlimit old_rlim, new_rlim;
662
663         ret = nonseekable_open(inode, filp);
664         if (unlikely(ret))
665                 return ret;
666
667         BUG_ON(dev != nvmap_dev);
668         priv = __nvmap_create_client(dev, "user");
669         if (!priv)
670                 return -ENOMEM;
671         trace_nvmap_open(priv, priv->name);
672
673         priv->kernel_client = false;
674         priv->super = (filp->f_op == &nvmap_super_fops);
675
676         filp->f_mapping->backing_dev_info = &nvmap_bdi;
677
678         filp->private_data = priv;
679         return 0;
680 }
681
682 static int nvmap_release(struct inode *inode, struct file *filp)
683 {
684         struct nvmap_client *priv = filp->private_data;
685
686         trace_nvmap_release(priv, priv->name);
687         nvmap_client_put(priv);
688         return 0;
689 }
690
691 int __nvmap_map(struct nvmap_handle *h, struct vm_area_struct *vma)
692 {
693         struct nvmap_vma_priv *priv;
694
695         h = nvmap_handle_get(h);
696         if (!h)
697                 return -EINVAL;
698
699         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
700         if (!priv)
701                 return -ENOMEM;
702
703         priv->offs = 0;
704         priv->handle = h;
705         atomic_set(&priv->count, 1);
706
707         vma->vm_flags |= (VM_SHARED | VM_IO | VM_DONTEXPAND |
708                           VM_MIXEDMAP | VM_DONTDUMP | VM_DONTCOPY);
709         vma->vm_ops = &nvmap_vma_ops;
710         BUG_ON(vma->vm_private_data != NULL);
711         vma->vm_private_data = priv;
712         vma->vm_page_prot = nvmap_pgprot(h, vma->vm_page_prot);
713         return 0;
714 }
715
716 static int nvmap_map(struct file *filp, struct vm_area_struct *vma)
717 {
718         struct nvmap_vma_priv *priv;
719
720         /* after NVMAP_IOC_MMAP, the handle that is mapped by this VMA
721          * will be stored in vm_private_data and faulted in. until the
722          * ioctl is made, the VMA is mapped no-access */
723         vma->vm_private_data = NULL;
724
725         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
726         if (!priv)
727                 return -ENOMEM;
728
729         priv->offs = 0;
730         priv->handle = NULL;
731         atomic_set(&priv->count, 1);
732
733         vma->vm_flags |= (VM_SHARED | VM_IO | VM_DONTEXPAND |
734                           VM_MIXEDMAP | VM_DONTDUMP | VM_DONTCOPY);
735         vma->vm_ops = &nvmap_vma_ops;
736         vma->vm_private_data = priv;
737
738         return 0;
739 }
740
741 static long nvmap_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
742 {
743         int err = 0;
744         void __user *uarg = (void __user *)arg;
745
746         if (_IOC_TYPE(cmd) != NVMAP_IOC_MAGIC)
747                 return -ENOTTY;
748
749         if (_IOC_NR(cmd) > NVMAP_IOC_MAXNR)
750                 return -ENOTTY;
751
752         if (_IOC_DIR(cmd) & _IOC_READ)
753                 err = !access_ok(VERIFY_WRITE, uarg, _IOC_SIZE(cmd));
754         if (_IOC_DIR(cmd) & _IOC_WRITE)
755                 err = !access_ok(VERIFY_READ, uarg, _IOC_SIZE(cmd));
756
757         if (err)
758                 return -EFAULT;
759
760         switch (cmd) {
761         case NVMAP_IOC_CLAIM:
762                 nvmap_warn(filp->private_data, "preserved handles not"
763                            "supported\n");
764                 err = -ENODEV;
765                 break;
766         case NVMAP_IOC_CREATE:
767         case NVMAP_IOC_FROM_ID:
768         case NVMAP_IOC_FROM_FD:
769                 err = nvmap_ioctl_create(filp, cmd, uarg);
770                 break;
771
772         case NVMAP_IOC_GET_ID:
773                 err = nvmap_ioctl_getid(filp, uarg);
774                 break;
775
776         case NVMAP_IOC_GET_FD:
777                 err = nvmap_ioctl_getfd(filp, uarg);
778                 break;
779
780         case NVMAP_IOC_PARAM:
781                 err = nvmap_ioctl_get_param(filp, uarg);
782                 break;
783
784         case NVMAP_IOC_UNPIN_MULT:
785         case NVMAP_IOC_PIN_MULT:
786                 err = nvmap_ioctl_pinop(filp, cmd == NVMAP_IOC_PIN_MULT, uarg);
787                 break;
788
789         case NVMAP_IOC_ALLOC:
790                 err = nvmap_ioctl_alloc(filp, uarg);
791                 break;
792
793         case NVMAP_IOC_ALLOC_KIND:
794                 err = nvmap_ioctl_alloc_kind(filp, uarg);
795                 break;
796
797         case NVMAP_IOC_FREE:
798                 err = nvmap_ioctl_free(filp, arg);
799                 break;
800
801         case NVMAP_IOC_MMAP:
802                 err = nvmap_map_into_caller_ptr(filp, uarg);
803                 break;
804
805         case NVMAP_IOC_WRITE:
806         case NVMAP_IOC_READ:
807                 err = nvmap_ioctl_rw_handle(filp, cmd == NVMAP_IOC_READ, uarg);
808                 break;
809
810         case NVMAP_IOC_CACHE:
811                 err = nvmap_ioctl_cache_maint(filp, uarg);
812                 break;
813
814         case NVMAP_IOC_SHARE:
815                 err = nvmap_ioctl_share_dmabuf(filp, uarg);
816                 break;
817
818         default:
819                 return -ENOTTY;
820         }
821         return err;
822 }
823
824 /* to ensure that the backing store for the VMA isn't freed while a fork'd
825  * reference still exists, nvmap_vma_open increments the reference count on
826  * the handle, and nvmap_vma_close decrements it. alternatively, we could
827  * disallow copying of the vma, or behave like pmem and zap the pages. FIXME.
828 */
829 static void nvmap_vma_open(struct vm_area_struct *vma)
830 {
831         struct nvmap_vma_priv *priv;
832
833         priv = vma->vm_private_data;
834         BUG_ON(!priv);
835
836         atomic_inc(&priv->count);
837 }
838
839 static void nvmap_vma_close(struct vm_area_struct *vma)
840 {
841         struct nvmap_vma_priv *priv = vma->vm_private_data;
842
843         if (priv) {
844                 if (!atomic_dec_return(&priv->count)) {
845                         if (priv->handle)
846                                 nvmap_handle_put(priv->handle);
847                         kfree(priv);
848                 }
849         }
850         vma->vm_private_data = NULL;
851 }
852
853 static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
854 {
855         struct page *page;
856         struct nvmap_vma_priv *priv;
857         unsigned long offs;
858
859         offs = (unsigned long)(vmf->virtual_address - vma->vm_start);
860         priv = vma->vm_private_data;
861         if (!priv || !priv->handle || !priv->handle->alloc)
862                 return VM_FAULT_SIGBUS;
863
864         offs += priv->offs;
865         /* if the VMA was split for some reason, vm_pgoff will be the VMA's
866          * offset from the original VMA */
867         offs += (vma->vm_pgoff << PAGE_SHIFT);
868
869         if (offs >= priv->handle->size)
870                 return VM_FAULT_SIGBUS;
871
872         if (!priv->handle->heap_pgalloc) {
873                 unsigned long pfn;
874                 BUG_ON(priv->handle->carveout->base & ~PAGE_MASK);
875                 pfn = ((priv->handle->carveout->base + offs) >> PAGE_SHIFT);
876                 if (!pfn_valid(pfn)) {
877                         vm_insert_pfn(vma,
878                                 (unsigned long)vmf->virtual_address, pfn);
879                         return VM_FAULT_NOPAGE;
880                 }
881                 /* CMA memory would get here */
882                 page = pfn_to_page(pfn);
883         } else {
884                 offs >>= PAGE_SHIFT;
885                 page = priv->handle->pgalloc.pages[offs];
886         }
887
888         if (page)
889                 get_page(page);
890         vmf->page = page;
891         return (page) ? 0 : VM_FAULT_SIGBUS;
892 }
893
894 #define DEBUGFS_OPEN_FOPS(name) \
895 static int nvmap_debug_##name##_open(struct inode *inode, \
896                                             struct file *file) \
897 { \
898         return single_open(file, nvmap_debug_##name##_show, \
899                             inode->i_private); \
900 } \
901 \
902 static const struct file_operations debug_##name##_fops = { \
903         .open = nvmap_debug_##name##_open, \
904         .read = seq_read, \
905         .llseek = seq_lseek, \
906         .release = single_release, \
907 }
908
909 #define K(x) (x >> 10)
910
911 static void client_stringify(struct nvmap_client *client, struct seq_file *s)
912 {
913         char task_comm[TASK_COMM_LEN];
914         if (!client->task) {
915                 seq_printf(s, "%-18s %18s %8u", client->name, "kernel", 0);
916                 return;
917         }
918         get_task_comm(task_comm, client->task);
919         seq_printf(s, "%-18s %18s %8u", client->name, task_comm,
920                    client->task->pid);
921 }
922
923 static void allocations_stringify(struct nvmap_client *client,
924                                   struct seq_file *s, bool iovmm)
925 {
926         struct rb_node *n;
927
928         nvmap_ref_lock(client);
929         n = rb_first(&client->handle_refs);
930         for (; n != NULL; n = rb_next(n)) {
931                 struct nvmap_handle_ref *ref =
932                         rb_entry(n, struct nvmap_handle_ref, node);
933                 struct nvmap_handle *handle = ref->handle;
934                 if (handle->alloc && handle->heap_pgalloc == iovmm) {
935                         phys_addr_t base = iovmm ? 0 :
936                                            (handle->carveout->base);
937                         seq_printf(s,
938                                 "%-18s %-18s %8llx %10zuK %8x %6u %6u %6u\n",
939                                 "", "",
940                                 (unsigned long long)base, K(handle->size),
941                                 handle->userflags,
942                                 atomic_read(&handle->ref),
943                                 atomic_read(&ref->dupes),
944                                 atomic_read(&ref->pin));
945                 }
946         }
947         nvmap_ref_unlock(client);
948 }
949
950 static int nvmap_debug_allocations_show(struct seq_file *s, void *unused)
951 {
952         struct nvmap_carveout_node *node = s->private;
953         struct nvmap_carveout_commit *commit;
954         unsigned int total = 0;
955
956         spin_lock(&node->clients_lock);
957         seq_printf(s, "%-18s %18s %8s %11s\n",
958                 "CLIENT", "PROCESS", "PID", "SIZE");
959         seq_printf(s, "%-18s %18s %8s %11s %8s %6s %6s %6s\n",
960                         "", "", "BASE", "SIZE", "FLAGS", "REFS",
961                         "DUPES", "PINS");
962         list_for_each_entry(commit, &node->clients, list) {
963                 struct nvmap_client *client =
964                         get_client_from_carveout_commit(node, commit);
965                 client_stringify(client, s);
966                 seq_printf(s, " %10zuK\n", K(commit->commit));
967                 allocations_stringify(client, s, false);
968                 seq_printf(s, "\n");
969                 total += commit->commit;
970         }
971         seq_printf(s, "%-18s %-18s %8s %10uK\n", "total", "", "", K(total));
972         spin_unlock(&node->clients_lock);
973         return 0;
974 }
975
976 DEBUGFS_OPEN_FOPS(allocations);
977
978 static int nvmap_debug_clients_show(struct seq_file *s, void *unused)
979 {
980         struct nvmap_carveout_node *node = s->private;
981         struct nvmap_carveout_commit *commit;
982         unsigned int total = 0;
983
984         spin_lock(&node->clients_lock);
985         seq_printf(s, "%-18s %18s %8s %11s\n",
986                 "CLIENT", "PROCESS", "PID", "SIZE");
987         list_for_each_entry(commit, &node->clients, list) {
988                 struct nvmap_client *client =
989                         get_client_from_carveout_commit(node, commit);
990                 client_stringify(client, s);
991                 seq_printf(s, " %10zu\n", K(commit->commit));
992                 total += commit->commit;
993         }
994         seq_printf(s, "%-18s %18s %8s %10uK\n", "total", "", "", K(total));
995         spin_unlock(&node->clients_lock);
996         return 0;
997 }
998
999 DEBUGFS_OPEN_FOPS(clients);
1000
1001 static void nvmap_iovmm_get_total_mss(u64 *pss, u64 *non_pss, u64 *total)
1002 {
1003         int i;
1004         struct rb_node *n;
1005         struct nvmap_device *dev = nvmap_dev;
1006
1007         *total = 0;
1008         if (pss)
1009                 *pss = 0;
1010         if (non_pss)
1011                 *non_pss = 0;
1012         if (!dev)
1013                 return;
1014         spin_lock(&dev->handle_lock);
1015         n = rb_first(&dev->handles);
1016         for (; n != NULL; n = rb_next(n)) {
1017                 struct nvmap_handle *h =
1018                         rb_entry(n, struct nvmap_handle, node);
1019
1020                 if (!h || !h->alloc || !h->heap_pgalloc)
1021                         continue;
1022                 if (!non_pss) {
1023                         *total += h->size;
1024                         continue;
1025                 }
1026
1027                 for (i = 0; i < h->size >> PAGE_SHIFT; i++) {
1028                         int mapcount = page_mapcount(h->pgalloc.pages[i]);
1029                         if (!mapcount)
1030                                 *non_pss += PAGE_SIZE;
1031                         *total += PAGE_SIZE;
1032                 }
1033         }
1034         if (pss && non_pss)
1035                 *pss = *total - *non_pss;
1036         spin_unlock(&dev->handle_lock);
1037 }
1038
1039 #define PRINT_MEM_STATS_NOTE(x) \
1040 do { \
1041         seq_printf(s, "Note: total memory is precise account of pages " \
1042                 "allocated by NvMap.\nIt doesn't match with all clients " \
1043                 "\"%s\" accumulated as shared memory \nis accounted in " \
1044                 "full in each clients \"%s\" that shared memory.\n", #x, #x); \
1045 } while (0)
1046
1047 static int nvmap_debug_iovmm_clients_show(struct seq_file *s, void *unused)
1048 {
1049         u64 total;
1050         struct nvmap_client *client;
1051         struct nvmap_device *dev = s->private;
1052
1053         spin_lock(&dev->clients_lock);
1054         seq_printf(s, "%-18s %18s %8s %11s\n",
1055                 "CLIENT", "PROCESS", "PID", "SIZE");
1056         list_for_each_entry(client, &dev->clients, list) {
1057                 int iovm_commit = atomic_read(&client->iovm_commit);
1058                 client_stringify(client, s);
1059                 seq_printf(s, " %10uK\n", K(iovm_commit));
1060         }
1061         spin_unlock(&dev->clients_lock);
1062         nvmap_iovmm_get_total_mss(NULL, NULL, &total);
1063         seq_printf(s, "%-18s %18s %8s %10lluK\n", "total", "", "", K(total));
1064         PRINT_MEM_STATS_NOTE(SIZE);
1065         return 0;
1066 }
1067
1068 DEBUGFS_OPEN_FOPS(iovmm_clients);
1069
1070 static int nvmap_debug_iovmm_allocations_show(struct seq_file *s, void *unused)
1071 {
1072         u64 total;
1073         struct nvmap_client *client;
1074         struct nvmap_device *dev = s->private;
1075
1076         spin_lock(&dev->clients_lock);
1077         seq_printf(s, "%-18s %18s %8s %11s\n",
1078                 "CLIENT", "PROCESS", "PID", "SIZE");
1079         seq_printf(s, "%-18s %18s %8s %11s %8s %6s %6s %6s\n",
1080                         "", "", "BASE", "SIZE", "FLAGS", "REFS",
1081                         "DUPES", "PINS");
1082         list_for_each_entry(client, &dev->clients, list) {
1083                 int iovm_commit = atomic_read(&client->iovm_commit);
1084                 client_stringify(client, s);
1085                 seq_printf(s, " %10uK\n", K(iovm_commit));
1086                 allocations_stringify(client, s, true);
1087                 seq_printf(s, "\n");
1088         }
1089         spin_unlock(&dev->clients_lock);
1090         nvmap_iovmm_get_total_mss(NULL, NULL, &total);
1091         seq_printf(s, "%-18s %-18s %8s %10lluK\n", "total", "", "", K(total));
1092         PRINT_MEM_STATS_NOTE(SIZE);
1093         return 0;
1094 }
1095
1096 DEBUGFS_OPEN_FOPS(iovmm_allocations);
1097
1098 static void nvmap_iovmm_get_client_mss(struct nvmap_client *client, u64 *pss,
1099                                    u64 *non_pss, u64 *total)
1100 {
1101         int i;
1102         struct rb_node *n;
1103
1104         *pss = *non_pss = *total = 0;
1105         nvmap_ref_lock(client);
1106         n = rb_first(&client->handle_refs);
1107         for (; n != NULL; n = rb_next(n)) {
1108                 struct nvmap_handle_ref *ref =
1109                         rb_entry(n, struct nvmap_handle_ref, node);
1110                 struct nvmap_handle *h = ref->handle;
1111
1112                 if (!h || !h->alloc || !h->heap_pgalloc)
1113                         continue;
1114
1115                 for (i = 0; i < h->size >> PAGE_SHIFT; i++) {
1116                         int mapcount = page_mapcount(h->pgalloc.pages[i]);
1117                         if (!mapcount)
1118                                 *non_pss += PAGE_SIZE;
1119                         *total += PAGE_SIZE;
1120                 }
1121                 *pss = *total - *non_pss;
1122         }
1123         nvmap_ref_unlock(client);
1124 }
1125
1126 static int nvmap_debug_iovmm_procrank_show(struct seq_file *s, void *unused)
1127 {
1128         u64 pss, non_pss, total;
1129         struct nvmap_client *client;
1130         struct nvmap_device *dev = s->private;
1131         u64 total_memory, total_pss, total_non_pss;
1132
1133         spin_lock(&dev->clients_lock);
1134         seq_printf(s, "%-18s %18s %8s %11s %11s %11s\n",
1135                 "CLIENT", "PROCESS", "PID", "PSS", "NON-PSS", "TOTAL");
1136         list_for_each_entry(client, &dev->clients, list) {
1137                 client_stringify(client, s);
1138                 nvmap_iovmm_get_client_mss(client, &pss, &non_pss, &total);
1139                 seq_printf(s, " %10lluK %10lluK %10lluK\n", K(pss),
1140                         K(non_pss), K(total));
1141         }
1142         spin_unlock(&dev->clients_lock);
1143
1144         nvmap_iovmm_get_total_mss(&total_pss, &total_non_pss, &total_memory);
1145         seq_printf(s, "%-18s %18s %8s %10lluK %10lluK %10lluK\n",
1146                 "total", "", "", K(total_pss),
1147                 K(total_non_pss), K(total_memory));
1148         PRINT_MEM_STATS_NOTE(TOTAL);
1149         return 0;
1150 }
1151
1152 DEBUGFS_OPEN_FOPS(iovmm_procrank);
1153
1154 ulong nvmap_iovmm_get_used_pages(void)
1155 {
1156         u64 total;
1157
1158         nvmap_iovmm_get_total_mss(NULL, NULL, &total);
1159         return total >> PAGE_SHIFT;
1160 }
1161
1162 static void nvmap_deferred_ops_init(struct nvmap_deferred_ops *deferred_ops)
1163 {
1164         INIT_LIST_HEAD(&deferred_ops->ops_list);
1165         spin_lock_init(&deferred_ops->deferred_ops_lock);
1166
1167 #ifdef CONFIG_NVMAP_DEFERRED_CACHE_MAINT
1168         deferred_ops->enable_deferred_cache_maintenance = 1;
1169 #else
1170         deferred_ops->enable_deferred_cache_maintenance = 0;
1171 #endif /* CONFIG_NVMAP_DEFERRED_CACHE_MAINT */
1172
1173         deferred_ops->deferred_maint_inner_requested = 0;
1174         deferred_ops->deferred_maint_inner_flushed = 0;
1175         deferred_ops->deferred_maint_outer_requested = 0;
1176         deferred_ops->deferred_maint_outer_flushed = 0;
1177 }
1178
1179 static int nvmap_probe(struct platform_device *pdev)
1180 {
1181         struct nvmap_platform_data *plat = pdev->dev.platform_data;
1182         struct nvmap_device *dev;
1183         struct dentry *nvmap_debug_root;
1184         unsigned int i;
1185         int e;
1186
1187         if (!plat) {
1188                 dev_err(&pdev->dev, "no platform data?\n");
1189                 return -ENODEV;
1190         }
1191
1192         /*
1193          * The DMA mapping API uses these parameters to decide how to map the
1194          * passed buffers. If the maximum physical segment size is set to
1195          * smaller than the size of the buffer, then the buffer will be mapped
1196          * as separate IO virtual address ranges.
1197          */
1198         pdev->dev.dma_parms = &nvmap_dma_parameters;
1199
1200         if (WARN_ON(nvmap_dev != NULL)) {
1201                 dev_err(&pdev->dev, "only one nvmap device may be present\n");
1202                 return -ENODEV;
1203         }
1204
1205         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1206         if (!dev) {
1207                 dev_err(&pdev->dev, "out of memory for device\n");
1208                 return -ENOMEM;
1209         }
1210
1211         dev->dev_user.minor = MISC_DYNAMIC_MINOR;
1212         dev->dev_user.name = "nvmap";
1213         dev->dev_user.fops = &nvmap_user_fops;
1214         dev->dev_user.parent = &pdev->dev;
1215
1216         dev->dev_super.minor = MISC_DYNAMIC_MINOR;
1217         dev->dev_super.name = "knvmap";
1218         dev->dev_super.fops = &nvmap_super_fops;
1219         dev->dev_super.parent = &pdev->dev;
1220
1221         dev->handles = RB_ROOT;
1222
1223         init_waitqueue_head(&dev->pte_wait);
1224
1225         init_waitqueue_head(&dev->iovmm_master.pin_wait);
1226
1227         nvmap_deferred_ops_init(&dev->deferred_ops);
1228
1229         mutex_init(&dev->iovmm_master.pin_lock);
1230 #ifdef CONFIG_NVMAP_PAGE_POOLS
1231         for (i = 0; i < NVMAP_NUM_POOLS; i++)
1232                 nvmap_page_pool_init(&dev->iovmm_master.pools[i], i);
1233 #endif
1234
1235         dev->vm_rgn = alloc_vm_area(NVMAP_NUM_PTES * PAGE_SIZE, NULL);
1236         if (!dev->vm_rgn) {
1237                 e = -ENOMEM;
1238                 dev_err(&pdev->dev, "couldn't allocate remapping region\n");
1239                 goto fail;
1240         }
1241
1242         spin_lock_init(&dev->ptelock);
1243         spin_lock_init(&dev->handle_lock);
1244         INIT_LIST_HEAD(&dev->clients);
1245         spin_lock_init(&dev->clients_lock);
1246
1247         for (i = 0; i < NVMAP_NUM_PTES; i++) {
1248                 unsigned long addr;
1249                 pgd_t *pgd;
1250                 pud_t *pud;
1251                 pmd_t *pmd;
1252
1253                 addr = (unsigned long)dev->vm_rgn->addr + (i * PAGE_SIZE);
1254                 pgd = pgd_offset_k(addr);
1255                 pud = pud_alloc(&init_mm, pgd, addr);
1256                 if (!pud) {
1257                         e = -ENOMEM;
1258                         dev_err(&pdev->dev, "couldn't allocate page tables\n");
1259                         goto fail;
1260                 }
1261                 pmd = pmd_alloc(&init_mm, pud, addr);
1262                 if (!pmd) {
1263                         e = -ENOMEM;
1264                         dev_err(&pdev->dev, "couldn't allocate page tables\n");
1265                         goto fail;
1266                 }
1267                 dev->ptes[i] = pte_alloc_kernel(pmd, addr);
1268                 if (!dev->ptes[i]) {
1269                         e = -ENOMEM;
1270                         dev_err(&pdev->dev, "couldn't allocate page tables\n");
1271                         goto fail;
1272                 }
1273         }
1274
1275         e = misc_register(&dev->dev_user);
1276         if (e) {
1277                 dev_err(&pdev->dev, "unable to register miscdevice %s\n",
1278                         dev->dev_user.name);
1279                 goto fail;
1280         }
1281
1282         e = misc_register(&dev->dev_super);
1283         if (e) {
1284                 dev_err(&pdev->dev, "unable to register miscdevice %s\n",
1285                         dev->dev_super.name);
1286                 goto fail;
1287         }
1288
1289         dev->nr_carveouts = 0;
1290         dev->heaps = kzalloc(sizeof(struct nvmap_carveout_node) *
1291                              plat->nr_carveouts, GFP_KERNEL);
1292         if (!dev->heaps) {
1293                 e = -ENOMEM;
1294                 dev_err(&pdev->dev, "couldn't allocate carveout memory\n");
1295                 goto fail;
1296         }
1297
1298         nvmap_debug_root = debugfs_create_dir("nvmap", NULL);
1299         if (IS_ERR_OR_NULL(nvmap_debug_root))
1300                 dev_err(&pdev->dev, "couldn't create debug files\n");
1301
1302         debugfs_create_bool("enable_deferred_cache_maintenance",
1303                 S_IRUGO|S_IWUSR, nvmap_debug_root,
1304                 (u32 *)&dev->deferred_ops.enable_deferred_cache_maintenance);
1305
1306         debugfs_create_u32("max_handle_count", S_IRUGO,
1307                         nvmap_debug_root, &nvmap_max_handle_count);
1308
1309         debugfs_create_u64("deferred_maint_inner_requested", S_IRUGO|S_IWUSR,
1310                         nvmap_debug_root,
1311                         &dev->deferred_ops.deferred_maint_inner_requested);
1312
1313         debugfs_create_u64("deferred_maint_inner_flushed", S_IRUGO|S_IWUSR,
1314                         nvmap_debug_root,
1315                         &dev->deferred_ops.deferred_maint_inner_flushed);
1316 #ifdef CONFIG_OUTER_CACHE
1317         debugfs_create_u64("deferred_maint_outer_requested", S_IRUGO|S_IWUSR,
1318                         nvmap_debug_root,
1319                         &dev->deferred_ops.deferred_maint_outer_requested);
1320
1321         debugfs_create_u64("deferred_maint_outer_flushed", S_IRUGO|S_IWUSR,
1322                         nvmap_debug_root,
1323                         &dev->deferred_ops.deferred_maint_outer_flushed);
1324 #endif /* CONFIG_OUTER_CACHE */
1325         for (i = 0; i < plat->nr_carveouts; i++) {
1326                 struct nvmap_carveout_node *node = &dev->heaps[dev->nr_carveouts];
1327                 const struct nvmap_platform_carveout *co = &plat->carveouts[i];
1328                 node->base = round_up(co->base, PAGE_SIZE);
1329                 node->size = round_down(co->size -
1330                                         (node->base - co->base), PAGE_SIZE);
1331                 if (!co->size)
1332                         continue;
1333
1334                 node->carveout = nvmap_heap_create(
1335                                 dev->dev_user.this_device, co,
1336                                 node->base, node->size, node);
1337
1338                 if (!node->carveout) {
1339                         e = -ENOMEM;
1340                         dev_err(&pdev->dev, "couldn't create %s\n", co->name);
1341                         goto fail_heaps;
1342                 }
1343                 node->index = dev->nr_carveouts;
1344                 dev->nr_carveouts++;
1345                 spin_lock_init(&node->clients_lock);
1346                 INIT_LIST_HEAD(&node->clients);
1347                 node->heap_bit = co->usage_mask;
1348
1349                 if (!IS_ERR_OR_NULL(nvmap_debug_root)) {
1350                         struct dentry *heap_root =
1351                                 debugfs_create_dir(co->name, nvmap_debug_root);
1352                         if (!IS_ERR_OR_NULL(heap_root)) {
1353                                 debugfs_create_file("clients", S_IRUGO,
1354                                         heap_root, node, &debug_clients_fops);
1355                                 debugfs_create_file("allocations", S_IRUGO,
1356                                         heap_root, node,
1357                                         &debug_allocations_fops);
1358                                 nvmap_heap_debugfs_init(heap_root,
1359                                         node->carveout);
1360                         }
1361                 }
1362         }
1363         if (!IS_ERR_OR_NULL(nvmap_debug_root)) {
1364                 struct dentry *iovmm_root =
1365                         debugfs_create_dir("iovmm", nvmap_debug_root);
1366                 if (!IS_ERR_OR_NULL(iovmm_root)) {
1367                         debugfs_create_file("clients", S_IRUGO, iovmm_root,
1368                                 dev, &debug_iovmm_clients_fops);
1369                         debugfs_create_file("allocations", S_IRUGO, iovmm_root,
1370                                 dev, &debug_iovmm_allocations_fops);
1371                         debugfs_create_file("procrank", S_IRUGO, iovmm_root,
1372                                 dev, &debug_iovmm_procrank_fops);
1373 #ifdef CONFIG_NVMAP_PAGE_POOLS
1374                         for (i = 0; i < NVMAP_NUM_POOLS; i++) {
1375                                 char name[40];
1376                                 char *memtype_string[] = {"uc", "wc",
1377                                                           "iwb", "wb"};
1378                                 sprintf(name, "%s_page_pool_available_pages",
1379                                         memtype_string[i]);
1380                                 debugfs_create_u32(name, S_IRUGO,
1381                                         iovmm_root,
1382                                         &dev->iovmm_master.pools[i].npages);
1383                         }
1384 #endif
1385                 }
1386 #ifdef CONFIG_NVMAP_CACHE_MAINT_BY_SET_WAYS
1387                 debugfs_create_size_t("cache_maint_inner_threshold",
1388                                       S_IRUSR | S_IWUSR,
1389                                       nvmap_debug_root,
1390                                       &cache_maint_inner_threshold);
1391
1392                 /* cortex-a9 */
1393                 if ((read_cpuid_id() >> 4 & 0xfff) == 0xc09)
1394                         cache_maint_inner_threshold = SZ_32K;
1395                 pr_info("nvmap:inner cache maint threshold=%d",
1396                         cache_maint_inner_threshold);
1397 #endif
1398 #ifdef CONFIG_NVMAP_OUTER_CACHE_MAINT_BY_SET_WAYS
1399                 debugfs_create_size_t("cache_maint_outer_threshold",
1400                                       S_IRUSR | S_IWUSR,
1401                                       nvmap_debug_root,
1402                                       &cache_maint_outer_threshold);
1403                 pr_info("nvmap:outer cache maint threshold=%d",
1404                         cache_maint_outer_threshold);
1405 #endif
1406         }
1407
1408         platform_set_drvdata(pdev, dev);
1409         nvmap_pdev = pdev;
1410         nvmap_dev = dev;
1411         nvmap_share = &dev->iovmm_master;
1412
1413         nvmap_dmabuf_debugfs_init(nvmap_debug_root);
1414         e = nvmap_dmabuf_stash_init();
1415         if (e)
1416                 goto fail_heaps;
1417
1418         return 0;
1419 fail_heaps:
1420         for (i = 0; i < dev->nr_carveouts; i++) {
1421                 struct nvmap_carveout_node *node = &dev->heaps[i];
1422                 nvmap_heap_destroy(node->carveout);
1423         }
1424 fail:
1425         kfree(dev->heaps);
1426         if (dev->dev_super.minor != MISC_DYNAMIC_MINOR)
1427                 misc_deregister(&dev->dev_super);
1428         if (dev->dev_user.minor != MISC_DYNAMIC_MINOR)
1429                 misc_deregister(&dev->dev_user);
1430         if (dev->vm_rgn)
1431                 free_vm_area(dev->vm_rgn);
1432         kfree(dev);
1433         nvmap_dev = NULL;
1434         return e;
1435 }
1436
1437 static int nvmap_remove(struct platform_device *pdev)
1438 {
1439         struct nvmap_device *dev = platform_get_drvdata(pdev);
1440         struct rb_node *n;
1441         struct nvmap_handle *h;
1442         int i;
1443
1444         misc_deregister(&dev->dev_super);
1445         misc_deregister(&dev->dev_user);
1446
1447         while ((n = rb_first(&dev->handles))) {
1448                 h = rb_entry(n, struct nvmap_handle, node);
1449                 rb_erase(&h->node, &dev->handles);
1450                 kfree(h);
1451         }
1452
1453         for (i = 0; i < dev->nr_carveouts; i++) {
1454                 struct nvmap_carveout_node *node = &dev->heaps[i];
1455                 nvmap_heap_destroy(node->carveout);
1456         }
1457         kfree(dev->heaps);
1458
1459         free_vm_area(dev->vm_rgn);
1460         kfree(dev);
1461         nvmap_dev = NULL;
1462         return 0;
1463 }
1464
1465 static int nvmap_suspend(struct platform_device *pdev, pm_message_t state)
1466 {
1467         return 0;
1468 }
1469
1470 static int nvmap_resume(struct platform_device *pdev)
1471 {
1472         return 0;
1473 }
1474
1475 static struct platform_driver nvmap_driver = {
1476         .probe          = nvmap_probe,
1477         .remove         = nvmap_remove,
1478         .suspend        = nvmap_suspend,
1479         .resume         = nvmap_resume,
1480
1481         .driver = {
1482                 .name   = "tegra-nvmap",
1483                 .owner  = THIS_MODULE,
1484         },
1485 };
1486
1487 static int __init nvmap_init_driver(void)
1488 {
1489         int e;
1490
1491         nvmap_dev = NULL;
1492
1493         e = nvmap_heap_init();
1494         if (e)
1495                 goto fail;
1496
1497         e = platform_driver_register(&nvmap_driver);
1498         if (e) {
1499                 nvmap_heap_deinit();
1500                 goto fail;
1501         }
1502
1503 fail:
1504         return e;
1505 }
1506 fs_initcall(nvmap_init_driver);
1507
1508 static void __exit nvmap_exit_driver(void)
1509 {
1510         platform_driver_unregister(&nvmap_driver);
1511         nvmap_heap_deinit();
1512         nvmap_dev = NULL;
1513 }
1514 module_exit(nvmap_exit_driver);