video: tegra: nvmap: replace mutex with spinlock
[linux-2.6.git] / drivers / video / tegra / nvmap / nvmap_dev.c
1 /*
2  * drivers/video/tegra/nvmap/nvmap_dev.c
3  *
4  * User-space interface to nvmap
5  *
6  * Copyright (c) 2010, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/backing-dev.h>
24 #include <linux/bitmap.h>
25 #include <linux/debugfs.h>
26 #include <linux/delay.h>
27 #include <linux/kernel.h>
28 #include <linux/miscdevice.h>
29 #include <linux/mm.h>
30 #include <linux/oom.h>
31 #include <linux/platform_device.h>
32 #include <linux/seq_file.h>
33 #include <linux/slab.h>
34 #include <linux/spinlock.h>
35 #include <linux/uaccess.h>
36 #include <linux/vmalloc.h>
37
38 #include <asm/cacheflush.h>
39 #include <asm/tlbflush.h>
40
41 #include <mach/iovmm.h>
42 #include <mach/nvmap.h>
43
44 #include "nvmap.h"
45 #include "nvmap_ioctl.h"
46 #include "nvmap_mru.h"
47
48 #define NVMAP_NUM_PTES          64
49 #define NVMAP_CARVEOUT_KILLER_RETRY_TIME 100 /* msecs */
50
51 #ifdef CONFIG_NVMAP_CARVEOUT_KILLER
52 static bool carveout_killer = true;
53 #else
54 static bool carveout_killer;
55 #endif
56 module_param(carveout_killer, bool, 0640);
57
58 struct nvmap_carveout_node {
59         unsigned int            heap_bit;
60         struct nvmap_heap       *carveout;
61         int                     index;
62         struct list_head        clients;
63         spinlock_t              clients_lock;
64 };
65
66 struct nvmap_device {
67         struct vm_struct *vm_rgn;
68         pte_t           *ptes[NVMAP_NUM_PTES];
69         unsigned long   ptebits[NVMAP_NUM_PTES / BITS_PER_LONG];
70         unsigned int    lastpte;
71         spinlock_t      ptelock;
72
73         struct rb_root  handles;
74         spinlock_t      handle_lock;
75         wait_queue_head_t pte_wait;
76         struct miscdevice dev_super;
77         struct miscdevice dev_user;
78         struct nvmap_carveout_node *heaps;
79         int nr_carveouts;
80         struct nvmap_share iovmm_master;
81 };
82
83 struct nvmap_device *nvmap_dev;
84
85 static struct backing_dev_info nvmap_bdi = {
86         .ra_pages       = 0,
87         .capabilities   = (BDI_CAP_NO_ACCT_AND_WRITEBACK |
88                            BDI_CAP_READ_MAP | BDI_CAP_WRITE_MAP),
89 };
90
91 static int nvmap_open(struct inode *inode, struct file *filp);
92 static int nvmap_release(struct inode *inode, struct file *filp);
93 static long nvmap_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
94 static int nvmap_map(struct file *filp, struct vm_area_struct *vma);
95 static void nvmap_vma_open(struct vm_area_struct *vma);
96 static void nvmap_vma_close(struct vm_area_struct *vma);
97 static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
98
99 static const struct file_operations nvmap_user_fops = {
100         .owner          = THIS_MODULE,
101         .open           = nvmap_open,
102         .release        = nvmap_release,
103         .unlocked_ioctl = nvmap_ioctl,
104         .mmap           = nvmap_map,
105 };
106
107 static const struct file_operations nvmap_super_fops = {
108         .owner          = THIS_MODULE,
109         .open           = nvmap_open,
110         .release        = nvmap_release,
111         .unlocked_ioctl = nvmap_ioctl,
112         .mmap           = nvmap_map,
113 };
114
115 static struct vm_operations_struct nvmap_vma_ops = {
116         .open           = nvmap_vma_open,
117         .close          = nvmap_vma_close,
118         .fault          = nvmap_vma_fault,
119 };
120
121 int is_nvmap_vma(struct vm_area_struct *vma)
122 {
123         return vma->vm_ops == &nvmap_vma_ops;
124 }
125
126 struct device *nvmap_client_to_device(struct nvmap_client *client)
127 {
128         if (client->super)
129                 return client->dev->dev_super.this_device;
130         else
131                 return client->dev->dev_user.this_device;
132 }
133
134 struct nvmap_share *nvmap_get_share_from_dev(struct nvmap_device *dev)
135 {
136         return &dev->iovmm_master;
137 }
138
139 /* allocates a PTE for the caller's use; returns the PTE pointer or
140  * a negative errno. may be called from IRQs */
141 pte_t **nvmap_alloc_pte_irq(struct nvmap_device *dev, void **vaddr)
142 {
143         unsigned long flags;
144         unsigned long bit;
145
146         spin_lock_irqsave(&dev->ptelock, flags);
147         bit = find_next_zero_bit(dev->ptebits, NVMAP_NUM_PTES, dev->lastpte);
148         if (bit == NVMAP_NUM_PTES) {
149                 bit = find_first_zero_bit(dev->ptebits, dev->lastpte);
150                 if (bit == dev->lastpte)
151                         bit = NVMAP_NUM_PTES;
152         }
153
154         if (bit == NVMAP_NUM_PTES) {
155                 spin_unlock_irqrestore(&dev->ptelock, flags);
156                 return ERR_PTR(-ENOMEM);
157         }
158
159         dev->lastpte = bit;
160         set_bit(bit, dev->ptebits);
161         spin_unlock_irqrestore(&dev->ptelock, flags);
162
163         *vaddr = dev->vm_rgn->addr + bit * PAGE_SIZE;
164         return &(dev->ptes[bit]);
165 }
166
167 /* allocates a PTE for the caller's use; returns the PTE pointer or
168  * a negative errno. must be called from sleepable contexts */
169 pte_t **nvmap_alloc_pte(struct nvmap_device *dev, void **vaddr)
170 {
171         int ret;
172         pte_t **pte;
173         ret = wait_event_interruptible(dev->pte_wait,
174                         !IS_ERR(pte = nvmap_alloc_pte_irq(dev, vaddr)));
175
176         if (ret == -ERESTARTSYS)
177                 return ERR_PTR(-EINTR);
178
179         return pte;
180 }
181
182 /* frees a PTE */
183 void nvmap_free_pte(struct nvmap_device *dev, pte_t **pte)
184 {
185         unsigned long addr;
186         unsigned int bit = pte - dev->ptes;
187         unsigned long flags;
188
189         if (WARN_ON(bit >= NVMAP_NUM_PTES))
190                 return;
191
192         addr = (unsigned long)dev->vm_rgn->addr + bit * PAGE_SIZE;
193         set_pte_at(&init_mm, addr, *pte, 0);
194
195         spin_lock_irqsave(&dev->ptelock, flags);
196         clear_bit(bit, dev->ptebits);
197         spin_unlock_irqrestore(&dev->ptelock, flags);
198         wake_up(&dev->pte_wait);
199 }
200
201 /* verifies that the handle ref value "ref" is a valid handle ref for the
202  * file. caller must hold the file's ref_lock prior to calling this function */
203 struct nvmap_handle_ref *_nvmap_validate_id_locked(struct nvmap_client *c,
204                                                    unsigned long id)
205 {
206         struct rb_node *n = c->handle_refs.rb_node;
207
208         while (n) {
209                 struct nvmap_handle_ref *ref;
210                 ref = rb_entry(n, struct nvmap_handle_ref, node);
211                 if ((unsigned long)ref->handle == id)
212                         return ref;
213                 else if (id > (unsigned long)ref->handle)
214                         n = n->rb_right;
215                 else
216                         n = n->rb_left;
217         }
218
219         return NULL;
220 }
221
222 struct nvmap_handle *nvmap_get_handle_id(struct nvmap_client *client,
223                                          unsigned long id)
224 {
225         struct nvmap_handle_ref *ref;
226         struct nvmap_handle *h = NULL;
227
228         nvmap_ref_lock(client);
229         ref = _nvmap_validate_id_locked(client, id);
230         if (ref)
231                 h = ref->handle;
232         if (h)
233                 h = nvmap_handle_get(h);
234         nvmap_ref_unlock(client);
235         return h;
236 }
237
238 unsigned long nvmap_carveout_usage(struct nvmap_client *c,
239                                    struct nvmap_heap_block *b)
240 {
241         struct nvmap_heap *h = nvmap_block_to_heap(b);
242         struct nvmap_carveout_node *n;
243         int i;
244
245         for (i = 0; i < c->dev->nr_carveouts; i++) {
246                 n = &c->dev->heaps[i];
247                 if (n->carveout == h)
248                         return n->heap_bit;
249         }
250         return 0;
251 }
252
253 static int nvmap_flush_heap_block(struct nvmap_client *client,
254                                   struct nvmap_heap_block *block, size_t len)
255 {
256         pte_t **pte;
257         void *addr;
258         unsigned long kaddr;
259         unsigned long phys = block->base;
260         unsigned long end = block->base + len;
261
262         pte = nvmap_alloc_pte(client->dev, &addr);
263         if (IS_ERR(pte))
264                 return PTR_ERR(pte);
265
266         kaddr = (unsigned long)addr;
267
268         while (phys < end) {
269                 unsigned long next = (phys + PAGE_SIZE) & PAGE_MASK;
270                 unsigned long pfn = __phys_to_pfn(phys);
271                 void *base = (void *)kaddr + (phys & ~PAGE_MASK);
272
273                 next = min(next, end);
274                 set_pte_at(&init_mm, kaddr, *pte, pfn_pte(pfn, pgprot_kernel));
275                 flush_tlb_kernel_page(kaddr);
276                 __cpuc_flush_dcache_area(base, next - phys);
277                 phys = next;
278         }
279
280         outer_flush_range(block->base, block->base + len);
281
282         nvmap_free_pte(client->dev, pte);
283         return 0;
284 }
285
286 void nvmap_carveout_commit_add(struct nvmap_client *client,
287                                struct nvmap_carveout_node *node,
288                                size_t len)
289 {
290         unsigned long flags;
291
292         nvmap_ref_lock(client);
293         spin_lock_irqsave(&node->clients_lock, flags);
294         BUG_ON(list_empty(&client->carveout_commit[node->index].list) &&
295                client->carveout_commit[node->index].commit != 0);
296
297         client->carveout_commit[node->index].commit += len;
298         /* if this client isn't already on the list of nodes for this heap,
299            add it */
300         if (list_empty(&client->carveout_commit[node->index].list)) {
301                 list_add(&client->carveout_commit[node->index].list,
302                          &node->clients);
303         }
304         spin_unlock_irqrestore(&node->clients_lock, flags);
305         nvmap_ref_unlock(client);
306 }
307
308 void nvmap_carveout_commit_subtract(struct nvmap_client *client,
309                                     struct nvmap_carveout_node *node,
310                                     size_t len)
311 {
312         unsigned long flags;
313
314         if (!client)
315                 return;
316
317         spin_lock_irqsave(&node->clients_lock, flags);
318         client->carveout_commit[node->index].commit -= len;
319         BUG_ON(client->carveout_commit[node->index].commit < 0);
320         /* if no more allocation in this carveout for this node, delete it */
321         if (!client->carveout_commit[node->index].commit)
322                 list_del_init(&client->carveout_commit[node->index].list);
323         spin_unlock_irqrestore(&node->clients_lock, flags);
324 }
325
326 static struct nvmap_client* get_client_from_carveout_commit(
327         struct nvmap_carveout_node *node, struct nvmap_carveout_commit *commit)
328 {
329         struct nvmap_carveout_commit *first_commit = commit - node->index;
330         return (void *)first_commit - offsetof(struct nvmap_client,
331                                                carveout_commit);
332 }
333
334 static DECLARE_WAIT_QUEUE_HEAD(wait_reclaim);
335 static int wait_count;
336 bool nvmap_shrink_carveout(struct nvmap_carveout_node *node)
337 {
338         struct nvmap_carveout_commit *commit;
339         size_t selected_size = 0;
340         int selected_oom_adj = OOM_ADJUST_MIN;
341         struct task_struct *selected_task = NULL;
342         unsigned long flags;
343         bool wait = false;
344         int current_oom_adj = OOM_ADJUST_MIN;
345
346         task_lock(current);
347         if (current->signal)
348                 current_oom_adj = current->signal->oom_adj;
349         task_unlock(current);
350
351         spin_lock_irqsave(&node->clients_lock, flags);
352         /* find the task with the smallest oom_adj (lowest priority)
353          * and largest carveout allocation -- ignore kernel allocations,
354          * there's no way to handle them */
355         list_for_each_entry(commit, &node->clients, list) {
356                 struct nvmap_client *client =
357                         get_client_from_carveout_commit(node, commit);
358                 size_t size = commit->commit;
359                 struct task_struct *task = client->task;
360                 struct signal_struct *sig;
361
362                 if (!task)
363                         continue;
364
365                 task_lock(task);
366                 sig = task->signal;
367                 if (!task->mm || !sig)
368                         goto end;
369                 /* don't try to kill current */
370                 if (task == current->group_leader)
371                         goto end;
372                 /* don't try to kill higher priority tasks */
373                 if (sig->oom_adj < current_oom_adj)
374                         goto end;
375                 if (sig->oom_adj < selected_oom_adj)
376                         goto end;
377                 if (sig->oom_adj == selected_oom_adj &&
378                     size <= selected_size)
379                         goto end;
380                 selected_oom_adj = sig->oom_adj;
381                 selected_size = size;
382                 selected_task = task;
383 end:
384                 task_unlock(task);
385         }
386         if (selected_task) {
387                 wait = true;
388                 if (fatal_signal_pending(selected_task)) {
389                         pr_warning("carveout_killer: process %d dying "
390                                    "slowly\n", selected_task->pid);
391                         goto out;
392                 }
393                 pr_info("carveout_killer: killing process %d with oom_adj %d "
394                         "to reclaim %d (for process with oom_adj %d)\n",
395                         selected_task->pid, selected_oom_adj,
396                         selected_size, current_oom_adj);
397                 force_sig(SIGKILL, selected_task);
398         }
399 out:
400         spin_unlock_irqrestore(&node->clients_lock, flags);
401         return wait;
402 }
403
404 struct nvmap_heap_block *do_nvmap_carveout_alloc(struct nvmap_client *client,
405                                               size_t len, size_t align,
406                                               unsigned long usage,
407                                               unsigned int prot,
408                                               struct nvmap_handle *handle)
409 {
410         struct nvmap_carveout_node *co_heap;
411         struct nvmap_device *dev = client->dev;
412         int i;
413
414         for (i = 0; i < dev->nr_carveouts; i++) {
415                 struct nvmap_heap_block *block;
416                 co_heap = &dev->heaps[i];
417
418                 if (!(co_heap->heap_bit & usage))
419                         continue;
420
421                 block = nvmap_heap_alloc(co_heap->carveout, len,
422                                         align, prot, handle);
423                 if (block) {
424                         /* flush any stale data that may be left in the
425                          * cache at the block's address, since the new
426                          * block may be mapped uncached */
427                         if (nvmap_flush_heap_block(client, block, len)) {
428                                 nvmap_heap_free(block);
429                                 block = NULL;
430                         }
431                         return block;
432                 }
433         }
434         return NULL;
435 }
436
437 static bool nvmap_carveout_freed(int count)
438 {
439         smp_rmb();
440         return count != wait_count;
441 }
442
443 struct nvmap_heap_block *nvmap_carveout_alloc(struct nvmap_client *client,
444                                               size_t len, size_t align,
445                                               unsigned long usage,
446                                               unsigned int prot,
447                                               struct nvmap_handle *handle)
448 {
449         struct nvmap_heap_block *block;
450         struct nvmap_carveout_node *co_heap;
451         struct nvmap_device *dev = client->dev;
452         int i;
453         unsigned long end = jiffies +
454                 msecs_to_jiffies(NVMAP_CARVEOUT_KILLER_RETRY_TIME);
455         int count = 0;
456
457         do {
458                 block = do_nvmap_carveout_alloc(client, len, align, usage,
459                                                 prot, handle);
460                 if (!carveout_killer)
461                         return block;
462
463                 if (block)
464                         return block;
465
466                 if (!count++) {
467                         char task_comm[TASK_COMM_LEN];
468                         if (client->task)
469                                 get_task_comm(task_comm, client->task);
470                         else
471                                 task_comm[0] = 0;
472                         pr_info("%s: failed to allocate %u bytes for "
473                                 "process %s, firing carveout "
474                                 "killer!\n", __func__, len, task_comm);
475
476                 } else {
477                         pr_info("%s: still can't allocate %u bytes, "
478                                 "attempt %d!\n", __func__, len, count);
479                 }
480
481                 /* shrink carveouts that matter and try again */
482                 for (i = 0; i < dev->nr_carveouts; i++) {
483                         int count;
484                         co_heap = &dev->heaps[i];
485
486                         if (!(co_heap->heap_bit & usage))
487                                 continue;
488
489                         count = wait_count;
490                         /* indicates we didn't find anything to kill,
491                            might as well stop trying */
492                         if (!nvmap_shrink_carveout(co_heap))
493                                 return NULL;
494
495                         if (time_is_after_jiffies(end))
496                                 wait_event_interruptible_timeout(wait_reclaim,
497                                          nvmap_carveout_freed(count),
498                                          end - jiffies);
499                 }
500         } while (time_is_after_jiffies(end));
501
502         if (time_is_before_jiffies(end))
503                 pr_info("carveout_killer: timeout expired without "
504                         "allocation succeeding.\n");
505
506         return NULL;
507 }
508
509 /* remove a handle from the device's tree of all handles; called
510  * when freeing handles. */
511 int nvmap_handle_remove(struct nvmap_device *dev, struct nvmap_handle *h)
512 {
513         spin_lock(&dev->handle_lock);
514
515         /* re-test inside the spinlock if the handle really has no clients;
516          * only remove the handle if it is unreferenced */
517         if (atomic_add_return(0, &h->ref) > 0) {
518                 spin_unlock(&dev->handle_lock);
519                 return -EBUSY;
520         }
521         smp_rmb();
522         BUG_ON(atomic_read(&h->ref) < 0);
523         BUG_ON(atomic_read(&h->pin) != 0);
524
525         rb_erase(&h->node, &dev->handles);
526
527         spin_unlock(&dev->handle_lock);
528         return 0;
529 }
530
531 /* adds a newly-created handle to the device master tree */
532 void nvmap_handle_add(struct nvmap_device *dev, struct nvmap_handle *h)
533 {
534         struct rb_node **p;
535         struct rb_node *parent = NULL;
536
537         spin_lock(&dev->handle_lock);
538         p = &dev->handles.rb_node;
539         while (*p) {
540                 struct nvmap_handle *b;
541
542                 parent = *p;
543                 b = rb_entry(parent, struct nvmap_handle, node);
544                 if (h > b)
545                         p = &parent->rb_right;
546                 else
547                         p = &parent->rb_left;
548         }
549         rb_link_node(&h->node, parent, p);
550         rb_insert_color(&h->node, &dev->handles);
551         spin_unlock(&dev->handle_lock);
552 }
553
554 /* validates that a handle is in the device master tree, and that the
555  * client has permission to access it */
556 struct nvmap_handle *nvmap_validate_get(struct nvmap_client *client,
557                                         unsigned long id)
558 {
559         struct nvmap_handle *h = NULL;
560         struct rb_node *n;
561
562         spin_lock(&client->dev->handle_lock);
563
564         n = client->dev->handles.rb_node;
565
566         while (n) {
567                 h = rb_entry(n, struct nvmap_handle, node);
568                 if ((unsigned long)h == id) {
569                         if (client->super || h->global || (h->owner == client))
570                                 h = nvmap_handle_get(h);
571                         else
572                                 h = NULL;
573                         spin_unlock(&client->dev->handle_lock);
574                         return h;
575                 }
576                 if (id > (unsigned long)h)
577                         n = n->rb_right;
578                 else
579                         n = n->rb_left;
580         }
581         spin_unlock(&client->dev->handle_lock);
582         return NULL;
583 }
584
585 struct nvmap_client *nvmap_create_client(struct nvmap_device *dev,
586                                          const char *name)
587 {
588         struct nvmap_client *client;
589         struct task_struct *task;
590         int i;
591
592         if (WARN_ON(!dev))
593                 return NULL;
594
595         client = kzalloc(sizeof(*client) + (sizeof(struct nvmap_carveout_commit)
596                          * dev->nr_carveouts), GFP_KERNEL);
597         if (!client)
598                 return NULL;
599
600         client->name = name;
601         client->super = true;
602         client->dev = dev;
603         /* TODO: allocate unique IOVMM client for each nvmap client */
604         client->share = &dev->iovmm_master;
605         client->handle_refs = RB_ROOT;
606
607         atomic_set(&client->iovm_commit, 0);
608
609         client->iovm_limit = nvmap_mru_vm_size(client->share->iovmm);
610
611         for (i = 0; i < dev->nr_carveouts; i++) {
612                 INIT_LIST_HEAD(&client->carveout_commit[i].list);
613                 client->carveout_commit[i].commit = 0;
614         }
615
616         get_task_struct(current->group_leader);
617         task_lock(current->group_leader);
618         /* don't bother to store task struct for kernel threads,
619            they can't be killed anyway */
620         if (current->flags & PF_KTHREAD) {
621                 put_task_struct(current->group_leader);
622                 task = NULL;
623         } else {
624                 task = current->group_leader;
625         }
626         task_unlock(current->group_leader);
627         client->task = task;
628
629         spin_lock_init(&client->ref_lock);
630         atomic_set(&client->count, 1);
631
632         return client;
633 }
634
635 static void destroy_client(struct nvmap_client *client)
636 {
637         struct rb_node *n;
638         int i;
639
640         if (!client)
641                 return;
642
643
644         while ((n = rb_first(&client->handle_refs))) {
645                 struct nvmap_handle_ref *ref;
646                 int pins, dupes;
647
648                 ref = rb_entry(n, struct nvmap_handle_ref, node);
649                 rb_erase(&ref->node, &client->handle_refs);
650
651                 smp_rmb();
652                 pins = atomic_read(&ref->pin);
653
654                 spin_lock(&ref->handle->lock);
655                 if (ref->handle->owner == client)
656                     ref->handle->owner = NULL;
657                 spin_unlock(&ref->handle->lock);
658
659                 while (pins--)
660                         nvmap_unpin_handles(client, &ref->handle, 1);
661
662                 dupes = atomic_read(&ref->dupes);
663                 while (dupes--)
664                         nvmap_handle_put(ref->handle);
665
666                 kfree(ref);
667         }
668
669         if (carveout_killer) {
670                 wait_count++;
671                 smp_wmb();
672                 wake_up_all(&wait_reclaim);
673         }
674
675         for (i = 0; i < client->dev->nr_carveouts; i++)
676                 list_del(&client->carveout_commit[i].list);
677
678         if (client->task)
679                 put_task_struct(client->task);
680
681         kfree(client);
682 }
683
684 struct nvmap_client *nvmap_client_get(struct nvmap_client *client)
685 {
686         if (WARN_ON(!client))
687                 return NULL;
688
689         if (WARN_ON(!atomic_add_unless(&client->count, 1, 0)))
690                 return NULL;
691
692         return client;
693 }
694
695 struct nvmap_client *nvmap_client_get_file(int fd)
696 {
697         struct nvmap_client *client = ERR_PTR(-EFAULT);
698         struct file *f = fget(fd);
699         if (!f)
700                 return ERR_PTR(-EINVAL);
701
702         if ((f->f_op == &nvmap_user_fops) || (f->f_op == &nvmap_super_fops)) {
703                 client = f->private_data;
704                 atomic_inc(&client->count);
705         }
706
707         fput(f);
708         return client;
709 }
710
711 void nvmap_client_put(struct nvmap_client *client)
712 {
713         if (!client)
714                 return;
715
716         if (!atomic_dec_return(&client->count))
717                 destroy_client(client);
718 }
719
720 static int nvmap_open(struct inode *inode, struct file *filp)
721 {
722         struct miscdevice *miscdev = filp->private_data;
723         struct nvmap_device *dev = dev_get_drvdata(miscdev->parent);
724         struct nvmap_client *priv;
725         int ret;
726
727         ret = nonseekable_open(inode, filp);
728         if (unlikely(ret))
729                 return ret;
730
731         BUG_ON(dev != nvmap_dev);
732         priv = nvmap_create_client(dev, "user");
733         if (!priv)
734                 return -ENOMEM;
735
736         priv->super = (filp->f_op == &nvmap_super_fops);
737
738         filp->f_mapping->backing_dev_info = &nvmap_bdi;
739
740         filp->private_data = priv;
741         return 0;
742 }
743
744 static int nvmap_release(struct inode *inode, struct file *filp)
745 {
746         nvmap_client_put(filp->private_data);
747         return 0;
748 }
749
750 static int nvmap_map(struct file *filp, struct vm_area_struct *vma)
751 {
752         struct nvmap_vma_priv *priv;
753
754         /* after NVMAP_IOC_MMAP, the handle that is mapped by this VMA
755          * will be stored in vm_private_data and faulted in. until the
756          * ioctl is made, the VMA is mapped no-access */
757         vma->vm_private_data = NULL;
758
759         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
760         if (!priv)
761                 return -ENOMEM;
762
763         priv->offs = 0;
764         priv->handle = NULL;
765         atomic_set(&priv->count, 1);
766
767         vma->vm_flags |= VM_SHARED;
768         vma->vm_flags |= (VM_IO | VM_DONTEXPAND | VM_MIXEDMAP | VM_RESERVED);
769         vma->vm_ops = &nvmap_vma_ops;
770         vma->vm_private_data = priv;
771
772         return 0;
773 }
774
775 static long nvmap_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
776 {
777         int err = 0;
778         void __user *uarg = (void __user *)arg;
779
780         if (_IOC_TYPE(cmd) != NVMAP_IOC_MAGIC)
781                 return -ENOTTY;
782
783         if (_IOC_NR(cmd) > NVMAP_IOC_MAXNR)
784                 return -ENOTTY;
785
786         if (_IOC_DIR(cmd) & _IOC_READ)
787                 err = !access_ok(VERIFY_WRITE, uarg, _IOC_SIZE(cmd));
788         if (_IOC_DIR(cmd) & _IOC_WRITE)
789                 err = !access_ok(VERIFY_READ, uarg, _IOC_SIZE(cmd));
790
791         if (err)
792                 return -EFAULT;
793
794         switch (cmd) {
795         case NVMAP_IOC_CLAIM:
796                 nvmap_warn(filp->private_data, "preserved handles not"
797                            "supported\n");
798                 err = -ENODEV;
799                 break;
800         case NVMAP_IOC_CREATE:
801         case NVMAP_IOC_FROM_ID:
802                 err = nvmap_ioctl_create(filp, cmd, uarg);
803                 break;
804
805         case NVMAP_IOC_GET_ID:
806                 err = nvmap_ioctl_getid(filp, uarg);
807                 break;
808
809         case NVMAP_IOC_PARAM:
810                 err = nvmap_ioctl_get_param(filp, uarg);
811                 break;
812
813         case NVMAP_IOC_UNPIN_MULT:
814         case NVMAP_IOC_PIN_MULT:
815                 err = nvmap_ioctl_pinop(filp, cmd == NVMAP_IOC_PIN_MULT, uarg);
816                 break;
817
818         case NVMAP_IOC_ALLOC:
819                 err = nvmap_ioctl_alloc(filp, uarg);
820                 break;
821
822         case NVMAP_IOC_FREE:
823                 err = nvmap_ioctl_free(filp, arg);
824                 break;
825
826         case NVMAP_IOC_MMAP:
827                 err = nvmap_map_into_caller_ptr(filp, uarg);
828                 break;
829
830         case NVMAP_IOC_WRITE:
831         case NVMAP_IOC_READ:
832                 err = nvmap_ioctl_rw_handle(filp, cmd == NVMAP_IOC_READ, uarg);
833                 break;
834
835         case NVMAP_IOC_CACHE:
836                 err = nvmap_ioctl_cache_maint(filp, uarg);
837                 break;
838
839         default:
840                 return -ENOTTY;
841         }
842         return err;
843 }
844
845 /* to ensure that the backing store for the VMA isn't freed while a fork'd
846  * reference still exists, nvmap_vma_open increments the reference count on
847  * the handle, and nvmap_vma_close decrements it. alternatively, we could
848  * disallow copying of the vma, or behave like pmem and zap the pages. FIXME.
849 */
850 static void nvmap_vma_open(struct vm_area_struct *vma)
851 {
852         struct nvmap_vma_priv *priv;
853
854         priv = vma->vm_private_data;
855
856         BUG_ON(!priv);
857
858         atomic_inc(&priv->count);
859 }
860
861 static void nvmap_vma_close(struct vm_area_struct *vma)
862 {
863         struct nvmap_vma_priv *priv = vma->vm_private_data;
864
865         if (priv) {
866                 if (priv->handle) {
867                         nvmap_usecount_dec(priv->handle);
868                         BUG_ON(priv->handle->usecount < 0);
869                 }
870                 if (!atomic_dec_return(&priv->count)) {
871                         if (priv->handle)
872                                 nvmap_handle_put(priv->handle);
873                         kfree(priv);
874                 }
875         }
876         vma->vm_private_data = NULL;
877 }
878
879 static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
880 {
881         struct nvmap_vma_priv *priv;
882         unsigned long offs;
883
884         offs = (unsigned long)(vmf->virtual_address - vma->vm_start);
885         priv = vma->vm_private_data;
886         if (!priv || !priv->handle || !priv->handle->alloc)
887                 return VM_FAULT_SIGBUS;
888
889         offs += priv->offs;
890         /* if the VMA was split for some reason, vm_pgoff will be the VMA's
891          * offset from the original VMA */
892         offs += (vma->vm_pgoff << PAGE_SHIFT);
893
894         if (offs >= priv->handle->size)
895                 return VM_FAULT_SIGBUS;
896
897         if (!priv->handle->heap_pgalloc) {
898                 unsigned long pfn;
899                 BUG_ON(priv->handle->carveout->base & ~PAGE_MASK);
900                 pfn = ((priv->handle->carveout->base + offs) >> PAGE_SHIFT);
901                 vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
902                 return VM_FAULT_NOPAGE;
903         } else {
904                 struct page *page;
905                 offs >>= PAGE_SHIFT;
906                 page = priv->handle->pgalloc.pages[offs];
907                 if (page)
908                         get_page(page);
909                 vmf->page = page;
910                 return (page) ? 0 : VM_FAULT_SIGBUS;
911         }
912 }
913
914 static ssize_t attr_show_usage(struct device *dev,
915                                struct device_attribute *attr, char *buf)
916 {
917         struct nvmap_carveout_node *node = nvmap_heap_device_to_arg(dev);
918
919         return sprintf(buf, "%08x\n", node->heap_bit);
920 }
921
922 static struct device_attribute heap_attr_show_usage =
923         __ATTR(usage, S_IRUGO, attr_show_usage, NULL);
924
925 static struct attribute *heap_extra_attrs[] = {
926         &heap_attr_show_usage.attr,
927         NULL,
928 };
929
930 static struct attribute_group heap_extra_attr_group = {
931         .attrs = heap_extra_attrs,
932 };
933
934 static void client_stringify(struct nvmap_client *client, struct seq_file *s)
935 {
936         char task_comm[TASK_COMM_LEN];
937         if (!client->task) {
938                 seq_printf(s, "%8s %16s %8u", client->name, "kernel", 0);
939                 return;
940         }
941         get_task_comm(task_comm, client->task);
942         seq_printf(s, "%8s %16s %8u", client->name, task_comm,
943                    client->task->pid);
944 }
945
946 static void allocations_stringify(struct nvmap_client *client,
947                                   struct seq_file *s)
948 {
949         struct rb_node *n = rb_first(&client->handle_refs);
950         unsigned long long total = 0;
951
952         for (; n != NULL; n = rb_next(n)) {
953                 struct nvmap_handle_ref *ref =
954                         rb_entry(n, struct nvmap_handle_ref, node);
955                 struct nvmap_handle *handle = ref->handle;
956                 if (handle->alloc && !handle->heap_pgalloc) {
957                         seq_printf(s, " %8u@%8lx ", handle->size,
958                                    handle->carveout->base);
959                         total += handle->size;
960                 }
961         }
962         seq_printf(s, " total: %llu\n", total);
963 }
964
965 static int nvmap_debug_allocations_show(struct seq_file *s, void *unused)
966 {
967         struct nvmap_carveout_node *node = s->private;
968         struct nvmap_carveout_commit *commit;
969         unsigned long flags;
970
971         spin_lock_irqsave(&node->clients_lock, flags);
972         list_for_each_entry(commit, &node->clients, list) {
973                 struct nvmap_client *client =
974                         get_client_from_carveout_commit(node, commit);
975                 client_stringify(client, s);
976                 allocations_stringify(client, s);
977         }
978         spin_unlock_irqrestore(&node->clients_lock, flags);
979
980         return 0;
981 }
982
983 static int nvmap_debug_allocations_open(struct inode *inode, struct file *file)
984 {
985         return single_open(file, nvmap_debug_allocations_show,
986                            inode->i_private);
987 }
988
989 static struct file_operations debug_allocations_fops = {
990         .open = nvmap_debug_allocations_open,
991         .read = seq_read,
992         .llseek = seq_lseek,
993         .release = single_release,
994 };
995
996 static int nvmap_debug_clients_show(struct seq_file *s, void *unused)
997 {
998         struct nvmap_carveout_node *node = s->private;
999         struct nvmap_carveout_commit *commit;
1000         unsigned long flags;
1001
1002         spin_lock_irqsave(&node->clients_lock, flags);
1003         list_for_each_entry(commit, &node->clients, list) {
1004                 struct nvmap_client *client =
1005                         get_client_from_carveout_commit(node, commit);
1006                 client_stringify(client, s);
1007                 seq_printf(s, " %8u\n", commit->commit);
1008         }
1009         spin_unlock_irqrestore(&node->clients_lock, flags);
1010
1011         return 0;
1012 }
1013
1014 static int nvmap_debug_clients_open(struct inode *inode, struct file *file)
1015 {
1016         return single_open(file, nvmap_debug_clients_show, inode->i_private);
1017 }
1018
1019 static struct file_operations debug_clients_fops = {
1020         .open = nvmap_debug_clients_open,
1021         .read = seq_read,
1022         .llseek = seq_lseek,
1023         .release = single_release,
1024 };
1025
1026 static int nvmap_probe(struct platform_device *pdev)
1027 {
1028         struct nvmap_platform_data *plat = pdev->dev.platform_data;
1029         struct nvmap_device *dev;
1030         struct dentry *nvmap_debug_root;
1031         unsigned int i;
1032         int e;
1033
1034         if (!plat) {
1035                 dev_err(&pdev->dev, "no platform data?\n");
1036                 return -ENODEV;
1037         }
1038
1039         if (WARN_ON(nvmap_dev != NULL)) {
1040                 dev_err(&pdev->dev, "only one nvmap device may be present\n");
1041                 return -ENODEV;
1042         }
1043
1044         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1045         if (!dev) {
1046                 dev_err(&pdev->dev, "out of memory for device\n");
1047                 return -ENOMEM;
1048         }
1049
1050         dev->dev_user.minor = MISC_DYNAMIC_MINOR;
1051         dev->dev_user.name = "nvmap";
1052         dev->dev_user.fops = &nvmap_user_fops;
1053         dev->dev_user.parent = &pdev->dev;
1054
1055         dev->dev_super.minor = MISC_DYNAMIC_MINOR;
1056         dev->dev_super.name = "knvmap";
1057         dev->dev_super.fops = &nvmap_super_fops;
1058         dev->dev_super.parent = &pdev->dev;
1059
1060         dev->handles = RB_ROOT;
1061
1062         init_waitqueue_head(&dev->pte_wait);
1063
1064         init_waitqueue_head(&dev->iovmm_master.pin_wait);
1065         mutex_init(&dev->iovmm_master.pin_lock);
1066         dev->iovmm_master.iovmm =
1067                 tegra_iovmm_alloc_client(dev_name(&pdev->dev), NULL);
1068         if (IS_ERR(dev->iovmm_master.iovmm)) {
1069                 e = PTR_ERR(dev->iovmm_master.iovmm);
1070                 dev_err(&pdev->dev, "couldn't create iovmm client\n");
1071                 goto fail;
1072         }
1073         dev->vm_rgn = alloc_vm_area(NVMAP_NUM_PTES * PAGE_SIZE);
1074         if (!dev->vm_rgn) {
1075                 e = -ENOMEM;
1076                 dev_err(&pdev->dev, "couldn't allocate remapping region\n");
1077                 goto fail;
1078         }
1079         e = nvmap_mru_init(&dev->iovmm_master);
1080         if (e) {
1081                 dev_err(&pdev->dev, "couldn't initialize MRU lists\n");
1082                 goto fail;
1083         }
1084
1085         spin_lock_init(&dev->ptelock);
1086         spin_lock_init(&dev->handle_lock);
1087
1088         for (i = 0; i < NVMAP_NUM_PTES; i++) {
1089                 unsigned long addr;
1090                 pgd_t *pgd;
1091                 pud_t *pud;
1092                 pmd_t *pmd;
1093
1094                 addr = (unsigned long)dev->vm_rgn->addr + (i * PAGE_SIZE);
1095                 pgd = pgd_offset_k(addr);
1096                 pud = pud_alloc(&init_mm, pgd, addr);
1097                 if (!pud) {
1098                         e = -ENOMEM;
1099                         dev_err(&pdev->dev, "couldn't allocate page tables\n");
1100                         goto fail;
1101                 }
1102                 pmd = pmd_alloc(&init_mm, pud, addr);
1103                 if (!pmd) {
1104                         e = -ENOMEM;
1105                         dev_err(&pdev->dev, "couldn't allocate page tables\n");
1106                         goto fail;
1107                 }
1108                 dev->ptes[i] = pte_alloc_kernel(pmd, addr);
1109                 if (!dev->ptes[i]) {
1110                         e = -ENOMEM;
1111                         dev_err(&pdev->dev, "couldn't allocate page tables\n");
1112                         goto fail;
1113                 }
1114         }
1115
1116         e = misc_register(&dev->dev_user);
1117         if (e) {
1118                 dev_err(&pdev->dev, "unable to register miscdevice %s\n",
1119                         dev->dev_user.name);
1120                 goto fail;
1121         }
1122
1123         e = misc_register(&dev->dev_super);
1124         if (e) {
1125                 dev_err(&pdev->dev, "unable to register miscdevice %s\n",
1126                         dev->dev_super.name);
1127                 goto fail;
1128         }
1129
1130         dev->nr_carveouts = 0;
1131         dev->heaps = kzalloc(sizeof(struct nvmap_carveout_node) *
1132                              plat->nr_carveouts, GFP_KERNEL);
1133         if (!dev->heaps) {
1134                 e = -ENOMEM;
1135                 dev_err(&pdev->dev, "couldn't allocate carveout memory\n");
1136                 goto fail;
1137         }
1138
1139         nvmap_debug_root = debugfs_create_dir("nvmap", NULL);
1140         if (IS_ERR_OR_NULL(nvmap_debug_root))
1141                 dev_err(&pdev->dev, "couldn't create debug files\n");
1142
1143         for (i = 0; i < plat->nr_carveouts; i++) {
1144                 struct nvmap_carveout_node *node = &dev->heaps[i];
1145                 const struct nvmap_platform_carveout *co = &plat->carveouts[i];
1146                 node->carveout = nvmap_heap_create(dev->dev_user.this_device,
1147                                    co->name, co->base, co->size,
1148                                    co->buddy_size, node);
1149                 if (!node->carveout) {
1150                         e = -ENOMEM;
1151                         dev_err(&pdev->dev, "couldn't create %s\n", co->name);
1152                         goto fail_heaps;
1153                 }
1154                 dev->nr_carveouts++;
1155                 spin_lock_init(&node->clients_lock);
1156                 node->index = i;
1157                 INIT_LIST_HEAD(&node->clients);
1158                 node->heap_bit = co->usage_mask;
1159                 if (nvmap_heap_create_group(node->carveout,
1160                                             &heap_extra_attr_group))
1161                         dev_warn(&pdev->dev, "couldn't add extra attributes\n");
1162
1163                 dev_info(&pdev->dev, "created carveout %s (%uKiB)\n",
1164                          co->name, co->size / 1024);
1165
1166                 if (!IS_ERR_OR_NULL(nvmap_debug_root)) {
1167                         struct dentry *heap_root =
1168                                 debugfs_create_dir(co->name, nvmap_debug_root);
1169                         if (!IS_ERR_OR_NULL(heap_root)) {
1170                                 debugfs_create_file("clients", 0664, heap_root,
1171                                     node, &debug_clients_fops);
1172                                 debugfs_create_file("allocations", 0664,
1173                                     heap_root, node, &debug_allocations_fops);
1174                         }
1175                 }
1176         }
1177
1178         platform_set_drvdata(pdev, dev);
1179         nvmap_dev = dev;
1180         return 0;
1181 fail_heaps:
1182         for (i = 0; i < dev->nr_carveouts; i++) {
1183                 struct nvmap_carveout_node *node = &dev->heaps[i];
1184                 nvmap_heap_remove_group(node->carveout, &heap_extra_attr_group);
1185                 nvmap_heap_destroy(node->carveout);
1186         }
1187 fail:
1188         kfree(dev->heaps);
1189         nvmap_mru_destroy(&dev->iovmm_master);
1190         if (dev->dev_super.minor != MISC_DYNAMIC_MINOR)
1191                 misc_deregister(&dev->dev_super);
1192         if (dev->dev_user.minor != MISC_DYNAMIC_MINOR)
1193                 misc_deregister(&dev->dev_user);
1194         if (!IS_ERR_OR_NULL(dev->iovmm_master.iovmm))
1195                 tegra_iovmm_free_client(dev->iovmm_master.iovmm);
1196         if (dev->vm_rgn)
1197                 free_vm_area(dev->vm_rgn);
1198         kfree(dev);
1199         nvmap_dev = NULL;
1200         return e;
1201 }
1202
1203 static int nvmap_remove(struct platform_device *pdev)
1204 {
1205         struct nvmap_device *dev = platform_get_drvdata(pdev);
1206         struct rb_node *n;
1207         struct nvmap_handle *h;
1208         int i;
1209
1210         misc_deregister(&dev->dev_super);
1211         misc_deregister(&dev->dev_user);
1212
1213         while ((n = rb_first(&dev->handles))) {
1214                 h = rb_entry(n, struct nvmap_handle, node);
1215                 rb_erase(&h->node, &dev->handles);
1216                 kfree(h);
1217         }
1218
1219         if (!IS_ERR_OR_NULL(dev->iovmm_master.iovmm))
1220                 tegra_iovmm_free_client(dev->iovmm_master.iovmm);
1221
1222         nvmap_mru_destroy(&dev->iovmm_master);
1223
1224         for (i = 0; i < dev->nr_carveouts; i++) {
1225                 struct nvmap_carveout_node *node = &dev->heaps[i];
1226                 nvmap_heap_remove_group(node->carveout, &heap_extra_attr_group);
1227                 nvmap_heap_destroy(node->carveout);
1228         }
1229         kfree(dev->heaps);
1230
1231         free_vm_area(dev->vm_rgn);
1232         kfree(dev);
1233         nvmap_dev = NULL;
1234         return 0;
1235 }
1236
1237 static int nvmap_suspend(struct platform_device *pdev, pm_message_t state)
1238 {
1239         return 0;
1240 }
1241
1242 static int nvmap_resume(struct platform_device *pdev)
1243 {
1244         return 0;
1245 }
1246
1247 static struct platform_driver nvmap_driver = {
1248         .probe          = nvmap_probe,
1249         .remove         = nvmap_remove,
1250         .suspend        = nvmap_suspend,
1251         .resume         = nvmap_resume,
1252
1253         .driver = {
1254                 .name   = "tegra-nvmap",
1255                 .owner  = THIS_MODULE,
1256         },
1257 };
1258
1259 static int __init nvmap_init_driver(void)
1260 {
1261         int e;
1262
1263         nvmap_dev = NULL;
1264
1265         e = nvmap_heap_init();
1266         if (e)
1267                 goto fail;
1268
1269         e = platform_driver_register(&nvmap_driver);
1270         if (e) {
1271                 nvmap_heap_deinit();
1272                 goto fail;
1273         }
1274
1275 fail:
1276         return e;
1277 }
1278 fs_initcall(nvmap_init_driver);
1279
1280 static void __exit nvmap_exit_driver(void)
1281 {
1282         platform_driver_unregister(&nvmap_driver);
1283         nvmap_heap_deinit();
1284         nvmap_dev = NULL;
1285 }
1286 module_exit(nvmap_exit_driver);