drm/radeon/kms: add initial colortiling support.
[linux-2.6.git] / drivers / gpu / drm / ttm / ttm_bo_vm.c
1 /**************************************************************************
2  *
3  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
4  * All Rights Reserved.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the
8  * "Software"), to deal in the Software without restriction, including
9  * without limitation the rights to use, copy, modify, merge, publish,
10  * distribute, sub license, and/or sell copies of the Software, and to
11  * permit persons to whom the Software is furnished to do so, subject to
12  * the following conditions:
13  *
14  * The above copyright notice and this permission notice (including the
15  * next paragraph) shall be included in all copies or substantial portions
16  * of the Software.
17  *
18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
25  *
26  **************************************************************************/
27 /*
28  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
29  */
30
31 #include <ttm/ttm_module.h>
32 #include <ttm/ttm_bo_driver.h>
33 #include <ttm/ttm_placement.h>
34 #include <linux/mm.h>
35 #include <linux/rbtree.h>
36 #include <linux/module.h>
37 #include <linux/uaccess.h>
38
39 #define TTM_BO_VM_NUM_PREFAULT 16
40
41 static struct ttm_buffer_object *ttm_bo_vm_lookup_rb(struct ttm_bo_device *bdev,
42                                                      unsigned long page_start,
43                                                      unsigned long num_pages)
44 {
45         struct rb_node *cur = bdev->addr_space_rb.rb_node;
46         unsigned long cur_offset;
47         struct ttm_buffer_object *bo;
48         struct ttm_buffer_object *best_bo = NULL;
49
50         while (likely(cur != NULL)) {
51                 bo = rb_entry(cur, struct ttm_buffer_object, vm_rb);
52                 cur_offset = bo->vm_node->start;
53                 if (page_start >= cur_offset) {
54                         cur = cur->rb_right;
55                         best_bo = bo;
56                         if (page_start == cur_offset)
57                                 break;
58                 } else
59                         cur = cur->rb_left;
60         }
61
62         if (unlikely(best_bo == NULL))
63                 return NULL;
64
65         if (unlikely((best_bo->vm_node->start + best_bo->num_pages) <
66                      (page_start + num_pages)))
67                 return NULL;
68
69         return best_bo;
70 }
71
72 static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
73 {
74         struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
75             vma->vm_private_data;
76         struct ttm_bo_device *bdev = bo->bdev;
77         unsigned long bus_base;
78         unsigned long bus_offset;
79         unsigned long bus_size;
80         unsigned long page_offset;
81         unsigned long page_last;
82         unsigned long pfn;
83         struct ttm_tt *ttm = NULL;
84         struct page *page;
85         int ret;
86         int i;
87         bool is_iomem;
88         unsigned long address = (unsigned long)vmf->virtual_address;
89         int retval = VM_FAULT_NOPAGE;
90
91         /*
92          * Work around locking order reversal in fault / nopfn
93          * between mmap_sem and bo_reserve: Perform a trylock operation
94          * for reserve, and if it fails, retry the fault after scheduling.
95          */
96
97         ret = ttm_bo_reserve(bo, true, true, false, 0);
98         if (unlikely(ret != 0)) {
99                 if (ret == -EBUSY)
100                         set_need_resched();
101                 return VM_FAULT_NOPAGE;
102         }
103
104         if (bdev->driver->fault_reserve_notify)
105                 bdev->driver->fault_reserve_notify(bo);
106
107         /*
108          * Wait for buffer data in transit, due to a pipelined
109          * move.
110          */
111
112         spin_lock(&bo->lock);
113         if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) {
114                 ret = ttm_bo_wait(bo, false, true, false);
115                 spin_unlock(&bo->lock);
116                 if (unlikely(ret != 0)) {
117                         retval = (ret != -ERESTART) ?
118                             VM_FAULT_SIGBUS : VM_FAULT_NOPAGE;
119                         goto out_unlock;
120                 }
121         } else
122                 spin_unlock(&bo->lock);
123
124
125         ret = ttm_bo_pci_offset(bdev, &bo->mem, &bus_base, &bus_offset,
126                                 &bus_size);
127         if (unlikely(ret != 0)) {
128                 retval = VM_FAULT_SIGBUS;
129                 goto out_unlock;
130         }
131
132         is_iomem = (bus_size != 0);
133
134         page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) +
135             bo->vm_node->start - vma->vm_pgoff;
136         page_last = ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) +
137             bo->vm_node->start - vma->vm_pgoff;
138
139         if (unlikely(page_offset >= bo->num_pages)) {
140                 retval = VM_FAULT_SIGBUS;
141                 goto out_unlock;
142         }
143
144         /*
145          * Strictly, we're not allowed to modify vma->vm_page_prot here,
146          * since the mmap_sem is only held in read mode. However, we
147          * modify only the caching bits of vma->vm_page_prot and
148          * consider those bits protected by
149          * the bo->mutex, as we should be the only writers.
150          * There shouldn't really be any readers of these bits except
151          * within vm_insert_mixed()? fork?
152          *
153          * TODO: Add a list of vmas to the bo, and change the
154          * vma->vm_page_prot when the object changes caching policy, with
155          * the correct locks held.
156          */
157
158         if (is_iomem) {
159                 vma->vm_page_prot = ttm_io_prot(bo->mem.placement,
160                                                 vma->vm_page_prot);
161         } else {
162                 ttm = bo->ttm;
163                 vma->vm_page_prot = (bo->mem.placement & TTM_PL_FLAG_CACHED) ?
164                     vm_get_page_prot(vma->vm_flags) :
165                     ttm_io_prot(bo->mem.placement, vma->vm_page_prot);
166         }
167
168         /*
169          * Speculatively prefault a number of pages. Only error on
170          * first page.
171          */
172
173         for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) {
174
175                 if (is_iomem)
176                         pfn = ((bus_base + bus_offset) >> PAGE_SHIFT) +
177                             page_offset;
178                 else {
179                         page = ttm_tt_get_page(ttm, page_offset);
180                         if (unlikely(!page && i == 0)) {
181                                 retval = VM_FAULT_OOM;
182                                 goto out_unlock;
183                         } else if (unlikely(!page)) {
184                                 break;
185                         }
186                         pfn = page_to_pfn(page);
187                 }
188
189                 ret = vm_insert_mixed(vma, address, pfn);
190                 /*
191                  * Somebody beat us to this PTE or prefaulting to
192                  * an already populated PTE, or prefaulting error.
193                  */
194
195                 if (unlikely((ret == -EBUSY) || (ret != 0 && i > 0)))
196                         break;
197                 else if (unlikely(ret != 0)) {
198                         retval =
199                             (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS;
200                         goto out_unlock;
201
202                 }
203
204                 address += PAGE_SIZE;
205                 if (unlikely(++page_offset >= page_last))
206                         break;
207         }
208
209 out_unlock:
210         ttm_bo_unreserve(bo);
211         return retval;
212 }
213
214 static void ttm_bo_vm_open(struct vm_area_struct *vma)
215 {
216         struct ttm_buffer_object *bo =
217             (struct ttm_buffer_object *)vma->vm_private_data;
218
219         (void)ttm_bo_reference(bo);
220 }
221
222 static void ttm_bo_vm_close(struct vm_area_struct *vma)
223 {
224         struct ttm_buffer_object *bo =
225             (struct ttm_buffer_object *)vma->vm_private_data;
226
227         ttm_bo_unref(&bo);
228         vma->vm_private_data = NULL;
229 }
230
231 static struct vm_operations_struct ttm_bo_vm_ops = {
232         .fault = ttm_bo_vm_fault,
233         .open = ttm_bo_vm_open,
234         .close = ttm_bo_vm_close
235 };
236
237 int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma,
238                 struct ttm_bo_device *bdev)
239 {
240         struct ttm_bo_driver *driver;
241         struct ttm_buffer_object *bo;
242         int ret;
243
244         read_lock(&bdev->vm_lock);
245         bo = ttm_bo_vm_lookup_rb(bdev, vma->vm_pgoff,
246                                  (vma->vm_end - vma->vm_start) >> PAGE_SHIFT);
247         if (likely(bo != NULL))
248                 ttm_bo_reference(bo);
249         read_unlock(&bdev->vm_lock);
250
251         if (unlikely(bo == NULL)) {
252                 printk(KERN_ERR TTM_PFX
253                        "Could not find buffer object to map.\n");
254                 return -EINVAL;
255         }
256
257         driver = bo->bdev->driver;
258         if (unlikely(!driver->verify_access)) {
259                 ret = -EPERM;
260                 goto out_unref;
261         }
262         ret = driver->verify_access(bo, filp);
263         if (unlikely(ret != 0))
264                 goto out_unref;
265
266         vma->vm_ops = &ttm_bo_vm_ops;
267
268         /*
269          * Note: We're transferring the bo reference to
270          * vma->vm_private_data here.
271          */
272
273         vma->vm_private_data = bo;
274         vma->vm_flags |= VM_RESERVED | VM_IO | VM_MIXEDMAP | VM_DONTEXPAND;
275         return 0;
276 out_unref:
277         ttm_bo_unref(&bo);
278         return ret;
279 }
280 EXPORT_SYMBOL(ttm_bo_mmap);
281
282 int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo)
283 {
284         if (vma->vm_pgoff != 0)
285                 return -EACCES;
286
287         vma->vm_ops = &ttm_bo_vm_ops;
288         vma->vm_private_data = ttm_bo_reference(bo);
289         vma->vm_flags |= VM_RESERVED | VM_IO | VM_MIXEDMAP | VM_DONTEXPAND;
290         return 0;
291 }
292 EXPORT_SYMBOL(ttm_fbdev_mmap);
293
294
295 ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp,
296                   const char __user *wbuf, char __user *rbuf, size_t count,
297                   loff_t *f_pos, bool write)
298 {
299         struct ttm_buffer_object *bo;
300         struct ttm_bo_driver *driver;
301         struct ttm_bo_kmap_obj map;
302         unsigned long dev_offset = (*f_pos >> PAGE_SHIFT);
303         unsigned long kmap_offset;
304         unsigned long kmap_end;
305         unsigned long kmap_num;
306         size_t io_size;
307         unsigned int page_offset;
308         char *virtual;
309         int ret;
310         bool no_wait = false;
311         bool dummy;
312
313         read_lock(&bdev->vm_lock);
314         bo = ttm_bo_vm_lookup_rb(bdev, dev_offset, 1);
315         if (likely(bo != NULL))
316                 ttm_bo_reference(bo);
317         read_unlock(&bdev->vm_lock);
318
319         if (unlikely(bo == NULL))
320                 return -EFAULT;
321
322         driver = bo->bdev->driver;
323         if (unlikely(driver->verify_access)) {
324                 ret = -EPERM;
325                 goto out_unref;
326         }
327
328         ret = driver->verify_access(bo, filp);
329         if (unlikely(ret != 0))
330                 goto out_unref;
331
332         kmap_offset = dev_offset - bo->vm_node->start;
333         if (unlikely(kmap_offset) >= bo->num_pages) {
334                 ret = -EFBIG;
335                 goto out_unref;
336         }
337
338         page_offset = *f_pos & ~PAGE_MASK;
339         io_size = bo->num_pages - kmap_offset;
340         io_size = (io_size << PAGE_SHIFT) - page_offset;
341         if (count < io_size)
342                 io_size = count;
343
344         kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT;
345         kmap_num = kmap_end - kmap_offset + 1;
346
347         ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
348
349         switch (ret) {
350         case 0:
351                 break;
352         case -ERESTART:
353                 ret = -EINTR;
354                 goto out_unref;
355         case -EBUSY:
356                 ret = -EAGAIN;
357                 goto out_unref;
358         default:
359                 goto out_unref;
360         }
361
362         ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map);
363         if (unlikely(ret != 0)) {
364                 ttm_bo_unreserve(bo);
365                 goto out_unref;
366         }
367
368         virtual = ttm_kmap_obj_virtual(&map, &dummy);
369         virtual += page_offset;
370
371         if (write)
372                 ret = copy_from_user(virtual, wbuf, io_size);
373         else
374                 ret = copy_to_user(rbuf, virtual, io_size);
375
376         ttm_bo_kunmap(&map);
377         ttm_bo_unreserve(bo);
378         ttm_bo_unref(&bo);
379
380         if (unlikely(ret != 0))
381                 return -EFBIG;
382
383         *f_pos += io_size;
384
385         return io_size;
386 out_unref:
387         ttm_bo_unref(&bo);
388         return ret;
389 }
390
391 ssize_t ttm_bo_fbdev_io(struct ttm_buffer_object *bo, const char __user *wbuf,
392                         char __user *rbuf, size_t count, loff_t *f_pos,
393                         bool write)
394 {
395         struct ttm_bo_kmap_obj map;
396         unsigned long kmap_offset;
397         unsigned long kmap_end;
398         unsigned long kmap_num;
399         size_t io_size;
400         unsigned int page_offset;
401         char *virtual;
402         int ret;
403         bool no_wait = false;
404         bool dummy;
405
406         kmap_offset = (*f_pos >> PAGE_SHIFT);
407         if (unlikely(kmap_offset) >= bo->num_pages)
408                 return -EFBIG;
409
410         page_offset = *f_pos & ~PAGE_MASK;
411         io_size = bo->num_pages - kmap_offset;
412         io_size = (io_size << PAGE_SHIFT) - page_offset;
413         if (count < io_size)
414                 io_size = count;
415
416         kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT;
417         kmap_num = kmap_end - kmap_offset + 1;
418
419         ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
420
421         switch (ret) {
422         case 0:
423                 break;
424         case -ERESTART:
425                 return -EINTR;
426         case -EBUSY:
427                 return -EAGAIN;
428         default:
429                 return ret;
430         }
431
432         ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map);
433         if (unlikely(ret != 0)) {
434                 ttm_bo_unreserve(bo);
435                 return ret;
436         }
437
438         virtual = ttm_kmap_obj_virtual(&map, &dummy);
439         virtual += page_offset;
440
441         if (write)
442                 ret = copy_from_user(virtual, wbuf, io_size);
443         else
444                 ret = copy_to_user(rbuf, virtual, io_size);
445
446         ttm_bo_kunmap(&map);
447         ttm_bo_unreserve(bo);
448         ttm_bo_unref(&bo);
449
450         if (unlikely(ret != 0))
451                 return ret;
452
453         *f_pos += io_size;
454
455         return io_size;
456 }