drm/radeon/kms: avoid corner case issue with unmappable vram V2
[linux-2.6.git] / drivers / gpu / drm / radeon / radeon_object.c
1 /*
2  * Copyright 2009 Jerome Glisse.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sub license, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
20  *
21  * The above copyright notice and this permission notice (including the
22  * next paragraph) shall be included in all copies or substantial portions
23  * of the Software.
24  *
25  */
26 /*
27  * Authors:
28  *    Jerome Glisse <glisse@freedesktop.org>
29  *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
30  *    Dave Airlie
31  */
32 #include <linux/list.h>
33 #include <linux/slab.h>
34 #include <drm/drmP.h>
35 #include "radeon_drm.h"
36 #include "radeon.h"
37
38
39 int radeon_ttm_init(struct radeon_device *rdev);
40 void radeon_ttm_fini(struct radeon_device *rdev);
41 static void radeon_bo_clear_surface_reg(struct radeon_bo *bo);
42
43 /*
44  * To exclude mutual BO access we rely on bo_reserve exclusion, as all
45  * function are calling it.
46  */
47
48 static void radeon_ttm_bo_destroy(struct ttm_buffer_object *tbo)
49 {
50         struct radeon_bo *bo;
51
52         bo = container_of(tbo, struct radeon_bo, tbo);
53         mutex_lock(&bo->rdev->gem.mutex);
54         list_del_init(&bo->list);
55         mutex_unlock(&bo->rdev->gem.mutex);
56         radeon_bo_clear_surface_reg(bo);
57         kfree(bo);
58 }
59
60 bool radeon_ttm_bo_is_radeon_bo(struct ttm_buffer_object *bo)
61 {
62         if (bo->destroy == &radeon_ttm_bo_destroy)
63                 return true;
64         return false;
65 }
66
67 void radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain)
68 {
69         u32 c = 0;
70
71         rbo->placement.fpfn = 0;
72         rbo->placement.lpfn = rbo->rdev->mc.active_vram_size >> PAGE_SHIFT;
73         rbo->placement.placement = rbo->placements;
74         rbo->placement.busy_placement = rbo->placements;
75         if (domain & RADEON_GEM_DOMAIN_VRAM)
76                 rbo->placements[c++] = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
77                                         TTM_PL_FLAG_VRAM;
78         if (domain & RADEON_GEM_DOMAIN_GTT)
79                 rbo->placements[c++] = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
80         if (domain & RADEON_GEM_DOMAIN_CPU)
81                 rbo->placements[c++] = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM;
82         if (!c)
83                 rbo->placements[c++] = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM;
84         rbo->placement.num_placement = c;
85         rbo->placement.num_busy_placement = c;
86 }
87
88 int radeon_bo_create(struct radeon_device *rdev, struct drm_gem_object *gobj,
89                         unsigned long size, bool kernel, u32 domain,
90                         struct radeon_bo **bo_ptr)
91 {
92         struct radeon_bo *bo;
93         enum ttm_bo_type type;
94         int r;
95
96         if (unlikely(rdev->mman.bdev.dev_mapping == NULL)) {
97                 rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping;
98         }
99         if (kernel) {
100                 type = ttm_bo_type_kernel;
101         } else {
102                 type = ttm_bo_type_device;
103         }
104         *bo_ptr = NULL;
105         bo = kzalloc(sizeof(struct radeon_bo), GFP_KERNEL);
106         if (bo == NULL)
107                 return -ENOMEM;
108         bo->rdev = rdev;
109         bo->gobj = gobj;
110         bo->surface_reg = -1;
111         INIT_LIST_HEAD(&bo->list);
112
113 retry:
114         radeon_ttm_placement_from_domain(bo, domain);
115         /* Kernel allocation are uninterruptible */
116         mutex_lock(&rdev->vram_mutex);
117         r = ttm_bo_init(&rdev->mman.bdev, &bo->tbo, size, type,
118                         &bo->placement, 0, 0, !kernel, NULL, size,
119                         &radeon_ttm_bo_destroy);
120         mutex_unlock(&rdev->vram_mutex);
121         if (unlikely(r != 0)) {
122                 if (r != -ERESTARTSYS) {
123                         if (domain == RADEON_GEM_DOMAIN_VRAM) {
124                                 domain |= RADEON_GEM_DOMAIN_GTT;
125                                 goto retry;
126                         }
127                         dev_err(rdev->dev,
128                                 "object_init failed for (%lu, 0x%08X)\n",
129                                 size, domain);
130                 }
131                 return r;
132         }
133         *bo_ptr = bo;
134         if (gobj) {
135                 mutex_lock(&bo->rdev->gem.mutex);
136                 list_add_tail(&bo->list, &rdev->gem.objects);
137                 mutex_unlock(&bo->rdev->gem.mutex);
138         }
139         return 0;
140 }
141
142 int radeon_bo_kmap(struct radeon_bo *bo, void **ptr)
143 {
144         bool is_iomem;
145         int r;
146
147         if (bo->kptr) {
148                 if (ptr) {
149                         *ptr = bo->kptr;
150                 }
151                 return 0;
152         }
153         r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap);
154         if (r) {
155                 return r;
156         }
157         bo->kptr = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
158         if (ptr) {
159                 *ptr = bo->kptr;
160         }
161         radeon_bo_check_tiling(bo, 0, 0);
162         return 0;
163 }
164
165 void radeon_bo_kunmap(struct radeon_bo *bo)
166 {
167         if (bo->kptr == NULL)
168                 return;
169         bo->kptr = NULL;
170         radeon_bo_check_tiling(bo, 0, 0);
171         ttm_bo_kunmap(&bo->kmap);
172 }
173
174 void radeon_bo_unref(struct radeon_bo **bo)
175 {
176         struct ttm_buffer_object *tbo;
177         struct radeon_device *rdev;
178
179         if ((*bo) == NULL)
180                 return;
181         rdev = (*bo)->rdev;
182         tbo = &((*bo)->tbo);
183         mutex_lock(&rdev->vram_mutex);
184         ttm_bo_unref(&tbo);
185         mutex_unlock(&rdev->vram_mutex);
186         if (tbo == NULL)
187                 *bo = NULL;
188 }
189
190 int radeon_bo_pin(struct radeon_bo *bo, u32 domain, u64 *gpu_addr)
191 {
192         int r, i;
193
194         if (bo->pin_count) {
195                 bo->pin_count++;
196                 if (gpu_addr)
197                         *gpu_addr = radeon_bo_gpu_offset(bo);
198                 return 0;
199         }
200         radeon_ttm_placement_from_domain(bo, domain);
201         if (domain == RADEON_GEM_DOMAIN_VRAM) {
202                 /* force to pin into visible video ram */
203                 bo->placement.lpfn = bo->rdev->mc.visible_vram_size >> PAGE_SHIFT;
204         }
205         for (i = 0; i < bo->placement.num_placement; i++)
206                 bo->placements[i] |= TTM_PL_FLAG_NO_EVICT;
207         r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false, false);
208         if (likely(r == 0)) {
209                 bo->pin_count = 1;
210                 if (gpu_addr != NULL)
211                         *gpu_addr = radeon_bo_gpu_offset(bo);
212         }
213         if (unlikely(r != 0))
214                 dev_err(bo->rdev->dev, "%p pin failed\n", bo);
215         return r;
216 }
217
218 int radeon_bo_unpin(struct radeon_bo *bo)
219 {
220         int r, i;
221
222         if (!bo->pin_count) {
223                 dev_warn(bo->rdev->dev, "%p unpin not necessary\n", bo);
224                 return 0;
225         }
226         bo->pin_count--;
227         if (bo->pin_count)
228                 return 0;
229         for (i = 0; i < bo->placement.num_placement; i++)
230                 bo->placements[i] &= ~TTM_PL_FLAG_NO_EVICT;
231         r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false, false);
232         if (unlikely(r != 0))
233                 dev_err(bo->rdev->dev, "%p validate failed for unpin\n", bo);
234         return r;
235 }
236
237 int radeon_bo_evict_vram(struct radeon_device *rdev)
238 {
239         /* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */
240         if (0 && (rdev->flags & RADEON_IS_IGP)) {
241                 if (rdev->mc.igp_sideport_enabled == false)
242                         /* Useless to evict on IGP chips */
243                         return 0;
244         }
245         return ttm_bo_evict_mm(&rdev->mman.bdev, TTM_PL_VRAM);
246 }
247
248 void radeon_bo_force_delete(struct radeon_device *rdev)
249 {
250         struct radeon_bo *bo, *n;
251         struct drm_gem_object *gobj;
252
253         if (list_empty(&rdev->gem.objects)) {
254                 return;
255         }
256         dev_err(rdev->dev, "Userspace still has active objects !\n");
257         list_for_each_entry_safe(bo, n, &rdev->gem.objects, list) {
258                 mutex_lock(&rdev->ddev->struct_mutex);
259                 gobj = bo->gobj;
260                 dev_err(rdev->dev, "%p %p %lu %lu force free\n",
261                         gobj, bo, (unsigned long)gobj->size,
262                         *((unsigned long *)&gobj->refcount));
263                 mutex_lock(&bo->rdev->gem.mutex);
264                 list_del_init(&bo->list);
265                 mutex_unlock(&bo->rdev->gem.mutex);
266                 radeon_bo_unref(&bo);
267                 gobj->driver_private = NULL;
268                 drm_gem_object_unreference(gobj);
269                 mutex_unlock(&rdev->ddev->struct_mutex);
270         }
271 }
272
273 int radeon_bo_init(struct radeon_device *rdev)
274 {
275         /* Add an MTRR for the VRAM */
276         rdev->mc.vram_mtrr = mtrr_add(rdev->mc.aper_base, rdev->mc.aper_size,
277                         MTRR_TYPE_WRCOMB, 1);
278         DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
279                 rdev->mc.mc_vram_size >> 20,
280                 (unsigned long long)rdev->mc.aper_size >> 20);
281         DRM_INFO("RAM width %dbits %cDR\n",
282                         rdev->mc.vram_width, rdev->mc.vram_is_ddr ? 'D' : 'S');
283         return radeon_ttm_init(rdev);
284 }
285
286 void radeon_bo_fini(struct radeon_device *rdev)
287 {
288         radeon_ttm_fini(rdev);
289 }
290
291 void radeon_bo_list_add_object(struct radeon_bo_list *lobj,
292                                 struct list_head *head)
293 {
294         if (lobj->wdomain) {
295                 list_add(&lobj->list, head);
296         } else {
297                 list_add_tail(&lobj->list, head);
298         }
299 }
300
301 int radeon_bo_list_reserve(struct list_head *head)
302 {
303         struct radeon_bo_list *lobj;
304         int r;
305
306         list_for_each_entry(lobj, head, list){
307                 r = radeon_bo_reserve(lobj->bo, false);
308                 if (unlikely(r != 0))
309                         return r;
310                 lobj->reserved = true;
311         }
312         return 0;
313 }
314
315 void radeon_bo_list_unreserve(struct list_head *head)
316 {
317         struct radeon_bo_list *lobj;
318
319         list_for_each_entry(lobj, head, list) {
320                 /* only unreserve object we successfully reserved */
321                 if (lobj->reserved && radeon_bo_is_reserved(lobj->bo))
322                         radeon_bo_unreserve(lobj->bo);
323         }
324 }
325
326 int radeon_bo_list_validate(struct list_head *head)
327 {
328         struct radeon_bo_list *lobj;
329         struct radeon_bo *bo;
330         u32 domain;
331         int r;
332
333         list_for_each_entry(lobj, head, list) {
334                 lobj->reserved = false;
335         }
336         r = radeon_bo_list_reserve(head);
337         if (unlikely(r != 0)) {
338                 return r;
339         }
340         list_for_each_entry(lobj, head, list) {
341                 bo = lobj->bo;
342                 if (!bo->pin_count) {
343                         domain = lobj->wdomain ? lobj->wdomain : lobj->rdomain;
344                         
345                 retry:
346                         radeon_ttm_placement_from_domain(bo, domain);
347                         r = ttm_bo_validate(&bo->tbo, &bo->placement,
348                                                 true, false, false);
349                         if (unlikely(r)) {
350                                 if (r != -ERESTARTSYS && domain == RADEON_GEM_DOMAIN_VRAM) {
351                                         domain |= RADEON_GEM_DOMAIN_GTT;
352                                         goto retry;
353                                 }
354                                 return r;
355                         }
356                 }
357                 lobj->gpu_offset = radeon_bo_gpu_offset(bo);
358                 lobj->tiling_flags = bo->tiling_flags;
359         }
360         return 0;
361 }
362
363 void radeon_bo_list_fence(struct list_head *head, void *fence)
364 {
365         struct radeon_bo_list *lobj;
366         struct radeon_bo *bo;
367         struct radeon_fence *old_fence = NULL;
368
369         list_for_each_entry(lobj, head, list) {
370                 bo = lobj->bo;
371                 spin_lock(&bo->tbo.lock);
372                 old_fence = (struct radeon_fence *)bo->tbo.sync_obj;
373                 bo->tbo.sync_obj = radeon_fence_ref(fence);
374                 bo->tbo.sync_obj_arg = NULL;
375                 spin_unlock(&bo->tbo.lock);
376                 if (old_fence) {
377                         radeon_fence_unref(&old_fence);
378                 }
379         }
380 }
381
382 int radeon_bo_fbdev_mmap(struct radeon_bo *bo,
383                              struct vm_area_struct *vma)
384 {
385         return ttm_fbdev_mmap(vma, &bo->tbo);
386 }
387
388 int radeon_bo_get_surface_reg(struct radeon_bo *bo)
389 {
390         struct radeon_device *rdev = bo->rdev;
391         struct radeon_surface_reg *reg;
392         struct radeon_bo *old_object;
393         int steal;
394         int i;
395
396         BUG_ON(!atomic_read(&bo->tbo.reserved));
397
398         if (!bo->tiling_flags)
399                 return 0;
400
401         if (bo->surface_reg >= 0) {
402                 reg = &rdev->surface_regs[bo->surface_reg];
403                 i = bo->surface_reg;
404                 goto out;
405         }
406
407         steal = -1;
408         for (i = 0; i < RADEON_GEM_MAX_SURFACES; i++) {
409
410                 reg = &rdev->surface_regs[i];
411                 if (!reg->bo)
412                         break;
413
414                 old_object = reg->bo;
415                 if (old_object->pin_count == 0)
416                         steal = i;
417         }
418
419         /* if we are all out */
420         if (i == RADEON_GEM_MAX_SURFACES) {
421                 if (steal == -1)
422                         return -ENOMEM;
423                 /* find someone with a surface reg and nuke their BO */
424                 reg = &rdev->surface_regs[steal];
425                 old_object = reg->bo;
426                 /* blow away the mapping */
427                 DRM_DEBUG("stealing surface reg %d from %p\n", steal, old_object);
428                 ttm_bo_unmap_virtual(&old_object->tbo);
429                 old_object->surface_reg = -1;
430                 i = steal;
431         }
432
433         bo->surface_reg = i;
434         reg->bo = bo;
435
436 out:
437         radeon_set_surface_reg(rdev, i, bo->tiling_flags, bo->pitch,
438                                bo->tbo.mem.mm_node->start << PAGE_SHIFT,
439                                bo->tbo.num_pages << PAGE_SHIFT);
440         return 0;
441 }
442
443 static void radeon_bo_clear_surface_reg(struct radeon_bo *bo)
444 {
445         struct radeon_device *rdev = bo->rdev;
446         struct radeon_surface_reg *reg;
447
448         if (bo->surface_reg == -1)
449                 return;
450
451         reg = &rdev->surface_regs[bo->surface_reg];
452         radeon_clear_surface_reg(rdev, bo->surface_reg);
453
454         reg->bo = NULL;
455         bo->surface_reg = -1;
456 }
457
458 int radeon_bo_set_tiling_flags(struct radeon_bo *bo,
459                                 uint32_t tiling_flags, uint32_t pitch)
460 {
461         int r;
462
463         r = radeon_bo_reserve(bo, false);
464         if (unlikely(r != 0))
465                 return r;
466         bo->tiling_flags = tiling_flags;
467         bo->pitch = pitch;
468         radeon_bo_unreserve(bo);
469         return 0;
470 }
471
472 void radeon_bo_get_tiling_flags(struct radeon_bo *bo,
473                                 uint32_t *tiling_flags,
474                                 uint32_t *pitch)
475 {
476         BUG_ON(!atomic_read(&bo->tbo.reserved));
477         if (tiling_flags)
478                 *tiling_flags = bo->tiling_flags;
479         if (pitch)
480                 *pitch = bo->pitch;
481 }
482
483 int radeon_bo_check_tiling(struct radeon_bo *bo, bool has_moved,
484                                 bool force_drop)
485 {
486         BUG_ON(!atomic_read(&bo->tbo.reserved));
487
488         if (!(bo->tiling_flags & RADEON_TILING_SURFACE))
489                 return 0;
490
491         if (force_drop) {
492                 radeon_bo_clear_surface_reg(bo);
493                 return 0;
494         }
495
496         if (bo->tbo.mem.mem_type != TTM_PL_VRAM) {
497                 if (!has_moved)
498                         return 0;
499
500                 if (bo->surface_reg >= 0)
501                         radeon_bo_clear_surface_reg(bo);
502                 return 0;
503         }
504
505         if ((bo->surface_reg >= 0) && !has_moved)
506                 return 0;
507
508         return radeon_bo_get_surface_reg(bo);
509 }
510
511 void radeon_bo_move_notify(struct ttm_buffer_object *bo,
512                            struct ttm_mem_reg *mem)
513 {
514         struct radeon_bo *rbo;
515         if (!radeon_ttm_bo_is_radeon_bo(bo))
516                 return;
517         rbo = container_of(bo, struct radeon_bo, tbo);
518         radeon_bo_check_tiling(rbo, 0, 1);
519 }
520
521 int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
522 {
523         struct radeon_device *rdev;
524         struct radeon_bo *rbo;
525         unsigned long offset, size;
526         int r;
527
528         if (!radeon_ttm_bo_is_radeon_bo(bo))
529                 return 0;
530         rbo = container_of(bo, struct radeon_bo, tbo);
531         radeon_bo_check_tiling(rbo, 0, 0);
532         rdev = rbo->rdev;
533         if (bo->mem.mem_type == TTM_PL_VRAM) {
534                 size = bo->mem.num_pages << PAGE_SHIFT;
535                 offset = bo->mem.mm_node->start << PAGE_SHIFT;
536                 if ((offset + size) > rdev->mc.visible_vram_size) {
537                         /* hurrah the memory is not visible ! */
538                         radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_VRAM);
539                         rbo->placement.lpfn = rdev->mc.visible_vram_size >> PAGE_SHIFT;
540                         r = ttm_bo_validate(bo, &rbo->placement, false, true, false);
541                         if (unlikely(r != 0))
542                                 return r;
543                         offset = bo->mem.mm_node->start << PAGE_SHIFT;
544                         /* this should not happen */
545                         if ((offset + size) > rdev->mc.visible_vram_size)
546                                 return -EINVAL;
547                 }
548         }
549         return 0;
550 }