drm: Add the TTM GPU memory manager subsystem.
[linux-2.6.git] / drivers / gpu / drm / ttm / ttm_bo_util.c
1 /**************************************************************************
2  *
3  * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
4  * All Rights Reserved.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the
8  * "Software"), to deal in the Software without restriction, including
9  * without limitation the rights to use, copy, modify, merge, publish,
10  * distribute, sub license, and/or sell copies of the Software, and to
11  * permit persons to whom the Software is furnished to do so, subject to
12  * the following conditions:
13  *
14  * The above copyright notice and this permission notice (including the
15  * next paragraph) shall be included in all copies or substantial portions
16  * of the Software.
17  *
18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
25  *
26  **************************************************************************/
27 /*
28  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
29  */
30
31 #include "ttm/ttm_bo_driver.h"
32 #include "ttm/ttm_placement.h"
33 #include <linux/io.h>
34 #include <linux/highmem.h>
35 #include <linux/wait.h>
36 #include <linux/vmalloc.h>
37 #include <linux/version.h>
38 #include <linux/module.h>
39
40 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
41 {
42         struct ttm_mem_reg *old_mem = &bo->mem;
43
44         if (old_mem->mm_node) {
45                 spin_lock(&bo->bdev->lru_lock);
46                 drm_mm_put_block(old_mem->mm_node);
47                 spin_unlock(&bo->bdev->lru_lock);
48         }
49         old_mem->mm_node = NULL;
50 }
51
52 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
53                     bool evict, bool no_wait, struct ttm_mem_reg *new_mem)
54 {
55         struct ttm_tt *ttm = bo->ttm;
56         struct ttm_mem_reg *old_mem = &bo->mem;
57         uint32_t save_flags = old_mem->placement;
58         int ret;
59
60         if (old_mem->mem_type != TTM_PL_SYSTEM) {
61                 ttm_tt_unbind(ttm);
62                 ttm_bo_free_old_node(bo);
63                 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
64                                 TTM_PL_MASK_MEM);
65                 old_mem->mem_type = TTM_PL_SYSTEM;
66                 save_flags = old_mem->placement;
67         }
68
69         ret = ttm_tt_set_placement_caching(ttm, new_mem->placement);
70         if (unlikely(ret != 0))
71                 return ret;
72
73         if (new_mem->mem_type != TTM_PL_SYSTEM) {
74                 ret = ttm_tt_bind(ttm, new_mem);
75                 if (unlikely(ret != 0))
76                         return ret;
77         }
78
79         *old_mem = *new_mem;
80         new_mem->mm_node = NULL;
81         ttm_flag_masked(&save_flags, new_mem->placement, TTM_PL_MASK_MEMTYPE);
82         return 0;
83 }
84 EXPORT_SYMBOL(ttm_bo_move_ttm);
85
86 int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
87                         void **virtual)
88 {
89         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
90         unsigned long bus_offset;
91         unsigned long bus_size;
92         unsigned long bus_base;
93         int ret;
94         void *addr;
95
96         *virtual = NULL;
97         ret = ttm_bo_pci_offset(bdev, mem, &bus_base, &bus_offset, &bus_size);
98         if (ret || bus_size == 0)
99                 return ret;
100
101         if (!(man->flags & TTM_MEMTYPE_FLAG_NEEDS_IOREMAP))
102                 addr = (void *)(((u8 *) man->io_addr) + bus_offset);
103         else {
104                 if (mem->placement & TTM_PL_FLAG_WC)
105                         addr = ioremap_wc(bus_base + bus_offset, bus_size);
106                 else
107                         addr = ioremap_nocache(bus_base + bus_offset, bus_size);
108                 if (!addr)
109                         return -ENOMEM;
110         }
111         *virtual = addr;
112         return 0;
113 }
114
115 void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
116                          void *virtual)
117 {
118         struct ttm_mem_type_manager *man;
119
120         man = &bdev->man[mem->mem_type];
121
122         if (virtual && (man->flags & TTM_MEMTYPE_FLAG_NEEDS_IOREMAP))
123                 iounmap(virtual);
124 }
125
126 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
127 {
128         uint32_t *dstP =
129             (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
130         uint32_t *srcP =
131             (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
132
133         int i;
134         for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
135                 iowrite32(ioread32(srcP++), dstP++);
136         return 0;
137 }
138
139 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
140                                 unsigned long page)
141 {
142         struct page *d = ttm_tt_get_page(ttm, page);
143         void *dst;
144
145         if (!d)
146                 return -ENOMEM;
147
148         src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
149         dst = kmap(d);
150         if (!dst)
151                 return -ENOMEM;
152
153         memcpy_fromio(dst, src, PAGE_SIZE);
154         kunmap(d);
155         return 0;
156 }
157
158 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
159                                 unsigned long page)
160 {
161         struct page *s = ttm_tt_get_page(ttm, page);
162         void *src;
163
164         if (!s)
165                 return -ENOMEM;
166
167         dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
168         src = kmap(s);
169         if (!src)
170                 return -ENOMEM;
171
172         memcpy_toio(dst, src, PAGE_SIZE);
173         kunmap(s);
174         return 0;
175 }
176
177 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
178                        bool evict, bool no_wait, struct ttm_mem_reg *new_mem)
179 {
180         struct ttm_bo_device *bdev = bo->bdev;
181         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
182         struct ttm_tt *ttm = bo->ttm;
183         struct ttm_mem_reg *old_mem = &bo->mem;
184         struct ttm_mem_reg old_copy = *old_mem;
185         void *old_iomap;
186         void *new_iomap;
187         int ret;
188         uint32_t save_flags = old_mem->placement;
189         unsigned long i;
190         unsigned long page;
191         unsigned long add = 0;
192         int dir;
193
194         ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
195         if (ret)
196                 return ret;
197         ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap);
198         if (ret)
199                 goto out;
200
201         if (old_iomap == NULL && new_iomap == NULL)
202                 goto out2;
203         if (old_iomap == NULL && ttm == NULL)
204                 goto out2;
205
206         add = 0;
207         dir = 1;
208
209         if ((old_mem->mem_type == new_mem->mem_type) &&
210             (new_mem->mm_node->start <
211              old_mem->mm_node->start + old_mem->mm_node->size)) {
212                 dir = -1;
213                 add = new_mem->num_pages - 1;
214         }
215
216         for (i = 0; i < new_mem->num_pages; ++i) {
217                 page = i * dir + add;
218                 if (old_iomap == NULL)
219                         ret = ttm_copy_ttm_io_page(ttm, new_iomap, page);
220                 else if (new_iomap == NULL)
221                         ret = ttm_copy_io_ttm_page(ttm, old_iomap, page);
222                 else
223                         ret = ttm_copy_io_page(new_iomap, old_iomap, page);
224                 if (ret)
225                         goto out1;
226         }
227         mb();
228 out2:
229         ttm_bo_free_old_node(bo);
230
231         *old_mem = *new_mem;
232         new_mem->mm_node = NULL;
233         ttm_flag_masked(&save_flags, new_mem->placement, TTM_PL_MASK_MEMTYPE);
234
235         if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) {
236                 ttm_tt_unbind(ttm);
237                 ttm_tt_destroy(ttm);
238                 bo->ttm = NULL;
239         }
240
241 out1:
242         ttm_mem_reg_iounmap(bdev, new_mem, new_iomap);
243 out:
244         ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
245         return ret;
246 }
247 EXPORT_SYMBOL(ttm_bo_move_memcpy);
248
249 static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
250 {
251         kfree(bo);
252 }
253
254 /**
255  * ttm_buffer_object_transfer
256  *
257  * @bo: A pointer to a struct ttm_buffer_object.
258  * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
259  * holding the data of @bo with the old placement.
260  *
261  * This is a utility function that may be called after an accelerated move
262  * has been scheduled. A new buffer object is created as a placeholder for
263  * the old data while it's being copied. When that buffer object is idle,
264  * it can be destroyed, releasing the space of the old placement.
265  * Returns:
266  * !0: Failure.
267  */
268
269 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
270                                       struct ttm_buffer_object **new_obj)
271 {
272         struct ttm_buffer_object *fbo;
273         struct ttm_bo_device *bdev = bo->bdev;
274         struct ttm_bo_driver *driver = bdev->driver;
275
276         fbo = kzalloc(sizeof(*fbo), GFP_KERNEL);
277         if (!fbo)
278                 return -ENOMEM;
279
280         *fbo = *bo;
281
282         /**
283          * Fix up members that we shouldn't copy directly:
284          * TODO: Explicit member copy would probably be better here.
285          */
286
287         spin_lock_init(&fbo->lock);
288         init_waitqueue_head(&fbo->event_queue);
289         INIT_LIST_HEAD(&fbo->ddestroy);
290         INIT_LIST_HEAD(&fbo->lru);
291         INIT_LIST_HEAD(&fbo->swap);
292         fbo->vm_node = NULL;
293
294         fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj);
295         if (fbo->mem.mm_node)
296                 fbo->mem.mm_node->private = (void *)fbo;
297         kref_init(&fbo->list_kref);
298         kref_init(&fbo->kref);
299         fbo->destroy = &ttm_transfered_destroy;
300
301         *new_obj = fbo;
302         return 0;
303 }
304
305 pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp)
306 {
307 #if defined(__i386__) || defined(__x86_64__)
308         if (caching_flags & TTM_PL_FLAG_WC)
309                 tmp = pgprot_writecombine(tmp);
310         else if (boot_cpu_data.x86 > 3)
311                 tmp = pgprot_noncached(tmp);
312
313 #elif defined(__powerpc__)
314         if (!(caching_flags & TTM_PL_FLAG_CACHED)) {
315                 pgprot_val(tmp) |= _PAGE_NO_CACHE;
316                 if (caching_flags & TTM_PL_FLAG_UNCACHED)
317                         pgprot_val(tmp) |= _PAGE_GUARDED;
318         }
319 #endif
320 #if defined(__ia64__)
321         if (caching_flags & TTM_PL_FLAG_WC)
322                 tmp = pgprot_writecombine(tmp);
323         else
324                 tmp = pgprot_noncached(tmp);
325 #endif
326 #if defined(__sparc__)
327         if (!(caching_flags & TTM_PL_FLAG_CACHED))
328                 tmp = pgprot_noncached(tmp);
329 #endif
330         return tmp;
331 }
332
333 static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
334                           unsigned long bus_base,
335                           unsigned long bus_offset,
336                           unsigned long bus_size,
337                           struct ttm_bo_kmap_obj *map)
338 {
339         struct ttm_bo_device *bdev = bo->bdev;
340         struct ttm_mem_reg *mem = &bo->mem;
341         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
342
343         if (!(man->flags & TTM_MEMTYPE_FLAG_NEEDS_IOREMAP)) {
344                 map->bo_kmap_type = ttm_bo_map_premapped;
345                 map->virtual = (void *)(((u8 *) man->io_addr) + bus_offset);
346         } else {
347                 map->bo_kmap_type = ttm_bo_map_iomap;
348                 if (mem->placement & TTM_PL_FLAG_WC)
349                         map->virtual = ioremap_wc(bus_base + bus_offset,
350                                                   bus_size);
351                 else
352                         map->virtual = ioremap_nocache(bus_base + bus_offset,
353                                                        bus_size);
354         }
355         return (!map->virtual) ? -ENOMEM : 0;
356 }
357
358 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
359                            unsigned long start_page,
360                            unsigned long num_pages,
361                            struct ttm_bo_kmap_obj *map)
362 {
363         struct ttm_mem_reg *mem = &bo->mem; pgprot_t prot;
364         struct ttm_tt *ttm = bo->ttm;
365         struct page *d;
366         int i;
367
368         BUG_ON(!ttm);
369         if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) {
370                 /*
371                  * We're mapping a single page, and the desired
372                  * page protection is consistent with the bo.
373                  */
374
375                 map->bo_kmap_type = ttm_bo_map_kmap;
376                 map->page = ttm_tt_get_page(ttm, start_page);
377                 map->virtual = kmap(map->page);
378         } else {
379             /*
380              * Populate the part we're mapping;
381              */
382                 for (i = start_page; i < start_page + num_pages; ++i) {
383                         d = ttm_tt_get_page(ttm, i);
384                         if (!d)
385                                 return -ENOMEM;
386                 }
387
388                 /*
389                  * We need to use vmap to get the desired page protection
390                  * or to make the buffer object look contigous.
391                  */
392                 prot = (mem->placement & TTM_PL_FLAG_CACHED) ?
393                         PAGE_KERNEL :
394                         ttm_io_prot(mem->placement, PAGE_KERNEL);
395                 map->bo_kmap_type = ttm_bo_map_vmap;
396                 map->virtual = vmap(ttm->pages + start_page, num_pages,
397                                     0, prot);
398         }
399         return (!map->virtual) ? -ENOMEM : 0;
400 }
401
402 int ttm_bo_kmap(struct ttm_buffer_object *bo,
403                 unsigned long start_page, unsigned long num_pages,
404                 struct ttm_bo_kmap_obj *map)
405 {
406         int ret;
407         unsigned long bus_base;
408         unsigned long bus_offset;
409         unsigned long bus_size;
410
411         BUG_ON(!list_empty(&bo->swap));
412         map->virtual = NULL;
413         if (num_pages > bo->num_pages)
414                 return -EINVAL;
415         if (start_page > bo->num_pages)
416                 return -EINVAL;
417 #if 0
418         if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC))
419                 return -EPERM;
420 #endif
421         ret = ttm_bo_pci_offset(bo->bdev, &bo->mem, &bus_base,
422                                 &bus_offset, &bus_size);
423         if (ret)
424                 return ret;
425         if (bus_size == 0) {
426                 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
427         } else {
428                 bus_offset += start_page << PAGE_SHIFT;
429                 bus_size = num_pages << PAGE_SHIFT;
430                 return ttm_bo_ioremap(bo, bus_base, bus_offset, bus_size, map);
431         }
432 }
433 EXPORT_SYMBOL(ttm_bo_kmap);
434
435 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
436 {
437         if (!map->virtual)
438                 return;
439         switch (map->bo_kmap_type) {
440         case ttm_bo_map_iomap:
441                 iounmap(map->virtual);
442                 break;
443         case ttm_bo_map_vmap:
444                 vunmap(map->virtual);
445                 break;
446         case ttm_bo_map_kmap:
447                 kunmap(map->page);
448                 break;
449         case ttm_bo_map_premapped:
450                 break;
451         default:
452                 BUG();
453         }
454         map->virtual = NULL;
455         map->page = NULL;
456 }
457 EXPORT_SYMBOL(ttm_bo_kunmap);
458
459 int ttm_bo_pfn_prot(struct ttm_buffer_object *bo,
460                     unsigned long dst_offset,
461                     unsigned long *pfn, pgprot_t *prot)
462 {
463         struct ttm_mem_reg *mem = &bo->mem;
464         struct ttm_bo_device *bdev = bo->bdev;
465         unsigned long bus_offset;
466         unsigned long bus_size;
467         unsigned long bus_base;
468         int ret;
469         ret = ttm_bo_pci_offset(bdev, mem, &bus_base, &bus_offset,
470                         &bus_size);
471         if (ret)
472                 return -EINVAL;
473         if (bus_size != 0)
474                 *pfn = (bus_base + bus_offset + dst_offset) >> PAGE_SHIFT;
475         else
476                 if (!bo->ttm)
477                         return -EINVAL;
478                 else
479                         *pfn = page_to_pfn(ttm_tt_get_page(bo->ttm,
480                                                            dst_offset >>
481                                                            PAGE_SHIFT));
482         *prot = (mem->placement & TTM_PL_FLAG_CACHED) ?
483                 PAGE_KERNEL : ttm_io_prot(mem->placement, PAGE_KERNEL);
484
485         return 0;
486 }
487
488 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
489                               void *sync_obj,
490                               void *sync_obj_arg,
491                               bool evict, bool no_wait,
492                               struct ttm_mem_reg *new_mem)
493 {
494         struct ttm_bo_device *bdev = bo->bdev;
495         struct ttm_bo_driver *driver = bdev->driver;
496         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
497         struct ttm_mem_reg *old_mem = &bo->mem;
498         int ret;
499         uint32_t save_flags = old_mem->placement;
500         struct ttm_buffer_object *ghost_obj;
501         void *tmp_obj = NULL;
502
503         spin_lock(&bo->lock);
504         if (bo->sync_obj) {
505                 tmp_obj = bo->sync_obj;
506                 bo->sync_obj = NULL;
507         }
508         bo->sync_obj = driver->sync_obj_ref(sync_obj);
509         bo->sync_obj_arg = sync_obj_arg;
510         if (evict) {
511                 ret = ttm_bo_wait(bo, false, false, false);
512                 spin_unlock(&bo->lock);
513                 driver->sync_obj_unref(&bo->sync_obj);
514
515                 if (ret)
516                         return ret;
517
518                 ttm_bo_free_old_node(bo);
519                 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
520                     (bo->ttm != NULL)) {
521                         ttm_tt_unbind(bo->ttm);
522                         ttm_tt_destroy(bo->ttm);
523                         bo->ttm = NULL;
524                 }
525         } else {
526                 /**
527                  * This should help pipeline ordinary buffer moves.
528                  *
529                  * Hang old buffer memory on a new buffer object,
530                  * and leave it to be released when the GPU
531                  * operation has completed.
532                  */
533
534                 set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
535                 spin_unlock(&bo->lock);
536
537                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
538                 if (ret)
539                         return ret;
540
541                 /**
542                  * If we're not moving to fixed memory, the TTM object
543                  * needs to stay alive. Otherwhise hang it on the ghost
544                  * bo to be unbound and destroyed.
545                  */
546
547                 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED))
548                         ghost_obj->ttm = NULL;
549                 else
550                         bo->ttm = NULL;
551
552                 ttm_bo_unreserve(ghost_obj);
553                 ttm_bo_unref(&ghost_obj);
554         }
555
556         *old_mem = *new_mem;
557         new_mem->mm_node = NULL;
558         ttm_flag_masked(&save_flags, new_mem->placement, TTM_PL_MASK_MEMTYPE);
559         return 0;
560 }
561 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);