drm/nouveau: fix assumption that semaphore dmaobj is valid in x-chan sync
[linux-2.6.git] / drivers / gpu / drm / nouveau / nouveau_fence.c
1 /*
2  * Copyright (C) 2007 Ben Skeggs.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining
6  * a copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sublicense, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * The above copyright notice and this permission notice (including the
14  * next paragraph) shall be included in all copies or substantial
15  * portions of the Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24  *
25  */
26
27 #include "drmP.h"
28 #include "drm.h"
29
30 #include <linux/ktime.h>
31 #include <linux/hrtimer.h>
32
33 #include "nouveau_drv.h"
34 #include "nouveau_ramht.h"
35 #include "nouveau_dma.h"
36
37 #define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10)
38 #define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17)
39
40 struct nouveau_fence {
41         struct nouveau_channel *channel;
42         struct kref refcount;
43         struct list_head entry;
44
45         uint32_t sequence;
46         bool signalled;
47
48         void (*work)(void *priv, bool signalled);
49         void *priv;
50 };
51
52 struct nouveau_semaphore {
53         struct kref ref;
54         struct drm_device *dev;
55         struct drm_mm_node *mem;
56 };
57
58 static inline struct nouveau_fence *
59 nouveau_fence(void *sync_obj)
60 {
61         return (struct nouveau_fence *)sync_obj;
62 }
63
64 static void
65 nouveau_fence_del(struct kref *ref)
66 {
67         struct nouveau_fence *fence =
68                 container_of(ref, struct nouveau_fence, refcount);
69
70         nouveau_channel_ref(NULL, &fence->channel);
71         kfree(fence);
72 }
73
74 void
75 nouveau_fence_update(struct nouveau_channel *chan)
76 {
77         struct drm_device *dev = chan->dev;
78         struct nouveau_fence *tmp, *fence;
79         uint32_t sequence;
80
81         spin_lock(&chan->fence.lock);
82
83         /* Fetch the last sequence if the channel is still up and running */
84         if (likely(!list_empty(&chan->fence.pending))) {
85                 if (USE_REFCNT(dev))
86                         sequence = nvchan_rd32(chan, 0x48);
87                 else
88                         sequence = atomic_read(&chan->fence.last_sequence_irq);
89
90                 if (chan->fence.sequence_ack == sequence)
91                         goto out;
92                 chan->fence.sequence_ack = sequence;
93         }
94
95         list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
96                 sequence = fence->sequence;
97                 fence->signalled = true;
98                 list_del(&fence->entry);
99
100                 if (unlikely(fence->work))
101                         fence->work(fence->priv, true);
102
103                 kref_put(&fence->refcount, nouveau_fence_del);
104
105                 if (sequence == chan->fence.sequence_ack)
106                         break;
107         }
108 out:
109         spin_unlock(&chan->fence.lock);
110 }
111
112 int
113 nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence,
114                   bool emit)
115 {
116         struct nouveau_fence *fence;
117         int ret = 0;
118
119         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
120         if (!fence)
121                 return -ENOMEM;
122         kref_init(&fence->refcount);
123         nouveau_channel_ref(chan, &fence->channel);
124
125         if (emit)
126                 ret = nouveau_fence_emit(fence);
127
128         if (ret)
129                 nouveau_fence_unref(&fence);
130         *pfence = fence;
131         return ret;
132 }
133
134 struct nouveau_channel *
135 nouveau_fence_channel(struct nouveau_fence *fence)
136 {
137         return fence ? nouveau_channel_get_unlocked(fence->channel) : NULL;
138 }
139
140 int
141 nouveau_fence_emit(struct nouveau_fence *fence)
142 {
143         struct nouveau_channel *chan = fence->channel;
144         struct drm_device *dev = chan->dev;
145         struct drm_nouveau_private *dev_priv = dev->dev_private;
146         int ret;
147
148         ret = RING_SPACE(chan, 2);
149         if (ret)
150                 return ret;
151
152         if (unlikely(chan->fence.sequence == chan->fence.sequence_ack - 1)) {
153                 nouveau_fence_update(chan);
154
155                 BUG_ON(chan->fence.sequence ==
156                        chan->fence.sequence_ack - 1);
157         }
158
159         fence->sequence = ++chan->fence.sequence;
160
161         kref_get(&fence->refcount);
162         spin_lock(&chan->fence.lock);
163         list_add_tail(&fence->entry, &chan->fence.pending);
164         spin_unlock(&chan->fence.lock);
165
166         if (USE_REFCNT(dev)) {
167                 if (dev_priv->card_type < NV_C0)
168                         BEGIN_RING(chan, NvSubSw, 0x0050, 1);
169                 else
170                         BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0050, 1);
171         } else {
172                 BEGIN_RING(chan, NvSubSw, 0x0150, 1);
173         }
174         OUT_RING (chan, fence->sequence);
175         FIRE_RING(chan);
176
177         return 0;
178 }
179
180 void
181 nouveau_fence_work(struct nouveau_fence *fence,
182                    void (*work)(void *priv, bool signalled),
183                    void *priv)
184 {
185         BUG_ON(fence->work);
186
187         spin_lock(&fence->channel->fence.lock);
188
189         if (fence->signalled) {
190                 work(priv, true);
191         } else {
192                 fence->work = work;
193                 fence->priv = priv;
194         }
195
196         spin_unlock(&fence->channel->fence.lock);
197 }
198
199 void
200 __nouveau_fence_unref(void **sync_obj)
201 {
202         struct nouveau_fence *fence = nouveau_fence(*sync_obj);
203
204         if (fence)
205                 kref_put(&fence->refcount, nouveau_fence_del);
206         *sync_obj = NULL;
207 }
208
209 void *
210 __nouveau_fence_ref(void *sync_obj)
211 {
212         struct nouveau_fence *fence = nouveau_fence(sync_obj);
213
214         kref_get(&fence->refcount);
215         return sync_obj;
216 }
217
218 bool
219 __nouveau_fence_signalled(void *sync_obj, void *sync_arg)
220 {
221         struct nouveau_fence *fence = nouveau_fence(sync_obj);
222         struct nouveau_channel *chan = fence->channel;
223
224         if (fence->signalled)
225                 return true;
226
227         nouveau_fence_update(chan);
228         return fence->signalled;
229 }
230
231 int
232 __nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
233 {
234         unsigned long timeout = jiffies + (3 * DRM_HZ);
235         unsigned long sleep_time = NSEC_PER_MSEC / 1000;
236         ktime_t t;
237         int ret = 0;
238
239         while (1) {
240                 if (__nouveau_fence_signalled(sync_obj, sync_arg))
241                         break;
242
243                 if (time_after_eq(jiffies, timeout)) {
244                         ret = -EBUSY;
245                         break;
246                 }
247
248                 __set_current_state(intr ? TASK_INTERRUPTIBLE
249                         : TASK_UNINTERRUPTIBLE);
250                 if (lazy) {
251                         t = ktime_set(0, sleep_time);
252                         schedule_hrtimeout(&t, HRTIMER_MODE_REL);
253                         sleep_time *= 2;
254                         if (sleep_time > NSEC_PER_MSEC)
255                                 sleep_time = NSEC_PER_MSEC;
256                 }
257
258                 if (intr && signal_pending(current)) {
259                         ret = -ERESTARTSYS;
260                         break;
261                 }
262         }
263
264         __set_current_state(TASK_RUNNING);
265
266         return ret;
267 }
268
269 static struct nouveau_semaphore *
270 semaphore_alloc(struct drm_device *dev)
271 {
272         struct drm_nouveau_private *dev_priv = dev->dev_private;
273         struct nouveau_semaphore *sema;
274         int size = (dev_priv->chipset < 0x84) ? 4 : 16;
275         int ret, i;
276
277         if (!USE_SEMA(dev))
278                 return NULL;
279
280         sema = kmalloc(sizeof(*sema), GFP_KERNEL);
281         if (!sema)
282                 goto fail;
283
284         ret = drm_mm_pre_get(&dev_priv->fence.heap);
285         if (ret)
286                 goto fail;
287
288         spin_lock(&dev_priv->fence.lock);
289         sema->mem = drm_mm_search_free(&dev_priv->fence.heap, size, 0, 0);
290         if (sema->mem)
291                 sema->mem = drm_mm_get_block_atomic(sema->mem, size, 0);
292         spin_unlock(&dev_priv->fence.lock);
293
294         if (!sema->mem)
295                 goto fail;
296
297         kref_init(&sema->ref);
298         sema->dev = dev;
299         for (i = sema->mem->start; i < sema->mem->start + size; i += 4)
300                 nouveau_bo_wr32(dev_priv->fence.bo, i / 4, 0);
301
302         return sema;
303 fail:
304         kfree(sema);
305         return NULL;
306 }
307
308 static void
309 semaphore_free(struct kref *ref)
310 {
311         struct nouveau_semaphore *sema =
312                 container_of(ref, struct nouveau_semaphore, ref);
313         struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
314
315         spin_lock(&dev_priv->fence.lock);
316         drm_mm_put_block(sema->mem);
317         spin_unlock(&dev_priv->fence.lock);
318
319         kfree(sema);
320 }
321
322 static void
323 semaphore_work(void *priv, bool signalled)
324 {
325         struct nouveau_semaphore *sema = priv;
326         struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
327
328         if (unlikely(!signalled))
329                 nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 1);
330
331         kref_put(&sema->ref, semaphore_free);
332 }
333
334 static int
335 semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
336 {
337         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
338         struct nouveau_fence *fence = NULL;
339         int ret;
340
341         if (dev_priv->chipset < 0x84) {
342                 ret = RING_SPACE(chan, 4);
343                 if (ret)
344                         return ret;
345
346                 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 3);
347                 OUT_RING  (chan, NvSema);
348                 OUT_RING  (chan, sema->mem->start);
349                 OUT_RING  (chan, 1);
350         } else
351         if (dev_priv->chipset < 0xc0) {
352                 struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
353                 u64 offset = vma->offset + sema->mem->start;
354
355                 ret = RING_SPACE(chan, 7);
356                 if (ret)
357                         return ret;
358
359                 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
360                 OUT_RING  (chan, chan->vram_handle);
361                 BEGIN_RING(chan, NvSubSw, 0x0010, 4);
362                 OUT_RING  (chan, upper_32_bits(offset));
363                 OUT_RING  (chan, lower_32_bits(offset));
364                 OUT_RING  (chan, 1);
365                 OUT_RING  (chan, 1); /* ACQUIRE_EQ */
366         } else {
367                 struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
368                 u64 offset = vma->offset + sema->mem->start;
369
370                 ret = RING_SPACE(chan, 5);
371                 if (ret)
372                         return ret;
373
374                 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4);
375                 OUT_RING  (chan, upper_32_bits(offset));
376                 OUT_RING  (chan, lower_32_bits(offset));
377                 OUT_RING  (chan, 1);
378                 OUT_RING  (chan, 0x1001); /* ACQUIRE_EQ */
379         }
380
381         /* Delay semaphore destruction until its work is done */
382         ret = nouveau_fence_new(chan, &fence, true);
383         if (ret)
384                 return ret;
385
386         kref_get(&sema->ref);
387         nouveau_fence_work(fence, semaphore_work, sema);
388         nouveau_fence_unref(&fence);
389         return 0;
390 }
391
392 static int
393 semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
394 {
395         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
396         struct nouveau_fence *fence = NULL;
397         int ret;
398
399         if (dev_priv->chipset < 0x84) {
400                 ret = RING_SPACE(chan, 5);
401                 if (ret)
402                         return ret;
403
404                 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 2);
405                 OUT_RING  (chan, NvSema);
406                 OUT_RING  (chan, sema->mem->start);
407                 BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_RELEASE, 1);
408                 OUT_RING  (chan, 1);
409         } else
410         if (dev_priv->chipset < 0xc0) {
411                 struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
412                 u64 offset = vma->offset + sema->mem->start;
413
414                 ret = RING_SPACE(chan, 7);
415                 if (ret)
416                         return ret;
417
418                 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
419                 OUT_RING  (chan, chan->vram_handle);
420                 BEGIN_RING(chan, NvSubSw, 0x0010, 4);
421                 OUT_RING  (chan, upper_32_bits(offset));
422                 OUT_RING  (chan, lower_32_bits(offset));
423                 OUT_RING  (chan, 1);
424                 OUT_RING  (chan, 2); /* RELEASE */
425         } else {
426                 struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
427                 u64 offset = vma->offset + sema->mem->start;
428
429                 ret = RING_SPACE(chan, 5);
430                 if (ret)
431                         return ret;
432
433                 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4);
434                 OUT_RING  (chan, upper_32_bits(offset));
435                 OUT_RING  (chan, lower_32_bits(offset));
436                 OUT_RING  (chan, 1);
437                 OUT_RING  (chan, 0x1002); /* RELEASE */
438         }
439
440         /* Delay semaphore destruction until its work is done */
441         ret = nouveau_fence_new(chan, &fence, true);
442         if (ret)
443                 return ret;
444
445         kref_get(&sema->ref);
446         nouveau_fence_work(fence, semaphore_work, sema);
447         nouveau_fence_unref(&fence);
448         return 0;
449 }
450
451 int
452 nouveau_fence_sync(struct nouveau_fence *fence,
453                    struct nouveau_channel *wchan)
454 {
455         struct nouveau_channel *chan = nouveau_fence_channel(fence);
456         struct drm_device *dev = wchan->dev;
457         struct nouveau_semaphore *sema;
458         int ret = 0;
459
460         if (likely(!chan || chan == wchan ||
461                    nouveau_fence_signalled(fence)))
462                 goto out;
463
464         sema = semaphore_alloc(dev);
465         if (!sema) {
466                 /* Early card or broken userspace, fall back to
467                  * software sync. */
468                 ret = nouveau_fence_wait(fence, true, false);
469                 goto out;
470         }
471
472         /* try to take chan's mutex, if we can't take it right away
473          * we have to fallback to software sync to prevent locking
474          * order issues
475          */
476         if (!mutex_trylock(&chan->mutex)) {
477                 ret = nouveau_fence_wait(fence, true, false);
478                 goto out_unref;
479         }
480
481         /* Make wchan wait until it gets signalled */
482         ret = semaphore_acquire(wchan, sema);
483         if (ret)
484                 goto out_unlock;
485
486         /* Signal the semaphore from chan */
487         ret = semaphore_release(chan, sema);
488
489 out_unlock:
490         mutex_unlock(&chan->mutex);
491 out_unref:
492         kref_put(&sema->ref, semaphore_free);
493 out:
494         if (chan)
495                 nouveau_channel_put_unlocked(&chan);
496         return ret;
497 }
498
499 int
500 __nouveau_fence_flush(void *sync_obj, void *sync_arg)
501 {
502         return 0;
503 }
504
505 int
506 nouveau_fence_channel_init(struct nouveau_channel *chan)
507 {
508         struct drm_device *dev = chan->dev;
509         struct drm_nouveau_private *dev_priv = dev->dev_private;
510         struct nouveau_gpuobj *obj = NULL;
511         int ret;
512
513         if (dev_priv->card_type < NV_C0) {
514                 /* Create an NV_SW object for various sync purposes */
515                 ret = nouveau_gpuobj_gr_new(chan, NvSw, NV_SW);
516                 if (ret)
517                         return ret;
518
519                 ret = RING_SPACE(chan, 2);
520                 if (ret)
521                         return ret;
522
523                 BEGIN_RING(chan, NvSubSw, 0, 1);
524                 OUT_RING  (chan, NvSw);
525                 FIRE_RING (chan);
526         }
527
528         /* Setup area of memory shared between all channels for x-chan sync */
529         if (USE_SEMA(dev) && dev_priv->chipset < 0x84) {
530                 struct ttm_mem_reg *mem = &dev_priv->fence.bo->bo.mem;
531
532                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
533                                              mem->start << PAGE_SHIFT,
534                                              mem->size, NV_MEM_ACCESS_RW,
535                                              NV_MEM_TARGET_VRAM, &obj);
536                 if (ret)
537                         return ret;
538
539                 ret = nouveau_ramht_insert(chan, NvSema, obj);
540                 nouveau_gpuobj_ref(NULL, &obj);
541                 if (ret)
542                         return ret;
543         }
544
545         INIT_LIST_HEAD(&chan->fence.pending);
546         spin_lock_init(&chan->fence.lock);
547         atomic_set(&chan->fence.last_sequence_irq, 0);
548         return 0;
549 }
550
551 void
552 nouveau_fence_channel_fini(struct nouveau_channel *chan)
553 {
554         struct nouveau_fence *tmp, *fence;
555
556         spin_lock(&chan->fence.lock);
557
558         list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
559                 fence->signalled = true;
560                 list_del(&fence->entry);
561
562                 if (unlikely(fence->work))
563                         fence->work(fence->priv, false);
564
565                 kref_put(&fence->refcount, nouveau_fence_del);
566         }
567
568         spin_unlock(&chan->fence.lock);
569 }
570
571 int
572 nouveau_fence_init(struct drm_device *dev)
573 {
574         struct drm_nouveau_private *dev_priv = dev->dev_private;
575         int size = (dev_priv->chipset < 0x84) ? 4096 : 16384;
576         int ret;
577
578         /* Create a shared VRAM heap for cross-channel sync. */
579         if (USE_SEMA(dev)) {
580                 ret = nouveau_bo_new(dev, NULL, size, 0, TTM_PL_FLAG_VRAM,
581                                      0, 0, &dev_priv->fence.bo);
582                 if (ret)
583                         return ret;
584
585                 ret = nouveau_bo_pin(dev_priv->fence.bo, TTM_PL_FLAG_VRAM);
586                 if (ret)
587                         goto fail;
588
589                 ret = nouveau_bo_map(dev_priv->fence.bo);
590                 if (ret)
591                         goto fail;
592
593                 ret = drm_mm_init(&dev_priv->fence.heap, 0,
594                                   dev_priv->fence.bo->bo.mem.size);
595                 if (ret)
596                         goto fail;
597
598                 spin_lock_init(&dev_priv->fence.lock);
599         }
600
601         return 0;
602 fail:
603         nouveau_bo_unmap(dev_priv->fence.bo);
604         nouveau_bo_ref(NULL, &dev_priv->fence.bo);
605         return ret;
606 }
607
608 void
609 nouveau_fence_fini(struct drm_device *dev)
610 {
611         struct drm_nouveau_private *dev_priv = dev->dev_private;
612
613         if (USE_SEMA(dev)) {
614                 drm_mm_takedown(&dev_priv->fence.heap);
615                 nouveau_bo_unmap(dev_priv->fence.bo);
616                 nouveau_bo_unpin(dev_priv->fence.bo);
617                 nouveau_bo_ref(NULL, &dev_priv->fence.bo);
618         }
619 }