drm/radeon/kms: drop lock in return path of radeon_fence_count_emitted.
[linux-2.6.git] / drivers / gpu / drm / radeon / radeon_fence.c
1 /*
2  * Copyright 2009 Jerome Glisse.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sub license, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
20  *
21  * The above copyright notice and this permission notice (including the
22  * next paragraph) shall be included in all copies or substantial portions
23  * of the Software.
24  *
25  */
26 /*
27  * Authors:
28  *    Jerome Glisse <glisse@freedesktop.org>
29  *    Dave Airlie
30  */
31 #include <linux/seq_file.h>
32 #include <linux/atomic.h>
33 #include <linux/wait.h>
34 #include <linux/list.h>
35 #include <linux/kref.h>
36 #include <linux/slab.h>
37 #include "drmP.h"
38 #include "drm.h"
39 #include "radeon_reg.h"
40 #include "radeon.h"
41 #include "radeon_trace.h"
42
43 static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring)
44 {
45         if (rdev->wb.enabled) {
46                 *rdev->fence_drv[ring].cpu_addr = cpu_to_le32(seq);
47         } else {
48                 WREG32(rdev->fence_drv[ring].scratch_reg, seq);
49         }
50 }
51
52 static u32 radeon_fence_read(struct radeon_device *rdev, int ring)
53 {
54         u32 seq = 0;
55
56         if (rdev->wb.enabled) {
57                 seq = le32_to_cpu(*rdev->fence_drv[ring].cpu_addr);
58         } else {
59                 seq = RREG32(rdev->fence_drv[ring].scratch_reg);
60         }
61         return seq;
62 }
63
64 int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence)
65 {
66         unsigned long irq_flags;
67
68         write_lock_irqsave(&rdev->fence_lock, irq_flags);
69         if (fence->emitted) {
70                 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
71                 return 0;
72         }
73         fence->seq = atomic_add_return(1, &rdev->fence_drv[fence->ring].seq);
74         if (!rdev->ring[fence->ring].ready)
75                 /* FIXME: cp is not running assume everythings is done right
76                  * away
77                  */
78                 radeon_fence_write(rdev, fence->seq, fence->ring);
79         else
80                 radeon_fence_ring_emit(rdev, fence->ring, fence);
81
82         trace_radeon_fence_emit(rdev->ddev, fence->seq);
83         fence->emitted = true;
84         list_move_tail(&fence->list, &rdev->fence_drv[fence->ring].emitted);
85         write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
86         return 0;
87 }
88
89 static bool radeon_fence_poll_locked(struct radeon_device *rdev, int ring)
90 {
91         struct radeon_fence *fence;
92         struct list_head *i, *n;
93         uint32_t seq;
94         bool wake = false;
95         unsigned long cjiffies;
96
97         seq = radeon_fence_read(rdev, ring);
98         if (seq != rdev->fence_drv[ring].last_seq) {
99                 rdev->fence_drv[ring].last_seq = seq;
100                 rdev->fence_drv[ring].last_jiffies = jiffies;
101                 rdev->fence_drv[ring].last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
102         } else {
103                 cjiffies = jiffies;
104                 if (time_after(cjiffies, rdev->fence_drv[ring].last_jiffies)) {
105                         cjiffies -= rdev->fence_drv[ring].last_jiffies;
106                         if (time_after(rdev->fence_drv[ring].last_timeout, cjiffies)) {
107                                 /* update the timeout */
108                                 rdev->fence_drv[ring].last_timeout -= cjiffies;
109                         } else {
110                                 /* the 500ms timeout is elapsed we should test
111                                  * for GPU lockup
112                                  */
113                                 rdev->fence_drv[ring].last_timeout = 1;
114                         }
115                 } else {
116                         /* wrap around update last jiffies, we will just wait
117                          * a little longer
118                          */
119                         rdev->fence_drv[ring].last_jiffies = cjiffies;
120                 }
121                 return false;
122         }
123         n = NULL;
124         list_for_each(i, &rdev->fence_drv[ring].emitted) {
125                 fence = list_entry(i, struct radeon_fence, list);
126                 if (fence->seq == seq) {
127                         n = i;
128                         break;
129                 }
130         }
131         /* all fence previous to this one are considered as signaled */
132         if (n) {
133                 i = n;
134                 do {
135                         n = i->prev;
136                         list_move_tail(i, &rdev->fence_drv[ring].signaled);
137                         fence = list_entry(i, struct radeon_fence, list);
138                         fence->signaled = true;
139                         i = n;
140                 } while (i != &rdev->fence_drv[ring].emitted);
141                 wake = true;
142         }
143         return wake;
144 }
145
146 static void radeon_fence_destroy(struct kref *kref)
147 {
148         unsigned long irq_flags;
149         struct radeon_fence *fence;
150
151         fence = container_of(kref, struct radeon_fence, kref);
152         write_lock_irqsave(&fence->rdev->fence_lock, irq_flags);
153         list_del(&fence->list);
154         fence->emitted = false;
155         write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags);
156         if (fence->semaphore)
157                 radeon_semaphore_free(fence->rdev, fence->semaphore);
158         kfree(fence);
159 }
160
161 int radeon_fence_create(struct radeon_device *rdev,
162                         struct radeon_fence **fence,
163                         int ring)
164 {
165         unsigned long irq_flags;
166
167         *fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);
168         if ((*fence) == NULL) {
169                 return -ENOMEM;
170         }
171         kref_init(&((*fence)->kref));
172         (*fence)->rdev = rdev;
173         (*fence)->emitted = false;
174         (*fence)->signaled = false;
175         (*fence)->seq = 0;
176         (*fence)->ring = ring;
177         (*fence)->semaphore = NULL;
178         INIT_LIST_HEAD(&(*fence)->list);
179
180         write_lock_irqsave(&rdev->fence_lock, irq_flags);
181         list_add_tail(&(*fence)->list, &rdev->fence_drv[ring].created);
182         write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
183         return 0;
184 }
185
186 bool radeon_fence_signaled(struct radeon_fence *fence)
187 {
188         unsigned long irq_flags;
189         bool signaled = false;
190
191         if (!fence)
192                 return true;
193
194         if (fence->rdev->gpu_lockup)
195                 return true;
196
197         write_lock_irqsave(&fence->rdev->fence_lock, irq_flags);
198         signaled = fence->signaled;
199         /* if we are shuting down report all fence as signaled */
200         if (fence->rdev->shutdown) {
201                 signaled = true;
202         }
203         if (!fence->emitted) {
204                 WARN(1, "Querying an unemitted fence : %p !\n", fence);
205                 signaled = true;
206         }
207         if (!signaled) {
208                 radeon_fence_poll_locked(fence->rdev, fence->ring);
209                 signaled = fence->signaled;
210         }
211         write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags);
212         return signaled;
213 }
214
215 int radeon_fence_wait(struct radeon_fence *fence, bool intr)
216 {
217         struct radeon_device *rdev;
218         unsigned long irq_flags, timeout;
219         u32 seq;
220         int r;
221
222         if (fence == NULL) {
223                 WARN(1, "Querying an invalid fence : %p !\n", fence);
224                 return 0;
225         }
226         rdev = fence->rdev;
227         if (radeon_fence_signaled(fence)) {
228                 return 0;
229         }
230         timeout = rdev->fence_drv[fence->ring].last_timeout;
231 retry:
232         /* save current sequence used to check for GPU lockup */
233         seq = rdev->fence_drv[fence->ring].last_seq;
234         trace_radeon_fence_wait_begin(rdev->ddev, seq);
235         if (intr) {
236                 radeon_irq_kms_sw_irq_get(rdev, fence->ring);
237                 r = wait_event_interruptible_timeout(rdev->fence_drv[fence->ring].queue,
238                                 radeon_fence_signaled(fence), timeout);
239                 radeon_irq_kms_sw_irq_put(rdev, fence->ring);
240                 if (unlikely(r < 0)) {
241                         return r;
242                 }
243         } else {
244                 radeon_irq_kms_sw_irq_get(rdev, fence->ring);
245                 r = wait_event_timeout(rdev->fence_drv[fence->ring].queue,
246                          radeon_fence_signaled(fence), timeout);
247                 radeon_irq_kms_sw_irq_put(rdev, fence->ring);
248         }
249         trace_radeon_fence_wait_end(rdev->ddev, seq);
250         if (unlikely(!radeon_fence_signaled(fence))) {
251                 /* we were interrupted for some reason and fence isn't
252                  * isn't signaled yet, resume wait
253                  */
254                 if (r) {
255                         timeout = r;
256                         goto retry;
257                 }
258                 /* don't protect read access to rdev->fence_drv[t].last_seq
259                  * if we experiencing a lockup the value doesn't change
260                  */
261                 if (seq == rdev->fence_drv[fence->ring].last_seq &&
262                     radeon_gpu_is_lockup(rdev, &rdev->ring[fence->ring])) {
263                         /* good news we believe it's a lockup */
264                         printk(KERN_WARNING "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n",
265                              fence->seq, seq);
266                         /* FIXME: what should we do ? marking everyone
267                          * as signaled for now
268                          */
269                         rdev->gpu_lockup = true;
270                         r = radeon_gpu_reset(rdev);
271                         if (r)
272                                 return r;
273                         radeon_fence_write(rdev, fence->seq, fence->ring);
274                         rdev->gpu_lockup = false;
275                 }
276                 timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
277                 write_lock_irqsave(&rdev->fence_lock, irq_flags);
278                 rdev->fence_drv[fence->ring].last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
279                 rdev->fence_drv[fence->ring].last_jiffies = jiffies;
280                 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
281                 goto retry;
282         }
283         return 0;
284 }
285
286 int radeon_fence_wait_next(struct radeon_device *rdev, int ring)
287 {
288         unsigned long irq_flags;
289         struct radeon_fence *fence;
290         int r;
291
292         if (rdev->gpu_lockup) {
293                 return 0;
294         }
295         write_lock_irqsave(&rdev->fence_lock, irq_flags);
296         if (list_empty(&rdev->fence_drv[ring].emitted)) {
297                 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
298                 return 0;
299         }
300         fence = list_entry(rdev->fence_drv[ring].emitted.next,
301                            struct radeon_fence, list);
302         radeon_fence_ref(fence);
303         write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
304         r = radeon_fence_wait(fence, false);
305         radeon_fence_unref(&fence);
306         return r;
307 }
308
309 int radeon_fence_wait_last(struct radeon_device *rdev, int ring)
310 {
311         unsigned long irq_flags;
312         struct radeon_fence *fence;
313         int r;
314
315         if (rdev->gpu_lockup) {
316                 return 0;
317         }
318         write_lock_irqsave(&rdev->fence_lock, irq_flags);
319         if (list_empty(&rdev->fence_drv[ring].emitted)) {
320                 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
321                 return 0;
322         }
323         fence = list_entry(rdev->fence_drv[ring].emitted.prev,
324                            struct radeon_fence, list);
325         radeon_fence_ref(fence);
326         write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
327         r = radeon_fence_wait(fence, false);
328         radeon_fence_unref(&fence);
329         return r;
330 }
331
332 struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence)
333 {
334         kref_get(&fence->kref);
335         return fence;
336 }
337
338 void radeon_fence_unref(struct radeon_fence **fence)
339 {
340         struct radeon_fence *tmp = *fence;
341
342         *fence = NULL;
343         if (tmp) {
344                 kref_put(&tmp->kref, radeon_fence_destroy);
345         }
346 }
347
348 void radeon_fence_process(struct radeon_device *rdev, int ring)
349 {
350         unsigned long irq_flags;
351         bool wake;
352
353         write_lock_irqsave(&rdev->fence_lock, irq_flags);
354         wake = radeon_fence_poll_locked(rdev, ring);
355         write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
356         if (wake) {
357                 wake_up_all(&rdev->fence_drv[ring].queue);
358         }
359 }
360
361 int radeon_fence_count_emitted(struct radeon_device *rdev, int ring)
362 {
363         unsigned long irq_flags;
364         int not_processed = 0;
365
366         read_lock_irqsave(&rdev->fence_lock, irq_flags);
367         if (!rdev->fence_drv[ring].initialized) {
368                 read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
369                 return 0;
370         }
371
372         if (!list_empty(&rdev->fence_drv[ring].emitted)) {
373                 struct list_head *ptr;
374                 list_for_each(ptr, &rdev->fence_drv[ring].emitted) {
375                         /* count up to 3, that's enought info */
376                         if (++not_processed >= 3)
377                                 break;
378                 }
379         }
380         read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
381         return not_processed;
382 }
383
384 int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring)
385 {
386         unsigned long irq_flags;
387         uint64_t index;
388         int r;
389
390         write_lock_irqsave(&rdev->fence_lock, irq_flags);
391         radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
392         if (rdev->wb.use_event) {
393                 rdev->fence_drv[ring].scratch_reg = 0;
394                 index = R600_WB_EVENT_OFFSET + ring * 4;
395         } else {
396                 r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg);
397                 if (r) {
398                         dev_err(rdev->dev, "fence failed to get scratch register\n");
399                         write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
400                         return r;
401                 }
402                 index = RADEON_WB_SCRATCH_OFFSET +
403                         rdev->fence_drv[ring].scratch_reg -
404                         rdev->scratch.reg_base;
405         }
406         rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4];
407         rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index;
408         radeon_fence_write(rdev, atomic_read(&rdev->fence_drv[ring].seq), ring);
409         rdev->fence_drv[ring].initialized = true;
410         DRM_INFO("fence driver on ring %d use gpu addr 0x%08Lx and cpu addr 0x%p\n",
411                  ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr);
412         write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
413         return 0;
414 }
415
416 static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring)
417 {
418         rdev->fence_drv[ring].scratch_reg = -1;
419         rdev->fence_drv[ring].cpu_addr = NULL;
420         rdev->fence_drv[ring].gpu_addr = 0;
421         atomic_set(&rdev->fence_drv[ring].seq, 0);
422         INIT_LIST_HEAD(&rdev->fence_drv[ring].created);
423         INIT_LIST_HEAD(&rdev->fence_drv[ring].emitted);
424         INIT_LIST_HEAD(&rdev->fence_drv[ring].signaled);
425         init_waitqueue_head(&rdev->fence_drv[ring].queue);
426         rdev->fence_drv[ring].initialized = false;
427 }
428
429 int radeon_fence_driver_init(struct radeon_device *rdev)
430 {
431         unsigned long irq_flags;
432         int ring;
433
434         write_lock_irqsave(&rdev->fence_lock, irq_flags);
435         for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
436                 radeon_fence_driver_init_ring(rdev, ring);
437         }
438         write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
439         if (radeon_debugfs_fence_init(rdev)) {
440                 dev_err(rdev->dev, "fence debugfs file creation failed\n");
441         }
442         return 0;
443 }
444
445 void radeon_fence_driver_fini(struct radeon_device *rdev)
446 {
447         unsigned long irq_flags;
448         int ring;
449
450         for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
451                 if (!rdev->fence_drv[ring].initialized)
452                         continue;
453                 radeon_fence_wait_last(rdev, ring);
454                 wake_up_all(&rdev->fence_drv[ring].queue);
455                 write_lock_irqsave(&rdev->fence_lock, irq_flags);
456                 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
457                 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
458                 rdev->fence_drv[ring].initialized = false;
459         }
460 }
461
462
463 /*
464  * Fence debugfs
465  */
466 #if defined(CONFIG_DEBUG_FS)
467 static int radeon_debugfs_fence_info(struct seq_file *m, void *data)
468 {
469         struct drm_info_node *node = (struct drm_info_node *)m->private;
470         struct drm_device *dev = node->minor->dev;
471         struct radeon_device *rdev = dev->dev_private;
472         struct radeon_fence *fence;
473         int i;
474
475         for (i = 0; i < RADEON_NUM_RINGS; ++i) {
476                 if (!rdev->fence_drv[i].initialized)
477                         continue;
478
479                 seq_printf(m, "--- ring %d ---\n", i);
480                 seq_printf(m, "Last signaled fence 0x%08X\n",
481                            radeon_fence_read(rdev, i));
482                 if (!list_empty(&rdev->fence_drv[i].emitted)) {
483                         fence = list_entry(rdev->fence_drv[i].emitted.prev,
484                                            struct radeon_fence, list);
485                         seq_printf(m, "Last emitted fence %p with 0x%08X\n",
486                                    fence,  fence->seq);
487                 }
488         }
489         return 0;
490 }
491
492 static struct drm_info_list radeon_debugfs_fence_list[] = {
493         {"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL},
494 };
495 #endif
496
497 int radeon_debugfs_fence_init(struct radeon_device *rdev)
498 {
499 #if defined(CONFIG_DEBUG_FS)
500         return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 1);
501 #else
502         return 0;
503 #endif
504 }