drm/i915: fix ACTHD for gen <= 3
[linux-2.6.git] / drivers / gpu / drm / i915 / intel_ringbuffer.c
1 /*
2  * Copyright © 2008-2010 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Eric Anholt <eric@anholt.net>
25  *    Zou Nan hai <nanhai.zou@intel.com>
26  *    Xiang Hai hao<haihao.xiang@intel.com>
27  *
28  */
29
30 #include "drmP.h"
31 #include "drm.h"
32 #include "i915_drv.h"
33 #include "i915_drm.h"
34 #include "i915_trace.h"
35 #include "intel_drv.h"
36
37 static u32 i915_gem_get_seqno(struct drm_device *dev)
38 {
39         drm_i915_private_t *dev_priv = dev->dev_private;
40         u32 seqno;
41
42         seqno = dev_priv->next_seqno;
43
44         /* reserve 0 for non-seqno */
45         if (++dev_priv->next_seqno == 0)
46                 dev_priv->next_seqno = 1;
47
48         return seqno;
49 }
50
51 static void
52 render_ring_flush(struct drm_device *dev,
53                   struct intel_ring_buffer *ring,
54                   u32   invalidate_domains,
55                   u32   flush_domains)
56 {
57         drm_i915_private_t *dev_priv = dev->dev_private;
58         u32 cmd;
59
60 #if WATCH_EXEC
61         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
62                   invalidate_domains, flush_domains);
63 #endif
64
65         trace_i915_gem_request_flush(dev, dev_priv->next_seqno,
66                                      invalidate_domains, flush_domains);
67
68         if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
69                 /*
70                  * read/write caches:
71                  *
72                  * I915_GEM_DOMAIN_RENDER is always invalidated, but is
73                  * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
74                  * also flushed at 2d versus 3d pipeline switches.
75                  *
76                  * read-only caches:
77                  *
78                  * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
79                  * MI_READ_FLUSH is set, and is always flushed on 965.
80                  *
81                  * I915_GEM_DOMAIN_COMMAND may not exist?
82                  *
83                  * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
84                  * invalidated when MI_EXE_FLUSH is set.
85                  *
86                  * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
87                  * invalidated with every MI_FLUSH.
88                  *
89                  * TLBs:
90                  *
91                  * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
92                  * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
93                  * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
94                  * are flushed at any MI_FLUSH.
95                  */
96
97                 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
98                 if ((invalidate_domains|flush_domains) &
99                     I915_GEM_DOMAIN_RENDER)
100                         cmd &= ~MI_NO_WRITE_FLUSH;
101                 if (INTEL_INFO(dev)->gen < 4) {
102                         /*
103                          * On the 965, the sampler cache always gets flushed
104                          * and this bit is reserved.
105                          */
106                         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
107                                 cmd |= MI_READ_FLUSH;
108                 }
109                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
110                         cmd |= MI_EXE_FLUSH;
111
112 #if WATCH_EXEC
113                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
114 #endif
115                 intel_ring_begin(dev, ring, 2);
116                 intel_ring_emit(dev, ring, cmd);
117                 intel_ring_emit(dev, ring, MI_NOOP);
118                 intel_ring_advance(dev, ring);
119         }
120 }
121
122 static void ring_set_tail(struct drm_device *dev,
123                           struct intel_ring_buffer *ring,
124                           u32 value)
125 {
126         drm_i915_private_t *dev_priv = dev->dev_private;
127         I915_WRITE_TAIL(ring, ring->tail);
128 }
129
130 static unsigned int render_ring_get_active_head(struct drm_device *dev,
131                                                 struct intel_ring_buffer *ring)
132 {
133         drm_i915_private_t *dev_priv = dev->dev_private;
134         u32 acthd_reg = INTEL_INFO(dev)->gen >= 4 ? ACTHD_I965 : ACTHD;
135
136         return I915_READ(acthd_reg);
137 }
138
139 static int init_ring_common(struct drm_device *dev,
140                             struct intel_ring_buffer *ring)
141 {
142         u32 head;
143         drm_i915_private_t *dev_priv = dev->dev_private;
144         struct drm_i915_gem_object *obj_priv;
145         obj_priv = to_intel_bo(ring->gem_object);
146
147         /* Stop the ring if it's running. */
148         I915_WRITE_CTL(ring, 0);
149         I915_WRITE_HEAD(ring, 0);
150         ring->set_tail(dev, ring, 0);
151
152         /* Initialize the ring. */
153         I915_WRITE_START(ring, obj_priv->gtt_offset);
154         head = I915_READ_HEAD(ring) & HEAD_ADDR;
155
156         /* G45 ring initialization fails to reset head to zero */
157         if (head != 0) {
158                 DRM_ERROR("%s head not reset to zero "
159                                 "ctl %08x head %08x tail %08x start %08x\n",
160                                 ring->name,
161                                 I915_READ_CTL(ring),
162                                 I915_READ_HEAD(ring),
163                                 I915_READ_TAIL(ring),
164                                 I915_READ_START(ring));
165
166                 I915_WRITE_HEAD(ring, 0);
167
168                 DRM_ERROR("%s head forced to zero "
169                                 "ctl %08x head %08x tail %08x start %08x\n",
170                                 ring->name,
171                                 I915_READ_CTL(ring),
172                                 I915_READ_HEAD(ring),
173                                 I915_READ_TAIL(ring),
174                                 I915_READ_START(ring));
175         }
176
177         I915_WRITE_CTL(ring,
178                         ((ring->gem_object->size - PAGE_SIZE) & RING_NR_PAGES)
179                         | RING_NO_REPORT | RING_VALID);
180
181         head = I915_READ_HEAD(ring) & HEAD_ADDR;
182         /* If the head is still not zero, the ring is dead */
183         if (head != 0) {
184                 DRM_ERROR("%s initialization failed "
185                                 "ctl %08x head %08x tail %08x start %08x\n",
186                                 ring->name,
187                                 I915_READ_CTL(ring),
188                                 I915_READ_HEAD(ring),
189                                 I915_READ_TAIL(ring),
190                                 I915_READ_START(ring));
191                 return -EIO;
192         }
193
194         if (!drm_core_check_feature(dev, DRIVER_MODESET))
195                 i915_kernel_lost_context(dev);
196         else {
197                 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
198                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
199                 ring->space = ring->head - (ring->tail + 8);
200                 if (ring->space < 0)
201                         ring->space += ring->size;
202         }
203         return 0;
204 }
205
206 static int init_render_ring(struct drm_device *dev,
207                             struct intel_ring_buffer *ring)
208 {
209         drm_i915_private_t *dev_priv = dev->dev_private;
210         int ret = init_ring_common(dev, ring);
211         int mode;
212
213         if (INTEL_INFO(dev)->gen > 3) {
214                 mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
215                 if (IS_GEN6(dev))
216                         mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
217                 I915_WRITE(MI_MODE, mode);
218         }
219         return ret;
220 }
221
222 #define PIPE_CONTROL_FLUSH(addr)                                        \
223 do {                                                                    \
224         OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |          \
225                  PIPE_CONTROL_DEPTH_STALL | 2);                         \
226         OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT);                       \
227         OUT_RING(0);                                                    \
228         OUT_RING(0);                                                    \
229 } while (0)
230
231 /**
232  * Creates a new sequence number, emitting a write of it to the status page
233  * plus an interrupt, which will trigger i915_user_interrupt_handler.
234  *
235  * Must be called with struct_lock held.
236  *
237  * Returned sequence numbers are nonzero on success.
238  */
239 static u32
240 render_ring_add_request(struct drm_device *dev,
241                         struct intel_ring_buffer *ring,
242                         u32 flush_domains)
243 {
244         drm_i915_private_t *dev_priv = dev->dev_private;
245         u32 seqno;
246
247         seqno = i915_gem_get_seqno(dev);
248
249         if (IS_GEN6(dev)) {
250                 BEGIN_LP_RING(6);
251                 OUT_RING(GFX_OP_PIPE_CONTROL | 3);
252                 OUT_RING(PIPE_CONTROL_QW_WRITE |
253                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH |
254                          PIPE_CONTROL_NOTIFY);
255                 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
256                 OUT_RING(seqno);
257                 OUT_RING(0);
258                 OUT_RING(0);
259                 ADVANCE_LP_RING();
260         } else if (HAS_PIPE_CONTROL(dev)) {
261                 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
262
263                 /*
264                  * Workaround qword write incoherence by flushing the
265                  * PIPE_NOTIFY buffers out to memory before requesting
266                  * an interrupt.
267                  */
268                 BEGIN_LP_RING(32);
269                 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
270                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
271                 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
272                 OUT_RING(seqno);
273                 OUT_RING(0);
274                 PIPE_CONTROL_FLUSH(scratch_addr);
275                 scratch_addr += 128; /* write to separate cachelines */
276                 PIPE_CONTROL_FLUSH(scratch_addr);
277                 scratch_addr += 128;
278                 PIPE_CONTROL_FLUSH(scratch_addr);
279                 scratch_addr += 128;
280                 PIPE_CONTROL_FLUSH(scratch_addr);
281                 scratch_addr += 128;
282                 PIPE_CONTROL_FLUSH(scratch_addr);
283                 scratch_addr += 128;
284                 PIPE_CONTROL_FLUSH(scratch_addr);
285                 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
286                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
287                          PIPE_CONTROL_NOTIFY);
288                 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
289                 OUT_RING(seqno);
290                 OUT_RING(0);
291                 ADVANCE_LP_RING();
292         } else {
293                 BEGIN_LP_RING(4);
294                 OUT_RING(MI_STORE_DWORD_INDEX);
295                 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
296                 OUT_RING(seqno);
297
298                 OUT_RING(MI_USER_INTERRUPT);
299                 ADVANCE_LP_RING();
300         }
301         return seqno;
302 }
303
304 static u32
305 render_ring_get_seqno(struct drm_device *dev,
306                       struct intel_ring_buffer *ring)
307 {
308         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
309         if (HAS_PIPE_CONTROL(dev))
310                 return ((volatile u32 *)(dev_priv->seqno_page))[0];
311         else
312                 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
313 }
314
315 static void
316 render_ring_get_user_irq(struct drm_device *dev,
317                          struct intel_ring_buffer *ring)
318 {
319         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
320         unsigned long irqflags;
321
322         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
323         if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) {
324                 if (HAS_PCH_SPLIT(dev))
325                         ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
326                 else
327                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
328         }
329         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
330 }
331
332 static void
333 render_ring_put_user_irq(struct drm_device *dev,
334                          struct intel_ring_buffer *ring)
335 {
336         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
337         unsigned long irqflags;
338
339         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
340         BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0);
341         if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
342                 if (HAS_PCH_SPLIT(dev))
343                         ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
344                 else
345                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
346         }
347         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
348 }
349
350 static void render_setup_status_page(struct drm_device *dev,
351                                      struct     intel_ring_buffer *ring)
352 {
353         drm_i915_private_t *dev_priv = dev->dev_private;
354         if (IS_GEN6(dev)) {
355                 I915_WRITE(HWS_PGA_GEN6, ring->status_page.gfx_addr);
356                 I915_READ(HWS_PGA_GEN6); /* posting read */
357         } else {
358                 I915_WRITE(HWS_PGA, ring->status_page.gfx_addr);
359                 I915_READ(HWS_PGA); /* posting read */
360         }
361
362 }
363
364 static void
365 bsd_ring_flush(struct drm_device *dev,
366                 struct intel_ring_buffer *ring,
367                 u32     invalidate_domains,
368                 u32     flush_domains)
369 {
370         intel_ring_begin(dev, ring, 2);
371         intel_ring_emit(dev, ring, MI_FLUSH);
372         intel_ring_emit(dev, ring, MI_NOOP);
373         intel_ring_advance(dev, ring);
374 }
375
376 static unsigned int bsd_ring_get_active_head(struct drm_device *dev,
377                                              struct intel_ring_buffer *ring)
378 {
379         drm_i915_private_t *dev_priv = dev->dev_private;
380         return I915_READ(BSD_RING_ACTHD);
381 }
382
383 static int init_bsd_ring(struct drm_device *dev,
384                          struct intel_ring_buffer *ring)
385 {
386         return init_ring_common(dev, ring);
387 }
388
389 static u32
390 bsd_ring_add_request(struct drm_device *dev,
391                      struct intel_ring_buffer *ring,
392                      u32 flush_domains)
393 {
394         u32 seqno;
395
396         seqno = i915_gem_get_seqno(dev);
397
398         intel_ring_begin(dev, ring, 4);
399         intel_ring_emit(dev, ring, MI_STORE_DWORD_INDEX);
400         intel_ring_emit(dev, ring,
401                         I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
402         intel_ring_emit(dev, ring, seqno);
403         intel_ring_emit(dev, ring, MI_USER_INTERRUPT);
404         intel_ring_advance(dev, ring);
405
406         DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
407
408         return seqno;
409 }
410
411 static void bsd_setup_status_page(struct drm_device *dev,
412                                   struct  intel_ring_buffer *ring)
413 {
414         drm_i915_private_t *dev_priv = dev->dev_private;
415         I915_WRITE(BSD_HWS_PGA, ring->status_page.gfx_addr);
416         I915_READ(BSD_HWS_PGA);
417 }
418
419 static void
420 bsd_ring_get_user_irq(struct drm_device *dev,
421                       struct intel_ring_buffer *ring)
422 {
423         /* do nothing */
424 }
425 static void
426 bsd_ring_put_user_irq(struct drm_device *dev,
427                       struct intel_ring_buffer *ring)
428 {
429         /* do nothing */
430 }
431
432 static u32
433 bsd_ring_get_seqno(struct drm_device *dev,
434                    struct intel_ring_buffer *ring)
435 {
436         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
437 }
438
439 static int
440 bsd_ring_dispatch_gem_execbuffer(struct drm_device *dev,
441                                  struct intel_ring_buffer *ring,
442                                  struct drm_i915_gem_execbuffer2 *exec,
443                                  struct drm_clip_rect *cliprects,
444                                  uint64_t exec_offset)
445 {
446         uint32_t exec_start;
447         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
448         intel_ring_begin(dev, ring, 2);
449         intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START |
450                         (2 << 6) | MI_BATCH_NON_SECURE_I965);
451         intel_ring_emit(dev, ring, exec_start);
452         intel_ring_advance(dev, ring);
453         return 0;
454 }
455
456
457 static int
458 render_ring_dispatch_gem_execbuffer(struct drm_device *dev,
459                                     struct intel_ring_buffer *ring,
460                                     struct drm_i915_gem_execbuffer2 *exec,
461                                     struct drm_clip_rect *cliprects,
462                                     uint64_t exec_offset)
463 {
464         drm_i915_private_t *dev_priv = dev->dev_private;
465         int nbox = exec->num_cliprects;
466         int i = 0, count;
467         uint32_t exec_start, exec_len;
468         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
469         exec_len = (uint32_t) exec->batch_len;
470
471         trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
472
473         count = nbox ? nbox : 1;
474
475         for (i = 0; i < count; i++) {
476                 if (i < nbox) {
477                         int ret = i915_emit_box(dev, cliprects, i,
478                                                 exec->DR1, exec->DR4);
479                         if (ret)
480                                 return ret;
481                 }
482
483                 if (IS_I830(dev) || IS_845G(dev)) {
484                         intel_ring_begin(dev, ring, 4);
485                         intel_ring_emit(dev, ring, MI_BATCH_BUFFER);
486                         intel_ring_emit(dev, ring,
487                                         exec_start | MI_BATCH_NON_SECURE);
488                         intel_ring_emit(dev, ring, exec_start + exec_len - 4);
489                         intel_ring_emit(dev, ring, 0);
490                 } else {
491                         intel_ring_begin(dev, ring, 4);
492                         if (INTEL_INFO(dev)->gen >= 4) {
493                                 intel_ring_emit(dev, ring,
494                                                 MI_BATCH_BUFFER_START | (2 << 6)
495                                                 | MI_BATCH_NON_SECURE_I965);
496                                 intel_ring_emit(dev, ring, exec_start);
497                         } else {
498                                 intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START
499                                                 | (2 << 6));
500                                 intel_ring_emit(dev, ring, exec_start |
501                                                 MI_BATCH_NON_SECURE);
502                         }
503                 }
504                 intel_ring_advance(dev, ring);
505         }
506
507         if (IS_G4X(dev) || IS_IRONLAKE(dev)) {
508                 intel_ring_begin(dev, ring, 2);
509                 intel_ring_emit(dev, ring, MI_FLUSH |
510                                 MI_NO_WRITE_FLUSH |
511                                 MI_INVALIDATE_ISP );
512                 intel_ring_emit(dev, ring, MI_NOOP);
513                 intel_ring_advance(dev, ring);
514         }
515         /* XXX breadcrumb */
516
517         return 0;
518 }
519
520 static void cleanup_status_page(struct drm_device *dev,
521                                 struct intel_ring_buffer *ring)
522 {
523         drm_i915_private_t *dev_priv = dev->dev_private;
524         struct drm_gem_object *obj;
525         struct drm_i915_gem_object *obj_priv;
526
527         obj = ring->status_page.obj;
528         if (obj == NULL)
529                 return;
530         obj_priv = to_intel_bo(obj);
531
532         kunmap(obj_priv->pages[0]);
533         i915_gem_object_unpin(obj);
534         drm_gem_object_unreference(obj);
535         ring->status_page.obj = NULL;
536
537         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
538 }
539
540 static int init_status_page(struct drm_device *dev,
541                             struct intel_ring_buffer *ring)
542 {
543         drm_i915_private_t *dev_priv = dev->dev_private;
544         struct drm_gem_object *obj;
545         struct drm_i915_gem_object *obj_priv;
546         int ret;
547
548         obj = i915_gem_alloc_object(dev, 4096);
549         if (obj == NULL) {
550                 DRM_ERROR("Failed to allocate status page\n");
551                 ret = -ENOMEM;
552                 goto err;
553         }
554         obj_priv = to_intel_bo(obj);
555         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
556
557         ret = i915_gem_object_pin(obj, 4096);
558         if (ret != 0) {
559                 goto err_unref;
560         }
561
562         ring->status_page.gfx_addr = obj_priv->gtt_offset;
563         ring->status_page.page_addr = kmap(obj_priv->pages[0]);
564         if (ring->status_page.page_addr == NULL) {
565                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
566                 goto err_unpin;
567         }
568         ring->status_page.obj = obj;
569         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
570
571         ring->setup_status_page(dev, ring);
572         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
573                         ring->name, ring->status_page.gfx_addr);
574
575         return 0;
576
577 err_unpin:
578         i915_gem_object_unpin(obj);
579 err_unref:
580         drm_gem_object_unreference(obj);
581 err:
582         return ret;
583 }
584
585 int intel_init_ring_buffer(struct drm_device *dev,
586                            struct intel_ring_buffer *ring)
587 {
588         struct drm_i915_private *dev_priv = dev->dev_private;
589         struct drm_i915_gem_object *obj_priv;
590         struct drm_gem_object *obj;
591         int ret;
592
593         ring->dev = dev;
594
595         if (I915_NEED_GFX_HWS(dev)) {
596                 ret = init_status_page(dev, ring);
597                 if (ret)
598                         return ret;
599         }
600
601         obj = i915_gem_alloc_object(dev, ring->size);
602         if (obj == NULL) {
603                 DRM_ERROR("Failed to allocate ringbuffer\n");
604                 ret = -ENOMEM;
605                 goto err_hws;
606         }
607
608         ring->gem_object = obj;
609
610         ret = i915_gem_object_pin(obj, PAGE_SIZE);
611         if (ret)
612                 goto err_unref;
613
614         obj_priv = to_intel_bo(obj);
615         ring->map.size = ring->size;
616         ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
617         ring->map.type = 0;
618         ring->map.flags = 0;
619         ring->map.mtrr = 0;
620
621         drm_core_ioremap_wc(&ring->map, dev);
622         if (ring->map.handle == NULL) {
623                 DRM_ERROR("Failed to map ringbuffer.\n");
624                 ret = -EINVAL;
625                 goto err_unpin;
626         }
627
628         ring->virtual_start = ring->map.handle;
629         ret = ring->init(dev, ring);
630         if (ret)
631                 goto err_unmap;
632
633         if (!drm_core_check_feature(dev, DRIVER_MODESET))
634                 i915_kernel_lost_context(dev);
635         else {
636                 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
637                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
638                 ring->space = ring->head - (ring->tail + 8);
639                 if (ring->space < 0)
640                         ring->space += ring->size;
641         }
642         INIT_LIST_HEAD(&ring->active_list);
643         INIT_LIST_HEAD(&ring->request_list);
644         return ret;
645
646 err_unmap:
647         drm_core_ioremapfree(&ring->map, dev);
648 err_unpin:
649         i915_gem_object_unpin(obj);
650 err_unref:
651         drm_gem_object_unreference(obj);
652         ring->gem_object = NULL;
653 err_hws:
654         cleanup_status_page(dev, ring);
655         return ret;
656 }
657
658 void intel_cleanup_ring_buffer(struct drm_device *dev,
659                                struct intel_ring_buffer *ring)
660 {
661         if (ring->gem_object == NULL)
662                 return;
663
664         drm_core_ioremapfree(&ring->map, dev);
665
666         i915_gem_object_unpin(ring->gem_object);
667         drm_gem_object_unreference(ring->gem_object);
668         ring->gem_object = NULL;
669         cleanup_status_page(dev, ring);
670 }
671
672 static int intel_wrap_ring_buffer(struct drm_device *dev,
673                                   struct intel_ring_buffer *ring)
674 {
675         unsigned int *virt;
676         int rem;
677         rem = ring->size - ring->tail;
678
679         if (ring->space < rem) {
680                 int ret = intel_wait_ring_buffer(dev, ring, rem);
681                 if (ret)
682                         return ret;
683         }
684
685         virt = (unsigned int *)(ring->virtual_start + ring->tail);
686         rem /= 8;
687         while (rem--) {
688                 *virt++ = MI_NOOP;
689                 *virt++ = MI_NOOP;
690         }
691
692         ring->tail = 0;
693         ring->space = ring->head - 8;
694
695         return 0;
696 }
697
698 int intel_wait_ring_buffer(struct drm_device *dev,
699                            struct intel_ring_buffer *ring, int n)
700 {
701         unsigned long end;
702         drm_i915_private_t *dev_priv = dev->dev_private;
703
704         trace_i915_ring_wait_begin (dev);
705         end = jiffies + 3 * HZ;
706         do {
707                 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
708                 ring->space = ring->head - (ring->tail + 8);
709                 if (ring->space < 0)
710                         ring->space += ring->size;
711                 if (ring->space >= n) {
712                         trace_i915_ring_wait_end (dev);
713                         return 0;
714                 }
715
716                 if (dev->primary->master) {
717                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
718                         if (master_priv->sarea_priv)
719                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
720                 }
721
722                 yield();
723         } while (!time_after(jiffies, end));
724         trace_i915_ring_wait_end (dev);
725         return -EBUSY;
726 }
727
728 void intel_ring_begin(struct drm_device *dev,
729                       struct intel_ring_buffer *ring,
730                       int num_dwords)
731 {
732         int n = 4*num_dwords;
733         if (unlikely(ring->tail + n > ring->size))
734                 intel_wrap_ring_buffer(dev, ring);
735         if (unlikely(ring->space < n))
736                 intel_wait_ring_buffer(dev, ring, n);
737
738         ring->space -= n;
739 }
740
741 void intel_ring_advance(struct drm_device *dev,
742                         struct intel_ring_buffer *ring)
743 {
744         ring->tail &= ring->size - 1;
745         ring->set_tail(dev, ring, ring->tail);
746 }
747
748 void intel_fill_struct(struct drm_device *dev,
749                        struct intel_ring_buffer *ring,
750                        void *data,
751                        unsigned int len)
752 {
753         unsigned int *virt = ring->virtual_start + ring->tail;
754         BUG_ON((len&~(4-1)) != 0);
755         intel_ring_begin(dev, ring, len/4);
756         memcpy(virt, data, len);
757         ring->tail += len;
758         ring->tail &= ring->size - 1;
759         ring->space -= len;
760         intel_ring_advance(dev, ring);
761 }
762
763 static const struct intel_ring_buffer render_ring = {
764         .name                   = "render ring",
765         .id                     = RING_RENDER,
766         .mmio_base              = RENDER_RING_BASE,
767         .size                   = 32 * PAGE_SIZE,
768         .setup_status_page      = render_setup_status_page,
769         .init                   = init_render_ring,
770         .set_tail               = ring_set_tail,
771         .get_active_head        = render_ring_get_active_head,
772         .flush                  = render_ring_flush,
773         .add_request            = render_ring_add_request,
774         .get_seqno              = render_ring_get_seqno,
775         .user_irq_get           = render_ring_get_user_irq,
776         .user_irq_put           = render_ring_put_user_irq,
777         .dispatch_gem_execbuffer = render_ring_dispatch_gem_execbuffer,
778 };
779
780 /* ring buffer for bit-stream decoder */
781
782 static const struct intel_ring_buffer bsd_ring = {
783         .name                   = "bsd ring",
784         .id                     = RING_BSD,
785         .mmio_base              = BSD_RING_BASE,
786         .size                   = 32 * PAGE_SIZE,
787         .setup_status_page      = bsd_setup_status_page,
788         .init                   = init_bsd_ring,
789         .set_tail               = ring_set_tail,
790         .get_active_head        = bsd_ring_get_active_head,
791         .flush                  = bsd_ring_flush,
792         .add_request            = bsd_ring_add_request,
793         .get_seqno              = bsd_ring_get_seqno,
794         .user_irq_get           = bsd_ring_get_user_irq,
795         .user_irq_put           = bsd_ring_put_user_irq,
796         .dispatch_gem_execbuffer = bsd_ring_dispatch_gem_execbuffer,
797 };
798
799
800 static void gen6_bsd_setup_status_page(struct drm_device *dev,
801                                        struct  intel_ring_buffer *ring)
802 {
803        drm_i915_private_t *dev_priv = dev->dev_private;
804        I915_WRITE(GEN6_BSD_HWS_PGA, ring->status_page.gfx_addr);
805        I915_READ(GEN6_BSD_HWS_PGA);
806 }
807
808 static void gen6_bsd_ring_set_tail(struct drm_device *dev,
809                                    struct intel_ring_buffer *ring,
810                                    u32 value)
811 {
812        drm_i915_private_t *dev_priv = dev->dev_private;
813
814        /* Every tail move must follow the sequence below */
815        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
816                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
817                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
818        I915_WRITE(GEN6_BSD_RNCID, 0x0);
819
820        if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
821                                GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
822                        50))
823                DRM_ERROR("timed out waiting for IDLE Indicator\n");
824
825        I915_WRITE_TAIL(ring, value);
826        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
827                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
828                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
829 }
830
831 static unsigned int gen6_bsd_ring_get_active_head(struct drm_device *dev,
832                                                   struct intel_ring_buffer *ring)
833 {
834        drm_i915_private_t *dev_priv = dev->dev_private;
835        return I915_READ(GEN6_BSD_RING_ACTHD);
836 }
837
838 static void gen6_bsd_ring_flush(struct drm_device *dev,
839                                 struct intel_ring_buffer *ring,
840                                 u32 invalidate_domains,
841                                 u32 flush_domains)
842 {
843        intel_ring_begin(dev, ring, 4);
844        intel_ring_emit(dev, ring, MI_FLUSH_DW);
845        intel_ring_emit(dev, ring, 0);
846        intel_ring_emit(dev, ring, 0);
847        intel_ring_emit(dev, ring, 0);
848        intel_ring_advance(dev, ring);
849 }
850
851 static int
852 gen6_bsd_ring_dispatch_gem_execbuffer(struct drm_device *dev,
853                                       struct intel_ring_buffer *ring,
854                                       struct drm_i915_gem_execbuffer2 *exec,
855                                       struct drm_clip_rect *cliprects,
856                                       uint64_t exec_offset)
857 {
858        uint32_t exec_start;
859
860        exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
861
862        intel_ring_begin(dev, ring, 2);
863        intel_ring_emit(dev, ring,
864                        MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
865        /* bit0-7 is the length on GEN6+ */
866        intel_ring_emit(dev, ring, exec_start);
867        intel_ring_advance(dev, ring);
868
869        return 0;
870 }
871
872 /* ring buffer for Video Codec for Gen6+ */
873 static const struct intel_ring_buffer gen6_bsd_ring = {
874        .name                    = "gen6 bsd ring",
875        .id                      = RING_BSD,
876        .mmio_base               = GEN6_BSD_RING_BASE,
877        .size                    = 32 * PAGE_SIZE,
878        .setup_status_page       = gen6_bsd_setup_status_page,
879        .init                    = init_bsd_ring,
880        .set_tail                = gen6_bsd_ring_set_tail,
881        .get_active_head         = gen6_bsd_ring_get_active_head,
882        .flush                   = gen6_bsd_ring_flush,
883        .add_request             = bsd_ring_add_request,
884        .get_seqno               = bsd_ring_get_seqno,
885        .user_irq_get            = bsd_ring_get_user_irq,
886        .user_irq_put            = bsd_ring_put_user_irq,
887        .dispatch_gem_execbuffer = gen6_bsd_ring_dispatch_gem_execbuffer,
888 };
889
890 int intel_init_render_ring_buffer(struct drm_device *dev)
891 {
892         drm_i915_private_t *dev_priv = dev->dev_private;
893
894         dev_priv->render_ring = render_ring;
895
896         if (!I915_NEED_GFX_HWS(dev)) {
897                 dev_priv->render_ring.status_page.page_addr
898                         = dev_priv->status_page_dmah->vaddr;
899                 memset(dev_priv->render_ring.status_page.page_addr,
900                                 0, PAGE_SIZE);
901         }
902
903         return intel_init_ring_buffer(dev, &dev_priv->render_ring);
904 }
905
906 int intel_init_bsd_ring_buffer(struct drm_device *dev)
907 {
908         drm_i915_private_t *dev_priv = dev->dev_private;
909
910         if (IS_GEN6(dev))
911                 dev_priv->bsd_ring = gen6_bsd_ring;
912         else
913                 dev_priv->bsd_ring = bsd_ring;
914
915         return intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
916 }