drm/i915: Move the implementation details of PIPE_CONTROL to the ringbuffer
[linux-2.6.git] / drivers / gpu / drm / i915 / intel_ringbuffer.c
1 /*
2  * Copyright © 2008-2010 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Eric Anholt <eric@anholt.net>
25  *    Zou Nan hai <nanhai.zou@intel.com>
26  *    Xiang Hai hao<haihao.xiang@intel.com>
27  *
28  */
29
30 #include "drmP.h"
31 #include "drm.h"
32 #include "i915_drv.h"
33 #include "i915_drm.h"
34 #include "i915_trace.h"
35 #include "intel_drv.h"
36
37 static u32 i915_gem_get_seqno(struct drm_device *dev)
38 {
39         drm_i915_private_t *dev_priv = dev->dev_private;
40         u32 seqno;
41
42         seqno = dev_priv->next_seqno;
43
44         /* reserve 0 for non-seqno */
45         if (++dev_priv->next_seqno == 0)
46                 dev_priv->next_seqno = 1;
47
48         return seqno;
49 }
50
51 static void
52 render_ring_flush(struct intel_ring_buffer *ring,
53                   u32   invalidate_domains,
54                   u32   flush_domains)
55 {
56         struct drm_device *dev = ring->dev;
57         drm_i915_private_t *dev_priv = dev->dev_private;
58         u32 cmd;
59
60 #if WATCH_EXEC
61         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
62                   invalidate_domains, flush_domains);
63 #endif
64
65         trace_i915_gem_request_flush(dev, dev_priv->next_seqno,
66                                      invalidate_domains, flush_domains);
67
68         if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
69                 /*
70                  * read/write caches:
71                  *
72                  * I915_GEM_DOMAIN_RENDER is always invalidated, but is
73                  * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
74                  * also flushed at 2d versus 3d pipeline switches.
75                  *
76                  * read-only caches:
77                  *
78                  * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
79                  * MI_READ_FLUSH is set, and is always flushed on 965.
80                  *
81                  * I915_GEM_DOMAIN_COMMAND may not exist?
82                  *
83                  * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
84                  * invalidated when MI_EXE_FLUSH is set.
85                  *
86                  * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
87                  * invalidated with every MI_FLUSH.
88                  *
89                  * TLBs:
90                  *
91                  * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
92                  * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
93                  * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
94                  * are flushed at any MI_FLUSH.
95                  */
96
97                 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
98                 if ((invalidate_domains|flush_domains) &
99                     I915_GEM_DOMAIN_RENDER)
100                         cmd &= ~MI_NO_WRITE_FLUSH;
101                 if (INTEL_INFO(dev)->gen < 4) {
102                         /*
103                          * On the 965, the sampler cache always gets flushed
104                          * and this bit is reserved.
105                          */
106                         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
107                                 cmd |= MI_READ_FLUSH;
108                 }
109                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
110                         cmd |= MI_EXE_FLUSH;
111
112 #if WATCH_EXEC
113                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
114 #endif
115                 if (intel_ring_begin(ring, 2) == 0) {
116                         intel_ring_emit(ring, cmd);
117                         intel_ring_emit(ring, MI_NOOP);
118                         intel_ring_advance(ring);
119                 }
120         }
121 }
122
123 static void ring_write_tail(struct intel_ring_buffer *ring,
124                             u32 value)
125 {
126         drm_i915_private_t *dev_priv = ring->dev->dev_private;
127         I915_WRITE_TAIL(ring, value);
128 }
129
130 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
131 {
132         drm_i915_private_t *dev_priv = ring->dev->dev_private;
133         u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
134                         RING_ACTHD(ring->mmio_base) : ACTHD;
135
136         return I915_READ(acthd_reg);
137 }
138
139 static int init_ring_common(struct intel_ring_buffer *ring)
140 {
141         drm_i915_private_t *dev_priv = ring->dev->dev_private;
142         struct drm_i915_gem_object *obj = ring->obj;
143         u32 head;
144
145         /* Stop the ring if it's running. */
146         I915_WRITE_CTL(ring, 0);
147         I915_WRITE_HEAD(ring, 0);
148         ring->write_tail(ring, 0);
149
150         /* Initialize the ring. */
151         I915_WRITE_START(ring, obj->gtt_offset);
152         head = I915_READ_HEAD(ring) & HEAD_ADDR;
153
154         /* G45 ring initialization fails to reset head to zero */
155         if (head != 0) {
156                 DRM_ERROR("%s head not reset to zero "
157                                 "ctl %08x head %08x tail %08x start %08x\n",
158                                 ring->name,
159                                 I915_READ_CTL(ring),
160                                 I915_READ_HEAD(ring),
161                                 I915_READ_TAIL(ring),
162                                 I915_READ_START(ring));
163
164                 I915_WRITE_HEAD(ring, 0);
165
166                 DRM_ERROR("%s head forced to zero "
167                                 "ctl %08x head %08x tail %08x start %08x\n",
168                                 ring->name,
169                                 I915_READ_CTL(ring),
170                                 I915_READ_HEAD(ring),
171                                 I915_READ_TAIL(ring),
172                                 I915_READ_START(ring));
173         }
174
175         I915_WRITE_CTL(ring,
176                         ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
177                         | RING_REPORT_64K | RING_VALID);
178
179         /* If the head is still not zero, the ring is dead */
180         if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
181             I915_READ_START(ring) != obj->gtt_offset ||
182             (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
183                 DRM_ERROR("%s initialization failed "
184                                 "ctl %08x head %08x tail %08x start %08x\n",
185                                 ring->name,
186                                 I915_READ_CTL(ring),
187                                 I915_READ_HEAD(ring),
188                                 I915_READ_TAIL(ring),
189                                 I915_READ_START(ring));
190                 return -EIO;
191         }
192
193         if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
194                 i915_kernel_lost_context(ring->dev);
195         else {
196                 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
197                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
198                 ring->space = ring->head - (ring->tail + 8);
199                 if (ring->space < 0)
200                         ring->space += ring->size;
201         }
202         return 0;
203 }
204
205 /*
206  * 965+ support PIPE_CONTROL commands, which provide finer grained control
207  * over cache flushing.
208  */
209 struct pipe_control {
210         struct drm_i915_gem_object *obj;
211         volatile u32 *cpu_page;
212         u32 gtt_offset;
213 };
214
215 static int
216 init_pipe_control(struct intel_ring_buffer *ring)
217 {
218         struct pipe_control *pc;
219         struct drm_i915_gem_object *obj;
220         int ret;
221
222         if (ring->private)
223                 return 0;
224
225         pc = kmalloc(sizeof(*pc), GFP_KERNEL);
226         if (!pc)
227                 return -ENOMEM;
228
229         obj = i915_gem_alloc_object(ring->dev, 4096);
230         if (obj == NULL) {
231                 DRM_ERROR("Failed to allocate seqno page\n");
232                 ret = -ENOMEM;
233                 goto err;
234         }
235         obj->agp_type = AGP_USER_CACHED_MEMORY;
236
237         ret = i915_gem_object_pin(obj, 4096, true);
238         if (ret)
239                 goto err_unref;
240
241         pc->gtt_offset = obj->gtt_offset;
242         pc->cpu_page =  kmap(obj->pages[0]);
243         if (pc->cpu_page == NULL)
244                 goto err_unpin;
245
246         pc->obj = obj;
247         ring->private = pc;
248         return 0;
249
250 err_unpin:
251         i915_gem_object_unpin(obj);
252 err_unref:
253         drm_gem_object_unreference(&obj->base);
254 err:
255         kfree(pc);
256         return ret;
257 }
258
259 static void
260 cleanup_pipe_control(struct intel_ring_buffer *ring)
261 {
262         struct pipe_control *pc = ring->private;
263         struct drm_i915_gem_object *obj;
264
265         if (!ring->private)
266                 return;
267
268         obj = pc->obj;
269         kunmap(obj->pages[0]);
270         i915_gem_object_unpin(obj);
271         drm_gem_object_unreference(&obj->base);
272
273         kfree(pc);
274         ring->private = NULL;
275 }
276
277 static int init_render_ring(struct intel_ring_buffer *ring)
278 {
279         struct drm_device *dev = ring->dev;
280         int ret = init_ring_common(ring);
281
282         if (INTEL_INFO(dev)->gen > 3) {
283                 drm_i915_private_t *dev_priv = dev->dev_private;
284                 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
285                 if (IS_GEN6(dev))
286                         mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
287                 I915_WRITE(MI_MODE, mode);
288         }
289
290         if (HAS_PIPE_CONTROL(dev)) {
291                 ret = init_pipe_control(ring);
292                 if (ret)
293                         return ret;
294         }
295
296         return ret;
297 }
298
299 static void render_ring_cleanup(struct intel_ring_buffer *ring)
300 {
301         if (!ring->private)
302                 return;
303
304         cleanup_pipe_control(ring);
305 }
306
307 #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
308 do {                                                                    \
309         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |           \
310                  PIPE_CONTROL_DEPTH_STALL | 2);                         \
311         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
312         intel_ring_emit(ring__, 0);                                                     \
313         intel_ring_emit(ring__, 0);                                                     \
314 } while (0)
315
316 /**
317  * Creates a new sequence number, emitting a write of it to the status page
318  * plus an interrupt, which will trigger i915_user_interrupt_handler.
319  *
320  * Must be called with struct_lock held.
321  *
322  * Returned sequence numbers are nonzero on success.
323  */
324 static int
325 render_ring_add_request(struct intel_ring_buffer *ring,
326                         u32 *result)
327 {
328         struct drm_device *dev = ring->dev;
329         u32 seqno = i915_gem_get_seqno(dev);
330         struct pipe_control *pc = ring->private;
331         int ret;
332
333         if (IS_GEN6(dev)) {
334                 ret = intel_ring_begin(ring, 6);
335                 if (ret)
336                     return ret;
337
338                 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | 3);
339                 intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE |
340                                 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH |
341                                 PIPE_CONTROL_NOTIFY);
342                 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
343                 intel_ring_emit(ring, seqno);
344                 intel_ring_emit(ring, 0);
345                 intel_ring_emit(ring, 0);
346         } else if (HAS_PIPE_CONTROL(dev)) {
347                 u32 scratch_addr = pc->gtt_offset + 128;
348
349                 /*
350                  * Workaround qword write incoherence by flushing the
351                  * PIPE_NOTIFY buffers out to memory before requesting
352                  * an interrupt.
353                  */
354                 ret = intel_ring_begin(ring, 32);
355                 if (ret)
356                         return ret;
357
358                 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
359                                 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
360                 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
361                 intel_ring_emit(ring, seqno);
362                 intel_ring_emit(ring, 0);
363                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
364                 scratch_addr += 128; /* write to separate cachelines */
365                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
366                 scratch_addr += 128;
367                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
368                 scratch_addr += 128;
369                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
370                 scratch_addr += 128;
371                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
372                 scratch_addr += 128;
373                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
374                 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
375                                 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
376                                 PIPE_CONTROL_NOTIFY);
377                 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
378                 intel_ring_emit(ring, seqno);
379                 intel_ring_emit(ring, 0);
380         } else {
381                 ret = intel_ring_begin(ring, 4);
382                 if (ret)
383                     return ret;
384
385                 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
386                 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
387                 intel_ring_emit(ring, seqno);
388
389                 intel_ring_emit(ring, MI_USER_INTERRUPT);
390         }
391
392         intel_ring_advance(ring);
393         *result = seqno;
394         return 0;
395 }
396
397 static u32
398 render_ring_get_seqno(struct intel_ring_buffer *ring)
399 {
400         struct drm_device *dev = ring->dev;
401         if (HAS_PIPE_CONTROL(dev)) {
402                 struct pipe_control *pc = ring->private;
403                 return pc->cpu_page[0];
404         } else
405                 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
406 }
407
408 static void
409 render_ring_get_user_irq(struct intel_ring_buffer *ring)
410 {
411         struct drm_device *dev = ring->dev;
412         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
413         unsigned long irqflags;
414
415         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
416         if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) {
417                 if (HAS_PCH_SPLIT(dev))
418                         ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
419                 else
420                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
421         }
422         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
423 }
424
425 static void
426 render_ring_put_user_irq(struct intel_ring_buffer *ring)
427 {
428         struct drm_device *dev = ring->dev;
429         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
430         unsigned long irqflags;
431
432         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
433         BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0);
434         if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
435                 if (HAS_PCH_SPLIT(dev))
436                         ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
437                 else
438                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
439         }
440         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
441 }
442
443 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
444 {
445         drm_i915_private_t *dev_priv = ring->dev->dev_private;
446         u32 mmio = IS_GEN6(ring->dev) ?
447                 RING_HWS_PGA_GEN6(ring->mmio_base) :
448                 RING_HWS_PGA(ring->mmio_base);
449         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
450         POSTING_READ(mmio);
451 }
452
453 static void
454 bsd_ring_flush(struct intel_ring_buffer *ring,
455                u32     invalidate_domains,
456                u32     flush_domains)
457 {
458         if (intel_ring_begin(ring, 2) == 0) {
459                 intel_ring_emit(ring, MI_FLUSH);
460                 intel_ring_emit(ring, MI_NOOP);
461                 intel_ring_advance(ring);
462         }
463 }
464
465 static int
466 ring_add_request(struct intel_ring_buffer *ring,
467                  u32 *result)
468 {
469         u32 seqno;
470         int ret;
471
472         ret = intel_ring_begin(ring, 4);
473         if (ret)
474                 return ret;
475
476         seqno = i915_gem_get_seqno(ring->dev);
477
478         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
479         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
480         intel_ring_emit(ring, seqno);
481         intel_ring_emit(ring, MI_USER_INTERRUPT);
482         intel_ring_advance(ring);
483
484         DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
485         *result = seqno;
486         return 0;
487 }
488
489 static void
490 bsd_ring_get_user_irq(struct intel_ring_buffer *ring)
491 {
492         /* do nothing */
493 }
494 static void
495 bsd_ring_put_user_irq(struct intel_ring_buffer *ring)
496 {
497         /* do nothing */
498 }
499
500 static u32
501 ring_status_page_get_seqno(struct intel_ring_buffer *ring)
502 {
503         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
504 }
505
506 static int
507 ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
508                          struct drm_i915_gem_execbuffer2 *exec,
509                          struct drm_clip_rect *cliprects,
510                          uint64_t exec_offset)
511 {
512         uint32_t exec_start;
513         int ret;
514
515         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
516
517         ret = intel_ring_begin(ring, 2);
518         if (ret)
519                 return ret;
520
521         intel_ring_emit(ring,
522                         MI_BATCH_BUFFER_START |
523                         (2 << 6) |
524                         MI_BATCH_NON_SECURE_I965);
525         intel_ring_emit(ring, exec_start);
526         intel_ring_advance(ring);
527
528         return 0;
529 }
530
531 static int
532 render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
533                                 struct drm_i915_gem_execbuffer2 *exec,
534                                 struct drm_clip_rect *cliprects,
535                                 uint64_t exec_offset)
536 {
537         struct drm_device *dev = ring->dev;
538         drm_i915_private_t *dev_priv = dev->dev_private;
539         int nbox = exec->num_cliprects;
540         uint32_t exec_start, exec_len;
541         int i, count, ret;
542
543         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
544         exec_len = (uint32_t) exec->batch_len;
545
546         trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
547
548         count = nbox ? nbox : 1;
549         for (i = 0; i < count; i++) {
550                 if (i < nbox) {
551                         ret = i915_emit_box(dev, cliprects, i,
552                                             exec->DR1, exec->DR4);
553                         if (ret)
554                                 return ret;
555                 }
556
557                 if (IS_I830(dev) || IS_845G(dev)) {
558                         ret = intel_ring_begin(ring, 4);
559                         if (ret)
560                                 return ret;
561
562                         intel_ring_emit(ring, MI_BATCH_BUFFER);
563                         intel_ring_emit(ring, exec_start | MI_BATCH_NON_SECURE);
564                         intel_ring_emit(ring, exec_start + exec_len - 4);
565                         intel_ring_emit(ring, 0);
566                 } else {
567                         ret = intel_ring_begin(ring, 2);
568                         if (ret)
569                                 return ret;
570
571                         if (INTEL_INFO(dev)->gen >= 4) {
572                                 intel_ring_emit(ring,
573                                                 MI_BATCH_BUFFER_START | (2 << 6)
574                                                 | MI_BATCH_NON_SECURE_I965);
575                                 intel_ring_emit(ring, exec_start);
576                         } else {
577                                 intel_ring_emit(ring, MI_BATCH_BUFFER_START
578                                                 | (2 << 6));
579                                 intel_ring_emit(ring, exec_start |
580                                                 MI_BATCH_NON_SECURE);
581                         }
582                 }
583                 intel_ring_advance(ring);
584         }
585
586         if (IS_G4X(dev) || IS_GEN5(dev)) {
587                 if (intel_ring_begin(ring, 2) == 0) {
588                         intel_ring_emit(ring, MI_FLUSH |
589                                         MI_NO_WRITE_FLUSH |
590                                         MI_INVALIDATE_ISP );
591                         intel_ring_emit(ring, MI_NOOP);
592                         intel_ring_advance(ring);
593                 }
594         }
595         /* XXX breadcrumb */
596
597         return 0;
598 }
599
600 static void cleanup_status_page(struct intel_ring_buffer *ring)
601 {
602         drm_i915_private_t *dev_priv = ring->dev->dev_private;
603         struct drm_i915_gem_object *obj;
604
605         obj = ring->status_page.obj;
606         if (obj == NULL)
607                 return;
608
609         kunmap(obj->pages[0]);
610         i915_gem_object_unpin(obj);
611         drm_gem_object_unreference(&obj->base);
612         ring->status_page.obj = NULL;
613
614         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
615 }
616
617 static int init_status_page(struct intel_ring_buffer *ring)
618 {
619         struct drm_device *dev = ring->dev;
620         drm_i915_private_t *dev_priv = dev->dev_private;
621         struct drm_i915_gem_object *obj;
622         int ret;
623
624         obj = i915_gem_alloc_object(dev, 4096);
625         if (obj == NULL) {
626                 DRM_ERROR("Failed to allocate status page\n");
627                 ret = -ENOMEM;
628                 goto err;
629         }
630         obj->agp_type = AGP_USER_CACHED_MEMORY;
631
632         ret = i915_gem_object_pin(obj, 4096, true);
633         if (ret != 0) {
634                 goto err_unref;
635         }
636
637         ring->status_page.gfx_addr = obj->gtt_offset;
638         ring->status_page.page_addr = kmap(obj->pages[0]);
639         if (ring->status_page.page_addr == NULL) {
640                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
641                 goto err_unpin;
642         }
643         ring->status_page.obj = obj;
644         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
645
646         intel_ring_setup_status_page(ring);
647         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
648                         ring->name, ring->status_page.gfx_addr);
649
650         return 0;
651
652 err_unpin:
653         i915_gem_object_unpin(obj);
654 err_unref:
655         drm_gem_object_unreference(&obj->base);
656 err:
657         return ret;
658 }
659
660 int intel_init_ring_buffer(struct drm_device *dev,
661                            struct intel_ring_buffer *ring)
662 {
663         struct drm_i915_gem_object *obj;
664         int ret;
665
666         ring->dev = dev;
667         INIT_LIST_HEAD(&ring->active_list);
668         INIT_LIST_HEAD(&ring->request_list);
669         INIT_LIST_HEAD(&ring->gpu_write_list);
670
671         if (I915_NEED_GFX_HWS(dev)) {
672                 ret = init_status_page(ring);
673                 if (ret)
674                         return ret;
675         }
676
677         obj = i915_gem_alloc_object(dev, ring->size);
678         if (obj == NULL) {
679                 DRM_ERROR("Failed to allocate ringbuffer\n");
680                 ret = -ENOMEM;
681                 goto err_hws;
682         }
683
684         ring->obj = obj;
685
686         ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
687         if (ret)
688                 goto err_unref;
689
690         ring->map.size = ring->size;
691         ring->map.offset = dev->agp->base + obj->gtt_offset;
692         ring->map.type = 0;
693         ring->map.flags = 0;
694         ring->map.mtrr = 0;
695
696         drm_core_ioremap_wc(&ring->map, dev);
697         if (ring->map.handle == NULL) {
698                 DRM_ERROR("Failed to map ringbuffer.\n");
699                 ret = -EINVAL;
700                 goto err_unpin;
701         }
702
703         ring->virtual_start = ring->map.handle;
704         ret = ring->init(ring);
705         if (ret)
706                 goto err_unmap;
707
708         return 0;
709
710 err_unmap:
711         drm_core_ioremapfree(&ring->map, dev);
712 err_unpin:
713         i915_gem_object_unpin(obj);
714 err_unref:
715         drm_gem_object_unreference(&obj->base);
716         ring->obj = NULL;
717 err_hws:
718         cleanup_status_page(ring);
719         return ret;
720 }
721
722 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
723 {
724         struct drm_i915_private *dev_priv;
725         int ret;
726
727         if (ring->obj == NULL)
728                 return;
729
730         /* Disable the ring buffer. The ring must be idle at this point */
731         dev_priv = ring->dev->dev_private;
732         ret = intel_wait_ring_buffer(ring, ring->size - 8);
733         I915_WRITE_CTL(ring, 0);
734
735         drm_core_ioremapfree(&ring->map, ring->dev);
736
737         i915_gem_object_unpin(ring->obj);
738         drm_gem_object_unreference(&ring->obj->base);
739         ring->obj = NULL;
740
741         if (ring->cleanup)
742                 ring->cleanup(ring);
743
744         cleanup_status_page(ring);
745 }
746
747 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
748 {
749         unsigned int *virt;
750         int rem;
751         rem = ring->size - ring->tail;
752
753         if (ring->space < rem) {
754                 int ret = intel_wait_ring_buffer(ring, rem);
755                 if (ret)
756                         return ret;
757         }
758
759         virt = (unsigned int *)(ring->virtual_start + ring->tail);
760         rem /= 8;
761         while (rem--) {
762                 *virt++ = MI_NOOP;
763                 *virt++ = MI_NOOP;
764         }
765
766         ring->tail = 0;
767         ring->space = ring->head - 8;
768
769         return 0;
770 }
771
772 int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
773 {
774         struct drm_device *dev = ring->dev;
775         struct drm_i915_private *dev_priv = dev->dev_private;
776         unsigned long end;
777         u32 head;
778
779         head = intel_read_status_page(ring, 4);
780         if (head) {
781                 ring->head = head & HEAD_ADDR;
782                 ring->space = ring->head - (ring->tail + 8);
783                 if (ring->space < 0)
784                         ring->space += ring->size;
785                 if (ring->space >= n)
786                         return 0;
787         }
788
789         trace_i915_ring_wait_begin (dev);
790         end = jiffies + 3 * HZ;
791         do {
792                 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
793                 ring->space = ring->head - (ring->tail + 8);
794                 if (ring->space < 0)
795                         ring->space += ring->size;
796                 if (ring->space >= n) {
797                         trace_i915_ring_wait_end(dev);
798                         return 0;
799                 }
800
801                 if (dev->primary->master) {
802                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
803                         if (master_priv->sarea_priv)
804                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
805                 }
806
807                 msleep(1);
808                 if (atomic_read(&dev_priv->mm.wedged))
809                         return -EAGAIN;
810         } while (!time_after(jiffies, end));
811         trace_i915_ring_wait_end (dev);
812         return -EBUSY;
813 }
814
815 int intel_ring_begin(struct intel_ring_buffer *ring,
816                      int num_dwords)
817 {
818         int n = 4*num_dwords;
819         int ret;
820
821         if (unlikely(ring->tail + n > ring->size)) {
822                 ret = intel_wrap_ring_buffer(ring);
823                 if (unlikely(ret))
824                         return ret;
825         }
826
827         if (unlikely(ring->space < n)) {
828                 ret = intel_wait_ring_buffer(ring, n);
829                 if (unlikely(ret))
830                         return ret;
831         }
832
833         ring->space -= n;
834         return 0;
835 }
836
837 void intel_ring_advance(struct intel_ring_buffer *ring)
838 {
839         ring->tail &= ring->size - 1;
840         ring->write_tail(ring, ring->tail);
841 }
842
843 static const struct intel_ring_buffer render_ring = {
844         .name                   = "render ring",
845         .id                     = RING_RENDER,
846         .mmio_base              = RENDER_RING_BASE,
847         .size                   = 32 * PAGE_SIZE,
848         .init                   = init_render_ring,
849         .write_tail             = ring_write_tail,
850         .flush                  = render_ring_flush,
851         .add_request            = render_ring_add_request,
852         .get_seqno              = render_ring_get_seqno,
853         .user_irq_get           = render_ring_get_user_irq,
854         .user_irq_put           = render_ring_put_user_irq,
855         .dispatch_execbuffer    = render_ring_dispatch_execbuffer,
856        .cleanup                 = render_ring_cleanup,
857 };
858
859 /* ring buffer for bit-stream decoder */
860
861 static const struct intel_ring_buffer bsd_ring = {
862         .name                   = "bsd ring",
863         .id                     = RING_BSD,
864         .mmio_base              = BSD_RING_BASE,
865         .size                   = 32 * PAGE_SIZE,
866         .init                   = init_ring_common,
867         .write_tail             = ring_write_tail,
868         .flush                  = bsd_ring_flush,
869         .add_request            = ring_add_request,
870         .get_seqno              = ring_status_page_get_seqno,
871         .user_irq_get           = bsd_ring_get_user_irq,
872         .user_irq_put           = bsd_ring_put_user_irq,
873         .dispatch_execbuffer    = ring_dispatch_execbuffer,
874 };
875
876
877 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
878                                      u32 value)
879 {
880        drm_i915_private_t *dev_priv = ring->dev->dev_private;
881
882        /* Every tail move must follow the sequence below */
883        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
884                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
885                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
886        I915_WRITE(GEN6_BSD_RNCID, 0x0);
887
888        if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
889                                GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
890                        50))
891                DRM_ERROR("timed out waiting for IDLE Indicator\n");
892
893        I915_WRITE_TAIL(ring, value);
894        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
895                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
896                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
897 }
898
899 static void gen6_ring_flush(struct intel_ring_buffer *ring,
900                             u32 invalidate_domains,
901                             u32 flush_domains)
902 {
903         if (intel_ring_begin(ring, 4) == 0) {
904                 intel_ring_emit(ring, MI_FLUSH_DW);
905                 intel_ring_emit(ring, 0);
906                 intel_ring_emit(ring, 0);
907                 intel_ring_emit(ring, 0);
908                 intel_ring_advance(ring);
909         }
910 }
911
912 static int
913 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
914                               struct drm_i915_gem_execbuffer2 *exec,
915                               struct drm_clip_rect *cliprects,
916                               uint64_t exec_offset)
917 {
918        uint32_t exec_start;
919        int ret;
920
921        exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
922
923        ret = intel_ring_begin(ring, 2);
924        if (ret)
925                return ret;
926
927        intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
928        /* bit0-7 is the length on GEN6+ */
929        intel_ring_emit(ring, exec_start);
930        intel_ring_advance(ring);
931
932        return 0;
933 }
934
935 /* ring buffer for Video Codec for Gen6+ */
936 static const struct intel_ring_buffer gen6_bsd_ring = {
937        .name                    = "gen6 bsd ring",
938        .id                      = RING_BSD,
939        .mmio_base               = GEN6_BSD_RING_BASE,
940        .size                    = 32 * PAGE_SIZE,
941        .init                    = init_ring_common,
942        .write_tail              = gen6_bsd_ring_write_tail,
943        .flush                   = gen6_ring_flush,
944        .add_request             = ring_add_request,
945        .get_seqno               = ring_status_page_get_seqno,
946        .user_irq_get            = bsd_ring_get_user_irq,
947        .user_irq_put            = bsd_ring_put_user_irq,
948        .dispatch_execbuffer     = gen6_ring_dispatch_execbuffer,
949 };
950
951 /* Blitter support (SandyBridge+) */
952
953 static void
954 blt_ring_get_user_irq(struct intel_ring_buffer *ring)
955 {
956         /* do nothing */
957 }
958 static void
959 blt_ring_put_user_irq(struct intel_ring_buffer *ring)
960 {
961         /* do nothing */
962 }
963
964
965 /* Workaround for some stepping of SNB,
966  * each time when BLT engine ring tail moved,
967  * the first command in the ring to be parsed
968  * should be MI_BATCH_BUFFER_START
969  */
970 #define NEED_BLT_WORKAROUND(dev) \
971         (IS_GEN6(dev) && (dev->pdev->revision < 8))
972
973 static inline struct drm_i915_gem_object *
974 to_blt_workaround(struct intel_ring_buffer *ring)
975 {
976         return ring->private;
977 }
978
979 static int blt_ring_init(struct intel_ring_buffer *ring)
980 {
981         if (NEED_BLT_WORKAROUND(ring->dev)) {
982                 struct drm_i915_gem_object *obj;
983                 u32 *ptr;
984                 int ret;
985
986                 obj = i915_gem_alloc_object(ring->dev, 4096);
987                 if (obj == NULL)
988                         return -ENOMEM;
989
990                 ret = i915_gem_object_pin(obj, 4096, true);
991                 if (ret) {
992                         drm_gem_object_unreference(&obj->base);
993                         return ret;
994                 }
995
996                 ptr = kmap(obj->pages[0]);
997                 *ptr++ = MI_BATCH_BUFFER_END;
998                 *ptr++ = MI_NOOP;
999                 kunmap(obj->pages[0]);
1000
1001                 ret = i915_gem_object_set_to_gtt_domain(obj, false);
1002                 if (ret) {
1003                         i915_gem_object_unpin(obj);
1004                         drm_gem_object_unreference(&obj->base);
1005                         return ret;
1006                 }
1007
1008                 ring->private = obj;
1009         }
1010
1011         return init_ring_common(ring);
1012 }
1013
1014 static int blt_ring_begin(struct intel_ring_buffer *ring,
1015                           int num_dwords)
1016 {
1017         if (ring->private) {
1018                 int ret = intel_ring_begin(ring, num_dwords+2);
1019                 if (ret)
1020                         return ret;
1021
1022                 intel_ring_emit(ring, MI_BATCH_BUFFER_START);
1023                 intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset);
1024
1025                 return 0;
1026         } else
1027                 return intel_ring_begin(ring, 4);
1028 }
1029
1030 static void blt_ring_flush(struct intel_ring_buffer *ring,
1031                            u32 invalidate_domains,
1032                            u32 flush_domains)
1033 {
1034         if (blt_ring_begin(ring, 4) == 0) {
1035                 intel_ring_emit(ring, MI_FLUSH_DW);
1036                 intel_ring_emit(ring, 0);
1037                 intel_ring_emit(ring, 0);
1038                 intel_ring_emit(ring, 0);
1039                 intel_ring_advance(ring);
1040         }
1041 }
1042
1043 static int
1044 blt_ring_add_request(struct intel_ring_buffer *ring,
1045                      u32 *result)
1046 {
1047         u32 seqno;
1048         int ret;
1049
1050         ret = blt_ring_begin(ring, 4);
1051         if (ret)
1052                 return ret;
1053
1054         seqno = i915_gem_get_seqno(ring->dev);
1055
1056         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
1057         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1058         intel_ring_emit(ring, seqno);
1059         intel_ring_emit(ring, MI_USER_INTERRUPT);
1060         intel_ring_advance(ring);
1061
1062         DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
1063         *result = seqno;
1064         return 0;
1065 }
1066
1067 static void blt_ring_cleanup(struct intel_ring_buffer *ring)
1068 {
1069         if (!ring->private)
1070                 return;
1071
1072         i915_gem_object_unpin(ring->private);
1073         drm_gem_object_unreference(ring->private);
1074         ring->private = NULL;
1075 }
1076
1077 static const struct intel_ring_buffer gen6_blt_ring = {
1078        .name                    = "blt ring",
1079        .id                      = RING_BLT,
1080        .mmio_base               = BLT_RING_BASE,
1081        .size                    = 32 * PAGE_SIZE,
1082        .init                    = blt_ring_init,
1083        .write_tail              = ring_write_tail,
1084        .flush                   = blt_ring_flush,
1085        .add_request             = blt_ring_add_request,
1086        .get_seqno               = ring_status_page_get_seqno,
1087        .user_irq_get            = blt_ring_get_user_irq,
1088        .user_irq_put            = blt_ring_put_user_irq,
1089        .dispatch_execbuffer     = gen6_ring_dispatch_execbuffer,
1090        .cleanup                 = blt_ring_cleanup,
1091 };
1092
1093 int intel_init_render_ring_buffer(struct drm_device *dev)
1094 {
1095         drm_i915_private_t *dev_priv = dev->dev_private;
1096
1097         dev_priv->render_ring = render_ring;
1098
1099         if (!I915_NEED_GFX_HWS(dev)) {
1100                 dev_priv->render_ring.status_page.page_addr
1101                         = dev_priv->status_page_dmah->vaddr;
1102                 memset(dev_priv->render_ring.status_page.page_addr,
1103                                 0, PAGE_SIZE);
1104         }
1105
1106         return intel_init_ring_buffer(dev, &dev_priv->render_ring);
1107 }
1108
1109 int intel_init_bsd_ring_buffer(struct drm_device *dev)
1110 {
1111         drm_i915_private_t *dev_priv = dev->dev_private;
1112
1113         if (IS_GEN6(dev))
1114                 dev_priv->bsd_ring = gen6_bsd_ring;
1115         else
1116                 dev_priv->bsd_ring = bsd_ring;
1117
1118         return intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
1119 }
1120
1121 int intel_init_blt_ring_buffer(struct drm_device *dev)
1122 {
1123         drm_i915_private_t *dev_priv = dev->dev_private;
1124
1125         dev_priv->blt_ring = gen6_blt_ring;
1126
1127         return intel_init_ring_buffer(dev, &dev_priv->blt_ring);
1128 }