drm/i915: Add wait_for in init_ring_common
[linux-2.6.git] / drivers / gpu / drm / i915 / intel_ringbuffer.c
index aa8f6ab..c17325c 100644 (file)
 #include "i915_trace.h"
 #include "intel_drv.h"
 
-static u32 i915_gem_get_seqno(struct drm_device *dev)
-{
-       drm_i915_private_t *dev_priv = dev->dev_private;
-       u32 seqno;
-
-       seqno = dev_priv->next_seqno;
-
-       /* reserve 0 for non-seqno */
-       if (++dev_priv->next_seqno == 0)
-               dev_priv->next_seqno = 1;
+/*
+ * 965+ support PIPE_CONTROL commands, which provide finer grained control
+ * over cache flushing.
+ */
+struct pipe_control {
+       struct drm_i915_gem_object *obj;
+       volatile u32 *cpu_page;
+       u32 gtt_offset;
+};
 
-       return seqno;
+static inline int ring_space(struct intel_ring_buffer *ring)
+{
+       int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
+       if (space < 0)
+               space += ring->size;
+       return space;
 }
 
 static int
@@ -54,77 +58,175 @@ render_ring_flush(struct intel_ring_buffer *ring,
                  u32   flush_domains)
 {
        struct drm_device *dev = ring->dev;
-       drm_i915_private_t *dev_priv = dev->dev_private;
        u32 cmd;
        int ret;
 
-#if WATCH_EXEC
-       DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
-                 invalidate_domains, flush_domains);
-#endif
-
-       trace_i915_gem_request_flush(dev, dev_priv->next_seqno,
-                                    invalidate_domains, flush_domains);
+       /*
+        * read/write caches:
+        *
+        * I915_GEM_DOMAIN_RENDER is always invalidated, but is
+        * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
+        * also flushed at 2d versus 3d pipeline switches.
+        *
+        * read-only caches:
+        *
+        * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
+        * MI_READ_FLUSH is set, and is always flushed on 965.
+        *
+        * I915_GEM_DOMAIN_COMMAND may not exist?
+        *
+        * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
+        * invalidated when MI_EXE_FLUSH is set.
+        *
+        * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
+        * invalidated with every MI_FLUSH.
+        *
+        * TLBs:
+        *
+        * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
+        * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
+        * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
+        * are flushed at any MI_FLUSH.
+        */
 
-       if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
+       cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
+       if ((invalidate_domains|flush_domains) &
+           I915_GEM_DOMAIN_RENDER)
+               cmd &= ~MI_NO_WRITE_FLUSH;
+       if (INTEL_INFO(dev)->gen < 4) {
                /*
-                * read/write caches:
-                *
-                * I915_GEM_DOMAIN_RENDER is always invalidated, but is
-                * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
-                * also flushed at 2d versus 3d pipeline switches.
-                *
-                * read-only caches:
-                *
-                * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
-                * MI_READ_FLUSH is set, and is always flushed on 965.
-                *
-                * I915_GEM_DOMAIN_COMMAND may not exist?
-                *
-                * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
-                * invalidated when MI_EXE_FLUSH is set.
-                *
-                * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
-                * invalidated with every MI_FLUSH.
-                *
-                * TLBs:
-                *
-                * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
-                * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
-                * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
-                * are flushed at any MI_FLUSH.
+                * On the 965, the sampler cache always gets flushed
+                * and this bit is reserved.
                 */
+               if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
+                       cmd |= MI_READ_FLUSH;
+       }
+       if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
+               cmd |= MI_EXE_FLUSH;
 
-               cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
-               if ((invalidate_domains|flush_domains) &
-                   I915_GEM_DOMAIN_RENDER)
-                       cmd &= ~MI_NO_WRITE_FLUSH;
-               if (INTEL_INFO(dev)->gen < 4) {
-                       /*
-                        * On the 965, the sampler cache always gets flushed
-                        * and this bit is reserved.
-                        */
-                       if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
-                               cmd |= MI_READ_FLUSH;
-               }
-               if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
-                       cmd |= MI_EXE_FLUSH;
+       if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
+           (IS_G4X(dev) || IS_GEN5(dev)))
+               cmd |= MI_INVALIDATE_ISP;
 
-               if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
-                   (IS_G4X(dev) || IS_GEN5(dev)))
-                       cmd |= MI_INVALIDATE_ISP;
+       ret = intel_ring_begin(ring, 2);
+       if (ret)
+               return ret;
 
-#if WATCH_EXEC
-               DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
-#endif
-               ret = intel_ring_begin(ring, 2);
-               if (ret)
-                       return ret;
+       intel_ring_emit(ring, cmd);
+       intel_ring_emit(ring, MI_NOOP);
+       intel_ring_advance(ring);
 
-               intel_ring_emit(ring, cmd);
-               intel_ring_emit(ring, MI_NOOP);
-               intel_ring_advance(ring);
-       }
+       return 0;
+}
+
+/**
+ * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
+ * implementing two workarounds on gen6.  From section 1.4.7.1
+ * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
+ *
+ * [DevSNB-C+{W/A}] Before any depth stall flush (including those
+ * produced by non-pipelined state commands), software needs to first
+ * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
+ * 0.
+ *
+ * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
+ * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
+ *
+ * And the workaround for these two requires this workaround first:
+ *
+ * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
+ * BEFORE the pipe-control with a post-sync op and no write-cache
+ * flushes.
+ *
+ * And this last workaround is tricky because of the requirements on
+ * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
+ * volume 2 part 1:
+ *
+ *     "1 of the following must also be set:
+ *      - Render Target Cache Flush Enable ([12] of DW1)
+ *      - Depth Cache Flush Enable ([0] of DW1)
+ *      - Stall at Pixel Scoreboard ([1] of DW1)
+ *      - Depth Stall ([13] of DW1)
+ *      - Post-Sync Operation ([13] of DW1)
+ *      - Notify Enable ([8] of DW1)"
+ *
+ * The cache flushes require the workaround flush that triggered this
+ * one, so we can't use it.  Depth stall would trigger the same.
+ * Post-sync nonzero is what triggered this second workaround, so we
+ * can't use that one either.  Notify enable is IRQs, which aren't
+ * really our business.  That leaves only stall at scoreboard.
+ */
+static int
+intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
+{
+       struct pipe_control *pc = ring->private;
+       u32 scratch_addr = pc->gtt_offset + 128;
+       int ret;
+
+
+       ret = intel_ring_begin(ring, 6);
+       if (ret)
+               return ret;
+
+       intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
+       intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
+                       PIPE_CONTROL_STALL_AT_SCOREBOARD);
+       intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
+       intel_ring_emit(ring, 0); /* low dword */
+       intel_ring_emit(ring, 0); /* high dword */
+       intel_ring_emit(ring, MI_NOOP);
+       intel_ring_advance(ring);
+
+       ret = intel_ring_begin(ring, 6);
+       if (ret)
+               return ret;
+
+       intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
+       intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
+       intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
+       intel_ring_emit(ring, 0);
+       intel_ring_emit(ring, 0);
+       intel_ring_emit(ring, MI_NOOP);
+       intel_ring_advance(ring);
+
+       return 0;
+}
+
+static int
+gen6_render_ring_flush(struct intel_ring_buffer *ring,
+                         u32 invalidate_domains, u32 flush_domains)
+{
+       u32 flags = 0;
+       struct pipe_control *pc = ring->private;
+       u32 scratch_addr = pc->gtt_offset + 128;
+       int ret;
+
+       /* Force SNB workarounds for PIPE_CONTROL flushes */
+       intel_emit_post_sync_nonzero_flush(ring);
+
+       /* Just flush everything.  Experiments have shown that reducing the
+        * number of bits based on the write domains has little performance
+        * impact.
+        */
+       flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
+       flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
+       flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
+       flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
+       flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
+       flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
+       flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
+
+       ret = intel_ring_begin(ring, 6);
+       if (ret)
+               return ret;
+
+       intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
+       intel_ring_emit(ring, flags);
+       intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
+       intel_ring_emit(ring, 0); /* lower dword */
+       intel_ring_emit(ring, 0); /* uppwer dword */
+       intel_ring_emit(ring, MI_NOOP);
+       intel_ring_advance(ring);
 
        return 0;
 }
@@ -147,10 +249,15 @@ u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
 
 static int init_ring_common(struct intel_ring_buffer *ring)
 {
-       drm_i915_private_t *dev_priv = ring->dev->dev_private;
+       struct drm_device *dev = ring->dev;
+       drm_i915_private_t *dev_priv = dev->dev_private;
        struct drm_i915_gem_object *obj = ring->obj;
+       int ret = 0;
        u32 head;
 
+       if (HAS_FORCE_WAKE(dev))
+               gen6_gt_force_wake_get(dev_priv);
+
        /* Stop the ring if it's running. */
        I915_WRITE_CTL(ring, 0);
        I915_WRITE_HEAD(ring, 0);
@@ -185,12 +292,12 @@ static int init_ring_common(struct intel_ring_buffer *ring)
 
        I915_WRITE_CTL(ring,
                        ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
-                       | RING_REPORT_64K | RING_VALID);
+                       | RING_VALID);
 
        /* If the head is still not zero, the ring is dead */
-       if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
-           I915_READ_START(ring) != obj->gtt_offset ||
-           (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
+       if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
+                    I915_READ_START(ring) == obj->gtt_offset &&
+                    (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
                DRM_ERROR("%s initialization failed "
                                "ctl %08x head %08x tail %08x start %08x\n",
                                ring->name,
@@ -198,31 +305,25 @@ static int init_ring_common(struct intel_ring_buffer *ring)
                                I915_READ_HEAD(ring),
                                I915_READ_TAIL(ring),
                                I915_READ_START(ring));
-               return -EIO;
+               ret = -EIO;
+               goto out;
        }
 
        if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
                i915_kernel_lost_context(ring->dev);
        else {
-               ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
+               ring->head = I915_READ_HEAD(ring);
                ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
-               ring->space = ring->head - (ring->tail + 8);
-               if (ring->space < 0)
-                       ring->space += ring->size;
+               ring->space = ring_space(ring);
+               ring->last_retired_head = -1;
        }
 
-       return 0;
-}
+out:
+       if (HAS_FORCE_WAKE(dev))
+               gen6_gt_force_wake_put(dev_priv);
 
-/*
- * 965+ support PIPE_CONTROL commands, which provide finer grained control
- * over cache flushing.
- */
-struct pipe_control {
-       struct drm_i915_gem_object *obj;
-       volatile u32 *cpu_page;
-       u32 gtt_offset;
-};
+       return ret;
+}
 
 static int
 init_pipe_control(struct intel_ring_buffer *ring)
@@ -244,7 +345,8 @@ init_pipe_control(struct intel_ring_buffer *ring)
                ret = -ENOMEM;
                goto err;
        }
-       obj->agp_type = AGP_USER_CACHED_MEMORY;
+
+       i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 
        ret = i915_gem_object_pin(obj, 4096, true);
        if (ret)
@@ -294,18 +396,35 @@ static int init_render_ring(struct intel_ring_buffer *ring)
 
        if (INTEL_INFO(dev)->gen > 3) {
                int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
-               if (IS_GEN6(dev))
-                       mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
                I915_WRITE(MI_MODE, mode);
+               if (IS_GEN7(dev))
+                       I915_WRITE(GFX_MODE_GEN7,
+                                  GFX_MODE_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
+                                  GFX_MODE_ENABLE(GFX_REPLAY_MODE));
        }
 
-       if (INTEL_INFO(dev)->gen >= 6) {
-       } else if (IS_GEN5(dev)) {
+       if (INTEL_INFO(dev)->gen >= 5) {
                ret = init_pipe_control(ring);
                if (ret)
                        return ret;
        }
 
+
+       if (IS_GEN6(dev)) {
+               /* From the Sandybridge PRM, volume 1 part 3, page 24:
+                * "If this bit is set, STCunit will have LRA as replacement
+                *  policy. [...] This bit must be reset.  LRA replacement
+                *  policy is not supported."
+                */
+               I915_WRITE(CACHE_MODE_0,
+                          CM0_STC_EVICT_DISABLE_LRA_SNB << CM0_MASK_SHIFT);
+       }
+
+       if (INTEL_INFO(dev)->gen >= 6) {
+               I915_WRITE(INSTPM,
+                          INSTPM_FORCE_ORDERING << 16 | INSTPM_FORCE_ORDERING);
+       }
+
        return ret;
 }
 
@@ -318,83 +437,131 @@ static void render_ring_cleanup(struct intel_ring_buffer *ring)
 }
 
 static void
-update_semaphore(struct intel_ring_buffer *ring, int i, u32 seqno)
+update_mboxes(struct intel_ring_buffer *ring,
+           u32 seqno,
+           u32 mmio_offset)
 {
-       struct drm_device *dev = ring->dev;
-       struct drm_i915_private *dev_priv = dev->dev_private;
-       int id;
-
-       /*
-        * cs -> 1 = vcs, 0 = bcs
-        * vcs -> 1 = bcs, 0 = cs,
-        * bcs -> 1 = cs, 0 = vcs.
-        */
-       id = ring - dev_priv->ring;
-       id += 2 - i;
-       id %= 3;
-
-       intel_ring_emit(ring,
-                       MI_SEMAPHORE_MBOX |
-                       MI_SEMAPHORE_REGISTER |
-                       MI_SEMAPHORE_UPDATE);
+       intel_ring_emit(ring, MI_SEMAPHORE_MBOX |
+                             MI_SEMAPHORE_GLOBAL_GTT |
+                             MI_SEMAPHORE_REGISTER |
+                             MI_SEMAPHORE_UPDATE);
        intel_ring_emit(ring, seqno);
-       intel_ring_emit(ring,
-                       RING_SYNC_0(dev_priv->ring[id].mmio_base) + 4*i);
+       intel_ring_emit(ring, mmio_offset);
 }
 
+/**
+ * gen6_add_request - Update the semaphore mailbox registers
+ * 
+ * @ring - ring that is adding a request
+ * @seqno - return seqno stuck into the ring
+ *
+ * Update the mailbox registers in the *other* rings with the current seqno.
+ * This acts like a signal in the canonical semaphore.
+ */
 static int
 gen6_add_request(struct intel_ring_buffer *ring,
-                u32 *result)
+                u32 *seqno)
 {
-       u32 seqno;
+       u32 mbox1_reg;
+       u32 mbox2_reg;
        int ret;
 
        ret = intel_ring_begin(ring, 10);
        if (ret)
                return ret;
 
-       seqno = i915_gem_get_seqno(ring->dev);
-       update_semaphore(ring, 0, seqno);
-       update_semaphore(ring, 1, seqno);
+       mbox1_reg = ring->signal_mbox[0];
+       mbox2_reg = ring->signal_mbox[1];
+
+       *seqno = i915_gem_next_request_seqno(ring);
 
+       update_mboxes(ring, *seqno, mbox1_reg);
+       update_mboxes(ring, *seqno, mbox2_reg);
        intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
        intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
-       intel_ring_emit(ring, seqno);
+       intel_ring_emit(ring, *seqno);
        intel_ring_emit(ring, MI_USER_INTERRUPT);
        intel_ring_advance(ring);
 
-       *result = seqno;
        return 0;
 }
 
-int
-intel_ring_sync(struct intel_ring_buffer *ring,
-               struct intel_ring_buffer *to,
+/**
+ * intel_ring_sync - sync the waiter to the signaller on seqno
+ *
+ * @waiter - ring that is waiting
+ * @signaller - ring which has, or will signal
+ * @seqno - seqno which the waiter will block on
+ */
+static int
+intel_ring_sync(struct intel_ring_buffer *waiter,
+               struct intel_ring_buffer *signaller,
+               int ring,
                u32 seqno)
 {
        int ret;
+       u32 dw1 = MI_SEMAPHORE_MBOX |
+                 MI_SEMAPHORE_COMPARE |
+                 MI_SEMAPHORE_REGISTER;
 
-       ret = intel_ring_begin(ring, 4);
+       ret = intel_ring_begin(waiter, 4);
        if (ret)
                return ret;
 
-       intel_ring_emit(ring,
-                       MI_SEMAPHORE_MBOX |
-                       MI_SEMAPHORE_REGISTER |
-                       intel_ring_sync_index(ring, to) << 17 |
-                       MI_SEMAPHORE_COMPARE);
-       intel_ring_emit(ring, seqno);
-       intel_ring_emit(ring, 0);
-       intel_ring_emit(ring, MI_NOOP);
-       intel_ring_advance(ring);
+       intel_ring_emit(waiter, dw1 | signaller->semaphore_register[ring]);
+       intel_ring_emit(waiter, seqno);
+       intel_ring_emit(waiter, 0);
+       intel_ring_emit(waiter, MI_NOOP);
+       intel_ring_advance(waiter);
 
        return 0;
 }
 
+/* VCS->RCS (RVSYNC) or BCS->RCS (RBSYNC) */
+int
+render_ring_sync_to(struct intel_ring_buffer *waiter,
+                   struct intel_ring_buffer *signaller,
+                   u32 seqno)
+{
+       WARN_ON(signaller->semaphore_register[RCS] == MI_SEMAPHORE_SYNC_INVALID);
+       return intel_ring_sync(waiter,
+                              signaller,
+                              RCS,
+                              seqno);
+}
+
+/* RCS->VCS (VRSYNC) or BCS->VCS (VBSYNC) */
+int
+gen6_bsd_ring_sync_to(struct intel_ring_buffer *waiter,
+                     struct intel_ring_buffer *signaller,
+                     u32 seqno)
+{
+       WARN_ON(signaller->semaphore_register[VCS] == MI_SEMAPHORE_SYNC_INVALID);
+       return intel_ring_sync(waiter,
+                              signaller,
+                              VCS,
+                              seqno);
+}
+
+/* RCS->BCS (BRSYNC) or VCS->BCS (BVSYNC) */
+int
+gen6_blt_ring_sync_to(struct intel_ring_buffer *waiter,
+                     struct intel_ring_buffer *signaller,
+                     u32 seqno)
+{
+       WARN_ON(signaller->semaphore_register[BCS] == MI_SEMAPHORE_SYNC_INVALID);
+       return intel_ring_sync(waiter,
+                              signaller,
+                              BCS,
+                              seqno);
+}
+
+
+
 #define PIPE_CONTROL_FLUSH(ring__, addr__)                                     \
 do {                                                                   \
-       intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |           \
-                PIPE_CONTROL_DEPTH_STALL | 2);                         \
+       intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |                \
+                PIPE_CONTROL_DEPTH_STALL);                             \
        intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
        intel_ring_emit(ring__, 0);                                                     \
        intel_ring_emit(ring__, 0);                                                     \
@@ -404,8 +571,7 @@ static int
 pc_render_add_request(struct intel_ring_buffer *ring,
                      u32 *result)
 {
-       struct drm_device *dev = ring->dev;
-       u32 seqno = i915_gem_get_seqno(dev);
+       u32 seqno = i915_gem_next_request_seqno(ring);
        struct pipe_control *pc = ring->private;
        u32 scratch_addr = pc->gtt_offset + 128;
        int ret;
@@ -422,8 +588,9 @@ pc_render_add_request(struct intel_ring_buffer *ring,
        if (ret)
                return ret;
 
-       intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
-                       PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
+       intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
+                       PIPE_CONTROL_WRITE_FLUSH |
+                       PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
        intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
        intel_ring_emit(ring, seqno);
        intel_ring_emit(ring, 0);
@@ -438,8 +605,10 @@ pc_render_add_request(struct intel_ring_buffer *ring,
        PIPE_CONTROL_FLUSH(ring, scratch_addr);
        scratch_addr += 128;
        PIPE_CONTROL_FLUSH(ring, scratch_addr);
-       intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
-                       PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
+
+       intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
+                       PIPE_CONTROL_WRITE_FLUSH |
+                       PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
                        PIPE_CONTROL_NOTIFY);
        intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
        intel_ring_emit(ring, seqno);
@@ -454,8 +623,7 @@ static int
 render_ring_add_request(struct intel_ring_buffer *ring,
                        u32 *result)
 {
-       struct drm_device *dev = ring->dev;
-       u32 seqno = i915_gem_get_seqno(dev);
+       u32 seqno = i915_gem_next_request_seqno(ring);
        int ret;
 
        ret = intel_ring_begin(ring, 4);
@@ -473,6 +641,19 @@ render_ring_add_request(struct intel_ring_buffer *ring,
 }
 
 static u32
+gen6_ring_get_seqno(struct intel_ring_buffer *ring)
+{
+       struct drm_device *dev = ring->dev;
+
+       /* Workaround to force correct ordering between irq and seqno writes on
+        * ivb (and maybe also on snb) by reading from a CS register (like
+        * ACTHD) before reading the status page. */
+       if (IS_GEN6(dev) || IS_GEN7(dev))
+               intel_ring_get_active_head(ring);
+       return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
+}
+
+static u32
 ring_get_seqno(struct intel_ring_buffer *ring)
 {
        return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
@@ -485,26 +666,56 @@ pc_render_get_seqno(struct intel_ring_buffer *ring)
        return pc->cpu_page[0];
 }
 
+static void
+ironlake_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
+{
+       dev_priv->gt_irq_mask &= ~mask;
+       I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
+       POSTING_READ(GTIMR);
+}
+
+static void
+ironlake_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
+{
+       dev_priv->gt_irq_mask |= mask;
+       I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
+       POSTING_READ(GTIMR);
+}
+
+static void
+i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
+{
+       dev_priv->irq_mask &= ~mask;
+       I915_WRITE(IMR, dev_priv->irq_mask);
+       POSTING_READ(IMR);
+}
+
+static void
+i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
+{
+       dev_priv->irq_mask |= mask;
+       I915_WRITE(IMR, dev_priv->irq_mask);
+       POSTING_READ(IMR);
+}
+
 static bool
 render_ring_get_irq(struct intel_ring_buffer *ring)
 {
        struct drm_device *dev = ring->dev;
+       drm_i915_private_t *dev_priv = dev->dev_private;
 
        if (!dev->irq_enabled)
                return false;
 
-       if (atomic_inc_return(&ring->irq_refcount) == 1) {
-               drm_i915_private_t *dev_priv = dev->dev_private;
-               unsigned long irqflags;
-
-               spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+       spin_lock(&ring->irq_lock);
+       if (ring->irq_refcount++ == 0) {
                if (HAS_PCH_SPLIT(dev))
-                       ironlake_enable_graphics_irq(dev_priv,
-                                                    GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
+                       ironlake_enable_irq(dev_priv,
+                                           GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
                else
                        i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
-               spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
        }
+       spin_unlock(&ring->irq_lock);
 
        return true;
 }
@@ -513,28 +724,47 @@ static void
 render_ring_put_irq(struct intel_ring_buffer *ring)
 {
        struct drm_device *dev = ring->dev;
+       drm_i915_private_t *dev_priv = dev->dev_private;
 
-       if (atomic_dec_and_test(&ring->irq_refcount)) {
-               drm_i915_private_t *dev_priv = dev->dev_private;
-               unsigned long irqflags;
-
-               spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+       spin_lock(&ring->irq_lock);
+       if (--ring->irq_refcount == 0) {
                if (HAS_PCH_SPLIT(dev))
-                       ironlake_disable_graphics_irq(dev_priv,
-                                                     GT_USER_INTERRUPT |
-                                                     GT_PIPE_NOTIFY);
+                       ironlake_disable_irq(dev_priv,
+                                            GT_USER_INTERRUPT |
+                                            GT_PIPE_NOTIFY);
                else
                        i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
-               spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
        }
+       spin_unlock(&ring->irq_lock);
 }
 
 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
 {
+       struct drm_device *dev = ring->dev;
        drm_i915_private_t *dev_priv = ring->dev->dev_private;
-       u32 mmio = IS_GEN6(ring->dev) ?
-               RING_HWS_PGA_GEN6(ring->mmio_base) :
-               RING_HWS_PGA(ring->mmio_base);
+       u32 mmio = 0;
+
+       /* The ring status page addresses are no longer next to the rest of
+        * the ring registers as of gen7.
+        */
+       if (IS_GEN7(dev)) {
+               switch (ring->id) {
+               case RCS:
+                       mmio = RENDER_HWS_PGA_GEN7;
+                       break;
+               case BCS:
+                       mmio = BLT_HWS_PGA_GEN7;
+                       break;
+               case VCS:
+                       mmio = BSD_HWS_PGA_GEN7;
+                       break;
+               }
+       } else if (IS_GEN6(ring->dev)) {
+               mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
+       } else {
+               mmio = RING_HWS_PGA(ring->mmio_base);
+       }
+
        I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
        POSTING_READ(mmio);
 }
@@ -546,9 +776,6 @@ bsd_ring_flush(struct intel_ring_buffer *ring,
 {
        int ret;
 
-       if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
-               return 0;
-
        ret = intel_ring_begin(ring, 2);
        if (ret)
                return ret;
@@ -570,7 +797,7 @@ ring_add_request(struct intel_ring_buffer *ring,
        if (ret)
                return ret;
 
-       seqno = i915_gem_get_seqno(ring->dev);
+       seqno = i915_gem_next_request_seqno(ring);
 
        intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
        intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
@@ -578,55 +805,86 @@ ring_add_request(struct intel_ring_buffer *ring,
        intel_ring_emit(ring, MI_USER_INTERRUPT);
        intel_ring_advance(ring);
 
-       DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
        *result = seqno;
        return 0;
 }
 
 static bool
-ring_get_irq(struct intel_ring_buffer *ring, u32 flag)
+gen6_ring_get_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
 {
        struct drm_device *dev = ring->dev;
+       drm_i915_private_t *dev_priv = dev->dev_private;
 
        if (!dev->irq_enabled)
               return false;
 
-       if (atomic_inc_return(&ring->irq_refcount) == 1) {
-               drm_i915_private_t *dev_priv = dev->dev_private;
-               unsigned long irqflags;
+       /* It looks like we need to prevent the gt from suspending while waiting
+        * for an notifiy irq, otherwise irqs seem to get lost on at least the
+        * blt/bsd rings on ivb. */
+       gen6_gt_force_wake_get(dev_priv);
 
-               spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-               ironlake_enable_graphics_irq(dev_priv, flag);
-               spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
+       spin_lock(&ring->irq_lock);
+       if (ring->irq_refcount++ == 0) {
+               ring->irq_mask &= ~rflag;
+               I915_WRITE_IMR(ring, ring->irq_mask);
+               ironlake_enable_irq(dev_priv, gflag);
        }
+       spin_unlock(&ring->irq_lock);
 
        return true;
 }
 
 static void
-ring_put_irq(struct intel_ring_buffer *ring, u32 flag)
+gen6_ring_put_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
 {
        struct drm_device *dev = ring->dev;
+       drm_i915_private_t *dev_priv = dev->dev_private;
 
-       if (atomic_dec_and_test(&ring->irq_refcount)) {
-               drm_i915_private_t *dev_priv = dev->dev_private;
-               unsigned long irqflags;
-
-               spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-               ironlake_disable_graphics_irq(dev_priv, flag);
-               spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
+       spin_lock(&ring->irq_lock);
+       if (--ring->irq_refcount == 0) {
+               ring->irq_mask |= rflag;
+               I915_WRITE_IMR(ring, ring->irq_mask);
+               ironlake_disable_irq(dev_priv, gflag);
        }
+       spin_unlock(&ring->irq_lock);
+
+       gen6_gt_force_wake_put(dev_priv);
 }
 
 static bool
 bsd_ring_get_irq(struct intel_ring_buffer *ring)
 {
-       return ring_get_irq(ring, GT_BSD_USER_INTERRUPT);
+       struct drm_device *dev = ring->dev;
+       drm_i915_private_t *dev_priv = dev->dev_private;
+
+       if (!dev->irq_enabled)
+               return false;
+
+       spin_lock(&ring->irq_lock);
+       if (ring->irq_refcount++ == 0) {
+               if (IS_G4X(dev))
+                       i915_enable_irq(dev_priv, I915_BSD_USER_INTERRUPT);
+               else
+                       ironlake_enable_irq(dev_priv, GT_BSD_USER_INTERRUPT);
+       }
+       spin_unlock(&ring->irq_lock);
+
+       return true;
 }
 static void
 bsd_ring_put_irq(struct intel_ring_buffer *ring)
 {
-       ring_put_irq(ring, GT_BSD_USER_INTERRUPT);
+       struct drm_device *dev = ring->dev;
+       drm_i915_private_t *dev_priv = dev->dev_private;
+
+       spin_lock(&ring->irq_lock);
+       if (--ring->irq_refcount == 0) {
+               if (IS_G4X(dev))
+                       i915_disable_irq(dev_priv, I915_BSD_USER_INTERRUPT);
+               else
+                       ironlake_disable_irq(dev_priv, GT_BSD_USER_INTERRUPT);
+       }
+       spin_unlock(&ring->irq_lock);
 }
 
 static int
@@ -652,11 +910,8 @@ render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
                                u32 offset, u32 len)
 {
        struct drm_device *dev = ring->dev;
-       drm_i915_private_t *dev_priv = dev->dev_private;
        int ret;
 
-       trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
-
        if (IS_I830(dev) || IS_845G(dev)) {
                ret = intel_ring_begin(ring, 4);
                if (ret)
@@ -717,7 +972,8 @@ static int init_status_page(struct intel_ring_buffer *ring)
                ret = -ENOMEM;
                goto err;
        }
-       obj->agp_type = AGP_USER_CACHED_MEMORY;
+
+       i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 
        ret = i915_gem_object_pin(obj, 4096, true);
        if (ret != 0) {
@@ -758,6 +1014,10 @@ int intel_init_ring_buffer(struct drm_device *dev,
        INIT_LIST_HEAD(&ring->request_list);
        INIT_LIST_HEAD(&ring->gpu_write_list);
 
+       init_waitqueue_head(&ring->irq_queue);
+       spin_lock_init(&ring->irq_lock);
+       ring->irq_mask = ~0;
+
        if (I915_NEED_GFX_HWS(dev)) {
                ret = init_status_page(ring);
                if (ret)
@@ -777,6 +1037,10 @@ int intel_init_ring_buffer(struct drm_device *dev,
        if (ret)
                goto err_unref;
 
+       ret = i915_gem_object_set_to_gtt_domain(obj, true);
+       if (ret)
+               goto err_unpin;
+
        ring->map.size = ring->size;
        ring->map.offset = dev->agp->base + obj->gtt_offset;
        ring->map.type = 0;
@@ -800,7 +1064,7 @@ int intel_init_ring_buffer(struct drm_device *dev,
         * of the buffer.
         */
        ring->effective_size = ring->size;
-       if (IS_I830(ring->dev))
+       if (IS_I830(ring->dev) || IS_845G(ring->dev))
                ring->effective_size -= 128;
 
        return 0;
@@ -827,7 +1091,11 @@ void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
 
        /* Disable the ring buffer. The ring must be idle at this point */
        dev_priv = ring->dev->dev_private;
-       ret = intel_wait_ring_buffer(ring, ring->size - 8);
+       ret = intel_wait_ring_idle(ring);
+       if (ret)
+               DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
+                         ring->name, ret);
+
        I915_WRITE_CTL(ring, 0);
 
        drm_core_ioremapfree(&ring->map, ring->dev);
@@ -861,7 +1129,84 @@ static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
        }
 
        ring->tail = 0;
-       ring->space = ring->head - 8;
+       ring->space = ring_space(ring);
+
+       return 0;
+}
+
+static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
+{
+       struct drm_i915_private *dev_priv = ring->dev->dev_private;
+       bool was_interruptible;
+       int ret;
+
+       /* XXX As we have not yet audited all the paths to check that
+        * they are ready for ERESTARTSYS from intel_ring_begin, do not
+        * allow us to be interruptible by a signal.
+        */
+       was_interruptible = dev_priv->mm.interruptible;
+       dev_priv->mm.interruptible = false;
+
+       ret = i915_wait_request(ring, seqno, true);
+
+       dev_priv->mm.interruptible = was_interruptible;
+
+       return ret;
+}
+
+static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
+{
+       struct drm_i915_gem_request *request;
+       u32 seqno = 0;
+       int ret;
+
+       i915_gem_retire_requests_ring(ring);
+
+       if (ring->last_retired_head != -1) {
+               ring->head = ring->last_retired_head;
+               ring->last_retired_head = -1;
+               ring->space = ring_space(ring);
+               if (ring->space >= n)
+                       return 0;
+       }
+
+       list_for_each_entry(request, &ring->request_list, list) {
+               int space;
+
+               if (request->tail == -1)
+                       continue;
+
+               space = request->tail - (ring->tail + 8);
+               if (space < 0)
+                       space += ring->size;
+               if (space >= n) {
+                       seqno = request->seqno;
+                       break;
+               }
+
+               /* Consume this request in case we need more space than
+                * is available and so need to prevent a race between
+                * updating last_retired_head and direct reads of
+                * I915_RING_HEAD. It also provides a nice sanity check.
+                */
+               request->tail = -1;
+       }
+
+       if (seqno == 0)
+               return -ENOSPC;
+
+       ret = intel_ring_wait_seqno(ring, seqno);
+       if (ret)
+               return ret;
+
+       if (WARN_ON(ring->last_retired_head == -1))
+               return -ENOSPC;
+
+       ring->head = ring->last_retired_head;
+       ring->last_retired_head = -1;
+       ring->space = ring_space(ring);
+       if (WARN_ON(ring->space < n))
+               return -ENOSPC;
 
        return 0;
 }
@@ -871,24 +1216,28 @@ int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
        struct drm_device *dev = ring->dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
        unsigned long end;
-       u32 head;
+       int ret;
+
+       ret = intel_ring_wait_request(ring, n);
+       if (ret != -ENOSPC)
+               return ret;
+
+       trace_i915_ring_wait_begin(ring);
+       if (drm_core_check_feature(dev, DRIVER_GEM))
+               /* With GEM the hangcheck timer should kick us out of the loop,
+                * leaving it early runs the risk of corrupting GEM state (due
+                * to running on almost untested codepaths). But on resume
+                * timers don't work yet, so prevent a complete hang in that
+                * case by choosing an insanely large timeout. */
+               end = jiffies + 60 * HZ;
+       else
+               end = jiffies + 3 * HZ;
 
-       trace_i915_ring_wait_begin (dev);
-       end = jiffies + 3 * HZ;
        do {
-               /* If the reported head position has wrapped or hasn't advanced,
-                * fallback to the slow and accurate path.
-                */
-               head = intel_read_status_page(ring, 4);
-               if (head < ring->actual_head)
-                       head = I915_READ_HEAD(ring);
-               ring->actual_head = head;
-               ring->head = head & HEAD_ADDR;
-               ring->space = ring->head - (ring->tail + 8);
-               if (ring->space < 0)
-                       ring->space += ring->size;
+               ring->head = I915_READ_HEAD(ring);
+               ring->space = ring_space(ring);
                if (ring->space >= n) {
-                       trace_i915_ring_wait_end(dev);
+                       trace_i915_ring_wait_end(ring);
                        return 0;
                }
 
@@ -902,16 +1251,20 @@ int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
                if (atomic_read(&dev_priv->mm.wedged))
                        return -EAGAIN;
        } while (!time_after(jiffies, end));
-       trace_i915_ring_wait_end (dev);
+       trace_i915_ring_wait_end(ring);
        return -EBUSY;
 }
 
 int intel_ring_begin(struct intel_ring_buffer *ring,
                     int num_dwords)
 {
+       struct drm_i915_private *dev_priv = ring->dev->dev_private;
        int n = 4*num_dwords;
        int ret;
 
+       if (unlikely(atomic_read(&dev_priv->mm.wedged)))
+               return -EIO;
+
        if (unlikely(ring->tail + n > ring->effective_size)) {
                ret = intel_wrap_ring_buffer(ring);
                if (unlikely(ret))
@@ -936,7 +1289,7 @@ void intel_ring_advance(struct intel_ring_buffer *ring)
 
 static const struct intel_ring_buffer render_ring = {
        .name                   = "render ring",
-       .id                     = RING_RENDER,
+       .id                     = RCS,
        .mmio_base              = RENDER_RING_BASE,
        .size                   = 32 * PAGE_SIZE,
        .init                   = init_render_ring,
@@ -947,14 +1300,19 @@ static const struct intel_ring_buffer render_ring = {
        .irq_get                = render_ring_get_irq,
        .irq_put                = render_ring_put_irq,
        .dispatch_execbuffer    = render_ring_dispatch_execbuffer,
-       .cleanup                        = render_ring_cleanup,
+       .cleanup                = render_ring_cleanup,
+       .sync_to                = render_ring_sync_to,
+       .semaphore_register     = {MI_SEMAPHORE_SYNC_INVALID,
+                                  MI_SEMAPHORE_SYNC_RV,
+                                  MI_SEMAPHORE_SYNC_RB},
+       .signal_mbox            = {GEN6_VRSYNC, GEN6_BRSYNC},
 };
 
 /* ring buffer for bit-stream decoder */
 
 static const struct intel_ring_buffer bsd_ring = {
        .name                   = "bsd ring",
-       .id                     = RING_BSD,
+       .id                     = VCS,
        .mmio_base              = BSD_RING_BASE,
        .size                   = 32 * PAGE_SIZE,
        .init                   = init_ring_common,
@@ -971,42 +1329,42 @@ static const struct intel_ring_buffer bsd_ring = {
 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
                                     u32 value)
 {
-       drm_i915_private_t *dev_priv = ring->dev->dev_private;
+       drm_i915_private_t *dev_priv = ring->dev->dev_private;
 
        /* Every tail move must follow the sequence below */
-       I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
-              GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
-              GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
-       I915_WRITE(GEN6_BSD_RNCID, 0x0);
-
-       if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
-                               GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
-                       50))
-               DRM_ERROR("timed out waiting for IDLE Indicator\n");
-
-       I915_WRITE_TAIL(ring, value);
-       I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
-              GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
-              GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
+       I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
+               GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
+               GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
+       I915_WRITE(GEN6_BSD_RNCID, 0x0);
+
+       if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
+               GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
+               50))
+       DRM_ERROR("timed out waiting for IDLE Indicator\n");
+
+       I915_WRITE_TAIL(ring, value);
+       I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
+               GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
+               GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
 }
 
 static int gen6_ring_flush(struct intel_ring_buffer *ring,
-                          u32 invalidate_domains,
-                          u32 flush_domains)
+                          u32 invalidate, u32 flush)
 {
+       uint32_t cmd;
        int ret;
 
-       if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
-               return 0;
-
        ret = intel_ring_begin(ring, 4);
        if (ret)
                return ret;
 
-       intel_ring_emit(ring, MI_FLUSH_DW);
-       intel_ring_emit(ring, 0);
+       cmd = MI_FLUSH_DW;
+       if (invalidate & I915_GEM_GPU_DOMAINS)
+               cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
+       intel_ring_emit(ring, cmd);
        intel_ring_emit(ring, 0);
        intel_ring_emit(ring, 0);
+       intel_ring_emit(ring, MI_NOOP);
        intel_ring_advance(ring);
        return 0;
 }
@@ -1015,46 +1373,71 @@ static int
 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
                              u32 offset, u32 len)
 {
-       int ret;
+       int ret;
+
+       ret = intel_ring_begin(ring, 2);
+       if (ret)
+               return ret;
+
+       intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
+       /* bit0-7 is the length on GEN6+ */
+       intel_ring_emit(ring, offset);
+       intel_ring_advance(ring);
 
-       ret = intel_ring_begin(ring, 2);
-       if (ret)
-              return ret;
+       return 0;
+}
 
-       intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
-       /* bit0-7 is the length on GEN6+ */
-       intel_ring_emit(ring, offset);
-       intel_ring_advance(ring);
+static bool
+gen6_render_ring_get_irq(struct intel_ring_buffer *ring)
+{
+       return gen6_ring_get_irq(ring,
+                                GT_USER_INTERRUPT,
+                                GEN6_RENDER_USER_INTERRUPT);
+}
 
-       return 0;
+static void
+gen6_render_ring_put_irq(struct intel_ring_buffer *ring)
+{
+       return gen6_ring_put_irq(ring,
+                                GT_USER_INTERRUPT,
+                                GEN6_RENDER_USER_INTERRUPT);
 }
 
 static bool
 gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring)
 {
-       return ring_get_irq(ring, GT_GEN6_BSD_USER_INTERRUPT);
+       return gen6_ring_get_irq(ring,
+                                GT_GEN6_BSD_USER_INTERRUPT,
+                                GEN6_BSD_USER_INTERRUPT);
 }
 
 static void
 gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring)
 {
-       ring_put_irq(ring, GT_GEN6_BSD_USER_INTERRUPT);
+       return gen6_ring_put_irq(ring,
+                                GT_GEN6_BSD_USER_INTERRUPT,
+                                GEN6_BSD_USER_INTERRUPT);
 }
 
 /* ring buffer for Video Codec for Gen6+ */
 static const struct intel_ring_buffer gen6_bsd_ring = {
        .name                   = "gen6 bsd ring",
-       .id                     = RING_BSD,
+       .id                     = VCS,
        .mmio_base              = GEN6_BSD_RING_BASE,
        .size                   = 32 * PAGE_SIZE,
        .init                   = init_ring_common,
        .write_tail             = gen6_bsd_ring_write_tail,
        .flush                  = gen6_ring_flush,
        .add_request            = gen6_add_request,
-       .get_seqno              = ring_get_seqno,
+       .get_seqno              = gen6_ring_get_seqno,
        .irq_get                = gen6_bsd_ring_get_irq,
        .irq_put                = gen6_bsd_ring_put_irq,
        .dispatch_execbuffer    = gen6_ring_dispatch_execbuffer,
+       .sync_to                = gen6_bsd_ring_sync_to,
+       .semaphore_register     = {MI_SEMAPHORE_SYNC_VR,
+                                  MI_SEMAPHORE_SYNC_INVALID,
+                                  MI_SEMAPHORE_SYNC_VB},
+       .signal_mbox            = {GEN6_RVSYNC, GEN6_BVSYNC},
 };
 
 /* Blitter support (SandyBridge+) */
@@ -1062,126 +1445,58 @@ static const struct intel_ring_buffer gen6_bsd_ring = {
 static bool
 blt_ring_get_irq(struct intel_ring_buffer *ring)
 {
-       return ring_get_irq(ring, GT_BLT_USER_INTERRUPT);
+       return gen6_ring_get_irq(ring,
+                                GT_BLT_USER_INTERRUPT,
+                                GEN6_BLITTER_USER_INTERRUPT);
 }
 
 static void
 blt_ring_put_irq(struct intel_ring_buffer *ring)
 {
-       ring_put_irq(ring, GT_BLT_USER_INTERRUPT);
-}
-
-
-/* Workaround for some stepping of SNB,
- * each time when BLT engine ring tail moved,
- * the first command in the ring to be parsed
- * should be MI_BATCH_BUFFER_START
- */
-#define NEED_BLT_WORKAROUND(dev) \
-       (IS_GEN6(dev) && (dev->pdev->revision < 8))
-
-static inline struct drm_i915_gem_object *
-to_blt_workaround(struct intel_ring_buffer *ring)
-{
-       return ring->private;
-}
-
-static int blt_ring_init(struct intel_ring_buffer *ring)
-{
-       if (NEED_BLT_WORKAROUND(ring->dev)) {
-               struct drm_i915_gem_object *obj;
-               u32 *ptr;
-               int ret;
-
-               obj = i915_gem_alloc_object(ring->dev, 4096);
-               if (obj == NULL)
-                       return -ENOMEM;
-
-               ret = i915_gem_object_pin(obj, 4096, true);
-               if (ret) {
-                       drm_gem_object_unreference(&obj->base);
-                       return ret;
-               }
-
-               ptr = kmap(obj->pages[0]);
-               *ptr++ = MI_BATCH_BUFFER_END;
-               *ptr++ = MI_NOOP;
-               kunmap(obj->pages[0]);
-
-               ret = i915_gem_object_set_to_gtt_domain(obj, false);
-               if (ret) {
-                       i915_gem_object_unpin(obj);
-                       drm_gem_object_unreference(&obj->base);
-                       return ret;
-               }
-
-               ring->private = obj;
-       }
-
-       return init_ring_common(ring);
-}
-
-static int blt_ring_begin(struct intel_ring_buffer *ring,
-                         int num_dwords)
-{
-       if (ring->private) {
-               int ret = intel_ring_begin(ring, num_dwords+2);
-               if (ret)
-                       return ret;
-
-               intel_ring_emit(ring, MI_BATCH_BUFFER_START);
-               intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset);
-
-               return 0;
-       } else
-               return intel_ring_begin(ring, 4);
+       gen6_ring_put_irq(ring,
+                         GT_BLT_USER_INTERRUPT,
+                         GEN6_BLITTER_USER_INTERRUPT);
 }
 
 static int blt_ring_flush(struct intel_ring_buffer *ring,
-                          u32 invalidate_domains,
-                          u32 flush_domains)
+                         u32 invalidate, u32 flush)
 {
+       uint32_t cmd;
        int ret;
 
-       if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
-               return 0;
-
-       ret = blt_ring_begin(ring, 4);
+       ret = intel_ring_begin(ring, 4);
        if (ret)
                return ret;
 
-       intel_ring_emit(ring, MI_FLUSH_DW);
-       intel_ring_emit(ring, 0);
+       cmd = MI_FLUSH_DW;
+       if (invalidate & I915_GEM_DOMAIN_RENDER)
+               cmd |= MI_INVALIDATE_TLB;
+       intel_ring_emit(ring, cmd);
        intel_ring_emit(ring, 0);
        intel_ring_emit(ring, 0);
+       intel_ring_emit(ring, MI_NOOP);
        intel_ring_advance(ring);
        return 0;
 }
 
-static void blt_ring_cleanup(struct intel_ring_buffer *ring)
-{
-       if (!ring->private)
-               return;
-
-       i915_gem_object_unpin(ring->private);
-       drm_gem_object_unreference(ring->private);
-       ring->private = NULL;
-}
-
 static const struct intel_ring_buffer gen6_blt_ring = {
-       .name                   = "blt ring",
-       .id                     = RING_BLT,
-       .mmio_base              = BLT_RING_BASE,
-       .size                   = 32 * PAGE_SIZE,
-       .init                   = blt_ring_init,
-       .write_tail             = ring_write_tail,
-       .flush                  = blt_ring_flush,
-       .add_request            = gen6_add_request,
-       .get_seqno              = ring_get_seqno,
-       .irq_get                        = blt_ring_get_irq,
-       .irq_put                        = blt_ring_put_irq,
-       .dispatch_execbuffer    = gen6_ring_dispatch_execbuffer,
-       .cleanup                        = blt_ring_cleanup,
+       .name                   = "blt ring",
+       .id                     = BCS,
+       .mmio_base              = BLT_RING_BASE,
+       .size                   = 32 * PAGE_SIZE,
+       .init                   = init_ring_common,
+       .write_tail             = ring_write_tail,
+       .flush                  = blt_ring_flush,
+       .add_request            = gen6_add_request,
+       .get_seqno              = gen6_ring_get_seqno,
+       .irq_get                = blt_ring_get_irq,
+       .irq_put                = blt_ring_put_irq,
+       .dispatch_execbuffer    = gen6_ring_dispatch_execbuffer,
+       .sync_to                = gen6_blt_ring_sync_to,
+       .semaphore_register     = {MI_SEMAPHORE_SYNC_BR,
+                                  MI_SEMAPHORE_SYNC_BV,
+                                  MI_SEMAPHORE_SYNC_INVALID},
+       .signal_mbox            = {GEN6_RBSYNC, GEN6_VBSYNC},
 };
 
 int intel_init_render_ring_buffer(struct drm_device *dev)
@@ -1192,6 +1507,10 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
        *ring = render_ring;
        if (INTEL_INFO(dev)->gen >= 6) {
                ring->add_request = gen6_add_request;
+               ring->flush = gen6_render_ring_flush;
+               ring->irq_get = gen6_render_ring_get_irq;
+               ring->irq_put = gen6_render_ring_put_irq;
+               ring->get_seqno = gen6_ring_get_seqno;
        } else if (IS_GEN5(dev)) {
                ring->add_request = pc_render_add_request;
                ring->get_seqno = pc_render_get_seqno;
@@ -1205,12 +1524,57 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
        return intel_init_ring_buffer(dev, ring);
 }
 
+int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
+{
+       drm_i915_private_t *dev_priv = dev->dev_private;
+       struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
+
+       *ring = render_ring;
+       if (INTEL_INFO(dev)->gen >= 6) {
+               ring->add_request = gen6_add_request;
+               ring->irq_get = gen6_render_ring_get_irq;
+               ring->irq_put = gen6_render_ring_put_irq;
+       } else if (IS_GEN5(dev)) {
+               ring->add_request = pc_render_add_request;
+               ring->get_seqno = pc_render_get_seqno;
+       }
+
+       if (!I915_NEED_GFX_HWS(dev))
+               ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
+
+       ring->dev = dev;
+       INIT_LIST_HEAD(&ring->active_list);
+       INIT_LIST_HEAD(&ring->request_list);
+       INIT_LIST_HEAD(&ring->gpu_write_list);
+
+       ring->size = size;
+       ring->effective_size = ring->size;
+       if (IS_I830(ring->dev))
+               ring->effective_size -= 128;
+
+       ring->map.offset = start;
+       ring->map.size = size;
+       ring->map.type = 0;
+       ring->map.flags = 0;
+       ring->map.mtrr = 0;
+
+       drm_core_ioremap_wc(&ring->map, dev);
+       if (ring->map.handle == NULL) {
+               DRM_ERROR("can not ioremap virtual address for"
+                         " ring buffer\n");
+               return -ENOMEM;
+       }
+
+       ring->virtual_start = (void __force __iomem *)ring->map.handle;
+       return 0;
+}
+
 int intel_init_bsd_ring_buffer(struct drm_device *dev)
 {
        drm_i915_private_t *dev_priv = dev->dev_private;
        struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
 
-       if (IS_GEN6(dev))
+       if (IS_GEN6(dev) || IS_GEN7(dev))
                *ring = gen6_bsd_ring;
        else
                *ring = bsd_ring;