drm/i915: Reset last_retired_head when resetting ring
[linux-2.6.git] / drivers / gpu / drm / i915 / intel_ringbuffer.c
1 /*
2  * Copyright © 2008-2010 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Eric Anholt <eric@anholt.net>
25  *    Zou Nan hai <nanhai.zou@intel.com>
26  *    Xiang Hai hao<haihao.xiang@intel.com>
27  *
28  */
29
30 #include "drmP.h"
31 #include "drm.h"
32 #include "i915_drv.h"
33 #include "i915_drm.h"
34 #include "i915_trace.h"
35 #include "intel_drv.h"
36
37 /*
38  * 965+ support PIPE_CONTROL commands, which provide finer grained control
39  * over cache flushing.
40  */
41 struct pipe_control {
42         struct drm_i915_gem_object *obj;
43         volatile u32 *cpu_page;
44         u32 gtt_offset;
45 };
46
47 static inline int ring_space(struct intel_ring_buffer *ring)
48 {
49         int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
50         if (space < 0)
51                 space += ring->size;
52         return space;
53 }
54
55 static int
56 render_ring_flush(struct intel_ring_buffer *ring,
57                   u32   invalidate_domains,
58                   u32   flush_domains)
59 {
60         struct drm_device *dev = ring->dev;
61         u32 cmd;
62         int ret;
63
64         /*
65          * read/write caches:
66          *
67          * I915_GEM_DOMAIN_RENDER is always invalidated, but is
68          * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
69          * also flushed at 2d versus 3d pipeline switches.
70          *
71          * read-only caches:
72          *
73          * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
74          * MI_READ_FLUSH is set, and is always flushed on 965.
75          *
76          * I915_GEM_DOMAIN_COMMAND may not exist?
77          *
78          * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
79          * invalidated when MI_EXE_FLUSH is set.
80          *
81          * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
82          * invalidated with every MI_FLUSH.
83          *
84          * TLBs:
85          *
86          * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
87          * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
88          * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
89          * are flushed at any MI_FLUSH.
90          */
91
92         cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
93         if ((invalidate_domains|flush_domains) &
94             I915_GEM_DOMAIN_RENDER)
95                 cmd &= ~MI_NO_WRITE_FLUSH;
96         if (INTEL_INFO(dev)->gen < 4) {
97                 /*
98                  * On the 965, the sampler cache always gets flushed
99                  * and this bit is reserved.
100                  */
101                 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
102                         cmd |= MI_READ_FLUSH;
103         }
104         if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
105                 cmd |= MI_EXE_FLUSH;
106
107         if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
108             (IS_G4X(dev) || IS_GEN5(dev)))
109                 cmd |= MI_INVALIDATE_ISP;
110
111         ret = intel_ring_begin(ring, 2);
112         if (ret)
113                 return ret;
114
115         intel_ring_emit(ring, cmd);
116         intel_ring_emit(ring, MI_NOOP);
117         intel_ring_advance(ring);
118
119         return 0;
120 }
121
122 /**
123  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
124  * implementing two workarounds on gen6.  From section 1.4.7.1
125  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
126  *
127  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
128  * produced by non-pipelined state commands), software needs to first
129  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
130  * 0.
131  *
132  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
133  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
134  *
135  * And the workaround for these two requires this workaround first:
136  *
137  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
138  * BEFORE the pipe-control with a post-sync op and no write-cache
139  * flushes.
140  *
141  * And this last workaround is tricky because of the requirements on
142  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
143  * volume 2 part 1:
144  *
145  *     "1 of the following must also be set:
146  *      - Render Target Cache Flush Enable ([12] of DW1)
147  *      - Depth Cache Flush Enable ([0] of DW1)
148  *      - Stall at Pixel Scoreboard ([1] of DW1)
149  *      - Depth Stall ([13] of DW1)
150  *      - Post-Sync Operation ([13] of DW1)
151  *      - Notify Enable ([8] of DW1)"
152  *
153  * The cache flushes require the workaround flush that triggered this
154  * one, so we can't use it.  Depth stall would trigger the same.
155  * Post-sync nonzero is what triggered this second workaround, so we
156  * can't use that one either.  Notify enable is IRQs, which aren't
157  * really our business.  That leaves only stall at scoreboard.
158  */
159 static int
160 intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
161 {
162         struct pipe_control *pc = ring->private;
163         u32 scratch_addr = pc->gtt_offset + 128;
164         int ret;
165
166
167         ret = intel_ring_begin(ring, 6);
168         if (ret)
169                 return ret;
170
171         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
172         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
173                         PIPE_CONTROL_STALL_AT_SCOREBOARD);
174         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
175         intel_ring_emit(ring, 0); /* low dword */
176         intel_ring_emit(ring, 0); /* high dword */
177         intel_ring_emit(ring, MI_NOOP);
178         intel_ring_advance(ring);
179
180         ret = intel_ring_begin(ring, 6);
181         if (ret)
182                 return ret;
183
184         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
185         intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
186         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
187         intel_ring_emit(ring, 0);
188         intel_ring_emit(ring, 0);
189         intel_ring_emit(ring, MI_NOOP);
190         intel_ring_advance(ring);
191
192         return 0;
193 }
194
195 static int
196 gen6_render_ring_flush(struct intel_ring_buffer *ring,
197                          u32 invalidate_domains, u32 flush_domains)
198 {
199         u32 flags = 0;
200         struct pipe_control *pc = ring->private;
201         u32 scratch_addr = pc->gtt_offset + 128;
202         int ret;
203
204         /* Force SNB workarounds for PIPE_CONTROL flushes */
205         intel_emit_post_sync_nonzero_flush(ring);
206
207         /* Just flush everything.  Experiments have shown that reducing the
208          * number of bits based on the write domains has little performance
209          * impact.
210          */
211         flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
212         flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
213         flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
214         flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
215         flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
216         flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
217         flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
218
219         ret = intel_ring_begin(ring, 6);
220         if (ret)
221                 return ret;
222
223         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
224         intel_ring_emit(ring, flags);
225         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
226         intel_ring_emit(ring, 0); /* lower dword */
227         intel_ring_emit(ring, 0); /* uppwer dword */
228         intel_ring_emit(ring, MI_NOOP);
229         intel_ring_advance(ring);
230
231         return 0;
232 }
233
234 static void ring_write_tail(struct intel_ring_buffer *ring,
235                             u32 value)
236 {
237         drm_i915_private_t *dev_priv = ring->dev->dev_private;
238         I915_WRITE_TAIL(ring, value);
239 }
240
241 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
242 {
243         drm_i915_private_t *dev_priv = ring->dev->dev_private;
244         u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
245                         RING_ACTHD(ring->mmio_base) : ACTHD;
246
247         return I915_READ(acthd_reg);
248 }
249
250 static int init_ring_common(struct intel_ring_buffer *ring)
251 {
252         drm_i915_private_t *dev_priv = ring->dev->dev_private;
253         struct drm_i915_gem_object *obj = ring->obj;
254         u32 head;
255
256         /* Stop the ring if it's running. */
257         I915_WRITE_CTL(ring, 0);
258         I915_WRITE_HEAD(ring, 0);
259         ring->write_tail(ring, 0);
260
261         /* Initialize the ring. */
262         I915_WRITE_START(ring, obj->gtt_offset);
263         head = I915_READ_HEAD(ring) & HEAD_ADDR;
264
265         /* G45 ring initialization fails to reset head to zero */
266         if (head != 0) {
267                 DRM_DEBUG_KMS("%s head not reset to zero "
268                               "ctl %08x head %08x tail %08x start %08x\n",
269                               ring->name,
270                               I915_READ_CTL(ring),
271                               I915_READ_HEAD(ring),
272                               I915_READ_TAIL(ring),
273                               I915_READ_START(ring));
274
275                 I915_WRITE_HEAD(ring, 0);
276
277                 if (I915_READ_HEAD(ring) & HEAD_ADDR) {
278                         DRM_ERROR("failed to set %s head to zero "
279                                   "ctl %08x head %08x tail %08x start %08x\n",
280                                   ring->name,
281                                   I915_READ_CTL(ring),
282                                   I915_READ_HEAD(ring),
283                                   I915_READ_TAIL(ring),
284                                   I915_READ_START(ring));
285                 }
286         }
287
288         I915_WRITE_CTL(ring,
289                         ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
290                         | RING_VALID);
291
292         /* If the head is still not zero, the ring is dead */
293         if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
294             I915_READ_START(ring) != obj->gtt_offset ||
295             (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
296                 DRM_ERROR("%s initialization failed "
297                                 "ctl %08x head %08x tail %08x start %08x\n",
298                                 ring->name,
299                                 I915_READ_CTL(ring),
300                                 I915_READ_HEAD(ring),
301                                 I915_READ_TAIL(ring),
302                                 I915_READ_START(ring));
303                 return -EIO;
304         }
305
306         if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
307                 i915_kernel_lost_context(ring->dev);
308         else {
309                 ring->head = I915_READ_HEAD(ring);
310                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
311                 ring->space = ring_space(ring);
312                 ring->last_retired_head = -1;
313         }
314
315         return 0;
316 }
317
318 static int
319 init_pipe_control(struct intel_ring_buffer *ring)
320 {
321         struct pipe_control *pc;
322         struct drm_i915_gem_object *obj;
323         int ret;
324
325         if (ring->private)
326                 return 0;
327
328         pc = kmalloc(sizeof(*pc), GFP_KERNEL);
329         if (!pc)
330                 return -ENOMEM;
331
332         obj = i915_gem_alloc_object(ring->dev, 4096);
333         if (obj == NULL) {
334                 DRM_ERROR("Failed to allocate seqno page\n");
335                 ret = -ENOMEM;
336                 goto err;
337         }
338
339         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
340
341         ret = i915_gem_object_pin(obj, 4096, true);
342         if (ret)
343                 goto err_unref;
344
345         pc->gtt_offset = obj->gtt_offset;
346         pc->cpu_page =  kmap(obj->pages[0]);
347         if (pc->cpu_page == NULL)
348                 goto err_unpin;
349
350         pc->obj = obj;
351         ring->private = pc;
352         return 0;
353
354 err_unpin:
355         i915_gem_object_unpin(obj);
356 err_unref:
357         drm_gem_object_unreference(&obj->base);
358 err:
359         kfree(pc);
360         return ret;
361 }
362
363 static void
364 cleanup_pipe_control(struct intel_ring_buffer *ring)
365 {
366         struct pipe_control *pc = ring->private;
367         struct drm_i915_gem_object *obj;
368
369         if (!ring->private)
370                 return;
371
372         obj = pc->obj;
373         kunmap(obj->pages[0]);
374         i915_gem_object_unpin(obj);
375         drm_gem_object_unreference(&obj->base);
376
377         kfree(pc);
378         ring->private = NULL;
379 }
380
381 static int init_render_ring(struct intel_ring_buffer *ring)
382 {
383         struct drm_device *dev = ring->dev;
384         struct drm_i915_private *dev_priv = dev->dev_private;
385         int ret = init_ring_common(ring);
386
387         if (INTEL_INFO(dev)->gen > 3) {
388                 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
389                 I915_WRITE(MI_MODE, mode);
390                 if (IS_GEN7(dev))
391                         I915_WRITE(GFX_MODE_GEN7,
392                                    GFX_MODE_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
393                                    GFX_MODE_ENABLE(GFX_REPLAY_MODE));
394         }
395
396         if (INTEL_INFO(dev)->gen >= 5) {
397                 ret = init_pipe_control(ring);
398                 if (ret)
399                         return ret;
400         }
401
402
403         if (IS_GEN6(dev)) {
404                 /* From the Sandybridge PRM, volume 1 part 3, page 24:
405                  * "If this bit is set, STCunit will have LRA as replacement
406                  *  policy. [...] This bit must be reset.  LRA replacement
407                  *  policy is not supported."
408                  */
409                 I915_WRITE(CACHE_MODE_0,
410                            CM0_STC_EVICT_DISABLE_LRA_SNB << CM0_MASK_SHIFT);
411         }
412
413         if (INTEL_INFO(dev)->gen >= 6) {
414                 I915_WRITE(INSTPM,
415                            INSTPM_FORCE_ORDERING << 16 | INSTPM_FORCE_ORDERING);
416         }
417
418         return ret;
419 }
420
421 static void render_ring_cleanup(struct intel_ring_buffer *ring)
422 {
423         if (!ring->private)
424                 return;
425
426         cleanup_pipe_control(ring);
427 }
428
429 static void
430 update_mboxes(struct intel_ring_buffer *ring,
431             u32 seqno,
432             u32 mmio_offset)
433 {
434         intel_ring_emit(ring, MI_SEMAPHORE_MBOX |
435                               MI_SEMAPHORE_GLOBAL_GTT |
436                               MI_SEMAPHORE_REGISTER |
437                               MI_SEMAPHORE_UPDATE);
438         intel_ring_emit(ring, seqno);
439         intel_ring_emit(ring, mmio_offset);
440 }
441
442 /**
443  * gen6_add_request - Update the semaphore mailbox registers
444  * 
445  * @ring - ring that is adding a request
446  * @seqno - return seqno stuck into the ring
447  *
448  * Update the mailbox registers in the *other* rings with the current seqno.
449  * This acts like a signal in the canonical semaphore.
450  */
451 static int
452 gen6_add_request(struct intel_ring_buffer *ring,
453                  u32 *seqno)
454 {
455         u32 mbox1_reg;
456         u32 mbox2_reg;
457         int ret;
458
459         ret = intel_ring_begin(ring, 10);
460         if (ret)
461                 return ret;
462
463         mbox1_reg = ring->signal_mbox[0];
464         mbox2_reg = ring->signal_mbox[1];
465
466         *seqno = i915_gem_next_request_seqno(ring);
467
468         update_mboxes(ring, *seqno, mbox1_reg);
469         update_mboxes(ring, *seqno, mbox2_reg);
470         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
471         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
472         intel_ring_emit(ring, *seqno);
473         intel_ring_emit(ring, MI_USER_INTERRUPT);
474         intel_ring_advance(ring);
475
476         return 0;
477 }
478
479 /**
480  * intel_ring_sync - sync the waiter to the signaller on seqno
481  *
482  * @waiter - ring that is waiting
483  * @signaller - ring which has, or will signal
484  * @seqno - seqno which the waiter will block on
485  */
486 static int
487 intel_ring_sync(struct intel_ring_buffer *waiter,
488                 struct intel_ring_buffer *signaller,
489                 int ring,
490                 u32 seqno)
491 {
492         int ret;
493         u32 dw1 = MI_SEMAPHORE_MBOX |
494                   MI_SEMAPHORE_COMPARE |
495                   MI_SEMAPHORE_REGISTER;
496
497         ret = intel_ring_begin(waiter, 4);
498         if (ret)
499                 return ret;
500
501         intel_ring_emit(waiter, dw1 | signaller->semaphore_register[ring]);
502         intel_ring_emit(waiter, seqno);
503         intel_ring_emit(waiter, 0);
504         intel_ring_emit(waiter, MI_NOOP);
505         intel_ring_advance(waiter);
506
507         return 0;
508 }
509
510 /* VCS->RCS (RVSYNC) or BCS->RCS (RBSYNC) */
511 int
512 render_ring_sync_to(struct intel_ring_buffer *waiter,
513                     struct intel_ring_buffer *signaller,
514                     u32 seqno)
515 {
516         WARN_ON(signaller->semaphore_register[RCS] == MI_SEMAPHORE_SYNC_INVALID);
517         return intel_ring_sync(waiter,
518                                signaller,
519                                RCS,
520                                seqno);
521 }
522
523 /* RCS->VCS (VRSYNC) or BCS->VCS (VBSYNC) */
524 int
525 gen6_bsd_ring_sync_to(struct intel_ring_buffer *waiter,
526                       struct intel_ring_buffer *signaller,
527                       u32 seqno)
528 {
529         WARN_ON(signaller->semaphore_register[VCS] == MI_SEMAPHORE_SYNC_INVALID);
530         return intel_ring_sync(waiter,
531                                signaller,
532                                VCS,
533                                seqno);
534 }
535
536 /* RCS->BCS (BRSYNC) or VCS->BCS (BVSYNC) */
537 int
538 gen6_blt_ring_sync_to(struct intel_ring_buffer *waiter,
539                       struct intel_ring_buffer *signaller,
540                       u32 seqno)
541 {
542         WARN_ON(signaller->semaphore_register[BCS] == MI_SEMAPHORE_SYNC_INVALID);
543         return intel_ring_sync(waiter,
544                                signaller,
545                                BCS,
546                                seqno);
547 }
548
549
550
551 #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
552 do {                                                                    \
553         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |                \
554                  PIPE_CONTROL_DEPTH_STALL);                             \
555         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
556         intel_ring_emit(ring__, 0);                                                     \
557         intel_ring_emit(ring__, 0);                                                     \
558 } while (0)
559
560 static int
561 pc_render_add_request(struct intel_ring_buffer *ring,
562                       u32 *result)
563 {
564         u32 seqno = i915_gem_next_request_seqno(ring);
565         struct pipe_control *pc = ring->private;
566         u32 scratch_addr = pc->gtt_offset + 128;
567         int ret;
568
569         /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
570          * incoherent with writes to memory, i.e. completely fubar,
571          * so we need to use PIPE_NOTIFY instead.
572          *
573          * However, we also need to workaround the qword write
574          * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
575          * memory before requesting an interrupt.
576          */
577         ret = intel_ring_begin(ring, 32);
578         if (ret)
579                 return ret;
580
581         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
582                         PIPE_CONTROL_WRITE_FLUSH |
583                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
584         intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
585         intel_ring_emit(ring, seqno);
586         intel_ring_emit(ring, 0);
587         PIPE_CONTROL_FLUSH(ring, scratch_addr);
588         scratch_addr += 128; /* write to separate cachelines */
589         PIPE_CONTROL_FLUSH(ring, scratch_addr);
590         scratch_addr += 128;
591         PIPE_CONTROL_FLUSH(ring, scratch_addr);
592         scratch_addr += 128;
593         PIPE_CONTROL_FLUSH(ring, scratch_addr);
594         scratch_addr += 128;
595         PIPE_CONTROL_FLUSH(ring, scratch_addr);
596         scratch_addr += 128;
597         PIPE_CONTROL_FLUSH(ring, scratch_addr);
598
599         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
600                         PIPE_CONTROL_WRITE_FLUSH |
601                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
602                         PIPE_CONTROL_NOTIFY);
603         intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
604         intel_ring_emit(ring, seqno);
605         intel_ring_emit(ring, 0);
606         intel_ring_advance(ring);
607
608         *result = seqno;
609         return 0;
610 }
611
612 static int
613 render_ring_add_request(struct intel_ring_buffer *ring,
614                         u32 *result)
615 {
616         u32 seqno = i915_gem_next_request_seqno(ring);
617         int ret;
618
619         ret = intel_ring_begin(ring, 4);
620         if (ret)
621                 return ret;
622
623         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
624         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
625         intel_ring_emit(ring, seqno);
626         intel_ring_emit(ring, MI_USER_INTERRUPT);
627         intel_ring_advance(ring);
628
629         *result = seqno;
630         return 0;
631 }
632
633 static u32
634 gen6_ring_get_seqno(struct intel_ring_buffer *ring)
635 {
636         struct drm_device *dev = ring->dev;
637
638         /* Workaround to force correct ordering between irq and seqno writes on
639          * ivb (and maybe also on snb) by reading from a CS register (like
640          * ACTHD) before reading the status page. */
641         if (IS_GEN6(dev) || IS_GEN7(dev))
642                 intel_ring_get_active_head(ring);
643         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
644 }
645
646 static u32
647 ring_get_seqno(struct intel_ring_buffer *ring)
648 {
649         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
650 }
651
652 static u32
653 pc_render_get_seqno(struct intel_ring_buffer *ring)
654 {
655         struct pipe_control *pc = ring->private;
656         return pc->cpu_page[0];
657 }
658
659 static void
660 ironlake_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
661 {
662         dev_priv->gt_irq_mask &= ~mask;
663         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
664         POSTING_READ(GTIMR);
665 }
666
667 static void
668 ironlake_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
669 {
670         dev_priv->gt_irq_mask |= mask;
671         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
672         POSTING_READ(GTIMR);
673 }
674
675 static void
676 i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
677 {
678         dev_priv->irq_mask &= ~mask;
679         I915_WRITE(IMR, dev_priv->irq_mask);
680         POSTING_READ(IMR);
681 }
682
683 static void
684 i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
685 {
686         dev_priv->irq_mask |= mask;
687         I915_WRITE(IMR, dev_priv->irq_mask);
688         POSTING_READ(IMR);
689 }
690
691 static bool
692 render_ring_get_irq(struct intel_ring_buffer *ring)
693 {
694         struct drm_device *dev = ring->dev;
695         drm_i915_private_t *dev_priv = dev->dev_private;
696
697         if (!dev->irq_enabled)
698                 return false;
699
700         spin_lock(&ring->irq_lock);
701         if (ring->irq_refcount++ == 0) {
702                 if (HAS_PCH_SPLIT(dev))
703                         ironlake_enable_irq(dev_priv,
704                                             GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
705                 else
706                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
707         }
708         spin_unlock(&ring->irq_lock);
709
710         return true;
711 }
712
713 static void
714 render_ring_put_irq(struct intel_ring_buffer *ring)
715 {
716         struct drm_device *dev = ring->dev;
717         drm_i915_private_t *dev_priv = dev->dev_private;
718
719         spin_lock(&ring->irq_lock);
720         if (--ring->irq_refcount == 0) {
721                 if (HAS_PCH_SPLIT(dev))
722                         ironlake_disable_irq(dev_priv,
723                                              GT_USER_INTERRUPT |
724                                              GT_PIPE_NOTIFY);
725                 else
726                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
727         }
728         spin_unlock(&ring->irq_lock);
729 }
730
731 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
732 {
733         struct drm_device *dev = ring->dev;
734         drm_i915_private_t *dev_priv = ring->dev->dev_private;
735         u32 mmio = 0;
736
737         /* The ring status page addresses are no longer next to the rest of
738          * the ring registers as of gen7.
739          */
740         if (IS_GEN7(dev)) {
741                 switch (ring->id) {
742                 case RCS:
743                         mmio = RENDER_HWS_PGA_GEN7;
744                         break;
745                 case BCS:
746                         mmio = BLT_HWS_PGA_GEN7;
747                         break;
748                 case VCS:
749                         mmio = BSD_HWS_PGA_GEN7;
750                         break;
751                 }
752         } else if (IS_GEN6(ring->dev)) {
753                 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
754         } else {
755                 mmio = RING_HWS_PGA(ring->mmio_base);
756         }
757
758         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
759         POSTING_READ(mmio);
760 }
761
762 static int
763 bsd_ring_flush(struct intel_ring_buffer *ring,
764                u32     invalidate_domains,
765                u32     flush_domains)
766 {
767         int ret;
768
769         ret = intel_ring_begin(ring, 2);
770         if (ret)
771                 return ret;
772
773         intel_ring_emit(ring, MI_FLUSH);
774         intel_ring_emit(ring, MI_NOOP);
775         intel_ring_advance(ring);
776         return 0;
777 }
778
779 static int
780 ring_add_request(struct intel_ring_buffer *ring,
781                  u32 *result)
782 {
783         u32 seqno;
784         int ret;
785
786         ret = intel_ring_begin(ring, 4);
787         if (ret)
788                 return ret;
789
790         seqno = i915_gem_next_request_seqno(ring);
791
792         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
793         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
794         intel_ring_emit(ring, seqno);
795         intel_ring_emit(ring, MI_USER_INTERRUPT);
796         intel_ring_advance(ring);
797
798         *result = seqno;
799         return 0;
800 }
801
802 static bool
803 gen6_ring_get_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
804 {
805         struct drm_device *dev = ring->dev;
806         drm_i915_private_t *dev_priv = dev->dev_private;
807
808         if (!dev->irq_enabled)
809                return false;
810
811         /* It looks like we need to prevent the gt from suspending while waiting
812          * for an notifiy irq, otherwise irqs seem to get lost on at least the
813          * blt/bsd rings on ivb. */
814         gen6_gt_force_wake_get(dev_priv);
815
816         spin_lock(&ring->irq_lock);
817         if (ring->irq_refcount++ == 0) {
818                 ring->irq_mask &= ~rflag;
819                 I915_WRITE_IMR(ring, ring->irq_mask);
820                 ironlake_enable_irq(dev_priv, gflag);
821         }
822         spin_unlock(&ring->irq_lock);
823
824         return true;
825 }
826
827 static void
828 gen6_ring_put_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
829 {
830         struct drm_device *dev = ring->dev;
831         drm_i915_private_t *dev_priv = dev->dev_private;
832
833         spin_lock(&ring->irq_lock);
834         if (--ring->irq_refcount == 0) {
835                 ring->irq_mask |= rflag;
836                 I915_WRITE_IMR(ring, ring->irq_mask);
837                 ironlake_disable_irq(dev_priv, gflag);
838         }
839         spin_unlock(&ring->irq_lock);
840
841         gen6_gt_force_wake_put(dev_priv);
842 }
843
844 static bool
845 bsd_ring_get_irq(struct intel_ring_buffer *ring)
846 {
847         struct drm_device *dev = ring->dev;
848         drm_i915_private_t *dev_priv = dev->dev_private;
849
850         if (!dev->irq_enabled)
851                 return false;
852
853         spin_lock(&ring->irq_lock);
854         if (ring->irq_refcount++ == 0) {
855                 if (IS_G4X(dev))
856                         i915_enable_irq(dev_priv, I915_BSD_USER_INTERRUPT);
857                 else
858                         ironlake_enable_irq(dev_priv, GT_BSD_USER_INTERRUPT);
859         }
860         spin_unlock(&ring->irq_lock);
861
862         return true;
863 }
864 static void
865 bsd_ring_put_irq(struct intel_ring_buffer *ring)
866 {
867         struct drm_device *dev = ring->dev;
868         drm_i915_private_t *dev_priv = dev->dev_private;
869
870         spin_lock(&ring->irq_lock);
871         if (--ring->irq_refcount == 0) {
872                 if (IS_G4X(dev))
873                         i915_disable_irq(dev_priv, I915_BSD_USER_INTERRUPT);
874                 else
875                         ironlake_disable_irq(dev_priv, GT_BSD_USER_INTERRUPT);
876         }
877         spin_unlock(&ring->irq_lock);
878 }
879
880 static int
881 ring_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
882 {
883         int ret;
884
885         ret = intel_ring_begin(ring, 2);
886         if (ret)
887                 return ret;
888
889         intel_ring_emit(ring,
890                         MI_BATCH_BUFFER_START | (2 << 6) |
891                         MI_BATCH_NON_SECURE_I965);
892         intel_ring_emit(ring, offset);
893         intel_ring_advance(ring);
894
895         return 0;
896 }
897
898 static int
899 render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
900                                 u32 offset, u32 len)
901 {
902         struct drm_device *dev = ring->dev;
903         int ret;
904
905         if (IS_I830(dev) || IS_845G(dev)) {
906                 ret = intel_ring_begin(ring, 4);
907                 if (ret)
908                         return ret;
909
910                 intel_ring_emit(ring, MI_BATCH_BUFFER);
911                 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
912                 intel_ring_emit(ring, offset + len - 8);
913                 intel_ring_emit(ring, 0);
914         } else {
915                 ret = intel_ring_begin(ring, 2);
916                 if (ret)
917                         return ret;
918
919                 if (INTEL_INFO(dev)->gen >= 4) {
920                         intel_ring_emit(ring,
921                                         MI_BATCH_BUFFER_START | (2 << 6) |
922                                         MI_BATCH_NON_SECURE_I965);
923                         intel_ring_emit(ring, offset);
924                 } else {
925                         intel_ring_emit(ring,
926                                         MI_BATCH_BUFFER_START | (2 << 6));
927                         intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
928                 }
929         }
930         intel_ring_advance(ring);
931
932         return 0;
933 }
934
935 static void cleanup_status_page(struct intel_ring_buffer *ring)
936 {
937         drm_i915_private_t *dev_priv = ring->dev->dev_private;
938         struct drm_i915_gem_object *obj;
939
940         obj = ring->status_page.obj;
941         if (obj == NULL)
942                 return;
943
944         kunmap(obj->pages[0]);
945         i915_gem_object_unpin(obj);
946         drm_gem_object_unreference(&obj->base);
947         ring->status_page.obj = NULL;
948
949         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
950 }
951
952 static int init_status_page(struct intel_ring_buffer *ring)
953 {
954         struct drm_device *dev = ring->dev;
955         drm_i915_private_t *dev_priv = dev->dev_private;
956         struct drm_i915_gem_object *obj;
957         int ret;
958
959         obj = i915_gem_alloc_object(dev, 4096);
960         if (obj == NULL) {
961                 DRM_ERROR("Failed to allocate status page\n");
962                 ret = -ENOMEM;
963                 goto err;
964         }
965
966         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
967
968         ret = i915_gem_object_pin(obj, 4096, true);
969         if (ret != 0) {
970                 goto err_unref;
971         }
972
973         ring->status_page.gfx_addr = obj->gtt_offset;
974         ring->status_page.page_addr = kmap(obj->pages[0]);
975         if (ring->status_page.page_addr == NULL) {
976                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
977                 goto err_unpin;
978         }
979         ring->status_page.obj = obj;
980         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
981
982         intel_ring_setup_status_page(ring);
983         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
984                         ring->name, ring->status_page.gfx_addr);
985
986         return 0;
987
988 err_unpin:
989         i915_gem_object_unpin(obj);
990 err_unref:
991         drm_gem_object_unreference(&obj->base);
992 err:
993         return ret;
994 }
995
996 int intel_init_ring_buffer(struct drm_device *dev,
997                            struct intel_ring_buffer *ring)
998 {
999         struct drm_i915_gem_object *obj;
1000         int ret;
1001
1002         ring->dev = dev;
1003         INIT_LIST_HEAD(&ring->active_list);
1004         INIT_LIST_HEAD(&ring->request_list);
1005         INIT_LIST_HEAD(&ring->gpu_write_list);
1006
1007         init_waitqueue_head(&ring->irq_queue);
1008         spin_lock_init(&ring->irq_lock);
1009         ring->irq_mask = ~0;
1010
1011         if (I915_NEED_GFX_HWS(dev)) {
1012                 ret = init_status_page(ring);
1013                 if (ret)
1014                         return ret;
1015         }
1016
1017         obj = i915_gem_alloc_object(dev, ring->size);
1018         if (obj == NULL) {
1019                 DRM_ERROR("Failed to allocate ringbuffer\n");
1020                 ret = -ENOMEM;
1021                 goto err_hws;
1022         }
1023
1024         ring->obj = obj;
1025
1026         ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
1027         if (ret)
1028                 goto err_unref;
1029
1030         ring->map.size = ring->size;
1031         ring->map.offset = dev->agp->base + obj->gtt_offset;
1032         ring->map.type = 0;
1033         ring->map.flags = 0;
1034         ring->map.mtrr = 0;
1035
1036         drm_core_ioremap_wc(&ring->map, dev);
1037         if (ring->map.handle == NULL) {
1038                 DRM_ERROR("Failed to map ringbuffer.\n");
1039                 ret = -EINVAL;
1040                 goto err_unpin;
1041         }
1042
1043         ring->virtual_start = ring->map.handle;
1044         ret = ring->init(ring);
1045         if (ret)
1046                 goto err_unmap;
1047
1048         /* Workaround an erratum on the i830 which causes a hang if
1049          * the TAIL pointer points to within the last 2 cachelines
1050          * of the buffer.
1051          */
1052         ring->effective_size = ring->size;
1053         if (IS_I830(ring->dev) || IS_845G(ring->dev))
1054                 ring->effective_size -= 128;
1055
1056         return 0;
1057
1058 err_unmap:
1059         drm_core_ioremapfree(&ring->map, dev);
1060 err_unpin:
1061         i915_gem_object_unpin(obj);
1062 err_unref:
1063         drm_gem_object_unreference(&obj->base);
1064         ring->obj = NULL;
1065 err_hws:
1066         cleanup_status_page(ring);
1067         return ret;
1068 }
1069
1070 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
1071 {
1072         struct drm_i915_private *dev_priv;
1073         int ret;
1074
1075         if (ring->obj == NULL)
1076                 return;
1077
1078         /* Disable the ring buffer. The ring must be idle at this point */
1079         dev_priv = ring->dev->dev_private;
1080         ret = intel_wait_ring_idle(ring);
1081         if (ret)
1082                 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
1083                           ring->name, ret);
1084
1085         I915_WRITE_CTL(ring, 0);
1086
1087         drm_core_ioremapfree(&ring->map, ring->dev);
1088
1089         i915_gem_object_unpin(ring->obj);
1090         drm_gem_object_unreference(&ring->obj->base);
1091         ring->obj = NULL;
1092
1093         if (ring->cleanup)
1094                 ring->cleanup(ring);
1095
1096         cleanup_status_page(ring);
1097 }
1098
1099 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
1100 {
1101         unsigned int *virt;
1102         int rem = ring->size - ring->tail;
1103
1104         if (ring->space < rem) {
1105                 int ret = intel_wait_ring_buffer(ring, rem);
1106                 if (ret)
1107                         return ret;
1108         }
1109
1110         virt = (unsigned int *)(ring->virtual_start + ring->tail);
1111         rem /= 8;
1112         while (rem--) {
1113                 *virt++ = MI_NOOP;
1114                 *virt++ = MI_NOOP;
1115         }
1116
1117         ring->tail = 0;
1118         ring->space = ring_space(ring);
1119
1120         return 0;
1121 }
1122
1123 static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
1124 {
1125         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1126         bool was_interruptible;
1127         int ret;
1128
1129         /* XXX As we have not yet audited all the paths to check that
1130          * they are ready for ERESTARTSYS from intel_ring_begin, do not
1131          * allow us to be interruptible by a signal.
1132          */
1133         was_interruptible = dev_priv->mm.interruptible;
1134         dev_priv->mm.interruptible = false;
1135
1136         ret = i915_wait_request(ring, seqno, true);
1137
1138         dev_priv->mm.interruptible = was_interruptible;
1139
1140         return ret;
1141 }
1142
1143 static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
1144 {
1145         struct drm_i915_gem_request *request;
1146         u32 seqno = 0;
1147         int ret;
1148
1149         i915_gem_retire_requests_ring(ring);
1150
1151         if (ring->last_retired_head != -1) {
1152                 ring->head = ring->last_retired_head;
1153                 ring->last_retired_head = -1;
1154                 ring->space = ring_space(ring);
1155                 if (ring->space >= n)
1156                         return 0;
1157         }
1158
1159         list_for_each_entry(request, &ring->request_list, list) {
1160                 int space;
1161
1162                 if (request->tail == -1)
1163                         continue;
1164
1165                 space = request->tail - (ring->tail + 8);
1166                 if (space < 0)
1167                         space += ring->size;
1168                 if (space >= n) {
1169                         seqno = request->seqno;
1170                         break;
1171                 }
1172
1173                 /* Consume this request in case we need more space than
1174                  * is available and so need to prevent a race between
1175                  * updating last_retired_head and direct reads of
1176                  * I915_RING_HEAD. It also provides a nice sanity check.
1177                  */
1178                 request->tail = -1;
1179         }
1180
1181         if (seqno == 0)
1182                 return -ENOSPC;
1183
1184         ret = intel_ring_wait_seqno(ring, seqno);
1185         if (ret)
1186                 return ret;
1187
1188         if (WARN_ON(ring->last_retired_head == -1))
1189                 return -ENOSPC;
1190
1191         ring->head = ring->last_retired_head;
1192         ring->last_retired_head = -1;
1193         ring->space = ring_space(ring);
1194         if (WARN_ON(ring->space < n))
1195                 return -ENOSPC;
1196
1197         return 0;
1198 }
1199
1200 int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
1201 {
1202         struct drm_device *dev = ring->dev;
1203         struct drm_i915_private *dev_priv = dev->dev_private;
1204         unsigned long end;
1205         int ret;
1206
1207         ret = intel_ring_wait_request(ring, n);
1208         if (ret != -ENOSPC)
1209                 return ret;
1210
1211         trace_i915_ring_wait_begin(ring);
1212         if (drm_core_check_feature(dev, DRIVER_GEM))
1213                 /* With GEM the hangcheck timer should kick us out of the loop,
1214                  * leaving it early runs the risk of corrupting GEM state (due
1215                  * to running on almost untested codepaths). But on resume
1216                  * timers don't work yet, so prevent a complete hang in that
1217                  * case by choosing an insanely large timeout. */
1218                 end = jiffies + 60 * HZ;
1219         else
1220                 end = jiffies + 3 * HZ;
1221
1222         do {
1223                 ring->head = I915_READ_HEAD(ring);
1224                 ring->space = ring_space(ring);
1225                 if (ring->space >= n) {
1226                         trace_i915_ring_wait_end(ring);
1227                         return 0;
1228                 }
1229
1230                 if (dev->primary->master) {
1231                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
1232                         if (master_priv->sarea_priv)
1233                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
1234                 }
1235
1236                 msleep(1);
1237                 if (atomic_read(&dev_priv->mm.wedged))
1238                         return -EAGAIN;
1239         } while (!time_after(jiffies, end));
1240         trace_i915_ring_wait_end(ring);
1241         return -EBUSY;
1242 }
1243
1244 int intel_ring_begin(struct intel_ring_buffer *ring,
1245                      int num_dwords)
1246 {
1247         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1248         int n = 4*num_dwords;
1249         int ret;
1250
1251         if (unlikely(atomic_read(&dev_priv->mm.wedged)))
1252                 return -EIO;
1253
1254         if (unlikely(ring->tail + n > ring->effective_size)) {
1255                 ret = intel_wrap_ring_buffer(ring);
1256                 if (unlikely(ret))
1257                         return ret;
1258         }
1259
1260         if (unlikely(ring->space < n)) {
1261                 ret = intel_wait_ring_buffer(ring, n);
1262                 if (unlikely(ret))
1263                         return ret;
1264         }
1265
1266         ring->space -= n;
1267         return 0;
1268 }
1269
1270 void intel_ring_advance(struct intel_ring_buffer *ring)
1271 {
1272         ring->tail &= ring->size - 1;
1273         ring->write_tail(ring, ring->tail);
1274 }
1275
1276 static const struct intel_ring_buffer render_ring = {
1277         .name                   = "render ring",
1278         .id                     = RCS,
1279         .mmio_base              = RENDER_RING_BASE,
1280         .size                   = 32 * PAGE_SIZE,
1281         .init                   = init_render_ring,
1282         .write_tail             = ring_write_tail,
1283         .flush                  = render_ring_flush,
1284         .add_request            = render_ring_add_request,
1285         .get_seqno              = ring_get_seqno,
1286         .irq_get                = render_ring_get_irq,
1287         .irq_put                = render_ring_put_irq,
1288         .dispatch_execbuffer    = render_ring_dispatch_execbuffer,
1289         .cleanup                = render_ring_cleanup,
1290         .sync_to                = render_ring_sync_to,
1291         .semaphore_register     = {MI_SEMAPHORE_SYNC_INVALID,
1292                                    MI_SEMAPHORE_SYNC_RV,
1293                                    MI_SEMAPHORE_SYNC_RB},
1294         .signal_mbox            = {GEN6_VRSYNC, GEN6_BRSYNC},
1295 };
1296
1297 /* ring buffer for bit-stream decoder */
1298
1299 static const struct intel_ring_buffer bsd_ring = {
1300         .name                   = "bsd ring",
1301         .id                     = VCS,
1302         .mmio_base              = BSD_RING_BASE,
1303         .size                   = 32 * PAGE_SIZE,
1304         .init                   = init_ring_common,
1305         .write_tail             = ring_write_tail,
1306         .flush                  = bsd_ring_flush,
1307         .add_request            = ring_add_request,
1308         .get_seqno              = ring_get_seqno,
1309         .irq_get                = bsd_ring_get_irq,
1310         .irq_put                = bsd_ring_put_irq,
1311         .dispatch_execbuffer    = ring_dispatch_execbuffer,
1312 };
1313
1314
1315 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
1316                                      u32 value)
1317 {
1318         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1319
1320        /* Every tail move must follow the sequence below */
1321         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1322                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1323                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
1324         I915_WRITE(GEN6_BSD_RNCID, 0x0);
1325
1326         if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
1327                 GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
1328                 50))
1329         DRM_ERROR("timed out waiting for IDLE Indicator\n");
1330
1331         I915_WRITE_TAIL(ring, value);
1332         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1333                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1334                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
1335 }
1336
1337 static int gen6_ring_flush(struct intel_ring_buffer *ring,
1338                            u32 invalidate, u32 flush)
1339 {
1340         uint32_t cmd;
1341         int ret;
1342
1343         ret = intel_ring_begin(ring, 4);
1344         if (ret)
1345                 return ret;
1346
1347         cmd = MI_FLUSH_DW;
1348         if (invalidate & I915_GEM_GPU_DOMAINS)
1349                 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
1350         intel_ring_emit(ring, cmd);
1351         intel_ring_emit(ring, 0);
1352         intel_ring_emit(ring, 0);
1353         intel_ring_emit(ring, MI_NOOP);
1354         intel_ring_advance(ring);
1355         return 0;
1356 }
1357
1358 static int
1359 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1360                               u32 offset, u32 len)
1361 {
1362         int ret;
1363
1364         ret = intel_ring_begin(ring, 2);
1365         if (ret)
1366                 return ret;
1367
1368         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
1369         /* bit0-7 is the length on GEN6+ */
1370         intel_ring_emit(ring, offset);
1371         intel_ring_advance(ring);
1372
1373         return 0;
1374 }
1375
1376 static bool
1377 gen6_render_ring_get_irq(struct intel_ring_buffer *ring)
1378 {
1379         return gen6_ring_get_irq(ring,
1380                                  GT_USER_INTERRUPT,
1381                                  GEN6_RENDER_USER_INTERRUPT);
1382 }
1383
1384 static void
1385 gen6_render_ring_put_irq(struct intel_ring_buffer *ring)
1386 {
1387         return gen6_ring_put_irq(ring,
1388                                  GT_USER_INTERRUPT,
1389                                  GEN6_RENDER_USER_INTERRUPT);
1390 }
1391
1392 static bool
1393 gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring)
1394 {
1395         return gen6_ring_get_irq(ring,
1396                                  GT_GEN6_BSD_USER_INTERRUPT,
1397                                  GEN6_BSD_USER_INTERRUPT);
1398 }
1399
1400 static void
1401 gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring)
1402 {
1403         return gen6_ring_put_irq(ring,
1404                                  GT_GEN6_BSD_USER_INTERRUPT,
1405                                  GEN6_BSD_USER_INTERRUPT);
1406 }
1407
1408 /* ring buffer for Video Codec for Gen6+ */
1409 static const struct intel_ring_buffer gen6_bsd_ring = {
1410         .name                   = "gen6 bsd ring",
1411         .id                     = VCS,
1412         .mmio_base              = GEN6_BSD_RING_BASE,
1413         .size                   = 32 * PAGE_SIZE,
1414         .init                   = init_ring_common,
1415         .write_tail             = gen6_bsd_ring_write_tail,
1416         .flush                  = gen6_ring_flush,
1417         .add_request            = gen6_add_request,
1418         .get_seqno              = gen6_ring_get_seqno,
1419         .irq_get                = gen6_bsd_ring_get_irq,
1420         .irq_put                = gen6_bsd_ring_put_irq,
1421         .dispatch_execbuffer    = gen6_ring_dispatch_execbuffer,
1422         .sync_to                = gen6_bsd_ring_sync_to,
1423         .semaphore_register     = {MI_SEMAPHORE_SYNC_VR,
1424                                    MI_SEMAPHORE_SYNC_INVALID,
1425                                    MI_SEMAPHORE_SYNC_VB},
1426         .signal_mbox            = {GEN6_RVSYNC, GEN6_BVSYNC},
1427 };
1428
1429 /* Blitter support (SandyBridge+) */
1430
1431 static bool
1432 blt_ring_get_irq(struct intel_ring_buffer *ring)
1433 {
1434         return gen6_ring_get_irq(ring,
1435                                  GT_BLT_USER_INTERRUPT,
1436                                  GEN6_BLITTER_USER_INTERRUPT);
1437 }
1438
1439 static void
1440 blt_ring_put_irq(struct intel_ring_buffer *ring)
1441 {
1442         gen6_ring_put_irq(ring,
1443                           GT_BLT_USER_INTERRUPT,
1444                           GEN6_BLITTER_USER_INTERRUPT);
1445 }
1446
1447 static int blt_ring_flush(struct intel_ring_buffer *ring,
1448                           u32 invalidate, u32 flush)
1449 {
1450         uint32_t cmd;
1451         int ret;
1452
1453         ret = intel_ring_begin(ring, 4);
1454         if (ret)
1455                 return ret;
1456
1457         cmd = MI_FLUSH_DW;
1458         if (invalidate & I915_GEM_DOMAIN_RENDER)
1459                 cmd |= MI_INVALIDATE_TLB;
1460         intel_ring_emit(ring, cmd);
1461         intel_ring_emit(ring, 0);
1462         intel_ring_emit(ring, 0);
1463         intel_ring_emit(ring, MI_NOOP);
1464         intel_ring_advance(ring);
1465         return 0;
1466 }
1467
1468 static const struct intel_ring_buffer gen6_blt_ring = {
1469         .name                   = "blt ring",
1470         .id                     = BCS,
1471         .mmio_base              = BLT_RING_BASE,
1472         .size                   = 32 * PAGE_SIZE,
1473         .init                   = init_ring_common,
1474         .write_tail             = ring_write_tail,
1475         .flush                  = blt_ring_flush,
1476         .add_request            = gen6_add_request,
1477         .get_seqno              = gen6_ring_get_seqno,
1478         .irq_get                = blt_ring_get_irq,
1479         .irq_put                = blt_ring_put_irq,
1480         .dispatch_execbuffer    = gen6_ring_dispatch_execbuffer,
1481         .sync_to                = gen6_blt_ring_sync_to,
1482         .semaphore_register     = {MI_SEMAPHORE_SYNC_BR,
1483                                    MI_SEMAPHORE_SYNC_BV,
1484                                    MI_SEMAPHORE_SYNC_INVALID},
1485         .signal_mbox            = {GEN6_RBSYNC, GEN6_VBSYNC},
1486 };
1487
1488 int intel_init_render_ring_buffer(struct drm_device *dev)
1489 {
1490         drm_i915_private_t *dev_priv = dev->dev_private;
1491         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1492
1493         *ring = render_ring;
1494         if (INTEL_INFO(dev)->gen >= 6) {
1495                 ring->add_request = gen6_add_request;
1496                 ring->flush = gen6_render_ring_flush;
1497                 ring->irq_get = gen6_render_ring_get_irq;
1498                 ring->irq_put = gen6_render_ring_put_irq;
1499                 ring->get_seqno = gen6_ring_get_seqno;
1500         } else if (IS_GEN5(dev)) {
1501                 ring->add_request = pc_render_add_request;
1502                 ring->get_seqno = pc_render_get_seqno;
1503         }
1504
1505         if (!I915_NEED_GFX_HWS(dev)) {
1506                 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1507                 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1508         }
1509
1510         return intel_init_ring_buffer(dev, ring);
1511 }
1512
1513 int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
1514 {
1515         drm_i915_private_t *dev_priv = dev->dev_private;
1516         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1517
1518         *ring = render_ring;
1519         if (INTEL_INFO(dev)->gen >= 6) {
1520                 ring->add_request = gen6_add_request;
1521                 ring->irq_get = gen6_render_ring_get_irq;
1522                 ring->irq_put = gen6_render_ring_put_irq;
1523         } else if (IS_GEN5(dev)) {
1524                 ring->add_request = pc_render_add_request;
1525                 ring->get_seqno = pc_render_get_seqno;
1526         }
1527
1528         if (!I915_NEED_GFX_HWS(dev))
1529                 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1530
1531         ring->dev = dev;
1532         INIT_LIST_HEAD(&ring->active_list);
1533         INIT_LIST_HEAD(&ring->request_list);
1534         INIT_LIST_HEAD(&ring->gpu_write_list);
1535
1536         ring->size = size;
1537         ring->effective_size = ring->size;
1538         if (IS_I830(ring->dev))
1539                 ring->effective_size -= 128;
1540
1541         ring->map.offset = start;
1542         ring->map.size = size;
1543         ring->map.type = 0;
1544         ring->map.flags = 0;
1545         ring->map.mtrr = 0;
1546
1547         drm_core_ioremap_wc(&ring->map, dev);
1548         if (ring->map.handle == NULL) {
1549                 DRM_ERROR("can not ioremap virtual address for"
1550                           " ring buffer\n");
1551                 return -ENOMEM;
1552         }
1553
1554         ring->virtual_start = (void __force __iomem *)ring->map.handle;
1555         return 0;
1556 }
1557
1558 int intel_init_bsd_ring_buffer(struct drm_device *dev)
1559 {
1560         drm_i915_private_t *dev_priv = dev->dev_private;
1561         struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
1562
1563         if (IS_GEN6(dev) || IS_GEN7(dev))
1564                 *ring = gen6_bsd_ring;
1565         else
1566                 *ring = bsd_ring;
1567
1568         return intel_init_ring_buffer(dev, ring);
1569 }
1570
1571 int intel_init_blt_ring_buffer(struct drm_device *dev)
1572 {
1573         drm_i915_private_t *dev_priv = dev->dev_private;
1574         struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
1575
1576         *ring = gen6_blt_ring;
1577
1578         return intel_init_ring_buffer(dev, ring);
1579 }