c17325ce3799ebf6b91d3160676dd01fd7b58e50
[linux-2.6.git] / drivers / gpu / drm / i915 / intel_ringbuffer.c
1 /*
2  * Copyright © 2008-2010 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Eric Anholt <eric@anholt.net>
25  *    Zou Nan hai <nanhai.zou@intel.com>
26  *    Xiang Hai hao<haihao.xiang@intel.com>
27  *
28  */
29
30 #include "drmP.h"
31 #include "drm.h"
32 #include "i915_drv.h"
33 #include "i915_drm.h"
34 #include "i915_trace.h"
35 #include "intel_drv.h"
36
37 /*
38  * 965+ support PIPE_CONTROL commands, which provide finer grained control
39  * over cache flushing.
40  */
41 struct pipe_control {
42         struct drm_i915_gem_object *obj;
43         volatile u32 *cpu_page;
44         u32 gtt_offset;
45 };
46
47 static inline int ring_space(struct intel_ring_buffer *ring)
48 {
49         int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
50         if (space < 0)
51                 space += ring->size;
52         return space;
53 }
54
55 static int
56 render_ring_flush(struct intel_ring_buffer *ring,
57                   u32   invalidate_domains,
58                   u32   flush_domains)
59 {
60         struct drm_device *dev = ring->dev;
61         u32 cmd;
62         int ret;
63
64         /*
65          * read/write caches:
66          *
67          * I915_GEM_DOMAIN_RENDER is always invalidated, but is
68          * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
69          * also flushed at 2d versus 3d pipeline switches.
70          *
71          * read-only caches:
72          *
73          * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
74          * MI_READ_FLUSH is set, and is always flushed on 965.
75          *
76          * I915_GEM_DOMAIN_COMMAND may not exist?
77          *
78          * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
79          * invalidated when MI_EXE_FLUSH is set.
80          *
81          * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
82          * invalidated with every MI_FLUSH.
83          *
84          * TLBs:
85          *
86          * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
87          * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
88          * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
89          * are flushed at any MI_FLUSH.
90          */
91
92         cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
93         if ((invalidate_domains|flush_domains) &
94             I915_GEM_DOMAIN_RENDER)
95                 cmd &= ~MI_NO_WRITE_FLUSH;
96         if (INTEL_INFO(dev)->gen < 4) {
97                 /*
98                  * On the 965, the sampler cache always gets flushed
99                  * and this bit is reserved.
100                  */
101                 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
102                         cmd |= MI_READ_FLUSH;
103         }
104         if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
105                 cmd |= MI_EXE_FLUSH;
106
107         if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
108             (IS_G4X(dev) || IS_GEN5(dev)))
109                 cmd |= MI_INVALIDATE_ISP;
110
111         ret = intel_ring_begin(ring, 2);
112         if (ret)
113                 return ret;
114
115         intel_ring_emit(ring, cmd);
116         intel_ring_emit(ring, MI_NOOP);
117         intel_ring_advance(ring);
118
119         return 0;
120 }
121
122 /**
123  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
124  * implementing two workarounds on gen6.  From section 1.4.7.1
125  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
126  *
127  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
128  * produced by non-pipelined state commands), software needs to first
129  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
130  * 0.
131  *
132  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
133  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
134  *
135  * And the workaround for these two requires this workaround first:
136  *
137  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
138  * BEFORE the pipe-control with a post-sync op and no write-cache
139  * flushes.
140  *
141  * And this last workaround is tricky because of the requirements on
142  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
143  * volume 2 part 1:
144  *
145  *     "1 of the following must also be set:
146  *      - Render Target Cache Flush Enable ([12] of DW1)
147  *      - Depth Cache Flush Enable ([0] of DW1)
148  *      - Stall at Pixel Scoreboard ([1] of DW1)
149  *      - Depth Stall ([13] of DW1)
150  *      - Post-Sync Operation ([13] of DW1)
151  *      - Notify Enable ([8] of DW1)"
152  *
153  * The cache flushes require the workaround flush that triggered this
154  * one, so we can't use it.  Depth stall would trigger the same.
155  * Post-sync nonzero is what triggered this second workaround, so we
156  * can't use that one either.  Notify enable is IRQs, which aren't
157  * really our business.  That leaves only stall at scoreboard.
158  */
159 static int
160 intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
161 {
162         struct pipe_control *pc = ring->private;
163         u32 scratch_addr = pc->gtt_offset + 128;
164         int ret;
165
166
167         ret = intel_ring_begin(ring, 6);
168         if (ret)
169                 return ret;
170
171         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
172         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
173                         PIPE_CONTROL_STALL_AT_SCOREBOARD);
174         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
175         intel_ring_emit(ring, 0); /* low dword */
176         intel_ring_emit(ring, 0); /* high dword */
177         intel_ring_emit(ring, MI_NOOP);
178         intel_ring_advance(ring);
179
180         ret = intel_ring_begin(ring, 6);
181         if (ret)
182                 return ret;
183
184         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
185         intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
186         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
187         intel_ring_emit(ring, 0);
188         intel_ring_emit(ring, 0);
189         intel_ring_emit(ring, MI_NOOP);
190         intel_ring_advance(ring);
191
192         return 0;
193 }
194
195 static int
196 gen6_render_ring_flush(struct intel_ring_buffer *ring,
197                          u32 invalidate_domains, u32 flush_domains)
198 {
199         u32 flags = 0;
200         struct pipe_control *pc = ring->private;
201         u32 scratch_addr = pc->gtt_offset + 128;
202         int ret;
203
204         /* Force SNB workarounds for PIPE_CONTROL flushes */
205         intel_emit_post_sync_nonzero_flush(ring);
206
207         /* Just flush everything.  Experiments have shown that reducing the
208          * number of bits based on the write domains has little performance
209          * impact.
210          */
211         flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
212         flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
213         flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
214         flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
215         flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
216         flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
217         flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
218
219         ret = intel_ring_begin(ring, 6);
220         if (ret)
221                 return ret;
222
223         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
224         intel_ring_emit(ring, flags);
225         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
226         intel_ring_emit(ring, 0); /* lower dword */
227         intel_ring_emit(ring, 0); /* uppwer dword */
228         intel_ring_emit(ring, MI_NOOP);
229         intel_ring_advance(ring);
230
231         return 0;
232 }
233
234 static void ring_write_tail(struct intel_ring_buffer *ring,
235                             u32 value)
236 {
237         drm_i915_private_t *dev_priv = ring->dev->dev_private;
238         I915_WRITE_TAIL(ring, value);
239 }
240
241 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
242 {
243         drm_i915_private_t *dev_priv = ring->dev->dev_private;
244         u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
245                         RING_ACTHD(ring->mmio_base) : ACTHD;
246
247         return I915_READ(acthd_reg);
248 }
249
250 static int init_ring_common(struct intel_ring_buffer *ring)
251 {
252         struct drm_device *dev = ring->dev;
253         drm_i915_private_t *dev_priv = dev->dev_private;
254         struct drm_i915_gem_object *obj = ring->obj;
255         int ret = 0;
256         u32 head;
257
258         if (HAS_FORCE_WAKE(dev))
259                 gen6_gt_force_wake_get(dev_priv);
260
261         /* Stop the ring if it's running. */
262         I915_WRITE_CTL(ring, 0);
263         I915_WRITE_HEAD(ring, 0);
264         ring->write_tail(ring, 0);
265
266         /* Initialize the ring. */
267         I915_WRITE_START(ring, obj->gtt_offset);
268         head = I915_READ_HEAD(ring) & HEAD_ADDR;
269
270         /* G45 ring initialization fails to reset head to zero */
271         if (head != 0) {
272                 DRM_DEBUG_KMS("%s head not reset to zero "
273                               "ctl %08x head %08x tail %08x start %08x\n",
274                               ring->name,
275                               I915_READ_CTL(ring),
276                               I915_READ_HEAD(ring),
277                               I915_READ_TAIL(ring),
278                               I915_READ_START(ring));
279
280                 I915_WRITE_HEAD(ring, 0);
281
282                 if (I915_READ_HEAD(ring) & HEAD_ADDR) {
283                         DRM_ERROR("failed to set %s head to zero "
284                                   "ctl %08x head %08x tail %08x start %08x\n",
285                                   ring->name,
286                                   I915_READ_CTL(ring),
287                                   I915_READ_HEAD(ring),
288                                   I915_READ_TAIL(ring),
289                                   I915_READ_START(ring));
290                 }
291         }
292
293         I915_WRITE_CTL(ring,
294                         ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
295                         | RING_VALID);
296
297         /* If the head is still not zero, the ring is dead */
298         if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
299                      I915_READ_START(ring) == obj->gtt_offset &&
300                      (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
301                 DRM_ERROR("%s initialization failed "
302                                 "ctl %08x head %08x tail %08x start %08x\n",
303                                 ring->name,
304                                 I915_READ_CTL(ring),
305                                 I915_READ_HEAD(ring),
306                                 I915_READ_TAIL(ring),
307                                 I915_READ_START(ring));
308                 ret = -EIO;
309                 goto out;
310         }
311
312         if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
313                 i915_kernel_lost_context(ring->dev);
314         else {
315                 ring->head = I915_READ_HEAD(ring);
316                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
317                 ring->space = ring_space(ring);
318                 ring->last_retired_head = -1;
319         }
320
321 out:
322         if (HAS_FORCE_WAKE(dev))
323                 gen6_gt_force_wake_put(dev_priv);
324
325         return ret;
326 }
327
328 static int
329 init_pipe_control(struct intel_ring_buffer *ring)
330 {
331         struct pipe_control *pc;
332         struct drm_i915_gem_object *obj;
333         int ret;
334
335         if (ring->private)
336                 return 0;
337
338         pc = kmalloc(sizeof(*pc), GFP_KERNEL);
339         if (!pc)
340                 return -ENOMEM;
341
342         obj = i915_gem_alloc_object(ring->dev, 4096);
343         if (obj == NULL) {
344                 DRM_ERROR("Failed to allocate seqno page\n");
345                 ret = -ENOMEM;
346                 goto err;
347         }
348
349         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
350
351         ret = i915_gem_object_pin(obj, 4096, true);
352         if (ret)
353                 goto err_unref;
354
355         pc->gtt_offset = obj->gtt_offset;
356         pc->cpu_page =  kmap(obj->pages[0]);
357         if (pc->cpu_page == NULL)
358                 goto err_unpin;
359
360         pc->obj = obj;
361         ring->private = pc;
362         return 0;
363
364 err_unpin:
365         i915_gem_object_unpin(obj);
366 err_unref:
367         drm_gem_object_unreference(&obj->base);
368 err:
369         kfree(pc);
370         return ret;
371 }
372
373 static void
374 cleanup_pipe_control(struct intel_ring_buffer *ring)
375 {
376         struct pipe_control *pc = ring->private;
377         struct drm_i915_gem_object *obj;
378
379         if (!ring->private)
380                 return;
381
382         obj = pc->obj;
383         kunmap(obj->pages[0]);
384         i915_gem_object_unpin(obj);
385         drm_gem_object_unreference(&obj->base);
386
387         kfree(pc);
388         ring->private = NULL;
389 }
390
391 static int init_render_ring(struct intel_ring_buffer *ring)
392 {
393         struct drm_device *dev = ring->dev;
394         struct drm_i915_private *dev_priv = dev->dev_private;
395         int ret = init_ring_common(ring);
396
397         if (INTEL_INFO(dev)->gen > 3) {
398                 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
399                 I915_WRITE(MI_MODE, mode);
400                 if (IS_GEN7(dev))
401                         I915_WRITE(GFX_MODE_GEN7,
402                                    GFX_MODE_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
403                                    GFX_MODE_ENABLE(GFX_REPLAY_MODE));
404         }
405
406         if (INTEL_INFO(dev)->gen >= 5) {
407                 ret = init_pipe_control(ring);
408                 if (ret)
409                         return ret;
410         }
411
412
413         if (IS_GEN6(dev)) {
414                 /* From the Sandybridge PRM, volume 1 part 3, page 24:
415                  * "If this bit is set, STCunit will have LRA as replacement
416                  *  policy. [...] This bit must be reset.  LRA replacement
417                  *  policy is not supported."
418                  */
419                 I915_WRITE(CACHE_MODE_0,
420                            CM0_STC_EVICT_DISABLE_LRA_SNB << CM0_MASK_SHIFT);
421         }
422
423         if (INTEL_INFO(dev)->gen >= 6) {
424                 I915_WRITE(INSTPM,
425                            INSTPM_FORCE_ORDERING << 16 | INSTPM_FORCE_ORDERING);
426         }
427
428         return ret;
429 }
430
431 static void render_ring_cleanup(struct intel_ring_buffer *ring)
432 {
433         if (!ring->private)
434                 return;
435
436         cleanup_pipe_control(ring);
437 }
438
439 static void
440 update_mboxes(struct intel_ring_buffer *ring,
441             u32 seqno,
442             u32 mmio_offset)
443 {
444         intel_ring_emit(ring, MI_SEMAPHORE_MBOX |
445                               MI_SEMAPHORE_GLOBAL_GTT |
446                               MI_SEMAPHORE_REGISTER |
447                               MI_SEMAPHORE_UPDATE);
448         intel_ring_emit(ring, seqno);
449         intel_ring_emit(ring, mmio_offset);
450 }
451
452 /**
453  * gen6_add_request - Update the semaphore mailbox registers
454  * 
455  * @ring - ring that is adding a request
456  * @seqno - return seqno stuck into the ring
457  *
458  * Update the mailbox registers in the *other* rings with the current seqno.
459  * This acts like a signal in the canonical semaphore.
460  */
461 static int
462 gen6_add_request(struct intel_ring_buffer *ring,
463                  u32 *seqno)
464 {
465         u32 mbox1_reg;
466         u32 mbox2_reg;
467         int ret;
468
469         ret = intel_ring_begin(ring, 10);
470         if (ret)
471                 return ret;
472
473         mbox1_reg = ring->signal_mbox[0];
474         mbox2_reg = ring->signal_mbox[1];
475
476         *seqno = i915_gem_next_request_seqno(ring);
477
478         update_mboxes(ring, *seqno, mbox1_reg);
479         update_mboxes(ring, *seqno, mbox2_reg);
480         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
481         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
482         intel_ring_emit(ring, *seqno);
483         intel_ring_emit(ring, MI_USER_INTERRUPT);
484         intel_ring_advance(ring);
485
486         return 0;
487 }
488
489 /**
490  * intel_ring_sync - sync the waiter to the signaller on seqno
491  *
492  * @waiter - ring that is waiting
493  * @signaller - ring which has, or will signal
494  * @seqno - seqno which the waiter will block on
495  */
496 static int
497 intel_ring_sync(struct intel_ring_buffer *waiter,
498                 struct intel_ring_buffer *signaller,
499                 int ring,
500                 u32 seqno)
501 {
502         int ret;
503         u32 dw1 = MI_SEMAPHORE_MBOX |
504                   MI_SEMAPHORE_COMPARE |
505                   MI_SEMAPHORE_REGISTER;
506
507         ret = intel_ring_begin(waiter, 4);
508         if (ret)
509                 return ret;
510
511         intel_ring_emit(waiter, dw1 | signaller->semaphore_register[ring]);
512         intel_ring_emit(waiter, seqno);
513         intel_ring_emit(waiter, 0);
514         intel_ring_emit(waiter, MI_NOOP);
515         intel_ring_advance(waiter);
516
517         return 0;
518 }
519
520 /* VCS->RCS (RVSYNC) or BCS->RCS (RBSYNC) */
521 int
522 render_ring_sync_to(struct intel_ring_buffer *waiter,
523                     struct intel_ring_buffer *signaller,
524                     u32 seqno)
525 {
526         WARN_ON(signaller->semaphore_register[RCS] == MI_SEMAPHORE_SYNC_INVALID);
527         return intel_ring_sync(waiter,
528                                signaller,
529                                RCS,
530                                seqno);
531 }
532
533 /* RCS->VCS (VRSYNC) or BCS->VCS (VBSYNC) */
534 int
535 gen6_bsd_ring_sync_to(struct intel_ring_buffer *waiter,
536                       struct intel_ring_buffer *signaller,
537                       u32 seqno)
538 {
539         WARN_ON(signaller->semaphore_register[VCS] == MI_SEMAPHORE_SYNC_INVALID);
540         return intel_ring_sync(waiter,
541                                signaller,
542                                VCS,
543                                seqno);
544 }
545
546 /* RCS->BCS (BRSYNC) or VCS->BCS (BVSYNC) */
547 int
548 gen6_blt_ring_sync_to(struct intel_ring_buffer *waiter,
549                       struct intel_ring_buffer *signaller,
550                       u32 seqno)
551 {
552         WARN_ON(signaller->semaphore_register[BCS] == MI_SEMAPHORE_SYNC_INVALID);
553         return intel_ring_sync(waiter,
554                                signaller,
555                                BCS,
556                                seqno);
557 }
558
559
560
561 #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
562 do {                                                                    \
563         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |                \
564                  PIPE_CONTROL_DEPTH_STALL);                             \
565         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
566         intel_ring_emit(ring__, 0);                                                     \
567         intel_ring_emit(ring__, 0);                                                     \
568 } while (0)
569
570 static int
571 pc_render_add_request(struct intel_ring_buffer *ring,
572                       u32 *result)
573 {
574         u32 seqno = i915_gem_next_request_seqno(ring);
575         struct pipe_control *pc = ring->private;
576         u32 scratch_addr = pc->gtt_offset + 128;
577         int ret;
578
579         /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
580          * incoherent with writes to memory, i.e. completely fubar,
581          * so we need to use PIPE_NOTIFY instead.
582          *
583          * However, we also need to workaround the qword write
584          * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
585          * memory before requesting an interrupt.
586          */
587         ret = intel_ring_begin(ring, 32);
588         if (ret)
589                 return ret;
590
591         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
592                         PIPE_CONTROL_WRITE_FLUSH |
593                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
594         intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
595         intel_ring_emit(ring, seqno);
596         intel_ring_emit(ring, 0);
597         PIPE_CONTROL_FLUSH(ring, scratch_addr);
598         scratch_addr += 128; /* write to separate cachelines */
599         PIPE_CONTROL_FLUSH(ring, scratch_addr);
600         scratch_addr += 128;
601         PIPE_CONTROL_FLUSH(ring, scratch_addr);
602         scratch_addr += 128;
603         PIPE_CONTROL_FLUSH(ring, scratch_addr);
604         scratch_addr += 128;
605         PIPE_CONTROL_FLUSH(ring, scratch_addr);
606         scratch_addr += 128;
607         PIPE_CONTROL_FLUSH(ring, scratch_addr);
608
609         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
610                         PIPE_CONTROL_WRITE_FLUSH |
611                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
612                         PIPE_CONTROL_NOTIFY);
613         intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
614         intel_ring_emit(ring, seqno);
615         intel_ring_emit(ring, 0);
616         intel_ring_advance(ring);
617
618         *result = seqno;
619         return 0;
620 }
621
622 static int
623 render_ring_add_request(struct intel_ring_buffer *ring,
624                         u32 *result)
625 {
626         u32 seqno = i915_gem_next_request_seqno(ring);
627         int ret;
628
629         ret = intel_ring_begin(ring, 4);
630         if (ret)
631                 return ret;
632
633         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
634         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
635         intel_ring_emit(ring, seqno);
636         intel_ring_emit(ring, MI_USER_INTERRUPT);
637         intel_ring_advance(ring);
638
639         *result = seqno;
640         return 0;
641 }
642
643 static u32
644 gen6_ring_get_seqno(struct intel_ring_buffer *ring)
645 {
646         struct drm_device *dev = ring->dev;
647
648         /* Workaround to force correct ordering between irq and seqno writes on
649          * ivb (and maybe also on snb) by reading from a CS register (like
650          * ACTHD) before reading the status page. */
651         if (IS_GEN6(dev) || IS_GEN7(dev))
652                 intel_ring_get_active_head(ring);
653         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
654 }
655
656 static u32
657 ring_get_seqno(struct intel_ring_buffer *ring)
658 {
659         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
660 }
661
662 static u32
663 pc_render_get_seqno(struct intel_ring_buffer *ring)
664 {
665         struct pipe_control *pc = ring->private;
666         return pc->cpu_page[0];
667 }
668
669 static void
670 ironlake_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
671 {
672         dev_priv->gt_irq_mask &= ~mask;
673         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
674         POSTING_READ(GTIMR);
675 }
676
677 static void
678 ironlake_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
679 {
680         dev_priv->gt_irq_mask |= mask;
681         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
682         POSTING_READ(GTIMR);
683 }
684
685 static void
686 i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
687 {
688         dev_priv->irq_mask &= ~mask;
689         I915_WRITE(IMR, dev_priv->irq_mask);
690         POSTING_READ(IMR);
691 }
692
693 static void
694 i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
695 {
696         dev_priv->irq_mask |= mask;
697         I915_WRITE(IMR, dev_priv->irq_mask);
698         POSTING_READ(IMR);
699 }
700
701 static bool
702 render_ring_get_irq(struct intel_ring_buffer *ring)
703 {
704         struct drm_device *dev = ring->dev;
705         drm_i915_private_t *dev_priv = dev->dev_private;
706
707         if (!dev->irq_enabled)
708                 return false;
709
710         spin_lock(&ring->irq_lock);
711         if (ring->irq_refcount++ == 0) {
712                 if (HAS_PCH_SPLIT(dev))
713                         ironlake_enable_irq(dev_priv,
714                                             GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
715                 else
716                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
717         }
718         spin_unlock(&ring->irq_lock);
719
720         return true;
721 }
722
723 static void
724 render_ring_put_irq(struct intel_ring_buffer *ring)
725 {
726         struct drm_device *dev = ring->dev;
727         drm_i915_private_t *dev_priv = dev->dev_private;
728
729         spin_lock(&ring->irq_lock);
730         if (--ring->irq_refcount == 0) {
731                 if (HAS_PCH_SPLIT(dev))
732                         ironlake_disable_irq(dev_priv,
733                                              GT_USER_INTERRUPT |
734                                              GT_PIPE_NOTIFY);
735                 else
736                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
737         }
738         spin_unlock(&ring->irq_lock);
739 }
740
741 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
742 {
743         struct drm_device *dev = ring->dev;
744         drm_i915_private_t *dev_priv = ring->dev->dev_private;
745         u32 mmio = 0;
746
747         /* The ring status page addresses are no longer next to the rest of
748          * the ring registers as of gen7.
749          */
750         if (IS_GEN7(dev)) {
751                 switch (ring->id) {
752                 case RCS:
753                         mmio = RENDER_HWS_PGA_GEN7;
754                         break;
755                 case BCS:
756                         mmio = BLT_HWS_PGA_GEN7;
757                         break;
758                 case VCS:
759                         mmio = BSD_HWS_PGA_GEN7;
760                         break;
761                 }
762         } else if (IS_GEN6(ring->dev)) {
763                 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
764         } else {
765                 mmio = RING_HWS_PGA(ring->mmio_base);
766         }
767
768         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
769         POSTING_READ(mmio);
770 }
771
772 static int
773 bsd_ring_flush(struct intel_ring_buffer *ring,
774                u32     invalidate_domains,
775                u32     flush_domains)
776 {
777         int ret;
778
779         ret = intel_ring_begin(ring, 2);
780         if (ret)
781                 return ret;
782
783         intel_ring_emit(ring, MI_FLUSH);
784         intel_ring_emit(ring, MI_NOOP);
785         intel_ring_advance(ring);
786         return 0;
787 }
788
789 static int
790 ring_add_request(struct intel_ring_buffer *ring,
791                  u32 *result)
792 {
793         u32 seqno;
794         int ret;
795
796         ret = intel_ring_begin(ring, 4);
797         if (ret)
798                 return ret;
799
800         seqno = i915_gem_next_request_seqno(ring);
801
802         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
803         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
804         intel_ring_emit(ring, seqno);
805         intel_ring_emit(ring, MI_USER_INTERRUPT);
806         intel_ring_advance(ring);
807
808         *result = seqno;
809         return 0;
810 }
811
812 static bool
813 gen6_ring_get_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
814 {
815         struct drm_device *dev = ring->dev;
816         drm_i915_private_t *dev_priv = dev->dev_private;
817
818         if (!dev->irq_enabled)
819                return false;
820
821         /* It looks like we need to prevent the gt from suspending while waiting
822          * for an notifiy irq, otherwise irqs seem to get lost on at least the
823          * blt/bsd rings on ivb. */
824         gen6_gt_force_wake_get(dev_priv);
825
826         spin_lock(&ring->irq_lock);
827         if (ring->irq_refcount++ == 0) {
828                 ring->irq_mask &= ~rflag;
829                 I915_WRITE_IMR(ring, ring->irq_mask);
830                 ironlake_enable_irq(dev_priv, gflag);
831         }
832         spin_unlock(&ring->irq_lock);
833
834         return true;
835 }
836
837 static void
838 gen6_ring_put_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
839 {
840         struct drm_device *dev = ring->dev;
841         drm_i915_private_t *dev_priv = dev->dev_private;
842
843         spin_lock(&ring->irq_lock);
844         if (--ring->irq_refcount == 0) {
845                 ring->irq_mask |= rflag;
846                 I915_WRITE_IMR(ring, ring->irq_mask);
847                 ironlake_disable_irq(dev_priv, gflag);
848         }
849         spin_unlock(&ring->irq_lock);
850
851         gen6_gt_force_wake_put(dev_priv);
852 }
853
854 static bool
855 bsd_ring_get_irq(struct intel_ring_buffer *ring)
856 {
857         struct drm_device *dev = ring->dev;
858         drm_i915_private_t *dev_priv = dev->dev_private;
859
860         if (!dev->irq_enabled)
861                 return false;
862
863         spin_lock(&ring->irq_lock);
864         if (ring->irq_refcount++ == 0) {
865                 if (IS_G4X(dev))
866                         i915_enable_irq(dev_priv, I915_BSD_USER_INTERRUPT);
867                 else
868                         ironlake_enable_irq(dev_priv, GT_BSD_USER_INTERRUPT);
869         }
870         spin_unlock(&ring->irq_lock);
871
872         return true;
873 }
874 static void
875 bsd_ring_put_irq(struct intel_ring_buffer *ring)
876 {
877         struct drm_device *dev = ring->dev;
878         drm_i915_private_t *dev_priv = dev->dev_private;
879
880         spin_lock(&ring->irq_lock);
881         if (--ring->irq_refcount == 0) {
882                 if (IS_G4X(dev))
883                         i915_disable_irq(dev_priv, I915_BSD_USER_INTERRUPT);
884                 else
885                         ironlake_disable_irq(dev_priv, GT_BSD_USER_INTERRUPT);
886         }
887         spin_unlock(&ring->irq_lock);
888 }
889
890 static int
891 ring_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
892 {
893         int ret;
894
895         ret = intel_ring_begin(ring, 2);
896         if (ret)
897                 return ret;
898
899         intel_ring_emit(ring,
900                         MI_BATCH_BUFFER_START | (2 << 6) |
901                         MI_BATCH_NON_SECURE_I965);
902         intel_ring_emit(ring, offset);
903         intel_ring_advance(ring);
904
905         return 0;
906 }
907
908 static int
909 render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
910                                 u32 offset, u32 len)
911 {
912         struct drm_device *dev = ring->dev;
913         int ret;
914
915         if (IS_I830(dev) || IS_845G(dev)) {
916                 ret = intel_ring_begin(ring, 4);
917                 if (ret)
918                         return ret;
919
920                 intel_ring_emit(ring, MI_BATCH_BUFFER);
921                 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
922                 intel_ring_emit(ring, offset + len - 8);
923                 intel_ring_emit(ring, 0);
924         } else {
925                 ret = intel_ring_begin(ring, 2);
926                 if (ret)
927                         return ret;
928
929                 if (INTEL_INFO(dev)->gen >= 4) {
930                         intel_ring_emit(ring,
931                                         MI_BATCH_BUFFER_START | (2 << 6) |
932                                         MI_BATCH_NON_SECURE_I965);
933                         intel_ring_emit(ring, offset);
934                 } else {
935                         intel_ring_emit(ring,
936                                         MI_BATCH_BUFFER_START | (2 << 6));
937                         intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
938                 }
939         }
940         intel_ring_advance(ring);
941
942         return 0;
943 }
944
945 static void cleanup_status_page(struct intel_ring_buffer *ring)
946 {
947         drm_i915_private_t *dev_priv = ring->dev->dev_private;
948         struct drm_i915_gem_object *obj;
949
950         obj = ring->status_page.obj;
951         if (obj == NULL)
952                 return;
953
954         kunmap(obj->pages[0]);
955         i915_gem_object_unpin(obj);
956         drm_gem_object_unreference(&obj->base);
957         ring->status_page.obj = NULL;
958
959         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
960 }
961
962 static int init_status_page(struct intel_ring_buffer *ring)
963 {
964         struct drm_device *dev = ring->dev;
965         drm_i915_private_t *dev_priv = dev->dev_private;
966         struct drm_i915_gem_object *obj;
967         int ret;
968
969         obj = i915_gem_alloc_object(dev, 4096);
970         if (obj == NULL) {
971                 DRM_ERROR("Failed to allocate status page\n");
972                 ret = -ENOMEM;
973                 goto err;
974         }
975
976         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
977
978         ret = i915_gem_object_pin(obj, 4096, true);
979         if (ret != 0) {
980                 goto err_unref;
981         }
982
983         ring->status_page.gfx_addr = obj->gtt_offset;
984         ring->status_page.page_addr = kmap(obj->pages[0]);
985         if (ring->status_page.page_addr == NULL) {
986                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
987                 goto err_unpin;
988         }
989         ring->status_page.obj = obj;
990         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
991
992         intel_ring_setup_status_page(ring);
993         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
994                         ring->name, ring->status_page.gfx_addr);
995
996         return 0;
997
998 err_unpin:
999         i915_gem_object_unpin(obj);
1000 err_unref:
1001         drm_gem_object_unreference(&obj->base);
1002 err:
1003         return ret;
1004 }
1005
1006 int intel_init_ring_buffer(struct drm_device *dev,
1007                            struct intel_ring_buffer *ring)
1008 {
1009         struct drm_i915_gem_object *obj;
1010         int ret;
1011
1012         ring->dev = dev;
1013         INIT_LIST_HEAD(&ring->active_list);
1014         INIT_LIST_HEAD(&ring->request_list);
1015         INIT_LIST_HEAD(&ring->gpu_write_list);
1016
1017         init_waitqueue_head(&ring->irq_queue);
1018         spin_lock_init(&ring->irq_lock);
1019         ring->irq_mask = ~0;
1020
1021         if (I915_NEED_GFX_HWS(dev)) {
1022                 ret = init_status_page(ring);
1023                 if (ret)
1024                         return ret;
1025         }
1026
1027         obj = i915_gem_alloc_object(dev, ring->size);
1028         if (obj == NULL) {
1029                 DRM_ERROR("Failed to allocate ringbuffer\n");
1030                 ret = -ENOMEM;
1031                 goto err_hws;
1032         }
1033
1034         ring->obj = obj;
1035
1036         ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
1037         if (ret)
1038                 goto err_unref;
1039
1040         ret = i915_gem_object_set_to_gtt_domain(obj, true);
1041         if (ret)
1042                 goto err_unpin;
1043
1044         ring->map.size = ring->size;
1045         ring->map.offset = dev->agp->base + obj->gtt_offset;
1046         ring->map.type = 0;
1047         ring->map.flags = 0;
1048         ring->map.mtrr = 0;
1049
1050         drm_core_ioremap_wc(&ring->map, dev);
1051         if (ring->map.handle == NULL) {
1052                 DRM_ERROR("Failed to map ringbuffer.\n");
1053                 ret = -EINVAL;
1054                 goto err_unpin;
1055         }
1056
1057         ring->virtual_start = ring->map.handle;
1058         ret = ring->init(ring);
1059         if (ret)
1060                 goto err_unmap;
1061
1062         /* Workaround an erratum on the i830 which causes a hang if
1063          * the TAIL pointer points to within the last 2 cachelines
1064          * of the buffer.
1065          */
1066         ring->effective_size = ring->size;
1067         if (IS_I830(ring->dev) || IS_845G(ring->dev))
1068                 ring->effective_size -= 128;
1069
1070         return 0;
1071
1072 err_unmap:
1073         drm_core_ioremapfree(&ring->map, dev);
1074 err_unpin:
1075         i915_gem_object_unpin(obj);
1076 err_unref:
1077         drm_gem_object_unreference(&obj->base);
1078         ring->obj = NULL;
1079 err_hws:
1080         cleanup_status_page(ring);
1081         return ret;
1082 }
1083
1084 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
1085 {
1086         struct drm_i915_private *dev_priv;
1087         int ret;
1088
1089         if (ring->obj == NULL)
1090                 return;
1091
1092         /* Disable the ring buffer. The ring must be idle at this point */
1093         dev_priv = ring->dev->dev_private;
1094         ret = intel_wait_ring_idle(ring);
1095         if (ret)
1096                 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
1097                           ring->name, ret);
1098
1099         I915_WRITE_CTL(ring, 0);
1100
1101         drm_core_ioremapfree(&ring->map, ring->dev);
1102
1103         i915_gem_object_unpin(ring->obj);
1104         drm_gem_object_unreference(&ring->obj->base);
1105         ring->obj = NULL;
1106
1107         if (ring->cleanup)
1108                 ring->cleanup(ring);
1109
1110         cleanup_status_page(ring);
1111 }
1112
1113 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
1114 {
1115         unsigned int *virt;
1116         int rem = ring->size - ring->tail;
1117
1118         if (ring->space < rem) {
1119                 int ret = intel_wait_ring_buffer(ring, rem);
1120                 if (ret)
1121                         return ret;
1122         }
1123
1124         virt = (unsigned int *)(ring->virtual_start + ring->tail);
1125         rem /= 8;
1126         while (rem--) {
1127                 *virt++ = MI_NOOP;
1128                 *virt++ = MI_NOOP;
1129         }
1130
1131         ring->tail = 0;
1132         ring->space = ring_space(ring);
1133
1134         return 0;
1135 }
1136
1137 static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
1138 {
1139         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1140         bool was_interruptible;
1141         int ret;
1142
1143         /* XXX As we have not yet audited all the paths to check that
1144          * they are ready for ERESTARTSYS from intel_ring_begin, do not
1145          * allow us to be interruptible by a signal.
1146          */
1147         was_interruptible = dev_priv->mm.interruptible;
1148         dev_priv->mm.interruptible = false;
1149
1150         ret = i915_wait_request(ring, seqno, true);
1151
1152         dev_priv->mm.interruptible = was_interruptible;
1153
1154         return ret;
1155 }
1156
1157 static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
1158 {
1159         struct drm_i915_gem_request *request;
1160         u32 seqno = 0;
1161         int ret;
1162
1163         i915_gem_retire_requests_ring(ring);
1164
1165         if (ring->last_retired_head != -1) {
1166                 ring->head = ring->last_retired_head;
1167                 ring->last_retired_head = -1;
1168                 ring->space = ring_space(ring);
1169                 if (ring->space >= n)
1170                         return 0;
1171         }
1172
1173         list_for_each_entry(request, &ring->request_list, list) {
1174                 int space;
1175
1176                 if (request->tail == -1)
1177                         continue;
1178
1179                 space = request->tail - (ring->tail + 8);
1180                 if (space < 0)
1181                         space += ring->size;
1182                 if (space >= n) {
1183                         seqno = request->seqno;
1184                         break;
1185                 }
1186
1187                 /* Consume this request in case we need more space than
1188                  * is available and so need to prevent a race between
1189                  * updating last_retired_head and direct reads of
1190                  * I915_RING_HEAD. It also provides a nice sanity check.
1191                  */
1192                 request->tail = -1;
1193         }
1194
1195         if (seqno == 0)
1196                 return -ENOSPC;
1197
1198         ret = intel_ring_wait_seqno(ring, seqno);
1199         if (ret)
1200                 return ret;
1201
1202         if (WARN_ON(ring->last_retired_head == -1))
1203                 return -ENOSPC;
1204
1205         ring->head = ring->last_retired_head;
1206         ring->last_retired_head = -1;
1207         ring->space = ring_space(ring);
1208         if (WARN_ON(ring->space < n))
1209                 return -ENOSPC;
1210
1211         return 0;
1212 }
1213
1214 int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
1215 {
1216         struct drm_device *dev = ring->dev;
1217         struct drm_i915_private *dev_priv = dev->dev_private;
1218         unsigned long end;
1219         int ret;
1220
1221         ret = intel_ring_wait_request(ring, n);
1222         if (ret != -ENOSPC)
1223                 return ret;
1224
1225         trace_i915_ring_wait_begin(ring);
1226         if (drm_core_check_feature(dev, DRIVER_GEM))
1227                 /* With GEM the hangcheck timer should kick us out of the loop,
1228                  * leaving it early runs the risk of corrupting GEM state (due
1229                  * to running on almost untested codepaths). But on resume
1230                  * timers don't work yet, so prevent a complete hang in that
1231                  * case by choosing an insanely large timeout. */
1232                 end = jiffies + 60 * HZ;
1233         else
1234                 end = jiffies + 3 * HZ;
1235
1236         do {
1237                 ring->head = I915_READ_HEAD(ring);
1238                 ring->space = ring_space(ring);
1239                 if (ring->space >= n) {
1240                         trace_i915_ring_wait_end(ring);
1241                         return 0;
1242                 }
1243
1244                 if (dev->primary->master) {
1245                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
1246                         if (master_priv->sarea_priv)
1247                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
1248                 }
1249
1250                 msleep(1);
1251                 if (atomic_read(&dev_priv->mm.wedged))
1252                         return -EAGAIN;
1253         } while (!time_after(jiffies, end));
1254         trace_i915_ring_wait_end(ring);
1255         return -EBUSY;
1256 }
1257
1258 int intel_ring_begin(struct intel_ring_buffer *ring,
1259                      int num_dwords)
1260 {
1261         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1262         int n = 4*num_dwords;
1263         int ret;
1264
1265         if (unlikely(atomic_read(&dev_priv->mm.wedged)))
1266                 return -EIO;
1267
1268         if (unlikely(ring->tail + n > ring->effective_size)) {
1269                 ret = intel_wrap_ring_buffer(ring);
1270                 if (unlikely(ret))
1271                         return ret;
1272         }
1273
1274         if (unlikely(ring->space < n)) {
1275                 ret = intel_wait_ring_buffer(ring, n);
1276                 if (unlikely(ret))
1277                         return ret;
1278         }
1279
1280         ring->space -= n;
1281         return 0;
1282 }
1283
1284 void intel_ring_advance(struct intel_ring_buffer *ring)
1285 {
1286         ring->tail &= ring->size - 1;
1287         ring->write_tail(ring, ring->tail);
1288 }
1289
1290 static const struct intel_ring_buffer render_ring = {
1291         .name                   = "render ring",
1292         .id                     = RCS,
1293         .mmio_base              = RENDER_RING_BASE,
1294         .size                   = 32 * PAGE_SIZE,
1295         .init                   = init_render_ring,
1296         .write_tail             = ring_write_tail,
1297         .flush                  = render_ring_flush,
1298         .add_request            = render_ring_add_request,
1299         .get_seqno              = ring_get_seqno,
1300         .irq_get                = render_ring_get_irq,
1301         .irq_put                = render_ring_put_irq,
1302         .dispatch_execbuffer    = render_ring_dispatch_execbuffer,
1303         .cleanup                = render_ring_cleanup,
1304         .sync_to                = render_ring_sync_to,
1305         .semaphore_register     = {MI_SEMAPHORE_SYNC_INVALID,
1306                                    MI_SEMAPHORE_SYNC_RV,
1307                                    MI_SEMAPHORE_SYNC_RB},
1308         .signal_mbox            = {GEN6_VRSYNC, GEN6_BRSYNC},
1309 };
1310
1311 /* ring buffer for bit-stream decoder */
1312
1313 static const struct intel_ring_buffer bsd_ring = {
1314         .name                   = "bsd ring",
1315         .id                     = VCS,
1316         .mmio_base              = BSD_RING_BASE,
1317         .size                   = 32 * PAGE_SIZE,
1318         .init                   = init_ring_common,
1319         .write_tail             = ring_write_tail,
1320         .flush                  = bsd_ring_flush,
1321         .add_request            = ring_add_request,
1322         .get_seqno              = ring_get_seqno,
1323         .irq_get                = bsd_ring_get_irq,
1324         .irq_put                = bsd_ring_put_irq,
1325         .dispatch_execbuffer    = ring_dispatch_execbuffer,
1326 };
1327
1328
1329 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
1330                                      u32 value)
1331 {
1332         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1333
1334        /* Every tail move must follow the sequence below */
1335         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1336                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1337                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
1338         I915_WRITE(GEN6_BSD_RNCID, 0x0);
1339
1340         if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
1341                 GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
1342                 50))
1343         DRM_ERROR("timed out waiting for IDLE Indicator\n");
1344
1345         I915_WRITE_TAIL(ring, value);
1346         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1347                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1348                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
1349 }
1350
1351 static int gen6_ring_flush(struct intel_ring_buffer *ring,
1352                            u32 invalidate, u32 flush)
1353 {
1354         uint32_t cmd;
1355         int ret;
1356
1357         ret = intel_ring_begin(ring, 4);
1358         if (ret)
1359                 return ret;
1360
1361         cmd = MI_FLUSH_DW;
1362         if (invalidate & I915_GEM_GPU_DOMAINS)
1363                 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
1364         intel_ring_emit(ring, cmd);
1365         intel_ring_emit(ring, 0);
1366         intel_ring_emit(ring, 0);
1367         intel_ring_emit(ring, MI_NOOP);
1368         intel_ring_advance(ring);
1369         return 0;
1370 }
1371
1372 static int
1373 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1374                               u32 offset, u32 len)
1375 {
1376         int ret;
1377
1378         ret = intel_ring_begin(ring, 2);
1379         if (ret)
1380                 return ret;
1381
1382         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
1383         /* bit0-7 is the length on GEN6+ */
1384         intel_ring_emit(ring, offset);
1385         intel_ring_advance(ring);
1386
1387         return 0;
1388 }
1389
1390 static bool
1391 gen6_render_ring_get_irq(struct intel_ring_buffer *ring)
1392 {
1393         return gen6_ring_get_irq(ring,
1394                                  GT_USER_INTERRUPT,
1395                                  GEN6_RENDER_USER_INTERRUPT);
1396 }
1397
1398 static void
1399 gen6_render_ring_put_irq(struct intel_ring_buffer *ring)
1400 {
1401         return gen6_ring_put_irq(ring,
1402                                  GT_USER_INTERRUPT,
1403                                  GEN6_RENDER_USER_INTERRUPT);
1404 }
1405
1406 static bool
1407 gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring)
1408 {
1409         return gen6_ring_get_irq(ring,
1410                                  GT_GEN6_BSD_USER_INTERRUPT,
1411                                  GEN6_BSD_USER_INTERRUPT);
1412 }
1413
1414 static void
1415 gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring)
1416 {
1417         return gen6_ring_put_irq(ring,
1418                                  GT_GEN6_BSD_USER_INTERRUPT,
1419                                  GEN6_BSD_USER_INTERRUPT);
1420 }
1421
1422 /* ring buffer for Video Codec for Gen6+ */
1423 static const struct intel_ring_buffer gen6_bsd_ring = {
1424         .name                   = "gen6 bsd ring",
1425         .id                     = VCS,
1426         .mmio_base              = GEN6_BSD_RING_BASE,
1427         .size                   = 32 * PAGE_SIZE,
1428         .init                   = init_ring_common,
1429         .write_tail             = gen6_bsd_ring_write_tail,
1430         .flush                  = gen6_ring_flush,
1431         .add_request            = gen6_add_request,
1432         .get_seqno              = gen6_ring_get_seqno,
1433         .irq_get                = gen6_bsd_ring_get_irq,
1434         .irq_put                = gen6_bsd_ring_put_irq,
1435         .dispatch_execbuffer    = gen6_ring_dispatch_execbuffer,
1436         .sync_to                = gen6_bsd_ring_sync_to,
1437         .semaphore_register     = {MI_SEMAPHORE_SYNC_VR,
1438                                    MI_SEMAPHORE_SYNC_INVALID,
1439                                    MI_SEMAPHORE_SYNC_VB},
1440         .signal_mbox            = {GEN6_RVSYNC, GEN6_BVSYNC},
1441 };
1442
1443 /* Blitter support (SandyBridge+) */
1444
1445 static bool
1446 blt_ring_get_irq(struct intel_ring_buffer *ring)
1447 {
1448         return gen6_ring_get_irq(ring,
1449                                  GT_BLT_USER_INTERRUPT,
1450                                  GEN6_BLITTER_USER_INTERRUPT);
1451 }
1452
1453 static void
1454 blt_ring_put_irq(struct intel_ring_buffer *ring)
1455 {
1456         gen6_ring_put_irq(ring,
1457                           GT_BLT_USER_INTERRUPT,
1458                           GEN6_BLITTER_USER_INTERRUPT);
1459 }
1460
1461 static int blt_ring_flush(struct intel_ring_buffer *ring,
1462                           u32 invalidate, u32 flush)
1463 {
1464         uint32_t cmd;
1465         int ret;
1466
1467         ret = intel_ring_begin(ring, 4);
1468         if (ret)
1469                 return ret;
1470
1471         cmd = MI_FLUSH_DW;
1472         if (invalidate & I915_GEM_DOMAIN_RENDER)
1473                 cmd |= MI_INVALIDATE_TLB;
1474         intel_ring_emit(ring, cmd);
1475         intel_ring_emit(ring, 0);
1476         intel_ring_emit(ring, 0);
1477         intel_ring_emit(ring, MI_NOOP);
1478         intel_ring_advance(ring);
1479         return 0;
1480 }
1481
1482 static const struct intel_ring_buffer gen6_blt_ring = {
1483         .name                   = "blt ring",
1484         .id                     = BCS,
1485         .mmio_base              = BLT_RING_BASE,
1486         .size                   = 32 * PAGE_SIZE,
1487         .init                   = init_ring_common,
1488         .write_tail             = ring_write_tail,
1489         .flush                  = blt_ring_flush,
1490         .add_request            = gen6_add_request,
1491         .get_seqno              = gen6_ring_get_seqno,
1492         .irq_get                = blt_ring_get_irq,
1493         .irq_put                = blt_ring_put_irq,
1494         .dispatch_execbuffer    = gen6_ring_dispatch_execbuffer,
1495         .sync_to                = gen6_blt_ring_sync_to,
1496         .semaphore_register     = {MI_SEMAPHORE_SYNC_BR,
1497                                    MI_SEMAPHORE_SYNC_BV,
1498                                    MI_SEMAPHORE_SYNC_INVALID},
1499         .signal_mbox            = {GEN6_RBSYNC, GEN6_VBSYNC},
1500 };
1501
1502 int intel_init_render_ring_buffer(struct drm_device *dev)
1503 {
1504         drm_i915_private_t *dev_priv = dev->dev_private;
1505         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1506
1507         *ring = render_ring;
1508         if (INTEL_INFO(dev)->gen >= 6) {
1509                 ring->add_request = gen6_add_request;
1510                 ring->flush = gen6_render_ring_flush;
1511                 ring->irq_get = gen6_render_ring_get_irq;
1512                 ring->irq_put = gen6_render_ring_put_irq;
1513                 ring->get_seqno = gen6_ring_get_seqno;
1514         } else if (IS_GEN5(dev)) {
1515                 ring->add_request = pc_render_add_request;
1516                 ring->get_seqno = pc_render_get_seqno;
1517         }
1518
1519         if (!I915_NEED_GFX_HWS(dev)) {
1520                 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1521                 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1522         }
1523
1524         return intel_init_ring_buffer(dev, ring);
1525 }
1526
1527 int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
1528 {
1529         drm_i915_private_t *dev_priv = dev->dev_private;
1530         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1531
1532         *ring = render_ring;
1533         if (INTEL_INFO(dev)->gen >= 6) {
1534                 ring->add_request = gen6_add_request;
1535                 ring->irq_get = gen6_render_ring_get_irq;
1536                 ring->irq_put = gen6_render_ring_put_irq;
1537         } else if (IS_GEN5(dev)) {
1538                 ring->add_request = pc_render_add_request;
1539                 ring->get_seqno = pc_render_get_seqno;
1540         }
1541
1542         if (!I915_NEED_GFX_HWS(dev))
1543                 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1544
1545         ring->dev = dev;
1546         INIT_LIST_HEAD(&ring->active_list);
1547         INIT_LIST_HEAD(&ring->request_list);
1548         INIT_LIST_HEAD(&ring->gpu_write_list);
1549
1550         ring->size = size;
1551         ring->effective_size = ring->size;
1552         if (IS_I830(ring->dev))
1553                 ring->effective_size -= 128;
1554
1555         ring->map.offset = start;
1556         ring->map.size = size;
1557         ring->map.type = 0;
1558         ring->map.flags = 0;
1559         ring->map.mtrr = 0;
1560
1561         drm_core_ioremap_wc(&ring->map, dev);
1562         if (ring->map.handle == NULL) {
1563                 DRM_ERROR("can not ioremap virtual address for"
1564                           " ring buffer\n");
1565                 return -ENOMEM;
1566         }
1567
1568         ring->virtual_start = (void __force __iomem *)ring->map.handle;
1569         return 0;
1570 }
1571
1572 int intel_init_bsd_ring_buffer(struct drm_device *dev)
1573 {
1574         drm_i915_private_t *dev_priv = dev->dev_private;
1575         struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
1576
1577         if (IS_GEN6(dev) || IS_GEN7(dev))
1578                 *ring = gen6_bsd_ring;
1579         else
1580                 *ring = bsd_ring;
1581
1582         return intel_init_ring_buffer(dev, ring);
1583 }
1584
1585 int intel_init_blt_ring_buffer(struct drm_device *dev)
1586 {
1587         drm_i915_private_t *dev_priv = dev->dev_private;
1588         struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
1589
1590         *ring = gen6_blt_ring;
1591
1592         return intel_init_ring_buffer(dev, ring);
1593 }