drm/i915: Use drm_i915_gem_object as the preferred type
[linux-2.6.git] / drivers / gpu / drm / i915 / intel_ringbuffer.c
1 /*
2  * Copyright © 2008-2010 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Eric Anholt <eric@anholt.net>
25  *    Zou Nan hai <nanhai.zou@intel.com>
26  *    Xiang Hai hao<haihao.xiang@intel.com>
27  *
28  */
29
30 #include "drmP.h"
31 #include "drm.h"
32 #include "i915_drv.h"
33 #include "i915_drm.h"
34 #include "i915_trace.h"
35 #include "intel_drv.h"
36
37 static u32 i915_gem_get_seqno(struct drm_device *dev)
38 {
39         drm_i915_private_t *dev_priv = dev->dev_private;
40         u32 seqno;
41
42         seqno = dev_priv->next_seqno;
43
44         /* reserve 0 for non-seqno */
45         if (++dev_priv->next_seqno == 0)
46                 dev_priv->next_seqno = 1;
47
48         return seqno;
49 }
50
51 static void
52 render_ring_flush(struct intel_ring_buffer *ring,
53                   u32   invalidate_domains,
54                   u32   flush_domains)
55 {
56         struct drm_device *dev = ring->dev;
57         drm_i915_private_t *dev_priv = dev->dev_private;
58         u32 cmd;
59
60 #if WATCH_EXEC
61         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
62                   invalidate_domains, flush_domains);
63 #endif
64
65         trace_i915_gem_request_flush(dev, dev_priv->next_seqno,
66                                      invalidate_domains, flush_domains);
67
68         if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
69                 /*
70                  * read/write caches:
71                  *
72                  * I915_GEM_DOMAIN_RENDER is always invalidated, but is
73                  * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
74                  * also flushed at 2d versus 3d pipeline switches.
75                  *
76                  * read-only caches:
77                  *
78                  * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
79                  * MI_READ_FLUSH is set, and is always flushed on 965.
80                  *
81                  * I915_GEM_DOMAIN_COMMAND may not exist?
82                  *
83                  * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
84                  * invalidated when MI_EXE_FLUSH is set.
85                  *
86                  * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
87                  * invalidated with every MI_FLUSH.
88                  *
89                  * TLBs:
90                  *
91                  * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
92                  * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
93                  * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
94                  * are flushed at any MI_FLUSH.
95                  */
96
97                 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
98                 if ((invalidate_domains|flush_domains) &
99                     I915_GEM_DOMAIN_RENDER)
100                         cmd &= ~MI_NO_WRITE_FLUSH;
101                 if (INTEL_INFO(dev)->gen < 4) {
102                         /*
103                          * On the 965, the sampler cache always gets flushed
104                          * and this bit is reserved.
105                          */
106                         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
107                                 cmd |= MI_READ_FLUSH;
108                 }
109                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
110                         cmd |= MI_EXE_FLUSH;
111
112 #if WATCH_EXEC
113                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
114 #endif
115                 if (intel_ring_begin(ring, 2) == 0) {
116                         intel_ring_emit(ring, cmd);
117                         intel_ring_emit(ring, MI_NOOP);
118                         intel_ring_advance(ring);
119                 }
120         }
121 }
122
123 static void ring_write_tail(struct intel_ring_buffer *ring,
124                             u32 value)
125 {
126         drm_i915_private_t *dev_priv = ring->dev->dev_private;
127         I915_WRITE_TAIL(ring, value);
128 }
129
130 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
131 {
132         drm_i915_private_t *dev_priv = ring->dev->dev_private;
133         u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
134                         RING_ACTHD(ring->mmio_base) : ACTHD;
135
136         return I915_READ(acthd_reg);
137 }
138
139 static int init_ring_common(struct intel_ring_buffer *ring)
140 {
141         drm_i915_private_t *dev_priv = ring->dev->dev_private;
142         struct drm_i915_gem_object *obj = ring->obj;
143         u32 head;
144
145         /* Stop the ring if it's running. */
146         I915_WRITE_CTL(ring, 0);
147         I915_WRITE_HEAD(ring, 0);
148         ring->write_tail(ring, 0);
149
150         /* Initialize the ring. */
151         I915_WRITE_START(ring, obj->gtt_offset);
152         head = I915_READ_HEAD(ring) & HEAD_ADDR;
153
154         /* G45 ring initialization fails to reset head to zero */
155         if (head != 0) {
156                 DRM_ERROR("%s head not reset to zero "
157                                 "ctl %08x head %08x tail %08x start %08x\n",
158                                 ring->name,
159                                 I915_READ_CTL(ring),
160                                 I915_READ_HEAD(ring),
161                                 I915_READ_TAIL(ring),
162                                 I915_READ_START(ring));
163
164                 I915_WRITE_HEAD(ring, 0);
165
166                 DRM_ERROR("%s head forced to zero "
167                                 "ctl %08x head %08x tail %08x start %08x\n",
168                                 ring->name,
169                                 I915_READ_CTL(ring),
170                                 I915_READ_HEAD(ring),
171                                 I915_READ_TAIL(ring),
172                                 I915_READ_START(ring));
173         }
174
175         I915_WRITE_CTL(ring,
176                         ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
177                         | RING_REPORT_64K | RING_VALID);
178
179         /* If the head is still not zero, the ring is dead */
180         if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
181             I915_READ_START(ring) != obj->gtt_offset ||
182             (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
183                 DRM_ERROR("%s initialization failed "
184                                 "ctl %08x head %08x tail %08x start %08x\n",
185                                 ring->name,
186                                 I915_READ_CTL(ring),
187                                 I915_READ_HEAD(ring),
188                                 I915_READ_TAIL(ring),
189                                 I915_READ_START(ring));
190                 return -EIO;
191         }
192
193         if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
194                 i915_kernel_lost_context(ring->dev);
195         else {
196                 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
197                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
198                 ring->space = ring->head - (ring->tail + 8);
199                 if (ring->space < 0)
200                         ring->space += ring->size;
201         }
202         return 0;
203 }
204
205 static int init_render_ring(struct intel_ring_buffer *ring)
206 {
207         struct drm_device *dev = ring->dev;
208         int ret = init_ring_common(ring);
209
210         if (INTEL_INFO(dev)->gen > 3) {
211                 drm_i915_private_t *dev_priv = dev->dev_private;
212                 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
213                 if (IS_GEN6(dev))
214                         mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
215                 I915_WRITE(MI_MODE, mode);
216         }
217
218         return ret;
219 }
220
221 #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
222 do {                                                                    \
223         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |           \
224                  PIPE_CONTROL_DEPTH_STALL | 2);                         \
225         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
226         intel_ring_emit(ring__, 0);                                                     \
227         intel_ring_emit(ring__, 0);                                                     \
228 } while (0)
229
230 /**
231  * Creates a new sequence number, emitting a write of it to the status page
232  * plus an interrupt, which will trigger i915_user_interrupt_handler.
233  *
234  * Must be called with struct_lock held.
235  *
236  * Returned sequence numbers are nonzero on success.
237  */
238 static int
239 render_ring_add_request(struct intel_ring_buffer *ring,
240                         u32 *result)
241 {
242         struct drm_device *dev = ring->dev;
243         drm_i915_private_t *dev_priv = dev->dev_private;
244         u32 seqno = i915_gem_get_seqno(dev);
245         int ret;
246
247         if (IS_GEN6(dev)) {
248                 ret = intel_ring_begin(ring, 6);
249                 if (ret)
250                     return ret;
251
252                 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | 3);
253                 intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE |
254                                 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH |
255                                 PIPE_CONTROL_NOTIFY);
256                 intel_ring_emit(ring, dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
257                 intel_ring_emit(ring, seqno);
258                 intel_ring_emit(ring, 0);
259                 intel_ring_emit(ring, 0);
260         } else if (HAS_PIPE_CONTROL(dev)) {
261                 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
262
263                 /*
264                  * Workaround qword write incoherence by flushing the
265                  * PIPE_NOTIFY buffers out to memory before requesting
266                  * an interrupt.
267                  */
268                 ret = intel_ring_begin(ring, 32);
269                 if (ret)
270                         return ret;
271
272                 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
273                                 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
274                 intel_ring_emit(ring, dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
275                 intel_ring_emit(ring, seqno);
276                 intel_ring_emit(ring, 0);
277                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
278                 scratch_addr += 128; /* write to separate cachelines */
279                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
280                 scratch_addr += 128;
281                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
282                 scratch_addr += 128;
283                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
284                 scratch_addr += 128;
285                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
286                 scratch_addr += 128;
287                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
288                 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
289                                 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
290                                 PIPE_CONTROL_NOTIFY);
291                 intel_ring_emit(ring, dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
292                 intel_ring_emit(ring, seqno);
293                 intel_ring_emit(ring, 0);
294         } else {
295                 ret = intel_ring_begin(ring, 4);
296                 if (ret)
297                     return ret;
298
299                 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
300                 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
301                 intel_ring_emit(ring, seqno);
302
303                 intel_ring_emit(ring, MI_USER_INTERRUPT);
304         }
305
306         intel_ring_advance(ring);
307         *result = seqno;
308         return 0;
309 }
310
311 static u32
312 render_ring_get_seqno(struct intel_ring_buffer *ring)
313 {
314         struct drm_device *dev = ring->dev;
315         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
316         if (HAS_PIPE_CONTROL(dev))
317                 return ((volatile u32 *)(dev_priv->seqno_page))[0];
318         else
319                 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
320 }
321
322 static void
323 render_ring_get_user_irq(struct intel_ring_buffer *ring)
324 {
325         struct drm_device *dev = ring->dev;
326         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
327         unsigned long irqflags;
328
329         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
330         if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) {
331                 if (HAS_PCH_SPLIT(dev))
332                         ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
333                 else
334                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
335         }
336         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
337 }
338
339 static void
340 render_ring_put_user_irq(struct intel_ring_buffer *ring)
341 {
342         struct drm_device *dev = ring->dev;
343         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
344         unsigned long irqflags;
345
346         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
347         BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0);
348         if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
349                 if (HAS_PCH_SPLIT(dev))
350                         ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
351                 else
352                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
353         }
354         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
355 }
356
357 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
358 {
359         drm_i915_private_t *dev_priv = ring->dev->dev_private;
360         u32 mmio = IS_GEN6(ring->dev) ?
361                 RING_HWS_PGA_GEN6(ring->mmio_base) :
362                 RING_HWS_PGA(ring->mmio_base);
363         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
364         POSTING_READ(mmio);
365 }
366
367 static void
368 bsd_ring_flush(struct intel_ring_buffer *ring,
369                u32     invalidate_domains,
370                u32     flush_domains)
371 {
372         if (intel_ring_begin(ring, 2) == 0) {
373                 intel_ring_emit(ring, MI_FLUSH);
374                 intel_ring_emit(ring, MI_NOOP);
375                 intel_ring_advance(ring);
376         }
377 }
378
379 static int
380 ring_add_request(struct intel_ring_buffer *ring,
381                  u32 *result)
382 {
383         u32 seqno;
384         int ret;
385
386         ret = intel_ring_begin(ring, 4);
387         if (ret)
388                 return ret;
389
390         seqno = i915_gem_get_seqno(ring->dev);
391
392         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
393         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
394         intel_ring_emit(ring, seqno);
395         intel_ring_emit(ring, MI_USER_INTERRUPT);
396         intel_ring_advance(ring);
397
398         DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
399         *result = seqno;
400         return 0;
401 }
402
403 static void
404 bsd_ring_get_user_irq(struct intel_ring_buffer *ring)
405 {
406         /* do nothing */
407 }
408 static void
409 bsd_ring_put_user_irq(struct intel_ring_buffer *ring)
410 {
411         /* do nothing */
412 }
413
414 static u32
415 ring_status_page_get_seqno(struct intel_ring_buffer *ring)
416 {
417         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
418 }
419
420 static int
421 ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
422                          struct drm_i915_gem_execbuffer2 *exec,
423                          struct drm_clip_rect *cliprects,
424                          uint64_t exec_offset)
425 {
426         uint32_t exec_start;
427         int ret;
428
429         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
430
431         ret = intel_ring_begin(ring, 2);
432         if (ret)
433                 return ret;
434
435         intel_ring_emit(ring,
436                         MI_BATCH_BUFFER_START |
437                         (2 << 6) |
438                         MI_BATCH_NON_SECURE_I965);
439         intel_ring_emit(ring, exec_start);
440         intel_ring_advance(ring);
441
442         return 0;
443 }
444
445 static int
446 render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
447                                 struct drm_i915_gem_execbuffer2 *exec,
448                                 struct drm_clip_rect *cliprects,
449                                 uint64_t exec_offset)
450 {
451         struct drm_device *dev = ring->dev;
452         drm_i915_private_t *dev_priv = dev->dev_private;
453         int nbox = exec->num_cliprects;
454         uint32_t exec_start, exec_len;
455         int i, count, ret;
456
457         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
458         exec_len = (uint32_t) exec->batch_len;
459
460         trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
461
462         count = nbox ? nbox : 1;
463         for (i = 0; i < count; i++) {
464                 if (i < nbox) {
465                         ret = i915_emit_box(dev, cliprects, i,
466                                             exec->DR1, exec->DR4);
467                         if (ret)
468                                 return ret;
469                 }
470
471                 if (IS_I830(dev) || IS_845G(dev)) {
472                         ret = intel_ring_begin(ring, 4);
473                         if (ret)
474                                 return ret;
475
476                         intel_ring_emit(ring, MI_BATCH_BUFFER);
477                         intel_ring_emit(ring, exec_start | MI_BATCH_NON_SECURE);
478                         intel_ring_emit(ring, exec_start + exec_len - 4);
479                         intel_ring_emit(ring, 0);
480                 } else {
481                         ret = intel_ring_begin(ring, 2);
482                         if (ret)
483                                 return ret;
484
485                         if (INTEL_INFO(dev)->gen >= 4) {
486                                 intel_ring_emit(ring,
487                                                 MI_BATCH_BUFFER_START | (2 << 6)
488                                                 | MI_BATCH_NON_SECURE_I965);
489                                 intel_ring_emit(ring, exec_start);
490                         } else {
491                                 intel_ring_emit(ring, MI_BATCH_BUFFER_START
492                                                 | (2 << 6));
493                                 intel_ring_emit(ring, exec_start |
494                                                 MI_BATCH_NON_SECURE);
495                         }
496                 }
497                 intel_ring_advance(ring);
498         }
499
500         if (IS_G4X(dev) || IS_GEN5(dev)) {
501                 if (intel_ring_begin(ring, 2) == 0) {
502                         intel_ring_emit(ring, MI_FLUSH |
503                                         MI_NO_WRITE_FLUSH |
504                                         MI_INVALIDATE_ISP );
505                         intel_ring_emit(ring, MI_NOOP);
506                         intel_ring_advance(ring);
507                 }
508         }
509         /* XXX breadcrumb */
510
511         return 0;
512 }
513
514 static void cleanup_status_page(struct intel_ring_buffer *ring)
515 {
516         drm_i915_private_t *dev_priv = ring->dev->dev_private;
517         struct drm_i915_gem_object *obj;
518
519         obj = ring->status_page.obj;
520         if (obj == NULL)
521                 return;
522
523         kunmap(obj->pages[0]);
524         i915_gem_object_unpin(obj);
525         drm_gem_object_unreference(&obj->base);
526         ring->status_page.obj = NULL;
527
528         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
529 }
530
531 static int init_status_page(struct intel_ring_buffer *ring)
532 {
533         struct drm_device *dev = ring->dev;
534         drm_i915_private_t *dev_priv = dev->dev_private;
535         struct drm_i915_gem_object *obj;
536         int ret;
537
538         obj = i915_gem_alloc_object(dev, 4096);
539         if (obj == NULL) {
540                 DRM_ERROR("Failed to allocate status page\n");
541                 ret = -ENOMEM;
542                 goto err;
543         }
544         obj->agp_type = AGP_USER_CACHED_MEMORY;
545
546         ret = i915_gem_object_pin(obj, 4096, true);
547         if (ret != 0) {
548                 goto err_unref;
549         }
550
551         ring->status_page.gfx_addr = obj->gtt_offset;
552         ring->status_page.page_addr = kmap(obj->pages[0]);
553         if (ring->status_page.page_addr == NULL) {
554                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
555                 goto err_unpin;
556         }
557         ring->status_page.obj = obj;
558         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
559
560         intel_ring_setup_status_page(ring);
561         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
562                         ring->name, ring->status_page.gfx_addr);
563
564         return 0;
565
566 err_unpin:
567         i915_gem_object_unpin(obj);
568 err_unref:
569         drm_gem_object_unreference(&obj->base);
570 err:
571         return ret;
572 }
573
574 int intel_init_ring_buffer(struct drm_device *dev,
575                            struct intel_ring_buffer *ring)
576 {
577         struct drm_i915_gem_object *obj;
578         int ret;
579
580         ring->dev = dev;
581         INIT_LIST_HEAD(&ring->active_list);
582         INIT_LIST_HEAD(&ring->request_list);
583         INIT_LIST_HEAD(&ring->gpu_write_list);
584
585         if (I915_NEED_GFX_HWS(dev)) {
586                 ret = init_status_page(ring);
587                 if (ret)
588                         return ret;
589         }
590
591         obj = i915_gem_alloc_object(dev, ring->size);
592         if (obj == NULL) {
593                 DRM_ERROR("Failed to allocate ringbuffer\n");
594                 ret = -ENOMEM;
595                 goto err_hws;
596         }
597
598         ring->obj = obj;
599
600         ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
601         if (ret)
602                 goto err_unref;
603
604         ring->map.size = ring->size;
605         ring->map.offset = dev->agp->base + obj->gtt_offset;
606         ring->map.type = 0;
607         ring->map.flags = 0;
608         ring->map.mtrr = 0;
609
610         drm_core_ioremap_wc(&ring->map, dev);
611         if (ring->map.handle == NULL) {
612                 DRM_ERROR("Failed to map ringbuffer.\n");
613                 ret = -EINVAL;
614                 goto err_unpin;
615         }
616
617         ring->virtual_start = ring->map.handle;
618         ret = ring->init(ring);
619         if (ret)
620                 goto err_unmap;
621
622         return 0;
623
624 err_unmap:
625         drm_core_ioremapfree(&ring->map, dev);
626 err_unpin:
627         i915_gem_object_unpin(obj);
628 err_unref:
629         drm_gem_object_unreference(&obj->base);
630         ring->obj = NULL;
631 err_hws:
632         cleanup_status_page(ring);
633         return ret;
634 }
635
636 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
637 {
638         struct drm_i915_private *dev_priv;
639         int ret;
640
641         if (ring->obj == NULL)
642                 return;
643
644         /* Disable the ring buffer. The ring must be idle at this point */
645         dev_priv = ring->dev->dev_private;
646         ret = intel_wait_ring_buffer(ring, ring->size - 8);
647         I915_WRITE_CTL(ring, 0);
648
649         drm_core_ioremapfree(&ring->map, ring->dev);
650
651         i915_gem_object_unpin(ring->obj);
652         drm_gem_object_unreference(&ring->obj->base);
653         ring->obj = NULL;
654
655         if (ring->cleanup)
656                 ring->cleanup(ring);
657
658         cleanup_status_page(ring);
659 }
660
661 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
662 {
663         unsigned int *virt;
664         int rem;
665         rem = ring->size - ring->tail;
666
667         if (ring->space < rem) {
668                 int ret = intel_wait_ring_buffer(ring, rem);
669                 if (ret)
670                         return ret;
671         }
672
673         virt = (unsigned int *)(ring->virtual_start + ring->tail);
674         rem /= 8;
675         while (rem--) {
676                 *virt++ = MI_NOOP;
677                 *virt++ = MI_NOOP;
678         }
679
680         ring->tail = 0;
681         ring->space = ring->head - 8;
682
683         return 0;
684 }
685
686 int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
687 {
688         struct drm_device *dev = ring->dev;
689         struct drm_i915_private *dev_priv = dev->dev_private;
690         unsigned long end;
691         u32 head;
692
693         head = intel_read_status_page(ring, 4);
694         if (head) {
695                 ring->head = head & HEAD_ADDR;
696                 ring->space = ring->head - (ring->tail + 8);
697                 if (ring->space < 0)
698                         ring->space += ring->size;
699                 if (ring->space >= n)
700                         return 0;
701         }
702
703         trace_i915_ring_wait_begin (dev);
704         end = jiffies + 3 * HZ;
705         do {
706                 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
707                 ring->space = ring->head - (ring->tail + 8);
708                 if (ring->space < 0)
709                         ring->space += ring->size;
710                 if (ring->space >= n) {
711                         trace_i915_ring_wait_end(dev);
712                         return 0;
713                 }
714
715                 if (dev->primary->master) {
716                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
717                         if (master_priv->sarea_priv)
718                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
719                 }
720
721                 msleep(1);
722                 if (atomic_read(&dev_priv->mm.wedged))
723                         return -EAGAIN;
724         } while (!time_after(jiffies, end));
725         trace_i915_ring_wait_end (dev);
726         return -EBUSY;
727 }
728
729 int intel_ring_begin(struct intel_ring_buffer *ring,
730                      int num_dwords)
731 {
732         int n = 4*num_dwords;
733         int ret;
734
735         if (unlikely(ring->tail + n > ring->size)) {
736                 ret = intel_wrap_ring_buffer(ring);
737                 if (unlikely(ret))
738                         return ret;
739         }
740
741         if (unlikely(ring->space < n)) {
742                 ret = intel_wait_ring_buffer(ring, n);
743                 if (unlikely(ret))
744                         return ret;
745         }
746
747         ring->space -= n;
748         return 0;
749 }
750
751 void intel_ring_advance(struct intel_ring_buffer *ring)
752 {
753         ring->tail &= ring->size - 1;
754         ring->write_tail(ring, ring->tail);
755 }
756
757 static const struct intel_ring_buffer render_ring = {
758         .name                   = "render ring",
759         .id                     = RING_RENDER,
760         .mmio_base              = RENDER_RING_BASE,
761         .size                   = 32 * PAGE_SIZE,
762         .init                   = init_render_ring,
763         .write_tail             = ring_write_tail,
764         .flush                  = render_ring_flush,
765         .add_request            = render_ring_add_request,
766         .get_seqno              = render_ring_get_seqno,
767         .user_irq_get           = render_ring_get_user_irq,
768         .user_irq_put           = render_ring_put_user_irq,
769         .dispatch_execbuffer    = render_ring_dispatch_execbuffer,
770 };
771
772 /* ring buffer for bit-stream decoder */
773
774 static const struct intel_ring_buffer bsd_ring = {
775         .name                   = "bsd ring",
776         .id                     = RING_BSD,
777         .mmio_base              = BSD_RING_BASE,
778         .size                   = 32 * PAGE_SIZE,
779         .init                   = init_ring_common,
780         .write_tail             = ring_write_tail,
781         .flush                  = bsd_ring_flush,
782         .add_request            = ring_add_request,
783         .get_seqno              = ring_status_page_get_seqno,
784         .user_irq_get           = bsd_ring_get_user_irq,
785         .user_irq_put           = bsd_ring_put_user_irq,
786         .dispatch_execbuffer    = ring_dispatch_execbuffer,
787 };
788
789
790 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
791                                      u32 value)
792 {
793        drm_i915_private_t *dev_priv = ring->dev->dev_private;
794
795        /* Every tail move must follow the sequence below */
796        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
797                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
798                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
799        I915_WRITE(GEN6_BSD_RNCID, 0x0);
800
801        if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
802                                GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
803                        50))
804                DRM_ERROR("timed out waiting for IDLE Indicator\n");
805
806        I915_WRITE_TAIL(ring, value);
807        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
808                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
809                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
810 }
811
812 static void gen6_ring_flush(struct intel_ring_buffer *ring,
813                             u32 invalidate_domains,
814                             u32 flush_domains)
815 {
816         if (intel_ring_begin(ring, 4) == 0) {
817                 intel_ring_emit(ring, MI_FLUSH_DW);
818                 intel_ring_emit(ring, 0);
819                 intel_ring_emit(ring, 0);
820                 intel_ring_emit(ring, 0);
821                 intel_ring_advance(ring);
822         }
823 }
824
825 static int
826 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
827                               struct drm_i915_gem_execbuffer2 *exec,
828                               struct drm_clip_rect *cliprects,
829                               uint64_t exec_offset)
830 {
831        uint32_t exec_start;
832        int ret;
833
834        exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
835
836        ret = intel_ring_begin(ring, 2);
837        if (ret)
838                return ret;
839
840        intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
841        /* bit0-7 is the length on GEN6+ */
842        intel_ring_emit(ring, exec_start);
843        intel_ring_advance(ring);
844
845        return 0;
846 }
847
848 /* ring buffer for Video Codec for Gen6+ */
849 static const struct intel_ring_buffer gen6_bsd_ring = {
850        .name                    = "gen6 bsd ring",
851        .id                      = RING_BSD,
852        .mmio_base               = GEN6_BSD_RING_BASE,
853        .size                    = 32 * PAGE_SIZE,
854        .init                    = init_ring_common,
855        .write_tail              = gen6_bsd_ring_write_tail,
856        .flush                   = gen6_ring_flush,
857        .add_request             = ring_add_request,
858        .get_seqno               = ring_status_page_get_seqno,
859        .user_irq_get            = bsd_ring_get_user_irq,
860        .user_irq_put            = bsd_ring_put_user_irq,
861        .dispatch_execbuffer     = gen6_ring_dispatch_execbuffer,
862 };
863
864 /* Blitter support (SandyBridge+) */
865
866 static void
867 blt_ring_get_user_irq(struct intel_ring_buffer *ring)
868 {
869         /* do nothing */
870 }
871 static void
872 blt_ring_put_user_irq(struct intel_ring_buffer *ring)
873 {
874         /* do nothing */
875 }
876
877
878 /* Workaround for some stepping of SNB,
879  * each time when BLT engine ring tail moved,
880  * the first command in the ring to be parsed
881  * should be MI_BATCH_BUFFER_START
882  */
883 #define NEED_BLT_WORKAROUND(dev) \
884         (IS_GEN6(dev) && (dev->pdev->revision < 8))
885
886 static inline struct drm_i915_gem_object *
887 to_blt_workaround(struct intel_ring_buffer *ring)
888 {
889         return ring->private;
890 }
891
892 static int blt_ring_init(struct intel_ring_buffer *ring)
893 {
894         if (NEED_BLT_WORKAROUND(ring->dev)) {
895                 struct drm_i915_gem_object *obj;
896                 u32 *ptr;
897                 int ret;
898
899                 obj = i915_gem_alloc_object(ring->dev, 4096);
900                 if (obj == NULL)
901                         return -ENOMEM;
902
903                 ret = i915_gem_object_pin(obj, 4096, true);
904                 if (ret) {
905                         drm_gem_object_unreference(&obj->base);
906                         return ret;
907                 }
908
909                 ptr = kmap(obj->pages[0]);
910                 *ptr++ = MI_BATCH_BUFFER_END;
911                 *ptr++ = MI_NOOP;
912                 kunmap(obj->pages[0]);
913
914                 ret = i915_gem_object_set_to_gtt_domain(obj, false);
915                 if (ret) {
916                         i915_gem_object_unpin(obj);
917                         drm_gem_object_unreference(&obj->base);
918                         return ret;
919                 }
920
921                 ring->private = obj;
922         }
923
924         return init_ring_common(ring);
925 }
926
927 static int blt_ring_begin(struct intel_ring_buffer *ring,
928                           int num_dwords)
929 {
930         if (ring->private) {
931                 int ret = intel_ring_begin(ring, num_dwords+2);
932                 if (ret)
933                         return ret;
934
935                 intel_ring_emit(ring, MI_BATCH_BUFFER_START);
936                 intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset);
937
938                 return 0;
939         } else
940                 return intel_ring_begin(ring, 4);
941 }
942
943 static void blt_ring_flush(struct intel_ring_buffer *ring,
944                            u32 invalidate_domains,
945                            u32 flush_domains)
946 {
947         if (blt_ring_begin(ring, 4) == 0) {
948                 intel_ring_emit(ring, MI_FLUSH_DW);
949                 intel_ring_emit(ring, 0);
950                 intel_ring_emit(ring, 0);
951                 intel_ring_emit(ring, 0);
952                 intel_ring_advance(ring);
953         }
954 }
955
956 static int
957 blt_ring_add_request(struct intel_ring_buffer *ring,
958                      u32 *result)
959 {
960         u32 seqno;
961         int ret;
962
963         ret = blt_ring_begin(ring, 4);
964         if (ret)
965                 return ret;
966
967         seqno = i915_gem_get_seqno(ring->dev);
968
969         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
970         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
971         intel_ring_emit(ring, seqno);
972         intel_ring_emit(ring, MI_USER_INTERRUPT);
973         intel_ring_advance(ring);
974
975         DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
976         *result = seqno;
977         return 0;
978 }
979
980 static void blt_ring_cleanup(struct intel_ring_buffer *ring)
981 {
982         if (!ring->private)
983                 return;
984
985         i915_gem_object_unpin(ring->private);
986         drm_gem_object_unreference(ring->private);
987         ring->private = NULL;
988 }
989
990 static const struct intel_ring_buffer gen6_blt_ring = {
991        .name                    = "blt ring",
992        .id                      = RING_BLT,
993        .mmio_base               = BLT_RING_BASE,
994        .size                    = 32 * PAGE_SIZE,
995        .init                    = blt_ring_init,
996        .write_tail              = ring_write_tail,
997        .flush                   = blt_ring_flush,
998        .add_request             = blt_ring_add_request,
999        .get_seqno               = ring_status_page_get_seqno,
1000        .user_irq_get            = blt_ring_get_user_irq,
1001        .user_irq_put            = blt_ring_put_user_irq,
1002        .dispatch_execbuffer     = gen6_ring_dispatch_execbuffer,
1003        .cleanup                 = blt_ring_cleanup,
1004 };
1005
1006 int intel_init_render_ring_buffer(struct drm_device *dev)
1007 {
1008         drm_i915_private_t *dev_priv = dev->dev_private;
1009
1010         dev_priv->render_ring = render_ring;
1011
1012         if (!I915_NEED_GFX_HWS(dev)) {
1013                 dev_priv->render_ring.status_page.page_addr
1014                         = dev_priv->status_page_dmah->vaddr;
1015                 memset(dev_priv->render_ring.status_page.page_addr,
1016                                 0, PAGE_SIZE);
1017         }
1018
1019         return intel_init_ring_buffer(dev, &dev_priv->render_ring);
1020 }
1021
1022 int intel_init_bsd_ring_buffer(struct drm_device *dev)
1023 {
1024         drm_i915_private_t *dev_priv = dev->dev_private;
1025
1026         if (IS_GEN6(dev))
1027                 dev_priv->bsd_ring = gen6_bsd_ring;
1028         else
1029                 dev_priv->bsd_ring = bsd_ring;
1030
1031         return intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
1032 }
1033
1034 int intel_init_blt_ring_buffer(struct drm_device *dev)
1035 {
1036         drm_i915_private_t *dev_priv = dev->dev_private;
1037
1038         dev_priv->blt_ring = gen6_blt_ring;
1039
1040         return intel_init_ring_buffer(dev, &dev_priv->blt_ring);
1041 }