drm/i915/ringbuffer: Check that we setup the ringbuffer
[linux-2.6.git] / drivers / gpu / drm / i915 / intel_ringbuffer.c
1 /*
2  * Copyright © 2008-2010 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Eric Anholt <eric@anholt.net>
25  *    Zou Nan hai <nanhai.zou@intel.com>
26  *    Xiang Hai hao<haihao.xiang@intel.com>
27  *
28  */
29
30 #include "drmP.h"
31 #include "drm.h"
32 #include "i915_drv.h"
33 #include "i915_drm.h"
34 #include "i915_trace.h"
35 #include "intel_drv.h"
36
37 static u32 i915_gem_get_seqno(struct drm_device *dev)
38 {
39         drm_i915_private_t *dev_priv = dev->dev_private;
40         u32 seqno;
41
42         seqno = dev_priv->next_seqno;
43
44         /* reserve 0 for non-seqno */
45         if (++dev_priv->next_seqno == 0)
46                 dev_priv->next_seqno = 1;
47
48         return seqno;
49 }
50
51 static void
52 render_ring_flush(struct intel_ring_buffer *ring,
53                   u32   invalidate_domains,
54                   u32   flush_domains)
55 {
56         struct drm_device *dev = ring->dev;
57         drm_i915_private_t *dev_priv = dev->dev_private;
58         u32 cmd;
59
60 #if WATCH_EXEC
61         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
62                   invalidate_domains, flush_domains);
63 #endif
64
65         trace_i915_gem_request_flush(dev, dev_priv->next_seqno,
66                                      invalidate_domains, flush_domains);
67
68         if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
69                 /*
70                  * read/write caches:
71                  *
72                  * I915_GEM_DOMAIN_RENDER is always invalidated, but is
73                  * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
74                  * also flushed at 2d versus 3d pipeline switches.
75                  *
76                  * read-only caches:
77                  *
78                  * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
79                  * MI_READ_FLUSH is set, and is always flushed on 965.
80                  *
81                  * I915_GEM_DOMAIN_COMMAND may not exist?
82                  *
83                  * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
84                  * invalidated when MI_EXE_FLUSH is set.
85                  *
86                  * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
87                  * invalidated with every MI_FLUSH.
88                  *
89                  * TLBs:
90                  *
91                  * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
92                  * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
93                  * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
94                  * are flushed at any MI_FLUSH.
95                  */
96
97                 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
98                 if ((invalidate_domains|flush_domains) &
99                     I915_GEM_DOMAIN_RENDER)
100                         cmd &= ~MI_NO_WRITE_FLUSH;
101                 if (INTEL_INFO(dev)->gen < 4) {
102                         /*
103                          * On the 965, the sampler cache always gets flushed
104                          * and this bit is reserved.
105                          */
106                         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
107                                 cmd |= MI_READ_FLUSH;
108                 }
109                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
110                         cmd |= MI_EXE_FLUSH;
111
112 #if WATCH_EXEC
113                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
114 #endif
115                 if (intel_ring_begin(ring, 2) == 0) {
116                         intel_ring_emit(ring, cmd);
117                         intel_ring_emit(ring, MI_NOOP);
118                         intel_ring_advance(ring);
119                 }
120         }
121 }
122
123 static void ring_write_tail(struct intel_ring_buffer *ring,
124                             u32 value)
125 {
126         drm_i915_private_t *dev_priv = ring->dev->dev_private;
127         I915_WRITE_TAIL(ring, value);
128 }
129
130 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
131 {
132         drm_i915_private_t *dev_priv = ring->dev->dev_private;
133         u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
134                         RING_ACTHD(ring->mmio_base) : ACTHD;
135
136         return I915_READ(acthd_reg);
137 }
138
139 static int init_ring_common(struct intel_ring_buffer *ring)
140 {
141         drm_i915_private_t *dev_priv = ring->dev->dev_private;
142         struct drm_i915_gem_object *obj_priv = to_intel_bo(ring->gem_object);
143         u32 head;
144
145         /* Stop the ring if it's running. */
146         I915_WRITE_CTL(ring, 0);
147         I915_WRITE_HEAD(ring, 0);
148         ring->write_tail(ring, 0);
149
150         /* Initialize the ring. */
151         I915_WRITE_START(ring, obj_priv->gtt_offset);
152         head = I915_READ_HEAD(ring) & HEAD_ADDR;
153
154         /* G45 ring initialization fails to reset head to zero */
155         if (head != 0) {
156                 DRM_ERROR("%s head not reset to zero "
157                                 "ctl %08x head %08x tail %08x start %08x\n",
158                                 ring->name,
159                                 I915_READ_CTL(ring),
160                                 I915_READ_HEAD(ring),
161                                 I915_READ_TAIL(ring),
162                                 I915_READ_START(ring));
163
164                 I915_WRITE_HEAD(ring, 0);
165
166                 DRM_ERROR("%s head forced to zero "
167                                 "ctl %08x head %08x tail %08x start %08x\n",
168                                 ring->name,
169                                 I915_READ_CTL(ring),
170                                 I915_READ_HEAD(ring),
171                                 I915_READ_TAIL(ring),
172                                 I915_READ_START(ring));
173         }
174
175         I915_WRITE_CTL(ring,
176                         ((ring->gem_object->size - PAGE_SIZE) & RING_NR_PAGES)
177                         | RING_NO_REPORT | RING_VALID);
178
179         /* If the head is still not zero, the ring is dead */
180         if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
181             I915_READ_START(ring) != obj_priv->gtt_offset ||
182             (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
183                 DRM_ERROR("%s initialization failed "
184                                 "ctl %08x head %08x tail %08x start %08x\n",
185                                 ring->name,
186                                 I915_READ_CTL(ring),
187                                 I915_READ_HEAD(ring),
188                                 I915_READ_TAIL(ring),
189                                 I915_READ_START(ring));
190                 return -EIO;
191         }
192
193         if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
194                 i915_kernel_lost_context(ring->dev);
195         else {
196                 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
197                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
198                 ring->space = ring->head - (ring->tail + 8);
199                 if (ring->space < 0)
200                         ring->space += ring->size;
201         }
202         return 0;
203 }
204
205 static int init_render_ring(struct intel_ring_buffer *ring)
206 {
207         struct drm_device *dev = ring->dev;
208         int ret = init_ring_common(ring);
209
210         if (INTEL_INFO(dev)->gen > 3) {
211                 drm_i915_private_t *dev_priv = dev->dev_private;
212                 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
213                 if (IS_GEN6(dev))
214                         mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
215                 I915_WRITE(MI_MODE, mode);
216         }
217
218         return ret;
219 }
220
221 #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
222 do {                                                                    \
223         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |           \
224                  PIPE_CONTROL_DEPTH_STALL | 2);                         \
225         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
226         intel_ring_emit(ring__, 0);                                                     \
227         intel_ring_emit(ring__, 0);                                                     \
228 } while (0)
229
230 /**
231  * Creates a new sequence number, emitting a write of it to the status page
232  * plus an interrupt, which will trigger i915_user_interrupt_handler.
233  *
234  * Must be called with struct_lock held.
235  *
236  * Returned sequence numbers are nonzero on success.
237  */
238 static int
239 render_ring_add_request(struct intel_ring_buffer *ring,
240                         u32 *result)
241 {
242         struct drm_device *dev = ring->dev;
243         drm_i915_private_t *dev_priv = dev->dev_private;
244         u32 seqno = i915_gem_get_seqno(dev);
245         int ret;
246
247         if (IS_GEN6(dev)) {
248                 ret = intel_ring_begin(ring, 6);
249                 if (ret)
250                     return ret;
251
252                 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | 3);
253                 intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE |
254                                 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH |
255                                 PIPE_CONTROL_NOTIFY);
256                 intel_ring_emit(ring, dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
257                 intel_ring_emit(ring, seqno);
258                 intel_ring_emit(ring, 0);
259                 intel_ring_emit(ring, 0);
260         } else if (HAS_PIPE_CONTROL(dev)) {
261                 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
262
263                 /*
264                  * Workaround qword write incoherence by flushing the
265                  * PIPE_NOTIFY buffers out to memory before requesting
266                  * an interrupt.
267                  */
268                 ret = intel_ring_begin(ring, 32);
269                 if (ret)
270                         return ret;
271
272                 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
273                                 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
274                 intel_ring_emit(ring, dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
275                 intel_ring_emit(ring, seqno);
276                 intel_ring_emit(ring, 0);
277                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
278                 scratch_addr += 128; /* write to separate cachelines */
279                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
280                 scratch_addr += 128;
281                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
282                 scratch_addr += 128;
283                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
284                 scratch_addr += 128;
285                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
286                 scratch_addr += 128;
287                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
288                 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
289                                 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
290                                 PIPE_CONTROL_NOTIFY);
291                 intel_ring_emit(ring, dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
292                 intel_ring_emit(ring, seqno);
293                 intel_ring_emit(ring, 0);
294         } else {
295                 ret = intel_ring_begin(ring, 4);
296                 if (ret)
297                     return ret;
298
299                 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
300                 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
301                 intel_ring_emit(ring, seqno);
302
303                 intel_ring_emit(ring, MI_USER_INTERRUPT);
304         }
305
306         intel_ring_advance(ring);
307         *result = seqno;
308         return 0;
309 }
310
311 static u32
312 render_ring_get_seqno(struct intel_ring_buffer *ring)
313 {
314         struct drm_device *dev = ring->dev;
315         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
316         if (HAS_PIPE_CONTROL(dev))
317                 return ((volatile u32 *)(dev_priv->seqno_page))[0];
318         else
319                 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
320 }
321
322 static void
323 render_ring_get_user_irq(struct intel_ring_buffer *ring)
324 {
325         struct drm_device *dev = ring->dev;
326         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
327         unsigned long irqflags;
328
329         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
330         if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) {
331                 if (HAS_PCH_SPLIT(dev))
332                         ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
333                 else
334                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
335         }
336         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
337 }
338
339 static void
340 render_ring_put_user_irq(struct intel_ring_buffer *ring)
341 {
342         struct drm_device *dev = ring->dev;
343         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
344         unsigned long irqflags;
345
346         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
347         BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0);
348         if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
349                 if (HAS_PCH_SPLIT(dev))
350                         ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
351                 else
352                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
353         }
354         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
355 }
356
357 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
358 {
359         drm_i915_private_t *dev_priv = ring->dev->dev_private;
360         u32 mmio = IS_GEN6(ring->dev) ?
361                 RING_HWS_PGA_GEN6(ring->mmio_base) :
362                 RING_HWS_PGA(ring->mmio_base);
363         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
364         POSTING_READ(mmio);
365 }
366
367 static void
368 bsd_ring_flush(struct intel_ring_buffer *ring,
369                u32     invalidate_domains,
370                u32     flush_domains)
371 {
372         if (intel_ring_begin(ring, 2) == 0) {
373                 intel_ring_emit(ring, MI_FLUSH);
374                 intel_ring_emit(ring, MI_NOOP);
375                 intel_ring_advance(ring);
376         }
377 }
378
379 static int
380 ring_add_request(struct intel_ring_buffer *ring,
381                  u32 *result)
382 {
383         u32 seqno;
384         int ret;
385
386         ret = intel_ring_begin(ring, 4);
387         if (ret)
388                 return ret;
389
390         seqno = i915_gem_get_seqno(ring->dev);
391
392         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
393         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
394         intel_ring_emit(ring, seqno);
395         intel_ring_emit(ring, MI_USER_INTERRUPT);
396         intel_ring_advance(ring);
397
398         DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
399         *result = seqno;
400         return 0;
401 }
402
403 static void
404 bsd_ring_get_user_irq(struct intel_ring_buffer *ring)
405 {
406         /* do nothing */
407 }
408 static void
409 bsd_ring_put_user_irq(struct intel_ring_buffer *ring)
410 {
411         /* do nothing */
412 }
413
414 static u32
415 ring_status_page_get_seqno(struct intel_ring_buffer *ring)
416 {
417         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
418 }
419
420 static int
421 ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
422                          struct drm_i915_gem_execbuffer2 *exec,
423                          struct drm_clip_rect *cliprects,
424                          uint64_t exec_offset)
425 {
426         uint32_t exec_start;
427         int ret;
428
429         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
430
431         ret = intel_ring_begin(ring, 2);
432         if (ret)
433                 return ret;
434
435         intel_ring_emit(ring,
436                         MI_BATCH_BUFFER_START |
437                         (2 << 6) |
438                         MI_BATCH_NON_SECURE_I965);
439         intel_ring_emit(ring, exec_start);
440         intel_ring_advance(ring);
441
442         return 0;
443 }
444
445 static int
446 render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
447                                 struct drm_i915_gem_execbuffer2 *exec,
448                                 struct drm_clip_rect *cliprects,
449                                 uint64_t exec_offset)
450 {
451         struct drm_device *dev = ring->dev;
452         drm_i915_private_t *dev_priv = dev->dev_private;
453         int nbox = exec->num_cliprects;
454         uint32_t exec_start, exec_len;
455         int i, count, ret;
456
457         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
458         exec_len = (uint32_t) exec->batch_len;
459
460         trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
461
462         count = nbox ? nbox : 1;
463         for (i = 0; i < count; i++) {
464                 if (i < nbox) {
465                         ret = i915_emit_box(dev, cliprects, i,
466                                             exec->DR1, exec->DR4);
467                         if (ret)
468                                 return ret;
469                 }
470
471                 if (IS_I830(dev) || IS_845G(dev)) {
472                         ret = intel_ring_begin(ring, 4);
473                         if (ret)
474                                 return ret;
475
476                         intel_ring_emit(ring, MI_BATCH_BUFFER);
477                         intel_ring_emit(ring, exec_start | MI_BATCH_NON_SECURE);
478                         intel_ring_emit(ring, exec_start + exec_len - 4);
479                         intel_ring_emit(ring, 0);
480                 } else {
481                         ret = intel_ring_begin(ring, 2);
482                         if (ret)
483                                 return ret;
484
485                         if (INTEL_INFO(dev)->gen >= 4) {
486                                 intel_ring_emit(ring,
487                                                 MI_BATCH_BUFFER_START | (2 << 6)
488                                                 | MI_BATCH_NON_SECURE_I965);
489                                 intel_ring_emit(ring, exec_start);
490                         } else {
491                                 intel_ring_emit(ring, MI_BATCH_BUFFER_START
492                                                 | (2 << 6));
493                                 intel_ring_emit(ring, exec_start |
494                                                 MI_BATCH_NON_SECURE);
495                         }
496                 }
497                 intel_ring_advance(ring);
498         }
499
500         if (IS_G4X(dev) || IS_GEN5(dev)) {
501                 if (intel_ring_begin(ring, 2) == 0) {
502                         intel_ring_emit(ring, MI_FLUSH |
503                                         MI_NO_WRITE_FLUSH |
504                                         MI_INVALIDATE_ISP );
505                         intel_ring_emit(ring, MI_NOOP);
506                         intel_ring_advance(ring);
507                 }
508         }
509         /* XXX breadcrumb */
510
511         return 0;
512 }
513
514 static void cleanup_status_page(struct intel_ring_buffer *ring)
515 {
516         drm_i915_private_t *dev_priv = ring->dev->dev_private;
517         struct drm_gem_object *obj;
518         struct drm_i915_gem_object *obj_priv;
519
520         obj = ring->status_page.obj;
521         if (obj == NULL)
522                 return;
523         obj_priv = to_intel_bo(obj);
524
525         kunmap(obj_priv->pages[0]);
526         i915_gem_object_unpin(obj);
527         drm_gem_object_unreference(obj);
528         ring->status_page.obj = NULL;
529
530         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
531 }
532
533 static int init_status_page(struct intel_ring_buffer *ring)
534 {
535         struct drm_device *dev = ring->dev;
536         drm_i915_private_t *dev_priv = dev->dev_private;
537         struct drm_gem_object *obj;
538         struct drm_i915_gem_object *obj_priv;
539         int ret;
540
541         obj = i915_gem_alloc_object(dev, 4096);
542         if (obj == NULL) {
543                 DRM_ERROR("Failed to allocate status page\n");
544                 ret = -ENOMEM;
545                 goto err;
546         }
547         obj_priv = to_intel_bo(obj);
548         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
549
550         ret = i915_gem_object_pin(obj, 4096, true);
551         if (ret != 0) {
552                 goto err_unref;
553         }
554
555         ring->status_page.gfx_addr = obj_priv->gtt_offset;
556         ring->status_page.page_addr = kmap(obj_priv->pages[0]);
557         if (ring->status_page.page_addr == NULL) {
558                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
559                 goto err_unpin;
560         }
561         ring->status_page.obj = obj;
562         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
563
564         intel_ring_setup_status_page(ring);
565         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
566                         ring->name, ring->status_page.gfx_addr);
567
568         return 0;
569
570 err_unpin:
571         i915_gem_object_unpin(obj);
572 err_unref:
573         drm_gem_object_unreference(obj);
574 err:
575         return ret;
576 }
577
578 int intel_init_ring_buffer(struct drm_device *dev,
579                            struct intel_ring_buffer *ring)
580 {
581         struct drm_i915_private *dev_priv = dev->dev_private;
582         struct drm_i915_gem_object *obj_priv;
583         struct drm_gem_object *obj;
584         int ret;
585
586         ring->dev = dev;
587         INIT_LIST_HEAD(&ring->active_list);
588         INIT_LIST_HEAD(&ring->request_list);
589         INIT_LIST_HEAD(&ring->gpu_write_list);
590
591         if (I915_NEED_GFX_HWS(dev)) {
592                 ret = init_status_page(ring);
593                 if (ret)
594                         return ret;
595         }
596
597         obj = i915_gem_alloc_object(dev, ring->size);
598         if (obj == NULL) {
599                 DRM_ERROR("Failed to allocate ringbuffer\n");
600                 ret = -ENOMEM;
601                 goto err_hws;
602         }
603
604         ring->gem_object = obj;
605
606         ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
607         if (ret)
608                 goto err_unref;
609
610         obj_priv = to_intel_bo(obj);
611         ring->map.size = ring->size;
612         ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
613         ring->map.type = 0;
614         ring->map.flags = 0;
615         ring->map.mtrr = 0;
616
617         drm_core_ioremap_wc(&ring->map, dev);
618         if (ring->map.handle == NULL) {
619                 DRM_ERROR("Failed to map ringbuffer.\n");
620                 ret = -EINVAL;
621                 goto err_unpin;
622         }
623
624         ring->virtual_start = ring->map.handle;
625         ret = ring->init(ring);
626         if (ret)
627                 goto err_unmap;
628
629         if (!drm_core_check_feature(dev, DRIVER_MODESET))
630                 i915_kernel_lost_context(dev);
631         else {
632                 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
633                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
634                 ring->space = ring->head - (ring->tail + 8);
635                 if (ring->space < 0)
636                         ring->space += ring->size;
637         }
638         return ret;
639
640 err_unmap:
641         drm_core_ioremapfree(&ring->map, dev);
642 err_unpin:
643         i915_gem_object_unpin(obj);
644 err_unref:
645         drm_gem_object_unreference(obj);
646         ring->gem_object = NULL;
647 err_hws:
648         cleanup_status_page(ring);
649         return ret;
650 }
651
652 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
653 {
654         if (ring->gem_object == NULL)
655                 return;
656
657         drm_core_ioremapfree(&ring->map, ring->dev);
658
659         i915_gem_object_unpin(ring->gem_object);
660         drm_gem_object_unreference(ring->gem_object);
661         ring->gem_object = NULL;
662
663         cleanup_status_page(ring);
664 }
665
666 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
667 {
668         unsigned int *virt;
669         int rem;
670         rem = ring->size - ring->tail;
671
672         if (ring->space < rem) {
673                 int ret = intel_wait_ring_buffer(ring, rem);
674                 if (ret)
675                         return ret;
676         }
677
678         virt = (unsigned int *)(ring->virtual_start + ring->tail);
679         rem /= 8;
680         while (rem--) {
681                 *virt++ = MI_NOOP;
682                 *virt++ = MI_NOOP;
683         }
684
685         ring->tail = 0;
686         ring->space = ring->head - 8;
687
688         return 0;
689 }
690
691 int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
692 {
693         struct drm_device *dev = ring->dev;
694         drm_i915_private_t *dev_priv = dev->dev_private;
695         unsigned long end;
696
697         trace_i915_ring_wait_begin (dev);
698         end = jiffies + 3 * HZ;
699         do {
700                 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
701                 ring->space = ring->head - (ring->tail + 8);
702                 if (ring->space < 0)
703                         ring->space += ring->size;
704                 if (ring->space >= n) {
705                         trace_i915_ring_wait_end(dev);
706                         return 0;
707                 }
708
709                 if (dev->primary->master) {
710                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
711                         if (master_priv->sarea_priv)
712                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
713                 }
714
715                 msleep(1);
716         } while (!time_after(jiffies, end));
717         trace_i915_ring_wait_end (dev);
718         return -EBUSY;
719 }
720
721 int intel_ring_begin(struct intel_ring_buffer *ring,
722                      int num_dwords)
723 {
724         int n = 4*num_dwords;
725         int ret;
726
727         if (unlikely(ring->tail + n > ring->size)) {
728                 ret = intel_wrap_ring_buffer(ring);
729                 if (unlikely(ret))
730                         return ret;
731         }
732
733         if (unlikely(ring->space < n)) {
734                 ret = intel_wait_ring_buffer(ring, n);
735                 if (unlikely(ret))
736                         return ret;
737         }
738
739         ring->space -= n;
740         return 0;
741 }
742
743 void intel_ring_advance(struct intel_ring_buffer *ring)
744 {
745         ring->tail &= ring->size - 1;
746         ring->write_tail(ring, ring->tail);
747 }
748
749 static const struct intel_ring_buffer render_ring = {
750         .name                   = "render ring",
751         .id                     = RING_RENDER,
752         .mmio_base              = RENDER_RING_BASE,
753         .size                   = 32 * PAGE_SIZE,
754         .init                   = init_render_ring,
755         .write_tail             = ring_write_tail,
756         .flush                  = render_ring_flush,
757         .add_request            = render_ring_add_request,
758         .get_seqno              = render_ring_get_seqno,
759         .user_irq_get           = render_ring_get_user_irq,
760         .user_irq_put           = render_ring_put_user_irq,
761         .dispatch_execbuffer    = render_ring_dispatch_execbuffer,
762 };
763
764 /* ring buffer for bit-stream decoder */
765
766 static const struct intel_ring_buffer bsd_ring = {
767         .name                   = "bsd ring",
768         .id                     = RING_BSD,
769         .mmio_base              = BSD_RING_BASE,
770         .size                   = 32 * PAGE_SIZE,
771         .init                   = init_ring_common,
772         .write_tail             = ring_write_tail,
773         .flush                  = bsd_ring_flush,
774         .add_request            = ring_add_request,
775         .get_seqno              = ring_status_page_get_seqno,
776         .user_irq_get           = bsd_ring_get_user_irq,
777         .user_irq_put           = bsd_ring_put_user_irq,
778         .dispatch_execbuffer    = ring_dispatch_execbuffer,
779 };
780
781
782 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
783                                      u32 value)
784 {
785        drm_i915_private_t *dev_priv = ring->dev->dev_private;
786
787        /* Every tail move must follow the sequence below */
788        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
789                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
790                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
791        I915_WRITE(GEN6_BSD_RNCID, 0x0);
792
793        if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
794                                GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
795                        50))
796                DRM_ERROR("timed out waiting for IDLE Indicator\n");
797
798        I915_WRITE_TAIL(ring, value);
799        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
800                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
801                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
802 }
803
804 static void gen6_ring_flush(struct intel_ring_buffer *ring,
805                             u32 invalidate_domains,
806                             u32 flush_domains)
807 {
808         if (intel_ring_begin(ring, 4) == 0) {
809                 intel_ring_emit(ring, MI_FLUSH_DW);
810                 intel_ring_emit(ring, 0);
811                 intel_ring_emit(ring, 0);
812                 intel_ring_emit(ring, 0);
813                 intel_ring_advance(ring);
814         }
815 }
816
817 static int
818 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
819                               struct drm_i915_gem_execbuffer2 *exec,
820                               struct drm_clip_rect *cliprects,
821                               uint64_t exec_offset)
822 {
823        uint32_t exec_start;
824        int ret;
825
826        exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
827
828        ret = intel_ring_begin(ring, 2);
829        if (ret)
830                return ret;
831
832        intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
833        /* bit0-7 is the length on GEN6+ */
834        intel_ring_emit(ring, exec_start);
835        intel_ring_advance(ring);
836
837        return 0;
838 }
839
840 /* ring buffer for Video Codec for Gen6+ */
841 static const struct intel_ring_buffer gen6_bsd_ring = {
842        .name                    = "gen6 bsd ring",
843        .id                      = RING_BSD,
844        .mmio_base               = GEN6_BSD_RING_BASE,
845        .size                    = 32 * PAGE_SIZE,
846        .init                    = init_ring_common,
847        .write_tail              = gen6_bsd_ring_write_tail,
848        .flush                   = gen6_ring_flush,
849        .add_request             = ring_add_request,
850        .get_seqno               = ring_status_page_get_seqno,
851        .user_irq_get            = bsd_ring_get_user_irq,
852        .user_irq_put            = bsd_ring_put_user_irq,
853        .dispatch_execbuffer     = gen6_ring_dispatch_execbuffer,
854 };
855
856 /* Blitter support (SandyBridge+) */
857
858 static void
859 blt_ring_get_user_irq(struct intel_ring_buffer *ring)
860 {
861         /* do nothing */
862 }
863 static void
864 blt_ring_put_user_irq(struct intel_ring_buffer *ring)
865 {
866         /* do nothing */
867 }
868
869 static const struct intel_ring_buffer gen6_blt_ring = {
870        .name                    = "blt ring",
871        .id                      = RING_BLT,
872        .mmio_base               = BLT_RING_BASE,
873        .size                    = 32 * PAGE_SIZE,
874        .init                    = init_ring_common,
875        .write_tail              = ring_write_tail,
876        .flush                   = gen6_ring_flush,
877        .add_request             = ring_add_request,
878        .get_seqno               = ring_status_page_get_seqno,
879        .user_irq_get            = blt_ring_get_user_irq,
880        .user_irq_put            = blt_ring_put_user_irq,
881        .dispatch_execbuffer     = gen6_ring_dispatch_execbuffer,
882 };
883
884 int intel_init_render_ring_buffer(struct drm_device *dev)
885 {
886         drm_i915_private_t *dev_priv = dev->dev_private;
887
888         dev_priv->render_ring = render_ring;
889
890         if (!I915_NEED_GFX_HWS(dev)) {
891                 dev_priv->render_ring.status_page.page_addr
892                         = dev_priv->status_page_dmah->vaddr;
893                 memset(dev_priv->render_ring.status_page.page_addr,
894                                 0, PAGE_SIZE);
895         }
896
897         return intel_init_ring_buffer(dev, &dev_priv->render_ring);
898 }
899
900 int intel_init_bsd_ring_buffer(struct drm_device *dev)
901 {
902         drm_i915_private_t *dev_priv = dev->dev_private;
903
904         if (IS_GEN6(dev))
905                 dev_priv->bsd_ring = gen6_bsd_ring;
906         else
907                 dev_priv->bsd_ring = bsd_ring;
908
909         return intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
910 }
911
912 int intel_init_blt_ring_buffer(struct drm_device *dev)
913 {
914         drm_i915_private_t *dev_priv = dev->dev_private;
915
916         dev_priv->blt_ring = gen6_blt_ring;
917
918         return intel_init_ring_buffer(dev, &dev_priv->blt_ring);
919 }