9ac73dd1b422db554758061d7d8786403bf64b9e
[linux-2.6.git] / drivers / gpu / drm / i915 / i915_gem.c
1 /*
2  * Copyright © 2008 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Eric Anholt <eric@anholt.net>
25  *
26  */
27
28 #include "drmP.h"
29 #include "drm.h"
30 #include "i915_drm.h"
31 #include "i915_drv.h"
32 #include <linux/swap.h>
33
34 static int
35 i915_gem_object_set_domain(struct drm_gem_object *obj,
36                             uint32_t read_domains,
37                             uint32_t write_domain);
38 static int
39 i915_gem_object_set_domain_range(struct drm_gem_object *obj,
40                                  uint64_t offset,
41                                  uint64_t size,
42                                  uint32_t read_domains,
43                                  uint32_t write_domain);
44 static int
45 i915_gem_set_domain(struct drm_gem_object *obj,
46                     struct drm_file *file_priv,
47                     uint32_t read_domains,
48                     uint32_t write_domain);
49 static int i915_gem_object_get_page_list(struct drm_gem_object *obj);
50 static void i915_gem_object_free_page_list(struct drm_gem_object *obj);
51 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
52
53 static void
54 i915_gem_cleanup_ringbuffer(struct drm_device *dev);
55
56 int
57 i915_gem_init_ioctl(struct drm_device *dev, void *data,
58                     struct drm_file *file_priv)
59 {
60         drm_i915_private_t *dev_priv = dev->dev_private;
61         struct drm_i915_gem_init *args = data;
62
63         mutex_lock(&dev->struct_mutex);
64
65         if (args->gtt_start >= args->gtt_end ||
66             (args->gtt_start & (PAGE_SIZE - 1)) != 0 ||
67             (args->gtt_end & (PAGE_SIZE - 1)) != 0) {
68                 mutex_unlock(&dev->struct_mutex);
69                 return -EINVAL;
70         }
71
72         drm_mm_init(&dev_priv->mm.gtt_space, args->gtt_start,
73             args->gtt_end - args->gtt_start);
74
75         dev->gtt_total = (uint32_t) (args->gtt_end - args->gtt_start);
76
77         mutex_unlock(&dev->struct_mutex);
78
79         return 0;
80 }
81
82
83 /**
84  * Creates a new mm object and returns a handle to it.
85  */
86 int
87 i915_gem_create_ioctl(struct drm_device *dev, void *data,
88                       struct drm_file *file_priv)
89 {
90         struct drm_i915_gem_create *args = data;
91         struct drm_gem_object *obj;
92         int handle, ret;
93
94         args->size = roundup(args->size, PAGE_SIZE);
95
96         /* Allocate the new object */
97         obj = drm_gem_object_alloc(dev, args->size);
98         if (obj == NULL)
99                 return -ENOMEM;
100
101         ret = drm_gem_handle_create(file_priv, obj, &handle);
102         mutex_lock(&dev->struct_mutex);
103         drm_gem_object_handle_unreference(obj);
104         mutex_unlock(&dev->struct_mutex);
105
106         if (ret)
107                 return ret;
108
109         args->handle = handle;
110
111         return 0;
112 }
113
114 /**
115  * Reads data from the object referenced by handle.
116  *
117  * On error, the contents of *data are undefined.
118  */
119 int
120 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
121                      struct drm_file *file_priv)
122 {
123         struct drm_i915_gem_pread *args = data;
124         struct drm_gem_object *obj;
125         struct drm_i915_gem_object *obj_priv;
126         ssize_t read;
127         loff_t offset;
128         int ret;
129
130         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
131         if (obj == NULL)
132                 return -EBADF;
133         obj_priv = obj->driver_private;
134
135         /* Bounds check source.
136          *
137          * XXX: This could use review for overflow issues...
138          */
139         if (args->offset > obj->size || args->size > obj->size ||
140             args->offset + args->size > obj->size) {
141                 drm_gem_object_unreference(obj);
142                 return -EINVAL;
143         }
144
145         mutex_lock(&dev->struct_mutex);
146
147         ret = i915_gem_object_set_domain_range(obj, args->offset, args->size,
148                                                I915_GEM_DOMAIN_CPU, 0);
149         if (ret != 0) {
150                 drm_gem_object_unreference(obj);
151                 mutex_unlock(&dev->struct_mutex);
152                 return ret;
153         }
154
155         offset = args->offset;
156
157         read = vfs_read(obj->filp, (char __user *)(uintptr_t)args->data_ptr,
158                         args->size, &offset);
159         if (read != args->size) {
160                 drm_gem_object_unreference(obj);
161                 mutex_unlock(&dev->struct_mutex);
162                 if (read < 0)
163                         return read;
164                 else
165                         return -EINVAL;
166         }
167
168         drm_gem_object_unreference(obj);
169         mutex_unlock(&dev->struct_mutex);
170
171         return 0;
172 }
173
174 static int
175 i915_gem_gtt_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
176                     struct drm_i915_gem_pwrite *args,
177                     struct drm_file *file_priv)
178 {
179         struct drm_i915_gem_object *obj_priv = obj->driver_private;
180         ssize_t remain;
181         loff_t offset;
182         char __user *user_data;
183         char __iomem *vaddr;
184         char *vaddr_atomic;
185         int i, o, l;
186         int ret = 0;
187         unsigned long pfn;
188         unsigned long unwritten;
189
190         user_data = (char __user *) (uintptr_t) args->data_ptr;
191         remain = args->size;
192         if (!access_ok(VERIFY_READ, user_data, remain))
193                 return -EFAULT;
194
195
196         mutex_lock(&dev->struct_mutex);
197         ret = i915_gem_object_pin(obj, 0);
198         if (ret) {
199                 mutex_unlock(&dev->struct_mutex);
200                 return ret;
201         }
202         ret = i915_gem_set_domain(obj, file_priv,
203                                   I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
204         if (ret)
205                 goto fail;
206
207         obj_priv = obj->driver_private;
208         offset = obj_priv->gtt_offset + args->offset;
209         obj_priv->dirty = 1;
210
211         while (remain > 0) {
212                 /* Operation in this page
213                  *
214                  * i = page number
215                  * o = offset within page
216                  * l = bytes to copy
217                  */
218                 i = offset >> PAGE_SHIFT;
219                 o = offset & (PAGE_SIZE-1);
220                 l = remain;
221                 if ((o + l) > PAGE_SIZE)
222                         l = PAGE_SIZE - o;
223
224                 pfn = (dev->agp->base >> PAGE_SHIFT) + i;
225
226 #ifdef CONFIG_HIGHMEM
227                 /* This is a workaround for the low performance of iounmap
228                  * (approximate 10% cpu cost on normal 3D workloads).
229                  * kmap_atomic on HIGHMEM kernels happens to let us map card
230                  * memory without taking IPIs.  When the vmap rework lands
231                  * we should be able to dump this hack.
232                  */
233                 vaddr_atomic = kmap_atomic_pfn(pfn, KM_USER0);
234 #if WATCH_PWRITE
235                 DRM_INFO("pwrite i %d o %d l %d pfn %ld vaddr %p\n",
236                          i, o, l, pfn, vaddr_atomic);
237 #endif
238                 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + o,
239                                                               user_data, l);
240                 kunmap_atomic(vaddr_atomic, KM_USER0);
241
242                 if (unwritten)
243 #endif /* CONFIG_HIGHMEM */
244                 {
245                         vaddr = ioremap_wc(pfn << PAGE_SHIFT, PAGE_SIZE);
246 #if WATCH_PWRITE
247                         DRM_INFO("pwrite slow i %d o %d l %d "
248                                  "pfn %ld vaddr %p\n",
249                                  i, o, l, pfn, vaddr);
250 #endif
251                         if (vaddr == NULL) {
252                                 ret = -EFAULT;
253                                 goto fail;
254                         }
255                         unwritten = __copy_from_user(vaddr + o, user_data, l);
256 #if WATCH_PWRITE
257                         DRM_INFO("unwritten %ld\n", unwritten);
258 #endif
259                         iounmap(vaddr);
260                         if (unwritten) {
261                                 ret = -EFAULT;
262                                 goto fail;
263                         }
264                 }
265
266                 remain -= l;
267                 user_data += l;
268                 offset += l;
269         }
270 #if WATCH_PWRITE && 1
271         i915_gem_clflush_object(obj);
272         i915_gem_dump_object(obj, args->offset + args->size, __func__, ~0);
273         i915_gem_clflush_object(obj);
274 #endif
275
276 fail:
277         i915_gem_object_unpin(obj);
278         mutex_unlock(&dev->struct_mutex);
279
280         return ret;
281 }
282
283 static int
284 i915_gem_shmem_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
285                       struct drm_i915_gem_pwrite *args,
286                       struct drm_file *file_priv)
287 {
288         int ret;
289         loff_t offset;
290         ssize_t written;
291
292         mutex_lock(&dev->struct_mutex);
293
294         ret = i915_gem_set_domain(obj, file_priv,
295                                   I915_GEM_DOMAIN_CPU, I915_GEM_DOMAIN_CPU);
296         if (ret) {
297                 mutex_unlock(&dev->struct_mutex);
298                 return ret;
299         }
300
301         offset = args->offset;
302
303         written = vfs_write(obj->filp,
304                             (char __user *)(uintptr_t) args->data_ptr,
305                             args->size, &offset);
306         if (written != args->size) {
307                 mutex_unlock(&dev->struct_mutex);
308                 if (written < 0)
309                         return written;
310                 else
311                         return -EINVAL;
312         }
313
314         mutex_unlock(&dev->struct_mutex);
315
316         return 0;
317 }
318
319 /**
320  * Writes data to the object referenced by handle.
321  *
322  * On error, the contents of the buffer that were to be modified are undefined.
323  */
324 int
325 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
326                       struct drm_file *file_priv)
327 {
328         struct drm_i915_gem_pwrite *args = data;
329         struct drm_gem_object *obj;
330         struct drm_i915_gem_object *obj_priv;
331         int ret = 0;
332
333         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
334         if (obj == NULL)
335                 return -EBADF;
336         obj_priv = obj->driver_private;
337
338         /* Bounds check destination.
339          *
340          * XXX: This could use review for overflow issues...
341          */
342         if (args->offset > obj->size || args->size > obj->size ||
343             args->offset + args->size > obj->size) {
344                 drm_gem_object_unreference(obj);
345                 return -EINVAL;
346         }
347
348         /* We can only do the GTT pwrite on untiled buffers, as otherwise
349          * it would end up going through the fenced access, and we'll get
350          * different detiling behavior between reading and writing.
351          * pread/pwrite currently are reading and writing from the CPU
352          * perspective, requiring manual detiling by the client.
353          */
354         if (obj_priv->tiling_mode == I915_TILING_NONE &&
355             dev->gtt_total != 0)
356                 ret = i915_gem_gtt_pwrite(dev, obj, args, file_priv);
357         else
358                 ret = i915_gem_shmem_pwrite(dev, obj, args, file_priv);
359
360 #if WATCH_PWRITE
361         if (ret)
362                 DRM_INFO("pwrite failed %d\n", ret);
363 #endif
364
365         drm_gem_object_unreference(obj);
366
367         return ret;
368 }
369
370 /**
371  * Called when user space prepares to use an object
372  */
373 int
374 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
375                           struct drm_file *file_priv)
376 {
377         struct drm_i915_gem_set_domain *args = data;
378         struct drm_gem_object *obj;
379         int ret;
380
381         if (!(dev->driver->driver_features & DRIVER_GEM))
382                 return -ENODEV;
383
384         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
385         if (obj == NULL)
386                 return -EBADF;
387
388         mutex_lock(&dev->struct_mutex);
389 #if WATCH_BUF
390         DRM_INFO("set_domain_ioctl %p(%d), %08x %08x\n",
391                  obj, obj->size, args->read_domains, args->write_domain);
392 #endif
393         ret = i915_gem_set_domain(obj, file_priv,
394                                   args->read_domains, args->write_domain);
395         drm_gem_object_unreference(obj);
396         mutex_unlock(&dev->struct_mutex);
397         return ret;
398 }
399
400 /**
401  * Called when user space has done writes to this buffer
402  */
403 int
404 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
405                       struct drm_file *file_priv)
406 {
407         struct drm_i915_gem_sw_finish *args = data;
408         struct drm_gem_object *obj;
409         struct drm_i915_gem_object *obj_priv;
410         int ret = 0;
411
412         if (!(dev->driver->driver_features & DRIVER_GEM))
413                 return -ENODEV;
414
415         mutex_lock(&dev->struct_mutex);
416         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
417         if (obj == NULL) {
418                 mutex_unlock(&dev->struct_mutex);
419                 return -EBADF;
420         }
421
422 #if WATCH_BUF
423         DRM_INFO("%s: sw_finish %d (%p %d)\n",
424                  __func__, args->handle, obj, obj->size);
425 #endif
426         obj_priv = obj->driver_private;
427
428         /* Pinned buffers may be scanout, so flush the cache */
429         if ((obj->write_domain & I915_GEM_DOMAIN_CPU) && obj_priv->pin_count) {
430                 i915_gem_clflush_object(obj);
431                 drm_agp_chipset_flush(dev);
432         }
433         drm_gem_object_unreference(obj);
434         mutex_unlock(&dev->struct_mutex);
435         return ret;
436 }
437
438 /**
439  * Maps the contents of an object, returning the address it is mapped
440  * into.
441  *
442  * While the mapping holds a reference on the contents of the object, it doesn't
443  * imply a ref on the object itself.
444  */
445 int
446 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
447                    struct drm_file *file_priv)
448 {
449         struct drm_i915_gem_mmap *args = data;
450         struct drm_gem_object *obj;
451         loff_t offset;
452         unsigned long addr;
453
454         if (!(dev->driver->driver_features & DRIVER_GEM))
455                 return -ENODEV;
456
457         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
458         if (obj == NULL)
459                 return -EBADF;
460
461         offset = args->offset;
462
463         down_write(&current->mm->mmap_sem);
464         addr = do_mmap(obj->filp, 0, args->size,
465                        PROT_READ | PROT_WRITE, MAP_SHARED,
466                        args->offset);
467         up_write(&current->mm->mmap_sem);
468         mutex_lock(&dev->struct_mutex);
469         drm_gem_object_unreference(obj);
470         mutex_unlock(&dev->struct_mutex);
471         if (IS_ERR((void *)addr))
472                 return addr;
473
474         args->addr_ptr = (uint64_t) addr;
475
476         return 0;
477 }
478
479 static void
480 i915_gem_object_free_page_list(struct drm_gem_object *obj)
481 {
482         struct drm_i915_gem_object *obj_priv = obj->driver_private;
483         int page_count = obj->size / PAGE_SIZE;
484         int i;
485
486         if (obj_priv->page_list == NULL)
487                 return;
488
489
490         for (i = 0; i < page_count; i++)
491                 if (obj_priv->page_list[i] != NULL) {
492                         if (obj_priv->dirty)
493                                 set_page_dirty(obj_priv->page_list[i]);
494                         mark_page_accessed(obj_priv->page_list[i]);
495                         page_cache_release(obj_priv->page_list[i]);
496                 }
497         obj_priv->dirty = 0;
498
499         drm_free(obj_priv->page_list,
500                  page_count * sizeof(struct page *),
501                  DRM_MEM_DRIVER);
502         obj_priv->page_list = NULL;
503 }
504
505 static void
506 i915_gem_object_move_to_active(struct drm_gem_object *obj)
507 {
508         struct drm_device *dev = obj->dev;
509         drm_i915_private_t *dev_priv = dev->dev_private;
510         struct drm_i915_gem_object *obj_priv = obj->driver_private;
511
512         /* Add a reference if we're newly entering the active list. */
513         if (!obj_priv->active) {
514                 drm_gem_object_reference(obj);
515                 obj_priv->active = 1;
516         }
517         /* Move from whatever list we were on to the tail of execution. */
518         list_move_tail(&obj_priv->list,
519                        &dev_priv->mm.active_list);
520 }
521
522
523 static void
524 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
525 {
526         struct drm_device *dev = obj->dev;
527         drm_i915_private_t *dev_priv = dev->dev_private;
528         struct drm_i915_gem_object *obj_priv = obj->driver_private;
529
530         i915_verify_inactive(dev, __FILE__, __LINE__);
531         if (obj_priv->pin_count != 0)
532                 list_del_init(&obj_priv->list);
533         else
534                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
535
536         if (obj_priv->active) {
537                 obj_priv->active = 0;
538                 drm_gem_object_unreference(obj);
539         }
540         i915_verify_inactive(dev, __FILE__, __LINE__);
541 }
542
543 /**
544  * Creates a new sequence number, emitting a write of it to the status page
545  * plus an interrupt, which will trigger i915_user_interrupt_handler.
546  *
547  * Must be called with struct_lock held.
548  *
549  * Returned sequence numbers are nonzero on success.
550  */
551 static uint32_t
552 i915_add_request(struct drm_device *dev, uint32_t flush_domains)
553 {
554         drm_i915_private_t *dev_priv = dev->dev_private;
555         struct drm_i915_gem_request *request;
556         uint32_t seqno;
557         int was_empty;
558         RING_LOCALS;
559
560         request = drm_calloc(1, sizeof(*request), DRM_MEM_DRIVER);
561         if (request == NULL)
562                 return 0;
563
564         /* Grab the seqno we're going to make this request be, and bump the
565          * next (skipping 0 so it can be the reserved no-seqno value).
566          */
567         seqno = dev_priv->mm.next_gem_seqno;
568         dev_priv->mm.next_gem_seqno++;
569         if (dev_priv->mm.next_gem_seqno == 0)
570                 dev_priv->mm.next_gem_seqno++;
571
572         BEGIN_LP_RING(4);
573         OUT_RING(MI_STORE_DWORD_INDEX);
574         OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
575         OUT_RING(seqno);
576
577         OUT_RING(MI_USER_INTERRUPT);
578         ADVANCE_LP_RING();
579
580         DRM_DEBUG("%d\n", seqno);
581
582         request->seqno = seqno;
583         request->emitted_jiffies = jiffies;
584         request->flush_domains = flush_domains;
585         was_empty = list_empty(&dev_priv->mm.request_list);
586         list_add_tail(&request->list, &dev_priv->mm.request_list);
587
588         if (was_empty && !dev_priv->mm.suspended)
589                 schedule_delayed_work(&dev_priv->mm.retire_work, HZ);
590         return seqno;
591 }
592
593 /**
594  * Command execution barrier
595  *
596  * Ensures that all commands in the ring are finished
597  * before signalling the CPU
598  */
599 static uint32_t
600 i915_retire_commands(struct drm_device *dev)
601 {
602         drm_i915_private_t *dev_priv = dev->dev_private;
603         uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
604         uint32_t flush_domains = 0;
605         RING_LOCALS;
606
607         /* The sampler always gets flushed on i965 (sigh) */
608         if (IS_I965G(dev))
609                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
610         BEGIN_LP_RING(2);
611         OUT_RING(cmd);
612         OUT_RING(0); /* noop */
613         ADVANCE_LP_RING();
614         return flush_domains;
615 }
616
617 /**
618  * Moves buffers associated only with the given active seqno from the active
619  * to inactive list, potentially freeing them.
620  */
621 static void
622 i915_gem_retire_request(struct drm_device *dev,
623                         struct drm_i915_gem_request *request)
624 {
625         drm_i915_private_t *dev_priv = dev->dev_private;
626
627         /* Move any buffers on the active list that are no longer referenced
628          * by the ringbuffer to the flushing/inactive lists as appropriate.
629          */
630         while (!list_empty(&dev_priv->mm.active_list)) {
631                 struct drm_gem_object *obj;
632                 struct drm_i915_gem_object *obj_priv;
633
634                 obj_priv = list_first_entry(&dev_priv->mm.active_list,
635                                             struct drm_i915_gem_object,
636                                             list);
637                 obj = obj_priv->obj;
638
639                 /* If the seqno being retired doesn't match the oldest in the
640                  * list, then the oldest in the list must still be newer than
641                  * this seqno.
642                  */
643                 if (obj_priv->last_rendering_seqno != request->seqno)
644                         return;
645 #if WATCH_LRU
646                 DRM_INFO("%s: retire %d moves to inactive list %p\n",
647                          __func__, request->seqno, obj);
648 #endif
649
650                 if (obj->write_domain != 0) {
651                         list_move_tail(&obj_priv->list,
652                                        &dev_priv->mm.flushing_list);
653                 } else {
654                         i915_gem_object_move_to_inactive(obj);
655                 }
656         }
657
658         if (request->flush_domains != 0) {
659                 struct drm_i915_gem_object *obj_priv, *next;
660
661                 /* Clear the write domain and activity from any buffers
662                  * that are just waiting for a flush matching the one retired.
663                  */
664                 list_for_each_entry_safe(obj_priv, next,
665                                          &dev_priv->mm.flushing_list, list) {
666                         struct drm_gem_object *obj = obj_priv->obj;
667
668                         if (obj->write_domain & request->flush_domains) {
669                                 obj->write_domain = 0;
670                                 i915_gem_object_move_to_inactive(obj);
671                         }
672                 }
673
674         }
675 }
676
677 /**
678  * Returns true if seq1 is later than seq2.
679  */
680 static int
681 i915_seqno_passed(uint32_t seq1, uint32_t seq2)
682 {
683         return (int32_t)(seq1 - seq2) >= 0;
684 }
685
686 uint32_t
687 i915_get_gem_seqno(struct drm_device *dev)
688 {
689         drm_i915_private_t *dev_priv = dev->dev_private;
690
691         return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
692 }
693
694 /**
695  * This function clears the request list as sequence numbers are passed.
696  */
697 void
698 i915_gem_retire_requests(struct drm_device *dev)
699 {
700         drm_i915_private_t *dev_priv = dev->dev_private;
701         uint32_t seqno;
702
703         seqno = i915_get_gem_seqno(dev);
704
705         while (!list_empty(&dev_priv->mm.request_list)) {
706                 struct drm_i915_gem_request *request;
707                 uint32_t retiring_seqno;
708
709                 request = list_first_entry(&dev_priv->mm.request_list,
710                                            struct drm_i915_gem_request,
711                                            list);
712                 retiring_seqno = request->seqno;
713
714                 if (i915_seqno_passed(seqno, retiring_seqno) ||
715                     dev_priv->mm.wedged) {
716                         i915_gem_retire_request(dev, request);
717
718                         list_del(&request->list);
719                         drm_free(request, sizeof(*request), DRM_MEM_DRIVER);
720                 } else
721                         break;
722         }
723 }
724
725 void
726 i915_gem_retire_work_handler(struct work_struct *work)
727 {
728         drm_i915_private_t *dev_priv;
729         struct drm_device *dev;
730
731         dev_priv = container_of(work, drm_i915_private_t,
732                                 mm.retire_work.work);
733         dev = dev_priv->dev;
734
735         mutex_lock(&dev->struct_mutex);
736         i915_gem_retire_requests(dev);
737         if (!dev_priv->mm.suspended &&
738             !list_empty(&dev_priv->mm.request_list))
739                 schedule_delayed_work(&dev_priv->mm.retire_work, HZ);
740         mutex_unlock(&dev->struct_mutex);
741 }
742
743 /**
744  * Waits for a sequence number to be signaled, and cleans up the
745  * request and object lists appropriately for that event.
746  */
747 static int
748 i915_wait_request(struct drm_device *dev, uint32_t seqno)
749 {
750         drm_i915_private_t *dev_priv = dev->dev_private;
751         int ret = 0;
752
753         BUG_ON(seqno == 0);
754
755         if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
756                 dev_priv->mm.waiting_gem_seqno = seqno;
757                 i915_user_irq_get(dev);
758                 ret = wait_event_interruptible(dev_priv->irq_queue,
759                                                i915_seqno_passed(i915_get_gem_seqno(dev),
760                                                                  seqno) ||
761                                                dev_priv->mm.wedged);
762                 i915_user_irq_put(dev);
763                 dev_priv->mm.waiting_gem_seqno = 0;
764         }
765         if (dev_priv->mm.wedged)
766                 ret = -EIO;
767
768         if (ret && ret != -ERESTARTSYS)
769                 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
770                           __func__, ret, seqno, i915_get_gem_seqno(dev));
771
772         /* Directly dispatch request retiring.  While we have the work queue
773          * to handle this, the waiter on a request often wants an associated
774          * buffer to have made it to the inactive list, and we would need
775          * a separate wait queue to handle that.
776          */
777         if (ret == 0)
778                 i915_gem_retire_requests(dev);
779
780         return ret;
781 }
782
783 static void
784 i915_gem_flush(struct drm_device *dev,
785                uint32_t invalidate_domains,
786                uint32_t flush_domains)
787 {
788         drm_i915_private_t *dev_priv = dev->dev_private;
789         uint32_t cmd;
790         RING_LOCALS;
791
792 #if WATCH_EXEC
793         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
794                   invalidate_domains, flush_domains);
795 #endif
796
797         if (flush_domains & I915_GEM_DOMAIN_CPU)
798                 drm_agp_chipset_flush(dev);
799
800         if ((invalidate_domains | flush_domains) & ~(I915_GEM_DOMAIN_CPU |
801                                                      I915_GEM_DOMAIN_GTT)) {
802                 /*
803                  * read/write caches:
804                  *
805                  * I915_GEM_DOMAIN_RENDER is always invalidated, but is
806                  * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
807                  * also flushed at 2d versus 3d pipeline switches.
808                  *
809                  * read-only caches:
810                  *
811                  * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
812                  * MI_READ_FLUSH is set, and is always flushed on 965.
813                  *
814                  * I915_GEM_DOMAIN_COMMAND may not exist?
815                  *
816                  * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
817                  * invalidated when MI_EXE_FLUSH is set.
818                  *
819                  * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
820                  * invalidated with every MI_FLUSH.
821                  *
822                  * TLBs:
823                  *
824                  * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
825                  * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
826                  * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
827                  * are flushed at any MI_FLUSH.
828                  */
829
830                 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
831                 if ((invalidate_domains|flush_domains) &
832                     I915_GEM_DOMAIN_RENDER)
833                         cmd &= ~MI_NO_WRITE_FLUSH;
834                 if (!IS_I965G(dev)) {
835                         /*
836                          * On the 965, the sampler cache always gets flushed
837                          * and this bit is reserved.
838                          */
839                         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
840                                 cmd |= MI_READ_FLUSH;
841                 }
842                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
843                         cmd |= MI_EXE_FLUSH;
844
845 #if WATCH_EXEC
846                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
847 #endif
848                 BEGIN_LP_RING(2);
849                 OUT_RING(cmd);
850                 OUT_RING(0); /* noop */
851                 ADVANCE_LP_RING();
852         }
853 }
854
855 /**
856  * Ensures that all rendering to the object has completed and the object is
857  * safe to unbind from the GTT or access from the CPU.
858  */
859 static int
860 i915_gem_object_wait_rendering(struct drm_gem_object *obj)
861 {
862         struct drm_device *dev = obj->dev;
863         struct drm_i915_gem_object *obj_priv = obj->driver_private;
864         int ret;
865
866         /* If there are writes queued to the buffer, flush and
867          * create a new seqno to wait for.
868          */
869         if (obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)) {
870                 uint32_t write_domain = obj->write_domain;
871 #if WATCH_BUF
872                 DRM_INFO("%s: flushing object %p from write domain %08x\n",
873                           __func__, obj, write_domain);
874 #endif
875                 i915_gem_flush(dev, 0, write_domain);
876
877                 i915_gem_object_move_to_active(obj);
878                 obj_priv->last_rendering_seqno = i915_add_request(dev,
879                                                                   write_domain);
880                 BUG_ON(obj_priv->last_rendering_seqno == 0);
881 #if WATCH_LRU
882                 DRM_INFO("%s: flush moves to exec list %p\n", __func__, obj);
883 #endif
884         }
885
886         /* If there is rendering queued on the buffer being evicted, wait for
887          * it.
888          */
889         if (obj_priv->active) {
890 #if WATCH_BUF
891                 DRM_INFO("%s: object %p wait for seqno %08x\n",
892                           __func__, obj, obj_priv->last_rendering_seqno);
893 #endif
894                 ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
895                 if (ret != 0)
896                         return ret;
897         }
898
899         return 0;
900 }
901
902 /**
903  * Unbinds an object from the GTT aperture.
904  */
905 static int
906 i915_gem_object_unbind(struct drm_gem_object *obj)
907 {
908         struct drm_device *dev = obj->dev;
909         struct drm_i915_gem_object *obj_priv = obj->driver_private;
910         int ret = 0;
911
912 #if WATCH_BUF
913         DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
914         DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
915 #endif
916         if (obj_priv->gtt_space == NULL)
917                 return 0;
918
919         if (obj_priv->pin_count != 0) {
920                 DRM_ERROR("Attempting to unbind pinned buffer\n");
921                 return -EINVAL;
922         }
923
924         /* Wait for any rendering to complete
925          */
926         ret = i915_gem_object_wait_rendering(obj);
927         if (ret) {
928                 DRM_ERROR("wait_rendering failed: %d\n", ret);
929                 return ret;
930         }
931
932         /* Move the object to the CPU domain to ensure that
933          * any possible CPU writes while it's not in the GTT
934          * are flushed when we go to remap it. This will
935          * also ensure that all pending GPU writes are finished
936          * before we unbind.
937          */
938         ret = i915_gem_object_set_domain(obj, I915_GEM_DOMAIN_CPU,
939                                          I915_GEM_DOMAIN_CPU);
940         if (ret) {
941                 DRM_ERROR("set_domain failed: %d\n", ret);
942                 return ret;
943         }
944
945         if (obj_priv->agp_mem != NULL) {
946                 drm_unbind_agp(obj_priv->agp_mem);
947                 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
948                 obj_priv->agp_mem = NULL;
949         }
950
951         BUG_ON(obj_priv->active);
952
953         i915_gem_object_free_page_list(obj);
954
955         if (obj_priv->gtt_space) {
956                 atomic_dec(&dev->gtt_count);
957                 atomic_sub(obj->size, &dev->gtt_memory);
958
959                 drm_mm_put_block(obj_priv->gtt_space);
960                 obj_priv->gtt_space = NULL;
961         }
962
963         /* Remove ourselves from the LRU list if present. */
964         if (!list_empty(&obj_priv->list))
965                 list_del_init(&obj_priv->list);
966
967         return 0;
968 }
969
970 static int
971 i915_gem_evict_something(struct drm_device *dev)
972 {
973         drm_i915_private_t *dev_priv = dev->dev_private;
974         struct drm_gem_object *obj;
975         struct drm_i915_gem_object *obj_priv;
976         int ret = 0;
977
978         for (;;) {
979                 /* If there's an inactive buffer available now, grab it
980                  * and be done.
981                  */
982                 if (!list_empty(&dev_priv->mm.inactive_list)) {
983                         obj_priv = list_first_entry(&dev_priv->mm.inactive_list,
984                                                     struct drm_i915_gem_object,
985                                                     list);
986                         obj = obj_priv->obj;
987                         BUG_ON(obj_priv->pin_count != 0);
988 #if WATCH_LRU
989                         DRM_INFO("%s: evicting %p\n", __func__, obj);
990 #endif
991                         BUG_ON(obj_priv->active);
992
993                         /* Wait on the rendering and unbind the buffer. */
994                         ret = i915_gem_object_unbind(obj);
995                         break;
996                 }
997
998                 /* If we didn't get anything, but the ring is still processing
999                  * things, wait for one of those things to finish and hopefully
1000                  * leave us a buffer to evict.
1001                  */
1002                 if (!list_empty(&dev_priv->mm.request_list)) {
1003                         struct drm_i915_gem_request *request;
1004
1005                         request = list_first_entry(&dev_priv->mm.request_list,
1006                                                    struct drm_i915_gem_request,
1007                                                    list);
1008
1009                         ret = i915_wait_request(dev, request->seqno);
1010                         if (ret)
1011                                 break;
1012
1013                         /* if waiting caused an object to become inactive,
1014                          * then loop around and wait for it. Otherwise, we
1015                          * assume that waiting freed and unbound something,
1016                          * so there should now be some space in the GTT
1017                          */
1018                         if (!list_empty(&dev_priv->mm.inactive_list))
1019                                 continue;
1020                         break;
1021                 }
1022
1023                 /* If we didn't have anything on the request list but there
1024                  * are buffers awaiting a flush, emit one and try again.
1025                  * When we wait on it, those buffers waiting for that flush
1026                  * will get moved to inactive.
1027                  */
1028                 if (!list_empty(&dev_priv->mm.flushing_list)) {
1029                         obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1030                                                     struct drm_i915_gem_object,
1031                                                     list);
1032                         obj = obj_priv->obj;
1033
1034                         i915_gem_flush(dev,
1035                                        obj->write_domain,
1036                                        obj->write_domain);
1037                         i915_add_request(dev, obj->write_domain);
1038
1039                         obj = NULL;
1040                         continue;
1041                 }
1042
1043                 DRM_ERROR("inactive empty %d request empty %d "
1044                           "flushing empty %d\n",
1045                           list_empty(&dev_priv->mm.inactive_list),
1046                           list_empty(&dev_priv->mm.request_list),
1047                           list_empty(&dev_priv->mm.flushing_list));
1048                 /* If we didn't do any of the above, there's nothing to be done
1049                  * and we just can't fit it in.
1050                  */
1051                 return -ENOMEM;
1052         }
1053         return ret;
1054 }
1055
1056 static int
1057 i915_gem_object_get_page_list(struct drm_gem_object *obj)
1058 {
1059         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1060         int page_count, i;
1061         struct address_space *mapping;
1062         struct inode *inode;
1063         struct page *page;
1064         int ret;
1065
1066         if (obj_priv->page_list)
1067                 return 0;
1068
1069         /* Get the list of pages out of our struct file.  They'll be pinned
1070          * at this point until we release them.
1071          */
1072         page_count = obj->size / PAGE_SIZE;
1073         BUG_ON(obj_priv->page_list != NULL);
1074         obj_priv->page_list = drm_calloc(page_count, sizeof(struct page *),
1075                                          DRM_MEM_DRIVER);
1076         if (obj_priv->page_list == NULL) {
1077                 DRM_ERROR("Faled to allocate page list\n");
1078                 return -ENOMEM;
1079         }
1080
1081         inode = obj->filp->f_path.dentry->d_inode;
1082         mapping = inode->i_mapping;
1083         for (i = 0; i < page_count; i++) {
1084                 page = read_mapping_page(mapping, i, NULL);
1085                 if (IS_ERR(page)) {
1086                         ret = PTR_ERR(page);
1087                         DRM_ERROR("read_mapping_page failed: %d\n", ret);
1088                         i915_gem_object_free_page_list(obj);
1089                         return ret;
1090                 }
1091                 obj_priv->page_list[i] = page;
1092         }
1093         return 0;
1094 }
1095
1096 /**
1097  * Finds free space in the GTT aperture and binds the object there.
1098  */
1099 static int
1100 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
1101 {
1102         struct drm_device *dev = obj->dev;
1103         drm_i915_private_t *dev_priv = dev->dev_private;
1104         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1105         struct drm_mm_node *free_space;
1106         int page_count, ret;
1107
1108         if (alignment == 0)
1109                 alignment = PAGE_SIZE;
1110         if (alignment & (PAGE_SIZE - 1)) {
1111                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
1112                 return -EINVAL;
1113         }
1114
1115  search_free:
1116         free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
1117                                         obj->size, alignment, 0);
1118         if (free_space != NULL) {
1119                 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
1120                                                        alignment);
1121                 if (obj_priv->gtt_space != NULL) {
1122                         obj_priv->gtt_space->private = obj;
1123                         obj_priv->gtt_offset = obj_priv->gtt_space->start;
1124                 }
1125         }
1126         if (obj_priv->gtt_space == NULL) {
1127                 /* If the gtt is empty and we're still having trouble
1128                  * fitting our object in, we're out of memory.
1129                  */
1130 #if WATCH_LRU
1131                 DRM_INFO("%s: GTT full, evicting something\n", __func__);
1132 #endif
1133                 if (list_empty(&dev_priv->mm.inactive_list) &&
1134                     list_empty(&dev_priv->mm.flushing_list) &&
1135                     list_empty(&dev_priv->mm.active_list)) {
1136                         DRM_ERROR("GTT full, but LRU list empty\n");
1137                         return -ENOMEM;
1138                 }
1139
1140                 ret = i915_gem_evict_something(dev);
1141                 if (ret != 0) {
1142                         DRM_ERROR("Failed to evict a buffer %d\n", ret);
1143                         return ret;
1144                 }
1145                 goto search_free;
1146         }
1147
1148 #if WATCH_BUF
1149         DRM_INFO("Binding object of size %d at 0x%08x\n",
1150                  obj->size, obj_priv->gtt_offset);
1151 #endif
1152         ret = i915_gem_object_get_page_list(obj);
1153         if (ret) {
1154                 drm_mm_put_block(obj_priv->gtt_space);
1155                 obj_priv->gtt_space = NULL;
1156                 return ret;
1157         }
1158
1159         page_count = obj->size / PAGE_SIZE;
1160         /* Create an AGP memory structure pointing at our pages, and bind it
1161          * into the GTT.
1162          */
1163         obj_priv->agp_mem = drm_agp_bind_pages(dev,
1164                                                obj_priv->page_list,
1165                                                page_count,
1166                                                obj_priv->gtt_offset,
1167                                                obj_priv->agp_type);
1168         if (obj_priv->agp_mem == NULL) {
1169                 i915_gem_object_free_page_list(obj);
1170                 drm_mm_put_block(obj_priv->gtt_space);
1171                 obj_priv->gtt_space = NULL;
1172                 return -ENOMEM;
1173         }
1174         atomic_inc(&dev->gtt_count);
1175         atomic_add(obj->size, &dev->gtt_memory);
1176
1177         /* Assert that the object is not currently in any GPU domain. As it
1178          * wasn't in the GTT, there shouldn't be any way it could have been in
1179          * a GPU cache
1180          */
1181         BUG_ON(obj->read_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
1182         BUG_ON(obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
1183
1184         return 0;
1185 }
1186
1187 void
1188 i915_gem_clflush_object(struct drm_gem_object *obj)
1189 {
1190         struct drm_i915_gem_object      *obj_priv = obj->driver_private;
1191
1192         /* If we don't have a page list set up, then we're not pinned
1193          * to GPU, and we can ignore the cache flush because it'll happen
1194          * again at bind time.
1195          */
1196         if (obj_priv->page_list == NULL)
1197                 return;
1198
1199         drm_clflush_pages(obj_priv->page_list, obj->size / PAGE_SIZE);
1200 }
1201
1202 /*
1203  * Set the next domain for the specified object. This
1204  * may not actually perform the necessary flushing/invaliding though,
1205  * as that may want to be batched with other set_domain operations
1206  *
1207  * This is (we hope) the only really tricky part of gem. The goal
1208  * is fairly simple -- track which caches hold bits of the object
1209  * and make sure they remain coherent. A few concrete examples may
1210  * help to explain how it works. For shorthand, we use the notation
1211  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
1212  * a pair of read and write domain masks.
1213  *
1214  * Case 1: the batch buffer
1215  *
1216  *      1. Allocated
1217  *      2. Written by CPU
1218  *      3. Mapped to GTT
1219  *      4. Read by GPU
1220  *      5. Unmapped from GTT
1221  *      6. Freed
1222  *
1223  *      Let's take these a step at a time
1224  *
1225  *      1. Allocated
1226  *              Pages allocated from the kernel may still have
1227  *              cache contents, so we set them to (CPU, CPU) always.
1228  *      2. Written by CPU (using pwrite)
1229  *              The pwrite function calls set_domain (CPU, CPU) and
1230  *              this function does nothing (as nothing changes)
1231  *      3. Mapped by GTT
1232  *              This function asserts that the object is not
1233  *              currently in any GPU-based read or write domains
1234  *      4. Read by GPU
1235  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
1236  *              As write_domain is zero, this function adds in the
1237  *              current read domains (CPU+COMMAND, 0).
1238  *              flush_domains is set to CPU.
1239  *              invalidate_domains is set to COMMAND
1240  *              clflush is run to get data out of the CPU caches
1241  *              then i915_dev_set_domain calls i915_gem_flush to
1242  *              emit an MI_FLUSH and drm_agp_chipset_flush
1243  *      5. Unmapped from GTT
1244  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
1245  *              flush_domains and invalidate_domains end up both zero
1246  *              so no flushing/invalidating happens
1247  *      6. Freed
1248  *              yay, done
1249  *
1250  * Case 2: The shared render buffer
1251  *
1252  *      1. Allocated
1253  *      2. Mapped to GTT
1254  *      3. Read/written by GPU
1255  *      4. set_domain to (CPU,CPU)
1256  *      5. Read/written by CPU
1257  *      6. Read/written by GPU
1258  *
1259  *      1. Allocated
1260  *              Same as last example, (CPU, CPU)
1261  *      2. Mapped to GTT
1262  *              Nothing changes (assertions find that it is not in the GPU)
1263  *      3. Read/written by GPU
1264  *              execbuffer calls set_domain (RENDER, RENDER)
1265  *              flush_domains gets CPU
1266  *              invalidate_domains gets GPU
1267  *              clflush (obj)
1268  *              MI_FLUSH and drm_agp_chipset_flush
1269  *      4. set_domain (CPU, CPU)
1270  *              flush_domains gets GPU
1271  *              invalidate_domains gets CPU
1272  *              wait_rendering (obj) to make sure all drawing is complete.
1273  *              This will include an MI_FLUSH to get the data from GPU
1274  *              to memory
1275  *              clflush (obj) to invalidate the CPU cache
1276  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
1277  *      5. Read/written by CPU
1278  *              cache lines are loaded and dirtied
1279  *      6. Read written by GPU
1280  *              Same as last GPU access
1281  *
1282  * Case 3: The constant buffer
1283  *
1284  *      1. Allocated
1285  *      2. Written by CPU
1286  *      3. Read by GPU
1287  *      4. Updated (written) by CPU again
1288  *      5. Read by GPU
1289  *
1290  *      1. Allocated
1291  *              (CPU, CPU)
1292  *      2. Written by CPU
1293  *              (CPU, CPU)
1294  *      3. Read by GPU
1295  *              (CPU+RENDER, 0)
1296  *              flush_domains = CPU
1297  *              invalidate_domains = RENDER
1298  *              clflush (obj)
1299  *              MI_FLUSH
1300  *              drm_agp_chipset_flush
1301  *      4. Updated (written) by CPU again
1302  *              (CPU, CPU)
1303  *              flush_domains = 0 (no previous write domain)
1304  *              invalidate_domains = 0 (no new read domains)
1305  *      5. Read by GPU
1306  *              (CPU+RENDER, 0)
1307  *              flush_domains = CPU
1308  *              invalidate_domains = RENDER
1309  *              clflush (obj)
1310  *              MI_FLUSH
1311  *              drm_agp_chipset_flush
1312  */
1313 static int
1314 i915_gem_object_set_domain(struct drm_gem_object *obj,
1315                             uint32_t read_domains,
1316                             uint32_t write_domain)
1317 {
1318         struct drm_device               *dev = obj->dev;
1319         struct drm_i915_gem_object      *obj_priv = obj->driver_private;
1320         uint32_t                        invalidate_domains = 0;
1321         uint32_t                        flush_domains = 0;
1322         int                             ret;
1323
1324 #if WATCH_BUF
1325         DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
1326                  __func__, obj,
1327                  obj->read_domains, read_domains,
1328                  obj->write_domain, write_domain);
1329 #endif
1330         /*
1331          * If the object isn't moving to a new write domain,
1332          * let the object stay in multiple read domains
1333          */
1334         if (write_domain == 0)
1335                 read_domains |= obj->read_domains;
1336         else
1337                 obj_priv->dirty = 1;
1338
1339         /*
1340          * Flush the current write domain if
1341          * the new read domains don't match. Invalidate
1342          * any read domains which differ from the old
1343          * write domain
1344          */
1345         if (obj->write_domain && obj->write_domain != read_domains) {
1346                 flush_domains |= obj->write_domain;
1347                 invalidate_domains |= read_domains & ~obj->write_domain;
1348         }
1349         /*
1350          * Invalidate any read caches which may have
1351          * stale data. That is, any new read domains.
1352          */
1353         invalidate_domains |= read_domains & ~obj->read_domains;
1354         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
1355 #if WATCH_BUF
1356                 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
1357                          __func__, flush_domains, invalidate_domains);
1358 #endif
1359                 /*
1360                  * If we're invaliding the CPU cache and flushing a GPU cache,
1361                  * then pause for rendering so that the GPU caches will be
1362                  * flushed before the cpu cache is invalidated
1363                  */
1364                 if ((invalidate_domains & I915_GEM_DOMAIN_CPU) &&
1365                     (flush_domains & ~(I915_GEM_DOMAIN_CPU |
1366                                        I915_GEM_DOMAIN_GTT))) {
1367                         ret = i915_gem_object_wait_rendering(obj);
1368                         if (ret)
1369                                 return ret;
1370                 }
1371                 i915_gem_clflush_object(obj);
1372         }
1373
1374         if ((write_domain | flush_domains) != 0)
1375                 obj->write_domain = write_domain;
1376
1377         /* If we're invalidating the CPU domain, clear the per-page CPU
1378          * domain list as well.
1379          */
1380         if (obj_priv->page_cpu_valid != NULL &&
1381             (write_domain != 0 ||
1382              read_domains & I915_GEM_DOMAIN_CPU)) {
1383                 drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE,
1384                          DRM_MEM_DRIVER);
1385                 obj_priv->page_cpu_valid = NULL;
1386         }
1387         obj->read_domains = read_domains;
1388
1389         dev->invalidate_domains |= invalidate_domains;
1390         dev->flush_domains |= flush_domains;
1391 #if WATCH_BUF
1392         DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
1393                  __func__,
1394                  obj->read_domains, obj->write_domain,
1395                  dev->invalidate_domains, dev->flush_domains);
1396 #endif
1397         return 0;
1398 }
1399
1400 /**
1401  * Set the read/write domain on a range of the object.
1402  *
1403  * Currently only implemented for CPU reads, otherwise drops to normal
1404  * i915_gem_object_set_domain().
1405  */
1406 static int
1407 i915_gem_object_set_domain_range(struct drm_gem_object *obj,
1408                                  uint64_t offset,
1409                                  uint64_t size,
1410                                  uint32_t read_domains,
1411                                  uint32_t write_domain)
1412 {
1413         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1414         int ret, i;
1415
1416         if (obj->read_domains & I915_GEM_DOMAIN_CPU)
1417                 return 0;
1418
1419         if (read_domains != I915_GEM_DOMAIN_CPU ||
1420             write_domain != 0)
1421                 return i915_gem_object_set_domain(obj,
1422                                                   read_domains, write_domain);
1423
1424         /* Wait on any GPU rendering to the object to be flushed. */
1425         if (obj->write_domain & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT)) {
1426                 ret = i915_gem_object_wait_rendering(obj);
1427                 if (ret)
1428                         return ret;
1429         }
1430
1431         if (obj_priv->page_cpu_valid == NULL) {
1432                 obj_priv->page_cpu_valid = drm_calloc(1, obj->size / PAGE_SIZE,
1433                                                       DRM_MEM_DRIVER);
1434         }
1435
1436         /* Flush the cache on any pages that are still invalid from the CPU's
1437          * perspective.
1438          */
1439         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; i++) {
1440                 if (obj_priv->page_cpu_valid[i])
1441                         continue;
1442
1443                 drm_clflush_pages(obj_priv->page_list + i, 1);
1444
1445                 obj_priv->page_cpu_valid[i] = 1;
1446         }
1447
1448         return 0;
1449 }
1450
1451 /**
1452  * Once all of the objects have been set in the proper domain,
1453  * perform the necessary flush and invalidate operations.
1454  *
1455  * Returns the write domains flushed, for use in flush tracking.
1456  */
1457 static uint32_t
1458 i915_gem_dev_set_domain(struct drm_device *dev)
1459 {
1460         uint32_t flush_domains = dev->flush_domains;
1461
1462         /*
1463          * Now that all the buffers are synced to the proper domains,
1464          * flush and invalidate the collected domains
1465          */
1466         if (dev->invalidate_domains | dev->flush_domains) {
1467 #if WATCH_EXEC
1468                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
1469                           __func__,
1470                          dev->invalidate_domains,
1471                          dev->flush_domains);
1472 #endif
1473                 i915_gem_flush(dev,
1474                                dev->invalidate_domains,
1475                                dev->flush_domains);
1476                 dev->invalidate_domains = 0;
1477                 dev->flush_domains = 0;
1478         }
1479
1480         return flush_domains;
1481 }
1482
1483 /**
1484  * Pin an object to the GTT and evaluate the relocations landing in it.
1485  */
1486 static int
1487 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
1488                                  struct drm_file *file_priv,
1489                                  struct drm_i915_gem_exec_object *entry)
1490 {
1491         struct drm_device *dev = obj->dev;
1492         struct drm_i915_gem_relocation_entry reloc;
1493         struct drm_i915_gem_relocation_entry __user *relocs;
1494         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1495         int i, ret;
1496         uint32_t last_reloc_offset = -1;
1497         void __iomem *reloc_page = NULL;
1498
1499         /* Choose the GTT offset for our buffer and put it there. */
1500         ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
1501         if (ret)
1502                 return ret;
1503
1504         entry->offset = obj_priv->gtt_offset;
1505
1506         relocs = (struct drm_i915_gem_relocation_entry __user *)
1507                  (uintptr_t) entry->relocs_ptr;
1508         /* Apply the relocations, using the GTT aperture to avoid cache
1509          * flushing requirements.
1510          */
1511         for (i = 0; i < entry->relocation_count; i++) {
1512                 struct drm_gem_object *target_obj;
1513                 struct drm_i915_gem_object *target_obj_priv;
1514                 uint32_t reloc_val, reloc_offset;
1515                 uint32_t __iomem *reloc_entry;
1516
1517                 ret = copy_from_user(&reloc, relocs + i, sizeof(reloc));
1518                 if (ret != 0) {
1519                         i915_gem_object_unpin(obj);
1520                         return ret;
1521                 }
1522
1523                 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
1524                                                    reloc.target_handle);
1525                 if (target_obj == NULL) {
1526                         i915_gem_object_unpin(obj);
1527                         return -EBADF;
1528                 }
1529                 target_obj_priv = target_obj->driver_private;
1530
1531                 /* The target buffer should have appeared before us in the
1532                  * exec_object list, so it should have a GTT space bound by now.
1533                  */
1534                 if (target_obj_priv->gtt_space == NULL) {
1535                         DRM_ERROR("No GTT space found for object %d\n",
1536                                   reloc.target_handle);
1537                         drm_gem_object_unreference(target_obj);
1538                         i915_gem_object_unpin(obj);
1539                         return -EINVAL;
1540                 }
1541
1542                 if (reloc.offset > obj->size - 4) {
1543                         DRM_ERROR("Relocation beyond object bounds: "
1544                                   "obj %p target %d offset %d size %d.\n",
1545                                   obj, reloc.target_handle,
1546                                   (int) reloc.offset, (int) obj->size);
1547                         drm_gem_object_unreference(target_obj);
1548                         i915_gem_object_unpin(obj);
1549                         return -EINVAL;
1550                 }
1551                 if (reloc.offset & 3) {
1552                         DRM_ERROR("Relocation not 4-byte aligned: "
1553                                   "obj %p target %d offset %d.\n",
1554                                   obj, reloc.target_handle,
1555                                   (int) reloc.offset);
1556                         drm_gem_object_unreference(target_obj);
1557                         i915_gem_object_unpin(obj);
1558                         return -EINVAL;
1559                 }
1560
1561                 if (reloc.write_domain && target_obj->pending_write_domain &&
1562                     reloc.write_domain != target_obj->pending_write_domain) {
1563                         DRM_ERROR("Write domain conflict: "
1564                                   "obj %p target %d offset %d "
1565                                   "new %08x old %08x\n",
1566                                   obj, reloc.target_handle,
1567                                   (int) reloc.offset,
1568                                   reloc.write_domain,
1569                                   target_obj->pending_write_domain);
1570                         drm_gem_object_unreference(target_obj);
1571                         i915_gem_object_unpin(obj);
1572                         return -EINVAL;
1573                 }
1574
1575 #if WATCH_RELOC
1576                 DRM_INFO("%s: obj %p offset %08x target %d "
1577                          "read %08x write %08x gtt %08x "
1578                          "presumed %08x delta %08x\n",
1579                          __func__,
1580                          obj,
1581                          (int) reloc.offset,
1582                          (int) reloc.target_handle,
1583                          (int) reloc.read_domains,
1584                          (int) reloc.write_domain,
1585                          (int) target_obj_priv->gtt_offset,
1586                          (int) reloc.presumed_offset,
1587                          reloc.delta);
1588 #endif
1589
1590                 target_obj->pending_read_domains |= reloc.read_domains;
1591                 target_obj->pending_write_domain |= reloc.write_domain;
1592
1593                 /* If the relocation already has the right value in it, no
1594                  * more work needs to be done.
1595                  */
1596                 if (target_obj_priv->gtt_offset == reloc.presumed_offset) {
1597                         drm_gem_object_unreference(target_obj);
1598                         continue;
1599                 }
1600
1601                 /* Now that we're going to actually write some data in,
1602                  * make sure that any rendering using this buffer's contents
1603                  * is completed.
1604                  */
1605                 i915_gem_object_wait_rendering(obj);
1606
1607                 /* As we're writing through the gtt, flush
1608                  * any CPU writes before we write the relocations
1609                  */
1610                 if (obj->write_domain & I915_GEM_DOMAIN_CPU) {
1611                         i915_gem_clflush_object(obj);
1612                         drm_agp_chipset_flush(dev);
1613                         obj->write_domain = 0;
1614                 }
1615
1616                 /* Map the page containing the relocation we're going to
1617                  * perform.
1618                  */
1619                 reloc_offset = obj_priv->gtt_offset + reloc.offset;
1620                 if (reloc_page == NULL ||
1621                     (last_reloc_offset & ~(PAGE_SIZE - 1)) !=
1622                     (reloc_offset & ~(PAGE_SIZE - 1))) {
1623                         if (reloc_page != NULL)
1624                                 iounmap(reloc_page);
1625
1626                         reloc_page = ioremap_wc(dev->agp->base +
1627                                                 (reloc_offset &
1628                                                  ~(PAGE_SIZE - 1)),
1629                                                 PAGE_SIZE);
1630                         last_reloc_offset = reloc_offset;
1631                         if (reloc_page == NULL) {
1632                                 drm_gem_object_unreference(target_obj);
1633                                 i915_gem_object_unpin(obj);
1634                                 return -ENOMEM;
1635                         }
1636                 }
1637
1638                 reloc_entry = (uint32_t __iomem *)(reloc_page +
1639                                            (reloc_offset & (PAGE_SIZE - 1)));
1640                 reloc_val = target_obj_priv->gtt_offset + reloc.delta;
1641
1642 #if WATCH_BUF
1643                 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
1644                           obj, (unsigned int) reloc.offset,
1645                           readl(reloc_entry), reloc_val);
1646 #endif
1647                 writel(reloc_val, reloc_entry);
1648
1649                 /* Write the updated presumed offset for this entry back out
1650                  * to the user.
1651                  */
1652                 reloc.presumed_offset = target_obj_priv->gtt_offset;
1653                 ret = copy_to_user(relocs + i, &reloc, sizeof(reloc));
1654                 if (ret != 0) {
1655                         drm_gem_object_unreference(target_obj);
1656                         i915_gem_object_unpin(obj);
1657                         return ret;
1658                 }
1659
1660                 drm_gem_object_unreference(target_obj);
1661         }
1662
1663         if (reloc_page != NULL)
1664                 iounmap(reloc_page);
1665
1666 #if WATCH_BUF
1667         if (0)
1668                 i915_gem_dump_object(obj, 128, __func__, ~0);
1669 #endif
1670         return 0;
1671 }
1672
1673 /** Dispatch a batchbuffer to the ring
1674  */
1675 static int
1676 i915_dispatch_gem_execbuffer(struct drm_device *dev,
1677                               struct drm_i915_gem_execbuffer *exec,
1678                               uint64_t exec_offset)
1679 {
1680         drm_i915_private_t *dev_priv = dev->dev_private;
1681         struct drm_clip_rect __user *boxes = (struct drm_clip_rect __user *)
1682                                              (uintptr_t) exec->cliprects_ptr;
1683         int nbox = exec->num_cliprects;
1684         int i = 0, count;
1685         uint32_t        exec_start, exec_len;
1686         RING_LOCALS;
1687
1688         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
1689         exec_len = (uint32_t) exec->batch_len;
1690
1691         if ((exec_start | exec_len) & 0x7) {
1692                 DRM_ERROR("alignment\n");
1693                 return -EINVAL;
1694         }
1695
1696         if (!exec_start)
1697                 return -EINVAL;
1698
1699         count = nbox ? nbox : 1;
1700
1701         for (i = 0; i < count; i++) {
1702                 if (i < nbox) {
1703                         int ret = i915_emit_box(dev, boxes, i,
1704                                                 exec->DR1, exec->DR4);
1705                         if (ret)
1706                                 return ret;
1707                 }
1708
1709                 if (IS_I830(dev) || IS_845G(dev)) {
1710                         BEGIN_LP_RING(4);
1711                         OUT_RING(MI_BATCH_BUFFER);
1712                         OUT_RING(exec_start | MI_BATCH_NON_SECURE);
1713                         OUT_RING(exec_start + exec_len - 4);
1714                         OUT_RING(0);
1715                         ADVANCE_LP_RING();
1716                 } else {
1717                         BEGIN_LP_RING(2);
1718                         if (IS_I965G(dev)) {
1719                                 OUT_RING(MI_BATCH_BUFFER_START |
1720                                          (2 << 6) |
1721                                          MI_BATCH_NON_SECURE_I965);
1722                                 OUT_RING(exec_start);
1723                         } else {
1724                                 OUT_RING(MI_BATCH_BUFFER_START |
1725                                          (2 << 6));
1726                                 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
1727                         }
1728                         ADVANCE_LP_RING();
1729                 }
1730         }
1731
1732         /* XXX breadcrumb */
1733         return 0;
1734 }
1735
1736 /* Throttle our rendering by waiting until the ring has completed our requests
1737  * emitted over 20 msec ago.
1738  *
1739  * This should get us reasonable parallelism between CPU and GPU but also
1740  * relatively low latency when blocking on a particular request to finish.
1741  */
1742 static int
1743 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
1744 {
1745         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
1746         int ret = 0;
1747         uint32_t seqno;
1748
1749         mutex_lock(&dev->struct_mutex);
1750         seqno = i915_file_priv->mm.last_gem_throttle_seqno;
1751         i915_file_priv->mm.last_gem_throttle_seqno =
1752                 i915_file_priv->mm.last_gem_seqno;
1753         if (seqno)
1754                 ret = i915_wait_request(dev, seqno);
1755         mutex_unlock(&dev->struct_mutex);
1756         return ret;
1757 }
1758
1759 int
1760 i915_gem_execbuffer(struct drm_device *dev, void *data,
1761                     struct drm_file *file_priv)
1762 {
1763         drm_i915_private_t *dev_priv = dev->dev_private;
1764         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
1765         struct drm_i915_gem_execbuffer *args = data;
1766         struct drm_i915_gem_exec_object *exec_list = NULL;
1767         struct drm_gem_object **object_list = NULL;
1768         struct drm_gem_object *batch_obj;
1769         int ret, i, pinned = 0;
1770         uint64_t exec_offset;
1771         uint32_t seqno, flush_domains;
1772
1773 #if WATCH_EXEC
1774         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
1775                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
1776 #endif
1777
1778         if (args->buffer_count < 1) {
1779                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
1780                 return -EINVAL;
1781         }
1782         /* Copy in the exec list from userland */
1783         exec_list = drm_calloc(sizeof(*exec_list), args->buffer_count,
1784                                DRM_MEM_DRIVER);
1785         object_list = drm_calloc(sizeof(*object_list), args->buffer_count,
1786                                  DRM_MEM_DRIVER);
1787         if (exec_list == NULL || object_list == NULL) {
1788                 DRM_ERROR("Failed to allocate exec or object list "
1789                           "for %d buffers\n",
1790                           args->buffer_count);
1791                 ret = -ENOMEM;
1792                 goto pre_mutex_err;
1793         }
1794         ret = copy_from_user(exec_list,
1795                              (struct drm_i915_relocation_entry __user *)
1796                              (uintptr_t) args->buffers_ptr,
1797                              sizeof(*exec_list) * args->buffer_count);
1798         if (ret != 0) {
1799                 DRM_ERROR("copy %d exec entries failed %d\n",
1800                           args->buffer_count, ret);
1801                 goto pre_mutex_err;
1802         }
1803
1804         mutex_lock(&dev->struct_mutex);
1805
1806         i915_verify_inactive(dev, __FILE__, __LINE__);
1807
1808         if (dev_priv->mm.wedged) {
1809                 DRM_ERROR("Execbuf while wedged\n");
1810                 mutex_unlock(&dev->struct_mutex);
1811                 return -EIO;
1812         }
1813
1814         if (dev_priv->mm.suspended) {
1815                 DRM_ERROR("Execbuf while VT-switched.\n");
1816                 mutex_unlock(&dev->struct_mutex);
1817                 return -EBUSY;
1818         }
1819
1820         /* Zero the gloabl flush/invalidate flags. These
1821          * will be modified as each object is bound to the
1822          * gtt
1823          */
1824         dev->invalidate_domains = 0;
1825         dev->flush_domains = 0;
1826
1827         /* Look up object handles and perform the relocations */
1828         for (i = 0; i < args->buffer_count; i++) {
1829                 object_list[i] = drm_gem_object_lookup(dev, file_priv,
1830                                                        exec_list[i].handle);
1831                 if (object_list[i] == NULL) {
1832                         DRM_ERROR("Invalid object handle %d at index %d\n",
1833                                    exec_list[i].handle, i);
1834                         ret = -EBADF;
1835                         goto err;
1836                 }
1837
1838                 object_list[i]->pending_read_domains = 0;
1839                 object_list[i]->pending_write_domain = 0;
1840                 ret = i915_gem_object_pin_and_relocate(object_list[i],
1841                                                        file_priv,
1842                                                        &exec_list[i]);
1843                 if (ret) {
1844                         DRM_ERROR("object bind and relocate failed %d\n", ret);
1845                         goto err;
1846                 }
1847                 pinned = i + 1;
1848         }
1849
1850         /* Set the pending read domains for the batch buffer to COMMAND */
1851         batch_obj = object_list[args->buffer_count-1];
1852         batch_obj->pending_read_domains = I915_GEM_DOMAIN_COMMAND;
1853         batch_obj->pending_write_domain = 0;
1854
1855         i915_verify_inactive(dev, __FILE__, __LINE__);
1856
1857         for (i = 0; i < args->buffer_count; i++) {
1858                 struct drm_gem_object *obj = object_list[i];
1859                 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1860
1861                 if (obj_priv->gtt_space == NULL) {
1862                         /* We evicted the buffer in the process of validating
1863                          * our set of buffers in.  We could try to recover by
1864                          * kicking them everything out and trying again from
1865                          * the start.
1866                          */
1867                         ret = -ENOMEM;
1868                         goto err;
1869                 }
1870
1871                 /* make sure all previous memory operations have passed */
1872                 ret = i915_gem_object_set_domain(obj,
1873                                                  obj->pending_read_domains,
1874                                                  obj->pending_write_domain);
1875                 if (ret)
1876                         goto err;
1877         }
1878
1879         i915_verify_inactive(dev, __FILE__, __LINE__);
1880
1881         /* Flush/invalidate caches and chipset buffer */
1882         flush_domains = i915_gem_dev_set_domain(dev);
1883
1884         i915_verify_inactive(dev, __FILE__, __LINE__);
1885
1886 #if WATCH_COHERENCY
1887         for (i = 0; i < args->buffer_count; i++) {
1888                 i915_gem_object_check_coherency(object_list[i],
1889                                                 exec_list[i].handle);
1890         }
1891 #endif
1892
1893         exec_offset = exec_list[args->buffer_count - 1].offset;
1894
1895 #if WATCH_EXEC
1896         i915_gem_dump_object(object_list[args->buffer_count - 1],
1897                               args->batch_len,
1898                               __func__,
1899                               ~0);
1900 #endif
1901
1902         (void)i915_add_request(dev, flush_domains);
1903
1904         /* Exec the batchbuffer */
1905         ret = i915_dispatch_gem_execbuffer(dev, args, exec_offset);
1906         if (ret) {
1907                 DRM_ERROR("dispatch failed %d\n", ret);
1908                 goto err;
1909         }
1910
1911         /*
1912          * Ensure that the commands in the batch buffer are
1913          * finished before the interrupt fires
1914          */
1915         flush_domains = i915_retire_commands(dev);
1916
1917         i915_verify_inactive(dev, __FILE__, __LINE__);
1918
1919         /*
1920          * Get a seqno representing the execution of the current buffer,
1921          * which we can wait on.  We would like to mitigate these interrupts,
1922          * likely by only creating seqnos occasionally (so that we have
1923          * *some* interrupts representing completion of buffers that we can
1924          * wait on when trying to clear up gtt space).
1925          */
1926         seqno = i915_add_request(dev, flush_domains);
1927         BUG_ON(seqno == 0);
1928         i915_file_priv->mm.last_gem_seqno = seqno;
1929         for (i = 0; i < args->buffer_count; i++) {
1930                 struct drm_gem_object *obj = object_list[i];
1931                 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1932
1933                 i915_gem_object_move_to_active(obj);
1934                 obj_priv->last_rendering_seqno = seqno;
1935 #if WATCH_LRU
1936                 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
1937 #endif
1938         }
1939 #if WATCH_LRU
1940         i915_dump_lru(dev, __func__);
1941 #endif
1942
1943         i915_verify_inactive(dev, __FILE__, __LINE__);
1944
1945         /* Copy the new buffer offsets back to the user's exec list. */
1946         ret = copy_to_user((struct drm_i915_relocation_entry __user *)
1947                            (uintptr_t) args->buffers_ptr,
1948                            exec_list,
1949                            sizeof(*exec_list) * args->buffer_count);
1950         if (ret)
1951                 DRM_ERROR("failed to copy %d exec entries "
1952                           "back to user (%d)\n",
1953                            args->buffer_count, ret);
1954 err:
1955         if (object_list != NULL) {
1956                 for (i = 0; i < pinned; i++)
1957                         i915_gem_object_unpin(object_list[i]);
1958
1959                 for (i = 0; i < args->buffer_count; i++)
1960                         drm_gem_object_unreference(object_list[i]);
1961         }
1962         mutex_unlock(&dev->struct_mutex);
1963
1964 pre_mutex_err:
1965         drm_free(object_list, sizeof(*object_list) * args->buffer_count,
1966                  DRM_MEM_DRIVER);
1967         drm_free(exec_list, sizeof(*exec_list) * args->buffer_count,
1968                  DRM_MEM_DRIVER);
1969
1970         return ret;
1971 }
1972
1973 int
1974 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
1975 {
1976         struct drm_device *dev = obj->dev;
1977         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1978         int ret;
1979
1980         i915_verify_inactive(dev, __FILE__, __LINE__);
1981         if (obj_priv->gtt_space == NULL) {
1982                 ret = i915_gem_object_bind_to_gtt(obj, alignment);
1983                 if (ret != 0) {
1984                         DRM_ERROR("Failure to bind: %d", ret);
1985                         return ret;
1986                 }
1987         }
1988         obj_priv->pin_count++;
1989
1990         /* If the object is not active and not pending a flush,
1991          * remove it from the inactive list
1992          */
1993         if (obj_priv->pin_count == 1) {
1994                 atomic_inc(&dev->pin_count);
1995                 atomic_add(obj->size, &dev->pin_memory);
1996                 if (!obj_priv->active &&
1997                     (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
1998                                            I915_GEM_DOMAIN_GTT)) == 0 &&
1999                     !list_empty(&obj_priv->list))
2000                         list_del_init(&obj_priv->list);
2001         }
2002         i915_verify_inactive(dev, __FILE__, __LINE__);
2003
2004         return 0;
2005 }
2006
2007 void
2008 i915_gem_object_unpin(struct drm_gem_object *obj)
2009 {
2010         struct drm_device *dev = obj->dev;
2011         drm_i915_private_t *dev_priv = dev->dev_private;
2012         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2013
2014         i915_verify_inactive(dev, __FILE__, __LINE__);
2015         obj_priv->pin_count--;
2016         BUG_ON(obj_priv->pin_count < 0);
2017         BUG_ON(obj_priv->gtt_space == NULL);
2018
2019         /* If the object is no longer pinned, and is
2020          * neither active nor being flushed, then stick it on
2021          * the inactive list
2022          */
2023         if (obj_priv->pin_count == 0) {
2024                 if (!obj_priv->active &&
2025                     (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
2026                                            I915_GEM_DOMAIN_GTT)) == 0)
2027                         list_move_tail(&obj_priv->list,
2028                                        &dev_priv->mm.inactive_list);
2029                 atomic_dec(&dev->pin_count);
2030                 atomic_sub(obj->size, &dev->pin_memory);
2031         }
2032         i915_verify_inactive(dev, __FILE__, __LINE__);
2033 }
2034
2035 int
2036 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
2037                    struct drm_file *file_priv)
2038 {
2039         struct drm_i915_gem_pin *args = data;
2040         struct drm_gem_object *obj;
2041         struct drm_i915_gem_object *obj_priv;
2042         int ret;
2043
2044         mutex_lock(&dev->struct_mutex);
2045
2046         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
2047         if (obj == NULL) {
2048                 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
2049                           args->handle);
2050                 mutex_unlock(&dev->struct_mutex);
2051                 return -EBADF;
2052         }
2053         obj_priv = obj->driver_private;
2054
2055         ret = i915_gem_object_pin(obj, args->alignment);
2056         if (ret != 0) {
2057                 drm_gem_object_unreference(obj);
2058                 mutex_unlock(&dev->struct_mutex);
2059                 return ret;
2060         }
2061
2062         /* XXX - flush the CPU caches for pinned objects
2063          * as the X server doesn't manage domains yet
2064          */
2065         if (obj->write_domain & I915_GEM_DOMAIN_CPU) {
2066                 i915_gem_clflush_object(obj);
2067                 drm_agp_chipset_flush(dev);
2068                 obj->write_domain = 0;
2069         }
2070         args->offset = obj_priv->gtt_offset;
2071         drm_gem_object_unreference(obj);
2072         mutex_unlock(&dev->struct_mutex);
2073
2074         return 0;
2075 }
2076
2077 int
2078 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
2079                      struct drm_file *file_priv)
2080 {
2081         struct drm_i915_gem_pin *args = data;
2082         struct drm_gem_object *obj;
2083
2084         mutex_lock(&dev->struct_mutex);
2085
2086         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
2087         if (obj == NULL) {
2088                 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
2089                           args->handle);
2090                 mutex_unlock(&dev->struct_mutex);
2091                 return -EBADF;
2092         }
2093
2094         i915_gem_object_unpin(obj);
2095
2096         drm_gem_object_unreference(obj);
2097         mutex_unlock(&dev->struct_mutex);
2098         return 0;
2099 }
2100
2101 int
2102 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
2103                     struct drm_file *file_priv)
2104 {
2105         struct drm_i915_gem_busy *args = data;
2106         struct drm_gem_object *obj;
2107         struct drm_i915_gem_object *obj_priv;
2108
2109         mutex_lock(&dev->struct_mutex);
2110         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
2111         if (obj == NULL) {
2112                 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
2113                           args->handle);
2114                 mutex_unlock(&dev->struct_mutex);
2115                 return -EBADF;
2116         }
2117
2118         obj_priv = obj->driver_private;
2119         args->busy = obj_priv->active;
2120
2121         drm_gem_object_unreference(obj);
2122         mutex_unlock(&dev->struct_mutex);
2123         return 0;
2124 }
2125
2126 int
2127 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
2128                         struct drm_file *file_priv)
2129 {
2130     return i915_gem_ring_throttle(dev, file_priv);
2131 }
2132
2133 int i915_gem_init_object(struct drm_gem_object *obj)
2134 {
2135         struct drm_i915_gem_object *obj_priv;
2136
2137         obj_priv = drm_calloc(1, sizeof(*obj_priv), DRM_MEM_DRIVER);
2138         if (obj_priv == NULL)
2139                 return -ENOMEM;
2140
2141         /*
2142          * We've just allocated pages from the kernel,
2143          * so they've just been written by the CPU with
2144          * zeros. They'll need to be clflushed before we
2145          * use them with the GPU.
2146          */
2147         obj->write_domain = I915_GEM_DOMAIN_CPU;
2148         obj->read_domains = I915_GEM_DOMAIN_CPU;
2149
2150         obj_priv->agp_type = AGP_USER_MEMORY;
2151
2152         obj->driver_private = obj_priv;
2153         obj_priv->obj = obj;
2154         INIT_LIST_HEAD(&obj_priv->list);
2155         return 0;
2156 }
2157
2158 void i915_gem_free_object(struct drm_gem_object *obj)
2159 {
2160         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2161
2162         while (obj_priv->pin_count > 0)
2163                 i915_gem_object_unpin(obj);
2164
2165         i915_gem_object_unbind(obj);
2166
2167         drm_free(obj_priv->page_cpu_valid, 1, DRM_MEM_DRIVER);
2168         drm_free(obj->driver_private, 1, DRM_MEM_DRIVER);
2169 }
2170
2171 static int
2172 i915_gem_set_domain(struct drm_gem_object *obj,
2173                     struct drm_file *file_priv,
2174                     uint32_t read_domains,
2175                     uint32_t write_domain)
2176 {
2177         struct drm_device *dev = obj->dev;
2178         int ret;
2179         uint32_t flush_domains;
2180
2181         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
2182
2183         ret = i915_gem_object_set_domain(obj, read_domains, write_domain);
2184         if (ret)
2185                 return ret;
2186         flush_domains = i915_gem_dev_set_domain(obj->dev);
2187
2188         if (flush_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT))
2189                 (void) i915_add_request(dev, flush_domains);
2190
2191         return 0;
2192 }
2193
2194 /** Unbinds all objects that are on the given buffer list. */
2195 static int
2196 i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head)
2197 {
2198         struct drm_gem_object *obj;
2199         struct drm_i915_gem_object *obj_priv;
2200         int ret;
2201
2202         while (!list_empty(head)) {
2203                 obj_priv = list_first_entry(head,
2204                                             struct drm_i915_gem_object,
2205                                             list);
2206                 obj = obj_priv->obj;
2207
2208                 if (obj_priv->pin_count != 0) {
2209                         DRM_ERROR("Pinned object in unbind list\n");
2210                         mutex_unlock(&dev->struct_mutex);
2211                         return -EINVAL;
2212                 }
2213
2214                 ret = i915_gem_object_unbind(obj);
2215                 if (ret != 0) {
2216                         DRM_ERROR("Error unbinding object in LeaveVT: %d\n",
2217                                   ret);
2218                         mutex_unlock(&dev->struct_mutex);
2219                         return ret;
2220                 }
2221         }
2222
2223
2224         return 0;
2225 }
2226
2227 static int
2228 i915_gem_idle(struct drm_device *dev)
2229 {
2230         drm_i915_private_t *dev_priv = dev->dev_private;
2231         uint32_t seqno, cur_seqno, last_seqno;
2232         int stuck, ret;
2233
2234         mutex_lock(&dev->struct_mutex);
2235
2236         if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) {
2237                 mutex_unlock(&dev->struct_mutex);
2238                 return 0;
2239         }
2240
2241         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
2242          * We need to replace this with a semaphore, or something.
2243          */
2244         dev_priv->mm.suspended = 1;
2245
2246         /* Cancel the retire work handler, wait for it to finish if running
2247          */
2248         mutex_unlock(&dev->struct_mutex);
2249         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
2250         mutex_lock(&dev->struct_mutex);
2251
2252         i915_kernel_lost_context(dev);
2253
2254         /* Flush the GPU along with all non-CPU write domains
2255          */
2256         i915_gem_flush(dev, ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT),
2257                        ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
2258         seqno = i915_add_request(dev, ~(I915_GEM_DOMAIN_CPU |
2259                                         I915_GEM_DOMAIN_GTT));
2260
2261         if (seqno == 0) {
2262                 mutex_unlock(&dev->struct_mutex);
2263                 return -ENOMEM;
2264         }
2265
2266         dev_priv->mm.waiting_gem_seqno = seqno;
2267         last_seqno = 0;
2268         stuck = 0;
2269         for (;;) {
2270                 cur_seqno = i915_get_gem_seqno(dev);
2271                 if (i915_seqno_passed(cur_seqno, seqno))
2272                         break;
2273                 if (last_seqno == cur_seqno) {
2274                         if (stuck++ > 100) {
2275                                 DRM_ERROR("hardware wedged\n");
2276                                 dev_priv->mm.wedged = 1;
2277                                 DRM_WAKEUP(&dev_priv->irq_queue);
2278                                 break;
2279                         }
2280                 }
2281                 msleep(10);
2282                 last_seqno = cur_seqno;
2283         }
2284         dev_priv->mm.waiting_gem_seqno = 0;
2285
2286         i915_gem_retire_requests(dev);
2287
2288         /* Active and flushing should now be empty as we've
2289          * waited for a sequence higher than any pending execbuffer
2290          */
2291         BUG_ON(!list_empty(&dev_priv->mm.active_list));
2292         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2293
2294         /* Request should now be empty as we've also waited
2295          * for the last request in the list
2296          */
2297         BUG_ON(!list_empty(&dev_priv->mm.request_list));
2298
2299         /* Move all buffers out of the GTT. */
2300         ret = i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list);
2301         if (ret) {
2302                 mutex_unlock(&dev->struct_mutex);
2303                 return ret;
2304         }
2305
2306         BUG_ON(!list_empty(&dev_priv->mm.active_list));
2307         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2308         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
2309         BUG_ON(!list_empty(&dev_priv->mm.request_list));
2310
2311         i915_gem_cleanup_ringbuffer(dev);
2312         mutex_unlock(&dev->struct_mutex);
2313
2314         return 0;
2315 }
2316
2317 static int
2318 i915_gem_init_hws(struct drm_device *dev)
2319 {
2320         drm_i915_private_t *dev_priv = dev->dev_private;
2321         struct drm_gem_object *obj;
2322         struct drm_i915_gem_object *obj_priv;
2323         int ret;
2324
2325         /* If we need a physical address for the status page, it's already
2326          * initialized at driver load time.
2327          */
2328         if (!I915_NEED_GFX_HWS(dev))
2329                 return 0;
2330
2331         obj = drm_gem_object_alloc(dev, 4096);
2332         if (obj == NULL) {
2333                 DRM_ERROR("Failed to allocate status page\n");
2334                 return -ENOMEM;
2335         }
2336         obj_priv = obj->driver_private;
2337         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
2338
2339         ret = i915_gem_object_pin(obj, 4096);
2340         if (ret != 0) {
2341                 drm_gem_object_unreference(obj);
2342                 return ret;
2343         }
2344
2345         dev_priv->status_gfx_addr = obj_priv->gtt_offset;
2346
2347         dev_priv->hw_status_page = kmap(obj_priv->page_list[0]);
2348         if (dev_priv->hw_status_page == NULL) {
2349                 DRM_ERROR("Failed to map status page.\n");
2350                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
2351                 drm_gem_object_unreference(obj);
2352                 return -EINVAL;
2353         }
2354         dev_priv->hws_obj = obj;
2355         memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
2356         I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
2357         I915_READ(HWS_PGA); /* posting read */
2358         DRM_DEBUG("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
2359
2360         return 0;
2361 }
2362
2363 static int
2364 i915_gem_init_ringbuffer(struct drm_device *dev)
2365 {
2366         drm_i915_private_t *dev_priv = dev->dev_private;
2367         struct drm_gem_object *obj;
2368         struct drm_i915_gem_object *obj_priv;
2369         int ret;
2370         u32 head;
2371
2372         ret = i915_gem_init_hws(dev);
2373         if (ret != 0)
2374                 return ret;
2375
2376         obj = drm_gem_object_alloc(dev, 128 * 1024);
2377         if (obj == NULL) {
2378                 DRM_ERROR("Failed to allocate ringbuffer\n");
2379                 return -ENOMEM;
2380         }
2381         obj_priv = obj->driver_private;
2382
2383         ret = i915_gem_object_pin(obj, 4096);
2384         if (ret != 0) {
2385                 drm_gem_object_unreference(obj);
2386                 return ret;
2387         }
2388
2389         /* Set up the kernel mapping for the ring. */
2390         dev_priv->ring.Size = obj->size;
2391         dev_priv->ring.tail_mask = obj->size - 1;
2392
2393         dev_priv->ring.map.offset = dev->agp->base + obj_priv->gtt_offset;
2394         dev_priv->ring.map.size = obj->size;
2395         dev_priv->ring.map.type = 0;
2396         dev_priv->ring.map.flags = 0;
2397         dev_priv->ring.map.mtrr = 0;
2398
2399         drm_core_ioremap_wc(&dev_priv->ring.map, dev);
2400         if (dev_priv->ring.map.handle == NULL) {
2401                 DRM_ERROR("Failed to map ringbuffer.\n");
2402                 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
2403                 drm_gem_object_unreference(obj);
2404                 return -EINVAL;
2405         }
2406         dev_priv->ring.ring_obj = obj;
2407         dev_priv->ring.virtual_start = dev_priv->ring.map.handle;
2408
2409         /* Stop the ring if it's running. */
2410         I915_WRITE(PRB0_CTL, 0);
2411         I915_WRITE(PRB0_TAIL, 0);
2412         I915_WRITE(PRB0_HEAD, 0);
2413
2414         /* Initialize the ring. */
2415         I915_WRITE(PRB0_START, obj_priv->gtt_offset);
2416         head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
2417
2418         /* G45 ring initialization fails to reset head to zero */
2419         if (head != 0) {
2420                 DRM_ERROR("Ring head not reset to zero "
2421                           "ctl %08x head %08x tail %08x start %08x\n",
2422                           I915_READ(PRB0_CTL),
2423                           I915_READ(PRB0_HEAD),
2424                           I915_READ(PRB0_TAIL),
2425                           I915_READ(PRB0_START));
2426                 I915_WRITE(PRB0_HEAD, 0);
2427
2428                 DRM_ERROR("Ring head forced to zero "
2429                           "ctl %08x head %08x tail %08x start %08x\n",
2430                           I915_READ(PRB0_CTL),
2431                           I915_READ(PRB0_HEAD),
2432                           I915_READ(PRB0_TAIL),
2433                           I915_READ(PRB0_START));
2434         }
2435
2436         I915_WRITE(PRB0_CTL,
2437                    ((obj->size - 4096) & RING_NR_PAGES) |
2438                    RING_NO_REPORT |
2439                    RING_VALID);
2440
2441         head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
2442
2443         /* If the head is still not zero, the ring is dead */
2444         if (head != 0) {
2445                 DRM_ERROR("Ring initialization failed "
2446                           "ctl %08x head %08x tail %08x start %08x\n",
2447                           I915_READ(PRB0_CTL),
2448                           I915_READ(PRB0_HEAD),
2449                           I915_READ(PRB0_TAIL),
2450                           I915_READ(PRB0_START));
2451                 return -EIO;
2452         }
2453
2454         /* Update our cache of the ring state */
2455         i915_kernel_lost_context(dev);
2456
2457         return 0;
2458 }
2459
2460 static void
2461 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
2462 {
2463         drm_i915_private_t *dev_priv = dev->dev_private;
2464
2465         if (dev_priv->ring.ring_obj == NULL)
2466                 return;
2467
2468         drm_core_ioremapfree(&dev_priv->ring.map, dev);
2469
2470         i915_gem_object_unpin(dev_priv->ring.ring_obj);
2471         drm_gem_object_unreference(dev_priv->ring.ring_obj);
2472         dev_priv->ring.ring_obj = NULL;
2473         memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
2474
2475         if (dev_priv->hws_obj != NULL) {
2476                 struct drm_gem_object *obj = dev_priv->hws_obj;
2477                 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2478
2479                 kunmap(obj_priv->page_list[0]);
2480                 i915_gem_object_unpin(obj);
2481                 drm_gem_object_unreference(obj);
2482                 dev_priv->hws_obj = NULL;
2483                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
2484                 dev_priv->hw_status_page = NULL;
2485
2486                 /* Write high address into HWS_PGA when disabling. */
2487                 I915_WRITE(HWS_PGA, 0x1ffff000);
2488         }
2489 }
2490
2491 int
2492 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
2493                        struct drm_file *file_priv)
2494 {
2495         drm_i915_private_t *dev_priv = dev->dev_private;
2496         int ret;
2497
2498         if (dev_priv->mm.wedged) {
2499                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
2500                 dev_priv->mm.wedged = 0;
2501         }
2502
2503         ret = i915_gem_init_ringbuffer(dev);
2504         if (ret != 0)
2505                 return ret;
2506
2507         mutex_lock(&dev->struct_mutex);
2508         BUG_ON(!list_empty(&dev_priv->mm.active_list));
2509         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2510         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
2511         BUG_ON(!list_empty(&dev_priv->mm.request_list));
2512         dev_priv->mm.suspended = 0;
2513         mutex_unlock(&dev->struct_mutex);
2514
2515         drm_irq_install(dev);
2516
2517         return 0;
2518 }
2519
2520 int
2521 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
2522                        struct drm_file *file_priv)
2523 {
2524         int ret;
2525
2526         ret = i915_gem_idle(dev);
2527         drm_irq_uninstall(dev);
2528
2529         return ret;
2530 }
2531
2532 void
2533 i915_gem_lastclose(struct drm_device *dev)
2534 {
2535         int ret;
2536
2537         ret = i915_gem_idle(dev);
2538         if (ret)
2539                 DRM_ERROR("failed to idle hardware: %d\n", ret);
2540 }
2541
2542 void
2543 i915_gem_load(struct drm_device *dev)
2544 {
2545         drm_i915_private_t *dev_priv = dev->dev_private;
2546
2547         INIT_LIST_HEAD(&dev_priv->mm.active_list);
2548         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
2549         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
2550         INIT_LIST_HEAD(&dev_priv->mm.request_list);
2551         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
2552                           i915_gem_retire_work_handler);
2553         INIT_WORK(&dev_priv->mm.vblank_work,
2554                   i915_gem_vblank_work_handler);
2555         dev_priv->mm.next_gem_seqno = 1;
2556
2557         i915_gem_detect_bit_6_swizzle(dev);
2558 }