Merge branch 'drm-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/airlied...
[linux-2.6.git] / drivers / gpu / drm / i915 / i915_gem.c
1 /*
2  * Copyright © 2008 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Eric Anholt <eric@anholt.net>
25  *
26  */
27
28 #include "drmP.h"
29 #include "drm.h"
30 #include "i915_drm.h"
31 #include "i915_drv.h"
32 #include "intel_drv.h"
33 #include <linux/swap.h>
34 #include <linux/pci.h>
35
36 #define I915_GEM_GPU_DOMAINS    (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
37
38 static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
39 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
40 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
41 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
42                                              int write);
43 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
44                                                      uint64_t offset,
45                                                      uint64_t size);
46 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
47 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
48 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
49                                            unsigned alignment);
50 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
51 static int i915_gem_evict_something(struct drm_device *dev);
52 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
53                                 struct drm_i915_gem_pwrite *args,
54                                 struct drm_file *file_priv);
55
56 int i915_gem_do_init(struct drm_device *dev, unsigned long start,
57                      unsigned long end)
58 {
59         drm_i915_private_t *dev_priv = dev->dev_private;
60
61         if (start >= end ||
62             (start & (PAGE_SIZE - 1)) != 0 ||
63             (end & (PAGE_SIZE - 1)) != 0) {
64                 return -EINVAL;
65         }
66
67         drm_mm_init(&dev_priv->mm.gtt_space, start,
68                     end - start);
69
70         dev->gtt_total = (uint32_t) (end - start);
71
72         return 0;
73 }
74
75 int
76 i915_gem_init_ioctl(struct drm_device *dev, void *data,
77                     struct drm_file *file_priv)
78 {
79         struct drm_i915_gem_init *args = data;
80         int ret;
81
82         mutex_lock(&dev->struct_mutex);
83         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
84         mutex_unlock(&dev->struct_mutex);
85
86         return ret;
87 }
88
89 int
90 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
91                             struct drm_file *file_priv)
92 {
93         struct drm_i915_gem_get_aperture *args = data;
94
95         if (!(dev->driver->driver_features & DRIVER_GEM))
96                 return -ENODEV;
97
98         args->aper_size = dev->gtt_total;
99         args->aper_available_size = (args->aper_size -
100                                      atomic_read(&dev->pin_memory));
101
102         return 0;
103 }
104
105
106 /**
107  * Creates a new mm object and returns a handle to it.
108  */
109 int
110 i915_gem_create_ioctl(struct drm_device *dev, void *data,
111                       struct drm_file *file_priv)
112 {
113         struct drm_i915_gem_create *args = data;
114         struct drm_gem_object *obj;
115         int ret;
116         u32 handle;
117
118         args->size = roundup(args->size, PAGE_SIZE);
119
120         /* Allocate the new object */
121         obj = drm_gem_object_alloc(dev, args->size);
122         if (obj == NULL)
123                 return -ENOMEM;
124
125         ret = drm_gem_handle_create(file_priv, obj, &handle);
126         mutex_lock(&dev->struct_mutex);
127         drm_gem_object_handle_unreference(obj);
128         mutex_unlock(&dev->struct_mutex);
129
130         if (ret)
131                 return ret;
132
133         args->handle = handle;
134
135         return 0;
136 }
137
138 static inline int
139 fast_shmem_read(struct page **pages,
140                 loff_t page_base, int page_offset,
141                 char __user *data,
142                 int length)
143 {
144         char __iomem *vaddr;
145         int unwritten;
146
147         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
148         if (vaddr == NULL)
149                 return -ENOMEM;
150         unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
151         kunmap_atomic(vaddr, KM_USER0);
152
153         if (unwritten)
154                 return -EFAULT;
155
156         return 0;
157 }
158
159 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
160 {
161         drm_i915_private_t *dev_priv = obj->dev->dev_private;
162         struct drm_i915_gem_object *obj_priv = obj->driver_private;
163
164         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
165                 obj_priv->tiling_mode != I915_TILING_NONE;
166 }
167
168 static inline int
169 slow_shmem_copy(struct page *dst_page,
170                 int dst_offset,
171                 struct page *src_page,
172                 int src_offset,
173                 int length)
174 {
175         char *dst_vaddr, *src_vaddr;
176
177         dst_vaddr = kmap_atomic(dst_page, KM_USER0);
178         if (dst_vaddr == NULL)
179                 return -ENOMEM;
180
181         src_vaddr = kmap_atomic(src_page, KM_USER1);
182         if (src_vaddr == NULL) {
183                 kunmap_atomic(dst_vaddr, KM_USER0);
184                 return -ENOMEM;
185         }
186
187         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
188
189         kunmap_atomic(src_vaddr, KM_USER1);
190         kunmap_atomic(dst_vaddr, KM_USER0);
191
192         return 0;
193 }
194
195 static inline int
196 slow_shmem_bit17_copy(struct page *gpu_page,
197                       int gpu_offset,
198                       struct page *cpu_page,
199                       int cpu_offset,
200                       int length,
201                       int is_read)
202 {
203         char *gpu_vaddr, *cpu_vaddr;
204
205         /* Use the unswizzled path if this page isn't affected. */
206         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
207                 if (is_read)
208                         return slow_shmem_copy(cpu_page, cpu_offset,
209                                                gpu_page, gpu_offset, length);
210                 else
211                         return slow_shmem_copy(gpu_page, gpu_offset,
212                                                cpu_page, cpu_offset, length);
213         }
214
215         gpu_vaddr = kmap_atomic(gpu_page, KM_USER0);
216         if (gpu_vaddr == NULL)
217                 return -ENOMEM;
218
219         cpu_vaddr = kmap_atomic(cpu_page, KM_USER1);
220         if (cpu_vaddr == NULL) {
221                 kunmap_atomic(gpu_vaddr, KM_USER0);
222                 return -ENOMEM;
223         }
224
225         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
226          * XORing with the other bits (A9 for Y, A9 and A10 for X)
227          */
228         while (length > 0) {
229                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
230                 int this_length = min(cacheline_end - gpu_offset, length);
231                 int swizzled_gpu_offset = gpu_offset ^ 64;
232
233                 if (is_read) {
234                         memcpy(cpu_vaddr + cpu_offset,
235                                gpu_vaddr + swizzled_gpu_offset,
236                                this_length);
237                 } else {
238                         memcpy(gpu_vaddr + swizzled_gpu_offset,
239                                cpu_vaddr + cpu_offset,
240                                this_length);
241                 }
242                 cpu_offset += this_length;
243                 gpu_offset += this_length;
244                 length -= this_length;
245         }
246
247         kunmap_atomic(cpu_vaddr, KM_USER1);
248         kunmap_atomic(gpu_vaddr, KM_USER0);
249
250         return 0;
251 }
252
253 /**
254  * This is the fast shmem pread path, which attempts to copy_from_user directly
255  * from the backing pages of the object to the user's address space.  On a
256  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
257  */
258 static int
259 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
260                           struct drm_i915_gem_pread *args,
261                           struct drm_file *file_priv)
262 {
263         struct drm_i915_gem_object *obj_priv = obj->driver_private;
264         ssize_t remain;
265         loff_t offset, page_base;
266         char __user *user_data;
267         int page_offset, page_length;
268         int ret;
269
270         user_data = (char __user *) (uintptr_t) args->data_ptr;
271         remain = args->size;
272
273         mutex_lock(&dev->struct_mutex);
274
275         ret = i915_gem_object_get_pages(obj);
276         if (ret != 0)
277                 goto fail_unlock;
278
279         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
280                                                         args->size);
281         if (ret != 0)
282                 goto fail_put_pages;
283
284         obj_priv = obj->driver_private;
285         offset = args->offset;
286
287         while (remain > 0) {
288                 /* Operation in this page
289                  *
290                  * page_base = page offset within aperture
291                  * page_offset = offset within page
292                  * page_length = bytes to copy for this page
293                  */
294                 page_base = (offset & ~(PAGE_SIZE-1));
295                 page_offset = offset & (PAGE_SIZE-1);
296                 page_length = remain;
297                 if ((page_offset + remain) > PAGE_SIZE)
298                         page_length = PAGE_SIZE - page_offset;
299
300                 ret = fast_shmem_read(obj_priv->pages,
301                                       page_base, page_offset,
302                                       user_data, page_length);
303                 if (ret)
304                         goto fail_put_pages;
305
306                 remain -= page_length;
307                 user_data += page_length;
308                 offset += page_length;
309         }
310
311 fail_put_pages:
312         i915_gem_object_put_pages(obj);
313 fail_unlock:
314         mutex_unlock(&dev->struct_mutex);
315
316         return ret;
317 }
318
319 /**
320  * This is the fallback shmem pread path, which allocates temporary storage
321  * in kernel space to copy_to_user into outside of the struct_mutex, so we
322  * can copy out of the object's backing pages while holding the struct mutex
323  * and not take page faults.
324  */
325 static int
326 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
327                           struct drm_i915_gem_pread *args,
328                           struct drm_file *file_priv)
329 {
330         struct drm_i915_gem_object *obj_priv = obj->driver_private;
331         struct mm_struct *mm = current->mm;
332         struct page **user_pages;
333         ssize_t remain;
334         loff_t offset, pinned_pages, i;
335         loff_t first_data_page, last_data_page, num_pages;
336         int shmem_page_index, shmem_page_offset;
337         int data_page_index,  data_page_offset;
338         int page_length;
339         int ret;
340         uint64_t data_ptr = args->data_ptr;
341         int do_bit17_swizzling;
342
343         remain = args->size;
344
345         /* Pin the user pages containing the data.  We can't fault while
346          * holding the struct mutex, yet we want to hold it while
347          * dereferencing the user data.
348          */
349         first_data_page = data_ptr / PAGE_SIZE;
350         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
351         num_pages = last_data_page - first_data_page + 1;
352
353         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
354         if (user_pages == NULL)
355                 return -ENOMEM;
356
357         down_read(&mm->mmap_sem);
358         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
359                                       num_pages, 1, 0, user_pages, NULL);
360         up_read(&mm->mmap_sem);
361         if (pinned_pages < num_pages) {
362                 ret = -EFAULT;
363                 goto fail_put_user_pages;
364         }
365
366         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
367
368         mutex_lock(&dev->struct_mutex);
369
370         ret = i915_gem_object_get_pages(obj);
371         if (ret != 0)
372                 goto fail_unlock;
373
374         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
375                                                         args->size);
376         if (ret != 0)
377                 goto fail_put_pages;
378
379         obj_priv = obj->driver_private;
380         offset = args->offset;
381
382         while (remain > 0) {
383                 /* Operation in this page
384                  *
385                  * shmem_page_index = page number within shmem file
386                  * shmem_page_offset = offset within page in shmem file
387                  * data_page_index = page number in get_user_pages return
388                  * data_page_offset = offset with data_page_index page.
389                  * page_length = bytes to copy for this page
390                  */
391                 shmem_page_index = offset / PAGE_SIZE;
392                 shmem_page_offset = offset & ~PAGE_MASK;
393                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
394                 data_page_offset = data_ptr & ~PAGE_MASK;
395
396                 page_length = remain;
397                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
398                         page_length = PAGE_SIZE - shmem_page_offset;
399                 if ((data_page_offset + page_length) > PAGE_SIZE)
400                         page_length = PAGE_SIZE - data_page_offset;
401
402                 if (do_bit17_swizzling) {
403                         ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
404                                                     shmem_page_offset,
405                                                     user_pages[data_page_index],
406                                                     data_page_offset,
407                                                     page_length,
408                                                     1);
409                 } else {
410                         ret = slow_shmem_copy(user_pages[data_page_index],
411                                               data_page_offset,
412                                               obj_priv->pages[shmem_page_index],
413                                               shmem_page_offset,
414                                               page_length);
415                 }
416                 if (ret)
417                         goto fail_put_pages;
418
419                 remain -= page_length;
420                 data_ptr += page_length;
421                 offset += page_length;
422         }
423
424 fail_put_pages:
425         i915_gem_object_put_pages(obj);
426 fail_unlock:
427         mutex_unlock(&dev->struct_mutex);
428 fail_put_user_pages:
429         for (i = 0; i < pinned_pages; i++) {
430                 SetPageDirty(user_pages[i]);
431                 page_cache_release(user_pages[i]);
432         }
433         drm_free_large(user_pages);
434
435         return ret;
436 }
437
438 /**
439  * Reads data from the object referenced by handle.
440  *
441  * On error, the contents of *data are undefined.
442  */
443 int
444 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
445                      struct drm_file *file_priv)
446 {
447         struct drm_i915_gem_pread *args = data;
448         struct drm_gem_object *obj;
449         struct drm_i915_gem_object *obj_priv;
450         int ret;
451
452         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
453         if (obj == NULL)
454                 return -EBADF;
455         obj_priv = obj->driver_private;
456
457         /* Bounds check source.
458          *
459          * XXX: This could use review for overflow issues...
460          */
461         if (args->offset > obj->size || args->size > obj->size ||
462             args->offset + args->size > obj->size) {
463                 drm_gem_object_unreference(obj);
464                 return -EINVAL;
465         }
466
467         if (i915_gem_object_needs_bit17_swizzle(obj)) {
468                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
469         } else {
470                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
471                 if (ret != 0)
472                         ret = i915_gem_shmem_pread_slow(dev, obj, args,
473                                                         file_priv);
474         }
475
476         drm_gem_object_unreference(obj);
477
478         return ret;
479 }
480
481 /* This is the fast write path which cannot handle
482  * page faults in the source data
483  */
484
485 static inline int
486 fast_user_write(struct io_mapping *mapping,
487                 loff_t page_base, int page_offset,
488                 char __user *user_data,
489                 int length)
490 {
491         char *vaddr_atomic;
492         unsigned long unwritten;
493
494         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
495         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
496                                                       user_data, length);
497         io_mapping_unmap_atomic(vaddr_atomic);
498         if (unwritten)
499                 return -EFAULT;
500         return 0;
501 }
502
503 /* Here's the write path which can sleep for
504  * page faults
505  */
506
507 static inline int
508 slow_kernel_write(struct io_mapping *mapping,
509                   loff_t gtt_base, int gtt_offset,
510                   struct page *user_page, int user_offset,
511                   int length)
512 {
513         char *src_vaddr, *dst_vaddr;
514         unsigned long unwritten;
515
516         dst_vaddr = io_mapping_map_atomic_wc(mapping, gtt_base);
517         src_vaddr = kmap_atomic(user_page, KM_USER1);
518         unwritten = __copy_from_user_inatomic_nocache(dst_vaddr + gtt_offset,
519                                                       src_vaddr + user_offset,
520                                                       length);
521         kunmap_atomic(src_vaddr, KM_USER1);
522         io_mapping_unmap_atomic(dst_vaddr);
523         if (unwritten)
524                 return -EFAULT;
525         return 0;
526 }
527
528 static inline int
529 fast_shmem_write(struct page **pages,
530                  loff_t page_base, int page_offset,
531                  char __user *data,
532                  int length)
533 {
534         char __iomem *vaddr;
535         unsigned long unwritten;
536
537         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
538         if (vaddr == NULL)
539                 return -ENOMEM;
540         unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
541         kunmap_atomic(vaddr, KM_USER0);
542
543         if (unwritten)
544                 return -EFAULT;
545         return 0;
546 }
547
548 /**
549  * This is the fast pwrite path, where we copy the data directly from the
550  * user into the GTT, uncached.
551  */
552 static int
553 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
554                          struct drm_i915_gem_pwrite *args,
555                          struct drm_file *file_priv)
556 {
557         struct drm_i915_gem_object *obj_priv = obj->driver_private;
558         drm_i915_private_t *dev_priv = dev->dev_private;
559         ssize_t remain;
560         loff_t offset, page_base;
561         char __user *user_data;
562         int page_offset, page_length;
563         int ret;
564
565         user_data = (char __user *) (uintptr_t) args->data_ptr;
566         remain = args->size;
567         if (!access_ok(VERIFY_READ, user_data, remain))
568                 return -EFAULT;
569
570
571         mutex_lock(&dev->struct_mutex);
572         ret = i915_gem_object_pin(obj, 0);
573         if (ret) {
574                 mutex_unlock(&dev->struct_mutex);
575                 return ret;
576         }
577         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
578         if (ret)
579                 goto fail;
580
581         obj_priv = obj->driver_private;
582         offset = obj_priv->gtt_offset + args->offset;
583
584         while (remain > 0) {
585                 /* Operation in this page
586                  *
587                  * page_base = page offset within aperture
588                  * page_offset = offset within page
589                  * page_length = bytes to copy for this page
590                  */
591                 page_base = (offset & ~(PAGE_SIZE-1));
592                 page_offset = offset & (PAGE_SIZE-1);
593                 page_length = remain;
594                 if ((page_offset + remain) > PAGE_SIZE)
595                         page_length = PAGE_SIZE - page_offset;
596
597                 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
598                                        page_offset, user_data, page_length);
599
600                 /* If we get a fault while copying data, then (presumably) our
601                  * source page isn't available.  Return the error and we'll
602                  * retry in the slow path.
603                  */
604                 if (ret)
605                         goto fail;
606
607                 remain -= page_length;
608                 user_data += page_length;
609                 offset += page_length;
610         }
611
612 fail:
613         i915_gem_object_unpin(obj);
614         mutex_unlock(&dev->struct_mutex);
615
616         return ret;
617 }
618
619 /**
620  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
621  * the memory and maps it using kmap_atomic for copying.
622  *
623  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
624  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
625  */
626 static int
627 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
628                          struct drm_i915_gem_pwrite *args,
629                          struct drm_file *file_priv)
630 {
631         struct drm_i915_gem_object *obj_priv = obj->driver_private;
632         drm_i915_private_t *dev_priv = dev->dev_private;
633         ssize_t remain;
634         loff_t gtt_page_base, offset;
635         loff_t first_data_page, last_data_page, num_pages;
636         loff_t pinned_pages, i;
637         struct page **user_pages;
638         struct mm_struct *mm = current->mm;
639         int gtt_page_offset, data_page_offset, data_page_index, page_length;
640         int ret;
641         uint64_t data_ptr = args->data_ptr;
642
643         remain = args->size;
644
645         /* Pin the user pages containing the data.  We can't fault while
646          * holding the struct mutex, and all of the pwrite implementations
647          * want to hold it while dereferencing the user data.
648          */
649         first_data_page = data_ptr / PAGE_SIZE;
650         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
651         num_pages = last_data_page - first_data_page + 1;
652
653         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
654         if (user_pages == NULL)
655                 return -ENOMEM;
656
657         down_read(&mm->mmap_sem);
658         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
659                                       num_pages, 0, 0, user_pages, NULL);
660         up_read(&mm->mmap_sem);
661         if (pinned_pages < num_pages) {
662                 ret = -EFAULT;
663                 goto out_unpin_pages;
664         }
665
666         mutex_lock(&dev->struct_mutex);
667         ret = i915_gem_object_pin(obj, 0);
668         if (ret)
669                 goto out_unlock;
670
671         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
672         if (ret)
673                 goto out_unpin_object;
674
675         obj_priv = obj->driver_private;
676         offset = obj_priv->gtt_offset + args->offset;
677
678         while (remain > 0) {
679                 /* Operation in this page
680                  *
681                  * gtt_page_base = page offset within aperture
682                  * gtt_page_offset = offset within page in aperture
683                  * data_page_index = page number in get_user_pages return
684                  * data_page_offset = offset with data_page_index page.
685                  * page_length = bytes to copy for this page
686                  */
687                 gtt_page_base = offset & PAGE_MASK;
688                 gtt_page_offset = offset & ~PAGE_MASK;
689                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
690                 data_page_offset = data_ptr & ~PAGE_MASK;
691
692                 page_length = remain;
693                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
694                         page_length = PAGE_SIZE - gtt_page_offset;
695                 if ((data_page_offset + page_length) > PAGE_SIZE)
696                         page_length = PAGE_SIZE - data_page_offset;
697
698                 ret = slow_kernel_write(dev_priv->mm.gtt_mapping,
699                                         gtt_page_base, gtt_page_offset,
700                                         user_pages[data_page_index],
701                                         data_page_offset,
702                                         page_length);
703
704                 /* If we get a fault while copying data, then (presumably) our
705                  * source page isn't available.  Return the error and we'll
706                  * retry in the slow path.
707                  */
708                 if (ret)
709                         goto out_unpin_object;
710
711                 remain -= page_length;
712                 offset += page_length;
713                 data_ptr += page_length;
714         }
715
716 out_unpin_object:
717         i915_gem_object_unpin(obj);
718 out_unlock:
719         mutex_unlock(&dev->struct_mutex);
720 out_unpin_pages:
721         for (i = 0; i < pinned_pages; i++)
722                 page_cache_release(user_pages[i]);
723         drm_free_large(user_pages);
724
725         return ret;
726 }
727
728 /**
729  * This is the fast shmem pwrite path, which attempts to directly
730  * copy_from_user into the kmapped pages backing the object.
731  */
732 static int
733 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
734                            struct drm_i915_gem_pwrite *args,
735                            struct drm_file *file_priv)
736 {
737         struct drm_i915_gem_object *obj_priv = obj->driver_private;
738         ssize_t remain;
739         loff_t offset, page_base;
740         char __user *user_data;
741         int page_offset, page_length;
742         int ret;
743
744         user_data = (char __user *) (uintptr_t) args->data_ptr;
745         remain = args->size;
746
747         mutex_lock(&dev->struct_mutex);
748
749         ret = i915_gem_object_get_pages(obj);
750         if (ret != 0)
751                 goto fail_unlock;
752
753         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
754         if (ret != 0)
755                 goto fail_put_pages;
756
757         obj_priv = obj->driver_private;
758         offset = args->offset;
759         obj_priv->dirty = 1;
760
761         while (remain > 0) {
762                 /* Operation in this page
763                  *
764                  * page_base = page offset within aperture
765                  * page_offset = offset within page
766                  * page_length = bytes to copy for this page
767                  */
768                 page_base = (offset & ~(PAGE_SIZE-1));
769                 page_offset = offset & (PAGE_SIZE-1);
770                 page_length = remain;
771                 if ((page_offset + remain) > PAGE_SIZE)
772                         page_length = PAGE_SIZE - page_offset;
773
774                 ret = fast_shmem_write(obj_priv->pages,
775                                        page_base, page_offset,
776                                        user_data, page_length);
777                 if (ret)
778                         goto fail_put_pages;
779
780                 remain -= page_length;
781                 user_data += page_length;
782                 offset += page_length;
783         }
784
785 fail_put_pages:
786         i915_gem_object_put_pages(obj);
787 fail_unlock:
788         mutex_unlock(&dev->struct_mutex);
789
790         return ret;
791 }
792
793 /**
794  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
795  * the memory and maps it using kmap_atomic for copying.
796  *
797  * This avoids taking mmap_sem for faulting on the user's address while the
798  * struct_mutex is held.
799  */
800 static int
801 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
802                            struct drm_i915_gem_pwrite *args,
803                            struct drm_file *file_priv)
804 {
805         struct drm_i915_gem_object *obj_priv = obj->driver_private;
806         struct mm_struct *mm = current->mm;
807         struct page **user_pages;
808         ssize_t remain;
809         loff_t offset, pinned_pages, i;
810         loff_t first_data_page, last_data_page, num_pages;
811         int shmem_page_index, shmem_page_offset;
812         int data_page_index,  data_page_offset;
813         int page_length;
814         int ret;
815         uint64_t data_ptr = args->data_ptr;
816         int do_bit17_swizzling;
817
818         remain = args->size;
819
820         /* Pin the user pages containing the data.  We can't fault while
821          * holding the struct mutex, and all of the pwrite implementations
822          * want to hold it while dereferencing the user data.
823          */
824         first_data_page = data_ptr / PAGE_SIZE;
825         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
826         num_pages = last_data_page - first_data_page + 1;
827
828         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
829         if (user_pages == NULL)
830                 return -ENOMEM;
831
832         down_read(&mm->mmap_sem);
833         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
834                                       num_pages, 0, 0, user_pages, NULL);
835         up_read(&mm->mmap_sem);
836         if (pinned_pages < num_pages) {
837                 ret = -EFAULT;
838                 goto fail_put_user_pages;
839         }
840
841         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
842
843         mutex_lock(&dev->struct_mutex);
844
845         ret = i915_gem_object_get_pages(obj);
846         if (ret != 0)
847                 goto fail_unlock;
848
849         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
850         if (ret != 0)
851                 goto fail_put_pages;
852
853         obj_priv = obj->driver_private;
854         offset = args->offset;
855         obj_priv->dirty = 1;
856
857         while (remain > 0) {
858                 /* Operation in this page
859                  *
860                  * shmem_page_index = page number within shmem file
861                  * shmem_page_offset = offset within page in shmem file
862                  * data_page_index = page number in get_user_pages return
863                  * data_page_offset = offset with data_page_index page.
864                  * page_length = bytes to copy for this page
865                  */
866                 shmem_page_index = offset / PAGE_SIZE;
867                 shmem_page_offset = offset & ~PAGE_MASK;
868                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
869                 data_page_offset = data_ptr & ~PAGE_MASK;
870
871                 page_length = remain;
872                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
873                         page_length = PAGE_SIZE - shmem_page_offset;
874                 if ((data_page_offset + page_length) > PAGE_SIZE)
875                         page_length = PAGE_SIZE - data_page_offset;
876
877                 if (do_bit17_swizzling) {
878                         ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
879                                                     shmem_page_offset,
880                                                     user_pages[data_page_index],
881                                                     data_page_offset,
882                                                     page_length,
883                                                     0);
884                 } else {
885                         ret = slow_shmem_copy(obj_priv->pages[shmem_page_index],
886                                               shmem_page_offset,
887                                               user_pages[data_page_index],
888                                               data_page_offset,
889                                               page_length);
890                 }
891                 if (ret)
892                         goto fail_put_pages;
893
894                 remain -= page_length;
895                 data_ptr += page_length;
896                 offset += page_length;
897         }
898
899 fail_put_pages:
900         i915_gem_object_put_pages(obj);
901 fail_unlock:
902         mutex_unlock(&dev->struct_mutex);
903 fail_put_user_pages:
904         for (i = 0; i < pinned_pages; i++)
905                 page_cache_release(user_pages[i]);
906         drm_free_large(user_pages);
907
908         return ret;
909 }
910
911 /**
912  * Writes data to the object referenced by handle.
913  *
914  * On error, the contents of the buffer that were to be modified are undefined.
915  */
916 int
917 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
918                       struct drm_file *file_priv)
919 {
920         struct drm_i915_gem_pwrite *args = data;
921         struct drm_gem_object *obj;
922         struct drm_i915_gem_object *obj_priv;
923         int ret = 0;
924
925         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
926         if (obj == NULL)
927                 return -EBADF;
928         obj_priv = obj->driver_private;
929
930         /* Bounds check destination.
931          *
932          * XXX: This could use review for overflow issues...
933          */
934         if (args->offset > obj->size || args->size > obj->size ||
935             args->offset + args->size > obj->size) {
936                 drm_gem_object_unreference(obj);
937                 return -EINVAL;
938         }
939
940         /* We can only do the GTT pwrite on untiled buffers, as otherwise
941          * it would end up going through the fenced access, and we'll get
942          * different detiling behavior between reading and writing.
943          * pread/pwrite currently are reading and writing from the CPU
944          * perspective, requiring manual detiling by the client.
945          */
946         if (obj_priv->phys_obj)
947                 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
948         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
949                  dev->gtt_total != 0) {
950                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
951                 if (ret == -EFAULT) {
952                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
953                                                        file_priv);
954                 }
955         } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
956                 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
957         } else {
958                 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
959                 if (ret == -EFAULT) {
960                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
961                                                          file_priv);
962                 }
963         }
964
965 #if WATCH_PWRITE
966         if (ret)
967                 DRM_INFO("pwrite failed %d\n", ret);
968 #endif
969
970         drm_gem_object_unreference(obj);
971
972         return ret;
973 }
974
975 /**
976  * Called when user space prepares to use an object with the CPU, either
977  * through the mmap ioctl's mapping or a GTT mapping.
978  */
979 int
980 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
981                           struct drm_file *file_priv)
982 {
983         struct drm_i915_private *dev_priv = dev->dev_private;
984         struct drm_i915_gem_set_domain *args = data;
985         struct drm_gem_object *obj;
986         struct drm_i915_gem_object *obj_priv;
987         uint32_t read_domains = args->read_domains;
988         uint32_t write_domain = args->write_domain;
989         int ret;
990
991         if (!(dev->driver->driver_features & DRIVER_GEM))
992                 return -ENODEV;
993
994         /* Only handle setting domains to types used by the CPU. */
995         if (write_domain & I915_GEM_GPU_DOMAINS)
996                 return -EINVAL;
997
998         if (read_domains & I915_GEM_GPU_DOMAINS)
999                 return -EINVAL;
1000
1001         /* Having something in the write domain implies it's in the read
1002          * domain, and only that read domain.  Enforce that in the request.
1003          */
1004         if (write_domain != 0 && read_domains != write_domain)
1005                 return -EINVAL;
1006
1007         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1008         if (obj == NULL)
1009                 return -EBADF;
1010         obj_priv = obj->driver_private;
1011
1012         mutex_lock(&dev->struct_mutex);
1013
1014         intel_mark_busy(dev, obj);
1015
1016 #if WATCH_BUF
1017         DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
1018                  obj, obj->size, read_domains, write_domain);
1019 #endif
1020         if (read_domains & I915_GEM_DOMAIN_GTT) {
1021                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1022
1023                 /* Update the LRU on the fence for the CPU access that's
1024                  * about to occur.
1025                  */
1026                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1027                         list_move_tail(&obj_priv->fence_list,
1028                                        &dev_priv->mm.fence_list);
1029                 }
1030
1031                 /* Silently promote "you're not bound, there was nothing to do"
1032                  * to success, since the client was just asking us to
1033                  * make sure everything was done.
1034                  */
1035                 if (ret == -EINVAL)
1036                         ret = 0;
1037         } else {
1038                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1039         }
1040
1041         drm_gem_object_unreference(obj);
1042         mutex_unlock(&dev->struct_mutex);
1043         return ret;
1044 }
1045
1046 /**
1047  * Called when user space has done writes to this buffer
1048  */
1049 int
1050 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1051                       struct drm_file *file_priv)
1052 {
1053         struct drm_i915_gem_sw_finish *args = data;
1054         struct drm_gem_object *obj;
1055         struct drm_i915_gem_object *obj_priv;
1056         int ret = 0;
1057
1058         if (!(dev->driver->driver_features & DRIVER_GEM))
1059                 return -ENODEV;
1060
1061         mutex_lock(&dev->struct_mutex);
1062         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1063         if (obj == NULL) {
1064                 mutex_unlock(&dev->struct_mutex);
1065                 return -EBADF;
1066         }
1067
1068 #if WATCH_BUF
1069         DRM_INFO("%s: sw_finish %d (%p %zd)\n",
1070                  __func__, args->handle, obj, obj->size);
1071 #endif
1072         obj_priv = obj->driver_private;
1073
1074         /* Pinned buffers may be scanout, so flush the cache */
1075         if (obj_priv->pin_count)
1076                 i915_gem_object_flush_cpu_write_domain(obj);
1077
1078         drm_gem_object_unreference(obj);
1079         mutex_unlock(&dev->struct_mutex);
1080         return ret;
1081 }
1082
1083 /**
1084  * Maps the contents of an object, returning the address it is mapped
1085  * into.
1086  *
1087  * While the mapping holds a reference on the contents of the object, it doesn't
1088  * imply a ref on the object itself.
1089  */
1090 int
1091 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1092                    struct drm_file *file_priv)
1093 {
1094         struct drm_i915_gem_mmap *args = data;
1095         struct drm_gem_object *obj;
1096         loff_t offset;
1097         unsigned long addr;
1098
1099         if (!(dev->driver->driver_features & DRIVER_GEM))
1100                 return -ENODEV;
1101
1102         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1103         if (obj == NULL)
1104                 return -EBADF;
1105
1106         offset = args->offset;
1107
1108         down_write(&current->mm->mmap_sem);
1109         addr = do_mmap(obj->filp, 0, args->size,
1110                        PROT_READ | PROT_WRITE, MAP_SHARED,
1111                        args->offset);
1112         up_write(&current->mm->mmap_sem);
1113         mutex_lock(&dev->struct_mutex);
1114         drm_gem_object_unreference(obj);
1115         mutex_unlock(&dev->struct_mutex);
1116         if (IS_ERR((void *)addr))
1117                 return addr;
1118
1119         args->addr_ptr = (uint64_t) addr;
1120
1121         return 0;
1122 }
1123
1124 /**
1125  * i915_gem_fault - fault a page into the GTT
1126  * vma: VMA in question
1127  * vmf: fault info
1128  *
1129  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1130  * from userspace.  The fault handler takes care of binding the object to
1131  * the GTT (if needed), allocating and programming a fence register (again,
1132  * only if needed based on whether the old reg is still valid or the object
1133  * is tiled) and inserting a new PTE into the faulting process.
1134  *
1135  * Note that the faulting process may involve evicting existing objects
1136  * from the GTT and/or fence registers to make room.  So performance may
1137  * suffer if the GTT working set is large or there are few fence registers
1138  * left.
1139  */
1140 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1141 {
1142         struct drm_gem_object *obj = vma->vm_private_data;
1143         struct drm_device *dev = obj->dev;
1144         struct drm_i915_private *dev_priv = dev->dev_private;
1145         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1146         pgoff_t page_offset;
1147         unsigned long pfn;
1148         int ret = 0;
1149         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1150
1151         /* We don't use vmf->pgoff since that has the fake offset */
1152         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1153                 PAGE_SHIFT;
1154
1155         /* Now bind it into the GTT if needed */
1156         mutex_lock(&dev->struct_mutex);
1157         if (!obj_priv->gtt_space) {
1158                 ret = i915_gem_object_bind_to_gtt(obj, obj_priv->gtt_alignment);
1159                 if (ret) {
1160                         mutex_unlock(&dev->struct_mutex);
1161                         return VM_FAULT_SIGBUS;
1162                 }
1163
1164                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1165                 if (ret) {
1166                         mutex_unlock(&dev->struct_mutex);
1167                         return VM_FAULT_SIGBUS;
1168                 }
1169
1170                 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1171         }
1172
1173         /* Need a new fence register? */
1174         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1175                 ret = i915_gem_object_get_fence_reg(obj);
1176                 if (ret) {
1177                         mutex_unlock(&dev->struct_mutex);
1178                         return VM_FAULT_SIGBUS;
1179                 }
1180         }
1181
1182         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1183                 page_offset;
1184
1185         /* Finally, remap it using the new GTT offset */
1186         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1187
1188         mutex_unlock(&dev->struct_mutex);
1189
1190         switch (ret) {
1191         case -ENOMEM:
1192         case -EAGAIN:
1193                 return VM_FAULT_OOM;
1194         case -EFAULT:
1195         case -EINVAL:
1196                 return VM_FAULT_SIGBUS;
1197         default:
1198                 return VM_FAULT_NOPAGE;
1199         }
1200 }
1201
1202 /**
1203  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1204  * @obj: obj in question
1205  *
1206  * GEM memory mapping works by handing back to userspace a fake mmap offset
1207  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1208  * up the object based on the offset and sets up the various memory mapping
1209  * structures.
1210  *
1211  * This routine allocates and attaches a fake offset for @obj.
1212  */
1213 static int
1214 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1215 {
1216         struct drm_device *dev = obj->dev;
1217         struct drm_gem_mm *mm = dev->mm_private;
1218         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1219         struct drm_map_list *list;
1220         struct drm_local_map *map;
1221         int ret = 0;
1222
1223         /* Set the object up for mmap'ing */
1224         list = &obj->map_list;
1225         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1226         if (!list->map)
1227                 return -ENOMEM;
1228
1229         map = list->map;
1230         map->type = _DRM_GEM;
1231         map->size = obj->size;
1232         map->handle = obj;
1233
1234         /* Get a DRM GEM mmap offset allocated... */
1235         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1236                                                     obj->size / PAGE_SIZE, 0, 0);
1237         if (!list->file_offset_node) {
1238                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1239                 ret = -ENOMEM;
1240                 goto out_free_list;
1241         }
1242
1243         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1244                                                   obj->size / PAGE_SIZE, 0);
1245         if (!list->file_offset_node) {
1246                 ret = -ENOMEM;
1247                 goto out_free_list;
1248         }
1249
1250         list->hash.key = list->file_offset_node->start;
1251         if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1252                 DRM_ERROR("failed to add to map hash\n");
1253                 goto out_free_mm;
1254         }
1255
1256         /* By now we should be all set, any drm_mmap request on the offset
1257          * below will get to our mmap & fault handler */
1258         obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1259
1260         return 0;
1261
1262 out_free_mm:
1263         drm_mm_put_block(list->file_offset_node);
1264 out_free_list:
1265         kfree(list->map);
1266
1267         return ret;
1268 }
1269
1270 /**
1271  * i915_gem_release_mmap - remove physical page mappings
1272  * @obj: obj in question
1273  *
1274  * Preserve the reservation of the mmaping with the DRM core code, but
1275  * relinquish ownership of the pages back to the system.
1276  *
1277  * It is vital that we remove the page mapping if we have mapped a tiled
1278  * object through the GTT and then lose the fence register due to
1279  * resource pressure. Similarly if the object has been moved out of the
1280  * aperture, than pages mapped into userspace must be revoked. Removing the
1281  * mapping will then trigger a page fault on the next user access, allowing
1282  * fixup by i915_gem_fault().
1283  */
1284 void
1285 i915_gem_release_mmap(struct drm_gem_object *obj)
1286 {
1287         struct drm_device *dev = obj->dev;
1288         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1289
1290         if (dev->dev_mapping)
1291                 unmap_mapping_range(dev->dev_mapping,
1292                                     obj_priv->mmap_offset, obj->size, 1);
1293 }
1294
1295 static void
1296 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1297 {
1298         struct drm_device *dev = obj->dev;
1299         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1300         struct drm_gem_mm *mm = dev->mm_private;
1301         struct drm_map_list *list;
1302
1303         list = &obj->map_list;
1304         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1305
1306         if (list->file_offset_node) {
1307                 drm_mm_put_block(list->file_offset_node);
1308                 list->file_offset_node = NULL;
1309         }
1310
1311         if (list->map) {
1312                 kfree(list->map);
1313                 list->map = NULL;
1314         }
1315
1316         obj_priv->mmap_offset = 0;
1317 }
1318
1319 /**
1320  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1321  * @obj: object to check
1322  *
1323  * Return the required GTT alignment for an object, taking into account
1324  * potential fence register mapping if needed.
1325  */
1326 static uint32_t
1327 i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1328 {
1329         struct drm_device *dev = obj->dev;
1330         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1331         int start, i;
1332
1333         /*
1334          * Minimum alignment is 4k (GTT page size), but might be greater
1335          * if a fence register is needed for the object.
1336          */
1337         if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1338                 return 4096;
1339
1340         /*
1341          * Previous chips need to be aligned to the size of the smallest
1342          * fence register that can contain the object.
1343          */
1344         if (IS_I9XX(dev))
1345                 start = 1024*1024;
1346         else
1347                 start = 512*1024;
1348
1349         for (i = start; i < obj->size; i <<= 1)
1350                 ;
1351
1352         return i;
1353 }
1354
1355 /**
1356  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1357  * @dev: DRM device
1358  * @data: GTT mapping ioctl data
1359  * @file_priv: GEM object info
1360  *
1361  * Simply returns the fake offset to userspace so it can mmap it.
1362  * The mmap call will end up in drm_gem_mmap(), which will set things
1363  * up so we can get faults in the handler above.
1364  *
1365  * The fault handler will take care of binding the object into the GTT
1366  * (since it may have been evicted to make room for something), allocating
1367  * a fence register, and mapping the appropriate aperture address into
1368  * userspace.
1369  */
1370 int
1371 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1372                         struct drm_file *file_priv)
1373 {
1374         struct drm_i915_gem_mmap_gtt *args = data;
1375         struct drm_i915_private *dev_priv = dev->dev_private;
1376         struct drm_gem_object *obj;
1377         struct drm_i915_gem_object *obj_priv;
1378         int ret;
1379
1380         if (!(dev->driver->driver_features & DRIVER_GEM))
1381                 return -ENODEV;
1382
1383         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1384         if (obj == NULL)
1385                 return -EBADF;
1386
1387         mutex_lock(&dev->struct_mutex);
1388
1389         obj_priv = obj->driver_private;
1390
1391         if (!obj_priv->mmap_offset) {
1392                 ret = i915_gem_create_mmap_offset(obj);
1393                 if (ret) {
1394                         drm_gem_object_unreference(obj);
1395                         mutex_unlock(&dev->struct_mutex);
1396                         return ret;
1397                 }
1398         }
1399
1400         args->offset = obj_priv->mmap_offset;
1401
1402         obj_priv->gtt_alignment = i915_gem_get_gtt_alignment(obj);
1403
1404         /* Make sure the alignment is correct for fence regs etc */
1405         if (obj_priv->agp_mem &&
1406             (obj_priv->gtt_offset & (obj_priv->gtt_alignment - 1))) {
1407                 drm_gem_object_unreference(obj);
1408                 mutex_unlock(&dev->struct_mutex);
1409                 return -EINVAL;
1410         }
1411
1412         /*
1413          * Pull it into the GTT so that we have a page list (makes the
1414          * initial fault faster and any subsequent flushing possible).
1415          */
1416         if (!obj_priv->agp_mem) {
1417                 ret = i915_gem_object_bind_to_gtt(obj, obj_priv->gtt_alignment);
1418                 if (ret) {
1419                         drm_gem_object_unreference(obj);
1420                         mutex_unlock(&dev->struct_mutex);
1421                         return ret;
1422                 }
1423                 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1424         }
1425
1426         drm_gem_object_unreference(obj);
1427         mutex_unlock(&dev->struct_mutex);
1428
1429         return 0;
1430 }
1431
1432 void
1433 i915_gem_object_put_pages(struct drm_gem_object *obj)
1434 {
1435         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1436         int page_count = obj->size / PAGE_SIZE;
1437         int i;
1438
1439         BUG_ON(obj_priv->pages_refcount == 0);
1440
1441         if (--obj_priv->pages_refcount != 0)
1442                 return;
1443
1444         if (obj_priv->tiling_mode != I915_TILING_NONE)
1445                 i915_gem_object_save_bit_17_swizzle(obj);
1446
1447         for (i = 0; i < page_count; i++)
1448                 if (obj_priv->pages[i] != NULL) {
1449                         if (obj_priv->dirty)
1450                                 set_page_dirty(obj_priv->pages[i]);
1451                         mark_page_accessed(obj_priv->pages[i]);
1452                         page_cache_release(obj_priv->pages[i]);
1453                 }
1454         obj_priv->dirty = 0;
1455
1456         drm_free_large(obj_priv->pages);
1457         obj_priv->pages = NULL;
1458 }
1459
1460 static void
1461 i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno)
1462 {
1463         struct drm_device *dev = obj->dev;
1464         drm_i915_private_t *dev_priv = dev->dev_private;
1465         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1466
1467         /* Add a reference if we're newly entering the active list. */
1468         if (!obj_priv->active) {
1469                 drm_gem_object_reference(obj);
1470                 obj_priv->active = 1;
1471         }
1472         /* Move from whatever list we were on to the tail of execution. */
1473         spin_lock(&dev_priv->mm.active_list_lock);
1474         list_move_tail(&obj_priv->list,
1475                        &dev_priv->mm.active_list);
1476         spin_unlock(&dev_priv->mm.active_list_lock);
1477         obj_priv->last_rendering_seqno = seqno;
1478 }
1479
1480 static void
1481 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1482 {
1483         struct drm_device *dev = obj->dev;
1484         drm_i915_private_t *dev_priv = dev->dev_private;
1485         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1486
1487         BUG_ON(!obj_priv->active);
1488         list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1489         obj_priv->last_rendering_seqno = 0;
1490 }
1491
1492 static void
1493 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1494 {
1495         struct drm_device *dev = obj->dev;
1496         drm_i915_private_t *dev_priv = dev->dev_private;
1497         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1498
1499         i915_verify_inactive(dev, __FILE__, __LINE__);
1500         if (obj_priv->pin_count != 0)
1501                 list_del_init(&obj_priv->list);
1502         else
1503                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1504
1505         obj_priv->last_rendering_seqno = 0;
1506         if (obj_priv->active) {
1507                 obj_priv->active = 0;
1508                 drm_gem_object_unreference(obj);
1509         }
1510         i915_verify_inactive(dev, __FILE__, __LINE__);
1511 }
1512
1513 /**
1514  * Creates a new sequence number, emitting a write of it to the status page
1515  * plus an interrupt, which will trigger i915_user_interrupt_handler.
1516  *
1517  * Must be called with struct_lock held.
1518  *
1519  * Returned sequence numbers are nonzero on success.
1520  */
1521 static uint32_t
1522 i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
1523                  uint32_t flush_domains)
1524 {
1525         drm_i915_private_t *dev_priv = dev->dev_private;
1526         struct drm_i915_file_private *i915_file_priv = NULL;
1527         struct drm_i915_gem_request *request;
1528         uint32_t seqno;
1529         int was_empty;
1530         RING_LOCALS;
1531
1532         if (file_priv != NULL)
1533                 i915_file_priv = file_priv->driver_priv;
1534
1535         request = kzalloc(sizeof(*request), GFP_KERNEL);
1536         if (request == NULL)
1537                 return 0;
1538
1539         /* Grab the seqno we're going to make this request be, and bump the
1540          * next (skipping 0 so it can be the reserved no-seqno value).
1541          */
1542         seqno = dev_priv->mm.next_gem_seqno;
1543         dev_priv->mm.next_gem_seqno++;
1544         if (dev_priv->mm.next_gem_seqno == 0)
1545                 dev_priv->mm.next_gem_seqno++;
1546
1547         BEGIN_LP_RING(4);
1548         OUT_RING(MI_STORE_DWORD_INDEX);
1549         OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1550         OUT_RING(seqno);
1551
1552         OUT_RING(MI_USER_INTERRUPT);
1553         ADVANCE_LP_RING();
1554
1555         DRM_DEBUG("%d\n", seqno);
1556
1557         request->seqno = seqno;
1558         request->emitted_jiffies = jiffies;
1559         was_empty = list_empty(&dev_priv->mm.request_list);
1560         list_add_tail(&request->list, &dev_priv->mm.request_list);
1561         if (i915_file_priv) {
1562                 list_add_tail(&request->client_list,
1563                               &i915_file_priv->mm.request_list);
1564         } else {
1565                 INIT_LIST_HEAD(&request->client_list);
1566         }
1567
1568         /* Associate any objects on the flushing list matching the write
1569          * domain we're flushing with our flush.
1570          */
1571         if (flush_domains != 0) {
1572                 struct drm_i915_gem_object *obj_priv, *next;
1573
1574                 list_for_each_entry_safe(obj_priv, next,
1575                                          &dev_priv->mm.flushing_list, list) {
1576                         struct drm_gem_object *obj = obj_priv->obj;
1577
1578                         if ((obj->write_domain & flush_domains) ==
1579                             obj->write_domain) {
1580                                 obj->write_domain = 0;
1581                                 i915_gem_object_move_to_active(obj, seqno);
1582                         }
1583                 }
1584
1585         }
1586
1587         if (was_empty && !dev_priv->mm.suspended)
1588                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1589         return seqno;
1590 }
1591
1592 /**
1593  * Command execution barrier
1594  *
1595  * Ensures that all commands in the ring are finished
1596  * before signalling the CPU
1597  */
1598 static uint32_t
1599 i915_retire_commands(struct drm_device *dev)
1600 {
1601         drm_i915_private_t *dev_priv = dev->dev_private;
1602         uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1603         uint32_t flush_domains = 0;
1604         RING_LOCALS;
1605
1606         /* The sampler always gets flushed on i965 (sigh) */
1607         if (IS_I965G(dev))
1608                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1609         BEGIN_LP_RING(2);
1610         OUT_RING(cmd);
1611         OUT_RING(0); /* noop */
1612         ADVANCE_LP_RING();
1613         return flush_domains;
1614 }
1615
1616 /**
1617  * Moves buffers associated only with the given active seqno from the active
1618  * to inactive list, potentially freeing them.
1619  */
1620 static void
1621 i915_gem_retire_request(struct drm_device *dev,
1622                         struct drm_i915_gem_request *request)
1623 {
1624         drm_i915_private_t *dev_priv = dev->dev_private;
1625
1626         /* Move any buffers on the active list that are no longer referenced
1627          * by the ringbuffer to the flushing/inactive lists as appropriate.
1628          */
1629         spin_lock(&dev_priv->mm.active_list_lock);
1630         while (!list_empty(&dev_priv->mm.active_list)) {
1631                 struct drm_gem_object *obj;
1632                 struct drm_i915_gem_object *obj_priv;
1633
1634                 obj_priv = list_first_entry(&dev_priv->mm.active_list,
1635                                             struct drm_i915_gem_object,
1636                                             list);
1637                 obj = obj_priv->obj;
1638
1639                 /* If the seqno being retired doesn't match the oldest in the
1640                  * list, then the oldest in the list must still be newer than
1641                  * this seqno.
1642                  */
1643                 if (obj_priv->last_rendering_seqno != request->seqno)
1644                         goto out;
1645
1646 #if WATCH_LRU
1647                 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1648                          __func__, request->seqno, obj);
1649 #endif
1650
1651                 if (obj->write_domain != 0)
1652                         i915_gem_object_move_to_flushing(obj);
1653                 else {
1654                         /* Take a reference on the object so it won't be
1655                          * freed while the spinlock is held.  The list
1656                          * protection for this spinlock is safe when breaking
1657                          * the lock like this since the next thing we do
1658                          * is just get the head of the list again.
1659                          */
1660                         drm_gem_object_reference(obj);
1661                         i915_gem_object_move_to_inactive(obj);
1662                         spin_unlock(&dev_priv->mm.active_list_lock);
1663                         drm_gem_object_unreference(obj);
1664                         spin_lock(&dev_priv->mm.active_list_lock);
1665                 }
1666         }
1667 out:
1668         spin_unlock(&dev_priv->mm.active_list_lock);
1669 }
1670
1671 /**
1672  * Returns true if seq1 is later than seq2.
1673  */
1674 static int
1675 i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1676 {
1677         return (int32_t)(seq1 - seq2) >= 0;
1678 }
1679
1680 uint32_t
1681 i915_get_gem_seqno(struct drm_device *dev)
1682 {
1683         drm_i915_private_t *dev_priv = dev->dev_private;
1684
1685         return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
1686 }
1687
1688 /**
1689  * This function clears the request list as sequence numbers are passed.
1690  */
1691 void
1692 i915_gem_retire_requests(struct drm_device *dev)
1693 {
1694         drm_i915_private_t *dev_priv = dev->dev_private;
1695         uint32_t seqno;
1696
1697         if (!dev_priv->hw_status_page)
1698                 return;
1699
1700         seqno = i915_get_gem_seqno(dev);
1701
1702         while (!list_empty(&dev_priv->mm.request_list)) {
1703                 struct drm_i915_gem_request *request;
1704                 uint32_t retiring_seqno;
1705
1706                 request = list_first_entry(&dev_priv->mm.request_list,
1707                                            struct drm_i915_gem_request,
1708                                            list);
1709                 retiring_seqno = request->seqno;
1710
1711                 if (i915_seqno_passed(seqno, retiring_seqno) ||
1712                     dev_priv->mm.wedged) {
1713                         i915_gem_retire_request(dev, request);
1714
1715                         list_del(&request->list);
1716                         list_del(&request->client_list);
1717                         kfree(request);
1718                 } else
1719                         break;
1720         }
1721 }
1722
1723 void
1724 i915_gem_retire_work_handler(struct work_struct *work)
1725 {
1726         drm_i915_private_t *dev_priv;
1727         struct drm_device *dev;
1728
1729         dev_priv = container_of(work, drm_i915_private_t,
1730                                 mm.retire_work.work);
1731         dev = dev_priv->dev;
1732
1733         mutex_lock(&dev->struct_mutex);
1734         i915_gem_retire_requests(dev);
1735         if (!dev_priv->mm.suspended &&
1736             !list_empty(&dev_priv->mm.request_list))
1737                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1738         mutex_unlock(&dev->struct_mutex);
1739 }
1740
1741 /**
1742  * Waits for a sequence number to be signaled, and cleans up the
1743  * request and object lists appropriately for that event.
1744  */
1745 static int
1746 i915_wait_request(struct drm_device *dev, uint32_t seqno)
1747 {
1748         drm_i915_private_t *dev_priv = dev->dev_private;
1749         u32 ier;
1750         int ret = 0;
1751
1752         BUG_ON(seqno == 0);
1753
1754         if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
1755                 if (IS_IGDNG(dev))
1756                         ier = I915_READ(DEIER) | I915_READ(GTIER);
1757                 else
1758                         ier = I915_READ(IER);
1759                 if (!ier) {
1760                         DRM_ERROR("something (likely vbetool) disabled "
1761                                   "interrupts, re-enabling\n");
1762                         i915_driver_irq_preinstall(dev);
1763                         i915_driver_irq_postinstall(dev);
1764                 }
1765
1766                 dev_priv->mm.waiting_gem_seqno = seqno;
1767                 i915_user_irq_get(dev);
1768                 ret = wait_event_interruptible(dev_priv->irq_queue,
1769                                                i915_seqno_passed(i915_get_gem_seqno(dev),
1770                                                                  seqno) ||
1771                                                dev_priv->mm.wedged);
1772                 i915_user_irq_put(dev);
1773                 dev_priv->mm.waiting_gem_seqno = 0;
1774         }
1775         if (dev_priv->mm.wedged)
1776                 ret = -EIO;
1777
1778         if (ret && ret != -ERESTARTSYS)
1779                 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
1780                           __func__, ret, seqno, i915_get_gem_seqno(dev));
1781
1782         /* Directly dispatch request retiring.  While we have the work queue
1783          * to handle this, the waiter on a request often wants an associated
1784          * buffer to have made it to the inactive list, and we would need
1785          * a separate wait queue to handle that.
1786          */
1787         if (ret == 0)
1788                 i915_gem_retire_requests(dev);
1789
1790         return ret;
1791 }
1792
1793 static void
1794 i915_gem_flush(struct drm_device *dev,
1795                uint32_t invalidate_domains,
1796                uint32_t flush_domains)
1797 {
1798         drm_i915_private_t *dev_priv = dev->dev_private;
1799         uint32_t cmd;
1800         RING_LOCALS;
1801
1802 #if WATCH_EXEC
1803         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
1804                   invalidate_domains, flush_domains);
1805 #endif
1806
1807         if (flush_domains & I915_GEM_DOMAIN_CPU)
1808                 drm_agp_chipset_flush(dev);
1809
1810         if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
1811                 /*
1812                  * read/write caches:
1813                  *
1814                  * I915_GEM_DOMAIN_RENDER is always invalidated, but is
1815                  * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
1816                  * also flushed at 2d versus 3d pipeline switches.
1817                  *
1818                  * read-only caches:
1819                  *
1820                  * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
1821                  * MI_READ_FLUSH is set, and is always flushed on 965.
1822                  *
1823                  * I915_GEM_DOMAIN_COMMAND may not exist?
1824                  *
1825                  * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
1826                  * invalidated when MI_EXE_FLUSH is set.
1827                  *
1828                  * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
1829                  * invalidated with every MI_FLUSH.
1830                  *
1831                  * TLBs:
1832                  *
1833                  * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
1834                  * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
1835                  * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
1836                  * are flushed at any MI_FLUSH.
1837                  */
1838
1839                 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1840                 if ((invalidate_domains|flush_domains) &
1841                     I915_GEM_DOMAIN_RENDER)
1842                         cmd &= ~MI_NO_WRITE_FLUSH;
1843                 if (!IS_I965G(dev)) {
1844                         /*
1845                          * On the 965, the sampler cache always gets flushed
1846                          * and this bit is reserved.
1847                          */
1848                         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
1849                                 cmd |= MI_READ_FLUSH;
1850                 }
1851                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
1852                         cmd |= MI_EXE_FLUSH;
1853
1854 #if WATCH_EXEC
1855                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
1856 #endif
1857                 BEGIN_LP_RING(2);
1858                 OUT_RING(cmd);
1859                 OUT_RING(0); /* noop */
1860                 ADVANCE_LP_RING();
1861         }
1862 }
1863
1864 /**
1865  * Ensures that all rendering to the object has completed and the object is
1866  * safe to unbind from the GTT or access from the CPU.
1867  */
1868 static int
1869 i915_gem_object_wait_rendering(struct drm_gem_object *obj)
1870 {
1871         struct drm_device *dev = obj->dev;
1872         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1873         int ret;
1874
1875         /* This function only exists to support waiting for existing rendering,
1876          * not for emitting required flushes.
1877          */
1878         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
1879
1880         /* If there is rendering queued on the buffer being evicted, wait for
1881          * it.
1882          */
1883         if (obj_priv->active) {
1884 #if WATCH_BUF
1885                 DRM_INFO("%s: object %p wait for seqno %08x\n",
1886                           __func__, obj, obj_priv->last_rendering_seqno);
1887 #endif
1888                 ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
1889                 if (ret != 0)
1890                         return ret;
1891         }
1892
1893         return 0;
1894 }
1895
1896 /**
1897  * Unbinds an object from the GTT aperture.
1898  */
1899 int
1900 i915_gem_object_unbind(struct drm_gem_object *obj)
1901 {
1902         struct drm_device *dev = obj->dev;
1903         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1904         int ret = 0;
1905
1906 #if WATCH_BUF
1907         DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
1908         DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
1909 #endif
1910         if (obj_priv->gtt_space == NULL)
1911                 return 0;
1912
1913         if (obj_priv->pin_count != 0) {
1914                 DRM_ERROR("Attempting to unbind pinned buffer\n");
1915                 return -EINVAL;
1916         }
1917
1918         /* Move the object to the CPU domain to ensure that
1919          * any possible CPU writes while it's not in the GTT
1920          * are flushed when we go to remap it. This will
1921          * also ensure that all pending GPU writes are finished
1922          * before we unbind.
1923          */
1924         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1925         if (ret) {
1926                 if (ret != -ERESTARTSYS)
1927                         DRM_ERROR("set_domain failed: %d\n", ret);
1928                 return ret;
1929         }
1930
1931         if (obj_priv->agp_mem != NULL) {
1932                 drm_unbind_agp(obj_priv->agp_mem);
1933                 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
1934                 obj_priv->agp_mem = NULL;
1935         }
1936
1937         BUG_ON(obj_priv->active);
1938
1939         /* blow away mappings if mapped through GTT */
1940         i915_gem_release_mmap(obj);
1941
1942         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
1943                 i915_gem_clear_fence_reg(obj);
1944
1945         i915_gem_object_put_pages(obj);
1946
1947         if (obj_priv->gtt_space) {
1948                 atomic_dec(&dev->gtt_count);
1949                 atomic_sub(obj->size, &dev->gtt_memory);
1950
1951                 drm_mm_put_block(obj_priv->gtt_space);
1952                 obj_priv->gtt_space = NULL;
1953         }
1954
1955         /* Remove ourselves from the LRU list if present. */
1956         if (!list_empty(&obj_priv->list))
1957                 list_del_init(&obj_priv->list);
1958
1959         return 0;
1960 }
1961
1962 static int
1963 i915_gem_evict_something(struct drm_device *dev)
1964 {
1965         drm_i915_private_t *dev_priv = dev->dev_private;
1966         struct drm_gem_object *obj;
1967         struct drm_i915_gem_object *obj_priv;
1968         int ret = 0;
1969
1970         for (;;) {
1971                 /* If there's an inactive buffer available now, grab it
1972                  * and be done.
1973                  */
1974                 if (!list_empty(&dev_priv->mm.inactive_list)) {
1975                         obj_priv = list_first_entry(&dev_priv->mm.inactive_list,
1976                                                     struct drm_i915_gem_object,
1977                                                     list);
1978                         obj = obj_priv->obj;
1979                         BUG_ON(obj_priv->pin_count != 0);
1980 #if WATCH_LRU
1981                         DRM_INFO("%s: evicting %p\n", __func__, obj);
1982 #endif
1983                         BUG_ON(obj_priv->active);
1984
1985                         /* Wait on the rendering and unbind the buffer. */
1986                         ret = i915_gem_object_unbind(obj);
1987                         break;
1988                 }
1989
1990                 /* If we didn't get anything, but the ring is still processing
1991                  * things, wait for one of those things to finish and hopefully
1992                  * leave us a buffer to evict.
1993                  */
1994                 if (!list_empty(&dev_priv->mm.request_list)) {
1995                         struct drm_i915_gem_request *request;
1996
1997                         request = list_first_entry(&dev_priv->mm.request_list,
1998                                                    struct drm_i915_gem_request,
1999                                                    list);
2000
2001                         ret = i915_wait_request(dev, request->seqno);
2002                         if (ret)
2003                                 break;
2004
2005                         /* if waiting caused an object to become inactive,
2006                          * then loop around and wait for it. Otherwise, we
2007                          * assume that waiting freed and unbound something,
2008                          * so there should now be some space in the GTT
2009                          */
2010                         if (!list_empty(&dev_priv->mm.inactive_list))
2011                                 continue;
2012                         break;
2013                 }
2014
2015                 /* If we didn't have anything on the request list but there
2016                  * are buffers awaiting a flush, emit one and try again.
2017                  * When we wait on it, those buffers waiting for that flush
2018                  * will get moved to inactive.
2019                  */
2020                 if (!list_empty(&dev_priv->mm.flushing_list)) {
2021                         obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
2022                                                     struct drm_i915_gem_object,
2023                                                     list);
2024                         obj = obj_priv->obj;
2025
2026                         i915_gem_flush(dev,
2027                                        obj->write_domain,
2028                                        obj->write_domain);
2029                         i915_add_request(dev, NULL, obj->write_domain);
2030
2031                         obj = NULL;
2032                         continue;
2033                 }
2034
2035                 DRM_ERROR("inactive empty %d request empty %d "
2036                           "flushing empty %d\n",
2037                           list_empty(&dev_priv->mm.inactive_list),
2038                           list_empty(&dev_priv->mm.request_list),
2039                           list_empty(&dev_priv->mm.flushing_list));
2040                 /* If we didn't do any of the above, there's nothing to be done
2041                  * and we just can't fit it in.
2042                  */
2043                 return -ENOSPC;
2044         }
2045         return ret;
2046 }
2047
2048 static int
2049 i915_gem_evict_everything(struct drm_device *dev)
2050 {
2051         int ret;
2052
2053         for (;;) {
2054                 ret = i915_gem_evict_something(dev);
2055                 if (ret != 0)
2056                         break;
2057         }
2058         if (ret == -ENOSPC)
2059                 return 0;
2060         return ret;
2061 }
2062
2063 int
2064 i915_gem_object_get_pages(struct drm_gem_object *obj)
2065 {
2066         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2067         int page_count, i;
2068         struct address_space *mapping;
2069         struct inode *inode;
2070         struct page *page;
2071         int ret;
2072
2073         if (obj_priv->pages_refcount++ != 0)
2074                 return 0;
2075
2076         /* Get the list of pages out of our struct file.  They'll be pinned
2077          * at this point until we release them.
2078          */
2079         page_count = obj->size / PAGE_SIZE;
2080         BUG_ON(obj_priv->pages != NULL);
2081         obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2082         if (obj_priv->pages == NULL) {
2083                 DRM_ERROR("Faled to allocate page list\n");
2084                 obj_priv->pages_refcount--;
2085                 return -ENOMEM;
2086         }
2087
2088         inode = obj->filp->f_path.dentry->d_inode;
2089         mapping = inode->i_mapping;
2090         for (i = 0; i < page_count; i++) {
2091                 page = read_mapping_page(mapping, i, NULL);
2092                 if (IS_ERR(page)) {
2093                         ret = PTR_ERR(page);
2094                         DRM_ERROR("read_mapping_page failed: %d\n", ret);
2095                         i915_gem_object_put_pages(obj);
2096                         return ret;
2097                 }
2098                 obj_priv->pages[i] = page;
2099         }
2100
2101         if (obj_priv->tiling_mode != I915_TILING_NONE)
2102                 i915_gem_object_do_bit_17_swizzle(obj);
2103
2104         return 0;
2105 }
2106
2107 static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2108 {
2109         struct drm_gem_object *obj = reg->obj;
2110         struct drm_device *dev = obj->dev;
2111         drm_i915_private_t *dev_priv = dev->dev_private;
2112         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2113         int regnum = obj_priv->fence_reg;
2114         uint64_t val;
2115
2116         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2117                     0xfffff000) << 32;
2118         val |= obj_priv->gtt_offset & 0xfffff000;
2119         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2120         if (obj_priv->tiling_mode == I915_TILING_Y)
2121                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2122         val |= I965_FENCE_REG_VALID;
2123
2124         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2125 }
2126
2127 static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2128 {
2129         struct drm_gem_object *obj = reg->obj;
2130         struct drm_device *dev = obj->dev;
2131         drm_i915_private_t *dev_priv = dev->dev_private;
2132         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2133         int regnum = obj_priv->fence_reg;
2134         int tile_width;
2135         uint32_t fence_reg, val;
2136         uint32_t pitch_val;
2137
2138         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2139             (obj_priv->gtt_offset & (obj->size - 1))) {
2140                 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
2141                      __func__, obj_priv->gtt_offset, obj->size);
2142                 return;
2143         }
2144
2145         if (obj_priv->tiling_mode == I915_TILING_Y &&
2146             HAS_128_BYTE_Y_TILING(dev))
2147                 tile_width = 128;
2148         else
2149                 tile_width = 512;
2150
2151         /* Note: pitch better be a power of two tile widths */
2152         pitch_val = obj_priv->stride / tile_width;
2153         pitch_val = ffs(pitch_val) - 1;
2154
2155         val = obj_priv->gtt_offset;
2156         if (obj_priv->tiling_mode == I915_TILING_Y)
2157                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2158         val |= I915_FENCE_SIZE_BITS(obj->size);
2159         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2160         val |= I830_FENCE_REG_VALID;
2161
2162         if (regnum < 8)
2163                 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2164         else
2165                 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2166         I915_WRITE(fence_reg, val);
2167 }
2168
2169 static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2170 {
2171         struct drm_gem_object *obj = reg->obj;
2172         struct drm_device *dev = obj->dev;
2173         drm_i915_private_t *dev_priv = dev->dev_private;
2174         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2175         int regnum = obj_priv->fence_reg;
2176         uint32_t val;
2177         uint32_t pitch_val;
2178         uint32_t fence_size_bits;
2179
2180         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2181             (obj_priv->gtt_offset & (obj->size - 1))) {
2182                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2183                      __func__, obj_priv->gtt_offset);
2184                 return;
2185         }
2186
2187         pitch_val = obj_priv->stride / 128;
2188         pitch_val = ffs(pitch_val) - 1;
2189         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2190
2191         val = obj_priv->gtt_offset;
2192         if (obj_priv->tiling_mode == I915_TILING_Y)
2193                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2194         fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2195         WARN_ON(fence_size_bits & ~0x00000f00);
2196         val |= fence_size_bits;
2197         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2198         val |= I830_FENCE_REG_VALID;
2199
2200         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2201 }
2202
2203 /**
2204  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2205  * @obj: object to map through a fence reg
2206  *
2207  * When mapping objects through the GTT, userspace wants to be able to write
2208  * to them without having to worry about swizzling if the object is tiled.
2209  *
2210  * This function walks the fence regs looking for a free one for @obj,
2211  * stealing one if it can't find any.
2212  *
2213  * It then sets up the reg based on the object's properties: address, pitch
2214  * and tiling format.
2215  */
2216 int
2217 i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
2218 {
2219         struct drm_device *dev = obj->dev;
2220         struct drm_i915_private *dev_priv = dev->dev_private;
2221         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2222         struct drm_i915_fence_reg *reg = NULL;
2223         struct drm_i915_gem_object *old_obj_priv = NULL;
2224         int i, ret, avail;
2225
2226         /* Just update our place in the LRU if our fence is getting used. */
2227         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2228                 list_move_tail(&obj_priv->fence_list, &dev_priv->mm.fence_list);
2229                 return 0;
2230         }
2231
2232         switch (obj_priv->tiling_mode) {
2233         case I915_TILING_NONE:
2234                 WARN(1, "allocating a fence for non-tiled object?\n");
2235                 break;
2236         case I915_TILING_X:
2237                 if (!obj_priv->stride)
2238                         return -EINVAL;
2239                 WARN((obj_priv->stride & (512 - 1)),
2240                      "object 0x%08x is X tiled but has non-512B pitch\n",
2241                      obj_priv->gtt_offset);
2242                 break;
2243         case I915_TILING_Y:
2244                 if (!obj_priv->stride)
2245                         return -EINVAL;
2246                 WARN((obj_priv->stride & (128 - 1)),
2247                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2248                      obj_priv->gtt_offset);
2249                 break;
2250         }
2251
2252         /* First try to find a free reg */
2253         avail = 0;
2254         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2255                 reg = &dev_priv->fence_regs[i];
2256                 if (!reg->obj)
2257                         break;
2258
2259                 old_obj_priv = reg->obj->driver_private;
2260                 if (!old_obj_priv->pin_count)
2261                     avail++;
2262         }
2263
2264         /* None available, try to steal one or wait for a user to finish */
2265         if (i == dev_priv->num_fence_regs) {
2266                 struct drm_gem_object *old_obj = NULL;
2267
2268                 if (avail == 0)
2269                         return -ENOSPC;
2270
2271                 list_for_each_entry(old_obj_priv, &dev_priv->mm.fence_list,
2272                                     fence_list) {
2273                         old_obj = old_obj_priv->obj;
2274
2275                         if (old_obj_priv->pin_count)
2276                                 continue;
2277
2278                         /* Take a reference, as otherwise the wait_rendering
2279                          * below may cause the object to get freed out from
2280                          * under us.
2281                          */
2282                         drm_gem_object_reference(old_obj);
2283
2284                         /* i915 uses fences for GPU access to tiled buffers */
2285                         if (IS_I965G(dev) || !old_obj_priv->active)
2286                                 break;
2287
2288                         /* This brings the object to the head of the LRU if it
2289                          * had been written to.  The only way this should
2290                          * result in us waiting longer than the expected
2291                          * optimal amount of time is if there was a
2292                          * fence-using buffer later that was read-only.
2293                          */
2294                         i915_gem_object_flush_gpu_write_domain(old_obj);
2295                         ret = i915_gem_object_wait_rendering(old_obj);
2296                         if (ret != 0) {
2297                                 drm_gem_object_unreference(old_obj);
2298                                 return ret;
2299                         }
2300
2301                         break;
2302                 }
2303
2304                 /*
2305                  * Zap this virtual mapping so we can set up a fence again
2306                  * for this object next time we need it.
2307                  */
2308                 i915_gem_release_mmap(old_obj);
2309
2310                 i = old_obj_priv->fence_reg;
2311                 reg = &dev_priv->fence_regs[i];
2312
2313                 old_obj_priv->fence_reg = I915_FENCE_REG_NONE;
2314                 list_del_init(&old_obj_priv->fence_list);
2315
2316                 drm_gem_object_unreference(old_obj);
2317         }
2318
2319         obj_priv->fence_reg = i;
2320         list_add_tail(&obj_priv->fence_list, &dev_priv->mm.fence_list);
2321
2322         reg->obj = obj;
2323
2324         if (IS_I965G(dev))
2325                 i965_write_fence_reg(reg);
2326         else if (IS_I9XX(dev))
2327                 i915_write_fence_reg(reg);
2328         else
2329                 i830_write_fence_reg(reg);
2330
2331         return 0;
2332 }
2333
2334 /**
2335  * i915_gem_clear_fence_reg - clear out fence register info
2336  * @obj: object to clear
2337  *
2338  * Zeroes out the fence register itself and clears out the associated
2339  * data structures in dev_priv and obj_priv.
2340  */
2341 static void
2342 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2343 {
2344         struct drm_device *dev = obj->dev;
2345         drm_i915_private_t *dev_priv = dev->dev_private;
2346         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2347
2348         if (IS_I965G(dev))
2349                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2350         else {
2351                 uint32_t fence_reg;
2352
2353                 if (obj_priv->fence_reg < 8)
2354                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2355                 else
2356                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
2357                                                        8) * 4;
2358
2359                 I915_WRITE(fence_reg, 0);
2360         }
2361
2362         dev_priv->fence_regs[obj_priv->fence_reg].obj = NULL;
2363         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2364         list_del_init(&obj_priv->fence_list);
2365 }
2366
2367 /**
2368  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2369  * to the buffer to finish, and then resets the fence register.
2370  * @obj: tiled object holding a fence register.
2371  *
2372  * Zeroes out the fence register itself and clears out the associated
2373  * data structures in dev_priv and obj_priv.
2374  */
2375 int
2376 i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
2377 {
2378         struct drm_device *dev = obj->dev;
2379         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2380
2381         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2382                 return 0;
2383
2384         /* On the i915, GPU access to tiled buffers is via a fence,
2385          * therefore we must wait for any outstanding access to complete
2386          * before clearing the fence.
2387          */
2388         if (!IS_I965G(dev)) {
2389                 int ret;
2390
2391                 i915_gem_object_flush_gpu_write_domain(obj);
2392                 i915_gem_object_flush_gtt_write_domain(obj);
2393                 ret = i915_gem_object_wait_rendering(obj);
2394                 if (ret != 0)
2395                         return ret;
2396         }
2397
2398         i915_gem_clear_fence_reg (obj);
2399
2400         return 0;
2401 }
2402
2403 /**
2404  * Finds free space in the GTT aperture and binds the object there.
2405  */
2406 static int
2407 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2408 {
2409         struct drm_device *dev = obj->dev;
2410         drm_i915_private_t *dev_priv = dev->dev_private;
2411         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2412         struct drm_mm_node *free_space;
2413         int page_count, ret;
2414
2415         if (dev_priv->mm.suspended)
2416                 return -EBUSY;
2417         if (alignment == 0)
2418                 alignment = i915_gem_get_gtt_alignment(obj);
2419         if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
2420                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2421                 return -EINVAL;
2422         }
2423
2424  search_free:
2425         free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2426                                         obj->size, alignment, 0);
2427         if (free_space != NULL) {
2428                 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2429                                                        alignment);
2430                 if (obj_priv->gtt_space != NULL) {
2431                         obj_priv->gtt_space->private = obj;
2432                         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2433                 }
2434         }
2435         if (obj_priv->gtt_space == NULL) {
2436                 bool lists_empty;
2437
2438                 /* If the gtt is empty and we're still having trouble
2439                  * fitting our object in, we're out of memory.
2440                  */
2441 #if WATCH_LRU
2442                 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2443 #endif
2444                 spin_lock(&dev_priv->mm.active_list_lock);
2445                 lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2446                                list_empty(&dev_priv->mm.flushing_list) &&
2447                                list_empty(&dev_priv->mm.active_list));
2448                 spin_unlock(&dev_priv->mm.active_list_lock);
2449                 if (lists_empty) {
2450                         DRM_ERROR("GTT full, but LRU list empty\n");
2451                         return -ENOSPC;
2452                 }
2453
2454                 ret = i915_gem_evict_something(dev);
2455                 if (ret != 0) {
2456                         if (ret != -ERESTARTSYS)
2457                                 DRM_ERROR("Failed to evict a buffer %d\n", ret);
2458                         return ret;
2459                 }
2460                 goto search_free;
2461         }
2462
2463 #if WATCH_BUF
2464         DRM_INFO("Binding object of size %zd at 0x%08x\n",
2465                  obj->size, obj_priv->gtt_offset);
2466 #endif
2467         ret = i915_gem_object_get_pages(obj);
2468         if (ret) {
2469                 drm_mm_put_block(obj_priv->gtt_space);
2470                 obj_priv->gtt_space = NULL;
2471                 return ret;
2472         }
2473
2474         page_count = obj->size / PAGE_SIZE;
2475         /* Create an AGP memory structure pointing at our pages, and bind it
2476          * into the GTT.
2477          */
2478         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2479                                                obj_priv->pages,
2480                                                page_count,
2481                                                obj_priv->gtt_offset,
2482                                                obj_priv->agp_type);
2483         if (obj_priv->agp_mem == NULL) {
2484                 i915_gem_object_put_pages(obj);
2485                 drm_mm_put_block(obj_priv->gtt_space);
2486                 obj_priv->gtt_space = NULL;
2487                 return -ENOMEM;
2488         }
2489         atomic_inc(&dev->gtt_count);
2490         atomic_add(obj->size, &dev->gtt_memory);
2491
2492         /* Assert that the object is not currently in any GPU domain. As it
2493          * wasn't in the GTT, there shouldn't be any way it could have been in
2494          * a GPU cache
2495          */
2496         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2497         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2498
2499         return 0;
2500 }
2501
2502 void
2503 i915_gem_clflush_object(struct drm_gem_object *obj)
2504 {
2505         struct drm_i915_gem_object      *obj_priv = obj->driver_private;
2506
2507         /* If we don't have a page list set up, then we're not pinned
2508          * to GPU, and we can ignore the cache flush because it'll happen
2509          * again at bind time.
2510          */
2511         if (obj_priv->pages == NULL)
2512                 return;
2513
2514         /* XXX: The 865 in particular appears to be weird in how it handles
2515          * cache flushing.  We haven't figured it out, but the
2516          * clflush+agp_chipset_flush doesn't appear to successfully get the
2517          * data visible to the PGU, while wbinvd + agp_chipset_flush does.
2518          */
2519         if (IS_I865G(obj->dev)) {
2520                 wbinvd();
2521                 return;
2522         }
2523
2524         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2525 }
2526
2527 /** Flushes any GPU write domain for the object if it's dirty. */
2528 static void
2529 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2530 {
2531         struct drm_device *dev = obj->dev;
2532         uint32_t seqno;
2533
2534         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2535                 return;
2536
2537         /* Queue the GPU write cache flushing we need. */
2538         i915_gem_flush(dev, 0, obj->write_domain);
2539         seqno = i915_add_request(dev, NULL, obj->write_domain);
2540         obj->write_domain = 0;
2541         i915_gem_object_move_to_active(obj, seqno);
2542 }
2543
2544 /** Flushes the GTT write domain for the object if it's dirty. */
2545 static void
2546 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2547 {
2548         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2549                 return;
2550
2551         /* No actual flushing is required for the GTT write domain.   Writes
2552          * to it immediately go to main memory as far as we know, so there's
2553          * no chipset flush.  It also doesn't land in render cache.
2554          */
2555         obj->write_domain = 0;
2556 }
2557
2558 /** Flushes the CPU write domain for the object if it's dirty. */
2559 static void
2560 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2561 {
2562         struct drm_device *dev = obj->dev;
2563
2564         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2565                 return;
2566
2567         i915_gem_clflush_object(obj);
2568         drm_agp_chipset_flush(dev);
2569         obj->write_domain = 0;
2570 }
2571
2572 /**
2573  * Moves a single object to the GTT read, and possibly write domain.
2574  *
2575  * This function returns when the move is complete, including waiting on
2576  * flushes to occur.
2577  */
2578 int
2579 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2580 {
2581         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2582         int ret;
2583
2584         /* Not valid to be called on unbound objects. */
2585         if (obj_priv->gtt_space == NULL)
2586                 return -EINVAL;
2587
2588         i915_gem_object_flush_gpu_write_domain(obj);
2589         /* Wait on any GPU rendering and flushing to occur. */
2590         ret = i915_gem_object_wait_rendering(obj);
2591         if (ret != 0)
2592                 return ret;
2593
2594         /* If we're writing through the GTT domain, then CPU and GPU caches
2595          * will need to be invalidated at next use.
2596          */
2597         if (write)
2598                 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2599
2600         i915_gem_object_flush_cpu_write_domain(obj);
2601
2602         /* It should now be out of any other write domains, and we can update
2603          * the domain values for our changes.
2604          */
2605         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2606         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2607         if (write) {
2608                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2609                 obj_priv->dirty = 1;
2610         }
2611
2612         return 0;
2613 }
2614
2615 /**
2616  * Moves a single object to the CPU read, and possibly write domain.
2617  *
2618  * This function returns when the move is complete, including waiting on
2619  * flushes to occur.
2620  */
2621 static int
2622 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2623 {
2624         int ret;
2625
2626         i915_gem_object_flush_gpu_write_domain(obj);
2627         /* Wait on any GPU rendering and flushing to occur. */
2628         ret = i915_gem_object_wait_rendering(obj);
2629         if (ret != 0)
2630                 return ret;
2631
2632         i915_gem_object_flush_gtt_write_domain(obj);
2633
2634         /* If we have a partially-valid cache of the object in the CPU,
2635          * finish invalidating it and free the per-page flags.
2636          */
2637         i915_gem_object_set_to_full_cpu_read_domain(obj);
2638
2639         /* Flush the CPU cache if it's still invalid. */
2640         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2641                 i915_gem_clflush_object(obj);
2642
2643                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2644         }
2645
2646         /* It should now be out of any other write domains, and we can update
2647          * the domain values for our changes.
2648          */
2649         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2650
2651         /* If we're writing through the CPU, then the GPU read domains will
2652          * need to be invalidated at next use.
2653          */
2654         if (write) {
2655                 obj->read_domains &= I915_GEM_DOMAIN_CPU;
2656                 obj->write_domain = I915_GEM_DOMAIN_CPU;
2657         }
2658
2659         return 0;
2660 }
2661
2662 /*
2663  * Set the next domain for the specified object. This
2664  * may not actually perform the necessary flushing/invaliding though,
2665  * as that may want to be batched with other set_domain operations
2666  *
2667  * This is (we hope) the only really tricky part of gem. The goal
2668  * is fairly simple -- track which caches hold bits of the object
2669  * and make sure they remain coherent. A few concrete examples may
2670  * help to explain how it works. For shorthand, we use the notation
2671  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2672  * a pair of read and write domain masks.
2673  *
2674  * Case 1: the batch buffer
2675  *
2676  *      1. Allocated
2677  *      2. Written by CPU
2678  *      3. Mapped to GTT
2679  *      4. Read by GPU
2680  *      5. Unmapped from GTT
2681  *      6. Freed
2682  *
2683  *      Let's take these a step at a time
2684  *
2685  *      1. Allocated
2686  *              Pages allocated from the kernel may still have
2687  *              cache contents, so we set them to (CPU, CPU) always.
2688  *      2. Written by CPU (using pwrite)
2689  *              The pwrite function calls set_domain (CPU, CPU) and
2690  *              this function does nothing (as nothing changes)
2691  *      3. Mapped by GTT
2692  *              This function asserts that the object is not
2693  *              currently in any GPU-based read or write domains
2694  *      4. Read by GPU
2695  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
2696  *              As write_domain is zero, this function adds in the
2697  *              current read domains (CPU+COMMAND, 0).
2698  *              flush_domains is set to CPU.
2699  *              invalidate_domains is set to COMMAND
2700  *              clflush is run to get data out of the CPU caches
2701  *              then i915_dev_set_domain calls i915_gem_flush to
2702  *              emit an MI_FLUSH and drm_agp_chipset_flush
2703  *      5. Unmapped from GTT
2704  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
2705  *              flush_domains and invalidate_domains end up both zero
2706  *              so no flushing/invalidating happens
2707  *      6. Freed
2708  *              yay, done
2709  *
2710  * Case 2: The shared render buffer
2711  *
2712  *      1. Allocated
2713  *      2. Mapped to GTT
2714  *      3. Read/written by GPU
2715  *      4. set_domain to (CPU,CPU)
2716  *      5. Read/written by CPU
2717  *      6. Read/written by GPU
2718  *
2719  *      1. Allocated
2720  *              Same as last example, (CPU, CPU)
2721  *      2. Mapped to GTT
2722  *              Nothing changes (assertions find that it is not in the GPU)
2723  *      3. Read/written by GPU
2724  *              execbuffer calls set_domain (RENDER, RENDER)
2725  *              flush_domains gets CPU
2726  *              invalidate_domains gets GPU
2727  *              clflush (obj)
2728  *              MI_FLUSH and drm_agp_chipset_flush
2729  *      4. set_domain (CPU, CPU)
2730  *              flush_domains gets GPU
2731  *              invalidate_domains gets CPU
2732  *              wait_rendering (obj) to make sure all drawing is complete.
2733  *              This will include an MI_FLUSH to get the data from GPU
2734  *              to memory
2735  *              clflush (obj) to invalidate the CPU cache
2736  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
2737  *      5. Read/written by CPU
2738  *              cache lines are loaded and dirtied
2739  *      6. Read written by GPU
2740  *              Same as last GPU access
2741  *
2742  * Case 3: The constant buffer
2743  *
2744  *      1. Allocated
2745  *      2. Written by CPU
2746  *      3. Read by GPU
2747  *      4. Updated (written) by CPU again
2748  *      5. Read by GPU
2749  *
2750  *      1. Allocated
2751  *              (CPU, CPU)
2752  *      2. Written by CPU
2753  *              (CPU, CPU)
2754  *      3. Read by GPU
2755  *              (CPU+RENDER, 0)
2756  *              flush_domains = CPU
2757  *              invalidate_domains = RENDER
2758  *              clflush (obj)
2759  *              MI_FLUSH
2760  *              drm_agp_chipset_flush
2761  *      4. Updated (written) by CPU again
2762  *              (CPU, CPU)
2763  *              flush_domains = 0 (no previous write domain)
2764  *              invalidate_domains = 0 (no new read domains)
2765  *      5. Read by GPU
2766  *              (CPU+RENDER, 0)
2767  *              flush_domains = CPU
2768  *              invalidate_domains = RENDER
2769  *              clflush (obj)
2770  *              MI_FLUSH
2771  *              drm_agp_chipset_flush
2772  */
2773 static void
2774 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
2775 {
2776         struct drm_device               *dev = obj->dev;
2777         struct drm_i915_gem_object      *obj_priv = obj->driver_private;
2778         uint32_t                        invalidate_domains = 0;
2779         uint32_t                        flush_domains = 0;
2780
2781         BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
2782         BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
2783
2784         intel_mark_busy(dev, obj);
2785
2786 #if WATCH_BUF
2787         DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
2788                  __func__, obj,
2789                  obj->read_domains, obj->pending_read_domains,
2790                  obj->write_domain, obj->pending_write_domain);
2791 #endif
2792         /*
2793          * If the object isn't moving to a new write domain,
2794          * let the object stay in multiple read domains
2795          */
2796         if (obj->pending_write_domain == 0)
2797                 obj->pending_read_domains |= obj->read_domains;
2798         else
2799                 obj_priv->dirty = 1;
2800
2801         /*
2802          * Flush the current write domain if
2803          * the new read domains don't match. Invalidate
2804          * any read domains which differ from the old
2805          * write domain
2806          */
2807         if (obj->write_domain &&
2808             obj->write_domain != obj->pending_read_domains) {
2809                 flush_domains |= obj->write_domain;
2810                 invalidate_domains |=
2811                         obj->pending_read_domains & ~obj->write_domain;
2812         }
2813         /*
2814          * Invalidate any read caches which may have
2815          * stale data. That is, any new read domains.
2816          */
2817         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
2818         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
2819 #if WATCH_BUF
2820                 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
2821                          __func__, flush_domains, invalidate_domains);
2822 #endif
2823                 i915_gem_clflush_object(obj);
2824         }
2825
2826         /* The actual obj->write_domain will be updated with
2827          * pending_write_domain after we emit the accumulated flush for all
2828          * of our domain changes in execbuffers (which clears objects'
2829          * write_domains).  So if we have a current write domain that we
2830          * aren't changing, set pending_write_domain to that.
2831          */
2832         if (flush_domains == 0 && obj->pending_write_domain == 0)
2833                 obj->pending_write_domain = obj->write_domain;
2834         obj->read_domains = obj->pending_read_domains;
2835
2836         dev->invalidate_domains |= invalidate_domains;
2837         dev->flush_domains |= flush_domains;
2838 #if WATCH_BUF
2839         DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
2840                  __func__,
2841                  obj->read_domains, obj->write_domain,
2842                  dev->invalidate_domains, dev->flush_domains);
2843 #endif
2844 }
2845
2846 /**
2847  * Moves the object from a partially CPU read to a full one.
2848  *
2849  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
2850  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
2851  */
2852 static void
2853 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
2854 {
2855         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2856
2857         if (!obj_priv->page_cpu_valid)
2858                 return;
2859
2860         /* If we're partially in the CPU read domain, finish moving it in.
2861          */
2862         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
2863                 int i;
2864
2865                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
2866                         if (obj_priv->page_cpu_valid[i])
2867                                 continue;
2868                         drm_clflush_pages(obj_priv->pages + i, 1);
2869                 }
2870         }
2871
2872         /* Free the page_cpu_valid mappings which are now stale, whether
2873          * or not we've got I915_GEM_DOMAIN_CPU.
2874          */
2875         kfree(obj_priv->page_cpu_valid);
2876         obj_priv->page_cpu_valid = NULL;
2877 }
2878
2879 /**
2880  * Set the CPU read domain on a range of the object.
2881  *
2882  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
2883  * not entirely valid.  The page_cpu_valid member of the object flags which
2884  * pages have been flushed, and will be respected by
2885  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
2886  * of the whole object.
2887  *
2888  * This function returns when the move is complete, including waiting on
2889  * flushes to occur.
2890  */
2891 static int
2892 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
2893                                           uint64_t offset, uint64_t size)
2894 {
2895         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2896         int i, ret;
2897
2898         if (offset == 0 && size == obj->size)
2899                 return i915_gem_object_set_to_cpu_domain(obj, 0);
2900
2901         i915_gem_object_flush_gpu_write_domain(obj);
2902         /* Wait on any GPU rendering and flushing to occur. */
2903         ret = i915_gem_object_wait_rendering(obj);
2904         if (ret != 0)
2905                 return ret;
2906         i915_gem_object_flush_gtt_write_domain(obj);
2907
2908         /* If we're already fully in the CPU read domain, we're done. */
2909         if (obj_priv->page_cpu_valid == NULL &&
2910             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
2911                 return 0;
2912
2913         /* Otherwise, create/clear the per-page CPU read domain flag if we're
2914          * newly adding I915_GEM_DOMAIN_CPU
2915          */
2916         if (obj_priv->page_cpu_valid == NULL) {
2917                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
2918                                                    GFP_KERNEL);
2919                 if (obj_priv->page_cpu_valid == NULL)
2920                         return -ENOMEM;
2921         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
2922                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
2923
2924         /* Flush the cache on any pages that are still invalid from the CPU's
2925          * perspective.
2926          */
2927         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
2928              i++) {
2929                 if (obj_priv->page_cpu_valid[i])
2930                         continue;
2931
2932                 drm_clflush_pages(obj_priv->pages + i, 1);
2933
2934                 obj_priv->page_cpu_valid[i] = 1;
2935         }
2936
2937         /* It should now be out of any other write domains, and we can update
2938          * the domain values for our changes.
2939          */
2940         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2941
2942         obj->read_domains |= I915_GEM_DOMAIN_CPU;
2943
2944         return 0;
2945 }
2946
2947 /**
2948  * Pin an object to the GTT and evaluate the relocations landing in it.
2949  */
2950 static int
2951 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
2952                                  struct drm_file *file_priv,
2953                                  struct drm_i915_gem_exec_object *entry,
2954                                  struct drm_i915_gem_relocation_entry *relocs)
2955 {
2956         struct drm_device *dev = obj->dev;
2957         drm_i915_private_t *dev_priv = dev->dev_private;
2958         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2959         int i, ret;
2960         void __iomem *reloc_page;
2961
2962         /* Choose the GTT offset for our buffer and put it there. */
2963         ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
2964         if (ret)
2965                 return ret;
2966
2967         entry->offset = obj_priv->gtt_offset;
2968
2969         /* Apply the relocations, using the GTT aperture to avoid cache
2970          * flushing requirements.
2971          */
2972         for (i = 0; i < entry->relocation_count; i++) {
2973                 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
2974                 struct drm_gem_object *target_obj;
2975                 struct drm_i915_gem_object *target_obj_priv;
2976                 uint32_t reloc_val, reloc_offset;
2977                 uint32_t __iomem *reloc_entry;
2978
2979                 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
2980                                                    reloc->target_handle);
2981                 if (target_obj == NULL) {
2982                         i915_gem_object_unpin(obj);
2983                         return -EBADF;
2984                 }
2985                 target_obj_priv = target_obj->driver_private;
2986
2987                 /* The target buffer should have appeared before us in the
2988                  * exec_object list, so it should have a GTT space bound by now.
2989                  */
2990                 if (target_obj_priv->gtt_space == NULL) {
2991                         DRM_ERROR("No GTT space found for object %d\n",
2992                                   reloc->target_handle);
2993                         drm_gem_object_unreference(target_obj);
2994                         i915_gem_object_unpin(obj);
2995                         return -EINVAL;
2996                 }
2997
2998                 if (reloc->offset > obj->size - 4) {
2999                         DRM_ERROR("Relocation beyond object bounds: "
3000                                   "obj %p target %d offset %d size %d.\n",
3001                                   obj, reloc->target_handle,
3002                                   (int) reloc->offset, (int) obj->size);
3003                         drm_gem_object_unreference(target_obj);
3004                         i915_gem_object_unpin(obj);
3005                         return -EINVAL;
3006                 }
3007                 if (reloc->offset & 3) {
3008                         DRM_ERROR("Relocation not 4-byte aligned: "
3009                                   "obj %p target %d offset %d.\n",
3010                                   obj, reloc->target_handle,
3011                                   (int) reloc->offset);
3012                         drm_gem_object_unreference(target_obj);
3013                         i915_gem_object_unpin(obj);
3014                         return -EINVAL;
3015                 }
3016
3017                 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3018                     reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3019                         DRM_ERROR("reloc with read/write CPU domains: "
3020                                   "obj %p target %d offset %d "
3021                                   "read %08x write %08x",
3022                                   obj, reloc->target_handle,
3023                                   (int) reloc->offset,
3024                                   reloc->read_domains,
3025                                   reloc->write_domain);
3026                         drm_gem_object_unreference(target_obj);
3027                         i915_gem_object_unpin(obj);
3028                         return -EINVAL;
3029                 }
3030
3031                 if (reloc->write_domain && target_obj->pending_write_domain &&
3032                     reloc->write_domain != target_obj->pending_write_domain) {
3033                         DRM_ERROR("Write domain conflict: "
3034                                   "obj %p target %d offset %d "
3035                                   "new %08x old %08x\n",
3036                                   obj, reloc->target_handle,
3037                                   (int) reloc->offset,
3038                                   reloc->write_domain,
3039                                   target_obj->pending_write_domain);
3040                         drm_gem_object_unreference(target_obj);
3041                         i915_gem_object_unpin(obj);
3042                         return -EINVAL;
3043                 }
3044
3045 #if WATCH_RELOC
3046                 DRM_INFO("%s: obj %p offset %08x target %d "
3047                          "read %08x write %08x gtt %08x "
3048                          "presumed %08x delta %08x\n",
3049                          __func__,
3050                          obj,
3051                          (int) reloc->offset,
3052                          (int) reloc->target_handle,
3053                          (int) reloc->read_domains,
3054                          (int) reloc->write_domain,
3055                          (int) target_obj_priv->gtt_offset,
3056                          (int) reloc->presumed_offset,
3057                          reloc->delta);
3058 #endif
3059
3060                 target_obj->pending_read_domains |= reloc->read_domains;
3061                 target_obj->pending_write_domain |= reloc->write_domain;
3062
3063                 /* If the relocation already has the right value in it, no
3064                  * more work needs to be done.
3065                  */
3066                 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3067                         drm_gem_object_unreference(target_obj);
3068                         continue;
3069                 }
3070
3071                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3072                 if (ret != 0) {
3073                         drm_gem_object_unreference(target_obj);
3074                         i915_gem_object_unpin(obj);
3075                         return -EINVAL;
3076                 }
3077
3078                 /* Map the page containing the relocation we're going to
3079                  * perform.
3080                  */
3081                 reloc_offset = obj_priv->gtt_offset + reloc->offset;
3082                 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3083                                                       (reloc_offset &
3084                                                        ~(PAGE_SIZE - 1)));
3085                 reloc_entry = (uint32_t __iomem *)(reloc_page +
3086                                                    (reloc_offset & (PAGE_SIZE - 1)));
3087                 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
3088
3089 #if WATCH_BUF
3090                 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
3091                           obj, (unsigned int) reloc->offset,
3092                           readl(reloc_entry), reloc_val);
3093 #endif
3094                 writel(reloc_val, reloc_entry);
3095                 io_mapping_unmap_atomic(reloc_page);
3096
3097                 /* The updated presumed offset for this entry will be
3098                  * copied back out to the user.
3099                  */
3100                 reloc->presumed_offset = target_obj_priv->gtt_offset;
3101
3102                 drm_gem_object_unreference(target_obj);
3103         }
3104
3105 #if WATCH_BUF
3106         if (0)
3107                 i915_gem_dump_object(obj, 128, __func__, ~0);
3108 #endif
3109         return 0;
3110 }
3111
3112 /** Dispatch a batchbuffer to the ring
3113  */
3114 static int
3115 i915_dispatch_gem_execbuffer(struct drm_device *dev,
3116                               struct drm_i915_gem_execbuffer *exec,
3117                               struct drm_clip_rect *cliprects,
3118                               uint64_t exec_offset)
3119 {
3120         drm_i915_private_t *dev_priv = dev->dev_private;
3121         int nbox = exec->num_cliprects;
3122         int i = 0, count;
3123         uint32_t exec_start, exec_len;
3124         RING_LOCALS;
3125
3126         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3127         exec_len = (uint32_t) exec->batch_len;
3128
3129         count = nbox ? nbox : 1;
3130
3131         for (i = 0; i < count; i++) {
3132                 if (i < nbox) {
3133                         int ret = i915_emit_box(dev, cliprects, i,
3134                                                 exec->DR1, exec->DR4);
3135                         if (ret)
3136                                 return ret;
3137                 }
3138
3139                 if (IS_I830(dev) || IS_845G(dev)) {
3140                         BEGIN_LP_RING(4);
3141                         OUT_RING(MI_BATCH_BUFFER);
3142                         OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3143                         OUT_RING(exec_start + exec_len - 4);
3144                         OUT_RING(0);
3145                         ADVANCE_LP_RING();
3146                 } else {
3147                         BEGIN_LP_RING(2);
3148                         if (IS_I965G(dev)) {
3149                                 OUT_RING(MI_BATCH_BUFFER_START |
3150                                          (2 << 6) |
3151                                          MI_BATCH_NON_SECURE_I965);
3152                                 OUT_RING(exec_start);
3153                         } else {
3154                                 OUT_RING(MI_BATCH_BUFFER_START |
3155                                          (2 << 6));
3156                                 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3157                         }
3158                         ADVANCE_LP_RING();
3159                 }
3160         }
3161
3162         /* XXX breadcrumb */
3163         return 0;
3164 }
3165
3166 /* Throttle our rendering by waiting until the ring has completed our requests
3167  * emitted over 20 msec ago.
3168  *
3169  * Note that if we were to use the current jiffies each time around the loop,
3170  * we wouldn't escape the function with any frames outstanding if the time to
3171  * render a frame was over 20ms.
3172  *
3173  * This should get us reasonable parallelism between CPU and GPU but also
3174  * relatively low latency when blocking on a particular request to finish.
3175  */
3176 static int
3177 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3178 {
3179         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3180         int ret = 0;
3181         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3182
3183         mutex_lock(&dev->struct_mutex);
3184         while (!list_empty(&i915_file_priv->mm.request_list)) {
3185                 struct drm_i915_gem_request *request;
3186
3187                 request = list_first_entry(&i915_file_priv->mm.request_list,
3188                                            struct drm_i915_gem_request,
3189                                            client_list);
3190
3191                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3192                         break;
3193
3194                 ret = i915_wait_request(dev, request->seqno);
3195                 if (ret != 0)
3196                         break;
3197         }
3198         mutex_unlock(&dev->struct_mutex);
3199
3200         return ret;
3201 }
3202
3203 static int
3204 i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object *exec_list,
3205                               uint32_t buffer_count,
3206                               struct drm_i915_gem_relocation_entry **relocs)
3207 {
3208         uint32_t reloc_count = 0, reloc_index = 0, i;
3209         int ret;
3210
3211         *relocs = NULL;
3212         for (i = 0; i < buffer_count; i++) {
3213                 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3214                         return -EINVAL;
3215                 reloc_count += exec_list[i].relocation_count;
3216         }
3217
3218         *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
3219         if (*relocs == NULL)
3220                 return -ENOMEM;
3221
3222         for (i = 0; i < buffer_count; i++) {
3223                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3224
3225                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3226
3227                 ret = copy_from_user(&(*relocs)[reloc_index],
3228                                      user_relocs,
3229                                      exec_list[i].relocation_count *
3230                                      sizeof(**relocs));
3231                 if (ret != 0) {
3232                         drm_free_large(*relocs);
3233                         *relocs = NULL;
3234                         return -EFAULT;
3235                 }
3236
3237                 reloc_index += exec_list[i].relocation_count;
3238         }
3239
3240         return 0;
3241 }
3242
3243 static int
3244 i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object *exec_list,
3245                             uint32_t buffer_count,
3246                             struct drm_i915_gem_relocation_entry *relocs)
3247 {
3248         uint32_t reloc_count = 0, i;
3249         int ret = 0;
3250
3251         for (i = 0; i < buffer_count; i++) {
3252                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3253                 int unwritten;
3254
3255                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3256
3257                 unwritten = copy_to_user(user_relocs,
3258                                          &relocs[reloc_count],
3259                                          exec_list[i].relocation_count *
3260                                          sizeof(*relocs));
3261
3262                 if (unwritten) {
3263                         ret = -EFAULT;
3264                         goto err;
3265                 }
3266
3267                 reloc_count += exec_list[i].relocation_count;
3268         }
3269
3270 err:
3271         drm_free_large(relocs);
3272
3273         return ret;
3274 }
3275
3276 static int
3277 i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer *exec,
3278                            uint64_t exec_offset)
3279 {
3280         uint32_t exec_start, exec_len;
3281
3282         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3283         exec_len = (uint32_t) exec->batch_len;
3284
3285         if ((exec_start | exec_len) & 0x7)
3286                 return -EINVAL;
3287
3288         if (!exec_start)
3289                 return -EINVAL;
3290
3291         return 0;
3292 }
3293
3294 int
3295 i915_gem_execbuffer(struct drm_device *dev, void *data,
3296                     struct drm_file *file_priv)
3297 {
3298         drm_i915_private_t *dev_priv = dev->dev_private;
3299         struct drm_i915_gem_execbuffer *args = data;
3300         struct drm_i915_gem_exec_object *exec_list = NULL;
3301         struct drm_gem_object **object_list = NULL;
3302         struct drm_gem_object *batch_obj;
3303         struct drm_i915_gem_object *obj_priv;
3304         struct drm_clip_rect *cliprects = NULL;
3305         struct drm_i915_gem_relocation_entry *relocs;
3306         int ret, ret2, i, pinned = 0;
3307         uint64_t exec_offset;
3308         uint32_t seqno, flush_domains, reloc_index;
3309         int pin_tries;
3310
3311 #if WATCH_EXEC
3312         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3313                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3314 #endif
3315
3316         if (args->buffer_count < 1) {
3317                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3318                 return -EINVAL;
3319         }
3320         /* Copy in the exec list from userland */
3321         exec_list = drm_calloc_large(sizeof(*exec_list), args->buffer_count);
3322         object_list = drm_calloc_large(sizeof(*object_list), args->buffer_count);
3323         if (exec_list == NULL || object_list == NULL) {
3324                 DRM_ERROR("Failed to allocate exec or object list "
3325                           "for %d buffers\n",
3326                           args->buffer_count);
3327                 ret = -ENOMEM;
3328                 goto pre_mutex_err;
3329         }
3330         ret = copy_from_user(exec_list,
3331                              (struct drm_i915_relocation_entry __user *)
3332                              (uintptr_t) args->buffers_ptr,
3333                              sizeof(*exec_list) * args->buffer_count);
3334         if (ret != 0) {
3335                 DRM_ERROR("copy %d exec entries failed %d\n",
3336                           args->buffer_count, ret);
3337                 goto pre_mutex_err;
3338         }
3339
3340         if (args->num_cliprects != 0) {
3341                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3342                                     GFP_KERNEL);
3343                 if (cliprects == NULL)
3344                         goto pre_mutex_err;
3345
3346                 ret = copy_from_user(cliprects,
3347                                      (struct drm_clip_rect __user *)
3348                                      (uintptr_t) args->cliprects_ptr,
3349                                      sizeof(*cliprects) * args->num_cliprects);
3350                 if (ret != 0) {
3351                         DRM_ERROR("copy %d cliprects failed: %d\n",
3352                                   args->num_cliprects, ret);
3353                         goto pre_mutex_err;
3354                 }
3355         }
3356
3357         ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3358                                             &relocs);
3359         if (ret != 0)
3360                 goto pre_mutex_err;
3361
3362         mutex_lock(&dev->struct_mutex);
3363
3364         i915_verify_inactive(dev, __FILE__, __LINE__);
3365
3366         if (dev_priv->mm.wedged) {
3367                 DRM_ERROR("Execbuf while wedged\n");
3368                 mutex_unlock(&dev->struct_mutex);
3369                 ret = -EIO;
3370                 goto pre_mutex_err;
3371         }
3372
3373         if (dev_priv->mm.suspended) {
3374                 DRM_ERROR("Execbuf while VT-switched.\n");
3375                 mutex_unlock(&dev->struct_mutex);
3376                 ret = -EBUSY;
3377                 goto pre_mutex_err;
3378         }
3379
3380         /* Look up object handles */
3381         for (i = 0; i < args->buffer_count; i++) {
3382                 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3383                                                        exec_list[i].handle);
3384                 if (object_list[i] == NULL) {
3385                         DRM_ERROR("Invalid object handle %d at index %d\n",
3386                                    exec_list[i].handle, i);
3387                         ret = -EBADF;
3388                         goto err;
3389                 }
3390
3391                 obj_priv = object_list[i]->driver_private;
3392                 if (obj_priv->in_execbuffer) {
3393                         DRM_ERROR("Object %p appears more than once in object list\n",
3394                                    object_list[i]);
3395                         ret = -EBADF;
3396                         goto err;
3397                 }
3398                 obj_priv->in_execbuffer = true;
3399         }
3400
3401         /* Pin and relocate */
3402         for (pin_tries = 0; ; pin_tries++) {
3403                 ret = 0;
3404                 reloc_index = 0;
3405
3406                 for (i = 0; i < args->buffer_count; i++) {
3407                         object_list[i]->pending_read_domains = 0;
3408                         object_list[i]->pending_write_domain = 0;
3409                         ret = i915_gem_object_pin_and_relocate(object_list[i],
3410                                                                file_priv,
3411                                                                &exec_list[i],
3412                                                                &relocs[reloc_index]);
3413                         if (ret)
3414                                 break;
3415                         pinned = i + 1;
3416                         reloc_index += exec_list[i].relocation_count;
3417                 }
3418                 /* success */
3419                 if (ret == 0)
3420                         break;
3421
3422                 /* error other than GTT full, or we've already tried again */
3423                 if (ret != -ENOSPC || pin_tries >= 1) {
3424                         if (ret != -ERESTARTSYS)
3425                                 DRM_ERROR("Failed to pin buffers %d\n", ret);
3426                         goto err;
3427                 }
3428
3429                 /* unpin all of our buffers */
3430                 for (i = 0; i < pinned; i++)
3431                         i915_gem_object_unpin(object_list[i]);
3432                 pinned = 0;
3433
3434                 /* evict everyone we can from the aperture */
3435                 ret = i915_gem_evict_everything(dev);
3436                 if (ret)
3437                         goto err;
3438         }
3439
3440         /* Set the pending read domains for the batch buffer to COMMAND */
3441         batch_obj = object_list[args->buffer_count-1];
3442         if (batch_obj->pending_write_domain) {
3443                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3444                 ret = -EINVAL;
3445                 goto err;
3446         }
3447         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3448
3449         /* Sanity check the batch buffer, prior to moving objects */
3450         exec_offset = exec_list[args->buffer_count - 1].offset;
3451         ret = i915_gem_check_execbuffer (args, exec_offset);
3452         if (ret != 0) {
3453                 DRM_ERROR("execbuf with invalid offset/length\n");
3454                 goto err;
3455         }
3456
3457         i915_verify_inactive(dev, __FILE__, __LINE__);
3458
3459         /* Zero the global flush/invalidate flags. These
3460          * will be modified as new domains are computed
3461          * for each object
3462          */
3463         dev->invalidate_domains = 0;
3464         dev->flush_domains = 0;
3465
3466         for (i = 0; i < args->buffer_count; i++) {
3467                 struct drm_gem_object *obj = object_list[i];
3468
3469                 /* Compute new gpu domains and update invalidate/flush */
3470                 i915_gem_object_set_to_gpu_domain(obj);
3471         }
3472
3473         i915_verify_inactive(dev, __FILE__, __LINE__);
3474
3475         if (dev->invalidate_domains | dev->flush_domains) {
3476 #if WATCH_EXEC
3477                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3478                           __func__,
3479                          dev->invalidate_domains,
3480                          dev->flush_domains);
3481 #endif
3482                 i915_gem_flush(dev,
3483                                dev->invalidate_domains,
3484                                dev->flush_domains);
3485                 if (dev->flush_domains)
3486                         (void)i915_add_request(dev, file_priv,
3487                                                dev->flush_domains);
3488         }
3489
3490         for (i = 0; i < args->buffer_count; i++) {
3491                 struct drm_gem_object *obj = object_list[i];
3492
3493                 obj->write_domain = obj->pending_write_domain;
3494         }
3495
3496         i915_verify_inactive(dev, __FILE__, __LINE__);
3497
3498 #if WATCH_COHERENCY
3499         for (i = 0; i < args->buffer_count; i++) {
3500                 i915_gem_object_check_coherency(object_list[i],
3501                                                 exec_list[i].handle);
3502         }
3503 #endif
3504
3505 #if WATCH_EXEC
3506         i915_gem_dump_object(batch_obj,
3507                               args->batch_len,
3508                               __func__,
3509                               ~0);
3510 #endif
3511
3512         /* Exec the batchbuffer */
3513         ret = i915_dispatch_gem_execbuffer(dev, args, cliprects, exec_offset);
3514         if (ret) {
3515                 DRM_ERROR("dispatch failed %d\n", ret);
3516                 goto err;
3517         }
3518
3519         /*
3520          * Ensure that the commands in the batch buffer are
3521          * finished before the interrupt fires
3522          */
3523         flush_domains = i915_retire_commands(dev);
3524
3525         i915_verify_inactive(dev, __FILE__, __LINE__);
3526
3527         /*
3528          * Get a seqno representing the execution of the current buffer,
3529          * which we can wait on.  We would like to mitigate these interrupts,
3530          * likely by only creating seqnos occasionally (so that we have
3531          * *some* interrupts representing completion of buffers that we can
3532          * wait on when trying to clear up gtt space).
3533          */
3534         seqno = i915_add_request(dev, file_priv, flush_domains);
3535         BUG_ON(seqno == 0);
3536         for (i = 0; i < args->buffer_count; i++) {
3537                 struct drm_gem_object *obj = object_list[i];
3538
3539                 i915_gem_object_move_to_active(obj, seqno);
3540 #if WATCH_LRU
3541                 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3542 #endif
3543         }
3544 #if WATCH_LRU
3545         i915_dump_lru(dev, __func__);
3546 #endif
3547
3548         i915_verify_inactive(dev, __FILE__, __LINE__);
3549
3550 err:
3551         for (i = 0; i < pinned; i++)
3552                 i915_gem_object_unpin(object_list[i]);
3553
3554         for (i = 0; i < args->buffer_count; i++) {
3555                 if (object_list[i]) {
3556                         obj_priv = object_list[i]->driver_private;
3557                         obj_priv->in_execbuffer = false;
3558                 }
3559                 drm_gem_object_unreference(object_list[i]);
3560         }
3561
3562         mutex_unlock(&dev->struct_mutex);
3563
3564         if (!ret) {
3565                 /* Copy the new buffer offsets back to the user's exec list. */
3566                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3567                                    (uintptr_t) args->buffers_ptr,
3568                                    exec_list,
3569                                    sizeof(*exec_list) * args->buffer_count);
3570                 if (ret) {
3571                         ret = -EFAULT;
3572                         DRM_ERROR("failed to copy %d exec entries "
3573                                   "back to user (%d)\n",
3574                                   args->buffer_count, ret);
3575                 }
3576         }
3577
3578         /* Copy the updated relocations out regardless of current error
3579          * state.  Failure to update the relocs would mean that the next
3580          * time userland calls execbuf, it would do so with presumed offset
3581          * state that didn't match the actual object state.
3582          */
3583         ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3584                                            relocs);
3585         if (ret2 != 0) {
3586                 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3587
3588                 if (ret == 0)
3589                         ret = ret2;
3590         }
3591
3592 pre_mutex_err:
3593         drm_free_large(object_list);
3594         drm_free_large(exec_list);
3595         kfree(cliprects);
3596
3597         return ret;
3598 }
3599
3600 int
3601 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
3602 {
3603         struct drm_device *dev = obj->dev;
3604         struct drm_i915_gem_object *obj_priv = obj->driver_private;
3605         int ret;
3606
3607         i915_verify_inactive(dev, __FILE__, __LINE__);
3608         if (obj_priv->gtt_space == NULL) {
3609                 ret = i915_gem_object_bind_to_gtt(obj, alignment);
3610                 if (ret != 0) {
3611                         if (ret != -EBUSY && ret != -ERESTARTSYS)
3612                                 DRM_ERROR("Failure to bind: %d\n", ret);
3613                         return ret;
3614                 }
3615         }
3616         /*
3617          * Pre-965 chips need a fence register set up in order to
3618          * properly handle tiled surfaces.
3619          */
3620         if (!IS_I965G(dev) && obj_priv->tiling_mode != I915_TILING_NONE) {
3621                 ret = i915_gem_object_get_fence_reg(obj);
3622                 if (ret != 0) {
3623                         if (ret != -EBUSY && ret != -ERESTARTSYS)
3624                                 DRM_ERROR("Failure to install fence: %d\n",
3625                                           ret);
3626                         return ret;
3627                 }
3628         }
3629         obj_priv->pin_count++;
3630
3631         /* If the object is not active and not pending a flush,
3632          * remove it from the inactive list
3633          */
3634         if (obj_priv->pin_count == 1) {
3635                 atomic_inc(&dev->pin_count);
3636                 atomic_add(obj->size, &dev->pin_memory);
3637                 if (!obj_priv->active &&
3638                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0 &&
3639                     !list_empty(&obj_priv->list))
3640                         list_del_init(&obj_priv->list);
3641         }
3642         i915_verify_inactive(dev, __FILE__, __LINE__);
3643
3644         return 0;
3645 }
3646
3647 void
3648 i915_gem_object_unpin(struct drm_gem_object *obj)
3649 {
3650         struct drm_device *dev = obj->dev;
3651         drm_i915_private_t *dev_priv = dev->dev_private;
3652         struct drm_i915_gem_object *obj_priv = obj->driver_private;
3653
3654         i915_verify_inactive(dev, __FILE__, __LINE__);
3655         obj_priv->pin_count--;
3656         BUG_ON(obj_priv->pin_count < 0);
3657         BUG_ON(obj_priv->gtt_space == NULL);
3658
3659         /* If the object is no longer pinned, and is
3660          * neither active nor being flushed, then stick it on
3661          * the inactive list
3662          */
3663         if (obj_priv->pin_count == 0) {
3664                 if (!obj_priv->active &&
3665                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
3666                         list_move_tail(&obj_priv->list,
3667                                        &dev_priv->mm.inactive_list);
3668                 atomic_dec(&dev->pin_count);
3669                 atomic_sub(obj->size, &dev->pin_memory);
3670         }
3671         i915_verify_inactive(dev, __FILE__, __LINE__);
3672 }
3673
3674 int
3675 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3676                    struct drm_file *file_priv)
3677 {
3678         struct drm_i915_gem_pin *args = data;
3679         struct drm_gem_object *obj;
3680         struct drm_i915_gem_object *obj_priv;
3681         int ret;
3682
3683         mutex_lock(&dev->struct_mutex);
3684
3685         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
3686         if (obj == NULL) {
3687                 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
3688                           args->handle);
3689                 mutex_unlock(&dev->struct_mutex);
3690                 return -EBADF;
3691         }
3692         obj_priv = obj->driver_private;
3693
3694         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
3695                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3696                           args->handle);
3697                 drm_gem_object_unreference(obj);
3698                 mutex_unlock(&dev->struct_mutex);
3699                 return -EINVAL;
3700         }
3701
3702         obj_priv->user_pin_count++;
3703         obj_priv->pin_filp = file_priv;
3704         if (obj_priv->user_pin_count == 1) {
3705                 ret = i915_gem_object_pin(obj, args->alignment);
3706                 if (ret != 0) {
3707                         drm_gem_object_unreference(obj);
3708                         mutex_unlock(&dev->struct_mutex);
3709                         return ret;
3710                 }
3711         }
3712
3713         /* XXX - flush the CPU caches for pinned objects
3714          * as the X server doesn't manage domains yet
3715          */
3716         i915_gem_object_flush_cpu_write_domain(obj);
3717         args->offset = obj_priv->gtt_offset;
3718         drm_gem_object_unreference(obj);
3719         mutex_unlock(&dev->struct_mutex);
3720
3721         return 0;
3722 }
3723
3724 int
3725 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3726                      struct drm_file *file_priv)
3727 {
3728         struct drm_i915_gem_pin *args = data;
3729         struct drm_gem_object *obj;
3730         struct drm_i915_gem_object *obj_priv;
3731
3732         mutex_lock(&dev->struct_mutex);
3733
3734         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
3735         if (obj == NULL) {
3736                 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
3737                           args->handle);
3738                 mutex_unlock(&dev->struct_mutex);
3739                 return -EBADF;
3740         }
3741
3742         obj_priv = obj->driver_private;
3743         if (obj_priv->pin_filp != file_priv) {
3744                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3745                           args->handle);
3746                 drm_gem_object_unreference(obj);
3747                 mutex_unlock(&dev->struct_mutex);
3748                 return -EINVAL;
3749         }
3750         obj_priv->user_pin_count--;
3751         if (obj_priv->user_pin_count == 0) {
3752                 obj_priv->pin_filp = NULL;
3753                 i915_gem_object_unpin(obj);
3754         }
3755
3756         drm_gem_object_unreference(obj);
3757         mutex_unlock(&dev->struct_mutex);
3758         return 0;
3759 }
3760
3761 int
3762 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3763                     struct drm_file *file_priv)
3764 {
3765         struct drm_i915_gem_busy *args = data;
3766         struct drm_gem_object *obj;
3767         struct drm_i915_gem_object *obj_priv;
3768
3769         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
3770         if (obj == NULL) {
3771                 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
3772                           args->handle);
3773                 return -EBADF;
3774         }
3775
3776         mutex_lock(&dev->struct_mutex);
3777         /* Update the active list for the hardware's current position.
3778          * Otherwise this only updates on a delayed timer or when irqs are
3779          * actually unmasked, and our working set ends up being larger than
3780          * required.
3781          */
3782         i915_gem_retire_requests(dev);
3783
3784         obj_priv = obj->driver_private;
3785         /* Don't count being on the flushing list against the object being
3786          * done.  Otherwise, a buffer left on the flushing list but not getting
3787          * flushed (because nobody's flushing that domain) won't ever return
3788          * unbusy and get reused by libdrm's bo cache.  The other expected
3789          * consumer of this interface, OpenGL's occlusion queries, also specs
3790          * that the objects get unbusy "eventually" without any interference.
3791          */
3792         args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
3793
3794         drm_gem_object_unreference(obj);
3795         mutex_unlock(&dev->struct_mutex);
3796         return 0;
3797 }
3798
3799 int
3800 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3801                         struct drm_file *file_priv)
3802 {
3803     return i915_gem_ring_throttle(dev, file_priv);
3804 }
3805
3806 int i915_gem_init_object(struct drm_gem_object *obj)
3807 {
3808         struct drm_i915_gem_object *obj_priv;
3809
3810         obj_priv = kzalloc(sizeof(*obj_priv), GFP_KERNEL);
3811         if (obj_priv == NULL)
3812                 return -ENOMEM;
3813
3814         /*
3815          * We've just allocated pages from the kernel,
3816          * so they've just been written by the CPU with
3817          * zeros. They'll need to be clflushed before we
3818          * use them with the GPU.
3819          */
3820         obj->write_domain = I915_GEM_DOMAIN_CPU;
3821         obj->read_domains = I915_GEM_DOMAIN_CPU;
3822
3823         obj_priv->agp_type = AGP_USER_MEMORY;
3824
3825         obj->driver_private = obj_priv;
3826         obj_priv->obj = obj;
3827         obj_priv->fence_reg = I915_FENCE_REG_NONE;
3828         INIT_LIST_HEAD(&obj_priv->list);
3829         INIT_LIST_HEAD(&obj_priv->fence_list);
3830
3831         return 0;
3832 }
3833
3834 void i915_gem_free_object(struct drm_gem_object *obj)
3835 {
3836         struct drm_device *dev = obj->dev;
3837         struct drm_i915_gem_object *obj_priv = obj->driver_private;
3838
3839         while (obj_priv->pin_count > 0)
3840                 i915_gem_object_unpin(obj);
3841
3842         if (obj_priv->phys_obj)
3843                 i915_gem_detach_phys_object(dev, obj);
3844
3845         i915_gem_object_unbind(obj);
3846
3847         i915_gem_free_mmap_offset(obj);
3848
3849         kfree(obj_priv->page_cpu_valid);
3850         kfree(obj_priv->bit_17);
3851         kfree(obj->driver_private);
3852 }
3853
3854 /** Unbinds all objects that are on the given buffer list. */
3855 static int
3856 i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head)
3857 {
3858         struct drm_gem_object *obj;
3859         struct drm_i915_gem_object *obj_priv;
3860         int ret;
3861
3862         while (!list_empty(head)) {
3863                 obj_priv = list_first_entry(head,
3864                                             struct drm_i915_gem_object,
3865                                             list);
3866                 obj = obj_priv->obj;
3867
3868                 if (obj_priv->pin_count != 0) {
3869                         DRM_ERROR("Pinned object in unbind list\n");
3870                         mutex_unlock(&dev->struct_mutex);
3871                         return -EINVAL;
3872                 }
3873
3874                 ret = i915_gem_object_unbind(obj);
3875                 if (ret != 0) {
3876                         DRM_ERROR("Error unbinding object in LeaveVT: %d\n",
3877                                   ret);
3878                         mutex_unlock(&dev->struct_mutex);
3879                         return ret;
3880                 }
3881         }
3882
3883
3884         return 0;
3885 }
3886
3887 int
3888 i915_gem_idle(struct drm_device *dev)
3889 {
3890         drm_i915_private_t *dev_priv = dev->dev_private;
3891         uint32_t seqno, cur_seqno, last_seqno;
3892         int stuck, ret;
3893
3894         mutex_lock(&dev->struct_mutex);
3895
3896         if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) {
3897                 mutex_unlock(&dev->struct_mutex);
3898                 return 0;
3899         }
3900
3901         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
3902          * We need to replace this with a semaphore, or something.
3903          */
3904         dev_priv->mm.suspended = 1;
3905
3906         /* Cancel the retire work handler, wait for it to finish if running
3907          */
3908         mutex_unlock(&dev->struct_mutex);
3909         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3910         mutex_lock(&dev->struct_mutex);
3911
3912         i915_kernel_lost_context(dev);
3913
3914         /* Flush the GPU along with all non-CPU write domains
3915          */
3916         i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
3917         seqno = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS);
3918
3919         if (seqno == 0) {
3920                 mutex_unlock(&dev->struct_mutex);
3921                 return -ENOMEM;
3922         }
3923
3924         dev_priv->mm.waiting_gem_seqno = seqno;
3925         last_seqno = 0;
3926         stuck = 0;
3927         for (;;) {
3928                 cur_seqno = i915_get_gem_seqno(dev);
3929                 if (i915_seqno_passed(cur_seqno, seqno))
3930                         break;
3931                 if (last_seqno == cur_seqno) {
3932                         if (stuck++ > 100) {
3933                                 DRM_ERROR("hardware wedged\n");
3934                                 dev_priv->mm.wedged = 1;
3935                                 DRM_WAKEUP(&dev_priv->irq_queue);
3936                                 break;
3937                         }
3938                 }
3939                 msleep(10);
3940                 last_seqno = cur_seqno;
3941         }
3942         dev_priv->mm.waiting_gem_seqno = 0;
3943
3944         i915_gem_retire_requests(dev);
3945
3946         spin_lock(&dev_priv->mm.active_list_lock);
3947         if (!dev_priv->mm.wedged) {
3948                 /* Active and flushing should now be empty as we've
3949                  * waited for a sequence higher than any pending execbuffer
3950                  */
3951                 WARN_ON(!list_empty(&dev_priv->mm.active_list));
3952                 WARN_ON(!list_empty(&dev_priv->mm.flushing_list));
3953                 /* Request should now be empty as we've also waited
3954                  * for the last request in the list
3955                  */
3956                 WARN_ON(!list_empty(&dev_priv->mm.request_list));
3957         }
3958
3959         /* Empty the active and flushing lists to inactive.  If there's
3960          * anything left at this point, it means that we're wedged and
3961          * nothing good's going to happen by leaving them there.  So strip
3962          * the GPU domains and just stuff them onto inactive.
3963          */
3964         while (!list_empty(&dev_priv->mm.active_list)) {
3965                 struct drm_i915_gem_object *obj_priv;
3966
3967                 obj_priv = list_first_entry(&dev_priv->mm.active_list,
3968                                             struct drm_i915_gem_object,
3969                                             list);
3970                 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
3971                 i915_gem_object_move_to_inactive(obj_priv->obj);
3972         }
3973         spin_unlock(&dev_priv->mm.active_list_lock);
3974
3975         while (!list_empty(&dev_priv->mm.flushing_list)) {
3976                 struct drm_i915_gem_object *obj_priv;
3977
3978                 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
3979                                             struct drm_i915_gem_object,
3980                                             list);
3981                 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
3982                 i915_gem_object_move_to_inactive(obj_priv->obj);
3983         }
3984
3985
3986         /* Move all inactive buffers out of the GTT. */
3987         ret = i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list);
3988         WARN_ON(!list_empty(&dev_priv->mm.inactive_list));
3989         if (ret) {
3990                 mutex_unlock(&dev->struct_mutex);
3991                 return ret;
3992         }
3993
3994         i915_gem_cleanup_ringbuffer(dev);
3995         mutex_unlock(&dev->struct_mutex);
3996
3997         return 0;
3998 }
3999
4000 static int
4001 i915_gem_init_hws(struct drm_device *dev)
4002 {
4003         drm_i915_private_t *dev_priv = dev->dev_private;
4004         struct drm_gem_object *obj;
4005         struct drm_i915_gem_object *obj_priv;
4006         int ret;
4007
4008         /* If we need a physical address for the status page, it's already
4009          * initialized at driver load time.
4010          */
4011         if (!I915_NEED_GFX_HWS(dev))
4012                 return 0;
4013
4014         obj = drm_gem_object_alloc(dev, 4096);
4015         if (obj == NULL) {
4016                 DRM_ERROR("Failed to allocate status page\n");
4017                 return -ENOMEM;
4018         }
4019         obj_priv = obj->driver_private;
4020         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4021
4022         ret = i915_gem_object_pin(obj, 4096);
4023         if (ret != 0) {
4024                 drm_gem_object_unreference(obj);
4025                 return ret;
4026         }
4027
4028         dev_priv->status_gfx_addr = obj_priv->gtt_offset;
4029
4030         dev_priv->hw_status_page = kmap(obj_priv->pages[0]);
4031         if (dev_priv->hw_status_page == NULL) {
4032                 DRM_ERROR("Failed to map status page.\n");
4033                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
4034                 i915_gem_object_unpin(obj);
4035                 drm_gem_object_unreference(obj);
4036                 return -EINVAL;
4037         }
4038         dev_priv->hws_obj = obj;
4039         memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
4040         I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
4041         I915_READ(HWS_PGA); /* posting read */
4042         DRM_DEBUG("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
4043
4044         return 0;
4045 }
4046
4047 static void
4048 i915_gem_cleanup_hws(struct drm_device *dev)
4049 {
4050         drm_i915_private_t *dev_priv = dev->dev_private;
4051         struct drm_gem_object *obj;
4052         struct drm_i915_gem_object *obj_priv;
4053
4054         if (dev_priv->hws_obj == NULL)
4055                 return;
4056
4057         obj = dev_priv->hws_obj;
4058         obj_priv = obj->driver_private;
4059
4060         kunmap(obj_priv->pages[0]);
4061         i915_gem_object_unpin(obj);
4062         drm_gem_object_unreference(obj);
4063         dev_priv->hws_obj = NULL;
4064
4065         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
4066         dev_priv->hw_status_page = NULL;
4067
4068         /* Write high address into HWS_PGA when disabling. */
4069         I915_WRITE(HWS_PGA, 0x1ffff000);
4070 }
4071
4072 int
4073 i915_gem_init_ringbuffer(struct drm_device *dev)
4074 {
4075         drm_i915_private_t *dev_priv = dev->dev_private;
4076         struct drm_gem_object *obj;
4077         struct drm_i915_gem_object *obj_priv;
4078         drm_i915_ring_buffer_t *ring = &dev_priv->ring;
4079         int ret;
4080         u32 head;
4081
4082         ret = i915_gem_init_hws(dev);
4083         if (ret != 0)
4084                 return ret;
4085
4086         obj = drm_gem_object_alloc(dev, 128 * 1024);
4087         if (obj == NULL) {
4088                 DRM_ERROR("Failed to allocate ringbuffer\n");
4089                 i915_gem_cleanup_hws(dev);
4090                 return -ENOMEM;
4091         }
4092         obj_priv = obj->driver_private;
4093
4094         ret = i915_gem_object_pin(obj, 4096);
4095         if (ret != 0) {
4096                 drm_gem_object_unreference(obj);
4097                 i915_gem_cleanup_hws(dev);
4098                 return ret;
4099         }
4100
4101         /* Set up the kernel mapping for the ring. */
4102         ring->Size = obj->size;
4103
4104         ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
4105         ring->map.size = obj->size;
4106         ring->map.type = 0;
4107         ring->map.flags = 0;
4108         ring->map.mtrr = 0;
4109
4110         drm_core_ioremap_wc(&ring->map, dev);
4111         if (ring->map.handle == NULL) {
4112                 DRM_ERROR("Failed to map ringbuffer.\n");
4113                 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
4114                 i915_gem_object_unpin(obj);
4115                 drm_gem_object_unreference(obj);
4116                 i915_gem_cleanup_hws(dev);
4117                 return -EINVAL;
4118         }
4119         ring->ring_obj = obj;
4120         ring->virtual_start = ring->map.handle;
4121
4122         /* Stop the ring if it's running. */
4123         I915_WRITE(PRB0_CTL, 0);
4124         I915_WRITE(PRB0_TAIL, 0);
4125         I915_WRITE(PRB0_HEAD, 0);
4126
4127         /* Initialize the ring. */
4128         I915_WRITE(PRB0_START, obj_priv->gtt_offset);
4129         head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4130
4131         /* G45 ring initialization fails to reset head to zero */
4132         if (head != 0) {
4133                 DRM_ERROR("Ring head not reset to zero "
4134                           "ctl %08x head %08x tail %08x start %08x\n",
4135                           I915_READ(PRB0_CTL),
4136                           I915_READ(PRB0_HEAD),
4137                           I915_READ(PRB0_TAIL),
4138                           I915_READ(PRB0_START));
4139                 I915_WRITE(PRB0_HEAD, 0);
4140
4141                 DRM_ERROR("Ring head forced to zero "
4142                           "ctl %08x head %08x tail %08x start %08x\n",
4143                           I915_READ(PRB0_CTL),
4144                           I915_READ(PRB0_HEAD),
4145                           I915_READ(PRB0_TAIL),
4146                           I915_READ(PRB0_START));
4147         }
4148
4149         I915_WRITE(PRB0_CTL,
4150                    ((obj->size - 4096) & RING_NR_PAGES) |
4151                    RING_NO_REPORT |
4152                    RING_VALID);
4153
4154         head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4155
4156         /* If the head is still not zero, the ring is dead */
4157         if (head != 0) {
4158                 DRM_ERROR("Ring initialization failed "
4159                           "ctl %08x head %08x tail %08x start %08x\n",
4160                           I915_READ(PRB0_CTL),
4161                           I915_READ(PRB0_HEAD),
4162                           I915_READ(PRB0_TAIL),
4163                           I915_READ(PRB0_START));
4164                 return -EIO;
4165         }
4166
4167         /* Update our cache of the ring state */
4168         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4169                 i915_kernel_lost_context(dev);
4170         else {
4171                 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4172                 ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
4173                 ring->space = ring->head - (ring->tail + 8);
4174                 if (ring->space < 0)
4175                         ring->space += ring->Size;
4176         }
4177
4178         return 0;
4179 }
4180
4181 void
4182 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4183 {
4184         drm_i915_private_t *dev_priv = dev->dev_private;
4185
4186         if (dev_priv->ring.ring_obj == NULL)
4187                 return;
4188
4189         drm_core_ioremapfree(&dev_priv->ring.map, dev);
4190
4191         i915_gem_object_unpin(dev_priv->ring.ring_obj);
4192         drm_gem_object_unreference(dev_priv->ring.ring_obj);
4193         dev_priv->ring.ring_obj = NULL;
4194         memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
4195
4196         i915_gem_cleanup_hws(dev);
4197 }
4198
4199 int
4200 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4201                        struct drm_file *file_priv)
4202 {
4203         drm_i915_private_t *dev_priv = dev->dev_private;
4204         int ret;
4205
4206         if (drm_core_check_feature(dev, DRIVER_MODESET))
4207                 return 0;
4208
4209         if (dev_priv->mm.wedged) {
4210                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4211                 dev_priv->mm.wedged = 0;
4212         }
4213
4214         mutex_lock(&dev->struct_mutex);
4215         dev_priv->mm.suspended = 0;
4216
4217         ret = i915_gem_init_ringbuffer(dev);
4218         if (ret != 0) {
4219                 mutex_unlock(&dev->struct_mutex);
4220                 return ret;
4221         }
4222
4223         spin_lock(&dev_priv->mm.active_list_lock);
4224         BUG_ON(!list_empty(&dev_priv->mm.active_list));
4225         spin_unlock(&dev_priv->mm.active_list_lock);
4226
4227         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4228         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4229         BUG_ON(!list_empty(&dev_priv->mm.request_list));
4230         mutex_unlock(&dev->struct_mutex);
4231
4232         drm_irq_install(dev);
4233
4234         return 0;
4235 }
4236
4237 int
4238 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4239                        struct drm_file *file_priv)
4240 {
4241         if (drm_core_check_feature(dev, DRIVER_MODESET))
4242                 return 0;
4243
4244         drm_irq_uninstall(dev);
4245         return i915_gem_idle(dev);
4246 }
4247
4248 void
4249 i915_gem_lastclose(struct drm_device *dev)
4250 {
4251         int ret;
4252
4253         if (drm_core_check_feature(dev, DRIVER_MODESET))
4254                 return;
4255
4256         ret = i915_gem_idle(dev);
4257         if (ret)
4258                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4259 }
4260
4261 void
4262 i915_gem_load(struct drm_device *dev)
4263 {
4264         int i;
4265         drm_i915_private_t *dev_priv = dev->dev_private;
4266
4267         spin_lock_init(&dev_priv->mm.active_list_lock);
4268         INIT_LIST_HEAD(&dev_priv->mm.active_list);
4269         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4270         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4271         INIT_LIST_HEAD(&dev_priv->mm.request_list);
4272         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4273         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4274                           i915_gem_retire_work_handler);
4275         dev_priv->mm.next_gem_seqno = 1;
4276
4277         /* Old X drivers will take 0-2 for front, back, depth buffers */
4278         dev_priv->fence_reg_start = 3;
4279
4280         if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4281                 dev_priv->num_fence_regs = 16;
4282         else
4283                 dev_priv->num_fence_regs = 8;
4284
4285         /* Initialize fence registers to zero */
4286         if (IS_I965G(dev)) {
4287                 for (i = 0; i < 16; i++)
4288                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4289         } else {
4290                 for (i = 0; i < 8; i++)
4291                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4292                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4293                         for (i = 0; i < 8; i++)
4294                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4295         }
4296
4297         i915_gem_detect_bit_6_swizzle(dev);
4298 }
4299
4300 /*
4301  * Create a physically contiguous memory object for this object
4302  * e.g. for cursor + overlay regs
4303  */
4304 int i915_gem_init_phys_object(struct drm_device *dev,
4305                               int id, int size)
4306 {
4307         drm_i915_private_t *dev_priv = dev->dev_private;
4308         struct drm_i915_gem_phys_object *phys_obj;
4309         int ret;
4310
4311         if (dev_priv->mm.phys_objs[id - 1] || !size)
4312                 return 0;
4313
4314         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4315         if (!phys_obj)
4316                 return -ENOMEM;
4317
4318         phys_obj->id = id;
4319
4320         phys_obj->handle = drm_pci_alloc(dev, size, 0, 0xffffffff);
4321         if (!phys_obj->handle) {
4322                 ret = -ENOMEM;
4323                 goto kfree_obj;
4324         }
4325 #ifdef CONFIG_X86
4326         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4327 #endif
4328
4329         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4330
4331         return 0;
4332 kfree_obj:
4333         kfree(phys_obj);
4334         return ret;
4335 }
4336
4337 void i915_gem_free_phys_object(struct drm_device *dev, int id)
4338 {
4339         drm_i915_private_t *dev_priv = dev->dev_private;
4340         struct drm_i915_gem_phys_object *phys_obj;
4341
4342         if (!dev_priv->mm.phys_objs[id - 1])
4343                 return;
4344
4345         phys_obj = dev_priv->mm.phys_objs[id - 1];
4346         if (phys_obj->cur_obj) {
4347                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4348         }
4349
4350 #ifdef CONFIG_X86
4351         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4352 #endif
4353         drm_pci_free(dev, phys_obj->handle);
4354         kfree(phys_obj);
4355         dev_priv->mm.phys_objs[id - 1] = NULL;
4356 }
4357
4358 void i915_gem_free_all_phys_object(struct drm_device *dev)
4359 {
4360         int i;
4361
4362         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4363                 i915_gem_free_phys_object(dev, i);
4364 }
4365
4366 void i915_gem_detach_phys_object(struct drm_device *dev,
4367                                  struct drm_gem_object *obj)
4368 {
4369         struct drm_i915_gem_object *obj_priv;
4370         int i;
4371         int ret;
4372         int page_count;
4373
4374         obj_priv = obj->driver_private;
4375         if (!obj_priv->phys_obj)
4376                 return;
4377
4378         ret = i915_gem_object_get_pages(obj);
4379         if (ret)
4380                 goto out;
4381
4382         page_count = obj->size / PAGE_SIZE;
4383
4384         for (i = 0; i < page_count; i++) {
4385                 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
4386                 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4387
4388                 memcpy(dst, src, PAGE_SIZE);
4389                 kunmap_atomic(dst, KM_USER0);
4390         }
4391         drm_clflush_pages(obj_priv->pages, page_count);
4392         drm_agp_chipset_flush(dev);
4393
4394         i915_gem_object_put_pages(obj);
4395 out:
4396         obj_priv->phys_obj->cur_obj = NULL;
4397         obj_priv->phys_obj = NULL;
4398 }
4399
4400 int
4401 i915_gem_attach_phys_object(struct drm_device *dev,
4402                             struct drm_gem_object *obj, int id)
4403 {
4404         drm_i915_private_t *dev_priv = dev->dev_private;
4405         struct drm_i915_gem_object *obj_priv;
4406         int ret = 0;
4407         int page_count;
4408         int i;
4409
4410         if (id > I915_MAX_PHYS_OBJECT)
4411                 return -EINVAL;
4412
4413         obj_priv = obj->driver_private;
4414
4415         if (obj_priv->phys_obj) {
4416                 if (obj_priv->phys_obj->id == id)
4417                         return 0;
4418                 i915_gem_detach_phys_object(dev, obj);
4419         }
4420
4421
4422         /* create a new object */
4423         if (!dev_priv->mm.phys_objs[id - 1]) {
4424                 ret = i915_gem_init_phys_object(dev, id,
4425                                                 obj->size);
4426                 if (ret) {
4427                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4428                         goto out;
4429                 }
4430         }
4431
4432         /* bind to the object */
4433         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4434         obj_priv->phys_obj->cur_obj = obj;
4435
4436         ret = i915_gem_object_get_pages(obj);
4437         if (ret) {
4438                 DRM_ERROR("failed to get page list\n");
4439                 goto out;
4440         }
4441
4442         page_count = obj->size / PAGE_SIZE;
4443
4444         for (i = 0; i < page_count; i++) {
4445                 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
4446                 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4447
4448                 memcpy(dst, src, PAGE_SIZE);
4449                 kunmap_atomic(src, KM_USER0);
4450         }
4451
4452         i915_gem_object_put_pages(obj);
4453
4454         return 0;
4455 out:
4456         return ret;
4457 }
4458
4459 static int
4460 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4461                      struct drm_i915_gem_pwrite *args,
4462                      struct drm_file *file_priv)
4463 {
4464         struct drm_i915_gem_object *obj_priv = obj->driver_private;
4465         void *obj_addr;
4466         int ret;
4467         char __user *user_data;
4468
4469         user_data = (char __user *) (uintptr_t) args->data_ptr;
4470         obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4471
4472         DRM_DEBUG("obj_addr %p, %lld\n", obj_addr, args->size);
4473         ret = copy_from_user(obj_addr, user_data, args->size);
4474         if (ret)
4475                 return -EFAULT;
4476
4477         drm_agp_chipset_flush(dev);
4478         return 0;
4479 }
4480
4481 void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
4482 {
4483         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
4484
4485         /* Clean up our request list when the client is going away, so that
4486          * later retire_requests won't dereference our soon-to-be-gone
4487          * file_priv.
4488          */
4489         mutex_lock(&dev->struct_mutex);
4490         while (!list_empty(&i915_file_priv->mm.request_list))
4491                 list_del_init(i915_file_priv->mm.request_list.next);
4492         mutex_unlock(&dev->struct_mutex);
4493 }