Merge branch 'drm-next' of git://people.freedesktop.org/~airlied/linux
[linux-3.10.git] / drivers / gpu / drm / i915 / i915_gem.c
1 /*
2  * Copyright © 2008 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Eric Anholt <eric@anholt.net>
25  *
26  */
27
28 #include <drm/drmP.h>
29 #include <drm/i915_drm.h>
30 #include "i915_drv.h"
31 #include "i915_trace.h"
32 #include "intel_drv.h"
33 #include <linux/shmem_fs.h>
34 #include <linux/slab.h>
35 #include <linux/swap.h>
36 #include <linux/pci.h>
37 #include <linux/dma-buf.h>
38
39 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
40 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
41 static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
42                                                     unsigned alignment,
43                                                     bool map_and_fenceable,
44                                                     bool nonblocking);
45 static int i915_gem_phys_pwrite(struct drm_device *dev,
46                                 struct drm_i915_gem_object *obj,
47                                 struct drm_i915_gem_pwrite *args,
48                                 struct drm_file *file);
49
50 static void i915_gem_write_fence(struct drm_device *dev, int reg,
51                                  struct drm_i915_gem_object *obj);
52 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
53                                          struct drm_i915_fence_reg *fence,
54                                          bool enable);
55
56 static int i915_gem_inactive_shrink(struct shrinker *shrinker,
57                                     struct shrink_control *sc);
58 static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
59 static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
60 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
61
62 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
63 {
64         if (obj->tiling_mode)
65                 i915_gem_release_mmap(obj);
66
67         /* As we do not have an associated fence register, we will force
68          * a tiling change if we ever need to acquire one.
69          */
70         obj->fence_dirty = false;
71         obj->fence_reg = I915_FENCE_REG_NONE;
72 }
73
74 /* some bookkeeping */
75 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
76                                   size_t size)
77 {
78         dev_priv->mm.object_count++;
79         dev_priv->mm.object_memory += size;
80 }
81
82 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
83                                      size_t size)
84 {
85         dev_priv->mm.object_count--;
86         dev_priv->mm.object_memory -= size;
87 }
88
89 static int
90 i915_gem_wait_for_error(struct drm_device *dev)
91 {
92         struct drm_i915_private *dev_priv = dev->dev_private;
93         struct completion *x = &dev_priv->error_completion;
94         unsigned long flags;
95         int ret;
96
97         if (!atomic_read(&dev_priv->mm.wedged))
98                 return 0;
99
100         /*
101          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
102          * userspace. If it takes that long something really bad is going on and
103          * we should simply try to bail out and fail as gracefully as possible.
104          */
105         ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
106         if (ret == 0) {
107                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
108                 return -EIO;
109         } else if (ret < 0) {
110                 return ret;
111         }
112
113         if (atomic_read(&dev_priv->mm.wedged)) {
114                 /* GPU is hung, bump the completion count to account for
115                  * the token we just consumed so that we never hit zero and
116                  * end up waiting upon a subsequent completion event that
117                  * will never happen.
118                  */
119                 spin_lock_irqsave(&x->wait.lock, flags);
120                 x->done++;
121                 spin_unlock_irqrestore(&x->wait.lock, flags);
122         }
123         return 0;
124 }
125
126 int i915_mutex_lock_interruptible(struct drm_device *dev)
127 {
128         int ret;
129
130         ret = i915_gem_wait_for_error(dev);
131         if (ret)
132                 return ret;
133
134         ret = mutex_lock_interruptible(&dev->struct_mutex);
135         if (ret)
136                 return ret;
137
138         WARN_ON(i915_verify_lists(dev));
139         return 0;
140 }
141
142 static inline bool
143 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
144 {
145         return obj->gtt_space && !obj->active;
146 }
147
148 int
149 i915_gem_init_ioctl(struct drm_device *dev, void *data,
150                     struct drm_file *file)
151 {
152         struct drm_i915_gem_init *args = data;
153
154         if (drm_core_check_feature(dev, DRIVER_MODESET))
155                 return -ENODEV;
156
157         if (args->gtt_start >= args->gtt_end ||
158             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
159                 return -EINVAL;
160
161         /* GEM with user mode setting was never supported on ilk and later. */
162         if (INTEL_INFO(dev)->gen >= 5)
163                 return -ENODEV;
164
165         mutex_lock(&dev->struct_mutex);
166         i915_gem_init_global_gtt(dev, args->gtt_start,
167                                  args->gtt_end, args->gtt_end);
168         mutex_unlock(&dev->struct_mutex);
169
170         return 0;
171 }
172
173 int
174 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
175                             struct drm_file *file)
176 {
177         struct drm_i915_private *dev_priv = dev->dev_private;
178         struct drm_i915_gem_get_aperture *args = data;
179         struct drm_i915_gem_object *obj;
180         size_t pinned;
181
182         pinned = 0;
183         mutex_lock(&dev->struct_mutex);
184         list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
185                 if (obj->pin_count)
186                         pinned += obj->gtt_space->size;
187         mutex_unlock(&dev->struct_mutex);
188
189         args->aper_size = dev_priv->mm.gtt_total;
190         args->aper_available_size = args->aper_size - pinned;
191
192         return 0;
193 }
194
195 static int
196 i915_gem_create(struct drm_file *file,
197                 struct drm_device *dev,
198                 uint64_t size,
199                 uint32_t *handle_p)
200 {
201         struct drm_i915_gem_object *obj;
202         int ret;
203         u32 handle;
204
205         size = roundup(size, PAGE_SIZE);
206         if (size == 0)
207                 return -EINVAL;
208
209         /* Allocate the new object */
210         obj = i915_gem_alloc_object(dev, size);
211         if (obj == NULL)
212                 return -ENOMEM;
213
214         ret = drm_gem_handle_create(file, &obj->base, &handle);
215         if (ret) {
216                 drm_gem_object_release(&obj->base);
217                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
218                 kfree(obj);
219                 return ret;
220         }
221
222         /* drop reference from allocate - handle holds it now */
223         drm_gem_object_unreference(&obj->base);
224         trace_i915_gem_object_create(obj);
225
226         *handle_p = handle;
227         return 0;
228 }
229
230 int
231 i915_gem_dumb_create(struct drm_file *file,
232                      struct drm_device *dev,
233                      struct drm_mode_create_dumb *args)
234 {
235         /* have to work out size/pitch and return them */
236         args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
237         args->size = args->pitch * args->height;
238         return i915_gem_create(file, dev,
239                                args->size, &args->handle);
240 }
241
242 int i915_gem_dumb_destroy(struct drm_file *file,
243                           struct drm_device *dev,
244                           uint32_t handle)
245 {
246         return drm_gem_handle_delete(file, handle);
247 }
248
249 /**
250  * Creates a new mm object and returns a handle to it.
251  */
252 int
253 i915_gem_create_ioctl(struct drm_device *dev, void *data,
254                       struct drm_file *file)
255 {
256         struct drm_i915_gem_create *args = data;
257
258         return i915_gem_create(file, dev,
259                                args->size, &args->handle);
260 }
261
262 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
263 {
264         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
265
266         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
267                 obj->tiling_mode != I915_TILING_NONE;
268 }
269
270 static inline int
271 __copy_to_user_swizzled(char __user *cpu_vaddr,
272                         const char *gpu_vaddr, int gpu_offset,
273                         int length)
274 {
275         int ret, cpu_offset = 0;
276
277         while (length > 0) {
278                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
279                 int this_length = min(cacheline_end - gpu_offset, length);
280                 int swizzled_gpu_offset = gpu_offset ^ 64;
281
282                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
283                                      gpu_vaddr + swizzled_gpu_offset,
284                                      this_length);
285                 if (ret)
286                         return ret + length;
287
288                 cpu_offset += this_length;
289                 gpu_offset += this_length;
290                 length -= this_length;
291         }
292
293         return 0;
294 }
295
296 static inline int
297 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
298                           const char __user *cpu_vaddr,
299                           int length)
300 {
301         int ret, cpu_offset = 0;
302
303         while (length > 0) {
304                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
305                 int this_length = min(cacheline_end - gpu_offset, length);
306                 int swizzled_gpu_offset = gpu_offset ^ 64;
307
308                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
309                                        cpu_vaddr + cpu_offset,
310                                        this_length);
311                 if (ret)
312                         return ret + length;
313
314                 cpu_offset += this_length;
315                 gpu_offset += this_length;
316                 length -= this_length;
317         }
318
319         return 0;
320 }
321
322 /* Per-page copy function for the shmem pread fastpath.
323  * Flushes invalid cachelines before reading the target if
324  * needs_clflush is set. */
325 static int
326 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
327                  char __user *user_data,
328                  bool page_do_bit17_swizzling, bool needs_clflush)
329 {
330         char *vaddr;
331         int ret;
332
333         if (unlikely(page_do_bit17_swizzling))
334                 return -EINVAL;
335
336         vaddr = kmap_atomic(page);
337         if (needs_clflush)
338                 drm_clflush_virt_range(vaddr + shmem_page_offset,
339                                        page_length);
340         ret = __copy_to_user_inatomic(user_data,
341                                       vaddr + shmem_page_offset,
342                                       page_length);
343         kunmap_atomic(vaddr);
344
345         return ret ? -EFAULT : 0;
346 }
347
348 static void
349 shmem_clflush_swizzled_range(char *addr, unsigned long length,
350                              bool swizzled)
351 {
352         if (unlikely(swizzled)) {
353                 unsigned long start = (unsigned long) addr;
354                 unsigned long end = (unsigned long) addr + length;
355
356                 /* For swizzling simply ensure that we always flush both
357                  * channels. Lame, but simple and it works. Swizzled
358                  * pwrite/pread is far from a hotpath - current userspace
359                  * doesn't use it at all. */
360                 start = round_down(start, 128);
361                 end = round_up(end, 128);
362
363                 drm_clflush_virt_range((void *)start, end - start);
364         } else {
365                 drm_clflush_virt_range(addr, length);
366         }
367
368 }
369
370 /* Only difference to the fast-path function is that this can handle bit17
371  * and uses non-atomic copy and kmap functions. */
372 static int
373 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
374                  char __user *user_data,
375                  bool page_do_bit17_swizzling, bool needs_clflush)
376 {
377         char *vaddr;
378         int ret;
379
380         vaddr = kmap(page);
381         if (needs_clflush)
382                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
383                                              page_length,
384                                              page_do_bit17_swizzling);
385
386         if (page_do_bit17_swizzling)
387                 ret = __copy_to_user_swizzled(user_data,
388                                               vaddr, shmem_page_offset,
389                                               page_length);
390         else
391                 ret = __copy_to_user(user_data,
392                                      vaddr + shmem_page_offset,
393                                      page_length);
394         kunmap(page);
395
396         return ret ? - EFAULT : 0;
397 }
398
399 static int
400 i915_gem_shmem_pread(struct drm_device *dev,
401                      struct drm_i915_gem_object *obj,
402                      struct drm_i915_gem_pread *args,
403                      struct drm_file *file)
404 {
405         char __user *user_data;
406         ssize_t remain;
407         loff_t offset;
408         int shmem_page_offset, page_length, ret = 0;
409         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
410         int hit_slowpath = 0;
411         int prefaulted = 0;
412         int needs_clflush = 0;
413         struct scatterlist *sg;
414         int i;
415
416         user_data = (char __user *) (uintptr_t) args->data_ptr;
417         remain = args->size;
418
419         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
420
421         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
422                 /* If we're not in the cpu read domain, set ourself into the gtt
423                  * read domain and manually flush cachelines (if required). This
424                  * optimizes for the case when the gpu will dirty the data
425                  * anyway again before the next pread happens. */
426                 if (obj->cache_level == I915_CACHE_NONE)
427                         needs_clflush = 1;
428                 if (obj->gtt_space) {
429                         ret = i915_gem_object_set_to_gtt_domain(obj, false);
430                         if (ret)
431                                 return ret;
432                 }
433         }
434
435         ret = i915_gem_object_get_pages(obj);
436         if (ret)
437                 return ret;
438
439         i915_gem_object_pin_pages(obj);
440
441         offset = args->offset;
442
443         for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
444                 struct page *page;
445
446                 if (i < offset >> PAGE_SHIFT)
447                         continue;
448
449                 if (remain <= 0)
450                         break;
451
452                 /* Operation in this page
453                  *
454                  * shmem_page_offset = offset within page in shmem file
455                  * page_length = bytes to copy for this page
456                  */
457                 shmem_page_offset = offset_in_page(offset);
458                 page_length = remain;
459                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
460                         page_length = PAGE_SIZE - shmem_page_offset;
461
462                 page = sg_page(sg);
463                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
464                         (page_to_phys(page) & (1 << 17)) != 0;
465
466                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
467                                        user_data, page_do_bit17_swizzling,
468                                        needs_clflush);
469                 if (ret == 0)
470                         goto next_page;
471
472                 hit_slowpath = 1;
473                 mutex_unlock(&dev->struct_mutex);
474
475                 if (!prefaulted) {
476                         ret = fault_in_multipages_writeable(user_data, remain);
477                         /* Userspace is tricking us, but we've already clobbered
478                          * its pages with the prefault and promised to write the
479                          * data up to the first fault. Hence ignore any errors
480                          * and just continue. */
481                         (void)ret;
482                         prefaulted = 1;
483                 }
484
485                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
486                                        user_data, page_do_bit17_swizzling,
487                                        needs_clflush);
488
489                 mutex_lock(&dev->struct_mutex);
490
491 next_page:
492                 mark_page_accessed(page);
493
494                 if (ret)
495                         goto out;
496
497                 remain -= page_length;
498                 user_data += page_length;
499                 offset += page_length;
500         }
501
502 out:
503         i915_gem_object_unpin_pages(obj);
504
505         if (hit_slowpath) {
506                 /* Fixup: Kill any reinstated backing storage pages */
507                 if (obj->madv == __I915_MADV_PURGED)
508                         i915_gem_object_truncate(obj);
509         }
510
511         return ret;
512 }
513
514 /**
515  * Reads data from the object referenced by handle.
516  *
517  * On error, the contents of *data are undefined.
518  */
519 int
520 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
521                      struct drm_file *file)
522 {
523         struct drm_i915_gem_pread *args = data;
524         struct drm_i915_gem_object *obj;
525         int ret = 0;
526
527         if (args->size == 0)
528                 return 0;
529
530         if (!access_ok(VERIFY_WRITE,
531                        (char __user *)(uintptr_t)args->data_ptr,
532                        args->size))
533                 return -EFAULT;
534
535         ret = i915_mutex_lock_interruptible(dev);
536         if (ret)
537                 return ret;
538
539         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
540         if (&obj->base == NULL) {
541                 ret = -ENOENT;
542                 goto unlock;
543         }
544
545         /* Bounds check source.  */
546         if (args->offset > obj->base.size ||
547             args->size > obj->base.size - args->offset) {
548                 ret = -EINVAL;
549                 goto out;
550         }
551
552         /* prime objects have no backing filp to GEM pread/pwrite
553          * pages from.
554          */
555         if (!obj->base.filp) {
556                 ret = -EINVAL;
557                 goto out;
558         }
559
560         trace_i915_gem_object_pread(obj, args->offset, args->size);
561
562         ret = i915_gem_shmem_pread(dev, obj, args, file);
563
564 out:
565         drm_gem_object_unreference(&obj->base);
566 unlock:
567         mutex_unlock(&dev->struct_mutex);
568         return ret;
569 }
570
571 /* This is the fast write path which cannot handle
572  * page faults in the source data
573  */
574
575 static inline int
576 fast_user_write(struct io_mapping *mapping,
577                 loff_t page_base, int page_offset,
578                 char __user *user_data,
579                 int length)
580 {
581         void __iomem *vaddr_atomic;
582         void *vaddr;
583         unsigned long unwritten;
584
585         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
586         /* We can use the cpu mem copy function because this is X86. */
587         vaddr = (void __force*)vaddr_atomic + page_offset;
588         unwritten = __copy_from_user_inatomic_nocache(vaddr,
589                                                       user_data, length);
590         io_mapping_unmap_atomic(vaddr_atomic);
591         return unwritten;
592 }
593
594 /**
595  * This is the fast pwrite path, where we copy the data directly from the
596  * user into the GTT, uncached.
597  */
598 static int
599 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
600                          struct drm_i915_gem_object *obj,
601                          struct drm_i915_gem_pwrite *args,
602                          struct drm_file *file)
603 {
604         drm_i915_private_t *dev_priv = dev->dev_private;
605         ssize_t remain;
606         loff_t offset, page_base;
607         char __user *user_data;
608         int page_offset, page_length, ret;
609
610         ret = i915_gem_object_pin(obj, 0, true, true);
611         if (ret)
612                 goto out;
613
614         ret = i915_gem_object_set_to_gtt_domain(obj, true);
615         if (ret)
616                 goto out_unpin;
617
618         ret = i915_gem_object_put_fence(obj);
619         if (ret)
620                 goto out_unpin;
621
622         user_data = (char __user *) (uintptr_t) args->data_ptr;
623         remain = args->size;
624
625         offset = obj->gtt_offset + args->offset;
626
627         while (remain > 0) {
628                 /* Operation in this page
629                  *
630                  * page_base = page offset within aperture
631                  * page_offset = offset within page
632                  * page_length = bytes to copy for this page
633                  */
634                 page_base = offset & PAGE_MASK;
635                 page_offset = offset_in_page(offset);
636                 page_length = remain;
637                 if ((page_offset + remain) > PAGE_SIZE)
638                         page_length = PAGE_SIZE - page_offset;
639
640                 /* If we get a fault while copying data, then (presumably) our
641                  * source page isn't available.  Return the error and we'll
642                  * retry in the slow path.
643                  */
644                 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
645                                     page_offset, user_data, page_length)) {
646                         ret = -EFAULT;
647                         goto out_unpin;
648                 }
649
650                 remain -= page_length;
651                 user_data += page_length;
652                 offset += page_length;
653         }
654
655 out_unpin:
656         i915_gem_object_unpin(obj);
657 out:
658         return ret;
659 }
660
661 /* Per-page copy function for the shmem pwrite fastpath.
662  * Flushes invalid cachelines before writing to the target if
663  * needs_clflush_before is set and flushes out any written cachelines after
664  * writing if needs_clflush is set. */
665 static int
666 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
667                   char __user *user_data,
668                   bool page_do_bit17_swizzling,
669                   bool needs_clflush_before,
670                   bool needs_clflush_after)
671 {
672         char *vaddr;
673         int ret;
674
675         if (unlikely(page_do_bit17_swizzling))
676                 return -EINVAL;
677
678         vaddr = kmap_atomic(page);
679         if (needs_clflush_before)
680                 drm_clflush_virt_range(vaddr + shmem_page_offset,
681                                        page_length);
682         ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
683                                                 user_data,
684                                                 page_length);
685         if (needs_clflush_after)
686                 drm_clflush_virt_range(vaddr + shmem_page_offset,
687                                        page_length);
688         kunmap_atomic(vaddr);
689
690         return ret ? -EFAULT : 0;
691 }
692
693 /* Only difference to the fast-path function is that this can handle bit17
694  * and uses non-atomic copy and kmap functions. */
695 static int
696 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
697                   char __user *user_data,
698                   bool page_do_bit17_swizzling,
699                   bool needs_clflush_before,
700                   bool needs_clflush_after)
701 {
702         char *vaddr;
703         int ret;
704
705         vaddr = kmap(page);
706         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
707                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
708                                              page_length,
709                                              page_do_bit17_swizzling);
710         if (page_do_bit17_swizzling)
711                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
712                                                 user_data,
713                                                 page_length);
714         else
715                 ret = __copy_from_user(vaddr + shmem_page_offset,
716                                        user_data,
717                                        page_length);
718         if (needs_clflush_after)
719                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
720                                              page_length,
721                                              page_do_bit17_swizzling);
722         kunmap(page);
723
724         return ret ? -EFAULT : 0;
725 }
726
727 static int
728 i915_gem_shmem_pwrite(struct drm_device *dev,
729                       struct drm_i915_gem_object *obj,
730                       struct drm_i915_gem_pwrite *args,
731                       struct drm_file *file)
732 {
733         ssize_t remain;
734         loff_t offset;
735         char __user *user_data;
736         int shmem_page_offset, page_length, ret = 0;
737         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
738         int hit_slowpath = 0;
739         int needs_clflush_after = 0;
740         int needs_clflush_before = 0;
741         int i;
742         struct scatterlist *sg;
743
744         user_data = (char __user *) (uintptr_t) args->data_ptr;
745         remain = args->size;
746
747         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
748
749         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
750                 /* If we're not in the cpu write domain, set ourself into the gtt
751                  * write domain and manually flush cachelines (if required). This
752                  * optimizes for the case when the gpu will use the data
753                  * right away and we therefore have to clflush anyway. */
754                 if (obj->cache_level == I915_CACHE_NONE)
755                         needs_clflush_after = 1;
756                 if (obj->gtt_space) {
757                         ret = i915_gem_object_set_to_gtt_domain(obj, true);
758                         if (ret)
759                                 return ret;
760                 }
761         }
762         /* Same trick applies for invalidate partially written cachelines before
763          * writing.  */
764         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
765             && obj->cache_level == I915_CACHE_NONE)
766                 needs_clflush_before = 1;
767
768         ret = i915_gem_object_get_pages(obj);
769         if (ret)
770                 return ret;
771
772         i915_gem_object_pin_pages(obj);
773
774         offset = args->offset;
775         obj->dirty = 1;
776
777         for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
778                 struct page *page;
779                 int partial_cacheline_write;
780
781                 if (i < offset >> PAGE_SHIFT)
782                         continue;
783
784                 if (remain <= 0)
785                         break;
786
787                 /* Operation in this page
788                  *
789                  * shmem_page_offset = offset within page in shmem file
790                  * page_length = bytes to copy for this page
791                  */
792                 shmem_page_offset = offset_in_page(offset);
793
794                 page_length = remain;
795                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
796                         page_length = PAGE_SIZE - shmem_page_offset;
797
798                 /* If we don't overwrite a cacheline completely we need to be
799                  * careful to have up-to-date data by first clflushing. Don't
800                  * overcomplicate things and flush the entire patch. */
801                 partial_cacheline_write = needs_clflush_before &&
802                         ((shmem_page_offset | page_length)
803                                 & (boot_cpu_data.x86_clflush_size - 1));
804
805                 page = sg_page(sg);
806                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
807                         (page_to_phys(page) & (1 << 17)) != 0;
808
809                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
810                                         user_data, page_do_bit17_swizzling,
811                                         partial_cacheline_write,
812                                         needs_clflush_after);
813                 if (ret == 0)
814                         goto next_page;
815
816                 hit_slowpath = 1;
817                 mutex_unlock(&dev->struct_mutex);
818                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
819                                         user_data, page_do_bit17_swizzling,
820                                         partial_cacheline_write,
821                                         needs_clflush_after);
822
823                 mutex_lock(&dev->struct_mutex);
824
825 next_page:
826                 set_page_dirty(page);
827                 mark_page_accessed(page);
828
829                 if (ret)
830                         goto out;
831
832                 remain -= page_length;
833                 user_data += page_length;
834                 offset += page_length;
835         }
836
837 out:
838         i915_gem_object_unpin_pages(obj);
839
840         if (hit_slowpath) {
841                 /* Fixup: Kill any reinstated backing storage pages */
842                 if (obj->madv == __I915_MADV_PURGED)
843                         i915_gem_object_truncate(obj);
844                 /* and flush dirty cachelines in case the object isn't in the cpu write
845                  * domain anymore. */
846                 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
847                         i915_gem_clflush_object(obj);
848                         intel_gtt_chipset_flush();
849                 }
850         }
851
852         if (needs_clflush_after)
853                 intel_gtt_chipset_flush();
854
855         return ret;
856 }
857
858 /**
859  * Writes data to the object referenced by handle.
860  *
861  * On error, the contents of the buffer that were to be modified are undefined.
862  */
863 int
864 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
865                       struct drm_file *file)
866 {
867         struct drm_i915_gem_pwrite *args = data;
868         struct drm_i915_gem_object *obj;
869         int ret;
870
871         if (args->size == 0)
872                 return 0;
873
874         if (!access_ok(VERIFY_READ,
875                        (char __user *)(uintptr_t)args->data_ptr,
876                        args->size))
877                 return -EFAULT;
878
879         ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr,
880                                            args->size);
881         if (ret)
882                 return -EFAULT;
883
884         ret = i915_mutex_lock_interruptible(dev);
885         if (ret)
886                 return ret;
887
888         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
889         if (&obj->base == NULL) {
890                 ret = -ENOENT;
891                 goto unlock;
892         }
893
894         /* Bounds check destination. */
895         if (args->offset > obj->base.size ||
896             args->size > obj->base.size - args->offset) {
897                 ret = -EINVAL;
898                 goto out;
899         }
900
901         /* prime objects have no backing filp to GEM pread/pwrite
902          * pages from.
903          */
904         if (!obj->base.filp) {
905                 ret = -EINVAL;
906                 goto out;
907         }
908
909         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
910
911         ret = -EFAULT;
912         /* We can only do the GTT pwrite on untiled buffers, as otherwise
913          * it would end up going through the fenced access, and we'll get
914          * different detiling behavior between reading and writing.
915          * pread/pwrite currently are reading and writing from the CPU
916          * perspective, requiring manual detiling by the client.
917          */
918         if (obj->phys_obj) {
919                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
920                 goto out;
921         }
922
923         if (obj->cache_level == I915_CACHE_NONE &&
924             obj->tiling_mode == I915_TILING_NONE &&
925             obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
926                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
927                 /* Note that the gtt paths might fail with non-page-backed user
928                  * pointers (e.g. gtt mappings when moving data between
929                  * textures). Fallback to the shmem path in that case. */
930         }
931
932         if (ret == -EFAULT || ret == -ENOSPC)
933                 ret = i915_gem_shmem_pwrite(dev, obj, args, file);
934
935 out:
936         drm_gem_object_unreference(&obj->base);
937 unlock:
938         mutex_unlock(&dev->struct_mutex);
939         return ret;
940 }
941
942 int
943 i915_gem_check_wedge(struct drm_i915_private *dev_priv,
944                      bool interruptible)
945 {
946         if (atomic_read(&dev_priv->mm.wedged)) {
947                 struct completion *x = &dev_priv->error_completion;
948                 bool recovery_complete;
949                 unsigned long flags;
950
951                 /* Give the error handler a chance to run. */
952                 spin_lock_irqsave(&x->wait.lock, flags);
953                 recovery_complete = x->done > 0;
954                 spin_unlock_irqrestore(&x->wait.lock, flags);
955
956                 /* Non-interruptible callers can't handle -EAGAIN, hence return
957                  * -EIO unconditionally for these. */
958                 if (!interruptible)
959                         return -EIO;
960
961                 /* Recovery complete, but still wedged means reset failure. */
962                 if (recovery_complete)
963                         return -EIO;
964
965                 return -EAGAIN;
966         }
967
968         return 0;
969 }
970
971 /*
972  * Compare seqno against outstanding lazy request. Emit a request if they are
973  * equal.
974  */
975 static int
976 i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
977 {
978         int ret;
979
980         BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
981
982         ret = 0;
983         if (seqno == ring->outstanding_lazy_request)
984                 ret = i915_add_request(ring, NULL, NULL);
985
986         return ret;
987 }
988
989 /**
990  * __wait_seqno - wait until execution of seqno has finished
991  * @ring: the ring expected to report seqno
992  * @seqno: duh!
993  * @interruptible: do an interruptible wait (normally yes)
994  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
995  *
996  * Returns 0 if the seqno was found within the alloted time. Else returns the
997  * errno with remaining time filled in timeout argument.
998  */
999 static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
1000                         bool interruptible, struct timespec *timeout)
1001 {
1002         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1003         struct timespec before, now, wait_time={1,0};
1004         unsigned long timeout_jiffies;
1005         long end;
1006         bool wait_forever = true;
1007         int ret;
1008
1009         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
1010                 return 0;
1011
1012         trace_i915_gem_request_wait_begin(ring, seqno);
1013
1014         if (timeout != NULL) {
1015                 wait_time = *timeout;
1016                 wait_forever = false;
1017         }
1018
1019         timeout_jiffies = timespec_to_jiffies(&wait_time);
1020
1021         if (WARN_ON(!ring->irq_get(ring)))
1022                 return -ENODEV;
1023
1024         /* Record current time in case interrupted by signal, or wedged * */
1025         getrawmonotonic(&before);
1026
1027 #define EXIT_COND \
1028         (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
1029         atomic_read(&dev_priv->mm.wedged))
1030         do {
1031                 if (interruptible)
1032                         end = wait_event_interruptible_timeout(ring->irq_queue,
1033                                                                EXIT_COND,
1034                                                                timeout_jiffies);
1035                 else
1036                         end = wait_event_timeout(ring->irq_queue, EXIT_COND,
1037                                                  timeout_jiffies);
1038
1039                 ret = i915_gem_check_wedge(dev_priv, interruptible);
1040                 if (ret)
1041                         end = ret;
1042         } while (end == 0 && wait_forever);
1043
1044         getrawmonotonic(&now);
1045
1046         ring->irq_put(ring);
1047         trace_i915_gem_request_wait_end(ring, seqno);
1048 #undef EXIT_COND
1049
1050         if (timeout) {
1051                 struct timespec sleep_time = timespec_sub(now, before);
1052                 *timeout = timespec_sub(*timeout, sleep_time);
1053         }
1054
1055         switch (end) {
1056         case -EIO:
1057         case -EAGAIN: /* Wedged */
1058         case -ERESTARTSYS: /* Signal */
1059                 return (int)end;
1060         case 0: /* Timeout */
1061                 if (timeout)
1062                         set_normalized_timespec(timeout, 0, 0);
1063                 return -ETIME;
1064         default: /* Completed */
1065                 WARN_ON(end < 0); /* We're not aware of other errors */
1066                 return 0;
1067         }
1068 }
1069
1070 /**
1071  * Waits for a sequence number to be signaled, and cleans up the
1072  * request and object lists appropriately for that event.
1073  */
1074 int
1075 i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
1076 {
1077         struct drm_device *dev = ring->dev;
1078         struct drm_i915_private *dev_priv = dev->dev_private;
1079         bool interruptible = dev_priv->mm.interruptible;
1080         int ret;
1081
1082         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1083         BUG_ON(seqno == 0);
1084
1085         ret = i915_gem_check_wedge(dev_priv, interruptible);
1086         if (ret)
1087                 return ret;
1088
1089         ret = i915_gem_check_olr(ring, seqno);
1090         if (ret)
1091                 return ret;
1092
1093         return __wait_seqno(ring, seqno, interruptible, NULL);
1094 }
1095
1096 /**
1097  * Ensures that all rendering to the object has completed and the object is
1098  * safe to unbind from the GTT or access from the CPU.
1099  */
1100 static __must_check int
1101 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1102                                bool readonly)
1103 {
1104         struct intel_ring_buffer *ring = obj->ring;
1105         u32 seqno;
1106         int ret;
1107
1108         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1109         if (seqno == 0)
1110                 return 0;
1111
1112         ret = i915_wait_seqno(ring, seqno);
1113         if (ret)
1114                 return ret;
1115
1116         i915_gem_retire_requests_ring(ring);
1117
1118         /* Manually manage the write flush as we may have not yet
1119          * retired the buffer.
1120          */
1121         if (obj->last_write_seqno &&
1122             i915_seqno_passed(seqno, obj->last_write_seqno)) {
1123                 obj->last_write_seqno = 0;
1124                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
1125         }
1126
1127         return 0;
1128 }
1129
1130 /* A nonblocking variant of the above wait. This is a highly dangerous routine
1131  * as the object state may change during this call.
1132  */
1133 static __must_check int
1134 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1135                                             bool readonly)
1136 {
1137         struct drm_device *dev = obj->base.dev;
1138         struct drm_i915_private *dev_priv = dev->dev_private;
1139         struct intel_ring_buffer *ring = obj->ring;
1140         u32 seqno;
1141         int ret;
1142
1143         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1144         BUG_ON(!dev_priv->mm.interruptible);
1145
1146         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1147         if (seqno == 0)
1148                 return 0;
1149
1150         ret = i915_gem_check_wedge(dev_priv, true);
1151         if (ret)
1152                 return ret;
1153
1154         ret = i915_gem_check_olr(ring, seqno);
1155         if (ret)
1156                 return ret;
1157
1158         mutex_unlock(&dev->struct_mutex);
1159         ret = __wait_seqno(ring, seqno, true, NULL);
1160         mutex_lock(&dev->struct_mutex);
1161
1162         i915_gem_retire_requests_ring(ring);
1163
1164         /* Manually manage the write flush as we may have not yet
1165          * retired the buffer.
1166          */
1167         if (obj->last_write_seqno &&
1168             i915_seqno_passed(seqno, obj->last_write_seqno)) {
1169                 obj->last_write_seqno = 0;
1170                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
1171         }
1172
1173         return ret;
1174 }
1175
1176 /**
1177  * Called when user space prepares to use an object with the CPU, either
1178  * through the mmap ioctl's mapping or a GTT mapping.
1179  */
1180 int
1181 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1182                           struct drm_file *file)
1183 {
1184         struct drm_i915_gem_set_domain *args = data;
1185         struct drm_i915_gem_object *obj;
1186         uint32_t read_domains = args->read_domains;
1187         uint32_t write_domain = args->write_domain;
1188         int ret;
1189
1190         /* Only handle setting domains to types used by the CPU. */
1191         if (write_domain & I915_GEM_GPU_DOMAINS)
1192                 return -EINVAL;
1193
1194         if (read_domains & I915_GEM_GPU_DOMAINS)
1195                 return -EINVAL;
1196
1197         /* Having something in the write domain implies it's in the read
1198          * domain, and only that read domain.  Enforce that in the request.
1199          */
1200         if (write_domain != 0 && read_domains != write_domain)
1201                 return -EINVAL;
1202
1203         ret = i915_mutex_lock_interruptible(dev);
1204         if (ret)
1205                 return ret;
1206
1207         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1208         if (&obj->base == NULL) {
1209                 ret = -ENOENT;
1210                 goto unlock;
1211         }
1212
1213         /* Try to flush the object off the GPU without holding the lock.
1214          * We will repeat the flush holding the lock in the normal manner
1215          * to catch cases where we are gazumped.
1216          */
1217         ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
1218         if (ret)
1219                 goto unref;
1220
1221         if (read_domains & I915_GEM_DOMAIN_GTT) {
1222                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1223
1224                 /* Silently promote "you're not bound, there was nothing to do"
1225                  * to success, since the client was just asking us to
1226                  * make sure everything was done.
1227                  */
1228                 if (ret == -EINVAL)
1229                         ret = 0;
1230         } else {
1231                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1232         }
1233
1234 unref:
1235         drm_gem_object_unreference(&obj->base);
1236 unlock:
1237         mutex_unlock(&dev->struct_mutex);
1238         return ret;
1239 }
1240
1241 /**
1242  * Called when user space has done writes to this buffer
1243  */
1244 int
1245 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1246                          struct drm_file *file)
1247 {
1248         struct drm_i915_gem_sw_finish *args = data;
1249         struct drm_i915_gem_object *obj;
1250         int ret = 0;
1251
1252         ret = i915_mutex_lock_interruptible(dev);
1253         if (ret)
1254                 return ret;
1255
1256         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1257         if (&obj->base == NULL) {
1258                 ret = -ENOENT;
1259                 goto unlock;
1260         }
1261
1262         /* Pinned buffers may be scanout, so flush the cache */
1263         if (obj->pin_count)
1264                 i915_gem_object_flush_cpu_write_domain(obj);
1265
1266         drm_gem_object_unreference(&obj->base);
1267 unlock:
1268         mutex_unlock(&dev->struct_mutex);
1269         return ret;
1270 }
1271
1272 /**
1273  * Maps the contents of an object, returning the address it is mapped
1274  * into.
1275  *
1276  * While the mapping holds a reference on the contents of the object, it doesn't
1277  * imply a ref on the object itself.
1278  */
1279 int
1280 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1281                     struct drm_file *file)
1282 {
1283         struct drm_i915_gem_mmap *args = data;
1284         struct drm_gem_object *obj;
1285         unsigned long addr;
1286
1287         obj = drm_gem_object_lookup(dev, file, args->handle);
1288         if (obj == NULL)
1289                 return -ENOENT;
1290
1291         /* prime objects have no backing filp to GEM mmap
1292          * pages from.
1293          */
1294         if (!obj->filp) {
1295                 drm_gem_object_unreference_unlocked(obj);
1296                 return -EINVAL;
1297         }
1298
1299         addr = vm_mmap(obj->filp, 0, args->size,
1300                        PROT_READ | PROT_WRITE, MAP_SHARED,
1301                        args->offset);
1302         drm_gem_object_unreference_unlocked(obj);
1303         if (IS_ERR((void *)addr))
1304                 return addr;
1305
1306         args->addr_ptr = (uint64_t) addr;
1307
1308         return 0;
1309 }
1310
1311 /**
1312  * i915_gem_fault - fault a page into the GTT
1313  * vma: VMA in question
1314  * vmf: fault info
1315  *
1316  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1317  * from userspace.  The fault handler takes care of binding the object to
1318  * the GTT (if needed), allocating and programming a fence register (again,
1319  * only if needed based on whether the old reg is still valid or the object
1320  * is tiled) and inserting a new PTE into the faulting process.
1321  *
1322  * Note that the faulting process may involve evicting existing objects
1323  * from the GTT and/or fence registers to make room.  So performance may
1324  * suffer if the GTT working set is large or there are few fence registers
1325  * left.
1326  */
1327 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1328 {
1329         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1330         struct drm_device *dev = obj->base.dev;
1331         drm_i915_private_t *dev_priv = dev->dev_private;
1332         pgoff_t page_offset;
1333         unsigned long pfn;
1334         int ret = 0;
1335         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1336
1337         /* We don't use vmf->pgoff since that has the fake offset */
1338         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1339                 PAGE_SHIFT;
1340
1341         ret = i915_mutex_lock_interruptible(dev);
1342         if (ret)
1343                 goto out;
1344
1345         trace_i915_gem_object_fault(obj, page_offset, true, write);
1346
1347         /* Now bind it into the GTT if needed */
1348         if (!obj->map_and_fenceable) {
1349                 ret = i915_gem_object_unbind(obj);
1350                 if (ret)
1351                         goto unlock;
1352         }
1353         if (!obj->gtt_space) {
1354                 ret = i915_gem_object_bind_to_gtt(obj, 0, true, false);
1355                 if (ret)
1356                         goto unlock;
1357
1358                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1359                 if (ret)
1360                         goto unlock;
1361         }
1362
1363         if (!obj->has_global_gtt_mapping)
1364                 i915_gem_gtt_bind_object(obj, obj->cache_level);
1365
1366         ret = i915_gem_object_get_fence(obj);
1367         if (ret)
1368                 goto unlock;
1369
1370         if (i915_gem_object_is_inactive(obj))
1371                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1372
1373         obj->fault_mappable = true;
1374
1375         pfn = ((dev_priv->mm.gtt_base_addr + obj->gtt_offset) >> PAGE_SHIFT) +
1376                 page_offset;
1377
1378         /* Finally, remap it using the new GTT offset */
1379         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1380 unlock:
1381         mutex_unlock(&dev->struct_mutex);
1382 out:
1383         switch (ret) {
1384         case -EIO:
1385                 /* If this -EIO is due to a gpu hang, give the reset code a
1386                  * chance to clean up the mess. Otherwise return the proper
1387                  * SIGBUS. */
1388                 if (!atomic_read(&dev_priv->mm.wedged))
1389                         return VM_FAULT_SIGBUS;
1390         case -EAGAIN:
1391                 /* Give the error handler a chance to run and move the
1392                  * objects off the GPU active list. Next time we service the
1393                  * fault, we should be able to transition the page into the
1394                  * GTT without touching the GPU (and so avoid further
1395                  * EIO/EGAIN). If the GPU is wedged, then there is no issue
1396                  * with coherency, just lost writes.
1397                  */
1398                 set_need_resched();
1399         case 0:
1400         case -ERESTARTSYS:
1401         case -EINTR:
1402                 return VM_FAULT_NOPAGE;
1403         case -ENOMEM:
1404                 return VM_FAULT_OOM;
1405         default:
1406                 return VM_FAULT_SIGBUS;
1407         }
1408 }
1409
1410 /**
1411  * i915_gem_release_mmap - remove physical page mappings
1412  * @obj: obj in question
1413  *
1414  * Preserve the reservation of the mmapping with the DRM core code, but
1415  * relinquish ownership of the pages back to the system.
1416  *
1417  * It is vital that we remove the page mapping if we have mapped a tiled
1418  * object through the GTT and then lose the fence register due to
1419  * resource pressure. Similarly if the object has been moved out of the
1420  * aperture, than pages mapped into userspace must be revoked. Removing the
1421  * mapping will then trigger a page fault on the next user access, allowing
1422  * fixup by i915_gem_fault().
1423  */
1424 void
1425 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1426 {
1427         if (!obj->fault_mappable)
1428                 return;
1429
1430         if (obj->base.dev->dev_mapping)
1431                 unmap_mapping_range(obj->base.dev->dev_mapping,
1432                                     (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1433                                     obj->base.size, 1);
1434
1435         obj->fault_mappable = false;
1436 }
1437
1438 static uint32_t
1439 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1440 {
1441         uint32_t gtt_size;
1442
1443         if (INTEL_INFO(dev)->gen >= 4 ||
1444             tiling_mode == I915_TILING_NONE)
1445                 return size;
1446
1447         /* Previous chips need a power-of-two fence region when tiling */
1448         if (INTEL_INFO(dev)->gen == 3)
1449                 gtt_size = 1024*1024;
1450         else
1451                 gtt_size = 512*1024;
1452
1453         while (gtt_size < size)
1454                 gtt_size <<= 1;
1455
1456         return gtt_size;
1457 }
1458
1459 /**
1460  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1461  * @obj: object to check
1462  *
1463  * Return the required GTT alignment for an object, taking into account
1464  * potential fence register mapping.
1465  */
1466 static uint32_t
1467 i915_gem_get_gtt_alignment(struct drm_device *dev,
1468                            uint32_t size,
1469                            int tiling_mode)
1470 {
1471         /*
1472          * Minimum alignment is 4k (GTT page size), but might be greater
1473          * if a fence register is needed for the object.
1474          */
1475         if (INTEL_INFO(dev)->gen >= 4 ||
1476             tiling_mode == I915_TILING_NONE)
1477                 return 4096;
1478
1479         /*
1480          * Previous chips need to be aligned to the size of the smallest
1481          * fence register that can contain the object.
1482          */
1483         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1484 }
1485
1486 /**
1487  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1488  *                                       unfenced object
1489  * @dev: the device
1490  * @size: size of the object
1491  * @tiling_mode: tiling mode of the object
1492  *
1493  * Return the required GTT alignment for an object, only taking into account
1494  * unfenced tiled surface requirements.
1495  */
1496 uint32_t
1497 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
1498                                     uint32_t size,
1499                                     int tiling_mode)
1500 {
1501         /*
1502          * Minimum alignment is 4k (GTT page size) for sane hw.
1503          */
1504         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
1505             tiling_mode == I915_TILING_NONE)
1506                 return 4096;
1507
1508         /* Previous hardware however needs to be aligned to a power-of-two
1509          * tile height. The simplest method for determining this is to reuse
1510          * the power-of-tile object size.
1511          */
1512         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1513 }
1514
1515 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
1516 {
1517         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1518         int ret;
1519
1520         if (obj->base.map_list.map)
1521                 return 0;
1522
1523         ret = drm_gem_create_mmap_offset(&obj->base);
1524         if (ret != -ENOSPC)
1525                 return ret;
1526
1527         /* Badly fragmented mmap space? The only way we can recover
1528          * space is by destroying unwanted objects. We can't randomly release
1529          * mmap_offsets as userspace expects them to be persistent for the
1530          * lifetime of the objects. The closest we can is to release the
1531          * offsets on purgeable objects by truncating it and marking it purged,
1532          * which prevents userspace from ever using that object again.
1533          */
1534         i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT);
1535         ret = drm_gem_create_mmap_offset(&obj->base);
1536         if (ret != -ENOSPC)
1537                 return ret;
1538
1539         i915_gem_shrink_all(dev_priv);
1540         return drm_gem_create_mmap_offset(&obj->base);
1541 }
1542
1543 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
1544 {
1545         if (!obj->base.map_list.map)
1546                 return;
1547
1548         drm_gem_free_mmap_offset(&obj->base);
1549 }
1550
1551 int
1552 i915_gem_mmap_gtt(struct drm_file *file,
1553                   struct drm_device *dev,
1554                   uint32_t handle,
1555                   uint64_t *offset)
1556 {
1557         struct drm_i915_private *dev_priv = dev->dev_private;
1558         struct drm_i915_gem_object *obj;
1559         int ret;
1560
1561         ret = i915_mutex_lock_interruptible(dev);
1562         if (ret)
1563                 return ret;
1564
1565         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1566         if (&obj->base == NULL) {
1567                 ret = -ENOENT;
1568                 goto unlock;
1569         }
1570
1571         if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
1572                 ret = -E2BIG;
1573                 goto out;
1574         }
1575
1576         if (obj->madv != I915_MADV_WILLNEED) {
1577                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1578                 ret = -EINVAL;
1579                 goto out;
1580         }
1581
1582         ret = i915_gem_object_create_mmap_offset(obj);
1583         if (ret)
1584                 goto out;
1585
1586         *offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
1587
1588 out:
1589         drm_gem_object_unreference(&obj->base);
1590 unlock:
1591         mutex_unlock(&dev->struct_mutex);
1592         return ret;
1593 }
1594
1595 /**
1596  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1597  * @dev: DRM device
1598  * @data: GTT mapping ioctl data
1599  * @file: GEM object info
1600  *
1601  * Simply returns the fake offset to userspace so it can mmap it.
1602  * The mmap call will end up in drm_gem_mmap(), which will set things
1603  * up so we can get faults in the handler above.
1604  *
1605  * The fault handler will take care of binding the object into the GTT
1606  * (since it may have been evicted to make room for something), allocating
1607  * a fence register, and mapping the appropriate aperture address into
1608  * userspace.
1609  */
1610 int
1611 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1612                         struct drm_file *file)
1613 {
1614         struct drm_i915_gem_mmap_gtt *args = data;
1615
1616         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1617 }
1618
1619 /* Immediately discard the backing storage */
1620 static void
1621 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1622 {
1623         struct inode *inode;
1624
1625         i915_gem_object_free_mmap_offset(obj);
1626
1627         if (obj->base.filp == NULL)
1628                 return;
1629
1630         /* Our goal here is to return as much of the memory as
1631          * is possible back to the system as we are called from OOM.
1632          * To do this we must instruct the shmfs to drop all of its
1633          * backing pages, *now*.
1634          */
1635         inode = obj->base.filp->f_path.dentry->d_inode;
1636         shmem_truncate_range(inode, 0, (loff_t)-1);
1637
1638         obj->madv = __I915_MADV_PURGED;
1639 }
1640
1641 static inline int
1642 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1643 {
1644         return obj->madv == I915_MADV_DONTNEED;
1645 }
1646
1647 static void
1648 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1649 {
1650         int page_count = obj->base.size / PAGE_SIZE;
1651         struct scatterlist *sg;
1652         int ret, i;
1653
1654         BUG_ON(obj->madv == __I915_MADV_PURGED);
1655
1656         ret = i915_gem_object_set_to_cpu_domain(obj, true);
1657         if (ret) {
1658                 /* In the event of a disaster, abandon all caches and
1659                  * hope for the best.
1660                  */
1661                 WARN_ON(ret != -EIO);
1662                 i915_gem_clflush_object(obj);
1663                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1664         }
1665
1666         if (i915_gem_object_needs_bit17_swizzle(obj))
1667                 i915_gem_object_save_bit_17_swizzle(obj);
1668
1669         if (obj->madv == I915_MADV_DONTNEED)
1670                 obj->dirty = 0;
1671
1672         for_each_sg(obj->pages->sgl, sg, page_count, i) {
1673                 struct page *page = sg_page(sg);
1674
1675                 if (obj->dirty)
1676                         set_page_dirty(page);
1677
1678                 if (obj->madv == I915_MADV_WILLNEED)
1679                         mark_page_accessed(page);
1680
1681                 page_cache_release(page);
1682         }
1683         obj->dirty = 0;
1684
1685         sg_free_table(obj->pages);
1686         kfree(obj->pages);
1687 }
1688
1689 static int
1690 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
1691 {
1692         const struct drm_i915_gem_object_ops *ops = obj->ops;
1693
1694         if (obj->pages == NULL)
1695                 return 0;
1696
1697         BUG_ON(obj->gtt_space);
1698
1699         if (obj->pages_pin_count)
1700                 return -EBUSY;
1701
1702         ops->put_pages(obj);
1703         obj->pages = NULL;
1704
1705         list_del(&obj->gtt_list);
1706         if (i915_gem_object_is_purgeable(obj))
1707                 i915_gem_object_truncate(obj);
1708
1709         return 0;
1710 }
1711
1712 static long
1713 i915_gem_purge(struct drm_i915_private *dev_priv, long target)
1714 {
1715         struct drm_i915_gem_object *obj, *next;
1716         long count = 0;
1717
1718         list_for_each_entry_safe(obj, next,
1719                                  &dev_priv->mm.unbound_list,
1720                                  gtt_list) {
1721                 if (i915_gem_object_is_purgeable(obj) &&
1722                     i915_gem_object_put_pages(obj) == 0) {
1723                         count += obj->base.size >> PAGE_SHIFT;
1724                         if (count >= target)
1725                                 return count;
1726                 }
1727         }
1728
1729         list_for_each_entry_safe(obj, next,
1730                                  &dev_priv->mm.inactive_list,
1731                                  mm_list) {
1732                 if (i915_gem_object_is_purgeable(obj) &&
1733                     i915_gem_object_unbind(obj) == 0 &&
1734                     i915_gem_object_put_pages(obj) == 0) {
1735                         count += obj->base.size >> PAGE_SHIFT;
1736                         if (count >= target)
1737                                 return count;
1738                 }
1739         }
1740
1741         return count;
1742 }
1743
1744 static void
1745 i915_gem_shrink_all(struct drm_i915_private *dev_priv)
1746 {
1747         struct drm_i915_gem_object *obj, *next;
1748
1749         i915_gem_evict_everything(dev_priv->dev);
1750
1751         list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list, gtt_list)
1752                 i915_gem_object_put_pages(obj);
1753 }
1754
1755 static int
1756 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
1757 {
1758         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1759         int page_count, i;
1760         struct address_space *mapping;
1761         struct sg_table *st;
1762         struct scatterlist *sg;
1763         struct page *page;
1764         gfp_t gfp;
1765
1766         /* Assert that the object is not currently in any GPU domain. As it
1767          * wasn't in the GTT, there shouldn't be any way it could have been in
1768          * a GPU cache
1769          */
1770         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
1771         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
1772
1773         st = kmalloc(sizeof(*st), GFP_KERNEL);
1774         if (st == NULL)
1775                 return -ENOMEM;
1776
1777         page_count = obj->base.size / PAGE_SIZE;
1778         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
1779                 sg_free_table(st);
1780                 kfree(st);
1781                 return -ENOMEM;
1782         }
1783
1784         /* Get the list of pages out of our struct file.  They'll be pinned
1785          * at this point until we release them.
1786          *
1787          * Fail silently without starting the shrinker
1788          */
1789         mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
1790         gfp = mapping_gfp_mask(mapping);
1791         gfp |= __GFP_NORETRY | __GFP_NOWARN;
1792         gfp &= ~(__GFP_IO | __GFP_WAIT);
1793         for_each_sg(st->sgl, sg, page_count, i) {
1794                 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1795                 if (IS_ERR(page)) {
1796                         i915_gem_purge(dev_priv, page_count);
1797                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1798                 }
1799                 if (IS_ERR(page)) {
1800                         /* We've tried hard to allocate the memory by reaping
1801                          * our own buffer, now let the real VM do its job and
1802                          * go down in flames if truly OOM.
1803                          */
1804                         gfp &= ~(__GFP_NORETRY | __GFP_NOWARN);
1805                         gfp |= __GFP_IO | __GFP_WAIT;
1806
1807                         i915_gem_shrink_all(dev_priv);
1808                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1809                         if (IS_ERR(page))
1810                                 goto err_pages;
1811
1812                         gfp |= __GFP_NORETRY | __GFP_NOWARN;
1813                         gfp &= ~(__GFP_IO | __GFP_WAIT);
1814                 }
1815
1816                 sg_set_page(sg, page, PAGE_SIZE, 0);
1817         }
1818
1819         if (i915_gem_object_needs_bit17_swizzle(obj))
1820                 i915_gem_object_do_bit_17_swizzle(obj);
1821
1822         obj->pages = st;
1823         return 0;
1824
1825 err_pages:
1826         for_each_sg(st->sgl, sg, i, page_count)
1827                 page_cache_release(sg_page(sg));
1828         sg_free_table(st);
1829         kfree(st);
1830         return PTR_ERR(page);
1831 }
1832
1833 /* Ensure that the associated pages are gathered from the backing storage
1834  * and pinned into our object. i915_gem_object_get_pages() may be called
1835  * multiple times before they are released by a single call to
1836  * i915_gem_object_put_pages() - once the pages are no longer referenced
1837  * either as a result of memory pressure (reaping pages under the shrinker)
1838  * or as the object is itself released.
1839  */
1840 int
1841 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
1842 {
1843         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1844         const struct drm_i915_gem_object_ops *ops = obj->ops;
1845         int ret;
1846
1847         if (obj->pages)
1848                 return 0;
1849
1850         BUG_ON(obj->pages_pin_count);
1851
1852         ret = ops->get_pages(obj);
1853         if (ret)
1854                 return ret;
1855
1856         list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
1857         return 0;
1858 }
1859
1860 void
1861 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1862                                struct intel_ring_buffer *ring,
1863                                u32 seqno)
1864 {
1865         struct drm_device *dev = obj->base.dev;
1866         struct drm_i915_private *dev_priv = dev->dev_private;
1867
1868         BUG_ON(ring == NULL);
1869         obj->ring = ring;
1870
1871         /* Add a reference if we're newly entering the active list. */
1872         if (!obj->active) {
1873                 drm_gem_object_reference(&obj->base);
1874                 obj->active = 1;
1875         }
1876
1877         /* Move from whatever list we were on to the tail of execution. */
1878         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1879         list_move_tail(&obj->ring_list, &ring->active_list);
1880
1881         obj->last_read_seqno = seqno;
1882
1883         if (obj->fenced_gpu_access) {
1884                 obj->last_fenced_seqno = seqno;
1885
1886                 /* Bump MRU to take account of the delayed flush */
1887                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
1888                         struct drm_i915_fence_reg *reg;
1889
1890                         reg = &dev_priv->fence_regs[obj->fence_reg];
1891                         list_move_tail(&reg->lru_list,
1892                                        &dev_priv->mm.fence_list);
1893                 }
1894         }
1895 }
1896
1897 static void
1898 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1899 {
1900         struct drm_device *dev = obj->base.dev;
1901         struct drm_i915_private *dev_priv = dev->dev_private;
1902
1903         BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
1904         BUG_ON(!obj->active);
1905
1906         if (obj->pin_count) /* are we a framebuffer? */
1907                 intel_mark_fb_idle(obj);
1908
1909         list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1910
1911         list_del_init(&obj->ring_list);
1912         obj->ring = NULL;
1913
1914         obj->last_read_seqno = 0;
1915         obj->last_write_seqno = 0;
1916         obj->base.write_domain = 0;
1917
1918         obj->last_fenced_seqno = 0;
1919         obj->fenced_gpu_access = false;
1920
1921         obj->active = 0;
1922         drm_gem_object_unreference(&obj->base);
1923
1924         WARN_ON(i915_verify_lists(dev));
1925 }
1926
1927 static u32
1928 i915_gem_get_seqno(struct drm_device *dev)
1929 {
1930         drm_i915_private_t *dev_priv = dev->dev_private;
1931         u32 seqno = dev_priv->next_seqno;
1932
1933         /* reserve 0 for non-seqno */
1934         if (++dev_priv->next_seqno == 0)
1935                 dev_priv->next_seqno = 1;
1936
1937         return seqno;
1938 }
1939
1940 u32
1941 i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
1942 {
1943         if (ring->outstanding_lazy_request == 0)
1944                 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
1945
1946         return ring->outstanding_lazy_request;
1947 }
1948
1949 int
1950 i915_add_request(struct intel_ring_buffer *ring,
1951                  struct drm_file *file,
1952                  struct drm_i915_gem_request *request)
1953 {
1954         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1955         uint32_t seqno;
1956         u32 request_ring_position;
1957         int was_empty;
1958         int ret;
1959
1960         /*
1961          * Emit any outstanding flushes - execbuf can fail to emit the flush
1962          * after having emitted the batchbuffer command. Hence we need to fix
1963          * things up similar to emitting the lazy request. The difference here
1964          * is that the flush _must_ happen before the next request, no matter
1965          * what.
1966          */
1967         ret = intel_ring_flush_all_caches(ring);
1968         if (ret)
1969                 return ret;
1970
1971         if (request == NULL) {
1972                 request = kmalloc(sizeof(*request), GFP_KERNEL);
1973                 if (request == NULL)
1974                         return -ENOMEM;
1975         }
1976
1977         seqno = i915_gem_next_request_seqno(ring);
1978
1979         /* Record the position of the start of the request so that
1980          * should we detect the updated seqno part-way through the
1981          * GPU processing the request, we never over-estimate the
1982          * position of the head.
1983          */
1984         request_ring_position = intel_ring_get_tail(ring);
1985
1986         ret = ring->add_request(ring, &seqno);
1987         if (ret) {
1988                 kfree(request);
1989                 return ret;
1990         }
1991
1992         trace_i915_gem_request_add(ring, seqno);
1993
1994         request->seqno = seqno;
1995         request->ring = ring;
1996         request->tail = request_ring_position;
1997         request->emitted_jiffies = jiffies;
1998         was_empty = list_empty(&ring->request_list);
1999         list_add_tail(&request->list, &ring->request_list);
2000         request->file_priv = NULL;
2001
2002         if (file) {
2003                 struct drm_i915_file_private *file_priv = file->driver_priv;
2004
2005                 spin_lock(&file_priv->mm.lock);
2006                 request->file_priv = file_priv;
2007                 list_add_tail(&request->client_list,
2008                               &file_priv->mm.request_list);
2009                 spin_unlock(&file_priv->mm.lock);
2010         }
2011
2012         ring->outstanding_lazy_request = 0;
2013
2014         if (!dev_priv->mm.suspended) {
2015                 if (i915_enable_hangcheck) {
2016                         mod_timer(&dev_priv->hangcheck_timer,
2017                                   jiffies +
2018                                   msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
2019                 }
2020                 if (was_empty) {
2021                         queue_delayed_work(dev_priv->wq,
2022                                            &dev_priv->mm.retire_work, HZ);
2023                         intel_mark_busy(dev_priv->dev);
2024                 }
2025         }
2026
2027         return 0;
2028 }
2029
2030 static inline void
2031 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
2032 {
2033         struct drm_i915_file_private *file_priv = request->file_priv;
2034
2035         if (!file_priv)
2036                 return;
2037
2038         spin_lock(&file_priv->mm.lock);
2039         if (request->file_priv) {
2040                 list_del(&request->client_list);
2041                 request->file_priv = NULL;
2042         }
2043         spin_unlock(&file_priv->mm.lock);
2044 }
2045
2046 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
2047                                       struct intel_ring_buffer *ring)
2048 {
2049         while (!list_empty(&ring->request_list)) {
2050                 struct drm_i915_gem_request *request;
2051
2052                 request = list_first_entry(&ring->request_list,
2053                                            struct drm_i915_gem_request,
2054                                            list);
2055
2056                 list_del(&request->list);
2057                 i915_gem_request_remove_from_client(request);
2058                 kfree(request);
2059         }
2060
2061         while (!list_empty(&ring->active_list)) {
2062                 struct drm_i915_gem_object *obj;
2063
2064                 obj = list_first_entry(&ring->active_list,
2065                                        struct drm_i915_gem_object,
2066                                        ring_list);
2067
2068                 i915_gem_object_move_to_inactive(obj);
2069         }
2070 }
2071
2072 static void i915_gem_reset_fences(struct drm_device *dev)
2073 {
2074         struct drm_i915_private *dev_priv = dev->dev_private;
2075         int i;
2076
2077         for (i = 0; i < dev_priv->num_fence_regs; i++) {
2078                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2079
2080                 i915_gem_write_fence(dev, i, NULL);
2081
2082                 if (reg->obj)
2083                         i915_gem_object_fence_lost(reg->obj);
2084
2085                 reg->pin_count = 0;
2086                 reg->obj = NULL;
2087                 INIT_LIST_HEAD(&reg->lru_list);
2088         }
2089
2090         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
2091 }
2092
2093 void i915_gem_reset(struct drm_device *dev)
2094 {
2095         struct drm_i915_private *dev_priv = dev->dev_private;
2096         struct drm_i915_gem_object *obj;
2097         struct intel_ring_buffer *ring;
2098         int i;
2099
2100         for_each_ring(ring, dev_priv, i)
2101                 i915_gem_reset_ring_lists(dev_priv, ring);
2102
2103         /* Move everything out of the GPU domains to ensure we do any
2104          * necessary invalidation upon reuse.
2105          */
2106         list_for_each_entry(obj,
2107                             &dev_priv->mm.inactive_list,
2108                             mm_list)
2109         {
2110                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2111         }
2112
2113         /* The fence registers are invalidated so clear them out */
2114         i915_gem_reset_fences(dev);
2115 }
2116
2117 /**
2118  * This function clears the request list as sequence numbers are passed.
2119  */
2120 void
2121 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
2122 {
2123         uint32_t seqno;
2124         int i;
2125
2126         if (list_empty(&ring->request_list))
2127                 return;
2128
2129         WARN_ON(i915_verify_lists(ring->dev));
2130
2131         seqno = ring->get_seqno(ring, true);
2132
2133         for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
2134                 if (seqno >= ring->sync_seqno[i])
2135                         ring->sync_seqno[i] = 0;
2136
2137         while (!list_empty(&ring->request_list)) {
2138                 struct drm_i915_gem_request *request;
2139
2140                 request = list_first_entry(&ring->request_list,
2141                                            struct drm_i915_gem_request,
2142                                            list);
2143
2144                 if (!i915_seqno_passed(seqno, request->seqno))
2145                         break;
2146
2147                 trace_i915_gem_request_retire(ring, request->seqno);
2148                 /* We know the GPU must have read the request to have
2149                  * sent us the seqno + interrupt, so use the position
2150                  * of tail of the request to update the last known position
2151                  * of the GPU head.
2152                  */
2153                 ring->last_retired_head = request->tail;
2154
2155                 list_del(&request->list);
2156                 i915_gem_request_remove_from_client(request);
2157                 kfree(request);
2158         }
2159
2160         /* Move any buffers on the active list that are no longer referenced
2161          * by the ringbuffer to the flushing/inactive lists as appropriate.
2162          */
2163         while (!list_empty(&ring->active_list)) {
2164                 struct drm_i915_gem_object *obj;
2165
2166                 obj = list_first_entry(&ring->active_list,
2167                                       struct drm_i915_gem_object,
2168                                       ring_list);
2169
2170                 if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2171                         break;
2172
2173                 i915_gem_object_move_to_inactive(obj);
2174         }
2175
2176         if (unlikely(ring->trace_irq_seqno &&
2177                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
2178                 ring->irq_put(ring);
2179                 ring->trace_irq_seqno = 0;
2180         }
2181
2182         WARN_ON(i915_verify_lists(ring->dev));
2183 }
2184
2185 void
2186 i915_gem_retire_requests(struct drm_device *dev)
2187 {
2188         drm_i915_private_t *dev_priv = dev->dev_private;
2189         struct intel_ring_buffer *ring;
2190         int i;
2191
2192         for_each_ring(ring, dev_priv, i)
2193                 i915_gem_retire_requests_ring(ring);
2194 }
2195
2196 static void
2197 i915_gem_retire_work_handler(struct work_struct *work)
2198 {
2199         drm_i915_private_t *dev_priv;
2200         struct drm_device *dev;
2201         struct intel_ring_buffer *ring;
2202         bool idle;
2203         int i;
2204
2205         dev_priv = container_of(work, drm_i915_private_t,
2206                                 mm.retire_work.work);
2207         dev = dev_priv->dev;
2208
2209         /* Come back later if the device is busy... */
2210         if (!mutex_trylock(&dev->struct_mutex)) {
2211                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2212                 return;
2213         }
2214
2215         i915_gem_retire_requests(dev);
2216
2217         /* Send a periodic flush down the ring so we don't hold onto GEM
2218          * objects indefinitely.
2219          */
2220         idle = true;
2221         for_each_ring(ring, dev_priv, i) {
2222                 if (ring->gpu_caches_dirty)
2223                         i915_add_request(ring, NULL, NULL);
2224
2225                 idle &= list_empty(&ring->request_list);
2226         }
2227
2228         if (!dev_priv->mm.suspended && !idle)
2229                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2230         if (idle)
2231                 intel_mark_idle(dev);
2232
2233         mutex_unlock(&dev->struct_mutex);
2234 }
2235
2236 /**
2237  * Ensures that an object will eventually get non-busy by flushing any required
2238  * write domains, emitting any outstanding lazy request and retiring and
2239  * completed requests.
2240  */
2241 static int
2242 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2243 {
2244         int ret;
2245
2246         if (obj->active) {
2247                 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
2248                 if (ret)
2249                         return ret;
2250
2251                 i915_gem_retire_requests_ring(obj->ring);
2252         }
2253
2254         return 0;
2255 }
2256
2257 /**
2258  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2259  * @DRM_IOCTL_ARGS: standard ioctl arguments
2260  *
2261  * Returns 0 if successful, else an error is returned with the remaining time in
2262  * the timeout parameter.
2263  *  -ETIME: object is still busy after timeout
2264  *  -ERESTARTSYS: signal interrupted the wait
2265  *  -ENONENT: object doesn't exist
2266  * Also possible, but rare:
2267  *  -EAGAIN: GPU wedged
2268  *  -ENOMEM: damn
2269  *  -ENODEV: Internal IRQ fail
2270  *  -E?: The add request failed
2271  *
2272  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2273  * non-zero timeout parameter the wait ioctl will wait for the given number of
2274  * nanoseconds on an object becoming unbusy. Since the wait itself does so
2275  * without holding struct_mutex the object may become re-busied before this
2276  * function completes. A similar but shorter * race condition exists in the busy
2277  * ioctl
2278  */
2279 int
2280 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2281 {
2282         struct drm_i915_gem_wait *args = data;
2283         struct drm_i915_gem_object *obj;
2284         struct intel_ring_buffer *ring = NULL;
2285         struct timespec timeout_stack, *timeout = NULL;
2286         u32 seqno = 0;
2287         int ret = 0;
2288
2289         if (args->timeout_ns >= 0) {
2290                 timeout_stack = ns_to_timespec(args->timeout_ns);
2291                 timeout = &timeout_stack;
2292         }
2293
2294         ret = i915_mutex_lock_interruptible(dev);
2295         if (ret)
2296                 return ret;
2297
2298         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
2299         if (&obj->base == NULL) {
2300                 mutex_unlock(&dev->struct_mutex);
2301                 return -ENOENT;
2302         }
2303
2304         /* Need to make sure the object gets inactive eventually. */
2305         ret = i915_gem_object_flush_active(obj);
2306         if (ret)
2307                 goto out;
2308
2309         if (obj->active) {
2310                 seqno = obj->last_read_seqno;
2311                 ring = obj->ring;
2312         }
2313
2314         if (seqno == 0)
2315                  goto out;
2316
2317         /* Do this after OLR check to make sure we make forward progress polling
2318          * on this IOCTL with a 0 timeout (like busy ioctl)
2319          */
2320         if (!args->timeout_ns) {
2321                 ret = -ETIME;
2322                 goto out;
2323         }
2324
2325         drm_gem_object_unreference(&obj->base);
2326         mutex_unlock(&dev->struct_mutex);
2327
2328         ret = __wait_seqno(ring, seqno, true, timeout);
2329         if (timeout) {
2330                 WARN_ON(!timespec_valid(timeout));
2331                 args->timeout_ns = timespec_to_ns(timeout);
2332         }
2333         return ret;
2334
2335 out:
2336         drm_gem_object_unreference(&obj->base);
2337         mutex_unlock(&dev->struct_mutex);
2338         return ret;
2339 }
2340
2341 /**
2342  * i915_gem_object_sync - sync an object to a ring.
2343  *
2344  * @obj: object which may be in use on another ring.
2345  * @to: ring we wish to use the object on. May be NULL.
2346  *
2347  * This code is meant to abstract object synchronization with the GPU.
2348  * Calling with NULL implies synchronizing the object with the CPU
2349  * rather than a particular GPU ring.
2350  *
2351  * Returns 0 if successful, else propagates up the lower layer error.
2352  */
2353 int
2354 i915_gem_object_sync(struct drm_i915_gem_object *obj,
2355                      struct intel_ring_buffer *to)
2356 {
2357         struct intel_ring_buffer *from = obj->ring;
2358         u32 seqno;
2359         int ret, idx;
2360
2361         if (from == NULL || to == from)
2362                 return 0;
2363
2364         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
2365                 return i915_gem_object_wait_rendering(obj, false);
2366
2367         idx = intel_ring_sync_index(from, to);
2368
2369         seqno = obj->last_read_seqno;
2370         if (seqno <= from->sync_seqno[idx])
2371                 return 0;
2372
2373         ret = i915_gem_check_olr(obj->ring, seqno);
2374         if (ret)
2375                 return ret;
2376
2377         ret = to->sync_to(to, from, seqno);
2378         if (!ret)
2379                 from->sync_seqno[idx] = seqno;
2380
2381         return ret;
2382 }
2383
2384 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2385 {
2386         u32 old_write_domain, old_read_domains;
2387
2388         /* Act a barrier for all accesses through the GTT */
2389         mb();
2390
2391         /* Force a pagefault for domain tracking on next user access */
2392         i915_gem_release_mmap(obj);
2393
2394         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2395                 return;
2396
2397         old_read_domains = obj->base.read_domains;
2398         old_write_domain = obj->base.write_domain;
2399
2400         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2401         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2402
2403         trace_i915_gem_object_change_domain(obj,
2404                                             old_read_domains,
2405                                             old_write_domain);
2406 }
2407
2408 /**
2409  * Unbinds an object from the GTT aperture.
2410  */
2411 int
2412 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2413 {
2414         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2415         int ret = 0;
2416
2417         if (obj->gtt_space == NULL)
2418                 return 0;
2419
2420         if (obj->pin_count)
2421                 return -EBUSY;
2422
2423         BUG_ON(obj->pages == NULL);
2424
2425         ret = i915_gem_object_finish_gpu(obj);
2426         if (ret)
2427                 return ret;
2428         /* Continue on if we fail due to EIO, the GPU is hung so we
2429          * should be safe and we need to cleanup or else we might
2430          * cause memory corruption through use-after-free.
2431          */
2432
2433         i915_gem_object_finish_gtt(obj);
2434
2435         /* release the fence reg _after_ flushing */
2436         ret = i915_gem_object_put_fence(obj);
2437         if (ret)
2438                 return ret;
2439
2440         trace_i915_gem_object_unbind(obj);
2441
2442         if (obj->has_global_gtt_mapping)
2443                 i915_gem_gtt_unbind_object(obj);
2444         if (obj->has_aliasing_ppgtt_mapping) {
2445                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
2446                 obj->has_aliasing_ppgtt_mapping = 0;
2447         }
2448         i915_gem_gtt_finish_object(obj);
2449
2450         list_del(&obj->mm_list);
2451         list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
2452         /* Avoid an unnecessary call to unbind on rebind. */
2453         obj->map_and_fenceable = true;
2454
2455         drm_mm_put_block(obj->gtt_space);
2456         obj->gtt_space = NULL;
2457         obj->gtt_offset = 0;
2458
2459         return 0;
2460 }
2461
2462 static int i915_ring_idle(struct intel_ring_buffer *ring)
2463 {
2464         if (list_empty(&ring->active_list))
2465                 return 0;
2466
2467         return i915_wait_seqno(ring, i915_gem_next_request_seqno(ring));
2468 }
2469
2470 int i915_gpu_idle(struct drm_device *dev)
2471 {
2472         drm_i915_private_t *dev_priv = dev->dev_private;
2473         struct intel_ring_buffer *ring;
2474         int ret, i;
2475
2476         /* Flush everything onto the inactive list. */
2477         for_each_ring(ring, dev_priv, i) {
2478                 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2479                 if (ret)
2480                         return ret;
2481
2482                 ret = i915_ring_idle(ring);
2483                 if (ret)
2484                         return ret;
2485         }
2486
2487         return 0;
2488 }
2489
2490 static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
2491                                         struct drm_i915_gem_object *obj)
2492 {
2493         drm_i915_private_t *dev_priv = dev->dev_private;
2494         uint64_t val;
2495
2496         if (obj) {
2497                 u32 size = obj->gtt_space->size;
2498
2499                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2500                                  0xfffff000) << 32;
2501                 val |= obj->gtt_offset & 0xfffff000;
2502                 val |= (uint64_t)((obj->stride / 128) - 1) <<
2503                         SANDYBRIDGE_FENCE_PITCH_SHIFT;
2504
2505                 if (obj->tiling_mode == I915_TILING_Y)
2506                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2507                 val |= I965_FENCE_REG_VALID;
2508         } else
2509                 val = 0;
2510
2511         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
2512         POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
2513 }
2514
2515 static void i965_write_fence_reg(struct drm_device *dev, int reg,
2516                                  struct drm_i915_gem_object *obj)
2517 {
2518         drm_i915_private_t *dev_priv = dev->dev_private;
2519         uint64_t val;
2520
2521         if (obj) {
2522                 u32 size = obj->gtt_space->size;
2523
2524                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2525                                  0xfffff000) << 32;
2526                 val |= obj->gtt_offset & 0xfffff000;
2527                 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2528                 if (obj->tiling_mode == I915_TILING_Y)
2529                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2530                 val |= I965_FENCE_REG_VALID;
2531         } else
2532                 val = 0;
2533
2534         I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
2535         POSTING_READ(FENCE_REG_965_0 + reg * 8);
2536 }
2537
2538 static void i915_write_fence_reg(struct drm_device *dev, int reg,
2539                                  struct drm_i915_gem_object *obj)
2540 {
2541         drm_i915_private_t *dev_priv = dev->dev_private;
2542         u32 val;
2543
2544         if (obj) {
2545                 u32 size = obj->gtt_space->size;
2546                 int pitch_val;
2547                 int tile_width;
2548
2549                 WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2550                      (size & -size) != size ||
2551                      (obj->gtt_offset & (size - 1)),
2552                      "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2553                      obj->gtt_offset, obj->map_and_fenceable, size);
2554
2555                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2556                         tile_width = 128;
2557                 else
2558                         tile_width = 512;
2559
2560                 /* Note: pitch better be a power of two tile widths */
2561                 pitch_val = obj->stride / tile_width;
2562                 pitch_val = ffs(pitch_val) - 1;
2563
2564                 val = obj->gtt_offset;
2565                 if (obj->tiling_mode == I915_TILING_Y)
2566                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2567                 val |= I915_FENCE_SIZE_BITS(size);
2568                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2569                 val |= I830_FENCE_REG_VALID;
2570         } else
2571                 val = 0;
2572
2573         if (reg < 8)
2574                 reg = FENCE_REG_830_0 + reg * 4;
2575         else
2576                 reg = FENCE_REG_945_8 + (reg - 8) * 4;
2577
2578         I915_WRITE(reg, val);
2579         POSTING_READ(reg);
2580 }
2581
2582 static void i830_write_fence_reg(struct drm_device *dev, int reg,
2583                                 struct drm_i915_gem_object *obj)
2584 {
2585         drm_i915_private_t *dev_priv = dev->dev_private;
2586         uint32_t val;
2587
2588         if (obj) {
2589                 u32 size = obj->gtt_space->size;
2590                 uint32_t pitch_val;
2591
2592                 WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2593                      (size & -size) != size ||
2594                      (obj->gtt_offset & (size - 1)),
2595                      "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2596                      obj->gtt_offset, size);
2597
2598                 pitch_val = obj->stride / 128;
2599                 pitch_val = ffs(pitch_val) - 1;
2600
2601                 val = obj->gtt_offset;
2602                 if (obj->tiling_mode == I915_TILING_Y)
2603                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2604                 val |= I830_FENCE_SIZE_BITS(size);
2605                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2606                 val |= I830_FENCE_REG_VALID;
2607         } else
2608                 val = 0;
2609
2610         I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
2611         POSTING_READ(FENCE_REG_830_0 + reg * 4);
2612 }
2613
2614 static void i915_gem_write_fence(struct drm_device *dev, int reg,
2615                                  struct drm_i915_gem_object *obj)
2616 {
2617         switch (INTEL_INFO(dev)->gen) {
2618         case 7:
2619         case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
2620         case 5:
2621         case 4: i965_write_fence_reg(dev, reg, obj); break;
2622         case 3: i915_write_fence_reg(dev, reg, obj); break;
2623         case 2: i830_write_fence_reg(dev, reg, obj); break;
2624         default: break;
2625         }
2626 }
2627
2628 static inline int fence_number(struct drm_i915_private *dev_priv,
2629                                struct drm_i915_fence_reg *fence)
2630 {
2631         return fence - dev_priv->fence_regs;
2632 }
2633
2634 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
2635                                          struct drm_i915_fence_reg *fence,
2636                                          bool enable)
2637 {
2638         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2639         int reg = fence_number(dev_priv, fence);
2640
2641         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
2642
2643         if (enable) {
2644                 obj->fence_reg = reg;
2645                 fence->obj = obj;
2646                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
2647         } else {
2648                 obj->fence_reg = I915_FENCE_REG_NONE;
2649                 fence->obj = NULL;
2650                 list_del_init(&fence->lru_list);
2651         }
2652 }
2653
2654 static int
2655 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
2656 {
2657         if (obj->last_fenced_seqno) {
2658                 int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2659                 if (ret)
2660                         return ret;
2661
2662                 obj->last_fenced_seqno = 0;
2663         }
2664
2665         /* Ensure that all CPU reads are completed before installing a fence
2666          * and all writes before removing the fence.
2667          */
2668         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
2669                 mb();
2670
2671         obj->fenced_gpu_access = false;
2672         return 0;
2673 }
2674
2675 int
2676 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2677 {
2678         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2679         int ret;
2680
2681         ret = i915_gem_object_flush_fence(obj);
2682         if (ret)
2683                 return ret;
2684
2685         if (obj->fence_reg == I915_FENCE_REG_NONE)
2686                 return 0;
2687
2688         i915_gem_object_update_fence(obj,
2689                                      &dev_priv->fence_regs[obj->fence_reg],
2690                                      false);
2691         i915_gem_object_fence_lost(obj);
2692
2693         return 0;
2694 }
2695
2696 static struct drm_i915_fence_reg *
2697 i915_find_fence_reg(struct drm_device *dev)
2698 {
2699         struct drm_i915_private *dev_priv = dev->dev_private;
2700         struct drm_i915_fence_reg *reg, *avail;
2701         int i;
2702
2703         /* First try to find a free reg */
2704         avail = NULL;
2705         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2706                 reg = &dev_priv->fence_regs[i];
2707                 if (!reg->obj)
2708                         return reg;
2709
2710                 if (!reg->pin_count)
2711                         avail = reg;
2712         }
2713
2714         if (avail == NULL)
2715                 return NULL;
2716
2717         /* None available, try to steal one or wait for a user to finish */
2718         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2719                 if (reg->pin_count)
2720                         continue;
2721
2722                 return reg;
2723         }
2724
2725         return NULL;
2726 }
2727
2728 /**
2729  * i915_gem_object_get_fence - set up fencing for an object
2730  * @obj: object to map through a fence reg
2731  *
2732  * When mapping objects through the GTT, userspace wants to be able to write
2733  * to them without having to worry about swizzling if the object is tiled.
2734  * This function walks the fence regs looking for a free one for @obj,
2735  * stealing one if it can't find any.
2736  *
2737  * It then sets up the reg based on the object's properties: address, pitch
2738  * and tiling format.
2739  *
2740  * For an untiled surface, this removes any existing fence.
2741  */
2742 int
2743 i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
2744 {
2745         struct drm_device *dev = obj->base.dev;
2746         struct drm_i915_private *dev_priv = dev->dev_private;
2747         bool enable = obj->tiling_mode != I915_TILING_NONE;
2748         struct drm_i915_fence_reg *reg;
2749         int ret;
2750
2751         /* Have we updated the tiling parameters upon the object and so
2752          * will need to serialise the write to the associated fence register?
2753          */
2754         if (obj->fence_dirty) {
2755                 ret = i915_gem_object_flush_fence(obj);
2756                 if (ret)
2757                         return ret;
2758         }
2759
2760         /* Just update our place in the LRU if our fence is getting reused. */
2761         if (obj->fence_reg != I915_FENCE_REG_NONE) {
2762                 reg = &dev_priv->fence_regs[obj->fence_reg];
2763                 if (!obj->fence_dirty) {
2764                         list_move_tail(&reg->lru_list,
2765                                        &dev_priv->mm.fence_list);
2766                         return 0;
2767                 }
2768         } else if (enable) {
2769                 reg = i915_find_fence_reg(dev);
2770                 if (reg == NULL)
2771                         return -EDEADLK;
2772
2773                 if (reg->obj) {
2774                         struct drm_i915_gem_object *old = reg->obj;
2775
2776                         ret = i915_gem_object_flush_fence(old);
2777                         if (ret)
2778                                 return ret;
2779
2780                         i915_gem_object_fence_lost(old);
2781                 }
2782         } else
2783                 return 0;
2784
2785         i915_gem_object_update_fence(obj, reg, enable);
2786         obj->fence_dirty = false;
2787
2788         return 0;
2789 }
2790
2791 static bool i915_gem_valid_gtt_space(struct drm_device *dev,
2792                                      struct drm_mm_node *gtt_space,
2793                                      unsigned long cache_level)
2794 {
2795         struct drm_mm_node *other;
2796
2797         /* On non-LLC machines we have to be careful when putting differing
2798          * types of snoopable memory together to avoid the prefetcher
2799          * crossing memory domains and dieing.
2800          */
2801         if (HAS_LLC(dev))
2802                 return true;
2803
2804         if (gtt_space == NULL)
2805                 return true;
2806
2807         if (list_empty(&gtt_space->node_list))
2808                 return true;
2809
2810         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
2811         if (other->allocated && !other->hole_follows && other->color != cache_level)
2812                 return false;
2813
2814         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
2815         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
2816                 return false;
2817
2818         return true;
2819 }
2820
2821 static void i915_gem_verify_gtt(struct drm_device *dev)
2822 {
2823 #if WATCH_GTT
2824         struct drm_i915_private *dev_priv = dev->dev_private;
2825         struct drm_i915_gem_object *obj;
2826         int err = 0;
2827
2828         list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
2829                 if (obj->gtt_space == NULL) {
2830                         printk(KERN_ERR "object found on GTT list with no space reserved\n");
2831                         err++;
2832                         continue;
2833                 }
2834
2835                 if (obj->cache_level != obj->gtt_space->color) {
2836                         printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
2837                                obj->gtt_space->start,
2838                                obj->gtt_space->start + obj->gtt_space->size,
2839                                obj->cache_level,
2840                                obj->gtt_space->color);
2841                         err++;
2842                         continue;
2843                 }
2844
2845                 if (!i915_gem_valid_gtt_space(dev,
2846                                               obj->gtt_space,
2847                                               obj->cache_level)) {
2848                         printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
2849                                obj->gtt_space->start,
2850                                obj->gtt_space->start + obj->gtt_space->size,
2851                                obj->cache_level);
2852                         err++;
2853                         continue;
2854                 }
2855         }
2856
2857         WARN_ON(err);
2858 #endif
2859 }
2860
2861 /**
2862  * Finds free space in the GTT aperture and binds the object there.
2863  */
2864 static int
2865 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
2866                             unsigned alignment,
2867                             bool map_and_fenceable,
2868                             bool nonblocking)
2869 {
2870         struct drm_device *dev = obj->base.dev;
2871         drm_i915_private_t *dev_priv = dev->dev_private;
2872         struct drm_mm_node *free_space;
2873         u32 size, fence_size, fence_alignment, unfenced_alignment;
2874         bool mappable, fenceable;
2875         int ret;
2876
2877         if (obj->madv != I915_MADV_WILLNEED) {
2878                 DRM_ERROR("Attempting to bind a purgeable object\n");
2879                 return -EINVAL;
2880         }
2881
2882         fence_size = i915_gem_get_gtt_size(dev,
2883                                            obj->base.size,
2884                                            obj->tiling_mode);
2885         fence_alignment = i915_gem_get_gtt_alignment(dev,
2886                                                      obj->base.size,
2887                                                      obj->tiling_mode);
2888         unfenced_alignment =
2889                 i915_gem_get_unfenced_gtt_alignment(dev,
2890                                                     obj->base.size,
2891                                                     obj->tiling_mode);
2892
2893         if (alignment == 0)
2894                 alignment = map_and_fenceable ? fence_alignment :
2895                                                 unfenced_alignment;
2896         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
2897                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2898                 return -EINVAL;
2899         }
2900
2901         size = map_and_fenceable ? fence_size : obj->base.size;
2902
2903         /* If the object is bigger than the entire aperture, reject it early
2904          * before evicting everything in a vain attempt to find space.
2905          */
2906         if (obj->base.size >
2907             (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2908                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2909                 return -E2BIG;
2910         }
2911
2912         ret = i915_gem_object_get_pages(obj);
2913         if (ret)
2914                 return ret;
2915
2916  search_free:
2917         if (map_and_fenceable)
2918                 free_space =
2919                         drm_mm_search_free_in_range_color(&dev_priv->mm.gtt_space,
2920                                                           size, alignment, obj->cache_level,
2921                                                           0, dev_priv->mm.gtt_mappable_end,
2922                                                           false);
2923         else
2924                 free_space = drm_mm_search_free_color(&dev_priv->mm.gtt_space,
2925                                                       size, alignment, obj->cache_level,
2926                                                       false);
2927
2928         if (free_space != NULL) {
2929                 if (map_and_fenceable)
2930                         obj->gtt_space =
2931                                 drm_mm_get_block_range_generic(free_space,
2932                                                                size, alignment, obj->cache_level,
2933                                                                0, dev_priv->mm.gtt_mappable_end,
2934                                                                false);
2935                 else
2936                         obj->gtt_space =
2937                                 drm_mm_get_block_generic(free_space,
2938                                                          size, alignment, obj->cache_level,
2939                                                          false);
2940         }
2941         if (obj->gtt_space == NULL) {
2942                 ret = i915_gem_evict_something(dev, size, alignment,
2943                                                obj->cache_level,
2944                                                map_and_fenceable,
2945                                                nonblocking);
2946                 if (ret)
2947                         return ret;
2948
2949                 goto search_free;
2950         }
2951         if (WARN_ON(!i915_gem_valid_gtt_space(dev,
2952                                               obj->gtt_space,
2953                                               obj->cache_level))) {
2954                 drm_mm_put_block(obj->gtt_space);
2955                 obj->gtt_space = NULL;
2956                 return -EINVAL;
2957         }
2958
2959
2960         ret = i915_gem_gtt_prepare_object(obj);
2961         if (ret) {
2962                 drm_mm_put_block(obj->gtt_space);
2963                 obj->gtt_space = NULL;
2964                 return ret;
2965         }
2966
2967         if (!dev_priv->mm.aliasing_ppgtt)
2968                 i915_gem_gtt_bind_object(obj, obj->cache_level);
2969
2970         list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
2971         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2972
2973         obj->gtt_offset = obj->gtt_space->start;
2974
2975         fenceable =
2976                 obj->gtt_space->size == fence_size &&
2977                 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
2978
2979         mappable =
2980                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
2981
2982         obj->map_and_fenceable = mappable && fenceable;
2983
2984         trace_i915_gem_object_bind(obj, map_and_fenceable);
2985         i915_gem_verify_gtt(dev);
2986         return 0;
2987 }
2988
2989 void
2990 i915_gem_clflush_object(struct drm_i915_gem_object *obj)
2991 {
2992         /* If we don't have a page list set up, then we're not pinned
2993          * to GPU, and we can ignore the cache flush because it'll happen
2994          * again at bind time.
2995          */
2996         if (obj->pages == NULL)
2997                 return;
2998
2999         /* If the GPU is snooping the contents of the CPU cache,
3000          * we do not need to manually clear the CPU cache lines.  However,
3001          * the caches are only snooped when the render cache is
3002          * flushed/invalidated.  As we always have to emit invalidations
3003          * and flushes when moving into and out of the RENDER domain, correct
3004          * snooping behaviour occurs naturally as the result of our domain
3005          * tracking.
3006          */
3007         if (obj->cache_level != I915_CACHE_NONE)
3008                 return;
3009
3010         trace_i915_gem_object_clflush(obj);
3011
3012         drm_clflush_sg(obj->pages);
3013 }
3014
3015 /** Flushes the GTT write domain for the object if it's dirty. */
3016 static void
3017 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3018 {
3019         uint32_t old_write_domain;
3020
3021         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3022                 return;
3023
3024         /* No actual flushing is required for the GTT write domain.  Writes
3025          * to it immediately go to main memory as far as we know, so there's
3026          * no chipset flush.  It also doesn't land in render cache.
3027          *
3028          * However, we do have to enforce the order so that all writes through
3029          * the GTT land before any writes to the device, such as updates to
3030          * the GATT itself.
3031          */
3032         wmb();
3033
3034         old_write_domain = obj->base.write_domain;
3035         obj->base.write_domain = 0;
3036
3037         trace_i915_gem_object_change_domain(obj,
3038                                             obj->base.read_domains,
3039                                             old_write_domain);
3040 }
3041
3042 /** Flushes the CPU write domain for the object if it's dirty. */
3043 static void
3044 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3045 {
3046         uint32_t old_write_domain;
3047
3048         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3049                 return;
3050
3051         i915_gem_clflush_object(obj);
3052         intel_gtt_chipset_flush();
3053         old_write_domain = obj->base.write_domain;
3054         obj->base.write_domain = 0;
3055
3056         trace_i915_gem_object_change_domain(obj,
3057                                             obj->base.read_domains,
3058                                             old_write_domain);
3059 }
3060
3061 /**
3062  * Moves a single object to the GTT read, and possibly write domain.
3063  *
3064  * This function returns when the move is complete, including waiting on
3065  * flushes to occur.
3066  */
3067 int
3068 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3069 {
3070         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
3071         uint32_t old_write_domain, old_read_domains;
3072         int ret;
3073
3074         /* Not valid to be called on unbound objects. */
3075         if (obj->gtt_space == NULL)
3076                 return -EINVAL;
3077
3078         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3079                 return 0;
3080
3081         ret = i915_gem_object_wait_rendering(obj, !write);
3082         if (ret)
3083                 return ret;
3084
3085         i915_gem_object_flush_cpu_write_domain(obj);
3086
3087         old_write_domain = obj->base.write_domain;
3088         old_read_domains = obj->base.read_domains;
3089
3090         /* It should now be out of any other write domains, and we can update
3091          * the domain values for our changes.
3092          */
3093         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3094         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3095         if (write) {
3096                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3097                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3098                 obj->dirty = 1;
3099         }
3100
3101         trace_i915_gem_object_change_domain(obj,
3102                                             old_read_domains,
3103                                             old_write_domain);
3104
3105         /* And bump the LRU for this access */
3106         if (i915_gem_object_is_inactive(obj))
3107                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
3108
3109         return 0;
3110 }
3111
3112 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3113                                     enum i915_cache_level cache_level)
3114 {
3115         struct drm_device *dev = obj->base.dev;
3116         drm_i915_private_t *dev_priv = dev->dev_private;
3117         int ret;
3118
3119         if (obj->cache_level == cache_level)
3120                 return 0;
3121
3122         if (obj->pin_count) {
3123                 DRM_DEBUG("can not change the cache level of pinned objects\n");
3124                 return -EBUSY;
3125         }
3126
3127         if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
3128                 ret = i915_gem_object_unbind(obj);
3129                 if (ret)
3130                         return ret;
3131         }
3132
3133         if (obj->gtt_space) {
3134                 ret = i915_gem_object_finish_gpu(obj);
3135                 if (ret)
3136                         return ret;
3137
3138                 i915_gem_object_finish_gtt(obj);
3139
3140                 /* Before SandyBridge, you could not use tiling or fence
3141                  * registers with snooped memory, so relinquish any fences
3142                  * currently pointing to our region in the aperture.
3143                  */
3144                 if (INTEL_INFO(dev)->gen < 6) {
3145                         ret = i915_gem_object_put_fence(obj);
3146                         if (ret)
3147                                 return ret;
3148                 }
3149
3150                 if (obj->has_global_gtt_mapping)
3151                         i915_gem_gtt_bind_object(obj, cache_level);
3152                 if (obj->has_aliasing_ppgtt_mapping)
3153                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
3154                                                obj, cache_level);
3155
3156                 obj->gtt_space->color = cache_level;
3157         }
3158
3159         if (cache_level == I915_CACHE_NONE) {
3160                 u32 old_read_domains, old_write_domain;
3161
3162                 /* If we're coming from LLC cached, then we haven't
3163                  * actually been tracking whether the data is in the
3164                  * CPU cache or not, since we only allow one bit set
3165                  * in obj->write_domain and have been skipping the clflushes.
3166                  * Just set it to the CPU cache for now.
3167                  */
3168                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
3169                 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
3170
3171                 old_read_domains = obj->base.read_domains;
3172                 old_write_domain = obj->base.write_domain;
3173
3174                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3175                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3176
3177                 trace_i915_gem_object_change_domain(obj,
3178                                                     old_read_domains,
3179                                                     old_write_domain);
3180         }
3181
3182         obj->cache_level = cache_level;
3183         i915_gem_verify_gtt(dev);
3184         return 0;
3185 }
3186
3187 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3188                                struct drm_file *file)
3189 {
3190         struct drm_i915_gem_caching *args = data;
3191         struct drm_i915_gem_object *obj;
3192         int ret;
3193
3194         ret = i915_mutex_lock_interruptible(dev);
3195         if (ret)
3196                 return ret;
3197
3198         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3199         if (&obj->base == NULL) {
3200                 ret = -ENOENT;
3201                 goto unlock;
3202         }
3203
3204         args->caching = obj->cache_level != I915_CACHE_NONE;
3205
3206         drm_gem_object_unreference(&obj->base);
3207 unlock:
3208         mutex_unlock(&dev->struct_mutex);
3209         return ret;
3210 }
3211
3212 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3213                                struct drm_file *file)
3214 {
3215         struct drm_i915_gem_caching *args = data;
3216         struct drm_i915_gem_object *obj;
3217         enum i915_cache_level level;
3218         int ret;
3219
3220         ret = i915_mutex_lock_interruptible(dev);
3221         if (ret)
3222                 return ret;
3223
3224         switch (args->caching) {
3225         case I915_CACHING_NONE:
3226                 level = I915_CACHE_NONE;
3227                 break;
3228         case I915_CACHING_CACHED:
3229                 level = I915_CACHE_LLC;
3230                 break;
3231         default:
3232                 return -EINVAL;
3233         }
3234
3235         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3236         if (&obj->base == NULL) {
3237                 ret = -ENOENT;
3238                 goto unlock;
3239         }
3240
3241         ret = i915_gem_object_set_cache_level(obj, level);
3242
3243         drm_gem_object_unreference(&obj->base);
3244 unlock:
3245         mutex_unlock(&dev->struct_mutex);
3246         return ret;
3247 }
3248
3249 /*
3250  * Prepare buffer for display plane (scanout, cursors, etc).
3251  * Can be called from an uninterruptible phase (modesetting) and allows
3252  * any flushes to be pipelined (for pageflips).
3253  */
3254 int
3255 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3256                                      u32 alignment,
3257                                      struct intel_ring_buffer *pipelined)
3258 {
3259         u32 old_read_domains, old_write_domain;
3260         int ret;
3261
3262         if (pipelined != obj->ring) {
3263                 ret = i915_gem_object_sync(obj, pipelined);
3264                 if (ret)
3265                         return ret;
3266         }
3267
3268         /* The display engine is not coherent with the LLC cache on gen6.  As
3269          * a result, we make sure that the pinning that is about to occur is
3270          * done with uncached PTEs. This is lowest common denominator for all
3271          * chipsets.
3272          *
3273          * However for gen6+, we could do better by using the GFDT bit instead
3274          * of uncaching, which would allow us to flush all the LLC-cached data
3275          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3276          */
3277         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
3278         if (ret)
3279                 return ret;
3280
3281         /* As the user may map the buffer once pinned in the display plane
3282          * (e.g. libkms for the bootup splash), we have to ensure that we
3283          * always use map_and_fenceable for all scanout buffers.
3284          */
3285         ret = i915_gem_object_pin(obj, alignment, true, false);
3286         if (ret)
3287                 return ret;
3288
3289         i915_gem_object_flush_cpu_write_domain(obj);
3290
3291         old_write_domain = obj->base.write_domain;
3292         old_read_domains = obj->base.read_domains;
3293
3294         /* It should now be out of any other write domains, and we can update
3295          * the domain values for our changes.
3296          */
3297         obj->base.write_domain = 0;
3298         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3299
3300         trace_i915_gem_object_change_domain(obj,
3301                                             old_read_domains,
3302                                             old_write_domain);
3303
3304         return 0;
3305 }
3306
3307 int
3308 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
3309 {
3310         int ret;
3311
3312         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
3313                 return 0;
3314
3315         ret = i915_gem_object_wait_rendering(obj, false);
3316         if (ret)
3317                 return ret;
3318
3319         /* Ensure that we invalidate the GPU's caches and TLBs. */
3320         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3321         return 0;
3322 }
3323
3324 /**
3325  * Moves a single object to the CPU read, and possibly write domain.
3326  *
3327  * This function returns when the move is complete, including waiting on
3328  * flushes to occur.
3329  */
3330 int
3331 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3332 {
3333         uint32_t old_write_domain, old_read_domains;
3334         int ret;
3335
3336         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3337                 return 0;
3338
3339         ret = i915_gem_object_wait_rendering(obj, !write);
3340         if (ret)
3341                 return ret;
3342
3343         i915_gem_object_flush_gtt_write_domain(obj);
3344
3345         old_write_domain = obj->base.write_domain;
3346         old_read_domains = obj->base.read_domains;
3347
3348         /* Flush the CPU cache if it's still invalid. */
3349         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3350                 i915_gem_clflush_object(obj);
3351
3352                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3353         }
3354
3355         /* It should now be out of any other write domains, and we can update
3356          * the domain values for our changes.
3357          */
3358         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3359
3360         /* If we're writing through the CPU, then the GPU read domains will
3361          * need to be invalidated at next use.
3362          */
3363         if (write) {
3364                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3365                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3366         }
3367
3368         trace_i915_gem_object_change_domain(obj,
3369                                             old_read_domains,
3370                                             old_write_domain);
3371
3372         return 0;
3373 }
3374
3375 /* Throttle our rendering by waiting until the ring has completed our requests
3376  * emitted over 20 msec ago.
3377  *
3378  * Note that if we were to use the current jiffies each time around the loop,
3379  * we wouldn't escape the function with any frames outstanding if the time to
3380  * render a frame was over 20ms.
3381  *
3382  * This should get us reasonable parallelism between CPU and GPU but also
3383  * relatively low latency when blocking on a particular request to finish.
3384  */
3385 static int
3386 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3387 {
3388         struct drm_i915_private *dev_priv = dev->dev_private;
3389         struct drm_i915_file_private *file_priv = file->driver_priv;
3390         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3391         struct drm_i915_gem_request *request;
3392         struct intel_ring_buffer *ring = NULL;
3393         u32 seqno = 0;
3394         int ret;
3395
3396         if (atomic_read(&dev_priv->mm.wedged))
3397                 return -EIO;
3398
3399         spin_lock(&file_priv->mm.lock);
3400         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3401                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3402                         break;
3403
3404                 ring = request->ring;
3405                 seqno = request->seqno;
3406         }
3407         spin_unlock(&file_priv->mm.lock);
3408
3409         if (seqno == 0)
3410                 return 0;
3411
3412         ret = __wait_seqno(ring, seqno, true, NULL);
3413         if (ret == 0)
3414                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3415
3416         return ret;
3417 }
3418
3419 int
3420 i915_gem_object_pin(struct drm_i915_gem_object *obj,
3421                     uint32_t alignment,
3422                     bool map_and_fenceable,
3423                     bool nonblocking)
3424 {
3425         int ret;
3426
3427         if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
3428                 return -EBUSY;
3429
3430         if (obj->gtt_space != NULL) {
3431                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
3432                     (map_and_fenceable && !obj->map_and_fenceable)) {
3433                         WARN(obj->pin_count,
3434                              "bo is already pinned with incorrect alignment:"
3435                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
3436                              " obj->map_and_fenceable=%d\n",
3437                              obj->gtt_offset, alignment,
3438                              map_and_fenceable,
3439                              obj->map_and_fenceable);
3440                         ret = i915_gem_object_unbind(obj);
3441                         if (ret)
3442                                 return ret;
3443                 }
3444         }
3445
3446         if (obj->gtt_space == NULL) {
3447                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
3448                                                   map_and_fenceable,
3449                                                   nonblocking);
3450                 if (ret)
3451                         return ret;
3452         }
3453
3454         if (!obj->has_global_gtt_mapping && map_and_fenceable)
3455                 i915_gem_gtt_bind_object(obj, obj->cache_level);
3456
3457         obj->pin_count++;
3458         obj->pin_mappable |= map_and_fenceable;
3459
3460         return 0;
3461 }
3462
3463 void
3464 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
3465 {
3466         BUG_ON(obj->pin_count == 0);
3467         BUG_ON(obj->gtt_space == NULL);
3468
3469         if (--obj->pin_count == 0)
3470                 obj->pin_mappable = false;
3471 }
3472
3473 int
3474 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3475                    struct drm_file *file)
3476 {
3477         struct drm_i915_gem_pin *args = data;
3478         struct drm_i915_gem_object *obj;
3479         int ret;
3480
3481         ret = i915_mutex_lock_interruptible(dev);
3482         if (ret)
3483                 return ret;
3484
3485         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3486         if (&obj->base == NULL) {
3487                 ret = -ENOENT;
3488                 goto unlock;
3489         }
3490
3491         if (obj->madv != I915_MADV_WILLNEED) {
3492                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
3493                 ret = -EINVAL;
3494                 goto out;
3495         }
3496
3497         if (obj->pin_filp != NULL && obj->pin_filp != file) {
3498                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3499                           args->handle);
3500                 ret = -EINVAL;
3501                 goto out;
3502         }
3503
3504         obj->user_pin_count++;
3505         obj->pin_filp = file;
3506         if (obj->user_pin_count == 1) {
3507                 ret = i915_gem_object_pin(obj, args->alignment, true, false);
3508                 if (ret)
3509                         goto out;
3510         }
3511
3512         /* XXX - flush the CPU caches for pinned objects
3513          * as the X server doesn't manage domains yet
3514          */
3515         i915_gem_object_flush_cpu_write_domain(obj);
3516         args->offset = obj->gtt_offset;
3517 out:
3518         drm_gem_object_unreference(&obj->base);
3519 unlock:
3520         mutex_unlock(&dev->struct_mutex);
3521         return ret;
3522 }
3523
3524 int
3525 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3526                      struct drm_file *file)
3527 {
3528         struct drm_i915_gem_pin *args = data;
3529         struct drm_i915_gem_object *obj;
3530         int ret;
3531
3532         ret = i915_mutex_lock_interruptible(dev);
3533         if (ret)
3534                 return ret;
3535
3536         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3537         if (&obj->base == NULL) {
3538                 ret = -ENOENT;
3539                 goto unlock;
3540         }
3541
3542         if (obj->pin_filp != file) {
3543                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3544                           args->handle);
3545                 ret = -EINVAL;
3546                 goto out;
3547         }
3548         obj->user_pin_count--;
3549         if (obj->user_pin_count == 0) {
3550                 obj->pin_filp = NULL;
3551                 i915_gem_object_unpin(obj);
3552         }
3553
3554 out:
3555         drm_gem_object_unreference(&obj->base);
3556 unlock:
3557         mutex_unlock(&dev->struct_mutex);
3558         return ret;
3559 }
3560
3561 int
3562 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3563                     struct drm_file *file)
3564 {
3565         struct drm_i915_gem_busy *args = data;
3566         struct drm_i915_gem_object *obj;
3567         int ret;
3568
3569         ret = i915_mutex_lock_interruptible(dev);
3570         if (ret)
3571                 return ret;
3572
3573         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3574         if (&obj->base == NULL) {
3575                 ret = -ENOENT;
3576                 goto unlock;
3577         }
3578
3579         /* Count all active objects as busy, even if they are currently not used
3580          * by the gpu. Users of this interface expect objects to eventually
3581          * become non-busy without any further actions, therefore emit any
3582          * necessary flushes here.
3583          */
3584         ret = i915_gem_object_flush_active(obj);
3585
3586         args->busy = obj->active;
3587         if (obj->ring) {
3588                 BUILD_BUG_ON(I915_NUM_RINGS > 16);
3589                 args->busy |= intel_ring_flag(obj->ring) << 16;
3590         }
3591
3592         drm_gem_object_unreference(&obj->base);
3593 unlock:
3594         mutex_unlock(&dev->struct_mutex);
3595         return ret;
3596 }
3597
3598 int
3599 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3600                         struct drm_file *file_priv)
3601 {
3602         return i915_gem_ring_throttle(dev, file_priv);
3603 }
3604
3605 int
3606 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3607                        struct drm_file *file_priv)
3608 {
3609         struct drm_i915_gem_madvise *args = data;
3610         struct drm_i915_gem_object *obj;
3611         int ret;
3612
3613         switch (args->madv) {
3614         case I915_MADV_DONTNEED:
3615         case I915_MADV_WILLNEED:
3616             break;
3617         default:
3618             return -EINVAL;
3619         }
3620
3621         ret = i915_mutex_lock_interruptible(dev);
3622         if (ret)
3623                 return ret;
3624
3625         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
3626         if (&obj->base == NULL) {
3627                 ret = -ENOENT;
3628                 goto unlock;
3629         }
3630
3631         if (obj->pin_count) {
3632                 ret = -EINVAL;
3633                 goto out;
3634         }
3635
3636         if (obj->madv != __I915_MADV_PURGED)
3637                 obj->madv = args->madv;
3638
3639         /* if the object is no longer attached, discard its backing storage */
3640         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
3641                 i915_gem_object_truncate(obj);
3642
3643         args->retained = obj->madv != __I915_MADV_PURGED;
3644
3645 out:
3646         drm_gem_object_unreference(&obj->base);
3647 unlock:
3648         mutex_unlock(&dev->struct_mutex);
3649         return ret;
3650 }
3651
3652 void i915_gem_object_init(struct drm_i915_gem_object *obj,
3653                           const struct drm_i915_gem_object_ops *ops)
3654 {
3655         INIT_LIST_HEAD(&obj->mm_list);
3656         INIT_LIST_HEAD(&obj->gtt_list);
3657         INIT_LIST_HEAD(&obj->ring_list);
3658         INIT_LIST_HEAD(&obj->exec_list);
3659
3660         obj->ops = ops;
3661
3662         obj->fence_reg = I915_FENCE_REG_NONE;
3663         obj->madv = I915_MADV_WILLNEED;
3664         /* Avoid an unnecessary call to unbind on the first bind. */
3665         obj->map_and_fenceable = true;
3666
3667         i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
3668 }
3669
3670 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
3671         .get_pages = i915_gem_object_get_pages_gtt,
3672         .put_pages = i915_gem_object_put_pages_gtt,
3673 };
3674
3675 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
3676                                                   size_t size)
3677 {
3678         struct drm_i915_gem_object *obj;
3679         struct address_space *mapping;
3680         u32 mask;
3681
3682         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
3683         if (obj == NULL)
3684                 return NULL;
3685
3686         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
3687                 kfree(obj);
3688                 return NULL;
3689         }
3690
3691         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
3692         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
3693                 /* 965gm cannot relocate objects above 4GiB. */
3694                 mask &= ~__GFP_HIGHMEM;
3695                 mask |= __GFP_DMA32;
3696         }
3697
3698         mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3699         mapping_set_gfp_mask(mapping, mask);
3700
3701         i915_gem_object_init(obj, &i915_gem_object_ops);
3702
3703         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3704         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3705
3706         if (HAS_LLC(dev)) {
3707                 /* On some devices, we can have the GPU use the LLC (the CPU
3708                  * cache) for about a 10% performance improvement
3709                  * compared to uncached.  Graphics requests other than
3710                  * display scanout are coherent with the CPU in
3711                  * accessing this cache.  This means in this mode we
3712                  * don't need to clflush on the CPU side, and on the
3713                  * GPU side we only need to flush internal caches to
3714                  * get data visible to the CPU.
3715                  *
3716                  * However, we maintain the display planes as UC, and so
3717                  * need to rebind when first used as such.
3718                  */
3719                 obj->cache_level = I915_CACHE_LLC;
3720         } else
3721                 obj->cache_level = I915_CACHE_NONE;
3722
3723         return obj;
3724 }
3725
3726 int i915_gem_init_object(struct drm_gem_object *obj)
3727 {
3728         BUG();
3729
3730         return 0;
3731 }
3732
3733 void i915_gem_free_object(struct drm_gem_object *gem_obj)
3734 {
3735         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
3736         struct drm_device *dev = obj->base.dev;
3737         drm_i915_private_t *dev_priv = dev->dev_private;
3738
3739         trace_i915_gem_object_destroy(obj);
3740
3741         if (obj->phys_obj)
3742                 i915_gem_detach_phys_object(dev, obj);
3743
3744         obj->pin_count = 0;
3745         if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
3746                 bool was_interruptible;
3747
3748                 was_interruptible = dev_priv->mm.interruptible;
3749                 dev_priv->mm.interruptible = false;
3750
3751                 WARN_ON(i915_gem_object_unbind(obj));
3752
3753                 dev_priv->mm.interruptible = was_interruptible;
3754         }
3755
3756         obj->pages_pin_count = 0;
3757         i915_gem_object_put_pages(obj);
3758         i915_gem_object_free_mmap_offset(obj);
3759
3760         BUG_ON(obj->pages);
3761
3762         if (obj->base.import_attach)
3763                 drm_prime_gem_destroy(&obj->base, NULL);
3764
3765         drm_gem_object_release(&obj->base);
3766         i915_gem_info_remove_obj(dev_priv, obj->base.size);
3767
3768         kfree(obj->bit_17);
3769         kfree(obj);
3770 }
3771
3772 int
3773 i915_gem_idle(struct drm_device *dev)
3774 {
3775         drm_i915_private_t *dev_priv = dev->dev_private;
3776         int ret;
3777
3778         mutex_lock(&dev->struct_mutex);
3779
3780         if (dev_priv->mm.suspended) {
3781                 mutex_unlock(&dev->struct_mutex);
3782                 return 0;
3783         }
3784
3785         ret = i915_gpu_idle(dev);
3786         if (ret) {
3787                 mutex_unlock(&dev->struct_mutex);
3788                 return ret;
3789         }
3790         i915_gem_retire_requests(dev);
3791
3792         /* Under UMS, be paranoid and evict. */
3793         if (!drm_core_check_feature(dev, DRIVER_MODESET))
3794                 i915_gem_evict_everything(dev);
3795
3796         i915_gem_reset_fences(dev);
3797
3798         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
3799          * We need to replace this with a semaphore, or something.
3800          * And not confound mm.suspended!
3801          */
3802         dev_priv->mm.suspended = 1;
3803         del_timer_sync(&dev_priv->hangcheck_timer);
3804
3805         i915_kernel_lost_context(dev);
3806         i915_gem_cleanup_ringbuffer(dev);
3807
3808         mutex_unlock(&dev->struct_mutex);
3809
3810         /* Cancel the retire work handler, which should be idle now. */
3811         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3812
3813         return 0;
3814 }
3815
3816 void i915_gem_l3_remap(struct drm_device *dev)
3817 {
3818         drm_i915_private_t *dev_priv = dev->dev_private;
3819         u32 misccpctl;
3820         int i;
3821
3822         if (!IS_IVYBRIDGE(dev))
3823                 return;
3824
3825         if (!dev_priv->mm.l3_remap_info)
3826                 return;
3827
3828         misccpctl = I915_READ(GEN7_MISCCPCTL);
3829         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
3830         POSTING_READ(GEN7_MISCCPCTL);
3831
3832         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
3833                 u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
3834                 if (remap && remap != dev_priv->mm.l3_remap_info[i/4])
3835                         DRM_DEBUG("0x%x was already programmed to %x\n",
3836                                   GEN7_L3LOG_BASE + i, remap);
3837                 if (remap && !dev_priv->mm.l3_remap_info[i/4])
3838                         DRM_DEBUG_DRIVER("Clearing remapped register\n");
3839                 I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->mm.l3_remap_info[i/4]);
3840         }
3841
3842         /* Make sure all the writes land before disabling dop clock gating */
3843         POSTING_READ(GEN7_L3LOG_BASE);
3844
3845         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
3846 }
3847
3848 void i915_gem_init_swizzling(struct drm_device *dev)
3849 {
3850         drm_i915_private_t *dev_priv = dev->dev_private;
3851
3852         if (INTEL_INFO(dev)->gen < 5 ||
3853             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
3854                 return;
3855
3856         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
3857                                  DISP_TILE_SURFACE_SWIZZLING);
3858
3859         if (IS_GEN5(dev))
3860                 return;
3861
3862         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
3863         if (IS_GEN6(dev))
3864                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
3865         else
3866                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
3867 }
3868
3869 void i915_gem_init_ppgtt(struct drm_device *dev)
3870 {
3871         drm_i915_private_t *dev_priv = dev->dev_private;
3872         uint32_t pd_offset;
3873         struct intel_ring_buffer *ring;
3874         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
3875         uint32_t __iomem *pd_addr;
3876         uint32_t pd_entry;
3877         int i;
3878
3879         if (!dev_priv->mm.aliasing_ppgtt)
3880                 return;
3881
3882
3883         pd_addr = dev_priv->mm.gtt->gtt + ppgtt->pd_offset/sizeof(uint32_t);
3884         for (i = 0; i < ppgtt->num_pd_entries; i++) {
3885                 dma_addr_t pt_addr;
3886
3887                 if (dev_priv->mm.gtt->needs_dmar)
3888                         pt_addr = ppgtt->pt_dma_addr[i];
3889                 else
3890                         pt_addr = page_to_phys(ppgtt->pt_pages[i]);
3891
3892                 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
3893                 pd_entry |= GEN6_PDE_VALID;
3894
3895                 writel(pd_entry, pd_addr + i);
3896         }
3897         readl(pd_addr);
3898
3899         pd_offset = ppgtt->pd_offset;
3900         pd_offset /= 64; /* in cachelines, */
3901         pd_offset <<= 16;
3902
3903         if (INTEL_INFO(dev)->gen == 6) {
3904                 uint32_t ecochk, gab_ctl, ecobits;
3905
3906                 ecobits = I915_READ(GAC_ECO_BITS); 
3907                 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
3908
3909                 gab_ctl = I915_READ(GAB_CTL);
3910                 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
3911
3912                 ecochk = I915_READ(GAM_ECOCHK);
3913                 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
3914                                        ECOCHK_PPGTT_CACHE64B);
3915                 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
3916         } else if (INTEL_INFO(dev)->gen >= 7) {
3917                 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B);
3918                 /* GFX_MODE is per-ring on gen7+ */
3919         }
3920
3921         for_each_ring(ring, dev_priv, i) {
3922                 if (INTEL_INFO(dev)->gen >= 7)
3923                         I915_WRITE(RING_MODE_GEN7(ring),
3924                                    _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
3925
3926                 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
3927                 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
3928         }
3929 }
3930
3931 static bool
3932 intel_enable_blt(struct drm_device *dev)
3933 {
3934         if (!HAS_BLT(dev))
3935                 return false;
3936
3937         /* The blitter was dysfunctional on early prototypes */
3938         if (IS_GEN6(dev) && dev->pdev->revision < 8) {
3939                 DRM_INFO("BLT not supported on this pre-production hardware;"
3940                          " graphics performance will be degraded.\n");
3941                 return false;
3942         }
3943
3944         return true;
3945 }
3946
3947 int
3948 i915_gem_init_hw(struct drm_device *dev)
3949 {
3950         drm_i915_private_t *dev_priv = dev->dev_private;
3951         int ret;
3952
3953         if (!intel_enable_gtt())
3954                 return -EIO;
3955
3956         i915_gem_l3_remap(dev);
3957
3958         i915_gem_init_swizzling(dev);
3959
3960         ret = intel_init_render_ring_buffer(dev);
3961         if (ret)
3962                 return ret;
3963
3964         if (HAS_BSD(dev)) {
3965                 ret = intel_init_bsd_ring_buffer(dev);
3966                 if (ret)
3967                         goto cleanup_render_ring;
3968         }
3969
3970         if (intel_enable_blt(dev)) {
3971                 ret = intel_init_blt_ring_buffer(dev);
3972                 if (ret)
3973                         goto cleanup_bsd_ring;
3974         }
3975
3976         dev_priv->next_seqno = 1;
3977
3978         /*
3979          * XXX: There was some w/a described somewhere suggesting loading
3980          * contexts before PPGTT.
3981          */
3982         i915_gem_context_init(dev);
3983         i915_gem_init_ppgtt(dev);
3984
3985         return 0;
3986
3987 cleanup_bsd_ring:
3988         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
3989 cleanup_render_ring:
3990         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
3991         return ret;
3992 }
3993
3994 static bool
3995 intel_enable_ppgtt(struct drm_device *dev)
3996 {
3997         if (i915_enable_ppgtt >= 0)
3998                 return i915_enable_ppgtt;
3999
4000 #ifdef CONFIG_INTEL_IOMMU
4001         /* Disable ppgtt on SNB if VT-d is on. */
4002         if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
4003                 return false;
4004 #endif
4005
4006         return true;
4007 }
4008
4009 int i915_gem_init(struct drm_device *dev)
4010 {
4011         struct drm_i915_private *dev_priv = dev->dev_private;
4012         unsigned long gtt_size, mappable_size;
4013         int ret;
4014
4015         gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
4016         mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
4017
4018         mutex_lock(&dev->struct_mutex);
4019         if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
4020                 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
4021                  * aperture accordingly when using aliasing ppgtt. */
4022                 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
4023
4024                 i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size);
4025
4026                 ret = i915_gem_init_aliasing_ppgtt(dev);
4027                 if (ret) {
4028                         mutex_unlock(&dev->struct_mutex);
4029                         return ret;
4030                 }
4031         } else {
4032                 /* Let GEM Manage all of the aperture.
4033                  *
4034                  * However, leave one page at the end still bound to the scratch
4035                  * page.  There are a number of places where the hardware
4036                  * apparently prefetches past the end of the object, and we've
4037                  * seen multiple hangs with the GPU head pointer stuck in a
4038                  * batchbuffer bound at the last page of the aperture.  One page
4039                  * should be enough to keep any prefetching inside of the
4040                  * aperture.
4041                  */
4042                 i915_gem_init_global_gtt(dev, 0, mappable_size,
4043                                          gtt_size);
4044         }
4045
4046         ret = i915_gem_init_hw(dev);
4047         mutex_unlock(&dev->struct_mutex);
4048         if (ret) {
4049                 i915_gem_cleanup_aliasing_ppgtt(dev);
4050                 return ret;
4051         }
4052
4053         /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
4054         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4055                 dev_priv->dri1.allow_batchbuffer = 1;
4056         return 0;
4057 }
4058
4059 void
4060 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4061 {
4062         drm_i915_private_t *dev_priv = dev->dev_private;
4063         struct intel_ring_buffer *ring;
4064         int i;
4065
4066         for_each_ring(ring, dev_priv, i)
4067                 intel_cleanup_ring_buffer(ring);
4068 }
4069
4070 int
4071 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4072                        struct drm_file *file_priv)
4073 {
4074         drm_i915_private_t *dev_priv = dev->dev_private;
4075         int ret;
4076
4077         if (drm_core_check_feature(dev, DRIVER_MODESET))
4078                 return 0;
4079
4080         if (atomic_read(&dev_priv->mm.wedged)) {
4081                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4082                 atomic_set(&dev_priv->mm.wedged, 0);
4083         }
4084
4085         mutex_lock(&dev->struct_mutex);
4086         dev_priv->mm.suspended = 0;
4087
4088         ret = i915_gem_init_hw(dev);
4089         if (ret != 0) {
4090                 mutex_unlock(&dev->struct_mutex);
4091                 return ret;
4092         }
4093
4094         BUG_ON(!list_empty(&dev_priv->mm.active_list));
4095         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4096         mutex_unlock(&dev->struct_mutex);
4097
4098         ret = drm_irq_install(dev);
4099         if (ret)
4100                 goto cleanup_ringbuffer;
4101
4102         return 0;
4103
4104 cleanup_ringbuffer:
4105         mutex_lock(&dev->struct_mutex);
4106         i915_gem_cleanup_ringbuffer(dev);
4107         dev_priv->mm.suspended = 1;
4108         mutex_unlock(&dev->struct_mutex);
4109
4110         return ret;
4111 }
4112
4113 int
4114 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4115                        struct drm_file *file_priv)
4116 {
4117         if (drm_core_check_feature(dev, DRIVER_MODESET))
4118                 return 0;
4119
4120         drm_irq_uninstall(dev);
4121         return i915_gem_idle(dev);
4122 }
4123
4124 void
4125 i915_gem_lastclose(struct drm_device *dev)
4126 {
4127         int ret;
4128
4129         if (drm_core_check_feature(dev, DRIVER_MODESET))
4130                 return;
4131
4132         ret = i915_gem_idle(dev);
4133         if (ret)
4134                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4135 }
4136
4137 static void
4138 init_ring_lists(struct intel_ring_buffer *ring)
4139 {
4140         INIT_LIST_HEAD(&ring->active_list);
4141         INIT_LIST_HEAD(&ring->request_list);
4142 }
4143
4144 void
4145 i915_gem_load(struct drm_device *dev)
4146 {
4147         int i;
4148         drm_i915_private_t *dev_priv = dev->dev_private;
4149
4150         INIT_LIST_HEAD(&dev_priv->mm.active_list);
4151         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4152         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
4153         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
4154         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4155         for (i = 0; i < I915_NUM_RINGS; i++)
4156                 init_ring_lists(&dev_priv->ring[i]);
4157         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
4158                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4159         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4160                           i915_gem_retire_work_handler);
4161         init_completion(&dev_priv->error_completion);
4162
4163         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4164         if (IS_GEN3(dev)) {
4165                 I915_WRITE(MI_ARB_STATE,
4166                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
4167         }
4168
4169         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
4170
4171         /* Old X drivers will take 0-2 for front, back, depth buffers */
4172         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4173                 dev_priv->fence_reg_start = 3;
4174
4175         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4176                 dev_priv->num_fence_regs = 16;
4177         else
4178                 dev_priv->num_fence_regs = 8;
4179
4180         /* Initialize fence registers to zero */
4181         i915_gem_reset_fences(dev);
4182
4183         i915_gem_detect_bit_6_swizzle(dev);
4184         init_waitqueue_head(&dev_priv->pending_flip_queue);
4185
4186         dev_priv->mm.interruptible = true;
4187
4188         dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
4189         dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
4190         register_shrinker(&dev_priv->mm.inactive_shrinker);
4191 }
4192
4193 /*
4194  * Create a physically contiguous memory object for this object
4195  * e.g. for cursor + overlay regs
4196  */
4197 static int i915_gem_init_phys_object(struct drm_device *dev,
4198                                      int id, int size, int align)
4199 {
4200         drm_i915_private_t *dev_priv = dev->dev_private;
4201         struct drm_i915_gem_phys_object *phys_obj;
4202         int ret;
4203
4204         if (dev_priv->mm.phys_objs[id - 1] || !size)
4205                 return 0;
4206
4207         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4208         if (!phys_obj)
4209                 return -ENOMEM;
4210
4211         phys_obj->id = id;
4212
4213         phys_obj->handle = drm_pci_alloc(dev, size, align);
4214         if (!phys_obj->handle) {
4215                 ret = -ENOMEM;
4216                 goto kfree_obj;
4217         }
4218 #ifdef CONFIG_X86
4219         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4220 #endif
4221
4222         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4223
4224         return 0;
4225 kfree_obj:
4226         kfree(phys_obj);
4227         return ret;
4228 }
4229
4230 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4231 {
4232         drm_i915_private_t *dev_priv = dev->dev_private;
4233         struct drm_i915_gem_phys_object *phys_obj;
4234
4235         if (!dev_priv->mm.phys_objs[id - 1])
4236                 return;
4237
4238         phys_obj = dev_priv->mm.phys_objs[id - 1];
4239         if (phys_obj->cur_obj) {
4240                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4241         }
4242
4243 #ifdef CONFIG_X86
4244         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4245 #endif
4246         drm_pci_free(dev, phys_obj->handle);
4247         kfree(phys_obj);
4248         dev_priv->mm.phys_objs[id - 1] = NULL;
4249 }
4250
4251 void i915_gem_free_all_phys_object(struct drm_device *dev)
4252 {
4253         int i;
4254
4255         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4256                 i915_gem_free_phys_object(dev, i);
4257 }
4258
4259 void i915_gem_detach_phys_object(struct drm_device *dev,
4260                                  struct drm_i915_gem_object *obj)
4261 {
4262         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
4263         char *vaddr;
4264         int i;
4265         int page_count;
4266
4267         if (!obj->phys_obj)
4268                 return;
4269         vaddr = obj->phys_obj->handle->vaddr;
4270
4271         page_count = obj->base.size / PAGE_SIZE;
4272         for (i = 0; i < page_count; i++) {
4273                 struct page *page = shmem_read_mapping_page(mapping, i);
4274                 if (!IS_ERR(page)) {
4275                         char *dst = kmap_atomic(page);
4276                         memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
4277                         kunmap_atomic(dst);
4278
4279                         drm_clflush_pages(&page, 1);
4280
4281                         set_page_dirty(page);
4282                         mark_page_accessed(page);
4283                         page_cache_release(page);
4284                 }
4285         }
4286         intel_gtt_chipset_flush();
4287
4288         obj->phys_obj->cur_obj = NULL;
4289         obj->phys_obj = NULL;
4290 }
4291
4292 int
4293 i915_gem_attach_phys_object(struct drm_device *dev,
4294                             struct drm_i915_gem_object *obj,
4295                             int id,
4296                             int align)
4297 {
4298         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
4299         drm_i915_private_t *dev_priv = dev->dev_private;
4300         int ret = 0;
4301         int page_count;
4302         int i;
4303
4304         if (id > I915_MAX_PHYS_OBJECT)
4305                 return -EINVAL;
4306
4307         if (obj->phys_obj) {
4308                 if (obj->phys_obj->id == id)
4309                         return 0;
4310                 i915_gem_detach_phys_object(dev, obj);
4311         }
4312
4313         /* create a new object */
4314         if (!dev_priv->mm.phys_objs[id - 1]) {
4315                 ret = i915_gem_init_phys_object(dev, id,
4316                                                 obj->base.size, align);
4317                 if (ret) {
4318                         DRM_ERROR("failed to init phys object %d size: %zu\n",
4319                                   id, obj->base.size);
4320                         return ret;
4321                 }
4322         }
4323
4324         /* bind to the object */
4325         obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
4326         obj->phys_obj->cur_obj = obj;
4327
4328         page_count = obj->base.size / PAGE_SIZE;
4329
4330         for (i = 0; i < page_count; i++) {
4331                 struct page *page;
4332                 char *dst, *src;
4333
4334                 page = shmem_read_mapping_page(mapping, i);
4335                 if (IS_ERR(page))
4336                         return PTR_ERR(page);
4337
4338                 src = kmap_atomic(page);
4339                 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4340                 memcpy(dst, src, PAGE_SIZE);
4341                 kunmap_atomic(src);
4342
4343                 mark_page_accessed(page);
4344                 page_cache_release(page);
4345         }
4346
4347         return 0;
4348 }
4349
4350 static int
4351 i915_gem_phys_pwrite(struct drm_device *dev,
4352                      struct drm_i915_gem_object *obj,
4353                      struct drm_i915_gem_pwrite *args,
4354                      struct drm_file *file_priv)
4355 {
4356         void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
4357         char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
4358
4359         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
4360                 unsigned long unwritten;
4361
4362                 /* The physical object once assigned is fixed for the lifetime
4363                  * of the obj, so we can safely drop the lock and continue
4364                  * to access vaddr.
4365                  */
4366                 mutex_unlock(&dev->struct_mutex);
4367                 unwritten = copy_from_user(vaddr, user_data, args->size);
4368                 mutex_lock(&dev->struct_mutex);
4369                 if (unwritten)
4370                         return -EFAULT;
4371         }
4372
4373         intel_gtt_chipset_flush();
4374         return 0;
4375 }
4376
4377 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4378 {
4379         struct drm_i915_file_private *file_priv = file->driver_priv;
4380
4381         /* Clean up our request list when the client is going away, so that
4382          * later retire_requests won't dereference our soon-to-be-gone
4383          * file_priv.
4384          */
4385         spin_lock(&file_priv->mm.lock);
4386         while (!list_empty(&file_priv->mm.request_list)) {
4387                 struct drm_i915_gem_request *request;
4388
4389                 request = list_first_entry(&file_priv->mm.request_list,
4390                                            struct drm_i915_gem_request,
4391                                            client_list);
4392                 list_del(&request->client_list);
4393                 request->file_priv = NULL;
4394         }
4395         spin_unlock(&file_priv->mm.lock);
4396 }
4397
4398 static int
4399 i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
4400 {
4401         struct drm_i915_private *dev_priv =
4402                 container_of(shrinker,
4403                              struct drm_i915_private,
4404                              mm.inactive_shrinker);
4405         struct drm_device *dev = dev_priv->dev;
4406         struct drm_i915_gem_object *obj;
4407         int nr_to_scan = sc->nr_to_scan;
4408         int cnt;
4409
4410         if (!mutex_trylock(&dev->struct_mutex))
4411                 return 0;
4412
4413         if (nr_to_scan) {
4414                 nr_to_scan -= i915_gem_purge(dev_priv, nr_to_scan);
4415                 if (nr_to_scan > 0)
4416                         i915_gem_shrink_all(dev_priv);
4417         }
4418
4419         cnt = 0;
4420         list_for_each_entry(obj, &dev_priv->mm.unbound_list, gtt_list)
4421                 if (obj->pages_pin_count == 0)
4422                         cnt += obj->base.size >> PAGE_SHIFT;
4423         list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
4424                 if (obj->pin_count == 0 && obj->pages_pin_count == 0)
4425                         cnt += obj->base.size >> PAGE_SHIFT;
4426
4427         mutex_unlock(&dev->struct_mutex);
4428         return cnt;
4429 }