drivers: cpuquiet: Fix hotplug stats
[linux-3.10.git] / mm / madvise.c
1 /*
2  *      linux/mm/madvise.c
3  *
4  * Copyright (C) 1999  Linus Torvalds
5  * Copyright (C) 2002  Christoph Hellwig
6  */
7
8 #include <linux/mman.h>
9 #include <linux/pagemap.h>
10 #include <linux/syscalls.h>
11 #include <linux/mempolicy.h>
12 #include <linux/page-isolation.h>
13 #include <linux/hugetlb.h>
14 #include <linux/falloc.h>
15 #include <linux/sched.h>
16 #include <linux/ksm.h>
17 #include <linux/fs.h>
18 #include <linux/file.h>
19 #include <linux/blkdev.h>
20 #include <linux/swap.h>
21 #include <linux/swapops.h>
22
23 /*
24  * Any behaviour which results in changes to the vma->vm_flags needs to
25  * take mmap_sem for writing. Others, which simply traverse vmas, need
26  * to only take it for reading.
27  */
28 static int madvise_need_mmap_write(int behavior)
29 {
30         switch (behavior) {
31         case MADV_REMOVE:
32         case MADV_WILLNEED:
33         case MADV_DONTNEED:
34                 return 0;
35         default:
36                 /* be safe, default to 1. list exceptions explicitly */
37                 return 1;
38         }
39 }
40
41 /*
42  * We can potentially split a vm area into separate
43  * areas, each area with its own behavior.
44  */
45 static long madvise_behavior(struct vm_area_struct * vma,
46                      struct vm_area_struct **prev,
47                      unsigned long start, unsigned long end, int behavior)
48 {
49         struct mm_struct * mm = vma->vm_mm;
50         int error = 0;
51         pgoff_t pgoff;
52         unsigned long new_flags = vma->vm_flags;
53
54         switch (behavior) {
55         case MADV_NORMAL:
56                 new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ;
57                 break;
58         case MADV_SEQUENTIAL:
59                 new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ;
60                 break;
61         case MADV_RANDOM:
62                 new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ;
63                 break;
64         case MADV_DONTFORK:
65                 new_flags |= VM_DONTCOPY;
66                 break;
67         case MADV_DOFORK:
68                 if (vma->vm_flags & VM_IO) {
69                         error = -EINVAL;
70                         goto out;
71                 }
72                 new_flags &= ~VM_DONTCOPY;
73                 break;
74         case MADV_DONTDUMP:
75                 new_flags |= VM_DONTDUMP;
76                 break;
77         case MADV_DODUMP:
78                 if (new_flags & VM_SPECIAL) {
79                         error = -EINVAL;
80                         goto out;
81                 }
82                 new_flags &= ~VM_DONTDUMP;
83                 break;
84         case MADV_MERGEABLE:
85         case MADV_UNMERGEABLE:
86                 error = ksm_madvise(vma, start, end, behavior, &new_flags);
87                 if (error)
88                         goto out;
89                 break;
90         case MADV_HUGEPAGE:
91         case MADV_NOHUGEPAGE:
92                 error = hugepage_madvise(vma, &new_flags, behavior);
93                 if (error)
94                         goto out;
95                 break;
96         }
97
98         if (new_flags == vma->vm_flags) {
99                 *prev = vma;
100                 goto out;
101         }
102
103         pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
104         *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma,
105                                 vma->vm_file, pgoff, vma_policy(vma),
106                                 vma_get_anon_name(vma));
107         if (*prev) {
108                 vma = *prev;
109                 goto success;
110         }
111
112         *prev = vma;
113
114         if (start != vma->vm_start) {
115                 error = split_vma(mm, vma, start, 1);
116                 if (error)
117                         goto out;
118         }
119
120         if (end != vma->vm_end) {
121                 error = split_vma(mm, vma, end, 0);
122                 if (error)
123                         goto out;
124         }
125
126 success:
127         /*
128          * vm_flags is protected by the mmap_sem held in write mode.
129          */
130         vma->vm_flags = new_flags;
131
132 out:
133         if (error == -ENOMEM)
134                 error = -EAGAIN;
135         return error;
136 }
137
138 #ifdef CONFIG_SWAP
139 static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
140         unsigned long end, struct mm_walk *walk)
141 {
142         pte_t *orig_pte;
143         struct vm_area_struct *vma = walk->private;
144         unsigned long index;
145
146         if (pmd_none_or_trans_huge_or_clear_bad(pmd))
147                 return 0;
148
149         for (index = start; index != end; index += PAGE_SIZE) {
150                 pte_t pte;
151                 swp_entry_t entry;
152                 struct page *page;
153                 spinlock_t *ptl;
154
155                 orig_pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl);
156                 pte = *(orig_pte + ((index - start) / PAGE_SIZE));
157                 pte_unmap_unlock(orig_pte, ptl);
158
159                 if (pte_present(pte) || pte_none(pte) || pte_file(pte))
160                         continue;
161                 entry = pte_to_swp_entry(pte);
162                 if (unlikely(non_swap_entry(entry)))
163                         continue;
164
165                 page = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,
166                                                                 vma, index);
167                 if (page)
168                         page_cache_release(page);
169         }
170
171         return 0;
172 }
173
174 static void force_swapin_readahead(struct vm_area_struct *vma,
175                 unsigned long start, unsigned long end)
176 {
177         struct mm_walk walk = {
178                 .mm = vma->vm_mm,
179                 .pmd_entry = swapin_walk_pmd_entry,
180                 .private = vma,
181         };
182
183         walk_page_range(start, end, &walk);
184
185         lru_add_drain();        /* Push any new pages onto the LRU now */
186 }
187
188 static void force_shm_swapin_readahead(struct vm_area_struct *vma,
189                 unsigned long start, unsigned long end,
190                 struct address_space *mapping)
191 {
192         pgoff_t index;
193         struct page *page;
194         swp_entry_t swap;
195
196         for (; start < end; start += PAGE_SIZE) {
197                 index = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
198
199                 page = find_get_page(mapping, index);
200                 if (!radix_tree_exceptional_entry(page)) {
201                         if (page)
202                                 page_cache_release(page);
203                         continue;
204                 }
205                 swap = radix_to_swp_entry(page);
206                 page = read_swap_cache_async(swap, GFP_HIGHUSER_MOVABLE,
207                                                                 NULL, 0);
208                 if (page)
209                         page_cache_release(page);
210         }
211
212         lru_add_drain();        /* Push any new pages onto the LRU now */
213 }
214 #endif          /* CONFIG_SWAP */
215
216 /*
217  * Schedule all required I/O operations.  Do not wait for completion.
218  */
219 static long madvise_willneed(struct vm_area_struct * vma,
220                              struct vm_area_struct ** prev,
221                              unsigned long start, unsigned long end)
222 {
223         struct file *file = vma->vm_file;
224
225 #ifdef CONFIG_SWAP
226         if (!file || mapping_cap_swap_backed(file->f_mapping)) {
227                 *prev = vma;
228                 if (!file)
229                         force_swapin_readahead(vma, start, end);
230                 else
231                         force_shm_swapin_readahead(vma, start, end,
232                                                 file->f_mapping);
233                 return 0;
234         }
235 #endif
236
237         if (!file)
238                 return -EBADF;
239
240         if (file->f_mapping->a_ops->get_xip_mem) {
241                 /* no bad return value, but ignore advice */
242                 return 0;
243         }
244
245         *prev = vma;
246         start = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
247         if (end > vma->vm_end)
248                 end = vma->vm_end;
249         end = ((end - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
250
251         force_page_cache_readahead(file->f_mapping, file, start, end - start);
252         return 0;
253 }
254
255 /*
256  * Application no longer needs these pages.  If the pages are dirty,
257  * it's OK to just throw them away.  The app will be more careful about
258  * data it wants to keep.  Be sure to free swap resources too.  The
259  * zap_page_range call sets things up for shrink_active_list to actually free
260  * these pages later if no one else has touched them in the meantime,
261  * although we could add these pages to a global reuse list for
262  * shrink_active_list to pick up before reclaiming other pages.
263  *
264  * NB: This interface discards data rather than pushes it out to swap,
265  * as some implementations do.  This has performance implications for
266  * applications like large transactional databases which want to discard
267  * pages in anonymous maps after committing to backing store the data
268  * that was kept in them.  There is no reason to write this data out to
269  * the swap area if the application is discarding it.
270  *
271  * An interface that causes the system to free clean pages and flush
272  * dirty pages is already available as msync(MS_INVALIDATE).
273  */
274 static long madvise_dontneed(struct vm_area_struct * vma,
275                              struct vm_area_struct ** prev,
276                              unsigned long start, unsigned long end)
277 {
278         *prev = vma;
279         if (vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP))
280                 return -EINVAL;
281
282         if (unlikely(vma->vm_flags & VM_NONLINEAR)) {
283                 struct zap_details details = {
284                         .nonlinear_vma = vma,
285                         .last_index = ULONG_MAX,
286                 };
287                 zap_page_range(vma, start, end - start, &details);
288         } else
289                 zap_page_range(vma, start, end - start, NULL);
290         return 0;
291 }
292
293 /*
294  * Application wants to free up the pages and associated backing store.
295  * This is effectively punching a hole into the middle of a file.
296  *
297  * NOTE: Currently, only shmfs/tmpfs is supported for this operation.
298  * Other filesystems return -ENOSYS.
299  */
300 static long madvise_remove(struct vm_area_struct *vma,
301                                 struct vm_area_struct **prev,
302                                 unsigned long start, unsigned long end)
303 {
304         loff_t offset;
305         int error;
306         struct file *f;
307
308         *prev = NULL;   /* tell sys_madvise we drop mmap_sem */
309
310         if (vma->vm_flags & (VM_LOCKED|VM_NONLINEAR|VM_HUGETLB))
311                 return -EINVAL;
312
313         f = vma->vm_file;
314
315         if (!f || !f->f_mapping || !f->f_mapping->host) {
316                         return -EINVAL;
317         }
318
319         if ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE))
320                 return -EACCES;
321
322         offset = (loff_t)(start - vma->vm_start)
323                         + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
324
325         /*
326          * Filesystem's fallocate may need to take i_mutex.  We need to
327          * explicitly grab a reference because the vma (and hence the
328          * vma's reference to the file) can go away as soon as we drop
329          * mmap_sem.
330          */
331         get_file(f);
332         up_read(&current->mm->mmap_sem);
333         error = do_fallocate(f,
334                                 FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
335                                 offset, end - start);
336         fput(f);
337         down_read(&current->mm->mmap_sem);
338         return error;
339 }
340
341 #ifdef CONFIG_MEMORY_FAILURE
342 /*
343  * Error injection support for memory error handling.
344  */
345 static int madvise_hwpoison(int bhv, unsigned long start, unsigned long end)
346 {
347         int ret = 0;
348
349         if (!capable(CAP_SYS_ADMIN))
350                 return -EPERM;
351         for (; start < end; start += PAGE_SIZE) {
352                 struct page *p;
353                 int ret = get_user_pages_fast(start, 1, 0, &p);
354                 if (ret != 1)
355                         return ret;
356                 if (bhv == MADV_SOFT_OFFLINE) {
357                         printk(KERN_INFO "Soft offlining page %lx at %lx\n",
358                                 page_to_pfn(p), start);
359                         ret = soft_offline_page(p, MF_COUNT_INCREASED);
360                         if (ret)
361                                 break;
362                         continue;
363                 }
364                 printk(KERN_INFO "Injecting memory failure for page %lx at %lx\n",
365                        page_to_pfn(p), start);
366                 /* Ignore return value for now */
367                 memory_failure(page_to_pfn(p), 0, MF_COUNT_INCREASED);
368         }
369         return ret;
370 }
371 #endif
372
373 static long
374 madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev,
375                 unsigned long start, unsigned long end, int behavior)
376 {
377         switch (behavior) {
378         case MADV_REMOVE:
379                 return madvise_remove(vma, prev, start, end);
380         case MADV_WILLNEED:
381                 return madvise_willneed(vma, prev, start, end);
382         case MADV_DONTNEED:
383                 return madvise_dontneed(vma, prev, start, end);
384         default:
385                 return madvise_behavior(vma, prev, start, end, behavior);
386         }
387 }
388
389 static int
390 madvise_behavior_valid(int behavior)
391 {
392         switch (behavior) {
393         case MADV_DOFORK:
394         case MADV_DONTFORK:
395         case MADV_NORMAL:
396         case MADV_SEQUENTIAL:
397         case MADV_RANDOM:
398         case MADV_REMOVE:
399         case MADV_WILLNEED:
400         case MADV_DONTNEED:
401 #ifdef CONFIG_KSM
402         case MADV_MERGEABLE:
403         case MADV_UNMERGEABLE:
404 #endif
405 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
406         case MADV_HUGEPAGE:
407         case MADV_NOHUGEPAGE:
408 #endif
409         case MADV_DONTDUMP:
410         case MADV_DODUMP:
411                 return 1;
412
413         default:
414                 return 0;
415         }
416 }
417
418 /*
419  * The madvise(2) system call.
420  *
421  * Applications can use madvise() to advise the kernel how it should
422  * handle paging I/O in this VM area.  The idea is to help the kernel
423  * use appropriate read-ahead and caching techniques.  The information
424  * provided is advisory only, and can be safely disregarded by the
425  * kernel without affecting the correct operation of the application.
426  *
427  * behavior values:
428  *  MADV_NORMAL - the default behavior is to read clusters.  This
429  *              results in some read-ahead and read-behind.
430  *  MADV_RANDOM - the system should read the minimum amount of data
431  *              on any access, since it is unlikely that the appli-
432  *              cation will need more than what it asks for.
433  *  MADV_SEQUENTIAL - pages in the given range will probably be accessed
434  *              once, so they can be aggressively read ahead, and
435  *              can be freed soon after they are accessed.
436  *  MADV_WILLNEED - the application is notifying the system to read
437  *              some pages ahead.
438  *  MADV_DONTNEED - the application is finished with the given range,
439  *              so the kernel can free resources associated with it.
440  *  MADV_REMOVE - the application wants to free up the given range of
441  *              pages and associated backing store.
442  *  MADV_DONTFORK - omit this area from child's address space when forking:
443  *              typically, to avoid COWing pages pinned by get_user_pages().
444  *  MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking.
445  *  MADV_MERGEABLE - the application recommends that KSM try to merge pages in
446  *              this area with pages of identical content from other such areas.
447  *  MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others.
448  *
449  * return values:
450  *  zero    - success
451  *  -EINVAL - start + len < 0, start is not page-aligned,
452  *              "behavior" is not a valid value, or application
453  *              is attempting to release locked or shared pages.
454  *  -ENOMEM - addresses in the specified range are not currently
455  *              mapped, or are outside the AS of the process.
456  *  -EIO    - an I/O error occurred while paging in data.
457  *  -EBADF  - map exists, but area maps something that isn't a file.
458  *  -EAGAIN - a kernel resource was temporarily unavailable.
459  */
460 SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
461 {
462         unsigned long end, tmp;
463         struct vm_area_struct * vma, *prev;
464         int unmapped_error = 0;
465         int error = -EINVAL;
466         int write;
467         size_t len;
468         struct blk_plug plug;
469
470 #ifdef CONFIG_MEMORY_FAILURE
471         if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE)
472                 return madvise_hwpoison(behavior, start, start+len_in);
473 #endif
474         if (!madvise_behavior_valid(behavior))
475                 return error;
476
477         if (start & ~PAGE_MASK)
478                 return error;
479         len = (len_in + ~PAGE_MASK) & PAGE_MASK;
480
481         /* Check to see whether len was rounded up from small -ve to zero */
482         if (len_in && !len)
483                 return error;
484
485         end = start + len;
486         if (end < start)
487                 return error;
488
489         error = 0;
490         if (end == start)
491                 return error;
492
493         write = madvise_need_mmap_write(behavior);
494         if (write)
495                 down_write(&current->mm->mmap_sem);
496         else
497                 down_read(&current->mm->mmap_sem);
498
499         /*
500          * If the interval [start,end) covers some unmapped address
501          * ranges, just ignore them, but return -ENOMEM at the end.
502          * - different from the way of handling in mlock etc.
503          */
504         vma = find_vma_prev(current->mm, start, &prev);
505         if (vma && start > vma->vm_start)
506                 prev = vma;
507
508         blk_start_plug(&plug);
509         for (;;) {
510                 /* Still start < end. */
511                 error = -ENOMEM;
512                 if (!vma)
513                         goto out;
514
515                 /* Here start < (end|vma->vm_end). */
516                 if (start < vma->vm_start) {
517                         unmapped_error = -ENOMEM;
518                         start = vma->vm_start;
519                         if (start >= end)
520                                 goto out;
521                 }
522
523                 /* Here vma->vm_start <= start < (end|vma->vm_end) */
524                 tmp = vma->vm_end;
525                 if (end < tmp)
526                         tmp = end;
527
528                 /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */
529                 error = madvise_vma(vma, &prev, start, tmp, behavior);
530                 if (error)
531                         goto out;
532                 start = tmp;
533                 if (prev && start < prev->vm_end)
534                         start = prev->vm_end;
535                 error = unmapped_error;
536                 if (start >= end)
537                         goto out;
538                 if (prev)
539                         vma = prev->vm_next;
540                 else    /* madvise_remove dropped mmap_sem */
541                         vma = find_vma(current->mm, start);
542         }
543 out:
544         blk_finish_plug(&plug);
545         if (write)
546                 up_write(&current->mm->mmap_sem);
547         else
548                 up_read(&current->mm->mmap_sem);
549
550         return error;
551 }