thp: madvise(MADV_HUGEPAGE)
[linux-2.6.git] / mm / huge_memory.c
1 /*
2  *  Copyright (C) 2009  Red Hat, Inc.
3  *
4  *  This work is licensed under the terms of the GNU GPL, version 2. See
5  *  the COPYING file in the top-level directory.
6  */
7
8 #include <linux/mm.h>
9 #include <linux/sched.h>
10 #include <linux/highmem.h>
11 #include <linux/hugetlb.h>
12 #include <linux/mmu_notifier.h>
13 #include <linux/rmap.h>
14 #include <linux/swap.h>
15 #include <asm/tlb.h>
16 #include <asm/pgalloc.h>
17 #include "internal.h"
18
19 unsigned long transparent_hugepage_flags __read_mostly =
20         (1<<TRANSPARENT_HUGEPAGE_FLAG);
21
22 #ifdef CONFIG_SYSFS
23 static ssize_t double_flag_show(struct kobject *kobj,
24                                 struct kobj_attribute *attr, char *buf,
25                                 enum transparent_hugepage_flag enabled,
26                                 enum transparent_hugepage_flag req_madv)
27 {
28         if (test_bit(enabled, &transparent_hugepage_flags)) {
29                 VM_BUG_ON(test_bit(req_madv, &transparent_hugepage_flags));
30                 return sprintf(buf, "[always] madvise never\n");
31         } else if (test_bit(req_madv, &transparent_hugepage_flags))
32                 return sprintf(buf, "always [madvise] never\n");
33         else
34                 return sprintf(buf, "always madvise [never]\n");
35 }
36 static ssize_t double_flag_store(struct kobject *kobj,
37                                  struct kobj_attribute *attr,
38                                  const char *buf, size_t count,
39                                  enum transparent_hugepage_flag enabled,
40                                  enum transparent_hugepage_flag req_madv)
41 {
42         if (!memcmp("always", buf,
43                     min(sizeof("always")-1, count))) {
44                 set_bit(enabled, &transparent_hugepage_flags);
45                 clear_bit(req_madv, &transparent_hugepage_flags);
46         } else if (!memcmp("madvise", buf,
47                            min(sizeof("madvise")-1, count))) {
48                 clear_bit(enabled, &transparent_hugepage_flags);
49                 set_bit(req_madv, &transparent_hugepage_flags);
50         } else if (!memcmp("never", buf,
51                            min(sizeof("never")-1, count))) {
52                 clear_bit(enabled, &transparent_hugepage_flags);
53                 clear_bit(req_madv, &transparent_hugepage_flags);
54         } else
55                 return -EINVAL;
56
57         return count;
58 }
59
60 static ssize_t enabled_show(struct kobject *kobj,
61                             struct kobj_attribute *attr, char *buf)
62 {
63         return double_flag_show(kobj, attr, buf,
64                                 TRANSPARENT_HUGEPAGE_FLAG,
65                                 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);
66 }
67 static ssize_t enabled_store(struct kobject *kobj,
68                              struct kobj_attribute *attr,
69                              const char *buf, size_t count)
70 {
71         return double_flag_store(kobj, attr, buf, count,
72                                  TRANSPARENT_HUGEPAGE_FLAG,
73                                  TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);
74 }
75 static struct kobj_attribute enabled_attr =
76         __ATTR(enabled, 0644, enabled_show, enabled_store);
77
78 static ssize_t single_flag_show(struct kobject *kobj,
79                                 struct kobj_attribute *attr, char *buf,
80                                 enum transparent_hugepage_flag flag)
81 {
82         if (test_bit(flag, &transparent_hugepage_flags))
83                 return sprintf(buf, "[yes] no\n");
84         else
85                 return sprintf(buf, "yes [no]\n");
86 }
87 static ssize_t single_flag_store(struct kobject *kobj,
88                                  struct kobj_attribute *attr,
89                                  const char *buf, size_t count,
90                                  enum transparent_hugepage_flag flag)
91 {
92         if (!memcmp("yes", buf,
93                     min(sizeof("yes")-1, count))) {
94                 set_bit(flag, &transparent_hugepage_flags);
95         } else if (!memcmp("no", buf,
96                            min(sizeof("no")-1, count))) {
97                 clear_bit(flag, &transparent_hugepage_flags);
98         } else
99                 return -EINVAL;
100
101         return count;
102 }
103
104 /*
105  * Currently defrag only disables __GFP_NOWAIT for allocation. A blind
106  * __GFP_REPEAT is too aggressive, it's never worth swapping tons of
107  * memory just to allocate one more hugepage.
108  */
109 static ssize_t defrag_show(struct kobject *kobj,
110                            struct kobj_attribute *attr, char *buf)
111 {
112         return double_flag_show(kobj, attr, buf,
113                                 TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
114                                 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
115 }
116 static ssize_t defrag_store(struct kobject *kobj,
117                             struct kobj_attribute *attr,
118                             const char *buf, size_t count)
119 {
120         return double_flag_store(kobj, attr, buf, count,
121                                  TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
122                                  TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
123 }
124 static struct kobj_attribute defrag_attr =
125         __ATTR(defrag, 0644, defrag_show, defrag_store);
126
127 #ifdef CONFIG_DEBUG_VM
128 static ssize_t debug_cow_show(struct kobject *kobj,
129                                 struct kobj_attribute *attr, char *buf)
130 {
131         return single_flag_show(kobj, attr, buf,
132                                 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
133 }
134 static ssize_t debug_cow_store(struct kobject *kobj,
135                                struct kobj_attribute *attr,
136                                const char *buf, size_t count)
137 {
138         return single_flag_store(kobj, attr, buf, count,
139                                  TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
140 }
141 static struct kobj_attribute debug_cow_attr =
142         __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store);
143 #endif /* CONFIG_DEBUG_VM */
144
145 static struct attribute *hugepage_attr[] = {
146         &enabled_attr.attr,
147         &defrag_attr.attr,
148 #ifdef CONFIG_DEBUG_VM
149         &debug_cow_attr.attr,
150 #endif
151         NULL,
152 };
153
154 static struct attribute_group hugepage_attr_group = {
155         .attrs = hugepage_attr,
156         .name = "transparent_hugepage",
157 };
158 #endif /* CONFIG_SYSFS */
159
160 static int __init hugepage_init(void)
161 {
162 #ifdef CONFIG_SYSFS
163         int err;
164
165         err = sysfs_create_group(mm_kobj, &hugepage_attr_group);
166         if (err)
167                 printk(KERN_ERR "hugepage: register sysfs failed\n");
168 #endif
169         return 0;
170 }
171 module_init(hugepage_init)
172
173 static int __init setup_transparent_hugepage(char *str)
174 {
175         int ret = 0;
176         if (!str)
177                 goto out;
178         if (!strcmp(str, "always")) {
179                 set_bit(TRANSPARENT_HUGEPAGE_FLAG,
180                         &transparent_hugepage_flags);
181                 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
182                           &transparent_hugepage_flags);
183                 ret = 1;
184         } else if (!strcmp(str, "madvise")) {
185                 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
186                           &transparent_hugepage_flags);
187                 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
188                         &transparent_hugepage_flags);
189                 ret = 1;
190         } else if (!strcmp(str, "never")) {
191                 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
192                           &transparent_hugepage_flags);
193                 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
194                           &transparent_hugepage_flags);
195                 ret = 1;
196         }
197 out:
198         if (!ret)
199                 printk(KERN_WARNING
200                        "transparent_hugepage= cannot parse, ignored\n");
201         return ret;
202 }
203 __setup("transparent_hugepage=", setup_transparent_hugepage);
204
205 static void prepare_pmd_huge_pte(pgtable_t pgtable,
206                                  struct mm_struct *mm)
207 {
208         assert_spin_locked(&mm->page_table_lock);
209
210         /* FIFO */
211         if (!mm->pmd_huge_pte)
212                 INIT_LIST_HEAD(&pgtable->lru);
213         else
214                 list_add(&pgtable->lru, &mm->pmd_huge_pte->lru);
215         mm->pmd_huge_pte = pgtable;
216 }
217
218 static inline pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
219 {
220         if (likely(vma->vm_flags & VM_WRITE))
221                 pmd = pmd_mkwrite(pmd);
222         return pmd;
223 }
224
225 static int __do_huge_pmd_anonymous_page(struct mm_struct *mm,
226                                         struct vm_area_struct *vma,
227                                         unsigned long haddr, pmd_t *pmd,
228                                         struct page *page)
229 {
230         int ret = 0;
231         pgtable_t pgtable;
232
233         VM_BUG_ON(!PageCompound(page));
234         pgtable = pte_alloc_one(mm, haddr);
235         if (unlikely(!pgtable)) {
236                 put_page(page);
237                 return VM_FAULT_OOM;
238         }
239
240         clear_huge_page(page, haddr, HPAGE_PMD_NR);
241         __SetPageUptodate(page);
242
243         spin_lock(&mm->page_table_lock);
244         if (unlikely(!pmd_none(*pmd))) {
245                 spin_unlock(&mm->page_table_lock);
246                 put_page(page);
247                 pte_free(mm, pgtable);
248         } else {
249                 pmd_t entry;
250                 entry = mk_pmd(page, vma->vm_page_prot);
251                 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
252                 entry = pmd_mkhuge(entry);
253                 /*
254                  * The spinlocking to take the lru_lock inside
255                  * page_add_new_anon_rmap() acts as a full memory
256                  * barrier to be sure clear_huge_page writes become
257                  * visible after the set_pmd_at() write.
258                  */
259                 page_add_new_anon_rmap(page, vma, haddr);
260                 set_pmd_at(mm, haddr, pmd, entry);
261                 prepare_pmd_huge_pte(pgtable, mm);
262                 add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR);
263                 spin_unlock(&mm->page_table_lock);
264         }
265
266         return ret;
267 }
268
269 static inline struct page *alloc_hugepage(int defrag)
270 {
271         return alloc_pages(GFP_TRANSHUGE & ~(defrag ? 0 : __GFP_WAIT),
272                            HPAGE_PMD_ORDER);
273 }
274
275 int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
276                                unsigned long address, pmd_t *pmd,
277                                unsigned int flags)
278 {
279         struct page *page;
280         unsigned long haddr = address & HPAGE_PMD_MASK;
281         pte_t *pte;
282
283         if (haddr >= vma->vm_start && haddr + HPAGE_PMD_SIZE <= vma->vm_end) {
284                 if (unlikely(anon_vma_prepare(vma)))
285                         return VM_FAULT_OOM;
286                 page = alloc_hugepage(transparent_hugepage_defrag(vma));
287                 if (unlikely(!page))
288                         goto out;
289
290                 return __do_huge_pmd_anonymous_page(mm, vma, haddr, pmd, page);
291         }
292 out:
293         /*
294          * Use __pte_alloc instead of pte_alloc_map, because we can't
295          * run pte_offset_map on the pmd, if an huge pmd could
296          * materialize from under us from a different thread.
297          */
298         if (unlikely(__pte_alloc(mm, vma, pmd, address)))
299                 return VM_FAULT_OOM;
300         /* if an huge pmd materialized from under us just retry later */
301         if (unlikely(pmd_trans_huge(*pmd)))
302                 return 0;
303         /*
304          * A regular pmd is established and it can't morph into a huge pmd
305          * from under us anymore at this point because we hold the mmap_sem
306          * read mode and khugepaged takes it in write mode. So now it's
307          * safe to run pte_offset_map().
308          */
309         pte = pte_offset_map(pmd, address);
310         return handle_pte_fault(mm, vma, address, pte, pmd, flags);
311 }
312
313 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
314                   pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
315                   struct vm_area_struct *vma)
316 {
317         struct page *src_page;
318         pmd_t pmd;
319         pgtable_t pgtable;
320         int ret;
321
322         ret = -ENOMEM;
323         pgtable = pte_alloc_one(dst_mm, addr);
324         if (unlikely(!pgtable))
325                 goto out;
326
327         spin_lock(&dst_mm->page_table_lock);
328         spin_lock_nested(&src_mm->page_table_lock, SINGLE_DEPTH_NESTING);
329
330         ret = -EAGAIN;
331         pmd = *src_pmd;
332         if (unlikely(!pmd_trans_huge(pmd))) {
333                 pte_free(dst_mm, pgtable);
334                 goto out_unlock;
335         }
336         if (unlikely(pmd_trans_splitting(pmd))) {
337                 /* split huge page running from under us */
338                 spin_unlock(&src_mm->page_table_lock);
339                 spin_unlock(&dst_mm->page_table_lock);
340                 pte_free(dst_mm, pgtable);
341
342                 wait_split_huge_page(vma->anon_vma, src_pmd); /* src_vma */
343                 goto out;
344         }
345         src_page = pmd_page(pmd);
346         VM_BUG_ON(!PageHead(src_page));
347         get_page(src_page);
348         page_dup_rmap(src_page);
349         add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
350
351         pmdp_set_wrprotect(src_mm, addr, src_pmd);
352         pmd = pmd_mkold(pmd_wrprotect(pmd));
353         set_pmd_at(dst_mm, addr, dst_pmd, pmd);
354         prepare_pmd_huge_pte(pgtable, dst_mm);
355
356         ret = 0;
357 out_unlock:
358         spin_unlock(&src_mm->page_table_lock);
359         spin_unlock(&dst_mm->page_table_lock);
360 out:
361         return ret;
362 }
363
364 /* no "address" argument so destroys page coloring of some arch */
365 pgtable_t get_pmd_huge_pte(struct mm_struct *mm)
366 {
367         pgtable_t pgtable;
368
369         assert_spin_locked(&mm->page_table_lock);
370
371         /* FIFO */
372         pgtable = mm->pmd_huge_pte;
373         if (list_empty(&pgtable->lru))
374                 mm->pmd_huge_pte = NULL;
375         else {
376                 mm->pmd_huge_pte = list_entry(pgtable->lru.next,
377                                               struct page, lru);
378                 list_del(&pgtable->lru);
379         }
380         return pgtable;
381 }
382
383 static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
384                                         struct vm_area_struct *vma,
385                                         unsigned long address,
386                                         pmd_t *pmd, pmd_t orig_pmd,
387                                         struct page *page,
388                                         unsigned long haddr)
389 {
390         pgtable_t pgtable;
391         pmd_t _pmd;
392         int ret = 0, i;
393         struct page **pages;
394
395         pages = kmalloc(sizeof(struct page *) * HPAGE_PMD_NR,
396                         GFP_KERNEL);
397         if (unlikely(!pages)) {
398                 ret |= VM_FAULT_OOM;
399                 goto out;
400         }
401
402         for (i = 0; i < HPAGE_PMD_NR; i++) {
403                 pages[i] = alloc_page_vma(GFP_HIGHUSER_MOVABLE,
404                                           vma, address);
405                 if (unlikely(!pages[i])) {
406                         while (--i >= 0)
407                                 put_page(pages[i]);
408                         kfree(pages);
409                         ret |= VM_FAULT_OOM;
410                         goto out;
411                 }
412         }
413
414         for (i = 0; i < HPAGE_PMD_NR; i++) {
415                 copy_user_highpage(pages[i], page + i,
416                                    haddr + PAGE_SHIFT*i, vma);
417                 __SetPageUptodate(pages[i]);
418                 cond_resched();
419         }
420
421         spin_lock(&mm->page_table_lock);
422         if (unlikely(!pmd_same(*pmd, orig_pmd)))
423                 goto out_free_pages;
424         VM_BUG_ON(!PageHead(page));
425
426         pmdp_clear_flush_notify(vma, haddr, pmd);
427         /* leave pmd empty until pte is filled */
428
429         pgtable = get_pmd_huge_pte(mm);
430         pmd_populate(mm, &_pmd, pgtable);
431
432         for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
433                 pte_t *pte, entry;
434                 entry = mk_pte(pages[i], vma->vm_page_prot);
435                 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
436                 page_add_new_anon_rmap(pages[i], vma, haddr);
437                 pte = pte_offset_map(&_pmd, haddr);
438                 VM_BUG_ON(!pte_none(*pte));
439                 set_pte_at(mm, haddr, pte, entry);
440                 pte_unmap(pte);
441         }
442         kfree(pages);
443
444         mm->nr_ptes++;
445         smp_wmb(); /* make pte visible before pmd */
446         pmd_populate(mm, pmd, pgtable);
447         page_remove_rmap(page);
448         spin_unlock(&mm->page_table_lock);
449
450         ret |= VM_FAULT_WRITE;
451         put_page(page);
452
453 out:
454         return ret;
455
456 out_free_pages:
457         spin_unlock(&mm->page_table_lock);
458         for (i = 0; i < HPAGE_PMD_NR; i++)
459                 put_page(pages[i]);
460         kfree(pages);
461         goto out;
462 }
463
464 int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
465                         unsigned long address, pmd_t *pmd, pmd_t orig_pmd)
466 {
467         int ret = 0;
468         struct page *page, *new_page;
469         unsigned long haddr;
470
471         VM_BUG_ON(!vma->anon_vma);
472         spin_lock(&mm->page_table_lock);
473         if (unlikely(!pmd_same(*pmd, orig_pmd)))
474                 goto out_unlock;
475
476         page = pmd_page(orig_pmd);
477         VM_BUG_ON(!PageCompound(page) || !PageHead(page));
478         haddr = address & HPAGE_PMD_MASK;
479         if (page_mapcount(page) == 1) {
480                 pmd_t entry;
481                 entry = pmd_mkyoung(orig_pmd);
482                 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
483                 if (pmdp_set_access_flags(vma, haddr, pmd, entry,  1))
484                         update_mmu_cache(vma, address, entry);
485                 ret |= VM_FAULT_WRITE;
486                 goto out_unlock;
487         }
488         get_page(page);
489         spin_unlock(&mm->page_table_lock);
490
491         if (transparent_hugepage_enabled(vma) &&
492             !transparent_hugepage_debug_cow())
493                 new_page = alloc_hugepage(transparent_hugepage_defrag(vma));
494         else
495                 new_page = NULL;
496
497         if (unlikely(!new_page)) {
498                 ret = do_huge_pmd_wp_page_fallback(mm, vma, address,
499                                                    pmd, orig_pmd, page, haddr);
500                 put_page(page);
501                 goto out;
502         }
503
504         copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR);
505         __SetPageUptodate(new_page);
506
507         spin_lock(&mm->page_table_lock);
508         put_page(page);
509         if (unlikely(!pmd_same(*pmd, orig_pmd)))
510                 put_page(new_page);
511         else {
512                 pmd_t entry;
513                 VM_BUG_ON(!PageHead(page));
514                 entry = mk_pmd(new_page, vma->vm_page_prot);
515                 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
516                 entry = pmd_mkhuge(entry);
517                 pmdp_clear_flush_notify(vma, haddr, pmd);
518                 page_add_new_anon_rmap(new_page, vma, haddr);
519                 set_pmd_at(mm, haddr, pmd, entry);
520                 update_mmu_cache(vma, address, entry);
521                 page_remove_rmap(page);
522                 put_page(page);
523                 ret |= VM_FAULT_WRITE;
524         }
525 out_unlock:
526         spin_unlock(&mm->page_table_lock);
527 out:
528         return ret;
529 }
530
531 struct page *follow_trans_huge_pmd(struct mm_struct *mm,
532                                    unsigned long addr,
533                                    pmd_t *pmd,
534                                    unsigned int flags)
535 {
536         struct page *page = NULL;
537
538         assert_spin_locked(&mm->page_table_lock);
539
540         if (flags & FOLL_WRITE && !pmd_write(*pmd))
541                 goto out;
542
543         page = pmd_page(*pmd);
544         VM_BUG_ON(!PageHead(page));
545         if (flags & FOLL_TOUCH) {
546                 pmd_t _pmd;
547                 /*
548                  * We should set the dirty bit only for FOLL_WRITE but
549                  * for now the dirty bit in the pmd is meaningless.
550                  * And if the dirty bit will become meaningful and
551                  * we'll only set it with FOLL_WRITE, an atomic
552                  * set_bit will be required on the pmd to set the
553                  * young bit, instead of the current set_pmd_at.
554                  */
555                 _pmd = pmd_mkyoung(pmd_mkdirty(*pmd));
556                 set_pmd_at(mm, addr & HPAGE_PMD_MASK, pmd, _pmd);
557         }
558         page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
559         VM_BUG_ON(!PageCompound(page));
560         if (flags & FOLL_GET)
561                 get_page(page);
562
563 out:
564         return page;
565 }
566
567 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
568                  pmd_t *pmd)
569 {
570         int ret = 0;
571
572         spin_lock(&tlb->mm->page_table_lock);
573         if (likely(pmd_trans_huge(*pmd))) {
574                 if (unlikely(pmd_trans_splitting(*pmd))) {
575                         spin_unlock(&tlb->mm->page_table_lock);
576                         wait_split_huge_page(vma->anon_vma,
577                                              pmd);
578                 } else {
579                         struct page *page;
580                         pgtable_t pgtable;
581                         pgtable = get_pmd_huge_pte(tlb->mm);
582                         page = pmd_page(*pmd);
583                         pmd_clear(pmd);
584                         page_remove_rmap(page);
585                         VM_BUG_ON(page_mapcount(page) < 0);
586                         add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
587                         VM_BUG_ON(!PageHead(page));
588                         spin_unlock(&tlb->mm->page_table_lock);
589                         tlb_remove_page(tlb, page);
590                         pte_free(tlb->mm, pgtable);
591                         ret = 1;
592                 }
593         } else
594                 spin_unlock(&tlb->mm->page_table_lock);
595
596         return ret;
597 }
598
599 pmd_t *page_check_address_pmd(struct page *page,
600                               struct mm_struct *mm,
601                               unsigned long address,
602                               enum page_check_address_pmd_flag flag)
603 {
604         pgd_t *pgd;
605         pud_t *pud;
606         pmd_t *pmd, *ret = NULL;
607
608         if (address & ~HPAGE_PMD_MASK)
609                 goto out;
610
611         pgd = pgd_offset(mm, address);
612         if (!pgd_present(*pgd))
613                 goto out;
614
615         pud = pud_offset(pgd, address);
616         if (!pud_present(*pud))
617                 goto out;
618
619         pmd = pmd_offset(pud, address);
620         if (pmd_none(*pmd))
621                 goto out;
622         if (pmd_page(*pmd) != page)
623                 goto out;
624         VM_BUG_ON(flag == PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG &&
625                   pmd_trans_splitting(*pmd));
626         if (pmd_trans_huge(*pmd)) {
627                 VM_BUG_ON(flag == PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG &&
628                           !pmd_trans_splitting(*pmd));
629                 ret = pmd;
630         }
631 out:
632         return ret;
633 }
634
635 static int __split_huge_page_splitting(struct page *page,
636                                        struct vm_area_struct *vma,
637                                        unsigned long address)
638 {
639         struct mm_struct *mm = vma->vm_mm;
640         pmd_t *pmd;
641         int ret = 0;
642
643         spin_lock(&mm->page_table_lock);
644         pmd = page_check_address_pmd(page, mm, address,
645                                      PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG);
646         if (pmd) {
647                 /*
648                  * We can't temporarily set the pmd to null in order
649                  * to split it, the pmd must remain marked huge at all
650                  * times or the VM won't take the pmd_trans_huge paths
651                  * and it won't wait on the anon_vma->root->lock to
652                  * serialize against split_huge_page*.
653                  */
654                 pmdp_splitting_flush_notify(vma, address, pmd);
655                 ret = 1;
656         }
657         spin_unlock(&mm->page_table_lock);
658
659         return ret;
660 }
661
662 static void __split_huge_page_refcount(struct page *page)
663 {
664         int i;
665         unsigned long head_index = page->index;
666         struct zone *zone = page_zone(page);
667
668         /* prevent PageLRU to go away from under us, and freeze lru stats */
669         spin_lock_irq(&zone->lru_lock);
670         compound_lock(page);
671
672         for (i = 1; i < HPAGE_PMD_NR; i++) {
673                 struct page *page_tail = page + i;
674
675                 /* tail_page->_count cannot change */
676                 atomic_sub(atomic_read(&page_tail->_count), &page->_count);
677                 BUG_ON(page_count(page) <= 0);
678                 atomic_add(page_mapcount(page) + 1, &page_tail->_count);
679                 BUG_ON(atomic_read(&page_tail->_count) <= 0);
680
681                 /* after clearing PageTail the gup refcount can be released */
682                 smp_mb();
683
684                 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
685                 page_tail->flags |= (page->flags &
686                                      ((1L << PG_referenced) |
687                                       (1L << PG_swapbacked) |
688                                       (1L << PG_mlocked) |
689                                       (1L << PG_uptodate)));
690                 page_tail->flags |= (1L << PG_dirty);
691
692                 /*
693                  * 1) clear PageTail before overwriting first_page
694                  * 2) clear PageTail before clearing PageHead for VM_BUG_ON
695                  */
696                 smp_wmb();
697
698                 /*
699                  * __split_huge_page_splitting() already set the
700                  * splitting bit in all pmd that could map this
701                  * hugepage, that will ensure no CPU can alter the
702                  * mapcount on the head page. The mapcount is only
703                  * accounted in the head page and it has to be
704                  * transferred to all tail pages in the below code. So
705                  * for this code to be safe, the split the mapcount
706                  * can't change. But that doesn't mean userland can't
707                  * keep changing and reading the page contents while
708                  * we transfer the mapcount, so the pmd splitting
709                  * status is achieved setting a reserved bit in the
710                  * pmd, not by clearing the present bit.
711                 */
712                 BUG_ON(page_mapcount(page_tail));
713                 page_tail->_mapcount = page->_mapcount;
714
715                 BUG_ON(page_tail->mapping);
716                 page_tail->mapping = page->mapping;
717
718                 page_tail->index = ++head_index;
719
720                 BUG_ON(!PageAnon(page_tail));
721                 BUG_ON(!PageUptodate(page_tail));
722                 BUG_ON(!PageDirty(page_tail));
723                 BUG_ON(!PageSwapBacked(page_tail));
724
725                 lru_add_page_tail(zone, page, page_tail);
726         }
727
728         ClearPageCompound(page);
729         compound_unlock(page);
730         spin_unlock_irq(&zone->lru_lock);
731
732         for (i = 1; i < HPAGE_PMD_NR; i++) {
733                 struct page *page_tail = page + i;
734                 BUG_ON(page_count(page_tail) <= 0);
735                 /*
736                  * Tail pages may be freed if there wasn't any mapping
737                  * like if add_to_swap() is running on a lru page that
738                  * had its mapping zapped. And freeing these pages
739                  * requires taking the lru_lock so we do the put_page
740                  * of the tail pages after the split is complete.
741                  */
742                 put_page(page_tail);
743         }
744
745         /*
746          * Only the head page (now become a regular page) is required
747          * to be pinned by the caller.
748          */
749         BUG_ON(page_count(page) <= 0);
750 }
751
752 static int __split_huge_page_map(struct page *page,
753                                  struct vm_area_struct *vma,
754                                  unsigned long address)
755 {
756         struct mm_struct *mm = vma->vm_mm;
757         pmd_t *pmd, _pmd;
758         int ret = 0, i;
759         pgtable_t pgtable;
760         unsigned long haddr;
761
762         spin_lock(&mm->page_table_lock);
763         pmd = page_check_address_pmd(page, mm, address,
764                                      PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG);
765         if (pmd) {
766                 pgtable = get_pmd_huge_pte(mm);
767                 pmd_populate(mm, &_pmd, pgtable);
768
769                 for (i = 0, haddr = address; i < HPAGE_PMD_NR;
770                      i++, haddr += PAGE_SIZE) {
771                         pte_t *pte, entry;
772                         BUG_ON(PageCompound(page+i));
773                         entry = mk_pte(page + i, vma->vm_page_prot);
774                         entry = maybe_mkwrite(pte_mkdirty(entry), vma);
775                         if (!pmd_write(*pmd))
776                                 entry = pte_wrprotect(entry);
777                         else
778                                 BUG_ON(page_mapcount(page) != 1);
779                         if (!pmd_young(*pmd))
780                                 entry = pte_mkold(entry);
781                         pte = pte_offset_map(&_pmd, haddr);
782                         BUG_ON(!pte_none(*pte));
783                         set_pte_at(mm, haddr, pte, entry);
784                         pte_unmap(pte);
785                 }
786
787                 mm->nr_ptes++;
788                 smp_wmb(); /* make pte visible before pmd */
789                 /*
790                  * Up to this point the pmd is present and huge and
791                  * userland has the whole access to the hugepage
792                  * during the split (which happens in place). If we
793                  * overwrite the pmd with the not-huge version
794                  * pointing to the pte here (which of course we could
795                  * if all CPUs were bug free), userland could trigger
796                  * a small page size TLB miss on the small sized TLB
797                  * while the hugepage TLB entry is still established
798                  * in the huge TLB. Some CPU doesn't like that. See
799                  * http://support.amd.com/us/Processor_TechDocs/41322.pdf,
800                  * Erratum 383 on page 93. Intel should be safe but is
801                  * also warns that it's only safe if the permission
802                  * and cache attributes of the two entries loaded in
803                  * the two TLB is identical (which should be the case
804                  * here). But it is generally safer to never allow
805                  * small and huge TLB entries for the same virtual
806                  * address to be loaded simultaneously. So instead of
807                  * doing "pmd_populate(); flush_tlb_range();" we first
808                  * mark the current pmd notpresent (atomically because
809                  * here the pmd_trans_huge and pmd_trans_splitting
810                  * must remain set at all times on the pmd until the
811                  * split is complete for this pmd), then we flush the
812                  * SMP TLB and finally we write the non-huge version
813                  * of the pmd entry with pmd_populate.
814                  */
815                 set_pmd_at(mm, address, pmd, pmd_mknotpresent(*pmd));
816                 flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
817                 pmd_populate(mm, pmd, pgtable);
818                 ret = 1;
819         }
820         spin_unlock(&mm->page_table_lock);
821
822         return ret;
823 }
824
825 /* must be called with anon_vma->root->lock hold */
826 static void __split_huge_page(struct page *page,
827                               struct anon_vma *anon_vma)
828 {
829         int mapcount, mapcount2;
830         struct anon_vma_chain *avc;
831
832         BUG_ON(!PageHead(page));
833         BUG_ON(PageTail(page));
834
835         mapcount = 0;
836         list_for_each_entry(avc, &anon_vma->head, same_anon_vma) {
837                 struct vm_area_struct *vma = avc->vma;
838                 unsigned long addr = vma_address(page, vma);
839                 BUG_ON(is_vma_temporary_stack(vma));
840                 if (addr == -EFAULT)
841                         continue;
842                 mapcount += __split_huge_page_splitting(page, vma, addr);
843         }
844         /*
845          * It is critical that new vmas are added to the tail of the
846          * anon_vma list. This guarantes that if copy_huge_pmd() runs
847          * and establishes a child pmd before
848          * __split_huge_page_splitting() freezes the parent pmd (so if
849          * we fail to prevent copy_huge_pmd() from running until the
850          * whole __split_huge_page() is complete), we will still see
851          * the newly established pmd of the child later during the
852          * walk, to be able to set it as pmd_trans_splitting too.
853          */
854         if (mapcount != page_mapcount(page))
855                 printk(KERN_ERR "mapcount %d page_mapcount %d\n",
856                        mapcount, page_mapcount(page));
857         BUG_ON(mapcount != page_mapcount(page));
858
859         __split_huge_page_refcount(page);
860
861         mapcount2 = 0;
862         list_for_each_entry(avc, &anon_vma->head, same_anon_vma) {
863                 struct vm_area_struct *vma = avc->vma;
864                 unsigned long addr = vma_address(page, vma);
865                 BUG_ON(is_vma_temporary_stack(vma));
866                 if (addr == -EFAULT)
867                         continue;
868                 mapcount2 += __split_huge_page_map(page, vma, addr);
869         }
870         if (mapcount != mapcount2)
871                 printk(KERN_ERR "mapcount %d mapcount2 %d page_mapcount %d\n",
872                        mapcount, mapcount2, page_mapcount(page));
873         BUG_ON(mapcount != mapcount2);
874 }
875
876 int split_huge_page(struct page *page)
877 {
878         struct anon_vma *anon_vma;
879         int ret = 1;
880
881         BUG_ON(!PageAnon(page));
882         anon_vma = page_lock_anon_vma(page);
883         if (!anon_vma)
884                 goto out;
885         ret = 0;
886         if (!PageCompound(page))
887                 goto out_unlock;
888
889         BUG_ON(!PageSwapBacked(page));
890         __split_huge_page(page, anon_vma);
891
892         BUG_ON(PageCompound(page));
893 out_unlock:
894         page_unlock_anon_vma(anon_vma);
895 out:
896         return ret;
897 }
898
899 int hugepage_madvise(unsigned long *vm_flags)
900 {
901         /*
902          * Be somewhat over-protective like KSM for now!
903          */
904         if (*vm_flags & (VM_HUGEPAGE | VM_SHARED  | VM_MAYSHARE   |
905                          VM_PFNMAP   | VM_IO      | VM_DONTEXPAND |
906                          VM_RESERVED | VM_HUGETLB | VM_INSERTPAGE |
907                          VM_MIXEDMAP | VM_SAO))
908                 return -EINVAL;
909
910         *vm_flags |= VM_HUGEPAGE;
911
912         return 0;
913 }
914
915 void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd)
916 {
917         struct page *page;
918
919         spin_lock(&mm->page_table_lock);
920         if (unlikely(!pmd_trans_huge(*pmd))) {
921                 spin_unlock(&mm->page_table_lock);
922                 return;
923         }
924         page = pmd_page(*pmd);
925         VM_BUG_ON(!page_count(page));
926         get_page(page);
927         spin_unlock(&mm->page_table_lock);
928
929         split_huge_page(page);
930
931         put_page(page);
932         BUG_ON(pmd_trans_huge(*pmd));
933 }