b8ce6f450956e6e3a9191cbf92afd1d490a32c9d
[linux-2.6.git] / mm / hugetlb.c
1 /*
2  * Generic hugetlb support.
3  * (C) William Irwin, April 2004
4  */
5 #include <linux/list.h>
6 #include <linux/init.h>
7 #include <linux/module.h>
8 #include <linux/mm.h>
9 #include <linux/seq_file.h>
10 #include <linux/sysctl.h>
11 #include <linux/highmem.h>
12 #include <linux/mmu_notifier.h>
13 #include <linux/nodemask.h>
14 #include <linux/pagemap.h>
15 #include <linux/mempolicy.h>
16 #include <linux/cpuset.h>
17 #include <linux/mutex.h>
18 #include <linux/bootmem.h>
19 #include <linux/sysfs.h>
20 #include <linux/slab.h>
21 #include <linux/rmap.h>
22 #include <linux/swap.h>
23 #include <linux/swapops.h>
24
25 #include <asm/page.h>
26 #include <asm/pgtable.h>
27 #include <linux/io.h>
28
29 #include <linux/hugetlb.h>
30 #include <linux/node.h>
31 #include "internal.h"
32
33 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
34 static gfp_t htlb_alloc_mask = GFP_HIGHUSER;
35 unsigned long hugepages_treat_as_movable;
36
37 static int max_hstate;
38 unsigned int default_hstate_idx;
39 struct hstate hstates[HUGE_MAX_HSTATE];
40
41 __initdata LIST_HEAD(huge_boot_pages);
42
43 /* for command line parsing */
44 static struct hstate * __initdata parsed_hstate;
45 static unsigned long __initdata default_hstate_max_huge_pages;
46 static unsigned long __initdata default_hstate_size;
47
48 #define for_each_hstate(h) \
49         for ((h) = hstates; (h) < &hstates[max_hstate]; (h)++)
50
51 /*
52  * Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
53  */
54 static DEFINE_SPINLOCK(hugetlb_lock);
55
56 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
57 {
58         bool free = (spool->count == 0) && (spool->used_hpages == 0);
59
60         spin_unlock(&spool->lock);
61
62         /* If no pages are used, and no other handles to the subpool
63          * remain, free the subpool the subpool remain */
64         if (free)
65                 kfree(spool);
66 }
67
68 struct hugepage_subpool *hugepage_new_subpool(long nr_blocks)
69 {
70         struct hugepage_subpool *spool;
71
72         spool = kmalloc(sizeof(*spool), GFP_KERNEL);
73         if (!spool)
74                 return NULL;
75
76         spin_lock_init(&spool->lock);
77         spool->count = 1;
78         spool->max_hpages = nr_blocks;
79         spool->used_hpages = 0;
80
81         return spool;
82 }
83
84 void hugepage_put_subpool(struct hugepage_subpool *spool)
85 {
86         spin_lock(&spool->lock);
87         BUG_ON(!spool->count);
88         spool->count--;
89         unlock_or_release_subpool(spool);
90 }
91
92 static int hugepage_subpool_get_pages(struct hugepage_subpool *spool,
93                                       long delta)
94 {
95         int ret = 0;
96
97         if (!spool)
98                 return 0;
99
100         spin_lock(&spool->lock);
101         if ((spool->used_hpages + delta) <= spool->max_hpages) {
102                 spool->used_hpages += delta;
103         } else {
104                 ret = -ENOMEM;
105         }
106         spin_unlock(&spool->lock);
107
108         return ret;
109 }
110
111 static void hugepage_subpool_put_pages(struct hugepage_subpool *spool,
112                                        long delta)
113 {
114         if (!spool)
115                 return;
116
117         spin_lock(&spool->lock);
118         spool->used_hpages -= delta;
119         /* If hugetlbfs_put_super couldn't free spool due to
120         * an outstanding quota reference, free it now. */
121         unlock_or_release_subpool(spool);
122 }
123
124 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
125 {
126         return HUGETLBFS_SB(inode->i_sb)->spool;
127 }
128
129 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
130 {
131         return subpool_inode(vma->vm_file->f_dentry->d_inode);
132 }
133
134 /*
135  * Region tracking -- allows tracking of reservations and instantiated pages
136  *                    across the pages in a mapping.
137  *
138  * The region data structures are protected by a combination of the mmap_sem
139  * and the hugetlb_instantion_mutex.  To access or modify a region the caller
140  * must either hold the mmap_sem for write, or the mmap_sem for read and
141  * the hugetlb_instantiation mutex:
142  *
143  *      down_write(&mm->mmap_sem);
144  * or
145  *      down_read(&mm->mmap_sem);
146  *      mutex_lock(&hugetlb_instantiation_mutex);
147  */
148 struct file_region {
149         struct list_head link;
150         long from;
151         long to;
152 };
153
154 static long region_add(struct list_head *head, long f, long t)
155 {
156         struct file_region *rg, *nrg, *trg;
157
158         /* Locate the region we are either in or before. */
159         list_for_each_entry(rg, head, link)
160                 if (f <= rg->to)
161                         break;
162
163         /* Round our left edge to the current segment if it encloses us. */
164         if (f > rg->from)
165                 f = rg->from;
166
167         /* Check for and consume any regions we now overlap with. */
168         nrg = rg;
169         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
170                 if (&rg->link == head)
171                         break;
172                 if (rg->from > t)
173                         break;
174
175                 /* If this area reaches higher then extend our area to
176                  * include it completely.  If this is not the first area
177                  * which we intend to reuse, free it. */
178                 if (rg->to > t)
179                         t = rg->to;
180                 if (rg != nrg) {
181                         list_del(&rg->link);
182                         kfree(rg);
183                 }
184         }
185         nrg->from = f;
186         nrg->to = t;
187         return 0;
188 }
189
190 static long region_chg(struct list_head *head, long f, long t)
191 {
192         struct file_region *rg, *nrg;
193         long chg = 0;
194
195         /* Locate the region we are before or in. */
196         list_for_each_entry(rg, head, link)
197                 if (f <= rg->to)
198                         break;
199
200         /* If we are below the current region then a new region is required.
201          * Subtle, allocate a new region at the position but make it zero
202          * size such that we can guarantee to record the reservation. */
203         if (&rg->link == head || t < rg->from) {
204                 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
205                 if (!nrg)
206                         return -ENOMEM;
207                 nrg->from = f;
208                 nrg->to   = f;
209                 INIT_LIST_HEAD(&nrg->link);
210                 list_add(&nrg->link, rg->link.prev);
211
212                 return t - f;
213         }
214
215         /* Round our left edge to the current segment if it encloses us. */
216         if (f > rg->from)
217                 f = rg->from;
218         chg = t - f;
219
220         /* Check for and consume any regions we now overlap with. */
221         list_for_each_entry(rg, rg->link.prev, link) {
222                 if (&rg->link == head)
223                         break;
224                 if (rg->from > t)
225                         return chg;
226
227                 /* We overlap with this area, if it extends further than
228                  * us then we must extend ourselves.  Account for its
229                  * existing reservation. */
230                 if (rg->to > t) {
231                         chg += rg->to - t;
232                         t = rg->to;
233                 }
234                 chg -= rg->to - rg->from;
235         }
236         return chg;
237 }
238
239 static long region_truncate(struct list_head *head, long end)
240 {
241         struct file_region *rg, *trg;
242         long chg = 0;
243
244         /* Locate the region we are either in or before. */
245         list_for_each_entry(rg, head, link)
246                 if (end <= rg->to)
247                         break;
248         if (&rg->link == head)
249                 return 0;
250
251         /* If we are in the middle of a region then adjust it. */
252         if (end > rg->from) {
253                 chg = rg->to - end;
254                 rg->to = end;
255                 rg = list_entry(rg->link.next, typeof(*rg), link);
256         }
257
258         /* Drop any remaining regions. */
259         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
260                 if (&rg->link == head)
261                         break;
262                 chg += rg->to - rg->from;
263                 list_del(&rg->link);
264                 kfree(rg);
265         }
266         return chg;
267 }
268
269 static long region_count(struct list_head *head, long f, long t)
270 {
271         struct file_region *rg;
272         long chg = 0;
273
274         /* Locate each segment we overlap with, and count that overlap. */
275         list_for_each_entry(rg, head, link) {
276                 int seg_from;
277                 int seg_to;
278
279                 if (rg->to <= f)
280                         continue;
281                 if (rg->from >= t)
282                         break;
283
284                 seg_from = max(rg->from, f);
285                 seg_to = min(rg->to, t);
286
287                 chg += seg_to - seg_from;
288         }
289
290         return chg;
291 }
292
293 /*
294  * Convert the address within this vma to the page offset within
295  * the mapping, in pagecache page units; huge pages here.
296  */
297 static pgoff_t vma_hugecache_offset(struct hstate *h,
298                         struct vm_area_struct *vma, unsigned long address)
299 {
300         return ((address - vma->vm_start) >> huge_page_shift(h)) +
301                         (vma->vm_pgoff >> huge_page_order(h));
302 }
303
304 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
305                                      unsigned long address)
306 {
307         return vma_hugecache_offset(hstate_vma(vma), vma, address);
308 }
309
310 /*
311  * Return the size of the pages allocated when backing a VMA. In the majority
312  * cases this will be same size as used by the page table entries.
313  */
314 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
315 {
316         struct hstate *hstate;
317
318         if (!is_vm_hugetlb_page(vma))
319                 return PAGE_SIZE;
320
321         hstate = hstate_vma(vma);
322
323         return 1UL << (hstate->order + PAGE_SHIFT);
324 }
325 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
326
327 /*
328  * Return the page size being used by the MMU to back a VMA. In the majority
329  * of cases, the page size used by the kernel matches the MMU size. On
330  * architectures where it differs, an architecture-specific version of this
331  * function is required.
332  */
333 #ifndef vma_mmu_pagesize
334 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
335 {
336         return vma_kernel_pagesize(vma);
337 }
338 #endif
339
340 /*
341  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
342  * bits of the reservation map pointer, which are always clear due to
343  * alignment.
344  */
345 #define HPAGE_RESV_OWNER    (1UL << 0)
346 #define HPAGE_RESV_UNMAPPED (1UL << 1)
347 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
348
349 /*
350  * These helpers are used to track how many pages are reserved for
351  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
352  * is guaranteed to have their future faults succeed.
353  *
354  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
355  * the reserve counters are updated with the hugetlb_lock held. It is safe
356  * to reset the VMA at fork() time as it is not in use yet and there is no
357  * chance of the global counters getting corrupted as a result of the values.
358  *
359  * The private mapping reservation is represented in a subtly different
360  * manner to a shared mapping.  A shared mapping has a region map associated
361  * with the underlying file, this region map represents the backing file
362  * pages which have ever had a reservation assigned which this persists even
363  * after the page is instantiated.  A private mapping has a region map
364  * associated with the original mmap which is attached to all VMAs which
365  * reference it, this region map represents those offsets which have consumed
366  * reservation ie. where pages have been instantiated.
367  */
368 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
369 {
370         return (unsigned long)vma->vm_private_data;
371 }
372
373 static void set_vma_private_data(struct vm_area_struct *vma,
374                                                         unsigned long value)
375 {
376         vma->vm_private_data = (void *)value;
377 }
378
379 struct resv_map {
380         struct kref refs;
381         struct list_head regions;
382 };
383
384 static struct resv_map *resv_map_alloc(void)
385 {
386         struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
387         if (!resv_map)
388                 return NULL;
389
390         kref_init(&resv_map->refs);
391         INIT_LIST_HEAD(&resv_map->regions);
392
393         return resv_map;
394 }
395
396 static void resv_map_release(struct kref *ref)
397 {
398         struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
399
400         /* Clear out any active regions before we release the map. */
401         region_truncate(&resv_map->regions, 0);
402         kfree(resv_map);
403 }
404
405 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
406 {
407         VM_BUG_ON(!is_vm_hugetlb_page(vma));
408         if (!(vma->vm_flags & VM_MAYSHARE))
409                 return (struct resv_map *)(get_vma_private_data(vma) &
410                                                         ~HPAGE_RESV_MASK);
411         return NULL;
412 }
413
414 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
415 {
416         VM_BUG_ON(!is_vm_hugetlb_page(vma));
417         VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
418
419         set_vma_private_data(vma, (get_vma_private_data(vma) &
420                                 HPAGE_RESV_MASK) | (unsigned long)map);
421 }
422
423 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
424 {
425         VM_BUG_ON(!is_vm_hugetlb_page(vma));
426         VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
427
428         set_vma_private_data(vma, get_vma_private_data(vma) | flags);
429 }
430
431 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
432 {
433         VM_BUG_ON(!is_vm_hugetlb_page(vma));
434
435         return (get_vma_private_data(vma) & flag) != 0;
436 }
437
438 /* Decrement the reserved pages in the hugepage pool by one */
439 static void decrement_hugepage_resv_vma(struct hstate *h,
440                         struct vm_area_struct *vma)
441 {
442         if (vma->vm_flags & VM_NORESERVE)
443                 return;
444
445         if (vma->vm_flags & VM_MAYSHARE) {
446                 /* Shared mappings always use reserves */
447                 h->resv_huge_pages--;
448         } else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
449                 /*
450                  * Only the process that called mmap() has reserves for
451                  * private mappings.
452                  */
453                 h->resv_huge_pages--;
454         }
455 }
456
457 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
458 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
459 {
460         VM_BUG_ON(!is_vm_hugetlb_page(vma));
461         if (!(vma->vm_flags & VM_MAYSHARE))
462                 vma->vm_private_data = (void *)0;
463 }
464
465 /* Returns true if the VMA has associated reserve pages */
466 static int vma_has_reserves(struct vm_area_struct *vma)
467 {
468         if (vma->vm_flags & VM_MAYSHARE)
469                 return 1;
470         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
471                 return 1;
472         return 0;
473 }
474
475 static void copy_gigantic_page(struct page *dst, struct page *src)
476 {
477         int i;
478         struct hstate *h = page_hstate(src);
479         struct page *dst_base = dst;
480         struct page *src_base = src;
481
482         for (i = 0; i < pages_per_huge_page(h); ) {
483                 cond_resched();
484                 copy_highpage(dst, src);
485
486                 i++;
487                 dst = mem_map_next(dst, dst_base, i);
488                 src = mem_map_next(src, src_base, i);
489         }
490 }
491
492 void copy_huge_page(struct page *dst, struct page *src)
493 {
494         int i;
495         struct hstate *h = page_hstate(src);
496
497         if (unlikely(pages_per_huge_page(h) > MAX_ORDER_NR_PAGES)) {
498                 copy_gigantic_page(dst, src);
499                 return;
500         }
501
502         might_sleep();
503         for (i = 0; i < pages_per_huge_page(h); i++) {
504                 cond_resched();
505                 copy_highpage(dst + i, src + i);
506         }
507 }
508
509 static void enqueue_huge_page(struct hstate *h, struct page *page)
510 {
511         int nid = page_to_nid(page);
512         list_add(&page->lru, &h->hugepage_freelists[nid]);
513         h->free_huge_pages++;
514         h->free_huge_pages_node[nid]++;
515 }
516
517 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
518 {
519         struct page *page;
520
521         if (list_empty(&h->hugepage_freelists[nid]))
522                 return NULL;
523         page = list_entry(h->hugepage_freelists[nid].next, struct page, lru);
524         list_del(&page->lru);
525         set_page_refcounted(page);
526         h->free_huge_pages--;
527         h->free_huge_pages_node[nid]--;
528         return page;
529 }
530
531 static struct page *dequeue_huge_page_vma(struct hstate *h,
532                                 struct vm_area_struct *vma,
533                                 unsigned long address, int avoid_reserve)
534 {
535         struct page *page;
536         struct mempolicy *mpol;
537         nodemask_t *nodemask;
538         struct zonelist *zonelist;
539         struct zone *zone;
540         struct zoneref *z;
541         unsigned int cpuset_mems_cookie;
542
543 retry_cpuset:
544         cpuset_mems_cookie = get_mems_allowed();
545         zonelist = huge_zonelist(vma, address,
546                                         htlb_alloc_mask, &mpol, &nodemask);
547         /*
548          * A child process with MAP_PRIVATE mappings created by their parent
549          * have no page reserves. This check ensures that reservations are
550          * not "stolen". The child may still get SIGKILLed
551          */
552         if (!vma_has_reserves(vma) &&
553                         h->free_huge_pages - h->resv_huge_pages == 0)
554                 goto err;
555
556         /* If reserves cannot be used, ensure enough pages are in the pool */
557         if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
558                 goto err;
559
560         for_each_zone_zonelist_nodemask(zone, z, zonelist,
561                                                 MAX_NR_ZONES - 1, nodemask) {
562                 if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask)) {
563                         page = dequeue_huge_page_node(h, zone_to_nid(zone));
564                         if (page) {
565                                 if (!avoid_reserve)
566                                         decrement_hugepage_resv_vma(h, vma);
567                                 break;
568                         }
569                 }
570         }
571
572         mpol_cond_put(mpol);
573         if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
574                 goto retry_cpuset;
575         return page;
576
577 err:
578         mpol_cond_put(mpol);
579         return NULL;
580 }
581
582 static void update_and_free_page(struct hstate *h, struct page *page)
583 {
584         int i;
585
586         VM_BUG_ON(h->order >= MAX_ORDER);
587
588         h->nr_huge_pages--;
589         h->nr_huge_pages_node[page_to_nid(page)]--;
590         for (i = 0; i < pages_per_huge_page(h); i++) {
591                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
592                                 1 << PG_referenced | 1 << PG_dirty |
593                                 1 << PG_active | 1 << PG_reserved |
594                                 1 << PG_private | 1 << PG_writeback);
595         }
596         set_compound_page_dtor(page, NULL);
597         set_page_refcounted(page);
598         arch_release_hugepage(page);
599         __free_pages(page, huge_page_order(h));
600 }
601
602 struct hstate *size_to_hstate(unsigned long size)
603 {
604         struct hstate *h;
605
606         for_each_hstate(h) {
607                 if (huge_page_size(h) == size)
608                         return h;
609         }
610         return NULL;
611 }
612
613 static void free_huge_page(struct page *page)
614 {
615         /*
616          * Can't pass hstate in here because it is called from the
617          * compound page destructor.
618          */
619         struct hstate *h = page_hstate(page);
620         int nid = page_to_nid(page);
621         struct hugepage_subpool *spool =
622                 (struct hugepage_subpool *)page_private(page);
623
624         set_page_private(page, 0);
625         page->mapping = NULL;
626         BUG_ON(page_count(page));
627         BUG_ON(page_mapcount(page));
628         INIT_LIST_HEAD(&page->lru);
629
630         spin_lock(&hugetlb_lock);
631         if (h->surplus_huge_pages_node[nid] && huge_page_order(h) < MAX_ORDER) {
632                 update_and_free_page(h, page);
633                 h->surplus_huge_pages--;
634                 h->surplus_huge_pages_node[nid]--;
635         } else {
636                 enqueue_huge_page(h, page);
637         }
638         spin_unlock(&hugetlb_lock);
639         hugepage_subpool_put_pages(spool, 1);
640 }
641
642 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
643 {
644         set_compound_page_dtor(page, free_huge_page);
645         spin_lock(&hugetlb_lock);
646         h->nr_huge_pages++;
647         h->nr_huge_pages_node[nid]++;
648         spin_unlock(&hugetlb_lock);
649         put_page(page); /* free it into the hugepage allocator */
650 }
651
652 static void prep_compound_gigantic_page(struct page *page, unsigned long order)
653 {
654         int i;
655         int nr_pages = 1 << order;
656         struct page *p = page + 1;
657
658         /* we rely on prep_new_huge_page to set the destructor */
659         set_compound_order(page, order);
660         __SetPageHead(page);
661         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
662                 __SetPageTail(p);
663                 set_page_count(p, 0);
664                 p->first_page = page;
665         }
666 }
667
668 int PageHuge(struct page *page)
669 {
670         compound_page_dtor *dtor;
671
672         if (!PageCompound(page))
673                 return 0;
674
675         page = compound_head(page);
676         dtor = get_compound_page_dtor(page);
677
678         return dtor == free_huge_page;
679 }
680 EXPORT_SYMBOL_GPL(PageHuge);
681
682 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
683 {
684         struct page *page;
685
686         if (h->order >= MAX_ORDER)
687                 return NULL;
688
689         page = alloc_pages_exact_node(nid,
690                 htlb_alloc_mask|__GFP_COMP|__GFP_THISNODE|
691                                                 __GFP_REPEAT|__GFP_NOWARN,
692                 huge_page_order(h));
693         if (page) {
694                 if (arch_prepare_hugepage(page)) {
695                         __free_pages(page, huge_page_order(h));
696                         return NULL;
697                 }
698                 prep_new_huge_page(h, page, nid);
699         }
700
701         return page;
702 }
703
704 /*
705  * common helper functions for hstate_next_node_to_{alloc|free}.
706  * We may have allocated or freed a huge page based on a different
707  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
708  * be outside of *nodes_allowed.  Ensure that we use an allowed
709  * node for alloc or free.
710  */
711 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
712 {
713         nid = next_node(nid, *nodes_allowed);
714         if (nid == MAX_NUMNODES)
715                 nid = first_node(*nodes_allowed);
716         VM_BUG_ON(nid >= MAX_NUMNODES);
717
718         return nid;
719 }
720
721 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
722 {
723         if (!node_isset(nid, *nodes_allowed))
724                 nid = next_node_allowed(nid, nodes_allowed);
725         return nid;
726 }
727
728 /*
729  * returns the previously saved node ["this node"] from which to
730  * allocate a persistent huge page for the pool and advance the
731  * next node from which to allocate, handling wrap at end of node
732  * mask.
733  */
734 static int hstate_next_node_to_alloc(struct hstate *h,
735                                         nodemask_t *nodes_allowed)
736 {
737         int nid;
738
739         VM_BUG_ON(!nodes_allowed);
740
741         nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
742         h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
743
744         return nid;
745 }
746
747 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
748 {
749         struct page *page;
750         int start_nid;
751         int next_nid;
752         int ret = 0;
753
754         start_nid = hstate_next_node_to_alloc(h, nodes_allowed);
755         next_nid = start_nid;
756
757         do {
758                 page = alloc_fresh_huge_page_node(h, next_nid);
759                 if (page) {
760                         ret = 1;
761                         break;
762                 }
763                 next_nid = hstate_next_node_to_alloc(h, nodes_allowed);
764         } while (next_nid != start_nid);
765
766         if (ret)
767                 count_vm_event(HTLB_BUDDY_PGALLOC);
768         else
769                 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
770
771         return ret;
772 }
773
774 /*
775  * helper for free_pool_huge_page() - return the previously saved
776  * node ["this node"] from which to free a huge page.  Advance the
777  * next node id whether or not we find a free huge page to free so
778  * that the next attempt to free addresses the next node.
779  */
780 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
781 {
782         int nid;
783
784         VM_BUG_ON(!nodes_allowed);
785
786         nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
787         h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
788
789         return nid;
790 }
791
792 /*
793  * Free huge page from pool from next node to free.
794  * Attempt to keep persistent huge pages more or less
795  * balanced over allowed nodes.
796  * Called with hugetlb_lock locked.
797  */
798 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
799                                                          bool acct_surplus)
800 {
801         int start_nid;
802         int next_nid;
803         int ret = 0;
804
805         start_nid = hstate_next_node_to_free(h, nodes_allowed);
806         next_nid = start_nid;
807
808         do {
809                 /*
810                  * If we're returning unused surplus pages, only examine
811                  * nodes with surplus pages.
812                  */
813                 if ((!acct_surplus || h->surplus_huge_pages_node[next_nid]) &&
814                     !list_empty(&h->hugepage_freelists[next_nid])) {
815                         struct page *page =
816                                 list_entry(h->hugepage_freelists[next_nid].next,
817                                           struct page, lru);
818                         list_del(&page->lru);
819                         h->free_huge_pages--;
820                         h->free_huge_pages_node[next_nid]--;
821                         if (acct_surplus) {
822                                 h->surplus_huge_pages--;
823                                 h->surplus_huge_pages_node[next_nid]--;
824                         }
825                         update_and_free_page(h, page);
826                         ret = 1;
827                         break;
828                 }
829                 next_nid = hstate_next_node_to_free(h, nodes_allowed);
830         } while (next_nid != start_nid);
831
832         return ret;
833 }
834
835 static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)
836 {
837         struct page *page;
838         unsigned int r_nid;
839
840         if (h->order >= MAX_ORDER)
841                 return NULL;
842
843         /*
844          * Assume we will successfully allocate the surplus page to
845          * prevent racing processes from causing the surplus to exceed
846          * overcommit
847          *
848          * This however introduces a different race, where a process B
849          * tries to grow the static hugepage pool while alloc_pages() is
850          * called by process A. B will only examine the per-node
851          * counters in determining if surplus huge pages can be
852          * converted to normal huge pages in adjust_pool_surplus(). A
853          * won't be able to increment the per-node counter, until the
854          * lock is dropped by B, but B doesn't drop hugetlb_lock until
855          * no more huge pages can be converted from surplus to normal
856          * state (and doesn't try to convert again). Thus, we have a
857          * case where a surplus huge page exists, the pool is grown, and
858          * the surplus huge page still exists after, even though it
859          * should just have been converted to a normal huge page. This
860          * does not leak memory, though, as the hugepage will be freed
861          * once it is out of use. It also does not allow the counters to
862          * go out of whack in adjust_pool_surplus() as we don't modify
863          * the node values until we've gotten the hugepage and only the
864          * per-node value is checked there.
865          */
866         spin_lock(&hugetlb_lock);
867         if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
868                 spin_unlock(&hugetlb_lock);
869                 return NULL;
870         } else {
871                 h->nr_huge_pages++;
872                 h->surplus_huge_pages++;
873         }
874         spin_unlock(&hugetlb_lock);
875
876         if (nid == NUMA_NO_NODE)
877                 page = alloc_pages(htlb_alloc_mask|__GFP_COMP|
878                                    __GFP_REPEAT|__GFP_NOWARN,
879                                    huge_page_order(h));
880         else
881                 page = alloc_pages_exact_node(nid,
882                         htlb_alloc_mask|__GFP_COMP|__GFP_THISNODE|
883                         __GFP_REPEAT|__GFP_NOWARN, huge_page_order(h));
884
885         if (page && arch_prepare_hugepage(page)) {
886                 __free_pages(page, huge_page_order(h));
887                 page = NULL;
888         }
889
890         spin_lock(&hugetlb_lock);
891         if (page) {
892                 r_nid = page_to_nid(page);
893                 set_compound_page_dtor(page, free_huge_page);
894                 /*
895                  * We incremented the global counters already
896                  */
897                 h->nr_huge_pages_node[r_nid]++;
898                 h->surplus_huge_pages_node[r_nid]++;
899                 __count_vm_event(HTLB_BUDDY_PGALLOC);
900         } else {
901                 h->nr_huge_pages--;
902                 h->surplus_huge_pages--;
903                 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
904         }
905         spin_unlock(&hugetlb_lock);
906
907         return page;
908 }
909
910 /*
911  * This allocation function is useful in the context where vma is irrelevant.
912  * E.g. soft-offlining uses this function because it only cares physical
913  * address of error page.
914  */
915 struct page *alloc_huge_page_node(struct hstate *h, int nid)
916 {
917         struct page *page;
918
919         spin_lock(&hugetlb_lock);
920         page = dequeue_huge_page_node(h, nid);
921         spin_unlock(&hugetlb_lock);
922
923         if (!page)
924                 page = alloc_buddy_huge_page(h, nid);
925
926         return page;
927 }
928
929 /*
930  * Increase the hugetlb pool such that it can accommodate a reservation
931  * of size 'delta'.
932  */
933 static int gather_surplus_pages(struct hstate *h, int delta)
934 {
935         struct list_head surplus_list;
936         struct page *page, *tmp;
937         int ret, i;
938         int needed, allocated;
939         bool alloc_ok = true;
940
941         needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
942         if (needed <= 0) {
943                 h->resv_huge_pages += delta;
944                 return 0;
945         }
946
947         allocated = 0;
948         INIT_LIST_HEAD(&surplus_list);
949
950         ret = -ENOMEM;
951 retry:
952         spin_unlock(&hugetlb_lock);
953         for (i = 0; i < needed; i++) {
954                 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
955                 if (!page) {
956                         alloc_ok = false;
957                         break;
958                 }
959                 list_add(&page->lru, &surplus_list);
960         }
961         allocated += i;
962
963         /*
964          * After retaking hugetlb_lock, we need to recalculate 'needed'
965          * because either resv_huge_pages or free_huge_pages may have changed.
966          */
967         spin_lock(&hugetlb_lock);
968         needed = (h->resv_huge_pages + delta) -
969                         (h->free_huge_pages + allocated);
970         if (needed > 0) {
971                 if (alloc_ok)
972                         goto retry;
973                 /*
974                  * We were not able to allocate enough pages to
975                  * satisfy the entire reservation so we free what
976                  * we've allocated so far.
977                  */
978                 goto free;
979         }
980         /*
981          * The surplus_list now contains _at_least_ the number of extra pages
982          * needed to accommodate the reservation.  Add the appropriate number
983          * of pages to the hugetlb pool and free the extras back to the buddy
984          * allocator.  Commit the entire reservation here to prevent another
985          * process from stealing the pages as they are added to the pool but
986          * before they are reserved.
987          */
988         needed += allocated;
989         h->resv_huge_pages += delta;
990         ret = 0;
991
992         /* Free the needed pages to the hugetlb pool */
993         list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
994                 if ((--needed) < 0)
995                         break;
996                 list_del(&page->lru);
997                 /*
998                  * This page is now managed by the hugetlb allocator and has
999                  * no users -- drop the buddy allocator's reference.
1000                  */
1001                 put_page_testzero(page);
1002                 VM_BUG_ON(page_count(page));
1003                 enqueue_huge_page(h, page);
1004         }
1005 free:
1006         spin_unlock(&hugetlb_lock);
1007
1008         /* Free unnecessary surplus pages to the buddy allocator */
1009         if (!list_empty(&surplus_list)) {
1010                 list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1011                         list_del(&page->lru);
1012                         put_page(page);
1013                 }
1014         }
1015         spin_lock(&hugetlb_lock);
1016
1017         return ret;
1018 }
1019
1020 /*
1021  * When releasing a hugetlb pool reservation, any surplus pages that were
1022  * allocated to satisfy the reservation must be explicitly freed if they were
1023  * never used.
1024  * Called with hugetlb_lock held.
1025  */
1026 static void return_unused_surplus_pages(struct hstate *h,
1027                                         unsigned long unused_resv_pages)
1028 {
1029         unsigned long nr_pages;
1030
1031         /* Uncommit the reservation */
1032         h->resv_huge_pages -= unused_resv_pages;
1033
1034         /* Cannot return gigantic pages currently */
1035         if (h->order >= MAX_ORDER)
1036                 return;
1037
1038         nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1039
1040         /*
1041          * We want to release as many surplus pages as possible, spread
1042          * evenly across all nodes with memory. Iterate across these nodes
1043          * until we can no longer free unreserved surplus pages. This occurs
1044          * when the nodes with surplus pages have no free pages.
1045          * free_pool_huge_page() will balance the the freed pages across the
1046          * on-line nodes with memory and will handle the hstate accounting.
1047          */
1048         while (nr_pages--) {
1049                 if (!free_pool_huge_page(h, &node_states[N_HIGH_MEMORY], 1))
1050                         break;
1051         }
1052 }
1053
1054 /*
1055  * Determine if the huge page at addr within the vma has an associated
1056  * reservation.  Where it does not we will need to logically increase
1057  * reservation and actually increase subpool usage before an allocation
1058  * can occur.  Where any new reservation would be required the
1059  * reservation change is prepared, but not committed.  Once the page
1060  * has been allocated from the subpool and instantiated the change should
1061  * be committed via vma_commit_reservation.  No action is required on
1062  * failure.
1063  */
1064 static long vma_needs_reservation(struct hstate *h,
1065                         struct vm_area_struct *vma, unsigned long addr)
1066 {
1067         struct address_space *mapping = vma->vm_file->f_mapping;
1068         struct inode *inode = mapping->host;
1069
1070         if (vma->vm_flags & VM_MAYSHARE) {
1071                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1072                 return region_chg(&inode->i_mapping->private_list,
1073                                                         idx, idx + 1);
1074
1075         } else if (!is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1076                 return 1;
1077
1078         } else  {
1079                 long err;
1080                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1081                 struct resv_map *reservations = vma_resv_map(vma);
1082
1083                 err = region_chg(&reservations->regions, idx, idx + 1);
1084                 if (err < 0)
1085                         return err;
1086                 return 0;
1087         }
1088 }
1089 static void vma_commit_reservation(struct hstate *h,
1090                         struct vm_area_struct *vma, unsigned long addr)
1091 {
1092         struct address_space *mapping = vma->vm_file->f_mapping;
1093         struct inode *inode = mapping->host;
1094
1095         if (vma->vm_flags & VM_MAYSHARE) {
1096                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1097                 region_add(&inode->i_mapping->private_list, idx, idx + 1);
1098
1099         } else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1100                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1101                 struct resv_map *reservations = vma_resv_map(vma);
1102
1103                 /* Mark this page used in the map. */
1104                 region_add(&reservations->regions, idx, idx + 1);
1105         }
1106 }
1107
1108 static struct page *alloc_huge_page(struct vm_area_struct *vma,
1109                                     unsigned long addr, int avoid_reserve)
1110 {
1111         struct hugepage_subpool *spool = subpool_vma(vma);
1112         struct hstate *h = hstate_vma(vma);
1113         struct page *page;
1114         long chg;
1115
1116         /*
1117          * Processes that did not create the mapping will have no
1118          * reserves and will not have accounted against subpool
1119          * limit. Check that the subpool limit can be made before
1120          * satisfying the allocation MAP_NORESERVE mappings may also
1121          * need pages and subpool limit allocated allocated if no reserve
1122          * mapping overlaps.
1123          */
1124         chg = vma_needs_reservation(h, vma, addr);
1125         if (chg < 0)
1126                 return ERR_PTR(-VM_FAULT_OOM);
1127         if (chg)
1128                 if (hugepage_subpool_get_pages(spool, chg))
1129                         return ERR_PTR(-VM_FAULT_SIGBUS);
1130
1131         spin_lock(&hugetlb_lock);
1132         page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve);
1133         spin_unlock(&hugetlb_lock);
1134
1135         if (!page) {
1136                 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1137                 if (!page) {
1138                         hugepage_subpool_put_pages(spool, chg);
1139                         return ERR_PTR(-VM_FAULT_SIGBUS);
1140                 }
1141         }
1142
1143         set_page_private(page, (unsigned long)spool);
1144
1145         vma_commit_reservation(h, vma, addr);
1146
1147         return page;
1148 }
1149
1150 int __weak alloc_bootmem_huge_page(struct hstate *h)
1151 {
1152         struct huge_bootmem_page *m;
1153         int nr_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
1154
1155         while (nr_nodes) {
1156                 void *addr;
1157
1158                 addr = __alloc_bootmem_node_nopanic(
1159                                 NODE_DATA(hstate_next_node_to_alloc(h,
1160                                                 &node_states[N_HIGH_MEMORY])),
1161                                 huge_page_size(h), huge_page_size(h), 0);
1162
1163                 if (addr) {
1164                         /*
1165                          * Use the beginning of the huge page to store the
1166                          * huge_bootmem_page struct (until gather_bootmem
1167                          * puts them into the mem_map).
1168                          */
1169                         m = addr;
1170                         goto found;
1171                 }
1172                 nr_nodes--;
1173         }
1174         return 0;
1175
1176 found:
1177         BUG_ON((unsigned long)virt_to_phys(m) & (huge_page_size(h) - 1));
1178         /* Put them into a private list first because mem_map is not up yet */
1179         list_add(&m->list, &huge_boot_pages);
1180         m->hstate = h;
1181         return 1;
1182 }
1183
1184 static void prep_compound_huge_page(struct page *page, int order)
1185 {
1186         if (unlikely(order > (MAX_ORDER - 1)))
1187                 prep_compound_gigantic_page(page, order);
1188         else
1189                 prep_compound_page(page, order);
1190 }
1191
1192 /* Put bootmem huge pages into the standard lists after mem_map is up */
1193 static void __init gather_bootmem_prealloc(void)
1194 {
1195         struct huge_bootmem_page *m;
1196
1197         list_for_each_entry(m, &huge_boot_pages, list) {
1198                 struct hstate *h = m->hstate;
1199                 struct page *page;
1200
1201 #ifdef CONFIG_HIGHMEM
1202                 page = pfn_to_page(m->phys >> PAGE_SHIFT);
1203                 free_bootmem_late((unsigned long)m,
1204                                   sizeof(struct huge_bootmem_page));
1205 #else
1206                 page = virt_to_page(m);
1207 #endif
1208                 __ClearPageReserved(page);
1209                 WARN_ON(page_count(page) != 1);
1210                 prep_compound_huge_page(page, h->order);
1211                 prep_new_huge_page(h, page, page_to_nid(page));
1212                 /*
1213                  * If we had gigantic hugepages allocated at boot time, we need
1214                  * to restore the 'stolen' pages to totalram_pages in order to
1215                  * fix confusing memory reports from free(1) and another
1216                  * side-effects, like CommitLimit going negative.
1217                  */
1218                 if (h->order > (MAX_ORDER - 1))
1219                         totalram_pages += 1 << h->order;
1220         }
1221 }
1222
1223 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
1224 {
1225         unsigned long i;
1226
1227         for (i = 0; i < h->max_huge_pages; ++i) {
1228                 if (h->order >= MAX_ORDER) {
1229                         if (!alloc_bootmem_huge_page(h))
1230                                 break;
1231                 } else if (!alloc_fresh_huge_page(h,
1232                                          &node_states[N_HIGH_MEMORY]))
1233                         break;
1234         }
1235         h->max_huge_pages = i;
1236 }
1237
1238 static void __init hugetlb_init_hstates(void)
1239 {
1240         struct hstate *h;
1241
1242         for_each_hstate(h) {
1243                 /* oversize hugepages were init'ed in early boot */
1244                 if (h->order < MAX_ORDER)
1245                         hugetlb_hstate_alloc_pages(h);
1246         }
1247 }
1248
1249 static char * __init memfmt(char *buf, unsigned long n)
1250 {
1251         if (n >= (1UL << 30))
1252                 sprintf(buf, "%lu GB", n >> 30);
1253         else if (n >= (1UL << 20))
1254                 sprintf(buf, "%lu MB", n >> 20);
1255         else
1256                 sprintf(buf, "%lu KB", n >> 10);
1257         return buf;
1258 }
1259
1260 static void __init report_hugepages(void)
1261 {
1262         struct hstate *h;
1263
1264         for_each_hstate(h) {
1265                 char buf[32];
1266                 printk(KERN_INFO "HugeTLB registered %s page size, "
1267                                  "pre-allocated %ld pages\n",
1268                         memfmt(buf, huge_page_size(h)),
1269                         h->free_huge_pages);
1270         }
1271 }
1272
1273 #ifdef CONFIG_HIGHMEM
1274 static void try_to_free_low(struct hstate *h, unsigned long count,
1275                                                 nodemask_t *nodes_allowed)
1276 {
1277         int i;
1278
1279         if (h->order >= MAX_ORDER)
1280                 return;
1281
1282         for_each_node_mask(i, *nodes_allowed) {
1283                 struct page *page, *next;
1284                 struct list_head *freel = &h->hugepage_freelists[i];
1285                 list_for_each_entry_safe(page, next, freel, lru) {
1286                         if (count >= h->nr_huge_pages)
1287                                 return;
1288                         if (PageHighMem(page))
1289                                 continue;
1290                         list_del(&page->lru);
1291                         update_and_free_page(h, page);
1292                         h->free_huge_pages--;
1293                         h->free_huge_pages_node[page_to_nid(page)]--;
1294                 }
1295         }
1296 }
1297 #else
1298 static inline void try_to_free_low(struct hstate *h, unsigned long count,
1299                                                 nodemask_t *nodes_allowed)
1300 {
1301 }
1302 #endif
1303
1304 /*
1305  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
1306  * balanced by operating on them in a round-robin fashion.
1307  * Returns 1 if an adjustment was made.
1308  */
1309 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
1310                                 int delta)
1311 {
1312         int start_nid, next_nid;
1313         int ret = 0;
1314
1315         VM_BUG_ON(delta != -1 && delta != 1);
1316
1317         if (delta < 0)
1318                 start_nid = hstate_next_node_to_alloc(h, nodes_allowed);
1319         else
1320                 start_nid = hstate_next_node_to_free(h, nodes_allowed);
1321         next_nid = start_nid;
1322
1323         do {
1324                 int nid = next_nid;
1325                 if (delta < 0)  {
1326                         /*
1327                          * To shrink on this node, there must be a surplus page
1328                          */
1329                         if (!h->surplus_huge_pages_node[nid]) {
1330                                 next_nid = hstate_next_node_to_alloc(h,
1331                                                                 nodes_allowed);
1332                                 continue;
1333                         }
1334                 }
1335                 if (delta > 0) {
1336                         /*
1337                          * Surplus cannot exceed the total number of pages
1338                          */
1339                         if (h->surplus_huge_pages_node[nid] >=
1340                                                 h->nr_huge_pages_node[nid]) {
1341                                 next_nid = hstate_next_node_to_free(h,
1342                                                                 nodes_allowed);
1343                                 continue;
1344                         }
1345                 }
1346
1347                 h->surplus_huge_pages += delta;
1348                 h->surplus_huge_pages_node[nid] += delta;
1349                 ret = 1;
1350                 break;
1351         } while (next_nid != start_nid);
1352
1353         return ret;
1354 }
1355
1356 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
1357 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
1358                                                 nodemask_t *nodes_allowed)
1359 {
1360         unsigned long min_count, ret;
1361
1362         if (h->order >= MAX_ORDER)
1363                 return h->max_huge_pages;
1364
1365         /*
1366          * Increase the pool size
1367          * First take pages out of surplus state.  Then make up the
1368          * remaining difference by allocating fresh huge pages.
1369          *
1370          * We might race with alloc_buddy_huge_page() here and be unable
1371          * to convert a surplus huge page to a normal huge page. That is
1372          * not critical, though, it just means the overall size of the
1373          * pool might be one hugepage larger than it needs to be, but
1374          * within all the constraints specified by the sysctls.
1375          */
1376         spin_lock(&hugetlb_lock);
1377         while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
1378                 if (!adjust_pool_surplus(h, nodes_allowed, -1))
1379                         break;
1380         }
1381
1382         while (count > persistent_huge_pages(h)) {
1383                 /*
1384                  * If this allocation races such that we no longer need the
1385                  * page, free_huge_page will handle it by freeing the page
1386                  * and reducing the surplus.
1387                  */
1388                 spin_unlock(&hugetlb_lock);
1389                 ret = alloc_fresh_huge_page(h, nodes_allowed);
1390                 spin_lock(&hugetlb_lock);
1391                 if (!ret)
1392                         goto out;
1393
1394                 /* Bail for signals. Probably ctrl-c from user */
1395                 if (signal_pending(current))
1396                         goto out;
1397         }
1398
1399         /*
1400          * Decrease the pool size
1401          * First return free pages to the buddy allocator (being careful
1402          * to keep enough around to satisfy reservations).  Then place
1403          * pages into surplus state as needed so the pool will shrink
1404          * to the desired size as pages become free.
1405          *
1406          * By placing pages into the surplus state independent of the
1407          * overcommit value, we are allowing the surplus pool size to
1408          * exceed overcommit. There are few sane options here. Since
1409          * alloc_buddy_huge_page() is checking the global counter,
1410          * though, we'll note that we're not allowed to exceed surplus
1411          * and won't grow the pool anywhere else. Not until one of the
1412          * sysctls are changed, or the surplus pages go out of use.
1413          */
1414         min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
1415         min_count = max(count, min_count);
1416         try_to_free_low(h, min_count, nodes_allowed);
1417         while (min_count < persistent_huge_pages(h)) {
1418                 if (!free_pool_huge_page(h, nodes_allowed, 0))
1419                         break;
1420         }
1421         while (count < persistent_huge_pages(h)) {
1422                 if (!adjust_pool_surplus(h, nodes_allowed, 1))
1423                         break;
1424         }
1425 out:
1426         ret = persistent_huge_pages(h);
1427         spin_unlock(&hugetlb_lock);
1428         return ret;
1429 }
1430
1431 #define HSTATE_ATTR_RO(_name) \
1432         static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
1433
1434 #define HSTATE_ATTR(_name) \
1435         static struct kobj_attribute _name##_attr = \
1436                 __ATTR(_name, 0644, _name##_show, _name##_store)
1437
1438 static struct kobject *hugepages_kobj;
1439 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
1440
1441 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
1442
1443 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
1444 {
1445         int i;
1446
1447         for (i = 0; i < HUGE_MAX_HSTATE; i++)
1448                 if (hstate_kobjs[i] == kobj) {
1449                         if (nidp)
1450                                 *nidp = NUMA_NO_NODE;
1451                         return &hstates[i];
1452                 }
1453
1454         return kobj_to_node_hstate(kobj, nidp);
1455 }
1456
1457 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
1458                                         struct kobj_attribute *attr, char *buf)
1459 {
1460         struct hstate *h;
1461         unsigned long nr_huge_pages;
1462         int nid;
1463
1464         h = kobj_to_hstate(kobj, &nid);
1465         if (nid == NUMA_NO_NODE)
1466                 nr_huge_pages = h->nr_huge_pages;
1467         else
1468                 nr_huge_pages = h->nr_huge_pages_node[nid];
1469
1470         return sprintf(buf, "%lu\n", nr_huge_pages);
1471 }
1472
1473 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
1474                         struct kobject *kobj, struct kobj_attribute *attr,
1475                         const char *buf, size_t len)
1476 {
1477         int err;
1478         int nid;
1479         unsigned long count;
1480         struct hstate *h;
1481         NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
1482
1483         err = strict_strtoul(buf, 10, &count);
1484         if (err)
1485                 goto out;
1486
1487         h = kobj_to_hstate(kobj, &nid);
1488         if (h->order >= MAX_ORDER) {
1489                 err = -EINVAL;
1490                 goto out;
1491         }
1492
1493         if (nid == NUMA_NO_NODE) {
1494                 /*
1495                  * global hstate attribute
1496                  */
1497                 if (!(obey_mempolicy &&
1498                                 init_nodemask_of_mempolicy(nodes_allowed))) {
1499                         NODEMASK_FREE(nodes_allowed);
1500                         nodes_allowed = &node_states[N_HIGH_MEMORY];
1501                 }
1502         } else if (nodes_allowed) {
1503                 /*
1504                  * per node hstate attribute: adjust count to global,
1505                  * but restrict alloc/free to the specified node.
1506                  */
1507                 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
1508                 init_nodemask_of_node(nodes_allowed, nid);
1509         } else
1510                 nodes_allowed = &node_states[N_HIGH_MEMORY];
1511
1512         h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
1513
1514         if (nodes_allowed != &node_states[N_HIGH_MEMORY])
1515                 NODEMASK_FREE(nodes_allowed);
1516
1517         return len;
1518 out:
1519         NODEMASK_FREE(nodes_allowed);
1520         return err;
1521 }
1522
1523 static ssize_t nr_hugepages_show(struct kobject *kobj,
1524                                        struct kobj_attribute *attr, char *buf)
1525 {
1526         return nr_hugepages_show_common(kobj, attr, buf);
1527 }
1528
1529 static ssize_t nr_hugepages_store(struct kobject *kobj,
1530                struct kobj_attribute *attr, const char *buf, size_t len)
1531 {
1532         return nr_hugepages_store_common(false, kobj, attr, buf, len);
1533 }
1534 HSTATE_ATTR(nr_hugepages);
1535
1536 #ifdef CONFIG_NUMA
1537
1538 /*
1539  * hstate attribute for optionally mempolicy-based constraint on persistent
1540  * huge page alloc/free.
1541  */
1542 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
1543                                        struct kobj_attribute *attr, char *buf)
1544 {
1545         return nr_hugepages_show_common(kobj, attr, buf);
1546 }
1547
1548 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
1549                struct kobj_attribute *attr, const char *buf, size_t len)
1550 {
1551         return nr_hugepages_store_common(true, kobj, attr, buf, len);
1552 }
1553 HSTATE_ATTR(nr_hugepages_mempolicy);
1554 #endif
1555
1556
1557 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
1558                                         struct kobj_attribute *attr, char *buf)
1559 {
1560         struct hstate *h = kobj_to_hstate(kobj, NULL);
1561         return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
1562 }
1563
1564 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
1565                 struct kobj_attribute *attr, const char *buf, size_t count)
1566 {
1567         int err;
1568         unsigned long input;
1569         struct hstate *h = kobj_to_hstate(kobj, NULL);
1570
1571         if (h->order >= MAX_ORDER)
1572                 return -EINVAL;
1573
1574         err = strict_strtoul(buf, 10, &input);
1575         if (err)
1576                 return err;
1577
1578         spin_lock(&hugetlb_lock);
1579         h->nr_overcommit_huge_pages = input;
1580         spin_unlock(&hugetlb_lock);
1581
1582         return count;
1583 }
1584 HSTATE_ATTR(nr_overcommit_hugepages);
1585
1586 static ssize_t free_hugepages_show(struct kobject *kobj,
1587                                         struct kobj_attribute *attr, char *buf)
1588 {
1589         struct hstate *h;
1590         unsigned long free_huge_pages;
1591         int nid;
1592
1593         h = kobj_to_hstate(kobj, &nid);
1594         if (nid == NUMA_NO_NODE)
1595                 free_huge_pages = h->free_huge_pages;
1596         else
1597                 free_huge_pages = h->free_huge_pages_node[nid];
1598
1599         return sprintf(buf, "%lu\n", free_huge_pages);
1600 }
1601 HSTATE_ATTR_RO(free_hugepages);
1602
1603 static ssize_t resv_hugepages_show(struct kobject *kobj,
1604                                         struct kobj_attribute *attr, char *buf)
1605 {
1606         struct hstate *h = kobj_to_hstate(kobj, NULL);
1607         return sprintf(buf, "%lu\n", h->resv_huge_pages);
1608 }
1609 HSTATE_ATTR_RO(resv_hugepages);
1610
1611 static ssize_t surplus_hugepages_show(struct kobject *kobj,
1612                                         struct kobj_attribute *attr, char *buf)
1613 {
1614         struct hstate *h;
1615         unsigned long surplus_huge_pages;
1616         int nid;
1617
1618         h = kobj_to_hstate(kobj, &nid);
1619         if (nid == NUMA_NO_NODE)
1620                 surplus_huge_pages = h->surplus_huge_pages;
1621         else
1622                 surplus_huge_pages = h->surplus_huge_pages_node[nid];
1623
1624         return sprintf(buf, "%lu\n", surplus_huge_pages);
1625 }
1626 HSTATE_ATTR_RO(surplus_hugepages);
1627
1628 static struct attribute *hstate_attrs[] = {
1629         &nr_hugepages_attr.attr,
1630         &nr_overcommit_hugepages_attr.attr,
1631         &free_hugepages_attr.attr,
1632         &resv_hugepages_attr.attr,
1633         &surplus_hugepages_attr.attr,
1634 #ifdef CONFIG_NUMA
1635         &nr_hugepages_mempolicy_attr.attr,
1636 #endif
1637         NULL,
1638 };
1639
1640 static struct attribute_group hstate_attr_group = {
1641         .attrs = hstate_attrs,
1642 };
1643
1644 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
1645                                     struct kobject **hstate_kobjs,
1646                                     struct attribute_group *hstate_attr_group)
1647 {
1648         int retval;
1649         int hi = h - hstates;
1650
1651         hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
1652         if (!hstate_kobjs[hi])
1653                 return -ENOMEM;
1654
1655         retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
1656         if (retval)
1657                 kobject_put(hstate_kobjs[hi]);
1658
1659         return retval;
1660 }
1661
1662 static void __init hugetlb_sysfs_init(void)
1663 {
1664         struct hstate *h;
1665         int err;
1666
1667         hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
1668         if (!hugepages_kobj)
1669                 return;
1670
1671         for_each_hstate(h) {
1672                 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
1673                                          hstate_kobjs, &hstate_attr_group);
1674                 if (err)
1675                         printk(KERN_ERR "Hugetlb: Unable to add hstate %s",
1676                                                                 h->name);
1677         }
1678 }
1679
1680 #ifdef CONFIG_NUMA
1681
1682 /*
1683  * node_hstate/s - associate per node hstate attributes, via their kobjects,
1684  * with node devices in node_devices[] using a parallel array.  The array
1685  * index of a node device or _hstate == node id.
1686  * This is here to avoid any static dependency of the node device driver, in
1687  * the base kernel, on the hugetlb module.
1688  */
1689 struct node_hstate {
1690         struct kobject          *hugepages_kobj;
1691         struct kobject          *hstate_kobjs[HUGE_MAX_HSTATE];
1692 };
1693 struct node_hstate node_hstates[MAX_NUMNODES];
1694
1695 /*
1696  * A subset of global hstate attributes for node devices
1697  */
1698 static struct attribute *per_node_hstate_attrs[] = {
1699         &nr_hugepages_attr.attr,
1700         &free_hugepages_attr.attr,
1701         &surplus_hugepages_attr.attr,
1702         NULL,
1703 };
1704
1705 static struct attribute_group per_node_hstate_attr_group = {
1706         .attrs = per_node_hstate_attrs,
1707 };
1708
1709 /*
1710  * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
1711  * Returns node id via non-NULL nidp.
1712  */
1713 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1714 {
1715         int nid;
1716
1717         for (nid = 0; nid < nr_node_ids; nid++) {
1718                 struct node_hstate *nhs = &node_hstates[nid];
1719                 int i;
1720                 for (i = 0; i < HUGE_MAX_HSTATE; i++)
1721                         if (nhs->hstate_kobjs[i] == kobj) {
1722                                 if (nidp)
1723                                         *nidp = nid;
1724                                 return &hstates[i];
1725                         }
1726         }
1727
1728         BUG();
1729         return NULL;
1730 }
1731
1732 /*
1733  * Unregister hstate attributes from a single node device.
1734  * No-op if no hstate attributes attached.
1735  */
1736 void hugetlb_unregister_node(struct node *node)
1737 {
1738         struct hstate *h;
1739         struct node_hstate *nhs = &node_hstates[node->dev.id];
1740
1741         if (!nhs->hugepages_kobj)
1742                 return;         /* no hstate attributes */
1743
1744         for_each_hstate(h)
1745                 if (nhs->hstate_kobjs[h - hstates]) {
1746                         kobject_put(nhs->hstate_kobjs[h - hstates]);
1747                         nhs->hstate_kobjs[h - hstates] = NULL;
1748                 }
1749
1750         kobject_put(nhs->hugepages_kobj);
1751         nhs->hugepages_kobj = NULL;
1752 }
1753
1754 /*
1755  * hugetlb module exit:  unregister hstate attributes from node devices
1756  * that have them.
1757  */
1758 static void hugetlb_unregister_all_nodes(void)
1759 {
1760         int nid;
1761
1762         /*
1763          * disable node device registrations.
1764          */
1765         register_hugetlbfs_with_node(NULL, NULL);
1766
1767         /*
1768          * remove hstate attributes from any nodes that have them.
1769          */
1770         for (nid = 0; nid < nr_node_ids; nid++)
1771                 hugetlb_unregister_node(&node_devices[nid]);
1772 }
1773
1774 /*
1775  * Register hstate attributes for a single node device.
1776  * No-op if attributes already registered.
1777  */
1778 void hugetlb_register_node(struct node *node)
1779 {
1780         struct hstate *h;
1781         struct node_hstate *nhs = &node_hstates[node->dev.id];
1782         int err;
1783
1784         if (nhs->hugepages_kobj)
1785                 return;         /* already allocated */
1786
1787         nhs->hugepages_kobj = kobject_create_and_add("hugepages",
1788                                                         &node->dev.kobj);
1789         if (!nhs->hugepages_kobj)
1790                 return;
1791
1792         for_each_hstate(h) {
1793                 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
1794                                                 nhs->hstate_kobjs,
1795                                                 &per_node_hstate_attr_group);
1796                 if (err) {
1797                         printk(KERN_ERR "Hugetlb: Unable to add hstate %s"
1798                                         " for node %d\n",
1799                                                 h->name, node->dev.id);
1800                         hugetlb_unregister_node(node);
1801                         break;
1802                 }
1803         }
1804 }
1805
1806 /*
1807  * hugetlb init time:  register hstate attributes for all registered node
1808  * devices of nodes that have memory.  All on-line nodes should have
1809  * registered their associated device by this time.
1810  */
1811 static void hugetlb_register_all_nodes(void)
1812 {
1813         int nid;
1814
1815         for_each_node_state(nid, N_HIGH_MEMORY) {
1816                 struct node *node = &node_devices[nid];
1817                 if (node->dev.id == nid)
1818                         hugetlb_register_node(node);
1819         }
1820
1821         /*
1822          * Let the node device driver know we're here so it can
1823          * [un]register hstate attributes on node hotplug.
1824          */
1825         register_hugetlbfs_with_node(hugetlb_register_node,
1826                                      hugetlb_unregister_node);
1827 }
1828 #else   /* !CONFIG_NUMA */
1829
1830 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1831 {
1832         BUG();
1833         if (nidp)
1834                 *nidp = -1;
1835         return NULL;
1836 }
1837
1838 static void hugetlb_unregister_all_nodes(void) { }
1839
1840 static void hugetlb_register_all_nodes(void) { }
1841
1842 #endif
1843
1844 static void __exit hugetlb_exit(void)
1845 {
1846         struct hstate *h;
1847
1848         hugetlb_unregister_all_nodes();
1849
1850         for_each_hstate(h) {
1851                 kobject_put(hstate_kobjs[h - hstates]);
1852         }
1853
1854         kobject_put(hugepages_kobj);
1855 }
1856 module_exit(hugetlb_exit);
1857
1858 static int __init hugetlb_init(void)
1859 {
1860         /* Some platform decide whether they support huge pages at boot
1861          * time. On these, such as powerpc, HPAGE_SHIFT is set to 0 when
1862          * there is no such support
1863          */
1864         if (HPAGE_SHIFT == 0)
1865                 return 0;
1866
1867         if (!size_to_hstate(default_hstate_size)) {
1868                 default_hstate_size = HPAGE_SIZE;
1869                 if (!size_to_hstate(default_hstate_size))
1870                         hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
1871         }
1872         default_hstate_idx = size_to_hstate(default_hstate_size) - hstates;
1873         if (default_hstate_max_huge_pages)
1874                 default_hstate.max_huge_pages = default_hstate_max_huge_pages;
1875
1876         hugetlb_init_hstates();
1877
1878         gather_bootmem_prealloc();
1879
1880         report_hugepages();
1881
1882         hugetlb_sysfs_init();
1883
1884         hugetlb_register_all_nodes();
1885
1886         return 0;
1887 }
1888 module_init(hugetlb_init);
1889
1890 /* Should be called on processing a hugepagesz=... option */
1891 void __init hugetlb_add_hstate(unsigned order)
1892 {
1893         struct hstate *h;
1894         unsigned long i;
1895
1896         if (size_to_hstate(PAGE_SIZE << order)) {
1897                 printk(KERN_WARNING "hugepagesz= specified twice, ignoring\n");
1898                 return;
1899         }
1900         BUG_ON(max_hstate >= HUGE_MAX_HSTATE);
1901         BUG_ON(order == 0);
1902         h = &hstates[max_hstate++];
1903         h->order = order;
1904         h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
1905         h->nr_huge_pages = 0;
1906         h->free_huge_pages = 0;
1907         for (i = 0; i < MAX_NUMNODES; ++i)
1908                 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
1909         h->next_nid_to_alloc = first_node(node_states[N_HIGH_MEMORY]);
1910         h->next_nid_to_free = first_node(node_states[N_HIGH_MEMORY]);
1911         snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
1912                                         huge_page_size(h)/1024);
1913
1914         parsed_hstate = h;
1915 }
1916
1917 static int __init hugetlb_nrpages_setup(char *s)
1918 {
1919         unsigned long *mhp;
1920         static unsigned long *last_mhp;
1921
1922         /*
1923          * !max_hstate means we haven't parsed a hugepagesz= parameter yet,
1924          * so this hugepages= parameter goes to the "default hstate".
1925          */
1926         if (!max_hstate)
1927                 mhp = &default_hstate_max_huge_pages;
1928         else
1929                 mhp = &parsed_hstate->max_huge_pages;
1930
1931         if (mhp == last_mhp) {
1932                 printk(KERN_WARNING "hugepages= specified twice without "
1933                         "interleaving hugepagesz=, ignoring\n");
1934                 return 1;
1935         }
1936
1937         if (sscanf(s, "%lu", mhp) <= 0)
1938                 *mhp = 0;
1939
1940         /*
1941          * Global state is always initialized later in hugetlb_init.
1942          * But we need to allocate >= MAX_ORDER hstates here early to still
1943          * use the bootmem allocator.
1944          */
1945         if (max_hstate && parsed_hstate->order >= MAX_ORDER)
1946                 hugetlb_hstate_alloc_pages(parsed_hstate);
1947
1948         last_mhp = mhp;
1949
1950         return 1;
1951 }
1952 __setup("hugepages=", hugetlb_nrpages_setup);
1953
1954 static int __init hugetlb_default_setup(char *s)
1955 {
1956         default_hstate_size = memparse(s, &s);
1957         return 1;
1958 }
1959 __setup("default_hugepagesz=", hugetlb_default_setup);
1960
1961 static unsigned int cpuset_mems_nr(unsigned int *array)
1962 {
1963         int node;
1964         unsigned int nr = 0;
1965
1966         for_each_node_mask(node, cpuset_current_mems_allowed)
1967                 nr += array[node];
1968
1969         return nr;
1970 }
1971
1972 #ifdef CONFIG_SYSCTL
1973 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
1974                          struct ctl_table *table, int write,
1975                          void __user *buffer, size_t *length, loff_t *ppos)
1976 {
1977         struct hstate *h = &default_hstate;
1978         unsigned long tmp;
1979         int ret;
1980
1981         tmp = h->max_huge_pages;
1982
1983         if (write && h->order >= MAX_ORDER)
1984                 return -EINVAL;
1985
1986         table->data = &tmp;
1987         table->maxlen = sizeof(unsigned long);
1988         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
1989         if (ret)
1990                 goto out;
1991
1992         if (write) {
1993                 NODEMASK_ALLOC(nodemask_t, nodes_allowed,
1994                                                 GFP_KERNEL | __GFP_NORETRY);
1995                 if (!(obey_mempolicy &&
1996                                init_nodemask_of_mempolicy(nodes_allowed))) {
1997                         NODEMASK_FREE(nodes_allowed);
1998                         nodes_allowed = &node_states[N_HIGH_MEMORY];
1999                 }
2000                 h->max_huge_pages = set_max_huge_pages(h, tmp, nodes_allowed);
2001
2002                 if (nodes_allowed != &node_states[N_HIGH_MEMORY])
2003                         NODEMASK_FREE(nodes_allowed);
2004         }
2005 out:
2006         return ret;
2007 }
2008
2009 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2010                           void __user *buffer, size_t *length, loff_t *ppos)
2011 {
2012
2013         return hugetlb_sysctl_handler_common(false, table, write,
2014                                                         buffer, length, ppos);
2015 }
2016
2017 #ifdef CONFIG_NUMA
2018 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2019                           void __user *buffer, size_t *length, loff_t *ppos)
2020 {
2021         return hugetlb_sysctl_handler_common(true, table, write,
2022                                                         buffer, length, ppos);
2023 }
2024 #endif /* CONFIG_NUMA */
2025
2026 int hugetlb_treat_movable_handler(struct ctl_table *table, int write,
2027                         void __user *buffer,
2028                         size_t *length, loff_t *ppos)
2029 {
2030         proc_dointvec(table, write, buffer, length, ppos);
2031         if (hugepages_treat_as_movable)
2032                 htlb_alloc_mask = GFP_HIGHUSER_MOVABLE;
2033         else
2034                 htlb_alloc_mask = GFP_HIGHUSER;
2035         return 0;
2036 }
2037
2038 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
2039                         void __user *buffer,
2040                         size_t *length, loff_t *ppos)
2041 {
2042         struct hstate *h = &default_hstate;
2043         unsigned long tmp;
2044         int ret;
2045
2046         tmp = h->nr_overcommit_huge_pages;
2047
2048         if (write && h->order >= MAX_ORDER)
2049                 return -EINVAL;
2050
2051         table->data = &tmp;
2052         table->maxlen = sizeof(unsigned long);
2053         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2054         if (ret)
2055                 goto out;
2056
2057         if (write) {
2058                 spin_lock(&hugetlb_lock);
2059                 h->nr_overcommit_huge_pages = tmp;
2060                 spin_unlock(&hugetlb_lock);
2061         }
2062 out:
2063         return ret;
2064 }
2065
2066 #endif /* CONFIG_SYSCTL */
2067
2068 void hugetlb_report_meminfo(struct seq_file *m)
2069 {
2070         struct hstate *h = &default_hstate;
2071         seq_printf(m,
2072                         "HugePages_Total:   %5lu\n"
2073                         "HugePages_Free:    %5lu\n"
2074                         "HugePages_Rsvd:    %5lu\n"
2075                         "HugePages_Surp:    %5lu\n"
2076                         "Hugepagesize:   %8lu kB\n",
2077                         h->nr_huge_pages,
2078                         h->free_huge_pages,
2079                         h->resv_huge_pages,
2080                         h->surplus_huge_pages,
2081                         1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2082 }
2083
2084 int hugetlb_report_node_meminfo(int nid, char *buf)
2085 {
2086         struct hstate *h = &default_hstate;
2087         return sprintf(buf,
2088                 "Node %d HugePages_Total: %5u\n"
2089                 "Node %d HugePages_Free:  %5u\n"
2090                 "Node %d HugePages_Surp:  %5u\n",
2091                 nid, h->nr_huge_pages_node[nid],
2092                 nid, h->free_huge_pages_node[nid],
2093                 nid, h->surplus_huge_pages_node[nid]);
2094 }
2095
2096 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
2097 unsigned long hugetlb_total_pages(void)
2098 {
2099         struct hstate *h = &default_hstate;
2100         return h->nr_huge_pages * pages_per_huge_page(h);
2101 }
2102
2103 static int hugetlb_acct_memory(struct hstate *h, long delta)
2104 {
2105         int ret = -ENOMEM;
2106
2107         spin_lock(&hugetlb_lock);
2108         /*
2109          * When cpuset is configured, it breaks the strict hugetlb page
2110          * reservation as the accounting is done on a global variable. Such
2111          * reservation is completely rubbish in the presence of cpuset because
2112          * the reservation is not checked against page availability for the
2113          * current cpuset. Application can still potentially OOM'ed by kernel
2114          * with lack of free htlb page in cpuset that the task is in.
2115          * Attempt to enforce strict accounting with cpuset is almost
2116          * impossible (or too ugly) because cpuset is too fluid that
2117          * task or memory node can be dynamically moved between cpusets.
2118          *
2119          * The change of semantics for shared hugetlb mapping with cpuset is
2120          * undesirable. However, in order to preserve some of the semantics,
2121          * we fall back to check against current free page availability as
2122          * a best attempt and hopefully to minimize the impact of changing
2123          * semantics that cpuset has.
2124          */
2125         if (delta > 0) {
2126                 if (gather_surplus_pages(h, delta) < 0)
2127                         goto out;
2128
2129                 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
2130                         return_unused_surplus_pages(h, delta);
2131                         goto out;
2132                 }
2133         }
2134
2135         ret = 0;
2136         if (delta < 0)
2137                 return_unused_surplus_pages(h, (unsigned long) -delta);
2138
2139 out:
2140         spin_unlock(&hugetlb_lock);
2141         return ret;
2142 }
2143
2144 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
2145 {
2146         struct resv_map *reservations = vma_resv_map(vma);
2147
2148         /*
2149          * This new VMA should share its siblings reservation map if present.
2150          * The VMA will only ever have a valid reservation map pointer where
2151          * it is being copied for another still existing VMA.  As that VMA
2152          * has a reference to the reservation map it cannot disappear until
2153          * after this open call completes.  It is therefore safe to take a
2154          * new reference here without additional locking.
2155          */
2156         if (reservations)
2157                 kref_get(&reservations->refs);
2158 }
2159
2160 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
2161 {
2162         struct hstate *h = hstate_vma(vma);
2163         struct resv_map *reservations = vma_resv_map(vma);
2164         struct hugepage_subpool *spool = subpool_vma(vma);
2165         unsigned long reserve;
2166         unsigned long start;
2167         unsigned long end;
2168
2169         if (reservations) {
2170                 start = vma_hugecache_offset(h, vma, vma->vm_start);
2171                 end = vma_hugecache_offset(h, vma, vma->vm_end);
2172
2173                 reserve = (end - start) -
2174                         region_count(&reservations->regions, start, end);
2175
2176                 kref_put(&reservations->refs, resv_map_release);
2177
2178                 if (reserve) {
2179                         hugetlb_acct_memory(h, -reserve);
2180                         hugepage_subpool_put_pages(spool, reserve);
2181                 }
2182         }
2183 }
2184
2185 /*
2186  * We cannot handle pagefaults against hugetlb pages at all.  They cause
2187  * handle_mm_fault() to try to instantiate regular-sized pages in the
2188  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
2189  * this far.
2190  */
2191 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2192 {
2193         BUG();
2194         return 0;
2195 }
2196
2197 const struct vm_operations_struct hugetlb_vm_ops = {
2198         .fault = hugetlb_vm_op_fault,
2199         .open = hugetlb_vm_op_open,
2200         .close = hugetlb_vm_op_close,
2201 };
2202
2203 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
2204                                 int writable)
2205 {
2206         pte_t entry;
2207
2208         if (writable) {
2209                 entry =
2210                     pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
2211         } else {
2212                 entry = huge_pte_wrprotect(mk_pte(page, vma->vm_page_prot));
2213         }
2214         entry = pte_mkyoung(entry);
2215         entry = pte_mkhuge(entry);
2216
2217         return entry;
2218 }
2219
2220 static void set_huge_ptep_writable(struct vm_area_struct *vma,
2221                                    unsigned long address, pte_t *ptep)
2222 {
2223         pte_t entry;
2224
2225         entry = pte_mkwrite(pte_mkdirty(huge_ptep_get(ptep)));
2226         if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
2227                 update_mmu_cache(vma, address, ptep);
2228 }
2229
2230
2231 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
2232                             struct vm_area_struct *vma)
2233 {
2234         pte_t *src_pte, *dst_pte, entry;
2235         struct page *ptepage;
2236         unsigned long addr;
2237         int cow;
2238         struct hstate *h = hstate_vma(vma);
2239         unsigned long sz = huge_page_size(h);
2240
2241         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
2242
2243         for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
2244                 src_pte = huge_pte_offset(src, addr);
2245                 if (!src_pte)
2246                         continue;
2247                 dst_pte = huge_pte_alloc(dst, addr, sz);
2248                 if (!dst_pte)
2249                         goto nomem;
2250
2251                 /* If the pagetables are shared don't copy or take references */
2252                 if (dst_pte == src_pte)
2253                         continue;
2254
2255                 spin_lock(&dst->page_table_lock);
2256                 spin_lock_nested(&src->page_table_lock, SINGLE_DEPTH_NESTING);
2257                 if (!huge_pte_none(huge_ptep_get(src_pte))) {
2258                         if (cow)
2259                                 huge_ptep_set_wrprotect(src, addr, src_pte);
2260                         entry = huge_ptep_get(src_pte);
2261                         ptepage = pte_page(entry);
2262                         get_page(ptepage);
2263                         page_dup_rmap(ptepage);
2264                         set_huge_pte_at(dst, addr, dst_pte, entry);
2265                 }
2266                 spin_unlock(&src->page_table_lock);
2267                 spin_unlock(&dst->page_table_lock);
2268         }
2269         return 0;
2270
2271 nomem:
2272         return -ENOMEM;
2273 }
2274
2275 static int is_hugetlb_entry_migration(pte_t pte)
2276 {
2277         swp_entry_t swp;
2278
2279         if (huge_pte_none(pte) || pte_present(pte))
2280                 return 0;
2281         swp = pte_to_swp_entry(pte);
2282         if (non_swap_entry(swp) && is_migration_entry(swp))
2283                 return 1;
2284         else
2285                 return 0;
2286 }
2287
2288 static int is_hugetlb_entry_hwpoisoned(pte_t pte)
2289 {
2290         swp_entry_t swp;
2291
2292         if (huge_pte_none(pte) || pte_present(pte))
2293                 return 0;
2294         swp = pte_to_swp_entry(pte);
2295         if (non_swap_entry(swp) && is_hwpoison_entry(swp))
2296                 return 1;
2297         else
2298                 return 0;
2299 }
2300
2301 void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2302                             unsigned long end, struct page *ref_page)
2303 {
2304         struct mm_struct *mm = vma->vm_mm;
2305         unsigned long address;
2306         pte_t *ptep;
2307         pte_t pte;
2308         struct page *page;
2309         struct page *tmp;
2310         struct hstate *h = hstate_vma(vma);
2311         unsigned long sz = huge_page_size(h);
2312
2313         /*
2314          * A page gathering list, protected by per file i_mmap_mutex. The
2315          * lock is used to avoid list corruption from multiple unmapping
2316          * of the same page since we are using page->lru.
2317          */
2318         LIST_HEAD(page_list);
2319
2320         WARN_ON(!is_vm_hugetlb_page(vma));
2321         BUG_ON(start & ~huge_page_mask(h));
2322         BUG_ON(end & ~huge_page_mask(h));
2323
2324         mmu_notifier_invalidate_range_start(mm, start, end);
2325         spin_lock(&mm->page_table_lock);
2326         for (address = start; address < end; address += sz) {
2327                 ptep = huge_pte_offset(mm, address);
2328                 if (!ptep)
2329                         continue;
2330
2331                 if (huge_pmd_unshare(mm, &address, ptep))
2332                         continue;
2333
2334                 pte = huge_ptep_get(ptep);
2335                 if (huge_pte_none(pte))
2336                         continue;
2337
2338                 /*
2339                  * HWPoisoned hugepage is already unmapped and dropped reference
2340                  */
2341                 if (unlikely(is_hugetlb_entry_hwpoisoned(pte)))
2342                         continue;
2343
2344                 page = pte_page(pte);
2345                 /*
2346                  * If a reference page is supplied, it is because a specific
2347                  * page is being unmapped, not a range. Ensure the page we
2348                  * are about to unmap is the actual page of interest.
2349                  */
2350                 if (ref_page) {
2351                         if (page != ref_page)
2352                                 continue;
2353
2354                         /*
2355                          * Mark the VMA as having unmapped its page so that
2356                          * future faults in this VMA will fail rather than
2357                          * looking like data was lost
2358                          */
2359                         set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
2360                 }
2361
2362                 pte = huge_ptep_get_and_clear(mm, address, ptep);
2363                 if (pte_dirty(pte))
2364                         set_page_dirty(page);
2365                 list_add(&page->lru, &page_list);
2366
2367                 /* Bail out after unmapping reference page if supplied */
2368                 if (ref_page)
2369                         break;
2370         }
2371         flush_tlb_range(vma, start, end);
2372         spin_unlock(&mm->page_table_lock);
2373         mmu_notifier_invalidate_range_end(mm, start, end);
2374         list_for_each_entry_safe(page, tmp, &page_list, lru) {
2375                 page_remove_rmap(page);
2376                 list_del(&page->lru);
2377                 put_page(page);
2378         }
2379 }
2380
2381 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2382                           unsigned long end, struct page *ref_page)
2383 {
2384         mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex);
2385         __unmap_hugepage_range(vma, start, end, ref_page);
2386         mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
2387 }
2388
2389 /*
2390  * This is called when the original mapper is failing to COW a MAP_PRIVATE
2391  * mappping it owns the reserve page for. The intention is to unmap the page
2392  * from other VMAs and let the children be SIGKILLed if they are faulting the
2393  * same region.
2394  */
2395 static int unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
2396                                 struct page *page, unsigned long address)
2397 {
2398         struct hstate *h = hstate_vma(vma);
2399         struct vm_area_struct *iter_vma;
2400         struct address_space *mapping;
2401         struct prio_tree_iter iter;
2402         pgoff_t pgoff;
2403
2404         /*
2405          * vm_pgoff is in PAGE_SIZE units, hence the different calculation
2406          * from page cache lookup which is in HPAGE_SIZE units.
2407          */
2408         address = address & huge_page_mask(h);
2409         pgoff = vma_hugecache_offset(h, vma, address);
2410         mapping = vma->vm_file->f_dentry->d_inode->i_mapping;
2411
2412         /*
2413          * Take the mapping lock for the duration of the table walk. As
2414          * this mapping should be shared between all the VMAs,
2415          * __unmap_hugepage_range() is called as the lock is already held
2416          */
2417         mutex_lock(&mapping->i_mmap_mutex);
2418         vma_prio_tree_foreach(iter_vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
2419                 /* Do not unmap the current VMA */
2420                 if (iter_vma == vma)
2421                         continue;
2422
2423                 /*
2424                  * Unmap the page from other VMAs without their own reserves.
2425                  * They get marked to be SIGKILLed if they fault in these
2426                  * areas. This is because a future no-page fault on this VMA
2427                  * could insert a zeroed page instead of the data existing
2428                  * from the time of fork. This would look like data corruption
2429                  */
2430                 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
2431                         __unmap_hugepage_range(iter_vma,
2432                                 address, address + huge_page_size(h),
2433                                 page);
2434         }
2435         mutex_unlock(&mapping->i_mmap_mutex);
2436
2437         return 1;
2438 }
2439
2440 /*
2441  * Hugetlb_cow() should be called with page lock of the original hugepage held.
2442  * Called with hugetlb_instantiation_mutex held and pte_page locked so we
2443  * cannot race with other handlers or page migration.
2444  * Keep the pte_same checks anyway to make transition from the mutex easier.
2445  */
2446 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
2447                         unsigned long address, pte_t *ptep, pte_t pte,
2448                         struct page *pagecache_page)
2449 {
2450         struct hstate *h = hstate_vma(vma);
2451         struct page *old_page, *new_page;
2452         int avoidcopy;
2453         int outside_reserve = 0;
2454
2455         old_page = pte_page(pte);
2456
2457 retry_avoidcopy:
2458         /* If no-one else is actually using this page, avoid the copy
2459          * and just make the page writable */
2460         avoidcopy = (page_mapcount(old_page) == 1);
2461         if (avoidcopy) {
2462                 if (PageAnon(old_page))
2463                         page_move_anon_rmap(old_page, vma, address);
2464                 set_huge_ptep_writable(vma, address, ptep);
2465                 return 0;
2466         }
2467
2468         /*
2469          * If the process that created a MAP_PRIVATE mapping is about to
2470          * perform a COW due to a shared page count, attempt to satisfy
2471          * the allocation without using the existing reserves. The pagecache
2472          * page is used to determine if the reserve at this address was
2473          * consumed or not. If reserves were used, a partial faulted mapping
2474          * at the time of fork() could consume its reserves on COW instead
2475          * of the full address range.
2476          */
2477         if (!(vma->vm_flags & VM_MAYSHARE) &&
2478                         is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
2479                         old_page != pagecache_page)
2480                 outside_reserve = 1;
2481
2482         page_cache_get(old_page);
2483
2484         /* Drop page_table_lock as buddy allocator may be called */
2485         spin_unlock(&mm->page_table_lock);
2486         new_page = alloc_huge_page(vma, address, outside_reserve);
2487
2488         if (IS_ERR(new_page)) {
2489                 page_cache_release(old_page);
2490
2491                 /*
2492                  * If a process owning a MAP_PRIVATE mapping fails to COW,
2493                  * it is due to references held by a child and an insufficient
2494                  * huge page pool. To guarantee the original mappers
2495                  * reliability, unmap the page from child processes. The child
2496                  * may get SIGKILLed if it later faults.
2497                  */
2498                 if (outside_reserve) {
2499                         BUG_ON(huge_pte_none(pte));
2500                         if (unmap_ref_private(mm, vma, old_page, address)) {
2501                                 BUG_ON(page_count(old_page) != 1);
2502                                 BUG_ON(huge_pte_none(pte));
2503                                 spin_lock(&mm->page_table_lock);
2504                                 ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2505                                 if (likely(pte_same(huge_ptep_get(ptep), pte)))
2506                                         goto retry_avoidcopy;
2507                                 /*
2508                                  * race occurs while re-acquiring page_table_lock, and
2509                                  * our job is done.
2510                                  */
2511                                 return 0;
2512                         }
2513                         WARN_ON_ONCE(1);
2514                 }
2515
2516                 /* Caller expects lock to be held */
2517                 spin_lock(&mm->page_table_lock);
2518                 return -PTR_ERR(new_page);
2519         }
2520
2521         /*
2522          * When the original hugepage is shared one, it does not have
2523          * anon_vma prepared.
2524          */
2525         if (unlikely(anon_vma_prepare(vma))) {
2526                 page_cache_release(new_page);
2527                 page_cache_release(old_page);
2528                 /* Caller expects lock to be held */
2529                 spin_lock(&mm->page_table_lock);
2530                 return VM_FAULT_OOM;
2531         }
2532
2533         copy_user_huge_page(new_page, old_page, address, vma,
2534                             pages_per_huge_page(h));
2535         __SetPageUptodate(new_page);
2536
2537         /*
2538          * Retake the page_table_lock to check for racing updates
2539          * before the page tables are altered
2540          */
2541         spin_lock(&mm->page_table_lock);
2542         ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2543         if (likely(pte_same(huge_ptep_get(ptep), pte))) {
2544                 /* Break COW */
2545                 mmu_notifier_invalidate_range_start(mm,
2546                         address & huge_page_mask(h),
2547                         (address & huge_page_mask(h)) + huge_page_size(h));
2548                 huge_ptep_clear_flush(vma, address, ptep);
2549                 set_huge_pte_at(mm, address, ptep,
2550                                 make_huge_pte(vma, new_page, 1));
2551                 page_remove_rmap(old_page);
2552                 hugepage_add_new_anon_rmap(new_page, vma, address);
2553                 /* Make the old page be freed below */
2554                 new_page = old_page;
2555                 mmu_notifier_invalidate_range_end(mm,
2556                         address & huge_page_mask(h),
2557                         (address & huge_page_mask(h)) + huge_page_size(h));
2558         }
2559         page_cache_release(new_page);
2560         page_cache_release(old_page);
2561         return 0;
2562 }
2563
2564 /* Return the pagecache page at a given address within a VMA */
2565 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
2566                         struct vm_area_struct *vma, unsigned long address)
2567 {
2568         struct address_space *mapping;
2569         pgoff_t idx;
2570
2571         mapping = vma->vm_file->f_mapping;
2572         idx = vma_hugecache_offset(h, vma, address);
2573
2574         return find_lock_page(mapping, idx);
2575 }
2576
2577 /*
2578  * Return whether there is a pagecache page to back given address within VMA.
2579  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
2580  */
2581 static bool hugetlbfs_pagecache_present(struct hstate *h,
2582                         struct vm_area_struct *vma, unsigned long address)
2583 {
2584         struct address_space *mapping;
2585         pgoff_t idx;
2586         struct page *page;
2587
2588         mapping = vma->vm_file->f_mapping;
2589         idx = vma_hugecache_offset(h, vma, address);
2590
2591         page = find_get_page(mapping, idx);
2592         if (page)
2593                 put_page(page);
2594         return page != NULL;
2595 }
2596
2597 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
2598                         unsigned long address, pte_t *ptep, unsigned int flags)
2599 {
2600         struct hstate *h = hstate_vma(vma);
2601         int ret = VM_FAULT_SIGBUS;
2602         int anon_rmap = 0;
2603         pgoff_t idx;
2604         unsigned long size;
2605         struct page *page;
2606         struct address_space *mapping;
2607         pte_t new_pte;
2608
2609         /*
2610          * Currently, we are forced to kill the process in the event the
2611          * original mapper has unmapped pages from the child due to a failed
2612          * COW. Warn that such a situation has occurred as it may not be obvious
2613          */
2614         if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
2615                 printk(KERN_WARNING
2616                         "PID %d killed due to inadequate hugepage pool\n",
2617                         current->pid);
2618                 return ret;
2619         }
2620
2621         mapping = vma->vm_file->f_mapping;
2622         idx = vma_hugecache_offset(h, vma, address);
2623
2624         /*
2625          * Use page lock to guard against racing truncation
2626          * before we get page_table_lock.
2627          */
2628 retry:
2629         page = find_lock_page(mapping, idx);
2630         if (!page) {
2631                 size = i_size_read(mapping->host) >> huge_page_shift(h);
2632                 if (idx >= size)
2633                         goto out;
2634                 page = alloc_huge_page(vma, address, 0);
2635                 if (IS_ERR(page)) {
2636                         ret = -PTR_ERR(page);
2637                         goto out;
2638                 }
2639                 clear_huge_page(page, address, pages_per_huge_page(h));
2640                 __SetPageUptodate(page);
2641
2642                 if (vma->vm_flags & VM_MAYSHARE) {
2643                         int err;
2644                         struct inode *inode = mapping->host;
2645
2646                         err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
2647                         if (err) {
2648                                 put_page(page);
2649                                 if (err == -EEXIST)
2650                                         goto retry;
2651                                 goto out;
2652                         }
2653
2654                         spin_lock(&inode->i_lock);
2655                         inode->i_blocks += blocks_per_huge_page(h);
2656                         spin_unlock(&inode->i_lock);
2657                 } else {
2658                         lock_page(page);
2659                         if (unlikely(anon_vma_prepare(vma))) {
2660                                 ret = VM_FAULT_OOM;
2661                                 goto backout_unlocked;
2662                         }
2663                         anon_rmap = 1;
2664                 }
2665         } else {
2666                 /*
2667                  * If memory error occurs between mmap() and fault, some process
2668                  * don't have hwpoisoned swap entry for errored virtual address.
2669                  * So we need to block hugepage fault by PG_hwpoison bit check.
2670                  */
2671                 if (unlikely(PageHWPoison(page))) {
2672                         ret = VM_FAULT_HWPOISON |
2673                               VM_FAULT_SET_HINDEX(h - hstates);
2674                         goto backout_unlocked;
2675                 }
2676         }
2677
2678         /*
2679          * If we are going to COW a private mapping later, we examine the
2680          * pending reservations for this page now. This will ensure that
2681          * any allocations necessary to record that reservation occur outside
2682          * the spinlock.
2683          */
2684         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
2685                 if (vma_needs_reservation(h, vma, address) < 0) {
2686                         ret = VM_FAULT_OOM;
2687                         goto backout_unlocked;
2688                 }
2689
2690         spin_lock(&mm->page_table_lock);
2691         size = i_size_read(mapping->host) >> huge_page_shift(h);
2692         if (idx >= size)
2693                 goto backout;
2694
2695         ret = 0;
2696         if (!huge_pte_none(huge_ptep_get(ptep)))
2697                 goto backout;
2698
2699         if (anon_rmap)
2700                 hugepage_add_new_anon_rmap(page, vma, address);
2701         else
2702                 page_dup_rmap(page);
2703         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
2704                                 && (vma->vm_flags & VM_SHARED)));
2705         set_huge_pte_at(mm, address, ptep, new_pte);
2706
2707         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
2708                 /* Optimization, do the COW without a second fault */
2709                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page);
2710         }
2711
2712         spin_unlock(&mm->page_table_lock);
2713         unlock_page(page);
2714 out:
2715         return ret;
2716
2717 backout:
2718         spin_unlock(&mm->page_table_lock);
2719 backout_unlocked:
2720         unlock_page(page);
2721         put_page(page);
2722         goto out;
2723 }
2724
2725 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
2726                         unsigned long address, unsigned int flags)
2727 {
2728         pte_t *ptep;
2729         pte_t entry;
2730         int ret;
2731         struct page *page = NULL;
2732         struct page *pagecache_page = NULL;
2733         static DEFINE_MUTEX(hugetlb_instantiation_mutex);
2734         struct hstate *h = hstate_vma(vma);
2735
2736         address &= huge_page_mask(h);
2737
2738         ptep = huge_pte_offset(mm, address);
2739         if (ptep) {
2740                 entry = huge_ptep_get(ptep);
2741                 if (unlikely(is_hugetlb_entry_migration(entry))) {
2742                         migration_entry_wait(mm, (pmd_t *)ptep, address);
2743                         return 0;
2744                 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
2745                         return VM_FAULT_HWPOISON_LARGE |
2746                                VM_FAULT_SET_HINDEX(h - hstates);
2747         }
2748
2749         ptep = huge_pte_alloc(mm, address, huge_page_size(h));
2750         if (!ptep)
2751                 return VM_FAULT_OOM;
2752
2753         /*
2754          * Serialize hugepage allocation and instantiation, so that we don't
2755          * get spurious allocation failures if two CPUs race to instantiate
2756          * the same page in the page cache.
2757          */
2758         mutex_lock(&hugetlb_instantiation_mutex);
2759         entry = huge_ptep_get(ptep);
2760         if (huge_pte_none(entry)) {
2761                 ret = hugetlb_no_page(mm, vma, address, ptep, flags);
2762                 goto out_mutex;
2763         }
2764
2765         ret = 0;
2766
2767         /*
2768          * If we are going to COW the mapping later, we examine the pending
2769          * reservations for this page now. This will ensure that any
2770          * allocations necessary to record that reservation occur outside the
2771          * spinlock. For private mappings, we also lookup the pagecache
2772          * page now as it is used to determine if a reservation has been
2773          * consumed.
2774          */
2775         if ((flags & FAULT_FLAG_WRITE) && !pte_write(entry)) {
2776                 if (vma_needs_reservation(h, vma, address) < 0) {
2777                         ret = VM_FAULT_OOM;
2778                         goto out_mutex;
2779                 }
2780
2781                 if (!(vma->vm_flags & VM_MAYSHARE))
2782                         pagecache_page = hugetlbfs_pagecache_page(h,
2783                                                                 vma, address);
2784         }
2785
2786         /*
2787          * hugetlb_cow() requires page locks of pte_page(entry) and
2788          * pagecache_page, so here we need take the former one
2789          * when page != pagecache_page or !pagecache_page.
2790          * Note that locking order is always pagecache_page -> page,
2791          * so no worry about deadlock.
2792          */
2793         page = pte_page(entry);
2794         if (page != pagecache_page)
2795                 lock_page(page);
2796
2797         spin_lock(&mm->page_table_lock);
2798         /* Check for a racing update before calling hugetlb_cow */
2799         if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
2800                 goto out_page_table_lock;
2801
2802
2803         if (flags & FAULT_FLAG_WRITE) {
2804                 if (!pte_write(entry)) {
2805                         ret = hugetlb_cow(mm, vma, address, ptep, entry,
2806                                                         pagecache_page);
2807                         goto out_page_table_lock;
2808                 }
2809                 entry = pte_mkdirty(entry);
2810         }
2811         entry = pte_mkyoung(entry);
2812         if (huge_ptep_set_access_flags(vma, address, ptep, entry,
2813                                                 flags & FAULT_FLAG_WRITE))
2814                 update_mmu_cache(vma, address, ptep);
2815
2816 out_page_table_lock:
2817         spin_unlock(&mm->page_table_lock);
2818
2819         if (pagecache_page) {
2820                 unlock_page(pagecache_page);
2821                 put_page(pagecache_page);
2822         }
2823         if (page != pagecache_page)
2824                 unlock_page(page);
2825
2826 out_mutex:
2827         mutex_unlock(&hugetlb_instantiation_mutex);
2828
2829         return ret;
2830 }
2831
2832 /* Can be overriden by architectures */
2833 __attribute__((weak)) struct page *
2834 follow_huge_pud(struct mm_struct *mm, unsigned long address,
2835                pud_t *pud, int write)
2836 {
2837         BUG();
2838         return NULL;
2839 }
2840
2841 int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
2842                         struct page **pages, struct vm_area_struct **vmas,
2843                         unsigned long *position, int *length, int i,
2844                         unsigned int flags)
2845 {
2846         unsigned long pfn_offset;
2847         unsigned long vaddr = *position;
2848         int remainder = *length;
2849         struct hstate *h = hstate_vma(vma);
2850
2851         spin_lock(&mm->page_table_lock);
2852         while (vaddr < vma->vm_end && remainder) {
2853                 pte_t *pte;
2854                 int absent;
2855                 struct page *page;
2856
2857                 /*
2858                  * Some archs (sparc64, sh*) have multiple pte_ts to
2859                  * each hugepage.  We have to make sure we get the
2860                  * first, for the page indexing below to work.
2861                  */
2862                 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
2863                 absent = !pte || huge_pte_none(huge_ptep_get(pte));
2864
2865                 /*
2866                  * When coredumping, it suits get_dump_page if we just return
2867                  * an error where there's an empty slot with no huge pagecache
2868                  * to back it.  This way, we avoid allocating a hugepage, and
2869                  * the sparse dumpfile avoids allocating disk blocks, but its
2870                  * huge holes still show up with zeroes where they need to be.
2871                  */
2872                 if (absent && (flags & FOLL_DUMP) &&
2873                     !hugetlbfs_pagecache_present(h, vma, vaddr)) {
2874                         remainder = 0;
2875                         break;
2876                 }
2877
2878                 if (absent ||
2879                     ((flags & FOLL_WRITE) && !pte_write(huge_ptep_get(pte)))) {
2880                         int ret;
2881
2882                         spin_unlock(&mm->page_table_lock);
2883                         ret = hugetlb_fault(mm, vma, vaddr,
2884                                 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
2885                         spin_lock(&mm->page_table_lock);
2886                         if (!(ret & VM_FAULT_ERROR))
2887                                 continue;
2888
2889                         remainder = 0;
2890                         break;
2891                 }
2892
2893                 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
2894                 page = pte_page(huge_ptep_get(pte));
2895 same_page:
2896                 if (pages) {
2897                         pages[i] = mem_map_offset(page, pfn_offset);
2898                         get_page(pages[i]);
2899                 }
2900
2901                 if (vmas)
2902                         vmas[i] = vma;
2903
2904                 vaddr += PAGE_SIZE;
2905                 ++pfn_offset;
2906                 --remainder;
2907                 ++i;
2908                 if (vaddr < vma->vm_end && remainder &&
2909                                 pfn_offset < pages_per_huge_page(h)) {
2910                         /*
2911                          * We use pfn_offset to avoid touching the pageframes
2912                          * of this compound page.
2913                          */
2914                         goto same_page;
2915                 }
2916         }
2917         spin_unlock(&mm->page_table_lock);
2918         *length = remainder;
2919         *position = vaddr;
2920
2921         return i ? i : -EFAULT;
2922 }
2923
2924 void hugetlb_change_protection(struct vm_area_struct *vma,
2925                 unsigned long address, unsigned long end, pgprot_t newprot)
2926 {
2927         struct mm_struct *mm = vma->vm_mm;
2928         unsigned long start = address;
2929         pte_t *ptep;
2930         pte_t pte;
2931         struct hstate *h = hstate_vma(vma);
2932
2933         BUG_ON(address >= end);
2934         flush_cache_range(vma, address, end);
2935
2936         mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex);
2937         spin_lock(&mm->page_table_lock);
2938         for (; address < end; address += huge_page_size(h)) {
2939                 ptep = huge_pte_offset(mm, address);
2940                 if (!ptep)
2941                         continue;
2942                 if (huge_pmd_unshare(mm, &address, ptep))
2943                         continue;
2944                 if (!huge_pte_none(huge_ptep_get(ptep))) {
2945                         pte = huge_ptep_get_and_clear(mm, address, ptep);
2946                         pte = pte_mkhuge(pte_modify(pte, newprot));
2947                         set_huge_pte_at(mm, address, ptep, pte);
2948                 }
2949         }
2950         spin_unlock(&mm->page_table_lock);
2951         mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
2952
2953         flush_tlb_range(vma, start, end);
2954 }
2955
2956 int hugetlb_reserve_pages(struct inode *inode,
2957                                         long from, long to,
2958                                         struct vm_area_struct *vma,
2959                                         vm_flags_t vm_flags)
2960 {
2961         long ret, chg;
2962         struct hstate *h = hstate_inode(inode);
2963         struct hugepage_subpool *spool = subpool_inode(inode);
2964
2965         /*
2966          * Only apply hugepage reservation if asked. At fault time, an
2967          * attempt will be made for VM_NORESERVE to allocate a page
2968          * without using reserves
2969          */
2970         if (vm_flags & VM_NORESERVE)
2971                 return 0;
2972
2973         /*
2974          * Shared mappings base their reservation on the number of pages that
2975          * are already allocated on behalf of the file. Private mappings need
2976          * to reserve the full area even if read-only as mprotect() may be
2977          * called to make the mapping read-write. Assume !vma is a shm mapping
2978          */
2979         if (!vma || vma->vm_flags & VM_MAYSHARE)
2980                 chg = region_chg(&inode->i_mapping->private_list, from, to);
2981         else {
2982                 struct resv_map *resv_map = resv_map_alloc();
2983                 if (!resv_map)
2984                         return -ENOMEM;
2985
2986                 chg = to - from;
2987
2988                 set_vma_resv_map(vma, resv_map);
2989                 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
2990         }
2991
2992         if (chg < 0)
2993                 return chg;
2994
2995         /* There must be enough pages in the subpool for the mapping */
2996         if (hugepage_subpool_get_pages(spool, chg))
2997                 return -ENOSPC;
2998
2999         /*
3000          * Check enough hugepages are available for the reservation.
3001          * Hand the pages back to the subpool if there are not
3002          */
3003         ret = hugetlb_acct_memory(h, chg);
3004         if (ret < 0) {
3005                 hugepage_subpool_put_pages(spool, chg);
3006                 return ret;
3007         }
3008
3009         /*
3010          * Account for the reservations made. Shared mappings record regions
3011          * that have reservations as they are shared by multiple VMAs.
3012          * When the last VMA disappears, the region map says how much
3013          * the reservation was and the page cache tells how much of
3014          * the reservation was consumed. Private mappings are per-VMA and
3015          * only the consumed reservations are tracked. When the VMA
3016          * disappears, the original reservation is the VMA size and the
3017          * consumed reservations are stored in the map. Hence, nothing
3018          * else has to be done for private mappings here
3019          */
3020         if (!vma || vma->vm_flags & VM_MAYSHARE)
3021                 region_add(&inode->i_mapping->private_list, from, to);
3022         return 0;
3023 }
3024
3025 void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
3026 {
3027         struct hstate *h = hstate_inode(inode);
3028         long chg = region_truncate(&inode->i_mapping->private_list, offset);
3029         struct hugepage_subpool *spool = subpool_inode(inode);
3030
3031         spin_lock(&inode->i_lock);
3032         inode->i_blocks -= (blocks_per_huge_page(h) * freed);
3033         spin_unlock(&inode->i_lock);
3034
3035         hugepage_subpool_put_pages(spool, (chg - freed));
3036         hugetlb_acct_memory(h, -(chg - freed));
3037 }
3038
3039 #ifdef CONFIG_MEMORY_FAILURE
3040
3041 /* Should be called in hugetlb_lock */
3042 static int is_hugepage_on_freelist(struct page *hpage)
3043 {
3044         struct page *page;
3045         struct page *tmp;
3046         struct hstate *h = page_hstate(hpage);
3047         int nid = page_to_nid(hpage);
3048
3049         list_for_each_entry_safe(page, tmp, &h->hugepage_freelists[nid], lru)
3050                 if (page == hpage)
3051                         return 1;
3052         return 0;
3053 }
3054
3055 /*
3056  * This function is called from memory failure code.
3057  * Assume the caller holds page lock of the head page.
3058  */
3059 int dequeue_hwpoisoned_huge_page(struct page *hpage)
3060 {
3061         struct hstate *h = page_hstate(hpage);
3062         int nid = page_to_nid(hpage);
3063         int ret = -EBUSY;
3064
3065         spin_lock(&hugetlb_lock);
3066         if (is_hugepage_on_freelist(hpage)) {
3067                 list_del(&hpage->lru);
3068                 set_page_refcounted(hpage);
3069                 h->free_huge_pages--;
3070                 h->free_huge_pages_node[nid]--;
3071                 ret = 0;
3072         }
3073         spin_unlock(&hugetlb_lock);
3074         return ret;
3075 }
3076 #endif