HWPOISON, hugetlb: isolate corrupted hugepage
[linux-2.6.git] / mm / hugetlb.c
1 /*
2  * Generic hugetlb support.
3  * (C) William Irwin, April 2004
4  */
5 #include <linux/list.h>
6 #include <linux/init.h>
7 #include <linux/module.h>
8 #include <linux/mm.h>
9 #include <linux/seq_file.h>
10 #include <linux/sysctl.h>
11 #include <linux/highmem.h>
12 #include <linux/mmu_notifier.h>
13 #include <linux/nodemask.h>
14 #include <linux/pagemap.h>
15 #include <linux/mempolicy.h>
16 #include <linux/cpuset.h>
17 #include <linux/mutex.h>
18 #include <linux/bootmem.h>
19 #include <linux/sysfs.h>
20 #include <linux/slab.h>
21 #include <linux/rmap.h>
22
23 #include <asm/page.h>
24 #include <asm/pgtable.h>
25 #include <asm/io.h>
26
27 #include <linux/hugetlb.h>
28 #include <linux/node.h>
29 #include "internal.h"
30
31 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
32 static gfp_t htlb_alloc_mask = GFP_HIGHUSER;
33 unsigned long hugepages_treat_as_movable;
34
35 static int max_hstate;
36 unsigned int default_hstate_idx;
37 struct hstate hstates[HUGE_MAX_HSTATE];
38
39 __initdata LIST_HEAD(huge_boot_pages);
40
41 /* for command line parsing */
42 static struct hstate * __initdata parsed_hstate;
43 static unsigned long __initdata default_hstate_max_huge_pages;
44 static unsigned long __initdata default_hstate_size;
45
46 #define for_each_hstate(h) \
47         for ((h) = hstates; (h) < &hstates[max_hstate]; (h)++)
48
49 /*
50  * Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
51  */
52 static DEFINE_SPINLOCK(hugetlb_lock);
53
54 /*
55  * Region tracking -- allows tracking of reservations and instantiated pages
56  *                    across the pages in a mapping.
57  *
58  * The region data structures are protected by a combination of the mmap_sem
59  * and the hugetlb_instantion_mutex.  To access or modify a region the caller
60  * must either hold the mmap_sem for write, or the mmap_sem for read and
61  * the hugetlb_instantiation mutex:
62  *
63  *      down_write(&mm->mmap_sem);
64  * or
65  *      down_read(&mm->mmap_sem);
66  *      mutex_lock(&hugetlb_instantiation_mutex);
67  */
68 struct file_region {
69         struct list_head link;
70         long from;
71         long to;
72 };
73
74 static long region_add(struct list_head *head, long f, long t)
75 {
76         struct file_region *rg, *nrg, *trg;
77
78         /* Locate the region we are either in or before. */
79         list_for_each_entry(rg, head, link)
80                 if (f <= rg->to)
81                         break;
82
83         /* Round our left edge to the current segment if it encloses us. */
84         if (f > rg->from)
85                 f = rg->from;
86
87         /* Check for and consume any regions we now overlap with. */
88         nrg = rg;
89         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
90                 if (&rg->link == head)
91                         break;
92                 if (rg->from > t)
93                         break;
94
95                 /* If this area reaches higher then extend our area to
96                  * include it completely.  If this is not the first area
97                  * which we intend to reuse, free it. */
98                 if (rg->to > t)
99                         t = rg->to;
100                 if (rg != nrg) {
101                         list_del(&rg->link);
102                         kfree(rg);
103                 }
104         }
105         nrg->from = f;
106         nrg->to = t;
107         return 0;
108 }
109
110 static long region_chg(struct list_head *head, long f, long t)
111 {
112         struct file_region *rg, *nrg;
113         long chg = 0;
114
115         /* Locate the region we are before or in. */
116         list_for_each_entry(rg, head, link)
117                 if (f <= rg->to)
118                         break;
119
120         /* If we are below the current region then a new region is required.
121          * Subtle, allocate a new region at the position but make it zero
122          * size such that we can guarantee to record the reservation. */
123         if (&rg->link == head || t < rg->from) {
124                 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
125                 if (!nrg)
126                         return -ENOMEM;
127                 nrg->from = f;
128                 nrg->to   = f;
129                 INIT_LIST_HEAD(&nrg->link);
130                 list_add(&nrg->link, rg->link.prev);
131
132                 return t - f;
133         }
134
135         /* Round our left edge to the current segment if it encloses us. */
136         if (f > rg->from)
137                 f = rg->from;
138         chg = t - f;
139
140         /* Check for and consume any regions we now overlap with. */
141         list_for_each_entry(rg, rg->link.prev, link) {
142                 if (&rg->link == head)
143                         break;
144                 if (rg->from > t)
145                         return chg;
146
147                 /* We overlap with this area, if it extends futher than
148                  * us then we must extend ourselves.  Account for its
149                  * existing reservation. */
150                 if (rg->to > t) {
151                         chg += rg->to - t;
152                         t = rg->to;
153                 }
154                 chg -= rg->to - rg->from;
155         }
156         return chg;
157 }
158
159 static long region_truncate(struct list_head *head, long end)
160 {
161         struct file_region *rg, *trg;
162         long chg = 0;
163
164         /* Locate the region we are either in or before. */
165         list_for_each_entry(rg, head, link)
166                 if (end <= rg->to)
167                         break;
168         if (&rg->link == head)
169                 return 0;
170
171         /* If we are in the middle of a region then adjust it. */
172         if (end > rg->from) {
173                 chg = rg->to - end;
174                 rg->to = end;
175                 rg = list_entry(rg->link.next, typeof(*rg), link);
176         }
177
178         /* Drop any remaining regions. */
179         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
180                 if (&rg->link == head)
181                         break;
182                 chg += rg->to - rg->from;
183                 list_del(&rg->link);
184                 kfree(rg);
185         }
186         return chg;
187 }
188
189 static long region_count(struct list_head *head, long f, long t)
190 {
191         struct file_region *rg;
192         long chg = 0;
193
194         /* Locate each segment we overlap with, and count that overlap. */
195         list_for_each_entry(rg, head, link) {
196                 int seg_from;
197                 int seg_to;
198
199                 if (rg->to <= f)
200                         continue;
201                 if (rg->from >= t)
202                         break;
203
204                 seg_from = max(rg->from, f);
205                 seg_to = min(rg->to, t);
206
207                 chg += seg_to - seg_from;
208         }
209
210         return chg;
211 }
212
213 /*
214  * Convert the address within this vma to the page offset within
215  * the mapping, in pagecache page units; huge pages here.
216  */
217 static pgoff_t vma_hugecache_offset(struct hstate *h,
218                         struct vm_area_struct *vma, unsigned long address)
219 {
220         return ((address - vma->vm_start) >> huge_page_shift(h)) +
221                         (vma->vm_pgoff >> huge_page_order(h));
222 }
223
224 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
225                                      unsigned long address)
226 {
227         return vma_hugecache_offset(hstate_vma(vma), vma, address);
228 }
229
230 /*
231  * Return the size of the pages allocated when backing a VMA. In the majority
232  * cases this will be same size as used by the page table entries.
233  */
234 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
235 {
236         struct hstate *hstate;
237
238         if (!is_vm_hugetlb_page(vma))
239                 return PAGE_SIZE;
240
241         hstate = hstate_vma(vma);
242
243         return 1UL << (hstate->order + PAGE_SHIFT);
244 }
245 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
246
247 /*
248  * Return the page size being used by the MMU to back a VMA. In the majority
249  * of cases, the page size used by the kernel matches the MMU size. On
250  * architectures where it differs, an architecture-specific version of this
251  * function is required.
252  */
253 #ifndef vma_mmu_pagesize
254 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
255 {
256         return vma_kernel_pagesize(vma);
257 }
258 #endif
259
260 /*
261  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
262  * bits of the reservation map pointer, which are always clear due to
263  * alignment.
264  */
265 #define HPAGE_RESV_OWNER    (1UL << 0)
266 #define HPAGE_RESV_UNMAPPED (1UL << 1)
267 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
268
269 /*
270  * These helpers are used to track how many pages are reserved for
271  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
272  * is guaranteed to have their future faults succeed.
273  *
274  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
275  * the reserve counters are updated with the hugetlb_lock held. It is safe
276  * to reset the VMA at fork() time as it is not in use yet and there is no
277  * chance of the global counters getting corrupted as a result of the values.
278  *
279  * The private mapping reservation is represented in a subtly different
280  * manner to a shared mapping.  A shared mapping has a region map associated
281  * with the underlying file, this region map represents the backing file
282  * pages which have ever had a reservation assigned which this persists even
283  * after the page is instantiated.  A private mapping has a region map
284  * associated with the original mmap which is attached to all VMAs which
285  * reference it, this region map represents those offsets which have consumed
286  * reservation ie. where pages have been instantiated.
287  */
288 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
289 {
290         return (unsigned long)vma->vm_private_data;
291 }
292
293 static void set_vma_private_data(struct vm_area_struct *vma,
294                                                         unsigned long value)
295 {
296         vma->vm_private_data = (void *)value;
297 }
298
299 struct resv_map {
300         struct kref refs;
301         struct list_head regions;
302 };
303
304 static struct resv_map *resv_map_alloc(void)
305 {
306         struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
307         if (!resv_map)
308                 return NULL;
309
310         kref_init(&resv_map->refs);
311         INIT_LIST_HEAD(&resv_map->regions);
312
313         return resv_map;
314 }
315
316 static void resv_map_release(struct kref *ref)
317 {
318         struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
319
320         /* Clear out any active regions before we release the map. */
321         region_truncate(&resv_map->regions, 0);
322         kfree(resv_map);
323 }
324
325 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
326 {
327         VM_BUG_ON(!is_vm_hugetlb_page(vma));
328         if (!(vma->vm_flags & VM_MAYSHARE))
329                 return (struct resv_map *)(get_vma_private_data(vma) &
330                                                         ~HPAGE_RESV_MASK);
331         return NULL;
332 }
333
334 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
335 {
336         VM_BUG_ON(!is_vm_hugetlb_page(vma));
337         VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
338
339         set_vma_private_data(vma, (get_vma_private_data(vma) &
340                                 HPAGE_RESV_MASK) | (unsigned long)map);
341 }
342
343 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
344 {
345         VM_BUG_ON(!is_vm_hugetlb_page(vma));
346         VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
347
348         set_vma_private_data(vma, get_vma_private_data(vma) | flags);
349 }
350
351 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
352 {
353         VM_BUG_ON(!is_vm_hugetlb_page(vma));
354
355         return (get_vma_private_data(vma) & flag) != 0;
356 }
357
358 /* Decrement the reserved pages in the hugepage pool by one */
359 static void decrement_hugepage_resv_vma(struct hstate *h,
360                         struct vm_area_struct *vma)
361 {
362         if (vma->vm_flags & VM_NORESERVE)
363                 return;
364
365         if (vma->vm_flags & VM_MAYSHARE) {
366                 /* Shared mappings always use reserves */
367                 h->resv_huge_pages--;
368         } else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
369                 /*
370                  * Only the process that called mmap() has reserves for
371                  * private mappings.
372                  */
373                 h->resv_huge_pages--;
374         }
375 }
376
377 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
378 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
379 {
380         VM_BUG_ON(!is_vm_hugetlb_page(vma));
381         if (!(vma->vm_flags & VM_MAYSHARE))
382                 vma->vm_private_data = (void *)0;
383 }
384
385 /* Returns true if the VMA has associated reserve pages */
386 static int vma_has_reserves(struct vm_area_struct *vma)
387 {
388         if (vma->vm_flags & VM_MAYSHARE)
389                 return 1;
390         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
391                 return 1;
392         return 0;
393 }
394
395 static void clear_gigantic_page(struct page *page,
396                         unsigned long addr, unsigned long sz)
397 {
398         int i;
399         struct page *p = page;
400
401         might_sleep();
402         for (i = 0; i < sz/PAGE_SIZE; i++, p = mem_map_next(p, page, i)) {
403                 cond_resched();
404                 clear_user_highpage(p, addr + i * PAGE_SIZE);
405         }
406 }
407 static void clear_huge_page(struct page *page,
408                         unsigned long addr, unsigned long sz)
409 {
410         int i;
411
412         if (unlikely(sz/PAGE_SIZE > MAX_ORDER_NR_PAGES)) {
413                 clear_gigantic_page(page, addr, sz);
414                 return;
415         }
416
417         might_sleep();
418         for (i = 0; i < sz/PAGE_SIZE; i++) {
419                 cond_resched();
420                 clear_user_highpage(page + i, addr + i * PAGE_SIZE);
421         }
422 }
423
424 static void copy_gigantic_page(struct page *dst, struct page *src,
425                            unsigned long addr, struct vm_area_struct *vma)
426 {
427         int i;
428         struct hstate *h = hstate_vma(vma);
429         struct page *dst_base = dst;
430         struct page *src_base = src;
431         might_sleep();
432         for (i = 0; i < pages_per_huge_page(h); ) {
433                 cond_resched();
434                 copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma);
435
436                 i++;
437                 dst = mem_map_next(dst, dst_base, i);
438                 src = mem_map_next(src, src_base, i);
439         }
440 }
441 static void copy_huge_page(struct page *dst, struct page *src,
442                            unsigned long addr, struct vm_area_struct *vma)
443 {
444         int i;
445         struct hstate *h = hstate_vma(vma);
446
447         if (unlikely(pages_per_huge_page(h) > MAX_ORDER_NR_PAGES)) {
448                 copy_gigantic_page(dst, src, addr, vma);
449                 return;
450         }
451
452         might_sleep();
453         for (i = 0; i < pages_per_huge_page(h); i++) {
454                 cond_resched();
455                 copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE, vma);
456         }
457 }
458
459 static void enqueue_huge_page(struct hstate *h, struct page *page)
460 {
461         int nid = page_to_nid(page);
462         list_add(&page->lru, &h->hugepage_freelists[nid]);
463         h->free_huge_pages++;
464         h->free_huge_pages_node[nid]++;
465 }
466
467 static struct page *dequeue_huge_page_vma(struct hstate *h,
468                                 struct vm_area_struct *vma,
469                                 unsigned long address, int avoid_reserve)
470 {
471         int nid;
472         struct page *page = NULL;
473         struct mempolicy *mpol;
474         nodemask_t *nodemask;
475         struct zonelist *zonelist;
476         struct zone *zone;
477         struct zoneref *z;
478
479         get_mems_allowed();
480         zonelist = huge_zonelist(vma, address,
481                                         htlb_alloc_mask, &mpol, &nodemask);
482         /*
483          * A child process with MAP_PRIVATE mappings created by their parent
484          * have no page reserves. This check ensures that reservations are
485          * not "stolen". The child may still get SIGKILLed
486          */
487         if (!vma_has_reserves(vma) &&
488                         h->free_huge_pages - h->resv_huge_pages == 0)
489                 goto err;
490
491         /* If reserves cannot be used, ensure enough pages are in the pool */
492         if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
493                 goto err;;
494
495         for_each_zone_zonelist_nodemask(zone, z, zonelist,
496                                                 MAX_NR_ZONES - 1, nodemask) {
497                 nid = zone_to_nid(zone);
498                 if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask) &&
499                     !list_empty(&h->hugepage_freelists[nid])) {
500                         page = list_entry(h->hugepage_freelists[nid].next,
501                                           struct page, lru);
502                         list_del(&page->lru);
503                         h->free_huge_pages--;
504                         h->free_huge_pages_node[nid]--;
505
506                         if (!avoid_reserve)
507                                 decrement_hugepage_resv_vma(h, vma);
508
509                         break;
510                 }
511         }
512 err:
513         mpol_cond_put(mpol);
514         put_mems_allowed();
515         return page;
516 }
517
518 static void update_and_free_page(struct hstate *h, struct page *page)
519 {
520         int i;
521
522         VM_BUG_ON(h->order >= MAX_ORDER);
523
524         h->nr_huge_pages--;
525         h->nr_huge_pages_node[page_to_nid(page)]--;
526         for (i = 0; i < pages_per_huge_page(h); i++) {
527                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
528                                 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
529                                 1 << PG_private | 1<< PG_writeback);
530         }
531         set_compound_page_dtor(page, NULL);
532         set_page_refcounted(page);
533         arch_release_hugepage(page);
534         __free_pages(page, huge_page_order(h));
535 }
536
537 struct hstate *size_to_hstate(unsigned long size)
538 {
539         struct hstate *h;
540
541         for_each_hstate(h) {
542                 if (huge_page_size(h) == size)
543                         return h;
544         }
545         return NULL;
546 }
547
548 static void free_huge_page(struct page *page)
549 {
550         /*
551          * Can't pass hstate in here because it is called from the
552          * compound page destructor.
553          */
554         struct hstate *h = page_hstate(page);
555         int nid = page_to_nid(page);
556         struct address_space *mapping;
557
558         mapping = (struct address_space *) page_private(page);
559         set_page_private(page, 0);
560         page->mapping = NULL;
561         BUG_ON(page_count(page));
562         BUG_ON(page_mapcount(page));
563         INIT_LIST_HEAD(&page->lru);
564
565         spin_lock(&hugetlb_lock);
566         if (h->surplus_huge_pages_node[nid] && huge_page_order(h) < MAX_ORDER) {
567                 update_and_free_page(h, page);
568                 h->surplus_huge_pages--;
569                 h->surplus_huge_pages_node[nid]--;
570         } else {
571                 enqueue_huge_page(h, page);
572         }
573         spin_unlock(&hugetlb_lock);
574         if (mapping)
575                 hugetlb_put_quota(mapping, 1);
576 }
577
578 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
579 {
580         set_compound_page_dtor(page, free_huge_page);
581         spin_lock(&hugetlb_lock);
582         h->nr_huge_pages++;
583         h->nr_huge_pages_node[nid]++;
584         spin_unlock(&hugetlb_lock);
585         put_page(page); /* free it into the hugepage allocator */
586 }
587
588 static void prep_compound_gigantic_page(struct page *page, unsigned long order)
589 {
590         int i;
591         int nr_pages = 1 << order;
592         struct page *p = page + 1;
593
594         /* we rely on prep_new_huge_page to set the destructor */
595         set_compound_order(page, order);
596         __SetPageHead(page);
597         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
598                 __SetPageTail(p);
599                 p->first_page = page;
600         }
601 }
602
603 int PageHuge(struct page *page)
604 {
605         compound_page_dtor *dtor;
606
607         if (!PageCompound(page))
608                 return 0;
609
610         page = compound_head(page);
611         dtor = get_compound_page_dtor(page);
612
613         return dtor == free_huge_page;
614 }
615
616 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
617 {
618         struct page *page;
619
620         if (h->order >= MAX_ORDER)
621                 return NULL;
622
623         page = alloc_pages_exact_node(nid,
624                 htlb_alloc_mask|__GFP_COMP|__GFP_THISNODE|
625                                                 __GFP_REPEAT|__GFP_NOWARN,
626                 huge_page_order(h));
627         if (page) {
628                 if (arch_prepare_hugepage(page)) {
629                         __free_pages(page, huge_page_order(h));
630                         return NULL;
631                 }
632                 prep_new_huge_page(h, page, nid);
633         }
634
635         return page;
636 }
637
638 /*
639  * common helper functions for hstate_next_node_to_{alloc|free}.
640  * We may have allocated or freed a huge page based on a different
641  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
642  * be outside of *nodes_allowed.  Ensure that we use an allowed
643  * node for alloc or free.
644  */
645 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
646 {
647         nid = next_node(nid, *nodes_allowed);
648         if (nid == MAX_NUMNODES)
649                 nid = first_node(*nodes_allowed);
650         VM_BUG_ON(nid >= MAX_NUMNODES);
651
652         return nid;
653 }
654
655 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
656 {
657         if (!node_isset(nid, *nodes_allowed))
658                 nid = next_node_allowed(nid, nodes_allowed);
659         return nid;
660 }
661
662 /*
663  * returns the previously saved node ["this node"] from which to
664  * allocate a persistent huge page for the pool and advance the
665  * next node from which to allocate, handling wrap at end of node
666  * mask.
667  */
668 static int hstate_next_node_to_alloc(struct hstate *h,
669                                         nodemask_t *nodes_allowed)
670 {
671         int nid;
672
673         VM_BUG_ON(!nodes_allowed);
674
675         nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
676         h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
677
678         return nid;
679 }
680
681 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
682 {
683         struct page *page;
684         int start_nid;
685         int next_nid;
686         int ret = 0;
687
688         start_nid = hstate_next_node_to_alloc(h, nodes_allowed);
689         next_nid = start_nid;
690
691         do {
692                 page = alloc_fresh_huge_page_node(h, next_nid);
693                 if (page) {
694                         ret = 1;
695                         break;
696                 }
697                 next_nid = hstate_next_node_to_alloc(h, nodes_allowed);
698         } while (next_nid != start_nid);
699
700         if (ret)
701                 count_vm_event(HTLB_BUDDY_PGALLOC);
702         else
703                 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
704
705         return ret;
706 }
707
708 /*
709  * helper for free_pool_huge_page() - return the previously saved
710  * node ["this node"] from which to free a huge page.  Advance the
711  * next node id whether or not we find a free huge page to free so
712  * that the next attempt to free addresses the next node.
713  */
714 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
715 {
716         int nid;
717
718         VM_BUG_ON(!nodes_allowed);
719
720         nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
721         h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
722
723         return nid;
724 }
725
726 /*
727  * Free huge page from pool from next node to free.
728  * Attempt to keep persistent huge pages more or less
729  * balanced over allowed nodes.
730  * Called with hugetlb_lock locked.
731  */
732 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
733                                                          bool acct_surplus)
734 {
735         int start_nid;
736         int next_nid;
737         int ret = 0;
738
739         start_nid = hstate_next_node_to_free(h, nodes_allowed);
740         next_nid = start_nid;
741
742         do {
743                 /*
744                  * If we're returning unused surplus pages, only examine
745                  * nodes with surplus pages.
746                  */
747                 if ((!acct_surplus || h->surplus_huge_pages_node[next_nid]) &&
748                     !list_empty(&h->hugepage_freelists[next_nid])) {
749                         struct page *page =
750                                 list_entry(h->hugepage_freelists[next_nid].next,
751                                           struct page, lru);
752                         list_del(&page->lru);
753                         h->free_huge_pages--;
754                         h->free_huge_pages_node[next_nid]--;
755                         if (acct_surplus) {
756                                 h->surplus_huge_pages--;
757                                 h->surplus_huge_pages_node[next_nid]--;
758                         }
759                         update_and_free_page(h, page);
760                         ret = 1;
761                         break;
762                 }
763                 next_nid = hstate_next_node_to_free(h, nodes_allowed);
764         } while (next_nid != start_nid);
765
766         return ret;
767 }
768
769 static struct page *alloc_buddy_huge_page(struct hstate *h,
770                         struct vm_area_struct *vma, unsigned long address)
771 {
772         struct page *page;
773         unsigned int nid;
774
775         if (h->order >= MAX_ORDER)
776                 return NULL;
777
778         /*
779          * Assume we will successfully allocate the surplus page to
780          * prevent racing processes from causing the surplus to exceed
781          * overcommit
782          *
783          * This however introduces a different race, where a process B
784          * tries to grow the static hugepage pool while alloc_pages() is
785          * called by process A. B will only examine the per-node
786          * counters in determining if surplus huge pages can be
787          * converted to normal huge pages in adjust_pool_surplus(). A
788          * won't be able to increment the per-node counter, until the
789          * lock is dropped by B, but B doesn't drop hugetlb_lock until
790          * no more huge pages can be converted from surplus to normal
791          * state (and doesn't try to convert again). Thus, we have a
792          * case where a surplus huge page exists, the pool is grown, and
793          * the surplus huge page still exists after, even though it
794          * should just have been converted to a normal huge page. This
795          * does not leak memory, though, as the hugepage will be freed
796          * once it is out of use. It also does not allow the counters to
797          * go out of whack in adjust_pool_surplus() as we don't modify
798          * the node values until we've gotten the hugepage and only the
799          * per-node value is checked there.
800          */
801         spin_lock(&hugetlb_lock);
802         if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
803                 spin_unlock(&hugetlb_lock);
804                 return NULL;
805         } else {
806                 h->nr_huge_pages++;
807                 h->surplus_huge_pages++;
808         }
809         spin_unlock(&hugetlb_lock);
810
811         page = alloc_pages(htlb_alloc_mask|__GFP_COMP|
812                                         __GFP_REPEAT|__GFP_NOWARN,
813                                         huge_page_order(h));
814
815         if (page && arch_prepare_hugepage(page)) {
816                 __free_pages(page, huge_page_order(h));
817                 return NULL;
818         }
819
820         spin_lock(&hugetlb_lock);
821         if (page) {
822                 /*
823                  * This page is now managed by the hugetlb allocator and has
824                  * no users -- drop the buddy allocator's reference.
825                  */
826                 put_page_testzero(page);
827                 VM_BUG_ON(page_count(page));
828                 nid = page_to_nid(page);
829                 set_compound_page_dtor(page, free_huge_page);
830                 /*
831                  * We incremented the global counters already
832                  */
833                 h->nr_huge_pages_node[nid]++;
834                 h->surplus_huge_pages_node[nid]++;
835                 __count_vm_event(HTLB_BUDDY_PGALLOC);
836         } else {
837                 h->nr_huge_pages--;
838                 h->surplus_huge_pages--;
839                 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
840         }
841         spin_unlock(&hugetlb_lock);
842
843         return page;
844 }
845
846 /*
847  * Increase the hugetlb pool such that it can accomodate a reservation
848  * of size 'delta'.
849  */
850 static int gather_surplus_pages(struct hstate *h, int delta)
851 {
852         struct list_head surplus_list;
853         struct page *page, *tmp;
854         int ret, i;
855         int needed, allocated;
856
857         needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
858         if (needed <= 0) {
859                 h->resv_huge_pages += delta;
860                 return 0;
861         }
862
863         allocated = 0;
864         INIT_LIST_HEAD(&surplus_list);
865
866         ret = -ENOMEM;
867 retry:
868         spin_unlock(&hugetlb_lock);
869         for (i = 0; i < needed; i++) {
870                 page = alloc_buddy_huge_page(h, NULL, 0);
871                 if (!page) {
872                         /*
873                          * We were not able to allocate enough pages to
874                          * satisfy the entire reservation so we free what
875                          * we've allocated so far.
876                          */
877                         spin_lock(&hugetlb_lock);
878                         needed = 0;
879                         goto free;
880                 }
881
882                 list_add(&page->lru, &surplus_list);
883         }
884         allocated += needed;
885
886         /*
887          * After retaking hugetlb_lock, we need to recalculate 'needed'
888          * because either resv_huge_pages or free_huge_pages may have changed.
889          */
890         spin_lock(&hugetlb_lock);
891         needed = (h->resv_huge_pages + delta) -
892                         (h->free_huge_pages + allocated);
893         if (needed > 0)
894                 goto retry;
895
896         /*
897          * The surplus_list now contains _at_least_ the number of extra pages
898          * needed to accomodate the reservation.  Add the appropriate number
899          * of pages to the hugetlb pool and free the extras back to the buddy
900          * allocator.  Commit the entire reservation here to prevent another
901          * process from stealing the pages as they are added to the pool but
902          * before they are reserved.
903          */
904         needed += allocated;
905         h->resv_huge_pages += delta;
906         ret = 0;
907 free:
908         /* Free the needed pages to the hugetlb pool */
909         list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
910                 if ((--needed) < 0)
911                         break;
912                 list_del(&page->lru);
913                 enqueue_huge_page(h, page);
914         }
915
916         /* Free unnecessary surplus pages to the buddy allocator */
917         if (!list_empty(&surplus_list)) {
918                 spin_unlock(&hugetlb_lock);
919                 list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
920                         list_del(&page->lru);
921                         /*
922                          * The page has a reference count of zero already, so
923                          * call free_huge_page directly instead of using
924                          * put_page.  This must be done with hugetlb_lock
925                          * unlocked which is safe because free_huge_page takes
926                          * hugetlb_lock before deciding how to free the page.
927                          */
928                         free_huge_page(page);
929                 }
930                 spin_lock(&hugetlb_lock);
931         }
932
933         return ret;
934 }
935
936 /*
937  * When releasing a hugetlb pool reservation, any surplus pages that were
938  * allocated to satisfy the reservation must be explicitly freed if they were
939  * never used.
940  * Called with hugetlb_lock held.
941  */
942 static void return_unused_surplus_pages(struct hstate *h,
943                                         unsigned long unused_resv_pages)
944 {
945         unsigned long nr_pages;
946
947         /* Uncommit the reservation */
948         h->resv_huge_pages -= unused_resv_pages;
949
950         /* Cannot return gigantic pages currently */
951         if (h->order >= MAX_ORDER)
952                 return;
953
954         nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
955
956         /*
957          * We want to release as many surplus pages as possible, spread
958          * evenly across all nodes with memory. Iterate across these nodes
959          * until we can no longer free unreserved surplus pages. This occurs
960          * when the nodes with surplus pages have no free pages.
961          * free_pool_huge_page() will balance the the freed pages across the
962          * on-line nodes with memory and will handle the hstate accounting.
963          */
964         while (nr_pages--) {
965                 if (!free_pool_huge_page(h, &node_states[N_HIGH_MEMORY], 1))
966                         break;
967         }
968 }
969
970 /*
971  * Determine if the huge page at addr within the vma has an associated
972  * reservation.  Where it does not we will need to logically increase
973  * reservation and actually increase quota before an allocation can occur.
974  * Where any new reservation would be required the reservation change is
975  * prepared, but not committed.  Once the page has been quota'd allocated
976  * an instantiated the change should be committed via vma_commit_reservation.
977  * No action is required on failure.
978  */
979 static long vma_needs_reservation(struct hstate *h,
980                         struct vm_area_struct *vma, unsigned long addr)
981 {
982         struct address_space *mapping = vma->vm_file->f_mapping;
983         struct inode *inode = mapping->host;
984
985         if (vma->vm_flags & VM_MAYSHARE) {
986                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
987                 return region_chg(&inode->i_mapping->private_list,
988                                                         idx, idx + 1);
989
990         } else if (!is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
991                 return 1;
992
993         } else  {
994                 long err;
995                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
996                 struct resv_map *reservations = vma_resv_map(vma);
997
998                 err = region_chg(&reservations->regions, idx, idx + 1);
999                 if (err < 0)
1000                         return err;
1001                 return 0;
1002         }
1003 }
1004 static void vma_commit_reservation(struct hstate *h,
1005                         struct vm_area_struct *vma, unsigned long addr)
1006 {
1007         struct address_space *mapping = vma->vm_file->f_mapping;
1008         struct inode *inode = mapping->host;
1009
1010         if (vma->vm_flags & VM_MAYSHARE) {
1011                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1012                 region_add(&inode->i_mapping->private_list, idx, idx + 1);
1013
1014         } else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1015                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1016                 struct resv_map *reservations = vma_resv_map(vma);
1017
1018                 /* Mark this page used in the map. */
1019                 region_add(&reservations->regions, idx, idx + 1);
1020         }
1021 }
1022
1023 static struct page *alloc_huge_page(struct vm_area_struct *vma,
1024                                     unsigned long addr, int avoid_reserve)
1025 {
1026         struct hstate *h = hstate_vma(vma);
1027         struct page *page;
1028         struct address_space *mapping = vma->vm_file->f_mapping;
1029         struct inode *inode = mapping->host;
1030         long chg;
1031
1032         /*
1033          * Processes that did not create the mapping will have no reserves and
1034          * will not have accounted against quota. Check that the quota can be
1035          * made before satisfying the allocation
1036          * MAP_NORESERVE mappings may also need pages and quota allocated
1037          * if no reserve mapping overlaps.
1038          */
1039         chg = vma_needs_reservation(h, vma, addr);
1040         if (chg < 0)
1041                 return ERR_PTR(chg);
1042         if (chg)
1043                 if (hugetlb_get_quota(inode->i_mapping, chg))
1044                         return ERR_PTR(-ENOSPC);
1045
1046         spin_lock(&hugetlb_lock);
1047         page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve);
1048         spin_unlock(&hugetlb_lock);
1049
1050         if (!page) {
1051                 page = alloc_buddy_huge_page(h, vma, addr);
1052                 if (!page) {
1053                         hugetlb_put_quota(inode->i_mapping, chg);
1054                         return ERR_PTR(-VM_FAULT_SIGBUS);
1055                 }
1056         }
1057
1058         set_page_refcounted(page);
1059         set_page_private(page, (unsigned long) mapping);
1060
1061         vma_commit_reservation(h, vma, addr);
1062
1063         return page;
1064 }
1065
1066 int __weak alloc_bootmem_huge_page(struct hstate *h)
1067 {
1068         struct huge_bootmem_page *m;
1069         int nr_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
1070
1071         while (nr_nodes) {
1072                 void *addr;
1073
1074                 addr = __alloc_bootmem_node_nopanic(
1075                                 NODE_DATA(hstate_next_node_to_alloc(h,
1076                                                 &node_states[N_HIGH_MEMORY])),
1077                                 huge_page_size(h), huge_page_size(h), 0);
1078
1079                 if (addr) {
1080                         /*
1081                          * Use the beginning of the huge page to store the
1082                          * huge_bootmem_page struct (until gather_bootmem
1083                          * puts them into the mem_map).
1084                          */
1085                         m = addr;
1086                         goto found;
1087                 }
1088                 nr_nodes--;
1089         }
1090         return 0;
1091
1092 found:
1093         BUG_ON((unsigned long)virt_to_phys(m) & (huge_page_size(h) - 1));
1094         /* Put them into a private list first because mem_map is not up yet */
1095         list_add(&m->list, &huge_boot_pages);
1096         m->hstate = h;
1097         return 1;
1098 }
1099
1100 static void prep_compound_huge_page(struct page *page, int order)
1101 {
1102         if (unlikely(order > (MAX_ORDER - 1)))
1103                 prep_compound_gigantic_page(page, order);
1104         else
1105                 prep_compound_page(page, order);
1106 }
1107
1108 /* Put bootmem huge pages into the standard lists after mem_map is up */
1109 static void __init gather_bootmem_prealloc(void)
1110 {
1111         struct huge_bootmem_page *m;
1112
1113         list_for_each_entry(m, &huge_boot_pages, list) {
1114                 struct page *page = virt_to_page(m);
1115                 struct hstate *h = m->hstate;
1116                 __ClearPageReserved(page);
1117                 WARN_ON(page_count(page) != 1);
1118                 prep_compound_huge_page(page, h->order);
1119                 prep_new_huge_page(h, page, page_to_nid(page));
1120         }
1121 }
1122
1123 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
1124 {
1125         unsigned long i;
1126
1127         for (i = 0; i < h->max_huge_pages; ++i) {
1128                 if (h->order >= MAX_ORDER) {
1129                         if (!alloc_bootmem_huge_page(h))
1130                                 break;
1131                 } else if (!alloc_fresh_huge_page(h,
1132                                          &node_states[N_HIGH_MEMORY]))
1133                         break;
1134         }
1135         h->max_huge_pages = i;
1136 }
1137
1138 static void __init hugetlb_init_hstates(void)
1139 {
1140         struct hstate *h;
1141
1142         for_each_hstate(h) {
1143                 /* oversize hugepages were init'ed in early boot */
1144                 if (h->order < MAX_ORDER)
1145                         hugetlb_hstate_alloc_pages(h);
1146         }
1147 }
1148
1149 static char * __init memfmt(char *buf, unsigned long n)
1150 {
1151         if (n >= (1UL << 30))
1152                 sprintf(buf, "%lu GB", n >> 30);
1153         else if (n >= (1UL << 20))
1154                 sprintf(buf, "%lu MB", n >> 20);
1155         else
1156                 sprintf(buf, "%lu KB", n >> 10);
1157         return buf;
1158 }
1159
1160 static void __init report_hugepages(void)
1161 {
1162         struct hstate *h;
1163
1164         for_each_hstate(h) {
1165                 char buf[32];
1166                 printk(KERN_INFO "HugeTLB registered %s page size, "
1167                                  "pre-allocated %ld pages\n",
1168                         memfmt(buf, huge_page_size(h)),
1169                         h->free_huge_pages);
1170         }
1171 }
1172
1173 #ifdef CONFIG_HIGHMEM
1174 static void try_to_free_low(struct hstate *h, unsigned long count,
1175                                                 nodemask_t *nodes_allowed)
1176 {
1177         int i;
1178
1179         if (h->order >= MAX_ORDER)
1180                 return;
1181
1182         for_each_node_mask(i, *nodes_allowed) {
1183                 struct page *page, *next;
1184                 struct list_head *freel = &h->hugepage_freelists[i];
1185                 list_for_each_entry_safe(page, next, freel, lru) {
1186                         if (count >= h->nr_huge_pages)
1187                                 return;
1188                         if (PageHighMem(page))
1189                                 continue;
1190                         list_del(&page->lru);
1191                         update_and_free_page(h, page);
1192                         h->free_huge_pages--;
1193                         h->free_huge_pages_node[page_to_nid(page)]--;
1194                 }
1195         }
1196 }
1197 #else
1198 static inline void try_to_free_low(struct hstate *h, unsigned long count,
1199                                                 nodemask_t *nodes_allowed)
1200 {
1201 }
1202 #endif
1203
1204 /*
1205  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
1206  * balanced by operating on them in a round-robin fashion.
1207  * Returns 1 if an adjustment was made.
1208  */
1209 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
1210                                 int delta)
1211 {
1212         int start_nid, next_nid;
1213         int ret = 0;
1214
1215         VM_BUG_ON(delta != -1 && delta != 1);
1216
1217         if (delta < 0)
1218                 start_nid = hstate_next_node_to_alloc(h, nodes_allowed);
1219         else
1220                 start_nid = hstate_next_node_to_free(h, nodes_allowed);
1221         next_nid = start_nid;
1222
1223         do {
1224                 int nid = next_nid;
1225                 if (delta < 0)  {
1226                         /*
1227                          * To shrink on this node, there must be a surplus page
1228                          */
1229                         if (!h->surplus_huge_pages_node[nid]) {
1230                                 next_nid = hstate_next_node_to_alloc(h,
1231                                                                 nodes_allowed);
1232                                 continue;
1233                         }
1234                 }
1235                 if (delta > 0) {
1236                         /*
1237                          * Surplus cannot exceed the total number of pages
1238                          */
1239                         if (h->surplus_huge_pages_node[nid] >=
1240                                                 h->nr_huge_pages_node[nid]) {
1241                                 next_nid = hstate_next_node_to_free(h,
1242                                                                 nodes_allowed);
1243                                 continue;
1244                         }
1245                 }
1246
1247                 h->surplus_huge_pages += delta;
1248                 h->surplus_huge_pages_node[nid] += delta;
1249                 ret = 1;
1250                 break;
1251         } while (next_nid != start_nid);
1252
1253         return ret;
1254 }
1255
1256 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
1257 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
1258                                                 nodemask_t *nodes_allowed)
1259 {
1260         unsigned long min_count, ret;
1261
1262         if (h->order >= MAX_ORDER)
1263                 return h->max_huge_pages;
1264
1265         /*
1266          * Increase the pool size
1267          * First take pages out of surplus state.  Then make up the
1268          * remaining difference by allocating fresh huge pages.
1269          *
1270          * We might race with alloc_buddy_huge_page() here and be unable
1271          * to convert a surplus huge page to a normal huge page. That is
1272          * not critical, though, it just means the overall size of the
1273          * pool might be one hugepage larger than it needs to be, but
1274          * within all the constraints specified by the sysctls.
1275          */
1276         spin_lock(&hugetlb_lock);
1277         while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
1278                 if (!adjust_pool_surplus(h, nodes_allowed, -1))
1279                         break;
1280         }
1281
1282         while (count > persistent_huge_pages(h)) {
1283                 /*
1284                  * If this allocation races such that we no longer need the
1285                  * page, free_huge_page will handle it by freeing the page
1286                  * and reducing the surplus.
1287                  */
1288                 spin_unlock(&hugetlb_lock);
1289                 ret = alloc_fresh_huge_page(h, nodes_allowed);
1290                 spin_lock(&hugetlb_lock);
1291                 if (!ret)
1292                         goto out;
1293
1294                 /* Bail for signals. Probably ctrl-c from user */
1295                 if (signal_pending(current))
1296                         goto out;
1297         }
1298
1299         /*
1300          * Decrease the pool size
1301          * First return free pages to the buddy allocator (being careful
1302          * to keep enough around to satisfy reservations).  Then place
1303          * pages into surplus state as needed so the pool will shrink
1304          * to the desired size as pages become free.
1305          *
1306          * By placing pages into the surplus state independent of the
1307          * overcommit value, we are allowing the surplus pool size to
1308          * exceed overcommit. There are few sane options here. Since
1309          * alloc_buddy_huge_page() is checking the global counter,
1310          * though, we'll note that we're not allowed to exceed surplus
1311          * and won't grow the pool anywhere else. Not until one of the
1312          * sysctls are changed, or the surplus pages go out of use.
1313          */
1314         min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
1315         min_count = max(count, min_count);
1316         try_to_free_low(h, min_count, nodes_allowed);
1317         while (min_count < persistent_huge_pages(h)) {
1318                 if (!free_pool_huge_page(h, nodes_allowed, 0))
1319                         break;
1320         }
1321         while (count < persistent_huge_pages(h)) {
1322                 if (!adjust_pool_surplus(h, nodes_allowed, 1))
1323                         break;
1324         }
1325 out:
1326         ret = persistent_huge_pages(h);
1327         spin_unlock(&hugetlb_lock);
1328         return ret;
1329 }
1330
1331 #define HSTATE_ATTR_RO(_name) \
1332         static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
1333
1334 #define HSTATE_ATTR(_name) \
1335         static struct kobj_attribute _name##_attr = \
1336                 __ATTR(_name, 0644, _name##_show, _name##_store)
1337
1338 static struct kobject *hugepages_kobj;
1339 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
1340
1341 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
1342
1343 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
1344 {
1345         int i;
1346
1347         for (i = 0; i < HUGE_MAX_HSTATE; i++)
1348                 if (hstate_kobjs[i] == kobj) {
1349                         if (nidp)
1350                                 *nidp = NUMA_NO_NODE;
1351                         return &hstates[i];
1352                 }
1353
1354         return kobj_to_node_hstate(kobj, nidp);
1355 }
1356
1357 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
1358                                         struct kobj_attribute *attr, char *buf)
1359 {
1360         struct hstate *h;
1361         unsigned long nr_huge_pages;
1362         int nid;
1363
1364         h = kobj_to_hstate(kobj, &nid);
1365         if (nid == NUMA_NO_NODE)
1366                 nr_huge_pages = h->nr_huge_pages;
1367         else
1368                 nr_huge_pages = h->nr_huge_pages_node[nid];
1369
1370         return sprintf(buf, "%lu\n", nr_huge_pages);
1371 }
1372 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
1373                         struct kobject *kobj, struct kobj_attribute *attr,
1374                         const char *buf, size_t len)
1375 {
1376         int err;
1377         int nid;
1378         unsigned long count;
1379         struct hstate *h;
1380         NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
1381
1382         err = strict_strtoul(buf, 10, &count);
1383         if (err)
1384                 return 0;
1385
1386         h = kobj_to_hstate(kobj, &nid);
1387         if (nid == NUMA_NO_NODE) {
1388                 /*
1389                  * global hstate attribute
1390                  */
1391                 if (!(obey_mempolicy &&
1392                                 init_nodemask_of_mempolicy(nodes_allowed))) {
1393                         NODEMASK_FREE(nodes_allowed);
1394                         nodes_allowed = &node_states[N_HIGH_MEMORY];
1395                 }
1396         } else if (nodes_allowed) {
1397                 /*
1398                  * per node hstate attribute: adjust count to global,
1399                  * but restrict alloc/free to the specified node.
1400                  */
1401                 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
1402                 init_nodemask_of_node(nodes_allowed, nid);
1403         } else
1404                 nodes_allowed = &node_states[N_HIGH_MEMORY];
1405
1406         h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
1407
1408         if (nodes_allowed != &node_states[N_HIGH_MEMORY])
1409                 NODEMASK_FREE(nodes_allowed);
1410
1411         return len;
1412 }
1413
1414 static ssize_t nr_hugepages_show(struct kobject *kobj,
1415                                        struct kobj_attribute *attr, char *buf)
1416 {
1417         return nr_hugepages_show_common(kobj, attr, buf);
1418 }
1419
1420 static ssize_t nr_hugepages_store(struct kobject *kobj,
1421                struct kobj_attribute *attr, const char *buf, size_t len)
1422 {
1423         return nr_hugepages_store_common(false, kobj, attr, buf, len);
1424 }
1425 HSTATE_ATTR(nr_hugepages);
1426
1427 #ifdef CONFIG_NUMA
1428
1429 /*
1430  * hstate attribute for optionally mempolicy-based constraint on persistent
1431  * huge page alloc/free.
1432  */
1433 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
1434                                        struct kobj_attribute *attr, char *buf)
1435 {
1436         return nr_hugepages_show_common(kobj, attr, buf);
1437 }
1438
1439 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
1440                struct kobj_attribute *attr, const char *buf, size_t len)
1441 {
1442         return nr_hugepages_store_common(true, kobj, attr, buf, len);
1443 }
1444 HSTATE_ATTR(nr_hugepages_mempolicy);
1445 #endif
1446
1447
1448 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
1449                                         struct kobj_attribute *attr, char *buf)
1450 {
1451         struct hstate *h = kobj_to_hstate(kobj, NULL);
1452         return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
1453 }
1454 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
1455                 struct kobj_attribute *attr, const char *buf, size_t count)
1456 {
1457         int err;
1458         unsigned long input;
1459         struct hstate *h = kobj_to_hstate(kobj, NULL);
1460
1461         err = strict_strtoul(buf, 10, &input);
1462         if (err)
1463                 return 0;
1464
1465         spin_lock(&hugetlb_lock);
1466         h->nr_overcommit_huge_pages = input;
1467         spin_unlock(&hugetlb_lock);
1468
1469         return count;
1470 }
1471 HSTATE_ATTR(nr_overcommit_hugepages);
1472
1473 static ssize_t free_hugepages_show(struct kobject *kobj,
1474                                         struct kobj_attribute *attr, char *buf)
1475 {
1476         struct hstate *h;
1477         unsigned long free_huge_pages;
1478         int nid;
1479
1480         h = kobj_to_hstate(kobj, &nid);
1481         if (nid == NUMA_NO_NODE)
1482                 free_huge_pages = h->free_huge_pages;
1483         else
1484                 free_huge_pages = h->free_huge_pages_node[nid];
1485
1486         return sprintf(buf, "%lu\n", free_huge_pages);
1487 }
1488 HSTATE_ATTR_RO(free_hugepages);
1489
1490 static ssize_t resv_hugepages_show(struct kobject *kobj,
1491                                         struct kobj_attribute *attr, char *buf)
1492 {
1493         struct hstate *h = kobj_to_hstate(kobj, NULL);
1494         return sprintf(buf, "%lu\n", h->resv_huge_pages);
1495 }
1496 HSTATE_ATTR_RO(resv_hugepages);
1497
1498 static ssize_t surplus_hugepages_show(struct kobject *kobj,
1499                                         struct kobj_attribute *attr, char *buf)
1500 {
1501         struct hstate *h;
1502         unsigned long surplus_huge_pages;
1503         int nid;
1504
1505         h = kobj_to_hstate(kobj, &nid);
1506         if (nid == NUMA_NO_NODE)
1507                 surplus_huge_pages = h->surplus_huge_pages;
1508         else
1509                 surplus_huge_pages = h->surplus_huge_pages_node[nid];
1510
1511         return sprintf(buf, "%lu\n", surplus_huge_pages);
1512 }
1513 HSTATE_ATTR_RO(surplus_hugepages);
1514
1515 static struct attribute *hstate_attrs[] = {
1516         &nr_hugepages_attr.attr,
1517         &nr_overcommit_hugepages_attr.attr,
1518         &free_hugepages_attr.attr,
1519         &resv_hugepages_attr.attr,
1520         &surplus_hugepages_attr.attr,
1521 #ifdef CONFIG_NUMA
1522         &nr_hugepages_mempolicy_attr.attr,
1523 #endif
1524         NULL,
1525 };
1526
1527 static struct attribute_group hstate_attr_group = {
1528         .attrs = hstate_attrs,
1529 };
1530
1531 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
1532                                     struct kobject **hstate_kobjs,
1533                                     struct attribute_group *hstate_attr_group)
1534 {
1535         int retval;
1536         int hi = h - hstates;
1537
1538         hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
1539         if (!hstate_kobjs[hi])
1540                 return -ENOMEM;
1541
1542         retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
1543         if (retval)
1544                 kobject_put(hstate_kobjs[hi]);
1545
1546         return retval;
1547 }
1548
1549 static void __init hugetlb_sysfs_init(void)
1550 {
1551         struct hstate *h;
1552         int err;
1553
1554         hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
1555         if (!hugepages_kobj)
1556                 return;
1557
1558         for_each_hstate(h) {
1559                 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
1560                                          hstate_kobjs, &hstate_attr_group);
1561                 if (err)
1562                         printk(KERN_ERR "Hugetlb: Unable to add hstate %s",
1563                                                                 h->name);
1564         }
1565 }
1566
1567 #ifdef CONFIG_NUMA
1568
1569 /*
1570  * node_hstate/s - associate per node hstate attributes, via their kobjects,
1571  * with node sysdevs in node_devices[] using a parallel array.  The array
1572  * index of a node sysdev or _hstate == node id.
1573  * This is here to avoid any static dependency of the node sysdev driver, in
1574  * the base kernel, on the hugetlb module.
1575  */
1576 struct node_hstate {
1577         struct kobject          *hugepages_kobj;
1578         struct kobject          *hstate_kobjs[HUGE_MAX_HSTATE];
1579 };
1580 struct node_hstate node_hstates[MAX_NUMNODES];
1581
1582 /*
1583  * A subset of global hstate attributes for node sysdevs
1584  */
1585 static struct attribute *per_node_hstate_attrs[] = {
1586         &nr_hugepages_attr.attr,
1587         &free_hugepages_attr.attr,
1588         &surplus_hugepages_attr.attr,
1589         NULL,
1590 };
1591
1592 static struct attribute_group per_node_hstate_attr_group = {
1593         .attrs = per_node_hstate_attrs,
1594 };
1595
1596 /*
1597  * kobj_to_node_hstate - lookup global hstate for node sysdev hstate attr kobj.
1598  * Returns node id via non-NULL nidp.
1599  */
1600 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1601 {
1602         int nid;
1603
1604         for (nid = 0; nid < nr_node_ids; nid++) {
1605                 struct node_hstate *nhs = &node_hstates[nid];
1606                 int i;
1607                 for (i = 0; i < HUGE_MAX_HSTATE; i++)
1608                         if (nhs->hstate_kobjs[i] == kobj) {
1609                                 if (nidp)
1610                                         *nidp = nid;
1611                                 return &hstates[i];
1612                         }
1613         }
1614
1615         BUG();
1616         return NULL;
1617 }
1618
1619 /*
1620  * Unregister hstate attributes from a single node sysdev.
1621  * No-op if no hstate attributes attached.
1622  */
1623 void hugetlb_unregister_node(struct node *node)
1624 {
1625         struct hstate *h;
1626         struct node_hstate *nhs = &node_hstates[node->sysdev.id];
1627
1628         if (!nhs->hugepages_kobj)
1629                 return;         /* no hstate attributes */
1630
1631         for_each_hstate(h)
1632                 if (nhs->hstate_kobjs[h - hstates]) {
1633                         kobject_put(nhs->hstate_kobjs[h - hstates]);
1634                         nhs->hstate_kobjs[h - hstates] = NULL;
1635                 }
1636
1637         kobject_put(nhs->hugepages_kobj);
1638         nhs->hugepages_kobj = NULL;
1639 }
1640
1641 /*
1642  * hugetlb module exit:  unregister hstate attributes from node sysdevs
1643  * that have them.
1644  */
1645 static void hugetlb_unregister_all_nodes(void)
1646 {
1647         int nid;
1648
1649         /*
1650          * disable node sysdev registrations.
1651          */
1652         register_hugetlbfs_with_node(NULL, NULL);
1653
1654         /*
1655          * remove hstate attributes from any nodes that have them.
1656          */
1657         for (nid = 0; nid < nr_node_ids; nid++)
1658                 hugetlb_unregister_node(&node_devices[nid]);
1659 }
1660
1661 /*
1662  * Register hstate attributes for a single node sysdev.
1663  * No-op if attributes already registered.
1664  */
1665 void hugetlb_register_node(struct node *node)
1666 {
1667         struct hstate *h;
1668         struct node_hstate *nhs = &node_hstates[node->sysdev.id];
1669         int err;
1670
1671         if (nhs->hugepages_kobj)
1672                 return;         /* already allocated */
1673
1674         nhs->hugepages_kobj = kobject_create_and_add("hugepages",
1675                                                         &node->sysdev.kobj);
1676         if (!nhs->hugepages_kobj)
1677                 return;
1678
1679         for_each_hstate(h) {
1680                 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
1681                                                 nhs->hstate_kobjs,
1682                                                 &per_node_hstate_attr_group);
1683                 if (err) {
1684                         printk(KERN_ERR "Hugetlb: Unable to add hstate %s"
1685                                         " for node %d\n",
1686                                                 h->name, node->sysdev.id);
1687                         hugetlb_unregister_node(node);
1688                         break;
1689                 }
1690         }
1691 }
1692
1693 /*
1694  * hugetlb init time:  register hstate attributes for all registered node
1695  * sysdevs of nodes that have memory.  All on-line nodes should have
1696  * registered their associated sysdev by this time.
1697  */
1698 static void hugetlb_register_all_nodes(void)
1699 {
1700         int nid;
1701
1702         for_each_node_state(nid, N_HIGH_MEMORY) {
1703                 struct node *node = &node_devices[nid];
1704                 if (node->sysdev.id == nid)
1705                         hugetlb_register_node(node);
1706         }
1707
1708         /*
1709          * Let the node sysdev driver know we're here so it can
1710          * [un]register hstate attributes on node hotplug.
1711          */
1712         register_hugetlbfs_with_node(hugetlb_register_node,
1713                                      hugetlb_unregister_node);
1714 }
1715 #else   /* !CONFIG_NUMA */
1716
1717 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1718 {
1719         BUG();
1720         if (nidp)
1721                 *nidp = -1;
1722         return NULL;
1723 }
1724
1725 static void hugetlb_unregister_all_nodes(void) { }
1726
1727 static void hugetlb_register_all_nodes(void) { }
1728
1729 #endif
1730
1731 static void __exit hugetlb_exit(void)
1732 {
1733         struct hstate *h;
1734
1735         hugetlb_unregister_all_nodes();
1736
1737         for_each_hstate(h) {
1738                 kobject_put(hstate_kobjs[h - hstates]);
1739         }
1740
1741         kobject_put(hugepages_kobj);
1742 }
1743 module_exit(hugetlb_exit);
1744
1745 static int __init hugetlb_init(void)
1746 {
1747         /* Some platform decide whether they support huge pages at boot
1748          * time. On these, such as powerpc, HPAGE_SHIFT is set to 0 when
1749          * there is no such support
1750          */
1751         if (HPAGE_SHIFT == 0)
1752                 return 0;
1753
1754         if (!size_to_hstate(default_hstate_size)) {
1755                 default_hstate_size = HPAGE_SIZE;
1756                 if (!size_to_hstate(default_hstate_size))
1757                         hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
1758         }
1759         default_hstate_idx = size_to_hstate(default_hstate_size) - hstates;
1760         if (default_hstate_max_huge_pages)
1761                 default_hstate.max_huge_pages = default_hstate_max_huge_pages;
1762
1763         hugetlb_init_hstates();
1764
1765         gather_bootmem_prealloc();
1766
1767         report_hugepages();
1768
1769         hugetlb_sysfs_init();
1770
1771         hugetlb_register_all_nodes();
1772
1773         return 0;
1774 }
1775 module_init(hugetlb_init);
1776
1777 /* Should be called on processing a hugepagesz=... option */
1778 void __init hugetlb_add_hstate(unsigned order)
1779 {
1780         struct hstate *h;
1781         unsigned long i;
1782
1783         if (size_to_hstate(PAGE_SIZE << order)) {
1784                 printk(KERN_WARNING "hugepagesz= specified twice, ignoring\n");
1785                 return;
1786         }
1787         BUG_ON(max_hstate >= HUGE_MAX_HSTATE);
1788         BUG_ON(order == 0);
1789         h = &hstates[max_hstate++];
1790         h->order = order;
1791         h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
1792         h->nr_huge_pages = 0;
1793         h->free_huge_pages = 0;
1794         for (i = 0; i < MAX_NUMNODES; ++i)
1795                 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
1796         h->next_nid_to_alloc = first_node(node_states[N_HIGH_MEMORY]);
1797         h->next_nid_to_free = first_node(node_states[N_HIGH_MEMORY]);
1798         snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
1799                                         huge_page_size(h)/1024);
1800
1801         parsed_hstate = h;
1802 }
1803
1804 static int __init hugetlb_nrpages_setup(char *s)
1805 {
1806         unsigned long *mhp;
1807         static unsigned long *last_mhp;
1808
1809         /*
1810          * !max_hstate means we haven't parsed a hugepagesz= parameter yet,
1811          * so this hugepages= parameter goes to the "default hstate".
1812          */
1813         if (!max_hstate)
1814                 mhp = &default_hstate_max_huge_pages;
1815         else
1816                 mhp = &parsed_hstate->max_huge_pages;
1817
1818         if (mhp == last_mhp) {
1819                 printk(KERN_WARNING "hugepages= specified twice without "
1820                         "interleaving hugepagesz=, ignoring\n");
1821                 return 1;
1822         }
1823
1824         if (sscanf(s, "%lu", mhp) <= 0)
1825                 *mhp = 0;
1826
1827         /*
1828          * Global state is always initialized later in hugetlb_init.
1829          * But we need to allocate >= MAX_ORDER hstates here early to still
1830          * use the bootmem allocator.
1831          */
1832         if (max_hstate && parsed_hstate->order >= MAX_ORDER)
1833                 hugetlb_hstate_alloc_pages(parsed_hstate);
1834
1835         last_mhp = mhp;
1836
1837         return 1;
1838 }
1839 __setup("hugepages=", hugetlb_nrpages_setup);
1840
1841 static int __init hugetlb_default_setup(char *s)
1842 {
1843         default_hstate_size = memparse(s, &s);
1844         return 1;
1845 }
1846 __setup("default_hugepagesz=", hugetlb_default_setup);
1847
1848 static unsigned int cpuset_mems_nr(unsigned int *array)
1849 {
1850         int node;
1851         unsigned int nr = 0;
1852
1853         for_each_node_mask(node, cpuset_current_mems_allowed)
1854                 nr += array[node];
1855
1856         return nr;
1857 }
1858
1859 #ifdef CONFIG_SYSCTL
1860 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
1861                          struct ctl_table *table, int write,
1862                          void __user *buffer, size_t *length, loff_t *ppos)
1863 {
1864         struct hstate *h = &default_hstate;
1865         unsigned long tmp;
1866
1867         if (!write)
1868                 tmp = h->max_huge_pages;
1869
1870         table->data = &tmp;
1871         table->maxlen = sizeof(unsigned long);
1872         proc_doulongvec_minmax(table, write, buffer, length, ppos);
1873
1874         if (write) {
1875                 NODEMASK_ALLOC(nodemask_t, nodes_allowed,
1876                                                 GFP_KERNEL | __GFP_NORETRY);
1877                 if (!(obey_mempolicy &&
1878                                init_nodemask_of_mempolicy(nodes_allowed))) {
1879                         NODEMASK_FREE(nodes_allowed);
1880                         nodes_allowed = &node_states[N_HIGH_MEMORY];
1881                 }
1882                 h->max_huge_pages = set_max_huge_pages(h, tmp, nodes_allowed);
1883
1884                 if (nodes_allowed != &node_states[N_HIGH_MEMORY])
1885                         NODEMASK_FREE(nodes_allowed);
1886         }
1887
1888         return 0;
1889 }
1890
1891 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
1892                           void __user *buffer, size_t *length, loff_t *ppos)
1893 {
1894
1895         return hugetlb_sysctl_handler_common(false, table, write,
1896                                                         buffer, length, ppos);
1897 }
1898
1899 #ifdef CONFIG_NUMA
1900 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
1901                           void __user *buffer, size_t *length, loff_t *ppos)
1902 {
1903         return hugetlb_sysctl_handler_common(true, table, write,
1904                                                         buffer, length, ppos);
1905 }
1906 #endif /* CONFIG_NUMA */
1907
1908 int hugetlb_treat_movable_handler(struct ctl_table *table, int write,
1909                         void __user *buffer,
1910                         size_t *length, loff_t *ppos)
1911 {
1912         proc_dointvec(table, write, buffer, length, ppos);
1913         if (hugepages_treat_as_movable)
1914                 htlb_alloc_mask = GFP_HIGHUSER_MOVABLE;
1915         else
1916                 htlb_alloc_mask = GFP_HIGHUSER;
1917         return 0;
1918 }
1919
1920 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
1921                         void __user *buffer,
1922                         size_t *length, loff_t *ppos)
1923 {
1924         struct hstate *h = &default_hstate;
1925         unsigned long tmp;
1926
1927         if (!write)
1928                 tmp = h->nr_overcommit_huge_pages;
1929
1930         table->data = &tmp;
1931         table->maxlen = sizeof(unsigned long);
1932         proc_doulongvec_minmax(table, write, buffer, length, ppos);
1933
1934         if (write) {
1935                 spin_lock(&hugetlb_lock);
1936                 h->nr_overcommit_huge_pages = tmp;
1937                 spin_unlock(&hugetlb_lock);
1938         }
1939
1940         return 0;
1941 }
1942
1943 #endif /* CONFIG_SYSCTL */
1944
1945 void hugetlb_report_meminfo(struct seq_file *m)
1946 {
1947         struct hstate *h = &default_hstate;
1948         seq_printf(m,
1949                         "HugePages_Total:   %5lu\n"
1950                         "HugePages_Free:    %5lu\n"
1951                         "HugePages_Rsvd:    %5lu\n"
1952                         "HugePages_Surp:    %5lu\n"
1953                         "Hugepagesize:   %8lu kB\n",
1954                         h->nr_huge_pages,
1955                         h->free_huge_pages,
1956                         h->resv_huge_pages,
1957                         h->surplus_huge_pages,
1958                         1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
1959 }
1960
1961 int hugetlb_report_node_meminfo(int nid, char *buf)
1962 {
1963         struct hstate *h = &default_hstate;
1964         return sprintf(buf,
1965                 "Node %d HugePages_Total: %5u\n"
1966                 "Node %d HugePages_Free:  %5u\n"
1967                 "Node %d HugePages_Surp:  %5u\n",
1968                 nid, h->nr_huge_pages_node[nid],
1969                 nid, h->free_huge_pages_node[nid],
1970                 nid, h->surplus_huge_pages_node[nid]);
1971 }
1972
1973 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
1974 unsigned long hugetlb_total_pages(void)
1975 {
1976         struct hstate *h = &default_hstate;
1977         return h->nr_huge_pages * pages_per_huge_page(h);
1978 }
1979
1980 static int hugetlb_acct_memory(struct hstate *h, long delta)
1981 {
1982         int ret = -ENOMEM;
1983
1984         spin_lock(&hugetlb_lock);
1985         /*
1986          * When cpuset is configured, it breaks the strict hugetlb page
1987          * reservation as the accounting is done on a global variable. Such
1988          * reservation is completely rubbish in the presence of cpuset because
1989          * the reservation is not checked against page availability for the
1990          * current cpuset. Application can still potentially OOM'ed by kernel
1991          * with lack of free htlb page in cpuset that the task is in.
1992          * Attempt to enforce strict accounting with cpuset is almost
1993          * impossible (or too ugly) because cpuset is too fluid that
1994          * task or memory node can be dynamically moved between cpusets.
1995          *
1996          * The change of semantics for shared hugetlb mapping with cpuset is
1997          * undesirable. However, in order to preserve some of the semantics,
1998          * we fall back to check against current free page availability as
1999          * a best attempt and hopefully to minimize the impact of changing
2000          * semantics that cpuset has.
2001          */
2002         if (delta > 0) {
2003                 if (gather_surplus_pages(h, delta) < 0)
2004                         goto out;
2005
2006                 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
2007                         return_unused_surplus_pages(h, delta);
2008                         goto out;
2009                 }
2010         }
2011
2012         ret = 0;
2013         if (delta < 0)
2014                 return_unused_surplus_pages(h, (unsigned long) -delta);
2015
2016 out:
2017         spin_unlock(&hugetlb_lock);
2018         return ret;
2019 }
2020
2021 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
2022 {
2023         struct resv_map *reservations = vma_resv_map(vma);
2024
2025         /*
2026          * This new VMA should share its siblings reservation map if present.
2027          * The VMA will only ever have a valid reservation map pointer where
2028          * it is being copied for another still existing VMA.  As that VMA
2029          * has a reference to the reservation map it cannot dissappear until
2030          * after this open call completes.  It is therefore safe to take a
2031          * new reference here without additional locking.
2032          */
2033         if (reservations)
2034                 kref_get(&reservations->refs);
2035 }
2036
2037 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
2038 {
2039         struct hstate *h = hstate_vma(vma);
2040         struct resv_map *reservations = vma_resv_map(vma);
2041         unsigned long reserve;
2042         unsigned long start;
2043         unsigned long end;
2044
2045         if (reservations) {
2046                 start = vma_hugecache_offset(h, vma, vma->vm_start);
2047                 end = vma_hugecache_offset(h, vma, vma->vm_end);
2048
2049                 reserve = (end - start) -
2050                         region_count(&reservations->regions, start, end);
2051
2052                 kref_put(&reservations->refs, resv_map_release);
2053
2054                 if (reserve) {
2055                         hugetlb_acct_memory(h, -reserve);
2056                         hugetlb_put_quota(vma->vm_file->f_mapping, reserve);
2057                 }
2058         }
2059 }
2060
2061 /*
2062  * We cannot handle pagefaults against hugetlb pages at all.  They cause
2063  * handle_mm_fault() to try to instantiate regular-sized pages in the
2064  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
2065  * this far.
2066  */
2067 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2068 {
2069         BUG();
2070         return 0;
2071 }
2072
2073 const struct vm_operations_struct hugetlb_vm_ops = {
2074         .fault = hugetlb_vm_op_fault,
2075         .open = hugetlb_vm_op_open,
2076         .close = hugetlb_vm_op_close,
2077 };
2078
2079 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
2080                                 int writable)
2081 {
2082         pte_t entry;
2083
2084         if (writable) {
2085                 entry =
2086                     pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
2087         } else {
2088                 entry = huge_pte_wrprotect(mk_pte(page, vma->vm_page_prot));
2089         }
2090         entry = pte_mkyoung(entry);
2091         entry = pte_mkhuge(entry);
2092
2093         return entry;
2094 }
2095
2096 static void set_huge_ptep_writable(struct vm_area_struct *vma,
2097                                    unsigned long address, pte_t *ptep)
2098 {
2099         pte_t entry;
2100
2101         entry = pte_mkwrite(pte_mkdirty(huge_ptep_get(ptep)));
2102         if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1)) {
2103                 update_mmu_cache(vma, address, ptep);
2104         }
2105 }
2106
2107
2108 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
2109                             struct vm_area_struct *vma)
2110 {
2111         pte_t *src_pte, *dst_pte, entry;
2112         struct page *ptepage;
2113         unsigned long addr;
2114         int cow;
2115         struct hstate *h = hstate_vma(vma);
2116         unsigned long sz = huge_page_size(h);
2117
2118         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
2119
2120         for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
2121                 src_pte = huge_pte_offset(src, addr);
2122                 if (!src_pte)
2123                         continue;
2124                 dst_pte = huge_pte_alloc(dst, addr, sz);
2125                 if (!dst_pte)
2126                         goto nomem;
2127
2128                 /* If the pagetables are shared don't copy or take references */
2129                 if (dst_pte == src_pte)
2130                         continue;
2131
2132                 spin_lock(&dst->page_table_lock);
2133                 spin_lock_nested(&src->page_table_lock, SINGLE_DEPTH_NESTING);
2134                 if (!huge_pte_none(huge_ptep_get(src_pte))) {
2135                         if (cow)
2136                                 huge_ptep_set_wrprotect(src, addr, src_pte);
2137                         entry = huge_ptep_get(src_pte);
2138                         ptepage = pte_page(entry);
2139                         get_page(ptepage);
2140                         page_dup_rmap(ptepage);
2141                         set_huge_pte_at(dst, addr, dst_pte, entry);
2142                 }
2143                 spin_unlock(&src->page_table_lock);
2144                 spin_unlock(&dst->page_table_lock);
2145         }
2146         return 0;
2147
2148 nomem:
2149         return -ENOMEM;
2150 }
2151
2152 void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2153                             unsigned long end, struct page *ref_page)
2154 {
2155         struct mm_struct *mm = vma->vm_mm;
2156         unsigned long address;
2157         pte_t *ptep;
2158         pte_t pte;
2159         struct page *page;
2160         struct page *tmp;
2161         struct hstate *h = hstate_vma(vma);
2162         unsigned long sz = huge_page_size(h);
2163
2164         /*
2165          * A page gathering list, protected by per file i_mmap_lock. The
2166          * lock is used to avoid list corruption from multiple unmapping
2167          * of the same page since we are using page->lru.
2168          */
2169         LIST_HEAD(page_list);
2170
2171         WARN_ON(!is_vm_hugetlb_page(vma));
2172         BUG_ON(start & ~huge_page_mask(h));
2173         BUG_ON(end & ~huge_page_mask(h));
2174
2175         mmu_notifier_invalidate_range_start(mm, start, end);
2176         spin_lock(&mm->page_table_lock);
2177         for (address = start; address < end; address += sz) {
2178                 ptep = huge_pte_offset(mm, address);
2179                 if (!ptep)
2180                         continue;
2181
2182                 if (huge_pmd_unshare(mm, &address, ptep))
2183                         continue;
2184
2185                 /*
2186                  * If a reference page is supplied, it is because a specific
2187                  * page is being unmapped, not a range. Ensure the page we
2188                  * are about to unmap is the actual page of interest.
2189                  */
2190                 if (ref_page) {
2191                         pte = huge_ptep_get(ptep);
2192                         if (huge_pte_none(pte))
2193                                 continue;
2194                         page = pte_page(pte);
2195                         if (page != ref_page)
2196                                 continue;
2197
2198                         /*
2199                          * Mark the VMA as having unmapped its page so that
2200                          * future faults in this VMA will fail rather than
2201                          * looking like data was lost
2202                          */
2203                         set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
2204                 }
2205
2206                 pte = huge_ptep_get_and_clear(mm, address, ptep);
2207                 if (huge_pte_none(pte))
2208                         continue;
2209
2210                 page = pte_page(pte);
2211                 if (pte_dirty(pte))
2212                         set_page_dirty(page);
2213                 list_add(&page->lru, &page_list);
2214         }
2215         spin_unlock(&mm->page_table_lock);
2216         flush_tlb_range(vma, start, end);
2217         mmu_notifier_invalidate_range_end(mm, start, end);
2218         list_for_each_entry_safe(page, tmp, &page_list, lru) {
2219                 page_remove_rmap(page);
2220                 list_del(&page->lru);
2221                 put_page(page);
2222         }
2223 }
2224
2225 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2226                           unsigned long end, struct page *ref_page)
2227 {
2228         spin_lock(&vma->vm_file->f_mapping->i_mmap_lock);
2229         __unmap_hugepage_range(vma, start, end, ref_page);
2230         spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
2231 }
2232
2233 /*
2234  * This is called when the original mapper is failing to COW a MAP_PRIVATE
2235  * mappping it owns the reserve page for. The intention is to unmap the page
2236  * from other VMAs and let the children be SIGKILLed if they are faulting the
2237  * same region.
2238  */
2239 static int unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
2240                                 struct page *page, unsigned long address)
2241 {
2242         struct hstate *h = hstate_vma(vma);
2243         struct vm_area_struct *iter_vma;
2244         struct address_space *mapping;
2245         struct prio_tree_iter iter;
2246         pgoff_t pgoff;
2247
2248         /*
2249          * vm_pgoff is in PAGE_SIZE units, hence the different calculation
2250          * from page cache lookup which is in HPAGE_SIZE units.
2251          */
2252         address = address & huge_page_mask(h);
2253         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT)
2254                 + (vma->vm_pgoff >> PAGE_SHIFT);
2255         mapping = (struct address_space *)page_private(page);
2256
2257         /*
2258          * Take the mapping lock for the duration of the table walk. As
2259          * this mapping should be shared between all the VMAs,
2260          * __unmap_hugepage_range() is called as the lock is already held
2261          */
2262         spin_lock(&mapping->i_mmap_lock);
2263         vma_prio_tree_foreach(iter_vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
2264                 /* Do not unmap the current VMA */
2265                 if (iter_vma == vma)
2266                         continue;
2267
2268                 /*
2269                  * Unmap the page from other VMAs without their own reserves.
2270                  * They get marked to be SIGKILLed if they fault in these
2271                  * areas. This is because a future no-page fault on this VMA
2272                  * could insert a zeroed page instead of the data existing
2273                  * from the time of fork. This would look like data corruption
2274                  */
2275                 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
2276                         __unmap_hugepage_range(iter_vma,
2277                                 address, address + huge_page_size(h),
2278                                 page);
2279         }
2280         spin_unlock(&mapping->i_mmap_lock);
2281
2282         return 1;
2283 }
2284
2285 /*
2286  * Hugetlb_cow() should be called with page lock of the original hugepage held.
2287  */
2288 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
2289                         unsigned long address, pte_t *ptep, pte_t pte,
2290                         struct page *pagecache_page)
2291 {
2292         struct hstate *h = hstate_vma(vma);
2293         struct page *old_page, *new_page;
2294         int avoidcopy;
2295         int outside_reserve = 0;
2296
2297         old_page = pte_page(pte);
2298
2299 retry_avoidcopy:
2300         /* If no-one else is actually using this page, avoid the copy
2301          * and just make the page writable */
2302         avoidcopy = (page_mapcount(old_page) == 1);
2303         if (avoidcopy) {
2304                 if (!trylock_page(old_page))
2305                         if (PageAnon(old_page))
2306                                 page_move_anon_rmap(old_page, vma, address);
2307                 set_huge_ptep_writable(vma, address, ptep);
2308                 return 0;
2309         }
2310
2311         /*
2312          * If the process that created a MAP_PRIVATE mapping is about to
2313          * perform a COW due to a shared page count, attempt to satisfy
2314          * the allocation without using the existing reserves. The pagecache
2315          * page is used to determine if the reserve at this address was
2316          * consumed or not. If reserves were used, a partial faulted mapping
2317          * at the time of fork() could consume its reserves on COW instead
2318          * of the full address range.
2319          */
2320         if (!(vma->vm_flags & VM_MAYSHARE) &&
2321                         is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
2322                         old_page != pagecache_page)
2323                 outside_reserve = 1;
2324
2325         page_cache_get(old_page);
2326
2327         /* Drop page_table_lock as buddy allocator may be called */
2328         spin_unlock(&mm->page_table_lock);
2329         new_page = alloc_huge_page(vma, address, outside_reserve);
2330
2331         if (IS_ERR(new_page)) {
2332                 page_cache_release(old_page);
2333
2334                 /*
2335                  * If a process owning a MAP_PRIVATE mapping fails to COW,
2336                  * it is due to references held by a child and an insufficient
2337                  * huge page pool. To guarantee the original mappers
2338                  * reliability, unmap the page from child processes. The child
2339                  * may get SIGKILLed if it later faults.
2340                  */
2341                 if (outside_reserve) {
2342                         BUG_ON(huge_pte_none(pte));
2343                         if (unmap_ref_private(mm, vma, old_page, address)) {
2344                                 BUG_ON(page_count(old_page) != 1);
2345                                 BUG_ON(huge_pte_none(pte));
2346                                 spin_lock(&mm->page_table_lock);
2347                                 goto retry_avoidcopy;
2348                         }
2349                         WARN_ON_ONCE(1);
2350                 }
2351
2352                 /* Caller expects lock to be held */
2353                 spin_lock(&mm->page_table_lock);
2354                 return -PTR_ERR(new_page);
2355         }
2356
2357         /*
2358          * When the original hugepage is shared one, it does not have
2359          * anon_vma prepared.
2360          */
2361         if (unlikely(anon_vma_prepare(vma)))
2362                 return VM_FAULT_OOM;
2363
2364         copy_huge_page(new_page, old_page, address, vma);
2365         __SetPageUptodate(new_page);
2366
2367         /*
2368          * Retake the page_table_lock to check for racing updates
2369          * before the page tables are altered
2370          */
2371         spin_lock(&mm->page_table_lock);
2372         ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2373         if (likely(pte_same(huge_ptep_get(ptep), pte))) {
2374                 /* Break COW */
2375                 huge_ptep_clear_flush(vma, address, ptep);
2376                 set_huge_pte_at(mm, address, ptep,
2377                                 make_huge_pte(vma, new_page, 1));
2378                 page_remove_rmap(old_page);
2379                 hugepage_add_anon_rmap(new_page, vma, address);
2380                 /* Make the old page be freed below */
2381                 new_page = old_page;
2382         }
2383         page_cache_release(new_page);
2384         page_cache_release(old_page);
2385         return 0;
2386 }
2387
2388 /* Return the pagecache page at a given address within a VMA */
2389 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
2390                         struct vm_area_struct *vma, unsigned long address)
2391 {
2392         struct address_space *mapping;
2393         pgoff_t idx;
2394
2395         mapping = vma->vm_file->f_mapping;
2396         idx = vma_hugecache_offset(h, vma, address);
2397
2398         return find_lock_page(mapping, idx);
2399 }
2400
2401 /*
2402  * Return whether there is a pagecache page to back given address within VMA.
2403  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
2404  */
2405 static bool hugetlbfs_pagecache_present(struct hstate *h,
2406                         struct vm_area_struct *vma, unsigned long address)
2407 {
2408         struct address_space *mapping;
2409         pgoff_t idx;
2410         struct page *page;
2411
2412         mapping = vma->vm_file->f_mapping;
2413         idx = vma_hugecache_offset(h, vma, address);
2414
2415         page = find_get_page(mapping, idx);
2416         if (page)
2417                 put_page(page);
2418         return page != NULL;
2419 }
2420
2421 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
2422                         unsigned long address, pte_t *ptep, unsigned int flags)
2423 {
2424         struct hstate *h = hstate_vma(vma);
2425         int ret = VM_FAULT_SIGBUS;
2426         pgoff_t idx;
2427         unsigned long size;
2428         struct page *page;
2429         struct address_space *mapping;
2430         pte_t new_pte;
2431
2432         /*
2433          * Currently, we are forced to kill the process in the event the
2434          * original mapper has unmapped pages from the child due to a failed
2435          * COW. Warn that such a situation has occured as it may not be obvious
2436          */
2437         if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
2438                 printk(KERN_WARNING
2439                         "PID %d killed due to inadequate hugepage pool\n",
2440                         current->pid);
2441                 return ret;
2442         }
2443
2444         mapping = vma->vm_file->f_mapping;
2445         idx = vma_hugecache_offset(h, vma, address);
2446
2447         /*
2448          * Use page lock to guard against racing truncation
2449          * before we get page_table_lock.
2450          */
2451 retry:
2452         page = find_lock_page(mapping, idx);
2453         if (!page) {
2454                 size = i_size_read(mapping->host) >> huge_page_shift(h);
2455                 if (idx >= size)
2456                         goto out;
2457                 page = alloc_huge_page(vma, address, 0);
2458                 if (IS_ERR(page)) {
2459                         ret = -PTR_ERR(page);
2460                         goto out;
2461                 }
2462                 clear_huge_page(page, address, huge_page_size(h));
2463                 __SetPageUptodate(page);
2464
2465                 if (vma->vm_flags & VM_MAYSHARE) {
2466                         int err;
2467                         struct inode *inode = mapping->host;
2468
2469                         err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
2470                         if (err) {
2471                                 put_page(page);
2472                                 if (err == -EEXIST)
2473                                         goto retry;
2474                                 goto out;
2475                         }
2476
2477                         spin_lock(&inode->i_lock);
2478                         inode->i_blocks += blocks_per_huge_page(h);
2479                         spin_unlock(&inode->i_lock);
2480                         page_dup_rmap(page);
2481                 } else {
2482                         lock_page(page);
2483                         if (unlikely(anon_vma_prepare(vma))) {
2484                                 ret = VM_FAULT_OOM;
2485                                 goto backout_unlocked;
2486                         }
2487                         hugepage_add_new_anon_rmap(page, vma, address);
2488                 }
2489         } else {
2490                 page_dup_rmap(page);
2491         }
2492
2493         /*
2494          * If we are going to COW a private mapping later, we examine the
2495          * pending reservations for this page now. This will ensure that
2496          * any allocations necessary to record that reservation occur outside
2497          * the spinlock.
2498          */
2499         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
2500                 if (vma_needs_reservation(h, vma, address) < 0) {
2501                         ret = VM_FAULT_OOM;
2502                         goto backout_unlocked;
2503                 }
2504
2505         spin_lock(&mm->page_table_lock);
2506         size = i_size_read(mapping->host) >> huge_page_shift(h);
2507         if (idx >= size)
2508                 goto backout;
2509
2510         ret = 0;
2511         if (!huge_pte_none(huge_ptep_get(ptep)))
2512                 goto backout;
2513
2514         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
2515                                 && (vma->vm_flags & VM_SHARED)));
2516         set_huge_pte_at(mm, address, ptep, new_pte);
2517
2518         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
2519                 /* Optimization, do the COW without a second fault */
2520                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page);
2521         }
2522
2523         spin_unlock(&mm->page_table_lock);
2524         unlock_page(page);
2525 out:
2526         return ret;
2527
2528 backout:
2529         spin_unlock(&mm->page_table_lock);
2530 backout_unlocked:
2531         unlock_page(page);
2532         put_page(page);
2533         goto out;
2534 }
2535
2536 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
2537                         unsigned long address, unsigned int flags)
2538 {
2539         pte_t *ptep;
2540         pte_t entry;
2541         int ret;
2542         struct page *page = NULL;
2543         struct page *pagecache_page = NULL;
2544         static DEFINE_MUTEX(hugetlb_instantiation_mutex);
2545         struct hstate *h = hstate_vma(vma);
2546
2547         ptep = huge_pte_alloc(mm, address, huge_page_size(h));
2548         if (!ptep)
2549                 return VM_FAULT_OOM;
2550
2551         /*
2552          * Serialize hugepage allocation and instantiation, so that we don't
2553          * get spurious allocation failures if two CPUs race to instantiate
2554          * the same page in the page cache.
2555          */
2556         mutex_lock(&hugetlb_instantiation_mutex);
2557         entry = huge_ptep_get(ptep);
2558         if (huge_pte_none(entry)) {
2559                 ret = hugetlb_no_page(mm, vma, address, ptep, flags);
2560                 goto out_mutex;
2561         }
2562
2563         ret = 0;
2564
2565         /*
2566          * If we are going to COW the mapping later, we examine the pending
2567          * reservations for this page now. This will ensure that any
2568          * allocations necessary to record that reservation occur outside the
2569          * spinlock. For private mappings, we also lookup the pagecache
2570          * page now as it is used to determine if a reservation has been
2571          * consumed.
2572          */
2573         if ((flags & FAULT_FLAG_WRITE) && !pte_write(entry)) {
2574                 if (vma_needs_reservation(h, vma, address) < 0) {
2575                         ret = VM_FAULT_OOM;
2576                         goto out_mutex;
2577                 }
2578
2579                 if (!(vma->vm_flags & VM_MAYSHARE))
2580                         pagecache_page = hugetlbfs_pagecache_page(h,
2581                                                                 vma, address);
2582         }
2583
2584         if (!pagecache_page) {
2585                 page = pte_page(entry);
2586                 lock_page(page);
2587         }
2588
2589         spin_lock(&mm->page_table_lock);
2590         /* Check for a racing update before calling hugetlb_cow */
2591         if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
2592                 goto out_page_table_lock;
2593
2594
2595         if (flags & FAULT_FLAG_WRITE) {
2596                 if (!pte_write(entry)) {
2597                         ret = hugetlb_cow(mm, vma, address, ptep, entry,
2598                                                         pagecache_page);
2599                         goto out_page_table_lock;
2600                 }
2601                 entry = pte_mkdirty(entry);
2602         }
2603         entry = pte_mkyoung(entry);
2604         if (huge_ptep_set_access_flags(vma, address, ptep, entry,
2605                                                 flags & FAULT_FLAG_WRITE))
2606                 update_mmu_cache(vma, address, ptep);
2607
2608 out_page_table_lock:
2609         spin_unlock(&mm->page_table_lock);
2610
2611         if (pagecache_page) {
2612                 unlock_page(pagecache_page);
2613                 put_page(pagecache_page);
2614         } else {
2615                 unlock_page(page);
2616         }
2617
2618 out_mutex:
2619         mutex_unlock(&hugetlb_instantiation_mutex);
2620
2621         return ret;
2622 }
2623
2624 /* Can be overriden by architectures */
2625 __attribute__((weak)) struct page *
2626 follow_huge_pud(struct mm_struct *mm, unsigned long address,
2627                pud_t *pud, int write)
2628 {
2629         BUG();
2630         return NULL;
2631 }
2632
2633 int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
2634                         struct page **pages, struct vm_area_struct **vmas,
2635                         unsigned long *position, int *length, int i,
2636                         unsigned int flags)
2637 {
2638         unsigned long pfn_offset;
2639         unsigned long vaddr = *position;
2640         int remainder = *length;
2641         struct hstate *h = hstate_vma(vma);
2642
2643         spin_lock(&mm->page_table_lock);
2644         while (vaddr < vma->vm_end && remainder) {
2645                 pte_t *pte;
2646                 int absent;
2647                 struct page *page;
2648
2649                 /*
2650                  * Some archs (sparc64, sh*) have multiple pte_ts to
2651                  * each hugepage.  We have to make sure we get the
2652                  * first, for the page indexing below to work.
2653                  */
2654                 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
2655                 absent = !pte || huge_pte_none(huge_ptep_get(pte));
2656
2657                 /*
2658                  * When coredumping, it suits get_dump_page if we just return
2659                  * an error where there's an empty slot with no huge pagecache
2660                  * to back it.  This way, we avoid allocating a hugepage, and
2661                  * the sparse dumpfile avoids allocating disk blocks, but its
2662                  * huge holes still show up with zeroes where they need to be.
2663                  */
2664                 if (absent && (flags & FOLL_DUMP) &&
2665                     !hugetlbfs_pagecache_present(h, vma, vaddr)) {
2666                         remainder = 0;
2667                         break;
2668                 }
2669
2670                 if (absent ||
2671                     ((flags & FOLL_WRITE) && !pte_write(huge_ptep_get(pte)))) {
2672                         int ret;
2673
2674                         spin_unlock(&mm->page_table_lock);
2675                         ret = hugetlb_fault(mm, vma, vaddr,
2676                                 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
2677                         spin_lock(&mm->page_table_lock);
2678                         if (!(ret & VM_FAULT_ERROR))
2679                                 continue;
2680
2681                         remainder = 0;
2682                         break;
2683                 }
2684
2685                 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
2686                 page = pte_page(huge_ptep_get(pte));
2687 same_page:
2688                 if (pages) {
2689                         pages[i] = mem_map_offset(page, pfn_offset);
2690                         get_page(pages[i]);
2691                 }
2692
2693                 if (vmas)
2694                         vmas[i] = vma;
2695
2696                 vaddr += PAGE_SIZE;
2697                 ++pfn_offset;
2698                 --remainder;
2699                 ++i;
2700                 if (vaddr < vma->vm_end && remainder &&
2701                                 pfn_offset < pages_per_huge_page(h)) {
2702                         /*
2703                          * We use pfn_offset to avoid touching the pageframes
2704                          * of this compound page.
2705                          */
2706                         goto same_page;
2707                 }
2708         }
2709         spin_unlock(&mm->page_table_lock);
2710         *length = remainder;
2711         *position = vaddr;
2712
2713         return i ? i : -EFAULT;
2714 }
2715
2716 void hugetlb_change_protection(struct vm_area_struct *vma,
2717                 unsigned long address, unsigned long end, pgprot_t newprot)
2718 {
2719         struct mm_struct *mm = vma->vm_mm;
2720         unsigned long start = address;
2721         pte_t *ptep;
2722         pte_t pte;
2723         struct hstate *h = hstate_vma(vma);
2724
2725         BUG_ON(address >= end);
2726         flush_cache_range(vma, address, end);
2727
2728         spin_lock(&vma->vm_file->f_mapping->i_mmap_lock);
2729         spin_lock(&mm->page_table_lock);
2730         for (; address < end; address += huge_page_size(h)) {
2731                 ptep = huge_pte_offset(mm, address);
2732                 if (!ptep)
2733                         continue;
2734                 if (huge_pmd_unshare(mm, &address, ptep))
2735                         continue;
2736                 if (!huge_pte_none(huge_ptep_get(ptep))) {
2737                         pte = huge_ptep_get_and_clear(mm, address, ptep);
2738                         pte = pte_mkhuge(pte_modify(pte, newprot));
2739                         set_huge_pte_at(mm, address, ptep, pte);
2740                 }
2741         }
2742         spin_unlock(&mm->page_table_lock);
2743         spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
2744
2745         flush_tlb_range(vma, start, end);
2746 }
2747
2748 int hugetlb_reserve_pages(struct inode *inode,
2749                                         long from, long to,
2750                                         struct vm_area_struct *vma,
2751                                         int acctflag)
2752 {
2753         long ret, chg;
2754         struct hstate *h = hstate_inode(inode);
2755
2756         /*
2757          * Only apply hugepage reservation if asked. At fault time, an
2758          * attempt will be made for VM_NORESERVE to allocate a page
2759          * and filesystem quota without using reserves
2760          */
2761         if (acctflag & VM_NORESERVE)
2762                 return 0;
2763
2764         /*
2765          * Shared mappings base their reservation on the number of pages that
2766          * are already allocated on behalf of the file. Private mappings need
2767          * to reserve the full area even if read-only as mprotect() may be
2768          * called to make the mapping read-write. Assume !vma is a shm mapping
2769          */
2770         if (!vma || vma->vm_flags & VM_MAYSHARE)
2771                 chg = region_chg(&inode->i_mapping->private_list, from, to);
2772         else {
2773                 struct resv_map *resv_map = resv_map_alloc();
2774                 if (!resv_map)
2775                         return -ENOMEM;
2776
2777                 chg = to - from;
2778
2779                 set_vma_resv_map(vma, resv_map);
2780                 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
2781         }
2782
2783         if (chg < 0)
2784                 return chg;
2785
2786         /* There must be enough filesystem quota for the mapping */
2787         if (hugetlb_get_quota(inode->i_mapping, chg))
2788                 return -ENOSPC;
2789
2790         /*
2791          * Check enough hugepages are available for the reservation.
2792          * Hand back the quota if there are not
2793          */
2794         ret = hugetlb_acct_memory(h, chg);
2795         if (ret < 0) {
2796                 hugetlb_put_quota(inode->i_mapping, chg);
2797                 return ret;
2798         }
2799
2800         /*
2801          * Account for the reservations made. Shared mappings record regions
2802          * that have reservations as they are shared by multiple VMAs.
2803          * When the last VMA disappears, the region map says how much
2804          * the reservation was and the page cache tells how much of
2805          * the reservation was consumed. Private mappings are per-VMA and
2806          * only the consumed reservations are tracked. When the VMA
2807          * disappears, the original reservation is the VMA size and the
2808          * consumed reservations are stored in the map. Hence, nothing
2809          * else has to be done for private mappings here
2810          */
2811         if (!vma || vma->vm_flags & VM_MAYSHARE)
2812                 region_add(&inode->i_mapping->private_list, from, to);
2813         return 0;
2814 }
2815
2816 void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
2817 {
2818         struct hstate *h = hstate_inode(inode);
2819         long chg = region_truncate(&inode->i_mapping->private_list, offset);
2820
2821         spin_lock(&inode->i_lock);
2822         inode->i_blocks -= (blocks_per_huge_page(h) * freed);
2823         spin_unlock(&inode->i_lock);
2824
2825         hugetlb_put_quota(inode->i_mapping, (chg - freed));
2826         hugetlb_acct_memory(h, -(chg - freed));
2827 }
2828
2829 /*
2830  * This function is called from memory failure code.
2831  * Assume the caller holds page lock of the head page.
2832  */
2833 void __isolate_hwpoisoned_huge_page(struct page *hpage)
2834 {
2835         struct hstate *h = page_hstate(hpage);
2836         int nid = page_to_nid(hpage);
2837
2838         spin_lock(&hugetlb_lock);
2839         list_del(&hpage->lru);
2840         h->free_huge_pages--;
2841         h->free_huge_pages_node[nid]--;
2842         spin_unlock(&hugetlb_lock);
2843 }