hugetlb: call mmu notifiers on hugepage cow
[linux-2.6.git] / mm / hugetlb.c
1 /*
2  * Generic hugetlb support.
3  * (C) William Irwin, April 2004
4  */
5 #include <linux/list.h>
6 #include <linux/init.h>
7 #include <linux/module.h>
8 #include <linux/mm.h>
9 #include <linux/seq_file.h>
10 #include <linux/sysctl.h>
11 #include <linux/highmem.h>
12 #include <linux/mmu_notifier.h>
13 #include <linux/nodemask.h>
14 #include <linux/pagemap.h>
15 #include <linux/mempolicy.h>
16 #include <linux/cpuset.h>
17 #include <linux/mutex.h>
18 #include <linux/bootmem.h>
19 #include <linux/sysfs.h>
20 #include <linux/slab.h>
21
22 #include <asm/page.h>
23 #include <asm/pgtable.h>
24 #include <asm/io.h>
25
26 #include <linux/hugetlb.h>
27 #include <linux/node.h>
28 #include "internal.h"
29
30 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
31 static gfp_t htlb_alloc_mask = GFP_HIGHUSER;
32 unsigned long hugepages_treat_as_movable;
33
34 static int max_hstate;
35 unsigned int default_hstate_idx;
36 struct hstate hstates[HUGE_MAX_HSTATE];
37
38 __initdata LIST_HEAD(huge_boot_pages);
39
40 /* for command line parsing */
41 static struct hstate * __initdata parsed_hstate;
42 static unsigned long __initdata default_hstate_max_huge_pages;
43 static unsigned long __initdata default_hstate_size;
44
45 #define for_each_hstate(h) \
46         for ((h) = hstates; (h) < &hstates[max_hstate]; (h)++)
47
48 /*
49  * Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
50  */
51 static DEFINE_SPINLOCK(hugetlb_lock);
52
53 /*
54  * Region tracking -- allows tracking of reservations and instantiated pages
55  *                    across the pages in a mapping.
56  *
57  * The region data structures are protected by a combination of the mmap_sem
58  * and the hugetlb_instantion_mutex.  To access or modify a region the caller
59  * must either hold the mmap_sem for write, or the mmap_sem for read and
60  * the hugetlb_instantiation mutex:
61  *
62  *      down_write(&mm->mmap_sem);
63  * or
64  *      down_read(&mm->mmap_sem);
65  *      mutex_lock(&hugetlb_instantiation_mutex);
66  */
67 struct file_region {
68         struct list_head link;
69         long from;
70         long to;
71 };
72
73 static long region_add(struct list_head *head, long f, long t)
74 {
75         struct file_region *rg, *nrg, *trg;
76
77         /* Locate the region we are either in or before. */
78         list_for_each_entry(rg, head, link)
79                 if (f <= rg->to)
80                         break;
81
82         /* Round our left edge to the current segment if it encloses us. */
83         if (f > rg->from)
84                 f = rg->from;
85
86         /* Check for and consume any regions we now overlap with. */
87         nrg = rg;
88         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
89                 if (&rg->link == head)
90                         break;
91                 if (rg->from > t)
92                         break;
93
94                 /* If this area reaches higher then extend our area to
95                  * include it completely.  If this is not the first area
96                  * which we intend to reuse, free it. */
97                 if (rg->to > t)
98                         t = rg->to;
99                 if (rg != nrg) {
100                         list_del(&rg->link);
101                         kfree(rg);
102                 }
103         }
104         nrg->from = f;
105         nrg->to = t;
106         return 0;
107 }
108
109 static long region_chg(struct list_head *head, long f, long t)
110 {
111         struct file_region *rg, *nrg;
112         long chg = 0;
113
114         /* Locate the region we are before or in. */
115         list_for_each_entry(rg, head, link)
116                 if (f <= rg->to)
117                         break;
118
119         /* If we are below the current region then a new region is required.
120          * Subtle, allocate a new region at the position but make it zero
121          * size such that we can guarantee to record the reservation. */
122         if (&rg->link == head || t < rg->from) {
123                 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
124                 if (!nrg)
125                         return -ENOMEM;
126                 nrg->from = f;
127                 nrg->to   = f;
128                 INIT_LIST_HEAD(&nrg->link);
129                 list_add(&nrg->link, rg->link.prev);
130
131                 return t - f;
132         }
133
134         /* Round our left edge to the current segment if it encloses us. */
135         if (f > rg->from)
136                 f = rg->from;
137         chg = t - f;
138
139         /* Check for and consume any regions we now overlap with. */
140         list_for_each_entry(rg, rg->link.prev, link) {
141                 if (&rg->link == head)
142                         break;
143                 if (rg->from > t)
144                         return chg;
145
146                 /* We overlap with this area, if it extends futher than
147                  * us then we must extend ourselves.  Account for its
148                  * existing reservation. */
149                 if (rg->to > t) {
150                         chg += rg->to - t;
151                         t = rg->to;
152                 }
153                 chg -= rg->to - rg->from;
154         }
155         return chg;
156 }
157
158 static long region_truncate(struct list_head *head, long end)
159 {
160         struct file_region *rg, *trg;
161         long chg = 0;
162
163         /* Locate the region we are either in or before. */
164         list_for_each_entry(rg, head, link)
165                 if (end <= rg->to)
166                         break;
167         if (&rg->link == head)
168                 return 0;
169
170         /* If we are in the middle of a region then adjust it. */
171         if (end > rg->from) {
172                 chg = rg->to - end;
173                 rg->to = end;
174                 rg = list_entry(rg->link.next, typeof(*rg), link);
175         }
176
177         /* Drop any remaining regions. */
178         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
179                 if (&rg->link == head)
180                         break;
181                 chg += rg->to - rg->from;
182                 list_del(&rg->link);
183                 kfree(rg);
184         }
185         return chg;
186 }
187
188 static long region_count(struct list_head *head, long f, long t)
189 {
190         struct file_region *rg;
191         long chg = 0;
192
193         /* Locate each segment we overlap with, and count that overlap. */
194         list_for_each_entry(rg, head, link) {
195                 int seg_from;
196                 int seg_to;
197
198                 if (rg->to <= f)
199                         continue;
200                 if (rg->from >= t)
201                         break;
202
203                 seg_from = max(rg->from, f);
204                 seg_to = min(rg->to, t);
205
206                 chg += seg_to - seg_from;
207         }
208
209         return chg;
210 }
211
212 /*
213  * Convert the address within this vma to the page offset within
214  * the mapping, in pagecache page units; huge pages here.
215  */
216 static pgoff_t vma_hugecache_offset(struct hstate *h,
217                         struct vm_area_struct *vma, unsigned long address)
218 {
219         return ((address - vma->vm_start) >> huge_page_shift(h)) +
220                         (vma->vm_pgoff >> huge_page_order(h));
221 }
222
223 /*
224  * Return the size of the pages allocated when backing a VMA. In the majority
225  * cases this will be same size as used by the page table entries.
226  */
227 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
228 {
229         struct hstate *hstate;
230
231         if (!is_vm_hugetlb_page(vma))
232                 return PAGE_SIZE;
233
234         hstate = hstate_vma(vma);
235
236         return 1UL << (hstate->order + PAGE_SHIFT);
237 }
238 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
239
240 /*
241  * Return the page size being used by the MMU to back a VMA. In the majority
242  * of cases, the page size used by the kernel matches the MMU size. On
243  * architectures where it differs, an architecture-specific version of this
244  * function is required.
245  */
246 #ifndef vma_mmu_pagesize
247 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
248 {
249         return vma_kernel_pagesize(vma);
250 }
251 #endif
252
253 /*
254  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
255  * bits of the reservation map pointer, which are always clear due to
256  * alignment.
257  */
258 #define HPAGE_RESV_OWNER    (1UL << 0)
259 #define HPAGE_RESV_UNMAPPED (1UL << 1)
260 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
261
262 /*
263  * These helpers are used to track how many pages are reserved for
264  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
265  * is guaranteed to have their future faults succeed.
266  *
267  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
268  * the reserve counters are updated with the hugetlb_lock held. It is safe
269  * to reset the VMA at fork() time as it is not in use yet and there is no
270  * chance of the global counters getting corrupted as a result of the values.
271  *
272  * The private mapping reservation is represented in a subtly different
273  * manner to a shared mapping.  A shared mapping has a region map associated
274  * with the underlying file, this region map represents the backing file
275  * pages which have ever had a reservation assigned which this persists even
276  * after the page is instantiated.  A private mapping has a region map
277  * associated with the original mmap which is attached to all VMAs which
278  * reference it, this region map represents those offsets which have consumed
279  * reservation ie. where pages have been instantiated.
280  */
281 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
282 {
283         return (unsigned long)vma->vm_private_data;
284 }
285
286 static void set_vma_private_data(struct vm_area_struct *vma,
287                                                         unsigned long value)
288 {
289         vma->vm_private_data = (void *)value;
290 }
291
292 struct resv_map {
293         struct kref refs;
294         struct list_head regions;
295 };
296
297 static struct resv_map *resv_map_alloc(void)
298 {
299         struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
300         if (!resv_map)
301                 return NULL;
302
303         kref_init(&resv_map->refs);
304         INIT_LIST_HEAD(&resv_map->regions);
305
306         return resv_map;
307 }
308
309 static void resv_map_release(struct kref *ref)
310 {
311         struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
312
313         /* Clear out any active regions before we release the map. */
314         region_truncate(&resv_map->regions, 0);
315         kfree(resv_map);
316 }
317
318 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
319 {
320         VM_BUG_ON(!is_vm_hugetlb_page(vma));
321         if (!(vma->vm_flags & VM_MAYSHARE))
322                 return (struct resv_map *)(get_vma_private_data(vma) &
323                                                         ~HPAGE_RESV_MASK);
324         return NULL;
325 }
326
327 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
328 {
329         VM_BUG_ON(!is_vm_hugetlb_page(vma));
330         VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
331
332         set_vma_private_data(vma, (get_vma_private_data(vma) &
333                                 HPAGE_RESV_MASK) | (unsigned long)map);
334 }
335
336 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
337 {
338         VM_BUG_ON(!is_vm_hugetlb_page(vma));
339         VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
340
341         set_vma_private_data(vma, get_vma_private_data(vma) | flags);
342 }
343
344 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
345 {
346         VM_BUG_ON(!is_vm_hugetlb_page(vma));
347
348         return (get_vma_private_data(vma) & flag) != 0;
349 }
350
351 /* Decrement the reserved pages in the hugepage pool by one */
352 static void decrement_hugepage_resv_vma(struct hstate *h,
353                         struct vm_area_struct *vma)
354 {
355         if (vma->vm_flags & VM_NORESERVE)
356                 return;
357
358         if (vma->vm_flags & VM_MAYSHARE) {
359                 /* Shared mappings always use reserves */
360                 h->resv_huge_pages--;
361         } else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
362                 /*
363                  * Only the process that called mmap() has reserves for
364                  * private mappings.
365                  */
366                 h->resv_huge_pages--;
367         }
368 }
369
370 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
371 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
372 {
373         VM_BUG_ON(!is_vm_hugetlb_page(vma));
374         if (!(vma->vm_flags & VM_MAYSHARE))
375                 vma->vm_private_data = (void *)0;
376 }
377
378 /* Returns true if the VMA has associated reserve pages */
379 static int vma_has_reserves(struct vm_area_struct *vma)
380 {
381         if (vma->vm_flags & VM_MAYSHARE)
382                 return 1;
383         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
384                 return 1;
385         return 0;
386 }
387
388 static void clear_gigantic_page(struct page *page,
389                         unsigned long addr, unsigned long sz)
390 {
391         int i;
392         struct page *p = page;
393
394         might_sleep();
395         for (i = 0; i < sz/PAGE_SIZE; i++, p = mem_map_next(p, page, i)) {
396                 cond_resched();
397                 clear_user_highpage(p, addr + i * PAGE_SIZE);
398         }
399 }
400 static void clear_huge_page(struct page *page,
401                         unsigned long addr, unsigned long sz)
402 {
403         int i;
404
405         if (unlikely(sz/PAGE_SIZE > MAX_ORDER_NR_PAGES)) {
406                 clear_gigantic_page(page, addr, sz);
407                 return;
408         }
409
410         might_sleep();
411         for (i = 0; i < sz/PAGE_SIZE; i++) {
412                 cond_resched();
413                 clear_user_highpage(page + i, addr + i * PAGE_SIZE);
414         }
415 }
416
417 static void copy_gigantic_page(struct page *dst, struct page *src,
418                            unsigned long addr, struct vm_area_struct *vma)
419 {
420         int i;
421         struct hstate *h = hstate_vma(vma);
422         struct page *dst_base = dst;
423         struct page *src_base = src;
424         might_sleep();
425         for (i = 0; i < pages_per_huge_page(h); ) {
426                 cond_resched();
427                 copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma);
428
429                 i++;
430                 dst = mem_map_next(dst, dst_base, i);
431                 src = mem_map_next(src, src_base, i);
432         }
433 }
434 static void copy_huge_page(struct page *dst, struct page *src,
435                            unsigned long addr, struct vm_area_struct *vma)
436 {
437         int i;
438         struct hstate *h = hstate_vma(vma);
439
440         if (unlikely(pages_per_huge_page(h) > MAX_ORDER_NR_PAGES)) {
441                 copy_gigantic_page(dst, src, addr, vma);
442                 return;
443         }
444
445         might_sleep();
446         for (i = 0; i < pages_per_huge_page(h); i++) {
447                 cond_resched();
448                 copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE, vma);
449         }
450 }
451
452 static void enqueue_huge_page(struct hstate *h, struct page *page)
453 {
454         int nid = page_to_nid(page);
455         list_add(&page->lru, &h->hugepage_freelists[nid]);
456         h->free_huge_pages++;
457         h->free_huge_pages_node[nid]++;
458 }
459
460 static struct page *dequeue_huge_page_vma(struct hstate *h,
461                                 struct vm_area_struct *vma,
462                                 unsigned long address, int avoid_reserve)
463 {
464         int nid;
465         struct page *page = NULL;
466         struct mempolicy *mpol;
467         nodemask_t *nodemask;
468         struct zonelist *zonelist;
469         struct zone *zone;
470         struct zoneref *z;
471
472         get_mems_allowed();
473         zonelist = huge_zonelist(vma, address,
474                                         htlb_alloc_mask, &mpol, &nodemask);
475         /*
476          * A child process with MAP_PRIVATE mappings created by their parent
477          * have no page reserves. This check ensures that reservations are
478          * not "stolen". The child may still get SIGKILLed
479          */
480         if (!vma_has_reserves(vma) &&
481                         h->free_huge_pages - h->resv_huge_pages == 0)
482                 goto err;
483
484         /* If reserves cannot be used, ensure enough pages are in the pool */
485         if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
486                 goto err;;
487
488         for_each_zone_zonelist_nodemask(zone, z, zonelist,
489                                                 MAX_NR_ZONES - 1, nodemask) {
490                 nid = zone_to_nid(zone);
491                 if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask) &&
492                     !list_empty(&h->hugepage_freelists[nid])) {
493                         page = list_entry(h->hugepage_freelists[nid].next,
494                                           struct page, lru);
495                         list_del(&page->lru);
496                         h->free_huge_pages--;
497                         h->free_huge_pages_node[nid]--;
498
499                         if (!avoid_reserve)
500                                 decrement_hugepage_resv_vma(h, vma);
501
502                         break;
503                 }
504         }
505 err:
506         mpol_cond_put(mpol);
507         put_mems_allowed();
508         return page;
509 }
510
511 static void update_and_free_page(struct hstate *h, struct page *page)
512 {
513         int i;
514
515         VM_BUG_ON(h->order >= MAX_ORDER);
516
517         h->nr_huge_pages--;
518         h->nr_huge_pages_node[page_to_nid(page)]--;
519         for (i = 0; i < pages_per_huge_page(h); i++) {
520                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
521                                 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
522                                 1 << PG_private | 1<< PG_writeback);
523         }
524         set_compound_page_dtor(page, NULL);
525         set_page_refcounted(page);
526         arch_release_hugepage(page);
527         __free_pages(page, huge_page_order(h));
528 }
529
530 struct hstate *size_to_hstate(unsigned long size)
531 {
532         struct hstate *h;
533
534         for_each_hstate(h) {
535                 if (huge_page_size(h) == size)
536                         return h;
537         }
538         return NULL;
539 }
540
541 static void free_huge_page(struct page *page)
542 {
543         /*
544          * Can't pass hstate in here because it is called from the
545          * compound page destructor.
546          */
547         struct hstate *h = page_hstate(page);
548         int nid = page_to_nid(page);
549         struct address_space *mapping;
550
551         mapping = (struct address_space *) page_private(page);
552         set_page_private(page, 0);
553         page->mapping = NULL;
554         BUG_ON(page_count(page));
555         INIT_LIST_HEAD(&page->lru);
556
557         spin_lock(&hugetlb_lock);
558         if (h->surplus_huge_pages_node[nid] && huge_page_order(h) < MAX_ORDER) {
559                 update_and_free_page(h, page);
560                 h->surplus_huge_pages--;
561                 h->surplus_huge_pages_node[nid]--;
562         } else {
563                 enqueue_huge_page(h, page);
564         }
565         spin_unlock(&hugetlb_lock);
566         if (mapping)
567                 hugetlb_put_quota(mapping, 1);
568 }
569
570 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
571 {
572         set_compound_page_dtor(page, free_huge_page);
573         spin_lock(&hugetlb_lock);
574         h->nr_huge_pages++;
575         h->nr_huge_pages_node[nid]++;
576         spin_unlock(&hugetlb_lock);
577         put_page(page); /* free it into the hugepage allocator */
578 }
579
580 static void prep_compound_gigantic_page(struct page *page, unsigned long order)
581 {
582         int i;
583         int nr_pages = 1 << order;
584         struct page *p = page + 1;
585
586         /* we rely on prep_new_huge_page to set the destructor */
587         set_compound_order(page, order);
588         __SetPageHead(page);
589         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
590                 __SetPageTail(p);
591                 p->first_page = page;
592         }
593 }
594
595 int PageHuge(struct page *page)
596 {
597         compound_page_dtor *dtor;
598
599         if (!PageCompound(page))
600                 return 0;
601
602         page = compound_head(page);
603         dtor = get_compound_page_dtor(page);
604
605         return dtor == free_huge_page;
606 }
607
608 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
609 {
610         struct page *page;
611
612         if (h->order >= MAX_ORDER)
613                 return NULL;
614
615         page = alloc_pages_exact_node(nid,
616                 htlb_alloc_mask|__GFP_COMP|__GFP_THISNODE|
617                                                 __GFP_REPEAT|__GFP_NOWARN,
618                 huge_page_order(h));
619         if (page) {
620                 if (arch_prepare_hugepage(page)) {
621                         __free_pages(page, huge_page_order(h));
622                         return NULL;
623                 }
624                 prep_new_huge_page(h, page, nid);
625         }
626
627         return page;
628 }
629
630 /*
631  * common helper functions for hstate_next_node_to_{alloc|free}.
632  * We may have allocated or freed a huge page based on a different
633  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
634  * be outside of *nodes_allowed.  Ensure that we use an allowed
635  * node for alloc or free.
636  */
637 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
638 {
639         nid = next_node(nid, *nodes_allowed);
640         if (nid == MAX_NUMNODES)
641                 nid = first_node(*nodes_allowed);
642         VM_BUG_ON(nid >= MAX_NUMNODES);
643
644         return nid;
645 }
646
647 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
648 {
649         if (!node_isset(nid, *nodes_allowed))
650                 nid = next_node_allowed(nid, nodes_allowed);
651         return nid;
652 }
653
654 /*
655  * returns the previously saved node ["this node"] from which to
656  * allocate a persistent huge page for the pool and advance the
657  * next node from which to allocate, handling wrap at end of node
658  * mask.
659  */
660 static int hstate_next_node_to_alloc(struct hstate *h,
661                                         nodemask_t *nodes_allowed)
662 {
663         int nid;
664
665         VM_BUG_ON(!nodes_allowed);
666
667         nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
668         h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
669
670         return nid;
671 }
672
673 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
674 {
675         struct page *page;
676         int start_nid;
677         int next_nid;
678         int ret = 0;
679
680         start_nid = hstate_next_node_to_alloc(h, nodes_allowed);
681         next_nid = start_nid;
682
683         do {
684                 page = alloc_fresh_huge_page_node(h, next_nid);
685                 if (page) {
686                         ret = 1;
687                         break;
688                 }
689                 next_nid = hstate_next_node_to_alloc(h, nodes_allowed);
690         } while (next_nid != start_nid);
691
692         if (ret)
693                 count_vm_event(HTLB_BUDDY_PGALLOC);
694         else
695                 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
696
697         return ret;
698 }
699
700 /*
701  * helper for free_pool_huge_page() - return the previously saved
702  * node ["this node"] from which to free a huge page.  Advance the
703  * next node id whether or not we find a free huge page to free so
704  * that the next attempt to free addresses the next node.
705  */
706 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
707 {
708         int nid;
709
710         VM_BUG_ON(!nodes_allowed);
711
712         nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
713         h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
714
715         return nid;
716 }
717
718 /*
719  * Free huge page from pool from next node to free.
720  * Attempt to keep persistent huge pages more or less
721  * balanced over allowed nodes.
722  * Called with hugetlb_lock locked.
723  */
724 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
725                                                          bool acct_surplus)
726 {
727         int start_nid;
728         int next_nid;
729         int ret = 0;
730
731         start_nid = hstate_next_node_to_free(h, nodes_allowed);
732         next_nid = start_nid;
733
734         do {
735                 /*
736                  * If we're returning unused surplus pages, only examine
737                  * nodes with surplus pages.
738                  */
739                 if ((!acct_surplus || h->surplus_huge_pages_node[next_nid]) &&
740                     !list_empty(&h->hugepage_freelists[next_nid])) {
741                         struct page *page =
742                                 list_entry(h->hugepage_freelists[next_nid].next,
743                                           struct page, lru);
744                         list_del(&page->lru);
745                         h->free_huge_pages--;
746                         h->free_huge_pages_node[next_nid]--;
747                         if (acct_surplus) {
748                                 h->surplus_huge_pages--;
749                                 h->surplus_huge_pages_node[next_nid]--;
750                         }
751                         update_and_free_page(h, page);
752                         ret = 1;
753                         break;
754                 }
755                 next_nid = hstate_next_node_to_free(h, nodes_allowed);
756         } while (next_nid != start_nid);
757
758         return ret;
759 }
760
761 static struct page *alloc_buddy_huge_page(struct hstate *h,
762                         struct vm_area_struct *vma, unsigned long address)
763 {
764         struct page *page;
765         unsigned int nid;
766
767         if (h->order >= MAX_ORDER)
768                 return NULL;
769
770         /*
771          * Assume we will successfully allocate the surplus page to
772          * prevent racing processes from causing the surplus to exceed
773          * overcommit
774          *
775          * This however introduces a different race, where a process B
776          * tries to grow the static hugepage pool while alloc_pages() is
777          * called by process A. B will only examine the per-node
778          * counters in determining if surplus huge pages can be
779          * converted to normal huge pages in adjust_pool_surplus(). A
780          * won't be able to increment the per-node counter, until the
781          * lock is dropped by B, but B doesn't drop hugetlb_lock until
782          * no more huge pages can be converted from surplus to normal
783          * state (and doesn't try to convert again). Thus, we have a
784          * case where a surplus huge page exists, the pool is grown, and
785          * the surplus huge page still exists after, even though it
786          * should just have been converted to a normal huge page. This
787          * does not leak memory, though, as the hugepage will be freed
788          * once it is out of use. It also does not allow the counters to
789          * go out of whack in adjust_pool_surplus() as we don't modify
790          * the node values until we've gotten the hugepage and only the
791          * per-node value is checked there.
792          */
793         spin_lock(&hugetlb_lock);
794         if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
795                 spin_unlock(&hugetlb_lock);
796                 return NULL;
797         } else {
798                 h->nr_huge_pages++;
799                 h->surplus_huge_pages++;
800         }
801         spin_unlock(&hugetlb_lock);
802
803         page = alloc_pages(htlb_alloc_mask|__GFP_COMP|
804                                         __GFP_REPEAT|__GFP_NOWARN,
805                                         huge_page_order(h));
806
807         if (page && arch_prepare_hugepage(page)) {
808                 __free_pages(page, huge_page_order(h));
809                 return NULL;
810         }
811
812         spin_lock(&hugetlb_lock);
813         if (page) {
814                 /*
815                  * This page is now managed by the hugetlb allocator and has
816                  * no users -- drop the buddy allocator's reference.
817                  */
818                 put_page_testzero(page);
819                 VM_BUG_ON(page_count(page));
820                 nid = page_to_nid(page);
821                 set_compound_page_dtor(page, free_huge_page);
822                 /*
823                  * We incremented the global counters already
824                  */
825                 h->nr_huge_pages_node[nid]++;
826                 h->surplus_huge_pages_node[nid]++;
827                 __count_vm_event(HTLB_BUDDY_PGALLOC);
828         } else {
829                 h->nr_huge_pages--;
830                 h->surplus_huge_pages--;
831                 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
832         }
833         spin_unlock(&hugetlb_lock);
834
835         return page;
836 }
837
838 /*
839  * Increase the hugetlb pool such that it can accomodate a reservation
840  * of size 'delta'.
841  */
842 static int gather_surplus_pages(struct hstate *h, int delta)
843 {
844         struct list_head surplus_list;
845         struct page *page, *tmp;
846         int ret, i;
847         int needed, allocated;
848
849         needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
850         if (needed <= 0) {
851                 h->resv_huge_pages += delta;
852                 return 0;
853         }
854
855         allocated = 0;
856         INIT_LIST_HEAD(&surplus_list);
857
858         ret = -ENOMEM;
859 retry:
860         spin_unlock(&hugetlb_lock);
861         for (i = 0; i < needed; i++) {
862                 page = alloc_buddy_huge_page(h, NULL, 0);
863                 if (!page) {
864                         /*
865                          * We were not able to allocate enough pages to
866                          * satisfy the entire reservation so we free what
867                          * we've allocated so far.
868                          */
869                         spin_lock(&hugetlb_lock);
870                         needed = 0;
871                         goto free;
872                 }
873
874                 list_add(&page->lru, &surplus_list);
875         }
876         allocated += needed;
877
878         /*
879          * After retaking hugetlb_lock, we need to recalculate 'needed'
880          * because either resv_huge_pages or free_huge_pages may have changed.
881          */
882         spin_lock(&hugetlb_lock);
883         needed = (h->resv_huge_pages + delta) -
884                         (h->free_huge_pages + allocated);
885         if (needed > 0)
886                 goto retry;
887
888         /*
889          * The surplus_list now contains _at_least_ the number of extra pages
890          * needed to accomodate the reservation.  Add the appropriate number
891          * of pages to the hugetlb pool and free the extras back to the buddy
892          * allocator.  Commit the entire reservation here to prevent another
893          * process from stealing the pages as they are added to the pool but
894          * before they are reserved.
895          */
896         needed += allocated;
897         h->resv_huge_pages += delta;
898         ret = 0;
899 free:
900         /* Free the needed pages to the hugetlb pool */
901         list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
902                 if ((--needed) < 0)
903                         break;
904                 list_del(&page->lru);
905                 enqueue_huge_page(h, page);
906         }
907
908         /* Free unnecessary surplus pages to the buddy allocator */
909         if (!list_empty(&surplus_list)) {
910                 spin_unlock(&hugetlb_lock);
911                 list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
912                         list_del(&page->lru);
913                         /*
914                          * The page has a reference count of zero already, so
915                          * call free_huge_page directly instead of using
916                          * put_page.  This must be done with hugetlb_lock
917                          * unlocked which is safe because free_huge_page takes
918                          * hugetlb_lock before deciding how to free the page.
919                          */
920                         free_huge_page(page);
921                 }
922                 spin_lock(&hugetlb_lock);
923         }
924
925         return ret;
926 }
927
928 /*
929  * When releasing a hugetlb pool reservation, any surplus pages that were
930  * allocated to satisfy the reservation must be explicitly freed if they were
931  * never used.
932  * Called with hugetlb_lock held.
933  */
934 static void return_unused_surplus_pages(struct hstate *h,
935                                         unsigned long unused_resv_pages)
936 {
937         unsigned long nr_pages;
938
939         /* Uncommit the reservation */
940         h->resv_huge_pages -= unused_resv_pages;
941
942         /* Cannot return gigantic pages currently */
943         if (h->order >= MAX_ORDER)
944                 return;
945
946         nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
947
948         /*
949          * We want to release as many surplus pages as possible, spread
950          * evenly across all nodes with memory. Iterate across these nodes
951          * until we can no longer free unreserved surplus pages. This occurs
952          * when the nodes with surplus pages have no free pages.
953          * free_pool_huge_page() will balance the the freed pages across the
954          * on-line nodes with memory and will handle the hstate accounting.
955          */
956         while (nr_pages--) {
957                 if (!free_pool_huge_page(h, &node_states[N_HIGH_MEMORY], 1))
958                         break;
959         }
960 }
961
962 /*
963  * Determine if the huge page at addr within the vma has an associated
964  * reservation.  Where it does not we will need to logically increase
965  * reservation and actually increase quota before an allocation can occur.
966  * Where any new reservation would be required the reservation change is
967  * prepared, but not committed.  Once the page has been quota'd allocated
968  * an instantiated the change should be committed via vma_commit_reservation.
969  * No action is required on failure.
970  */
971 static long vma_needs_reservation(struct hstate *h,
972                         struct vm_area_struct *vma, unsigned long addr)
973 {
974         struct address_space *mapping = vma->vm_file->f_mapping;
975         struct inode *inode = mapping->host;
976
977         if (vma->vm_flags & VM_MAYSHARE) {
978                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
979                 return region_chg(&inode->i_mapping->private_list,
980                                                         idx, idx + 1);
981
982         } else if (!is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
983                 return 1;
984
985         } else  {
986                 long err;
987                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
988                 struct resv_map *reservations = vma_resv_map(vma);
989
990                 err = region_chg(&reservations->regions, idx, idx + 1);
991                 if (err < 0)
992                         return err;
993                 return 0;
994         }
995 }
996 static void vma_commit_reservation(struct hstate *h,
997                         struct vm_area_struct *vma, unsigned long addr)
998 {
999         struct address_space *mapping = vma->vm_file->f_mapping;
1000         struct inode *inode = mapping->host;
1001
1002         if (vma->vm_flags & VM_MAYSHARE) {
1003                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1004                 region_add(&inode->i_mapping->private_list, idx, idx + 1);
1005
1006         } else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1007                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1008                 struct resv_map *reservations = vma_resv_map(vma);
1009
1010                 /* Mark this page used in the map. */
1011                 region_add(&reservations->regions, idx, idx + 1);
1012         }
1013 }
1014
1015 static struct page *alloc_huge_page(struct vm_area_struct *vma,
1016                                     unsigned long addr, int avoid_reserve)
1017 {
1018         struct hstate *h = hstate_vma(vma);
1019         struct page *page;
1020         struct address_space *mapping = vma->vm_file->f_mapping;
1021         struct inode *inode = mapping->host;
1022         long chg;
1023
1024         /*
1025          * Processes that did not create the mapping will have no reserves and
1026          * will not have accounted against quota. Check that the quota can be
1027          * made before satisfying the allocation
1028          * MAP_NORESERVE mappings may also need pages and quota allocated
1029          * if no reserve mapping overlaps.
1030          */
1031         chg = vma_needs_reservation(h, vma, addr);
1032         if (chg < 0)
1033                 return ERR_PTR(chg);
1034         if (chg)
1035                 if (hugetlb_get_quota(inode->i_mapping, chg))
1036                         return ERR_PTR(-ENOSPC);
1037
1038         spin_lock(&hugetlb_lock);
1039         page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve);
1040         spin_unlock(&hugetlb_lock);
1041
1042         if (!page) {
1043                 page = alloc_buddy_huge_page(h, vma, addr);
1044                 if (!page) {
1045                         hugetlb_put_quota(inode->i_mapping, chg);
1046                         return ERR_PTR(-VM_FAULT_SIGBUS);
1047                 }
1048         }
1049
1050         set_page_refcounted(page);
1051         set_page_private(page, (unsigned long) mapping);
1052
1053         vma_commit_reservation(h, vma, addr);
1054
1055         return page;
1056 }
1057
1058 int __weak alloc_bootmem_huge_page(struct hstate *h)
1059 {
1060         struct huge_bootmem_page *m;
1061         int nr_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
1062
1063         while (nr_nodes) {
1064                 void *addr;
1065
1066                 addr = __alloc_bootmem_node_nopanic(
1067                                 NODE_DATA(hstate_next_node_to_alloc(h,
1068                                                 &node_states[N_HIGH_MEMORY])),
1069                                 huge_page_size(h), huge_page_size(h), 0);
1070
1071                 if (addr) {
1072                         /*
1073                          * Use the beginning of the huge page to store the
1074                          * huge_bootmem_page struct (until gather_bootmem
1075                          * puts them into the mem_map).
1076                          */
1077                         m = addr;
1078                         goto found;
1079                 }
1080                 nr_nodes--;
1081         }
1082         return 0;
1083
1084 found:
1085         BUG_ON((unsigned long)virt_to_phys(m) & (huge_page_size(h) - 1));
1086         /* Put them into a private list first because mem_map is not up yet */
1087         list_add(&m->list, &huge_boot_pages);
1088         m->hstate = h;
1089         return 1;
1090 }
1091
1092 static void prep_compound_huge_page(struct page *page, int order)
1093 {
1094         if (unlikely(order > (MAX_ORDER - 1)))
1095                 prep_compound_gigantic_page(page, order);
1096         else
1097                 prep_compound_page(page, order);
1098 }
1099
1100 /* Put bootmem huge pages into the standard lists after mem_map is up */
1101 static void __init gather_bootmem_prealloc(void)
1102 {
1103         struct huge_bootmem_page *m;
1104
1105         list_for_each_entry(m, &huge_boot_pages, list) {
1106                 struct page *page = virt_to_page(m);
1107                 struct hstate *h = m->hstate;
1108                 __ClearPageReserved(page);
1109                 WARN_ON(page_count(page) != 1);
1110                 prep_compound_huge_page(page, h->order);
1111                 prep_new_huge_page(h, page, page_to_nid(page));
1112         }
1113 }
1114
1115 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
1116 {
1117         unsigned long i;
1118
1119         for (i = 0; i < h->max_huge_pages; ++i) {
1120                 if (h->order >= MAX_ORDER) {
1121                         if (!alloc_bootmem_huge_page(h))
1122                                 break;
1123                 } else if (!alloc_fresh_huge_page(h,
1124                                          &node_states[N_HIGH_MEMORY]))
1125                         break;
1126         }
1127         h->max_huge_pages = i;
1128 }
1129
1130 static void __init hugetlb_init_hstates(void)
1131 {
1132         struct hstate *h;
1133
1134         for_each_hstate(h) {
1135                 /* oversize hugepages were init'ed in early boot */
1136                 if (h->order < MAX_ORDER)
1137                         hugetlb_hstate_alloc_pages(h);
1138         }
1139 }
1140
1141 static char * __init memfmt(char *buf, unsigned long n)
1142 {
1143         if (n >= (1UL << 30))
1144                 sprintf(buf, "%lu GB", n >> 30);
1145         else if (n >= (1UL << 20))
1146                 sprintf(buf, "%lu MB", n >> 20);
1147         else
1148                 sprintf(buf, "%lu KB", n >> 10);
1149         return buf;
1150 }
1151
1152 static void __init report_hugepages(void)
1153 {
1154         struct hstate *h;
1155
1156         for_each_hstate(h) {
1157                 char buf[32];
1158                 printk(KERN_INFO "HugeTLB registered %s page size, "
1159                                  "pre-allocated %ld pages\n",
1160                         memfmt(buf, huge_page_size(h)),
1161                         h->free_huge_pages);
1162         }
1163 }
1164
1165 #ifdef CONFIG_HIGHMEM
1166 static void try_to_free_low(struct hstate *h, unsigned long count,
1167                                                 nodemask_t *nodes_allowed)
1168 {
1169         int i;
1170
1171         if (h->order >= MAX_ORDER)
1172                 return;
1173
1174         for_each_node_mask(i, *nodes_allowed) {
1175                 struct page *page, *next;
1176                 struct list_head *freel = &h->hugepage_freelists[i];
1177                 list_for_each_entry_safe(page, next, freel, lru) {
1178                         if (count >= h->nr_huge_pages)
1179                                 return;
1180                         if (PageHighMem(page))
1181                                 continue;
1182                         list_del(&page->lru);
1183                         update_and_free_page(h, page);
1184                         h->free_huge_pages--;
1185                         h->free_huge_pages_node[page_to_nid(page)]--;
1186                 }
1187         }
1188 }
1189 #else
1190 static inline void try_to_free_low(struct hstate *h, unsigned long count,
1191                                                 nodemask_t *nodes_allowed)
1192 {
1193 }
1194 #endif
1195
1196 /*
1197  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
1198  * balanced by operating on them in a round-robin fashion.
1199  * Returns 1 if an adjustment was made.
1200  */
1201 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
1202                                 int delta)
1203 {
1204         int start_nid, next_nid;
1205         int ret = 0;
1206
1207         VM_BUG_ON(delta != -1 && delta != 1);
1208
1209         if (delta < 0)
1210                 start_nid = hstate_next_node_to_alloc(h, nodes_allowed);
1211         else
1212                 start_nid = hstate_next_node_to_free(h, nodes_allowed);
1213         next_nid = start_nid;
1214
1215         do {
1216                 int nid = next_nid;
1217                 if (delta < 0)  {
1218                         /*
1219                          * To shrink on this node, there must be a surplus page
1220                          */
1221                         if (!h->surplus_huge_pages_node[nid]) {
1222                                 next_nid = hstate_next_node_to_alloc(h,
1223                                                                 nodes_allowed);
1224                                 continue;
1225                         }
1226                 }
1227                 if (delta > 0) {
1228                         /*
1229                          * Surplus cannot exceed the total number of pages
1230                          */
1231                         if (h->surplus_huge_pages_node[nid] >=
1232                                                 h->nr_huge_pages_node[nid]) {
1233                                 next_nid = hstate_next_node_to_free(h,
1234                                                                 nodes_allowed);
1235                                 continue;
1236                         }
1237                 }
1238
1239                 h->surplus_huge_pages += delta;
1240                 h->surplus_huge_pages_node[nid] += delta;
1241                 ret = 1;
1242                 break;
1243         } while (next_nid != start_nid);
1244
1245         return ret;
1246 }
1247
1248 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
1249 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
1250                                                 nodemask_t *nodes_allowed)
1251 {
1252         unsigned long min_count, ret;
1253
1254         if (h->order >= MAX_ORDER)
1255                 return h->max_huge_pages;
1256
1257         /*
1258          * Increase the pool size
1259          * First take pages out of surplus state.  Then make up the
1260          * remaining difference by allocating fresh huge pages.
1261          *
1262          * We might race with alloc_buddy_huge_page() here and be unable
1263          * to convert a surplus huge page to a normal huge page. That is
1264          * not critical, though, it just means the overall size of the
1265          * pool might be one hugepage larger than it needs to be, but
1266          * within all the constraints specified by the sysctls.
1267          */
1268         spin_lock(&hugetlb_lock);
1269         while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
1270                 if (!adjust_pool_surplus(h, nodes_allowed, -1))
1271                         break;
1272         }
1273
1274         while (count > persistent_huge_pages(h)) {
1275                 /*
1276                  * If this allocation races such that we no longer need the
1277                  * page, free_huge_page will handle it by freeing the page
1278                  * and reducing the surplus.
1279                  */
1280                 spin_unlock(&hugetlb_lock);
1281                 ret = alloc_fresh_huge_page(h, nodes_allowed);
1282                 spin_lock(&hugetlb_lock);
1283                 if (!ret)
1284                         goto out;
1285
1286                 /* Bail for signals. Probably ctrl-c from user */
1287                 if (signal_pending(current))
1288                         goto out;
1289         }
1290
1291         /*
1292          * Decrease the pool size
1293          * First return free pages to the buddy allocator (being careful
1294          * to keep enough around to satisfy reservations).  Then place
1295          * pages into surplus state as needed so the pool will shrink
1296          * to the desired size as pages become free.
1297          *
1298          * By placing pages into the surplus state independent of the
1299          * overcommit value, we are allowing the surplus pool size to
1300          * exceed overcommit. There are few sane options here. Since
1301          * alloc_buddy_huge_page() is checking the global counter,
1302          * though, we'll note that we're not allowed to exceed surplus
1303          * and won't grow the pool anywhere else. Not until one of the
1304          * sysctls are changed, or the surplus pages go out of use.
1305          */
1306         min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
1307         min_count = max(count, min_count);
1308         try_to_free_low(h, min_count, nodes_allowed);
1309         while (min_count < persistent_huge_pages(h)) {
1310                 if (!free_pool_huge_page(h, nodes_allowed, 0))
1311                         break;
1312         }
1313         while (count < persistent_huge_pages(h)) {
1314                 if (!adjust_pool_surplus(h, nodes_allowed, 1))
1315                         break;
1316         }
1317 out:
1318         ret = persistent_huge_pages(h);
1319         spin_unlock(&hugetlb_lock);
1320         return ret;
1321 }
1322
1323 #define HSTATE_ATTR_RO(_name) \
1324         static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
1325
1326 #define HSTATE_ATTR(_name) \
1327         static struct kobj_attribute _name##_attr = \
1328                 __ATTR(_name, 0644, _name##_show, _name##_store)
1329
1330 static struct kobject *hugepages_kobj;
1331 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
1332
1333 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
1334
1335 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
1336 {
1337         int i;
1338
1339         for (i = 0; i < HUGE_MAX_HSTATE; i++)
1340                 if (hstate_kobjs[i] == kobj) {
1341                         if (nidp)
1342                                 *nidp = NUMA_NO_NODE;
1343                         return &hstates[i];
1344                 }
1345
1346         return kobj_to_node_hstate(kobj, nidp);
1347 }
1348
1349 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
1350                                         struct kobj_attribute *attr, char *buf)
1351 {
1352         struct hstate *h;
1353         unsigned long nr_huge_pages;
1354         int nid;
1355
1356         h = kobj_to_hstate(kobj, &nid);
1357         if (nid == NUMA_NO_NODE)
1358                 nr_huge_pages = h->nr_huge_pages;
1359         else
1360                 nr_huge_pages = h->nr_huge_pages_node[nid];
1361
1362         return sprintf(buf, "%lu\n", nr_huge_pages);
1363 }
1364 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
1365                         struct kobject *kobj, struct kobj_attribute *attr,
1366                         const char *buf, size_t len)
1367 {
1368         int err;
1369         int nid;
1370         unsigned long count;
1371         struct hstate *h;
1372         NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
1373
1374         err = strict_strtoul(buf, 10, &count);
1375         if (err)
1376                 return 0;
1377
1378         h = kobj_to_hstate(kobj, &nid);
1379         if (nid == NUMA_NO_NODE) {
1380                 /*
1381                  * global hstate attribute
1382                  */
1383                 if (!(obey_mempolicy &&
1384                                 init_nodemask_of_mempolicy(nodes_allowed))) {
1385                         NODEMASK_FREE(nodes_allowed);
1386                         nodes_allowed = &node_states[N_HIGH_MEMORY];
1387                 }
1388         } else if (nodes_allowed) {
1389                 /*
1390                  * per node hstate attribute: adjust count to global,
1391                  * but restrict alloc/free to the specified node.
1392                  */
1393                 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
1394                 init_nodemask_of_node(nodes_allowed, nid);
1395         } else
1396                 nodes_allowed = &node_states[N_HIGH_MEMORY];
1397
1398         h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
1399
1400         if (nodes_allowed != &node_states[N_HIGH_MEMORY])
1401                 NODEMASK_FREE(nodes_allowed);
1402
1403         return len;
1404 }
1405
1406 static ssize_t nr_hugepages_show(struct kobject *kobj,
1407                                        struct kobj_attribute *attr, char *buf)
1408 {
1409         return nr_hugepages_show_common(kobj, attr, buf);
1410 }
1411
1412 static ssize_t nr_hugepages_store(struct kobject *kobj,
1413                struct kobj_attribute *attr, const char *buf, size_t len)
1414 {
1415         return nr_hugepages_store_common(false, kobj, attr, buf, len);
1416 }
1417 HSTATE_ATTR(nr_hugepages);
1418
1419 #ifdef CONFIG_NUMA
1420
1421 /*
1422  * hstate attribute for optionally mempolicy-based constraint on persistent
1423  * huge page alloc/free.
1424  */
1425 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
1426                                        struct kobj_attribute *attr, char *buf)
1427 {
1428         return nr_hugepages_show_common(kobj, attr, buf);
1429 }
1430
1431 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
1432                struct kobj_attribute *attr, const char *buf, size_t len)
1433 {
1434         return nr_hugepages_store_common(true, kobj, attr, buf, len);
1435 }
1436 HSTATE_ATTR(nr_hugepages_mempolicy);
1437 #endif
1438
1439
1440 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
1441                                         struct kobj_attribute *attr, char *buf)
1442 {
1443         struct hstate *h = kobj_to_hstate(kobj, NULL);
1444         return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
1445 }
1446 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
1447                 struct kobj_attribute *attr, const char *buf, size_t count)
1448 {
1449         int err;
1450         unsigned long input;
1451         struct hstate *h = kobj_to_hstate(kobj, NULL);
1452
1453         err = strict_strtoul(buf, 10, &input);
1454         if (err)
1455                 return 0;
1456
1457         spin_lock(&hugetlb_lock);
1458         h->nr_overcommit_huge_pages = input;
1459         spin_unlock(&hugetlb_lock);
1460
1461         return count;
1462 }
1463 HSTATE_ATTR(nr_overcommit_hugepages);
1464
1465 static ssize_t free_hugepages_show(struct kobject *kobj,
1466                                         struct kobj_attribute *attr, char *buf)
1467 {
1468         struct hstate *h;
1469         unsigned long free_huge_pages;
1470         int nid;
1471
1472         h = kobj_to_hstate(kobj, &nid);
1473         if (nid == NUMA_NO_NODE)
1474                 free_huge_pages = h->free_huge_pages;
1475         else
1476                 free_huge_pages = h->free_huge_pages_node[nid];
1477
1478         return sprintf(buf, "%lu\n", free_huge_pages);
1479 }
1480 HSTATE_ATTR_RO(free_hugepages);
1481
1482 static ssize_t resv_hugepages_show(struct kobject *kobj,
1483                                         struct kobj_attribute *attr, char *buf)
1484 {
1485         struct hstate *h = kobj_to_hstate(kobj, NULL);
1486         return sprintf(buf, "%lu\n", h->resv_huge_pages);
1487 }
1488 HSTATE_ATTR_RO(resv_hugepages);
1489
1490 static ssize_t surplus_hugepages_show(struct kobject *kobj,
1491                                         struct kobj_attribute *attr, char *buf)
1492 {
1493         struct hstate *h;
1494         unsigned long surplus_huge_pages;
1495         int nid;
1496
1497         h = kobj_to_hstate(kobj, &nid);
1498         if (nid == NUMA_NO_NODE)
1499                 surplus_huge_pages = h->surplus_huge_pages;
1500         else
1501                 surplus_huge_pages = h->surplus_huge_pages_node[nid];
1502
1503         return sprintf(buf, "%lu\n", surplus_huge_pages);
1504 }
1505 HSTATE_ATTR_RO(surplus_hugepages);
1506
1507 static struct attribute *hstate_attrs[] = {
1508         &nr_hugepages_attr.attr,
1509         &nr_overcommit_hugepages_attr.attr,
1510         &free_hugepages_attr.attr,
1511         &resv_hugepages_attr.attr,
1512         &surplus_hugepages_attr.attr,
1513 #ifdef CONFIG_NUMA
1514         &nr_hugepages_mempolicy_attr.attr,
1515 #endif
1516         NULL,
1517 };
1518
1519 static struct attribute_group hstate_attr_group = {
1520         .attrs = hstate_attrs,
1521 };
1522
1523 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
1524                                     struct kobject **hstate_kobjs,
1525                                     struct attribute_group *hstate_attr_group)
1526 {
1527         int retval;
1528         int hi = h - hstates;
1529
1530         hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
1531         if (!hstate_kobjs[hi])
1532                 return -ENOMEM;
1533
1534         retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
1535         if (retval)
1536                 kobject_put(hstate_kobjs[hi]);
1537
1538         return retval;
1539 }
1540
1541 static void __init hugetlb_sysfs_init(void)
1542 {
1543         struct hstate *h;
1544         int err;
1545
1546         hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
1547         if (!hugepages_kobj)
1548                 return;
1549
1550         for_each_hstate(h) {
1551                 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
1552                                          hstate_kobjs, &hstate_attr_group);
1553                 if (err)
1554                         printk(KERN_ERR "Hugetlb: Unable to add hstate %s",
1555                                                                 h->name);
1556         }
1557 }
1558
1559 #ifdef CONFIG_NUMA
1560
1561 /*
1562  * node_hstate/s - associate per node hstate attributes, via their kobjects,
1563  * with node sysdevs in node_devices[] using a parallel array.  The array
1564  * index of a node sysdev or _hstate == node id.
1565  * This is here to avoid any static dependency of the node sysdev driver, in
1566  * the base kernel, on the hugetlb module.
1567  */
1568 struct node_hstate {
1569         struct kobject          *hugepages_kobj;
1570         struct kobject          *hstate_kobjs[HUGE_MAX_HSTATE];
1571 };
1572 struct node_hstate node_hstates[MAX_NUMNODES];
1573
1574 /*
1575  * A subset of global hstate attributes for node sysdevs
1576  */
1577 static struct attribute *per_node_hstate_attrs[] = {
1578         &nr_hugepages_attr.attr,
1579         &free_hugepages_attr.attr,
1580         &surplus_hugepages_attr.attr,
1581         NULL,
1582 };
1583
1584 static struct attribute_group per_node_hstate_attr_group = {
1585         .attrs = per_node_hstate_attrs,
1586 };
1587
1588 /*
1589  * kobj_to_node_hstate - lookup global hstate for node sysdev hstate attr kobj.
1590  * Returns node id via non-NULL nidp.
1591  */
1592 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1593 {
1594         int nid;
1595
1596         for (nid = 0; nid < nr_node_ids; nid++) {
1597                 struct node_hstate *nhs = &node_hstates[nid];
1598                 int i;
1599                 for (i = 0; i < HUGE_MAX_HSTATE; i++)
1600                         if (nhs->hstate_kobjs[i] == kobj) {
1601                                 if (nidp)
1602                                         *nidp = nid;
1603                                 return &hstates[i];
1604                         }
1605         }
1606
1607         BUG();
1608         return NULL;
1609 }
1610
1611 /*
1612  * Unregister hstate attributes from a single node sysdev.
1613  * No-op if no hstate attributes attached.
1614  */
1615 void hugetlb_unregister_node(struct node *node)
1616 {
1617         struct hstate *h;
1618         struct node_hstate *nhs = &node_hstates[node->sysdev.id];
1619
1620         if (!nhs->hugepages_kobj)
1621                 return;         /* no hstate attributes */
1622
1623         for_each_hstate(h)
1624                 if (nhs->hstate_kobjs[h - hstates]) {
1625                         kobject_put(nhs->hstate_kobjs[h - hstates]);
1626                         nhs->hstate_kobjs[h - hstates] = NULL;
1627                 }
1628
1629         kobject_put(nhs->hugepages_kobj);
1630         nhs->hugepages_kobj = NULL;
1631 }
1632
1633 /*
1634  * hugetlb module exit:  unregister hstate attributes from node sysdevs
1635  * that have them.
1636  */
1637 static void hugetlb_unregister_all_nodes(void)
1638 {
1639         int nid;
1640
1641         /*
1642          * disable node sysdev registrations.
1643          */
1644         register_hugetlbfs_with_node(NULL, NULL);
1645
1646         /*
1647          * remove hstate attributes from any nodes that have them.
1648          */
1649         for (nid = 0; nid < nr_node_ids; nid++)
1650                 hugetlb_unregister_node(&node_devices[nid]);
1651 }
1652
1653 /*
1654  * Register hstate attributes for a single node sysdev.
1655  * No-op if attributes already registered.
1656  */
1657 void hugetlb_register_node(struct node *node)
1658 {
1659         struct hstate *h;
1660         struct node_hstate *nhs = &node_hstates[node->sysdev.id];
1661         int err;
1662
1663         if (nhs->hugepages_kobj)
1664                 return;         /* already allocated */
1665
1666         nhs->hugepages_kobj = kobject_create_and_add("hugepages",
1667                                                         &node->sysdev.kobj);
1668         if (!nhs->hugepages_kobj)
1669                 return;
1670
1671         for_each_hstate(h) {
1672                 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
1673                                                 nhs->hstate_kobjs,
1674                                                 &per_node_hstate_attr_group);
1675                 if (err) {
1676                         printk(KERN_ERR "Hugetlb: Unable to add hstate %s"
1677                                         " for node %d\n",
1678                                                 h->name, node->sysdev.id);
1679                         hugetlb_unregister_node(node);
1680                         break;
1681                 }
1682         }
1683 }
1684
1685 /*
1686  * hugetlb init time:  register hstate attributes for all registered node
1687  * sysdevs of nodes that have memory.  All on-line nodes should have
1688  * registered their associated sysdev by this time.
1689  */
1690 static void hugetlb_register_all_nodes(void)
1691 {
1692         int nid;
1693
1694         for_each_node_state(nid, N_HIGH_MEMORY) {
1695                 struct node *node = &node_devices[nid];
1696                 if (node->sysdev.id == nid)
1697                         hugetlb_register_node(node);
1698         }
1699
1700         /*
1701          * Let the node sysdev driver know we're here so it can
1702          * [un]register hstate attributes on node hotplug.
1703          */
1704         register_hugetlbfs_with_node(hugetlb_register_node,
1705                                      hugetlb_unregister_node);
1706 }
1707 #else   /* !CONFIG_NUMA */
1708
1709 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1710 {
1711         BUG();
1712         if (nidp)
1713                 *nidp = -1;
1714         return NULL;
1715 }
1716
1717 static void hugetlb_unregister_all_nodes(void) { }
1718
1719 static void hugetlb_register_all_nodes(void) { }
1720
1721 #endif
1722
1723 static void __exit hugetlb_exit(void)
1724 {
1725         struct hstate *h;
1726
1727         hugetlb_unregister_all_nodes();
1728
1729         for_each_hstate(h) {
1730                 kobject_put(hstate_kobjs[h - hstates]);
1731         }
1732
1733         kobject_put(hugepages_kobj);
1734 }
1735 module_exit(hugetlb_exit);
1736
1737 static int __init hugetlb_init(void)
1738 {
1739         /* Some platform decide whether they support huge pages at boot
1740          * time. On these, such as powerpc, HPAGE_SHIFT is set to 0 when
1741          * there is no such support
1742          */
1743         if (HPAGE_SHIFT == 0)
1744                 return 0;
1745
1746         if (!size_to_hstate(default_hstate_size)) {
1747                 default_hstate_size = HPAGE_SIZE;
1748                 if (!size_to_hstate(default_hstate_size))
1749                         hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
1750         }
1751         default_hstate_idx = size_to_hstate(default_hstate_size) - hstates;
1752         if (default_hstate_max_huge_pages)
1753                 default_hstate.max_huge_pages = default_hstate_max_huge_pages;
1754
1755         hugetlb_init_hstates();
1756
1757         gather_bootmem_prealloc();
1758
1759         report_hugepages();
1760
1761         hugetlb_sysfs_init();
1762
1763         hugetlb_register_all_nodes();
1764
1765         return 0;
1766 }
1767 module_init(hugetlb_init);
1768
1769 /* Should be called on processing a hugepagesz=... option */
1770 void __init hugetlb_add_hstate(unsigned order)
1771 {
1772         struct hstate *h;
1773         unsigned long i;
1774
1775         if (size_to_hstate(PAGE_SIZE << order)) {
1776                 printk(KERN_WARNING "hugepagesz= specified twice, ignoring\n");
1777                 return;
1778         }
1779         BUG_ON(max_hstate >= HUGE_MAX_HSTATE);
1780         BUG_ON(order == 0);
1781         h = &hstates[max_hstate++];
1782         h->order = order;
1783         h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
1784         h->nr_huge_pages = 0;
1785         h->free_huge_pages = 0;
1786         for (i = 0; i < MAX_NUMNODES; ++i)
1787                 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
1788         h->next_nid_to_alloc = first_node(node_states[N_HIGH_MEMORY]);
1789         h->next_nid_to_free = first_node(node_states[N_HIGH_MEMORY]);
1790         snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
1791                                         huge_page_size(h)/1024);
1792
1793         parsed_hstate = h;
1794 }
1795
1796 static int __init hugetlb_nrpages_setup(char *s)
1797 {
1798         unsigned long *mhp;
1799         static unsigned long *last_mhp;
1800
1801         /*
1802          * !max_hstate means we haven't parsed a hugepagesz= parameter yet,
1803          * so this hugepages= parameter goes to the "default hstate".
1804          */
1805         if (!max_hstate)
1806                 mhp = &default_hstate_max_huge_pages;
1807         else
1808                 mhp = &parsed_hstate->max_huge_pages;
1809
1810         if (mhp == last_mhp) {
1811                 printk(KERN_WARNING "hugepages= specified twice without "
1812                         "interleaving hugepagesz=, ignoring\n");
1813                 return 1;
1814         }
1815
1816         if (sscanf(s, "%lu", mhp) <= 0)
1817                 *mhp = 0;
1818
1819         /*
1820          * Global state is always initialized later in hugetlb_init.
1821          * But we need to allocate >= MAX_ORDER hstates here early to still
1822          * use the bootmem allocator.
1823          */
1824         if (max_hstate && parsed_hstate->order >= MAX_ORDER)
1825                 hugetlb_hstate_alloc_pages(parsed_hstate);
1826
1827         last_mhp = mhp;
1828
1829         return 1;
1830 }
1831 __setup("hugepages=", hugetlb_nrpages_setup);
1832
1833 static int __init hugetlb_default_setup(char *s)
1834 {
1835         default_hstate_size = memparse(s, &s);
1836         return 1;
1837 }
1838 __setup("default_hugepagesz=", hugetlb_default_setup);
1839
1840 static unsigned int cpuset_mems_nr(unsigned int *array)
1841 {
1842         int node;
1843         unsigned int nr = 0;
1844
1845         for_each_node_mask(node, cpuset_current_mems_allowed)
1846                 nr += array[node];
1847
1848         return nr;
1849 }
1850
1851 #ifdef CONFIG_SYSCTL
1852 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
1853                          struct ctl_table *table, int write,
1854                          void __user *buffer, size_t *length, loff_t *ppos)
1855 {
1856         struct hstate *h = &default_hstate;
1857         unsigned long tmp;
1858
1859         if (!write)
1860                 tmp = h->max_huge_pages;
1861
1862         table->data = &tmp;
1863         table->maxlen = sizeof(unsigned long);
1864         proc_doulongvec_minmax(table, write, buffer, length, ppos);
1865
1866         if (write) {
1867                 NODEMASK_ALLOC(nodemask_t, nodes_allowed,
1868                                                 GFP_KERNEL | __GFP_NORETRY);
1869                 if (!(obey_mempolicy &&
1870                                init_nodemask_of_mempolicy(nodes_allowed))) {
1871                         NODEMASK_FREE(nodes_allowed);
1872                         nodes_allowed = &node_states[N_HIGH_MEMORY];
1873                 }
1874                 h->max_huge_pages = set_max_huge_pages(h, tmp, nodes_allowed);
1875
1876                 if (nodes_allowed != &node_states[N_HIGH_MEMORY])
1877                         NODEMASK_FREE(nodes_allowed);
1878         }
1879
1880         return 0;
1881 }
1882
1883 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
1884                           void __user *buffer, size_t *length, loff_t *ppos)
1885 {
1886
1887         return hugetlb_sysctl_handler_common(false, table, write,
1888                                                         buffer, length, ppos);
1889 }
1890
1891 #ifdef CONFIG_NUMA
1892 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
1893                           void __user *buffer, size_t *length, loff_t *ppos)
1894 {
1895         return hugetlb_sysctl_handler_common(true, table, write,
1896                                                         buffer, length, ppos);
1897 }
1898 #endif /* CONFIG_NUMA */
1899
1900 int hugetlb_treat_movable_handler(struct ctl_table *table, int write,
1901                         void __user *buffer,
1902                         size_t *length, loff_t *ppos)
1903 {
1904         proc_dointvec(table, write, buffer, length, ppos);
1905         if (hugepages_treat_as_movable)
1906                 htlb_alloc_mask = GFP_HIGHUSER_MOVABLE;
1907         else
1908                 htlb_alloc_mask = GFP_HIGHUSER;
1909         return 0;
1910 }
1911
1912 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
1913                         void __user *buffer,
1914                         size_t *length, loff_t *ppos)
1915 {
1916         struct hstate *h = &default_hstate;
1917         unsigned long tmp;
1918
1919         if (!write)
1920                 tmp = h->nr_overcommit_huge_pages;
1921
1922         table->data = &tmp;
1923         table->maxlen = sizeof(unsigned long);
1924         proc_doulongvec_minmax(table, write, buffer, length, ppos);
1925
1926         if (write) {
1927                 spin_lock(&hugetlb_lock);
1928                 h->nr_overcommit_huge_pages = tmp;
1929                 spin_unlock(&hugetlb_lock);
1930         }
1931
1932         return 0;
1933 }
1934
1935 #endif /* CONFIG_SYSCTL */
1936
1937 void hugetlb_report_meminfo(struct seq_file *m)
1938 {
1939         struct hstate *h = &default_hstate;
1940         seq_printf(m,
1941                         "HugePages_Total:   %5lu\n"
1942                         "HugePages_Free:    %5lu\n"
1943                         "HugePages_Rsvd:    %5lu\n"
1944                         "HugePages_Surp:    %5lu\n"
1945                         "Hugepagesize:   %8lu kB\n",
1946                         h->nr_huge_pages,
1947                         h->free_huge_pages,
1948                         h->resv_huge_pages,
1949                         h->surplus_huge_pages,
1950                         1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
1951 }
1952
1953 int hugetlb_report_node_meminfo(int nid, char *buf)
1954 {
1955         struct hstate *h = &default_hstate;
1956         return sprintf(buf,
1957                 "Node %d HugePages_Total: %5u\n"
1958                 "Node %d HugePages_Free:  %5u\n"
1959                 "Node %d HugePages_Surp:  %5u\n",
1960                 nid, h->nr_huge_pages_node[nid],
1961                 nid, h->free_huge_pages_node[nid],
1962                 nid, h->surplus_huge_pages_node[nid]);
1963 }
1964
1965 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
1966 unsigned long hugetlb_total_pages(void)
1967 {
1968         struct hstate *h = &default_hstate;
1969         return h->nr_huge_pages * pages_per_huge_page(h);
1970 }
1971
1972 static int hugetlb_acct_memory(struct hstate *h, long delta)
1973 {
1974         int ret = -ENOMEM;
1975
1976         spin_lock(&hugetlb_lock);
1977         /*
1978          * When cpuset is configured, it breaks the strict hugetlb page
1979          * reservation as the accounting is done on a global variable. Such
1980          * reservation is completely rubbish in the presence of cpuset because
1981          * the reservation is not checked against page availability for the
1982          * current cpuset. Application can still potentially OOM'ed by kernel
1983          * with lack of free htlb page in cpuset that the task is in.
1984          * Attempt to enforce strict accounting with cpuset is almost
1985          * impossible (or too ugly) because cpuset is too fluid that
1986          * task or memory node can be dynamically moved between cpusets.
1987          *
1988          * The change of semantics for shared hugetlb mapping with cpuset is
1989          * undesirable. However, in order to preserve some of the semantics,
1990          * we fall back to check against current free page availability as
1991          * a best attempt and hopefully to minimize the impact of changing
1992          * semantics that cpuset has.
1993          */
1994         if (delta > 0) {
1995                 if (gather_surplus_pages(h, delta) < 0)
1996                         goto out;
1997
1998                 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
1999                         return_unused_surplus_pages(h, delta);
2000                         goto out;
2001                 }
2002         }
2003
2004         ret = 0;
2005         if (delta < 0)
2006                 return_unused_surplus_pages(h, (unsigned long) -delta);
2007
2008 out:
2009         spin_unlock(&hugetlb_lock);
2010         return ret;
2011 }
2012
2013 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
2014 {
2015         struct resv_map *reservations = vma_resv_map(vma);
2016
2017         /*
2018          * This new VMA should share its siblings reservation map if present.
2019          * The VMA will only ever have a valid reservation map pointer where
2020          * it is being copied for another still existing VMA.  As that VMA
2021          * has a reference to the reservation map it cannot dissappear until
2022          * after this open call completes.  It is therefore safe to take a
2023          * new reference here without additional locking.
2024          */
2025         if (reservations)
2026                 kref_get(&reservations->refs);
2027 }
2028
2029 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
2030 {
2031         struct hstate *h = hstate_vma(vma);
2032         struct resv_map *reservations = vma_resv_map(vma);
2033         unsigned long reserve;
2034         unsigned long start;
2035         unsigned long end;
2036
2037         if (reservations) {
2038                 start = vma_hugecache_offset(h, vma, vma->vm_start);
2039                 end = vma_hugecache_offset(h, vma, vma->vm_end);
2040
2041                 reserve = (end - start) -
2042                         region_count(&reservations->regions, start, end);
2043
2044                 kref_put(&reservations->refs, resv_map_release);
2045
2046                 if (reserve) {
2047                         hugetlb_acct_memory(h, -reserve);
2048                         hugetlb_put_quota(vma->vm_file->f_mapping, reserve);
2049                 }
2050         }
2051 }
2052
2053 /*
2054  * We cannot handle pagefaults against hugetlb pages at all.  They cause
2055  * handle_mm_fault() to try to instantiate regular-sized pages in the
2056  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
2057  * this far.
2058  */
2059 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2060 {
2061         BUG();
2062         return 0;
2063 }
2064
2065 const struct vm_operations_struct hugetlb_vm_ops = {
2066         .fault = hugetlb_vm_op_fault,
2067         .open = hugetlb_vm_op_open,
2068         .close = hugetlb_vm_op_close,
2069 };
2070
2071 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
2072                                 int writable)
2073 {
2074         pte_t entry;
2075
2076         if (writable) {
2077                 entry =
2078                     pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
2079         } else {
2080                 entry = huge_pte_wrprotect(mk_pte(page, vma->vm_page_prot));
2081         }
2082         entry = pte_mkyoung(entry);
2083         entry = pte_mkhuge(entry);
2084
2085         return entry;
2086 }
2087
2088 static void set_huge_ptep_writable(struct vm_area_struct *vma,
2089                                    unsigned long address, pte_t *ptep)
2090 {
2091         pte_t entry;
2092
2093         entry = pte_mkwrite(pte_mkdirty(huge_ptep_get(ptep)));
2094         if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1)) {
2095                 update_mmu_cache(vma, address, ptep);
2096         }
2097 }
2098
2099
2100 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
2101                             struct vm_area_struct *vma)
2102 {
2103         pte_t *src_pte, *dst_pte, entry;
2104         struct page *ptepage;
2105         unsigned long addr;
2106         int cow;
2107         struct hstate *h = hstate_vma(vma);
2108         unsigned long sz = huge_page_size(h);
2109
2110         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
2111
2112         for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
2113                 src_pte = huge_pte_offset(src, addr);
2114                 if (!src_pte)
2115                         continue;
2116                 dst_pte = huge_pte_alloc(dst, addr, sz);
2117                 if (!dst_pte)
2118                         goto nomem;
2119
2120                 /* If the pagetables are shared don't copy or take references */
2121                 if (dst_pte == src_pte)
2122                         continue;
2123
2124                 spin_lock(&dst->page_table_lock);
2125                 spin_lock_nested(&src->page_table_lock, SINGLE_DEPTH_NESTING);
2126                 if (!huge_pte_none(huge_ptep_get(src_pte))) {
2127                         if (cow)
2128                                 huge_ptep_set_wrprotect(src, addr, src_pte);
2129                         entry = huge_ptep_get(src_pte);
2130                         ptepage = pte_page(entry);
2131                         get_page(ptepage);
2132                         set_huge_pte_at(dst, addr, dst_pte, entry);
2133                 }
2134                 spin_unlock(&src->page_table_lock);
2135                 spin_unlock(&dst->page_table_lock);
2136         }
2137         return 0;
2138
2139 nomem:
2140         return -ENOMEM;
2141 }
2142
2143 void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2144                             unsigned long end, struct page *ref_page)
2145 {
2146         struct mm_struct *mm = vma->vm_mm;
2147         unsigned long address;
2148         pte_t *ptep;
2149         pte_t pte;
2150         struct page *page;
2151         struct page *tmp;
2152         struct hstate *h = hstate_vma(vma);
2153         unsigned long sz = huge_page_size(h);
2154
2155         /*
2156          * A page gathering list, protected by per file i_mmap_lock. The
2157          * lock is used to avoid list corruption from multiple unmapping
2158          * of the same page since we are using page->lru.
2159          */
2160         LIST_HEAD(page_list);
2161
2162         WARN_ON(!is_vm_hugetlb_page(vma));
2163         BUG_ON(start & ~huge_page_mask(h));
2164         BUG_ON(end & ~huge_page_mask(h));
2165
2166         mmu_notifier_invalidate_range_start(mm, start, end);
2167         spin_lock(&mm->page_table_lock);
2168         for (address = start; address < end; address += sz) {
2169                 ptep = huge_pte_offset(mm, address);
2170                 if (!ptep)
2171                         continue;
2172
2173                 if (huge_pmd_unshare(mm, &address, ptep))
2174                         continue;
2175
2176                 /*
2177                  * If a reference page is supplied, it is because a specific
2178                  * page is being unmapped, not a range. Ensure the page we
2179                  * are about to unmap is the actual page of interest.
2180                  */
2181                 if (ref_page) {
2182                         pte = huge_ptep_get(ptep);
2183                         if (huge_pte_none(pte))
2184                                 continue;
2185                         page = pte_page(pte);
2186                         if (page != ref_page)
2187                                 continue;
2188
2189                         /*
2190                          * Mark the VMA as having unmapped its page so that
2191                          * future faults in this VMA will fail rather than
2192                          * looking like data was lost
2193                          */
2194                         set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
2195                 }
2196
2197                 pte = huge_ptep_get_and_clear(mm, address, ptep);
2198                 if (huge_pte_none(pte))
2199                         continue;
2200
2201                 page = pte_page(pte);
2202                 if (pte_dirty(pte))
2203                         set_page_dirty(page);
2204                 list_add(&page->lru, &page_list);
2205         }
2206         spin_unlock(&mm->page_table_lock);
2207         flush_tlb_range(vma, start, end);
2208         mmu_notifier_invalidate_range_end(mm, start, end);
2209         list_for_each_entry_safe(page, tmp, &page_list, lru) {
2210                 list_del(&page->lru);
2211                 put_page(page);
2212         }
2213 }
2214
2215 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2216                           unsigned long end, struct page *ref_page)
2217 {
2218         spin_lock(&vma->vm_file->f_mapping->i_mmap_lock);
2219         __unmap_hugepage_range(vma, start, end, ref_page);
2220         spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
2221 }
2222
2223 /*
2224  * This is called when the original mapper is failing to COW a MAP_PRIVATE
2225  * mappping it owns the reserve page for. The intention is to unmap the page
2226  * from other VMAs and let the children be SIGKILLed if they are faulting the
2227  * same region.
2228  */
2229 static int unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
2230                                 struct page *page, unsigned long address)
2231 {
2232         struct hstate *h = hstate_vma(vma);
2233         struct vm_area_struct *iter_vma;
2234         struct address_space *mapping;
2235         struct prio_tree_iter iter;
2236         pgoff_t pgoff;
2237
2238         /*
2239          * vm_pgoff is in PAGE_SIZE units, hence the different calculation
2240          * from page cache lookup which is in HPAGE_SIZE units.
2241          */
2242         address = address & huge_page_mask(h);
2243         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT)
2244                 + (vma->vm_pgoff >> PAGE_SHIFT);
2245         mapping = (struct address_space *)page_private(page);
2246
2247         /*
2248          * Take the mapping lock for the duration of the table walk. As
2249          * this mapping should be shared between all the VMAs,
2250          * __unmap_hugepage_range() is called as the lock is already held
2251          */
2252         spin_lock(&mapping->i_mmap_lock);
2253         vma_prio_tree_foreach(iter_vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
2254                 /* Do not unmap the current VMA */
2255                 if (iter_vma == vma)
2256                         continue;
2257
2258                 /*
2259                  * Unmap the page from other VMAs without their own reserves.
2260                  * They get marked to be SIGKILLed if they fault in these
2261                  * areas. This is because a future no-page fault on this VMA
2262                  * could insert a zeroed page instead of the data existing
2263                  * from the time of fork. This would look like data corruption
2264                  */
2265                 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
2266                         __unmap_hugepage_range(iter_vma,
2267                                 address, address + huge_page_size(h),
2268                                 page);
2269         }
2270         spin_unlock(&mapping->i_mmap_lock);
2271
2272         return 1;
2273 }
2274
2275 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
2276                         unsigned long address, pte_t *ptep, pte_t pte,
2277                         struct page *pagecache_page)
2278 {
2279         struct hstate *h = hstate_vma(vma);
2280         struct page *old_page, *new_page;
2281         int avoidcopy;
2282         int outside_reserve = 0;
2283
2284         old_page = pte_page(pte);
2285
2286 retry_avoidcopy:
2287         /* If no-one else is actually using this page, avoid the copy
2288          * and just make the page writable */
2289         avoidcopy = (page_count(old_page) == 1);
2290         if (avoidcopy) {
2291                 set_huge_ptep_writable(vma, address, ptep);
2292                 return 0;
2293         }
2294
2295         /*
2296          * If the process that created a MAP_PRIVATE mapping is about to
2297          * perform a COW due to a shared page count, attempt to satisfy
2298          * the allocation without using the existing reserves. The pagecache
2299          * page is used to determine if the reserve at this address was
2300          * consumed or not. If reserves were used, a partial faulted mapping
2301          * at the time of fork() could consume its reserves on COW instead
2302          * of the full address range.
2303          */
2304         if (!(vma->vm_flags & VM_MAYSHARE) &&
2305                         is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
2306                         old_page != pagecache_page)
2307                 outside_reserve = 1;
2308
2309         page_cache_get(old_page);
2310
2311         /* Drop page_table_lock as buddy allocator may be called */
2312         spin_unlock(&mm->page_table_lock);
2313         new_page = alloc_huge_page(vma, address, outside_reserve);
2314
2315         if (IS_ERR(new_page)) {
2316                 page_cache_release(old_page);
2317
2318                 /*
2319                  * If a process owning a MAP_PRIVATE mapping fails to COW,
2320                  * it is due to references held by a child and an insufficient
2321                  * huge page pool. To guarantee the original mappers
2322                  * reliability, unmap the page from child processes. The child
2323                  * may get SIGKILLed if it later faults.
2324                  */
2325                 if (outside_reserve) {
2326                         BUG_ON(huge_pte_none(pte));
2327                         if (unmap_ref_private(mm, vma, old_page, address)) {
2328                                 BUG_ON(page_count(old_page) != 1);
2329                                 BUG_ON(huge_pte_none(pte));
2330                                 spin_lock(&mm->page_table_lock);
2331                                 goto retry_avoidcopy;
2332                         }
2333                         WARN_ON_ONCE(1);
2334                 }
2335
2336                 /* Caller expects lock to be held */
2337                 spin_lock(&mm->page_table_lock);
2338                 return -PTR_ERR(new_page);
2339         }
2340
2341         copy_huge_page(new_page, old_page, address, vma);
2342         __SetPageUptodate(new_page);
2343
2344         /*
2345          * Retake the page_table_lock to check for racing updates
2346          * before the page tables are altered
2347          */
2348         spin_lock(&mm->page_table_lock);
2349         ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2350         if (likely(pte_same(huge_ptep_get(ptep), pte))) {
2351                 /* Break COW */
2352                 mmu_notifier_invalidate_range_start(mm,
2353                         address & huge_page_mask(h),
2354                         (address & huge_page_mask(h)) + huge_page_size(h));
2355                 huge_ptep_clear_flush(vma, address, ptep);
2356                 set_huge_pte_at(mm, address, ptep,
2357                                 make_huge_pte(vma, new_page, 1));
2358                 /* Make the old page be freed below */
2359                 new_page = old_page;
2360                 mmu_notifier_invalidate_range_end(mm,
2361                         address & huge_page_mask(h),
2362                         (address & huge_page_mask(h)) + huge_page_size(h));
2363         }
2364         page_cache_release(new_page);
2365         page_cache_release(old_page);
2366         return 0;
2367 }
2368
2369 /* Return the pagecache page at a given address within a VMA */
2370 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
2371                         struct vm_area_struct *vma, unsigned long address)
2372 {
2373         struct address_space *mapping;
2374         pgoff_t idx;
2375
2376         mapping = vma->vm_file->f_mapping;
2377         idx = vma_hugecache_offset(h, vma, address);
2378
2379         return find_lock_page(mapping, idx);
2380 }
2381
2382 /*
2383  * Return whether there is a pagecache page to back given address within VMA.
2384  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
2385  */
2386 static bool hugetlbfs_pagecache_present(struct hstate *h,
2387                         struct vm_area_struct *vma, unsigned long address)
2388 {
2389         struct address_space *mapping;
2390         pgoff_t idx;
2391         struct page *page;
2392
2393         mapping = vma->vm_file->f_mapping;
2394         idx = vma_hugecache_offset(h, vma, address);
2395
2396         page = find_get_page(mapping, idx);
2397         if (page)
2398                 put_page(page);
2399         return page != NULL;
2400 }
2401
2402 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
2403                         unsigned long address, pte_t *ptep, unsigned int flags)
2404 {
2405         struct hstate *h = hstate_vma(vma);
2406         int ret = VM_FAULT_SIGBUS;
2407         pgoff_t idx;
2408         unsigned long size;
2409         struct page *page;
2410         struct address_space *mapping;
2411         pte_t new_pte;
2412
2413         /*
2414          * Currently, we are forced to kill the process in the event the
2415          * original mapper has unmapped pages from the child due to a failed
2416          * COW. Warn that such a situation has occured as it may not be obvious
2417          */
2418         if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
2419                 printk(KERN_WARNING
2420                         "PID %d killed due to inadequate hugepage pool\n",
2421                         current->pid);
2422                 return ret;
2423         }
2424
2425         mapping = vma->vm_file->f_mapping;
2426         idx = vma_hugecache_offset(h, vma, address);
2427
2428         /*
2429          * Use page lock to guard against racing truncation
2430          * before we get page_table_lock.
2431          */
2432 retry:
2433         page = find_lock_page(mapping, idx);
2434         if (!page) {
2435                 size = i_size_read(mapping->host) >> huge_page_shift(h);
2436                 if (idx >= size)
2437                         goto out;
2438                 page = alloc_huge_page(vma, address, 0);
2439                 if (IS_ERR(page)) {
2440                         ret = -PTR_ERR(page);
2441                         goto out;
2442                 }
2443                 clear_huge_page(page, address, huge_page_size(h));
2444                 __SetPageUptodate(page);
2445
2446                 if (vma->vm_flags & VM_MAYSHARE) {
2447                         int err;
2448                         struct inode *inode = mapping->host;
2449
2450                         err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
2451                         if (err) {
2452                                 put_page(page);
2453                                 if (err == -EEXIST)
2454                                         goto retry;
2455                                 goto out;
2456                         }
2457
2458                         spin_lock(&inode->i_lock);
2459                         inode->i_blocks += blocks_per_huge_page(h);
2460                         spin_unlock(&inode->i_lock);
2461                 } else {
2462                         lock_page(page);
2463                         page->mapping = HUGETLB_POISON;
2464                 }
2465         }
2466
2467         /*
2468          * If we are going to COW a private mapping later, we examine the
2469          * pending reservations for this page now. This will ensure that
2470          * any allocations necessary to record that reservation occur outside
2471          * the spinlock.
2472          */
2473         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
2474                 if (vma_needs_reservation(h, vma, address) < 0) {
2475                         ret = VM_FAULT_OOM;
2476                         goto backout_unlocked;
2477                 }
2478
2479         spin_lock(&mm->page_table_lock);
2480         size = i_size_read(mapping->host) >> huge_page_shift(h);
2481         if (idx >= size)
2482                 goto backout;
2483
2484         ret = 0;
2485         if (!huge_pte_none(huge_ptep_get(ptep)))
2486                 goto backout;
2487
2488         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
2489                                 && (vma->vm_flags & VM_SHARED)));
2490         set_huge_pte_at(mm, address, ptep, new_pte);
2491
2492         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
2493                 /* Optimization, do the COW without a second fault */
2494                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page);
2495         }
2496
2497         spin_unlock(&mm->page_table_lock);
2498         unlock_page(page);
2499 out:
2500         return ret;
2501
2502 backout:
2503         spin_unlock(&mm->page_table_lock);
2504 backout_unlocked:
2505         unlock_page(page);
2506         put_page(page);
2507         goto out;
2508 }
2509
2510 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
2511                         unsigned long address, unsigned int flags)
2512 {
2513         pte_t *ptep;
2514         pte_t entry;
2515         int ret;
2516         struct page *pagecache_page = NULL;
2517         static DEFINE_MUTEX(hugetlb_instantiation_mutex);
2518         struct hstate *h = hstate_vma(vma);
2519
2520         ptep = huge_pte_alloc(mm, address, huge_page_size(h));
2521         if (!ptep)
2522                 return VM_FAULT_OOM;
2523
2524         /*
2525          * Serialize hugepage allocation and instantiation, so that we don't
2526          * get spurious allocation failures if two CPUs race to instantiate
2527          * the same page in the page cache.
2528          */
2529         mutex_lock(&hugetlb_instantiation_mutex);
2530         entry = huge_ptep_get(ptep);
2531         if (huge_pte_none(entry)) {
2532                 ret = hugetlb_no_page(mm, vma, address, ptep, flags);
2533                 goto out_mutex;
2534         }
2535
2536         ret = 0;
2537
2538         /*
2539          * If we are going to COW the mapping later, we examine the pending
2540          * reservations for this page now. This will ensure that any
2541          * allocations necessary to record that reservation occur outside the
2542          * spinlock. For private mappings, we also lookup the pagecache
2543          * page now as it is used to determine if a reservation has been
2544          * consumed.
2545          */
2546         if ((flags & FAULT_FLAG_WRITE) && !pte_write(entry)) {
2547                 if (vma_needs_reservation(h, vma, address) < 0) {
2548                         ret = VM_FAULT_OOM;
2549                         goto out_mutex;
2550                 }
2551
2552                 if (!(vma->vm_flags & VM_MAYSHARE))
2553                         pagecache_page = hugetlbfs_pagecache_page(h,
2554                                                                 vma, address);
2555         }
2556
2557         spin_lock(&mm->page_table_lock);
2558         /* Check for a racing update before calling hugetlb_cow */
2559         if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
2560                 goto out_page_table_lock;
2561
2562
2563         if (flags & FAULT_FLAG_WRITE) {
2564                 if (!pte_write(entry)) {
2565                         ret = hugetlb_cow(mm, vma, address, ptep, entry,
2566                                                         pagecache_page);
2567                         goto out_page_table_lock;
2568                 }
2569                 entry = pte_mkdirty(entry);
2570         }
2571         entry = pte_mkyoung(entry);
2572         if (huge_ptep_set_access_flags(vma, address, ptep, entry,
2573                                                 flags & FAULT_FLAG_WRITE))
2574                 update_mmu_cache(vma, address, ptep);
2575
2576 out_page_table_lock:
2577         spin_unlock(&mm->page_table_lock);
2578
2579         if (pagecache_page) {
2580                 unlock_page(pagecache_page);
2581                 put_page(pagecache_page);
2582         }
2583
2584 out_mutex:
2585         mutex_unlock(&hugetlb_instantiation_mutex);
2586
2587         return ret;
2588 }
2589
2590 /* Can be overriden by architectures */
2591 __attribute__((weak)) struct page *
2592 follow_huge_pud(struct mm_struct *mm, unsigned long address,
2593                pud_t *pud, int write)
2594 {
2595         BUG();
2596         return NULL;
2597 }
2598
2599 int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
2600                         struct page **pages, struct vm_area_struct **vmas,
2601                         unsigned long *position, int *length, int i,
2602                         unsigned int flags)
2603 {
2604         unsigned long pfn_offset;
2605         unsigned long vaddr = *position;
2606         int remainder = *length;
2607         struct hstate *h = hstate_vma(vma);
2608
2609         spin_lock(&mm->page_table_lock);
2610         while (vaddr < vma->vm_end && remainder) {
2611                 pte_t *pte;
2612                 int absent;
2613                 struct page *page;
2614
2615                 /*
2616                  * Some archs (sparc64, sh*) have multiple pte_ts to
2617                  * each hugepage.  We have to make sure we get the
2618                  * first, for the page indexing below to work.
2619                  */
2620                 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
2621                 absent = !pte || huge_pte_none(huge_ptep_get(pte));
2622
2623                 /*
2624                  * When coredumping, it suits get_dump_page if we just return
2625                  * an error where there's an empty slot with no huge pagecache
2626                  * to back it.  This way, we avoid allocating a hugepage, and
2627                  * the sparse dumpfile avoids allocating disk blocks, but its
2628                  * huge holes still show up with zeroes where they need to be.
2629                  */
2630                 if (absent && (flags & FOLL_DUMP) &&
2631                     !hugetlbfs_pagecache_present(h, vma, vaddr)) {
2632                         remainder = 0;
2633                         break;
2634                 }
2635
2636                 if (absent ||
2637                     ((flags & FOLL_WRITE) && !pte_write(huge_ptep_get(pte)))) {
2638                         int ret;
2639
2640                         spin_unlock(&mm->page_table_lock);
2641                         ret = hugetlb_fault(mm, vma, vaddr,
2642                                 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
2643                         spin_lock(&mm->page_table_lock);
2644                         if (!(ret & VM_FAULT_ERROR))
2645                                 continue;
2646
2647                         remainder = 0;
2648                         break;
2649                 }
2650
2651                 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
2652                 page = pte_page(huge_ptep_get(pte));
2653 same_page:
2654                 if (pages) {
2655                         pages[i] = mem_map_offset(page, pfn_offset);
2656                         get_page(pages[i]);
2657                 }
2658
2659                 if (vmas)
2660                         vmas[i] = vma;
2661
2662                 vaddr += PAGE_SIZE;
2663                 ++pfn_offset;
2664                 --remainder;
2665                 ++i;
2666                 if (vaddr < vma->vm_end && remainder &&
2667                                 pfn_offset < pages_per_huge_page(h)) {
2668                         /*
2669                          * We use pfn_offset to avoid touching the pageframes
2670                          * of this compound page.
2671                          */
2672                         goto same_page;
2673                 }
2674         }
2675         spin_unlock(&mm->page_table_lock);
2676         *length = remainder;
2677         *position = vaddr;
2678
2679         return i ? i : -EFAULT;
2680 }
2681
2682 void hugetlb_change_protection(struct vm_area_struct *vma,
2683                 unsigned long address, unsigned long end, pgprot_t newprot)
2684 {
2685         struct mm_struct *mm = vma->vm_mm;
2686         unsigned long start = address;
2687         pte_t *ptep;
2688         pte_t pte;
2689         struct hstate *h = hstate_vma(vma);
2690
2691         BUG_ON(address >= end);
2692         flush_cache_range(vma, address, end);
2693
2694         spin_lock(&vma->vm_file->f_mapping->i_mmap_lock);
2695         spin_lock(&mm->page_table_lock);
2696         for (; address < end; address += huge_page_size(h)) {
2697                 ptep = huge_pte_offset(mm, address);
2698                 if (!ptep)
2699                         continue;
2700                 if (huge_pmd_unshare(mm, &address, ptep))
2701                         continue;
2702                 if (!huge_pte_none(huge_ptep_get(ptep))) {
2703                         pte = huge_ptep_get_and_clear(mm, address, ptep);
2704                         pte = pte_mkhuge(pte_modify(pte, newprot));
2705                         set_huge_pte_at(mm, address, ptep, pte);
2706                 }
2707         }
2708         spin_unlock(&mm->page_table_lock);
2709         spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
2710
2711         flush_tlb_range(vma, start, end);
2712 }
2713
2714 int hugetlb_reserve_pages(struct inode *inode,
2715                                         long from, long to,
2716                                         struct vm_area_struct *vma,
2717                                         int acctflag)
2718 {
2719         long ret, chg;
2720         struct hstate *h = hstate_inode(inode);
2721
2722         /*
2723          * Only apply hugepage reservation if asked. At fault time, an
2724          * attempt will be made for VM_NORESERVE to allocate a page
2725          * and filesystem quota without using reserves
2726          */
2727         if (acctflag & VM_NORESERVE)
2728                 return 0;
2729
2730         /*
2731          * Shared mappings base their reservation on the number of pages that
2732          * are already allocated on behalf of the file. Private mappings need
2733          * to reserve the full area even if read-only as mprotect() may be
2734          * called to make the mapping read-write. Assume !vma is a shm mapping
2735          */
2736         if (!vma || vma->vm_flags & VM_MAYSHARE)
2737                 chg = region_chg(&inode->i_mapping->private_list, from, to);
2738         else {
2739                 struct resv_map *resv_map = resv_map_alloc();
2740                 if (!resv_map)
2741                         return -ENOMEM;
2742
2743                 chg = to - from;
2744
2745                 set_vma_resv_map(vma, resv_map);
2746                 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
2747         }
2748
2749         if (chg < 0)
2750                 return chg;
2751
2752         /* There must be enough filesystem quota for the mapping */
2753         if (hugetlb_get_quota(inode->i_mapping, chg))
2754                 return -ENOSPC;
2755
2756         /*
2757          * Check enough hugepages are available for the reservation.
2758          * Hand back the quota if there are not
2759          */
2760         ret = hugetlb_acct_memory(h, chg);
2761         if (ret < 0) {
2762                 hugetlb_put_quota(inode->i_mapping, chg);
2763                 return ret;
2764         }
2765
2766         /*
2767          * Account for the reservations made. Shared mappings record regions
2768          * that have reservations as they are shared by multiple VMAs.
2769          * When the last VMA disappears, the region map says how much
2770          * the reservation was and the page cache tells how much of
2771          * the reservation was consumed. Private mappings are per-VMA and
2772          * only the consumed reservations are tracked. When the VMA
2773          * disappears, the original reservation is the VMA size and the
2774          * consumed reservations are stored in the map. Hence, nothing
2775          * else has to be done for private mappings here
2776          */
2777         if (!vma || vma->vm_flags & VM_MAYSHARE)
2778                 region_add(&inode->i_mapping->private_list, from, to);
2779         return 0;
2780 }
2781
2782 void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
2783 {
2784         struct hstate *h = hstate_inode(inode);
2785         long chg = region_truncate(&inode->i_mapping->private_list, offset);
2786
2787         spin_lock(&inode->i_lock);
2788         inode->i_blocks -= (blocks_per_huge_page(h) * freed);
2789         spin_unlock(&inode->i_lock);
2790
2791         hugetlb_put_quota(inode->i_mapping, (chg - freed));
2792         hugetlb_acct_memory(h, -(chg - freed));
2793 }