hugetlb: reserve huge pages for reliable MAP_PRIVATE hugetlbfs mappings until fork()
[linux-3.10.git] / mm / hugetlb.c
1 /*
2  * Generic hugetlb support.
3  * (C) William Irwin, April 2004
4  */
5 #include <linux/gfp.h>
6 #include <linux/list.h>
7 #include <linux/init.h>
8 #include <linux/module.h>
9 #include <linux/mm.h>
10 #include <linux/sysctl.h>
11 #include <linux/highmem.h>
12 #include <linux/nodemask.h>
13 #include <linux/pagemap.h>
14 #include <linux/mempolicy.h>
15 #include <linux/cpuset.h>
16 #include <linux/mutex.h>
17
18 #include <asm/page.h>
19 #include <asm/pgtable.h>
20
21 #include <linux/hugetlb.h>
22 #include "internal.h"
23
24 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
25 static unsigned long nr_huge_pages, free_huge_pages, resv_huge_pages;
26 static unsigned long surplus_huge_pages;
27 static unsigned long nr_overcommit_huge_pages;
28 unsigned long max_huge_pages;
29 unsigned long sysctl_overcommit_huge_pages;
30 static struct list_head hugepage_freelists[MAX_NUMNODES];
31 static unsigned int nr_huge_pages_node[MAX_NUMNODES];
32 static unsigned int free_huge_pages_node[MAX_NUMNODES];
33 static unsigned int surplus_huge_pages_node[MAX_NUMNODES];
34 static gfp_t htlb_alloc_mask = GFP_HIGHUSER;
35 unsigned long hugepages_treat_as_movable;
36 static int hugetlb_next_nid;
37
38 /*
39  * Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
40  */
41 static DEFINE_SPINLOCK(hugetlb_lock);
42
43 /*
44  * These helpers are used to track how many pages are reserved for
45  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
46  * is guaranteed to have their future faults succeed.
47  *
48  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
49  * the reserve counters are updated with the hugetlb_lock held. It is safe
50  * to reset the VMA at fork() time as it is not in use yet and there is no
51  * chance of the global counters getting corrupted as a result of the values.
52  */
53 static unsigned long vma_resv_huge_pages(struct vm_area_struct *vma)
54 {
55         VM_BUG_ON(!is_vm_hugetlb_page(vma));
56         if (!(vma->vm_flags & VM_SHARED))
57                 return (unsigned long)vma->vm_private_data;
58         return 0;
59 }
60
61 static void set_vma_resv_huge_pages(struct vm_area_struct *vma,
62                                                         unsigned long reserve)
63 {
64         VM_BUG_ON(!is_vm_hugetlb_page(vma));
65         VM_BUG_ON(vma->vm_flags & VM_SHARED);
66
67         vma->vm_private_data = (void *)reserve;
68 }
69
70 /* Decrement the reserved pages in the hugepage pool by one */
71 static void decrement_hugepage_resv_vma(struct vm_area_struct *vma)
72 {
73         if (vma->vm_flags & VM_SHARED) {
74                 /* Shared mappings always use reserves */
75                 resv_huge_pages--;
76         } else {
77                 /*
78                  * Only the process that called mmap() has reserves for
79                  * private mappings.
80                  */
81                 if (vma_resv_huge_pages(vma)) {
82                         resv_huge_pages--;
83                         reserve = (unsigned long)vma->vm_private_data - 1;
84                         vma->vm_private_data = (void *)reserve;
85                 }
86         }
87 }
88
89 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
90 {
91         VM_BUG_ON(!is_vm_hugetlb_page(vma));
92         if (!(vma->vm_flags & VM_SHARED))
93                 vma->vm_private_data = (void *)0;
94 }
95
96 /* Returns true if the VMA has associated reserve pages */
97 static int vma_has_private_reserves(struct vm_area_struct *vma)
98 {
99         if (vma->vm_flags & VM_SHARED)
100                 return 0;
101         if (!vma_resv_huge_pages(vma))
102                 return 0;
103         return 1;
104 }
105
106 static void clear_huge_page(struct page *page, unsigned long addr)
107 {
108         int i;
109
110         might_sleep();
111         for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); i++) {
112                 cond_resched();
113                 clear_user_highpage(page + i, addr + i * PAGE_SIZE);
114         }
115 }
116
117 static void copy_huge_page(struct page *dst, struct page *src,
118                            unsigned long addr, struct vm_area_struct *vma)
119 {
120         int i;
121
122         might_sleep();
123         for (i = 0; i < HPAGE_SIZE/PAGE_SIZE; i++) {
124                 cond_resched();
125                 copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE, vma);
126         }
127 }
128
129 static void enqueue_huge_page(struct page *page)
130 {
131         int nid = page_to_nid(page);
132         list_add(&page->lru, &hugepage_freelists[nid]);
133         free_huge_pages++;
134         free_huge_pages_node[nid]++;
135 }
136
137 static struct page *dequeue_huge_page(void)
138 {
139         int nid;
140         struct page *page = NULL;
141
142         for (nid = 0; nid < MAX_NUMNODES; ++nid) {
143                 if (!list_empty(&hugepage_freelists[nid])) {
144                         page = list_entry(hugepage_freelists[nid].next,
145                                           struct page, lru);
146                         list_del(&page->lru);
147                         free_huge_pages--;
148                         free_huge_pages_node[nid]--;
149                         break;
150                 }
151         }
152         return page;
153 }
154
155 static struct page *dequeue_huge_page_vma(struct vm_area_struct *vma,
156                                 unsigned long address)
157 {
158         int nid;
159         struct page *page = NULL;
160         struct mempolicy *mpol;
161         nodemask_t *nodemask;
162         struct zonelist *zonelist = huge_zonelist(vma, address,
163                                         htlb_alloc_mask, &mpol, &nodemask);
164         struct zone *zone;
165         struct zoneref *z;
166
167         /*
168          * A child process with MAP_PRIVATE mappings created by their parent
169          * have no page reserves. This check ensures that reservations are
170          * not "stolen". The child may still get SIGKILLed
171          */
172         if (!vma_has_private_reserves(vma) &&
173                         free_huge_pages - resv_huge_pages == 0)
174                 return NULL;
175
176         for_each_zone_zonelist_nodemask(zone, z, zonelist,
177                                                 MAX_NR_ZONES - 1, nodemask) {
178                 nid = zone_to_nid(zone);
179                 if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask) &&
180                     !list_empty(&hugepage_freelists[nid])) {
181                         page = list_entry(hugepage_freelists[nid].next,
182                                           struct page, lru);
183                         list_del(&page->lru);
184                         free_huge_pages--;
185                         free_huge_pages_node[nid]--;
186                         decrement_hugepage_resv_vma(vma);
187
188                         break;
189                 }
190         }
191         mpol_cond_put(mpol);
192         return page;
193 }
194
195 static void update_and_free_page(struct page *page)
196 {
197         int i;
198         nr_huge_pages--;
199         nr_huge_pages_node[page_to_nid(page)]--;
200         for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) {
201                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
202                                 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
203                                 1 << PG_private | 1<< PG_writeback);
204         }
205         set_compound_page_dtor(page, NULL);
206         set_page_refcounted(page);
207         arch_release_hugepage(page);
208         __free_pages(page, HUGETLB_PAGE_ORDER);
209 }
210
211 static void free_huge_page(struct page *page)
212 {
213         int nid = page_to_nid(page);
214         struct address_space *mapping;
215
216         mapping = (struct address_space *) page_private(page);
217         set_page_private(page, 0);
218         BUG_ON(page_count(page));
219         INIT_LIST_HEAD(&page->lru);
220
221         spin_lock(&hugetlb_lock);
222         if (surplus_huge_pages_node[nid]) {
223                 update_and_free_page(page);
224                 surplus_huge_pages--;
225                 surplus_huge_pages_node[nid]--;
226         } else {
227                 enqueue_huge_page(page);
228         }
229         spin_unlock(&hugetlb_lock);
230         if (mapping)
231                 hugetlb_put_quota(mapping, 1);
232 }
233
234 /*
235  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
236  * balanced by operating on them in a round-robin fashion.
237  * Returns 1 if an adjustment was made.
238  */
239 static int adjust_pool_surplus(int delta)
240 {
241         static int prev_nid;
242         int nid = prev_nid;
243         int ret = 0;
244
245         VM_BUG_ON(delta != -1 && delta != 1);
246         do {
247                 nid = next_node(nid, node_online_map);
248                 if (nid == MAX_NUMNODES)
249                         nid = first_node(node_online_map);
250
251                 /* To shrink on this node, there must be a surplus page */
252                 if (delta < 0 && !surplus_huge_pages_node[nid])
253                         continue;
254                 /* Surplus cannot exceed the total number of pages */
255                 if (delta > 0 && surplus_huge_pages_node[nid] >=
256                                                 nr_huge_pages_node[nid])
257                         continue;
258
259                 surplus_huge_pages += delta;
260                 surplus_huge_pages_node[nid] += delta;
261                 ret = 1;
262                 break;
263         } while (nid != prev_nid);
264
265         prev_nid = nid;
266         return ret;
267 }
268
269 static struct page *alloc_fresh_huge_page_node(int nid)
270 {
271         struct page *page;
272
273         page = alloc_pages_node(nid,
274                 htlb_alloc_mask|__GFP_COMP|__GFP_THISNODE|
275                                                 __GFP_REPEAT|__GFP_NOWARN,
276                 HUGETLB_PAGE_ORDER);
277         if (page) {
278                 if (arch_prepare_hugepage(page)) {
279                         __free_pages(page, HUGETLB_PAGE_ORDER);
280                         return NULL;
281                 }
282                 set_compound_page_dtor(page, free_huge_page);
283                 spin_lock(&hugetlb_lock);
284                 nr_huge_pages++;
285                 nr_huge_pages_node[nid]++;
286                 spin_unlock(&hugetlb_lock);
287                 put_page(page); /* free it into the hugepage allocator */
288         }
289
290         return page;
291 }
292
293 static int alloc_fresh_huge_page(void)
294 {
295         struct page *page;
296         int start_nid;
297         int next_nid;
298         int ret = 0;
299
300         start_nid = hugetlb_next_nid;
301
302         do {
303                 page = alloc_fresh_huge_page_node(hugetlb_next_nid);
304                 if (page)
305                         ret = 1;
306                 /*
307                  * Use a helper variable to find the next node and then
308                  * copy it back to hugetlb_next_nid afterwards:
309                  * otherwise there's a window in which a racer might
310                  * pass invalid nid MAX_NUMNODES to alloc_pages_node.
311                  * But we don't need to use a spin_lock here: it really
312                  * doesn't matter if occasionally a racer chooses the
313                  * same nid as we do.  Move nid forward in the mask even
314                  * if we just successfully allocated a hugepage so that
315                  * the next caller gets hugepages on the next node.
316                  */
317                 next_nid = next_node(hugetlb_next_nid, node_online_map);
318                 if (next_nid == MAX_NUMNODES)
319                         next_nid = first_node(node_online_map);
320                 hugetlb_next_nid = next_nid;
321         } while (!page && hugetlb_next_nid != start_nid);
322
323         if (ret)
324                 count_vm_event(HTLB_BUDDY_PGALLOC);
325         else
326                 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
327
328         return ret;
329 }
330
331 static struct page *alloc_buddy_huge_page(struct vm_area_struct *vma,
332                                                 unsigned long address)
333 {
334         struct page *page;
335         unsigned int nid;
336
337         /*
338          * Assume we will successfully allocate the surplus page to
339          * prevent racing processes from causing the surplus to exceed
340          * overcommit
341          *
342          * This however introduces a different race, where a process B
343          * tries to grow the static hugepage pool while alloc_pages() is
344          * called by process A. B will only examine the per-node
345          * counters in determining if surplus huge pages can be
346          * converted to normal huge pages in adjust_pool_surplus(). A
347          * won't be able to increment the per-node counter, until the
348          * lock is dropped by B, but B doesn't drop hugetlb_lock until
349          * no more huge pages can be converted from surplus to normal
350          * state (and doesn't try to convert again). Thus, we have a
351          * case where a surplus huge page exists, the pool is grown, and
352          * the surplus huge page still exists after, even though it
353          * should just have been converted to a normal huge page. This
354          * does not leak memory, though, as the hugepage will be freed
355          * once it is out of use. It also does not allow the counters to
356          * go out of whack in adjust_pool_surplus() as we don't modify
357          * the node values until we've gotten the hugepage and only the
358          * per-node value is checked there.
359          */
360         spin_lock(&hugetlb_lock);
361         if (surplus_huge_pages >= nr_overcommit_huge_pages) {
362                 spin_unlock(&hugetlb_lock);
363                 return NULL;
364         } else {
365                 nr_huge_pages++;
366                 surplus_huge_pages++;
367         }
368         spin_unlock(&hugetlb_lock);
369
370         page = alloc_pages(htlb_alloc_mask|__GFP_COMP|
371                                         __GFP_REPEAT|__GFP_NOWARN,
372                                         HUGETLB_PAGE_ORDER);
373
374         spin_lock(&hugetlb_lock);
375         if (page) {
376                 /*
377                  * This page is now managed by the hugetlb allocator and has
378                  * no users -- drop the buddy allocator's reference.
379                  */
380                 put_page_testzero(page);
381                 VM_BUG_ON(page_count(page));
382                 nid = page_to_nid(page);
383                 set_compound_page_dtor(page, free_huge_page);
384                 /*
385                  * We incremented the global counters already
386                  */
387                 nr_huge_pages_node[nid]++;
388                 surplus_huge_pages_node[nid]++;
389                 __count_vm_event(HTLB_BUDDY_PGALLOC);
390         } else {
391                 nr_huge_pages--;
392                 surplus_huge_pages--;
393                 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
394         }
395         spin_unlock(&hugetlb_lock);
396
397         return page;
398 }
399
400 /*
401  * Increase the hugetlb pool such that it can accomodate a reservation
402  * of size 'delta'.
403  */
404 static int gather_surplus_pages(int delta)
405 {
406         struct list_head surplus_list;
407         struct page *page, *tmp;
408         int ret, i;
409         int needed, allocated;
410
411         needed = (resv_huge_pages + delta) - free_huge_pages;
412         if (needed <= 0) {
413                 resv_huge_pages += delta;
414                 return 0;
415         }
416
417         allocated = 0;
418         INIT_LIST_HEAD(&surplus_list);
419
420         ret = -ENOMEM;
421 retry:
422         spin_unlock(&hugetlb_lock);
423         for (i = 0; i < needed; i++) {
424                 page = alloc_buddy_huge_page(NULL, 0);
425                 if (!page) {
426                         /*
427                          * We were not able to allocate enough pages to
428                          * satisfy the entire reservation so we free what
429                          * we've allocated so far.
430                          */
431                         spin_lock(&hugetlb_lock);
432                         needed = 0;
433                         goto free;
434                 }
435
436                 list_add(&page->lru, &surplus_list);
437         }
438         allocated += needed;
439
440         /*
441          * After retaking hugetlb_lock, we need to recalculate 'needed'
442          * because either resv_huge_pages or free_huge_pages may have changed.
443          */
444         spin_lock(&hugetlb_lock);
445         needed = (resv_huge_pages + delta) - (free_huge_pages + allocated);
446         if (needed > 0)
447                 goto retry;
448
449         /*
450          * The surplus_list now contains _at_least_ the number of extra pages
451          * needed to accomodate the reservation.  Add the appropriate number
452          * of pages to the hugetlb pool and free the extras back to the buddy
453          * allocator.  Commit the entire reservation here to prevent another
454          * process from stealing the pages as they are added to the pool but
455          * before they are reserved.
456          */
457         needed += allocated;
458         resv_huge_pages += delta;
459         ret = 0;
460 free:
461         /* Free the needed pages to the hugetlb pool */
462         list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
463                 if ((--needed) < 0)
464                         break;
465                 list_del(&page->lru);
466                 enqueue_huge_page(page);
467         }
468
469         /* Free unnecessary surplus pages to the buddy allocator */
470         if (!list_empty(&surplus_list)) {
471                 spin_unlock(&hugetlb_lock);
472                 list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
473                         list_del(&page->lru);
474                         /*
475                          * The page has a reference count of zero already, so
476                          * call free_huge_page directly instead of using
477                          * put_page.  This must be done with hugetlb_lock
478                          * unlocked which is safe because free_huge_page takes
479                          * hugetlb_lock before deciding how to free the page.
480                          */
481                         free_huge_page(page);
482                 }
483                 spin_lock(&hugetlb_lock);
484         }
485
486         return ret;
487 }
488
489 /*
490  * When releasing a hugetlb pool reservation, any surplus pages that were
491  * allocated to satisfy the reservation must be explicitly freed if they were
492  * never used.
493  */
494 static void return_unused_surplus_pages(unsigned long unused_resv_pages)
495 {
496         static int nid = -1;
497         struct page *page;
498         unsigned long nr_pages;
499
500         /*
501          * We want to release as many surplus pages as possible, spread
502          * evenly across all nodes. Iterate across all nodes until we
503          * can no longer free unreserved surplus pages. This occurs when
504          * the nodes with surplus pages have no free pages.
505          */
506         unsigned long remaining_iterations = num_online_nodes();
507
508         /* Uncommit the reservation */
509         resv_huge_pages -= unused_resv_pages;
510
511         nr_pages = min(unused_resv_pages, surplus_huge_pages);
512
513         while (remaining_iterations-- && nr_pages) {
514                 nid = next_node(nid, node_online_map);
515                 if (nid == MAX_NUMNODES)
516                         nid = first_node(node_online_map);
517
518                 if (!surplus_huge_pages_node[nid])
519                         continue;
520
521                 if (!list_empty(&hugepage_freelists[nid])) {
522                         page = list_entry(hugepage_freelists[nid].next,
523                                           struct page, lru);
524                         list_del(&page->lru);
525                         update_and_free_page(page);
526                         free_huge_pages--;
527                         free_huge_pages_node[nid]--;
528                         surplus_huge_pages--;
529                         surplus_huge_pages_node[nid]--;
530                         nr_pages--;
531                         remaining_iterations = num_online_nodes();
532                 }
533         }
534 }
535
536 static struct page *alloc_huge_page(struct vm_area_struct *vma,
537                                     unsigned long addr)
538 {
539         struct page *page;
540         struct address_space *mapping = vma->vm_file->f_mapping;
541         struct inode *inode = mapping->host;
542         unsigned int chg = 0;
543
544         /*
545          * Processes that did not create the mapping will have no reserves and
546          * will not have accounted against quota. Check that the quota can be
547          * made before satisfying the allocation
548          */
549         if (!vma_has_private_reserves(vma)) {
550                 chg = 1;
551                 if (hugetlb_get_quota(inode->i_mapping, chg))
552                         return ERR_PTR(-ENOSPC);
553         }
554
555         spin_lock(&hugetlb_lock);
556         page = dequeue_huge_page_vma(vma, addr);
557         spin_unlock(&hugetlb_lock);
558
559         if (!page) {
560                 page = alloc_buddy_huge_page(vma, addr);
561                 if (!page) {
562                         hugetlb_put_quota(inode->i_mapping, chg);
563                         return ERR_PTR(-VM_FAULT_OOM);
564                 }
565         }
566
567         set_page_refcounted(page);
568         set_page_private(page, (unsigned long) mapping);
569
570         return page;
571 }
572
573 static int __init hugetlb_init(void)
574 {
575         unsigned long i;
576
577         if (HPAGE_SHIFT == 0)
578                 return 0;
579
580         for (i = 0; i < MAX_NUMNODES; ++i)
581                 INIT_LIST_HEAD(&hugepage_freelists[i]);
582
583         hugetlb_next_nid = first_node(node_online_map);
584
585         for (i = 0; i < max_huge_pages; ++i) {
586                 if (!alloc_fresh_huge_page())
587                         break;
588         }
589         max_huge_pages = free_huge_pages = nr_huge_pages = i;
590         printk("Total HugeTLB memory allocated, %ld\n", free_huge_pages);
591         return 0;
592 }
593 module_init(hugetlb_init);
594
595 static int __init hugetlb_setup(char *s)
596 {
597         if (sscanf(s, "%lu", &max_huge_pages) <= 0)
598                 max_huge_pages = 0;
599         return 1;
600 }
601 __setup("hugepages=", hugetlb_setup);
602
603 static unsigned int cpuset_mems_nr(unsigned int *array)
604 {
605         int node;
606         unsigned int nr = 0;
607
608         for_each_node_mask(node, cpuset_current_mems_allowed)
609                 nr += array[node];
610
611         return nr;
612 }
613
614 #ifdef CONFIG_SYSCTL
615 #ifdef CONFIG_HIGHMEM
616 static void try_to_free_low(unsigned long count)
617 {
618         int i;
619
620         for (i = 0; i < MAX_NUMNODES; ++i) {
621                 struct page *page, *next;
622                 list_for_each_entry_safe(page, next, &hugepage_freelists[i], lru) {
623                         if (count >= nr_huge_pages)
624                                 return;
625                         if (PageHighMem(page))
626                                 continue;
627                         list_del(&page->lru);
628                         update_and_free_page(page);
629                         free_huge_pages--;
630                         free_huge_pages_node[page_to_nid(page)]--;
631                 }
632         }
633 }
634 #else
635 static inline void try_to_free_low(unsigned long count)
636 {
637 }
638 #endif
639
640 #define persistent_huge_pages (nr_huge_pages - surplus_huge_pages)
641 static unsigned long set_max_huge_pages(unsigned long count)
642 {
643         unsigned long min_count, ret;
644
645         /*
646          * Increase the pool size
647          * First take pages out of surplus state.  Then make up the
648          * remaining difference by allocating fresh huge pages.
649          *
650          * We might race with alloc_buddy_huge_page() here and be unable
651          * to convert a surplus huge page to a normal huge page. That is
652          * not critical, though, it just means the overall size of the
653          * pool might be one hugepage larger than it needs to be, but
654          * within all the constraints specified by the sysctls.
655          */
656         spin_lock(&hugetlb_lock);
657         while (surplus_huge_pages && count > persistent_huge_pages) {
658                 if (!adjust_pool_surplus(-1))
659                         break;
660         }
661
662         while (count > persistent_huge_pages) {
663                 /*
664                  * If this allocation races such that we no longer need the
665                  * page, free_huge_page will handle it by freeing the page
666                  * and reducing the surplus.
667                  */
668                 spin_unlock(&hugetlb_lock);
669                 ret = alloc_fresh_huge_page();
670                 spin_lock(&hugetlb_lock);
671                 if (!ret)
672                         goto out;
673
674         }
675
676         /*
677          * Decrease the pool size
678          * First return free pages to the buddy allocator (being careful
679          * to keep enough around to satisfy reservations).  Then place
680          * pages into surplus state as needed so the pool will shrink
681          * to the desired size as pages become free.
682          *
683          * By placing pages into the surplus state independent of the
684          * overcommit value, we are allowing the surplus pool size to
685          * exceed overcommit. There are few sane options here. Since
686          * alloc_buddy_huge_page() is checking the global counter,
687          * though, we'll note that we're not allowed to exceed surplus
688          * and won't grow the pool anywhere else. Not until one of the
689          * sysctls are changed, or the surplus pages go out of use.
690          */
691         min_count = resv_huge_pages + nr_huge_pages - free_huge_pages;
692         min_count = max(count, min_count);
693         try_to_free_low(min_count);
694         while (min_count < persistent_huge_pages) {
695                 struct page *page = dequeue_huge_page();
696                 if (!page)
697                         break;
698                 update_and_free_page(page);
699         }
700         while (count < persistent_huge_pages) {
701                 if (!adjust_pool_surplus(1))
702                         break;
703         }
704 out:
705         ret = persistent_huge_pages;
706         spin_unlock(&hugetlb_lock);
707         return ret;
708 }
709
710 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
711                            struct file *file, void __user *buffer,
712                            size_t *length, loff_t *ppos)
713 {
714         proc_doulongvec_minmax(table, write, file, buffer, length, ppos);
715         max_huge_pages = set_max_huge_pages(max_huge_pages);
716         return 0;
717 }
718
719 int hugetlb_treat_movable_handler(struct ctl_table *table, int write,
720                         struct file *file, void __user *buffer,
721                         size_t *length, loff_t *ppos)
722 {
723         proc_dointvec(table, write, file, buffer, length, ppos);
724         if (hugepages_treat_as_movable)
725                 htlb_alloc_mask = GFP_HIGHUSER_MOVABLE;
726         else
727                 htlb_alloc_mask = GFP_HIGHUSER;
728         return 0;
729 }
730
731 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
732                         struct file *file, void __user *buffer,
733                         size_t *length, loff_t *ppos)
734 {
735         proc_doulongvec_minmax(table, write, file, buffer, length, ppos);
736         spin_lock(&hugetlb_lock);
737         nr_overcommit_huge_pages = sysctl_overcommit_huge_pages;
738         spin_unlock(&hugetlb_lock);
739         return 0;
740 }
741
742 #endif /* CONFIG_SYSCTL */
743
744 int hugetlb_report_meminfo(char *buf)
745 {
746         return sprintf(buf,
747                         "HugePages_Total: %5lu\n"
748                         "HugePages_Free:  %5lu\n"
749                         "HugePages_Rsvd:  %5lu\n"
750                         "HugePages_Surp:  %5lu\n"
751                         "Hugepagesize:    %5lu kB\n",
752                         nr_huge_pages,
753                         free_huge_pages,
754                         resv_huge_pages,
755                         surplus_huge_pages,
756                         HPAGE_SIZE/1024);
757 }
758
759 int hugetlb_report_node_meminfo(int nid, char *buf)
760 {
761         return sprintf(buf,
762                 "Node %d HugePages_Total: %5u\n"
763                 "Node %d HugePages_Free:  %5u\n"
764                 "Node %d HugePages_Surp:  %5u\n",
765                 nid, nr_huge_pages_node[nid],
766                 nid, free_huge_pages_node[nid],
767                 nid, surplus_huge_pages_node[nid]);
768 }
769
770 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
771 unsigned long hugetlb_total_pages(void)
772 {
773         return nr_huge_pages * (HPAGE_SIZE / PAGE_SIZE);
774 }
775
776 static int hugetlb_acct_memory(long delta)
777 {
778         int ret = -ENOMEM;
779
780         spin_lock(&hugetlb_lock);
781         /*
782          * When cpuset is configured, it breaks the strict hugetlb page
783          * reservation as the accounting is done on a global variable. Such
784          * reservation is completely rubbish in the presence of cpuset because
785          * the reservation is not checked against page availability for the
786          * current cpuset. Application can still potentially OOM'ed by kernel
787          * with lack of free htlb page in cpuset that the task is in.
788          * Attempt to enforce strict accounting with cpuset is almost
789          * impossible (or too ugly) because cpuset is too fluid that
790          * task or memory node can be dynamically moved between cpusets.
791          *
792          * The change of semantics for shared hugetlb mapping with cpuset is
793          * undesirable. However, in order to preserve some of the semantics,
794          * we fall back to check against current free page availability as
795          * a best attempt and hopefully to minimize the impact of changing
796          * semantics that cpuset has.
797          */
798         if (delta > 0) {
799                 if (gather_surplus_pages(delta) < 0)
800                         goto out;
801
802                 if (delta > cpuset_mems_nr(free_huge_pages_node)) {
803                         return_unused_surplus_pages(delta);
804                         goto out;
805                 }
806         }
807
808         ret = 0;
809         if (delta < 0)
810                 return_unused_surplus_pages((unsigned long) -delta);
811
812 out:
813         spin_unlock(&hugetlb_lock);
814         return ret;
815 }
816
817 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
818 {
819         unsigned long reserve = vma_resv_huge_pages(vma);
820         if (reserve)
821                 hugetlb_acct_memory(-reserve);
822 }
823
824 /*
825  * We cannot handle pagefaults against hugetlb pages at all.  They cause
826  * handle_mm_fault() to try to instantiate regular-sized pages in the
827  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
828  * this far.
829  */
830 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
831 {
832         BUG();
833         return 0;
834 }
835
836 struct vm_operations_struct hugetlb_vm_ops = {
837         .fault = hugetlb_vm_op_fault,
838         .close = hugetlb_vm_op_close,
839 };
840
841 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
842                                 int writable)
843 {
844         pte_t entry;
845
846         if (writable) {
847                 entry =
848                     pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
849         } else {
850                 entry = huge_pte_wrprotect(mk_pte(page, vma->vm_page_prot));
851         }
852         entry = pte_mkyoung(entry);
853         entry = pte_mkhuge(entry);
854
855         return entry;
856 }
857
858 static void set_huge_ptep_writable(struct vm_area_struct *vma,
859                                    unsigned long address, pte_t *ptep)
860 {
861         pte_t entry;
862
863         entry = pte_mkwrite(pte_mkdirty(huge_ptep_get(ptep)));
864         if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1)) {
865                 update_mmu_cache(vma, address, entry);
866         }
867 }
868
869
870 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
871                             struct vm_area_struct *vma)
872 {
873         pte_t *src_pte, *dst_pte, entry;
874         struct page *ptepage;
875         unsigned long addr;
876         int cow;
877
878         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
879
880         for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) {
881                 src_pte = huge_pte_offset(src, addr);
882                 if (!src_pte)
883                         continue;
884                 dst_pte = huge_pte_alloc(dst, addr);
885                 if (!dst_pte)
886                         goto nomem;
887
888                 /* If the pagetables are shared don't copy or take references */
889                 if (dst_pte == src_pte)
890                         continue;
891
892                 spin_lock(&dst->page_table_lock);
893                 spin_lock_nested(&src->page_table_lock, SINGLE_DEPTH_NESTING);
894                 if (!huge_pte_none(huge_ptep_get(src_pte))) {
895                         if (cow)
896                                 huge_ptep_set_wrprotect(src, addr, src_pte);
897                         entry = huge_ptep_get(src_pte);
898                         ptepage = pte_page(entry);
899                         get_page(ptepage);
900                         set_huge_pte_at(dst, addr, dst_pte, entry);
901                 }
902                 spin_unlock(&src->page_table_lock);
903                 spin_unlock(&dst->page_table_lock);
904         }
905         return 0;
906
907 nomem:
908         return -ENOMEM;
909 }
910
911 void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
912                             unsigned long end)
913 {
914         struct mm_struct *mm = vma->vm_mm;
915         unsigned long address;
916         pte_t *ptep;
917         pte_t pte;
918         struct page *page;
919         struct page *tmp;
920         /*
921          * A page gathering list, protected by per file i_mmap_lock. The
922          * lock is used to avoid list corruption from multiple unmapping
923          * of the same page since we are using page->lru.
924          */
925         LIST_HEAD(page_list);
926
927         WARN_ON(!is_vm_hugetlb_page(vma));
928         BUG_ON(start & ~HPAGE_MASK);
929         BUG_ON(end & ~HPAGE_MASK);
930
931         spin_lock(&mm->page_table_lock);
932         for (address = start; address < end; address += HPAGE_SIZE) {
933                 ptep = huge_pte_offset(mm, address);
934                 if (!ptep)
935                         continue;
936
937                 if (huge_pmd_unshare(mm, &address, ptep))
938                         continue;
939
940                 pte = huge_ptep_get_and_clear(mm, address, ptep);
941                 if (huge_pte_none(pte))
942                         continue;
943
944                 page = pte_page(pte);
945                 if (pte_dirty(pte))
946                         set_page_dirty(page);
947                 list_add(&page->lru, &page_list);
948         }
949         spin_unlock(&mm->page_table_lock);
950         flush_tlb_range(vma, start, end);
951         list_for_each_entry_safe(page, tmp, &page_list, lru) {
952                 list_del(&page->lru);
953                 put_page(page);
954         }
955 }
956
957 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
958                           unsigned long end)
959 {
960         /*
961          * It is undesirable to test vma->vm_file as it should be non-null
962          * for valid hugetlb area. However, vm_file will be NULL in the error
963          * cleanup path of do_mmap_pgoff. When hugetlbfs ->mmap method fails,
964          * do_mmap_pgoff() nullifies vma->vm_file before calling this function
965          * to clean up. Since no pte has actually been setup, it is safe to
966          * do nothing in this case.
967          */
968         if (vma->vm_file) {
969                 spin_lock(&vma->vm_file->f_mapping->i_mmap_lock);
970                 __unmap_hugepage_range(vma, start, end);
971                 spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
972         }
973 }
974
975 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
976                         unsigned long address, pte_t *ptep, pte_t pte)
977 {
978         struct page *old_page, *new_page;
979         int avoidcopy;
980
981         old_page = pte_page(pte);
982
983         /* If no-one else is actually using this page, avoid the copy
984          * and just make the page writable */
985         avoidcopy = (page_count(old_page) == 1);
986         if (avoidcopy) {
987                 set_huge_ptep_writable(vma, address, ptep);
988                 return 0;
989         }
990
991         page_cache_get(old_page);
992         new_page = alloc_huge_page(vma, address);
993
994         if (IS_ERR(new_page)) {
995                 page_cache_release(old_page);
996                 return -PTR_ERR(new_page);
997         }
998
999         spin_unlock(&mm->page_table_lock);
1000         copy_huge_page(new_page, old_page, address, vma);
1001         __SetPageUptodate(new_page);
1002         spin_lock(&mm->page_table_lock);
1003
1004         ptep = huge_pte_offset(mm, address & HPAGE_MASK);
1005         if (likely(pte_same(huge_ptep_get(ptep), pte))) {
1006                 /* Break COW */
1007                 huge_ptep_clear_flush(vma, address, ptep);
1008                 set_huge_pte_at(mm, address, ptep,
1009                                 make_huge_pte(vma, new_page, 1));
1010                 /* Make the old page be freed below */
1011                 new_page = old_page;
1012         }
1013         page_cache_release(new_page);
1014         page_cache_release(old_page);
1015         return 0;
1016 }
1017
1018 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
1019                         unsigned long address, pte_t *ptep, int write_access)
1020 {
1021         int ret = VM_FAULT_SIGBUS;
1022         unsigned long idx;
1023         unsigned long size;
1024         struct page *page;
1025         struct address_space *mapping;
1026         pte_t new_pte;
1027
1028         mapping = vma->vm_file->f_mapping;
1029         idx = ((address - vma->vm_start) >> HPAGE_SHIFT)
1030                 + (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));
1031
1032         /*
1033          * Use page lock to guard against racing truncation
1034          * before we get page_table_lock.
1035          */
1036 retry:
1037         page = find_lock_page(mapping, idx);
1038         if (!page) {
1039                 size = i_size_read(mapping->host) >> HPAGE_SHIFT;
1040                 if (idx >= size)
1041                         goto out;
1042                 page = alloc_huge_page(vma, address);
1043                 if (IS_ERR(page)) {
1044                         ret = -PTR_ERR(page);
1045                         goto out;
1046                 }
1047                 clear_huge_page(page, address);
1048                 __SetPageUptodate(page);
1049
1050                 if (vma->vm_flags & VM_SHARED) {
1051                         int err;
1052                         struct inode *inode = mapping->host;
1053
1054                         err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
1055                         if (err) {
1056                                 put_page(page);
1057                                 if (err == -EEXIST)
1058                                         goto retry;
1059                                 goto out;
1060                         }
1061
1062                         spin_lock(&inode->i_lock);
1063                         inode->i_blocks += BLOCKS_PER_HUGEPAGE;
1064                         spin_unlock(&inode->i_lock);
1065                 } else
1066                         lock_page(page);
1067         }
1068
1069         spin_lock(&mm->page_table_lock);
1070         size = i_size_read(mapping->host) >> HPAGE_SHIFT;
1071         if (idx >= size)
1072                 goto backout;
1073
1074         ret = 0;
1075         if (!huge_pte_none(huge_ptep_get(ptep)))
1076                 goto backout;
1077
1078         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
1079                                 && (vma->vm_flags & VM_SHARED)));
1080         set_huge_pte_at(mm, address, ptep, new_pte);
1081
1082         if (write_access && !(vma->vm_flags & VM_SHARED)) {
1083                 /* Optimization, do the COW without a second fault */
1084                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte);
1085         }
1086
1087         spin_unlock(&mm->page_table_lock);
1088         unlock_page(page);
1089 out:
1090         return ret;
1091
1092 backout:
1093         spin_unlock(&mm->page_table_lock);
1094         unlock_page(page);
1095         put_page(page);
1096         goto out;
1097 }
1098
1099 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
1100                         unsigned long address, int write_access)
1101 {
1102         pte_t *ptep;
1103         pte_t entry;
1104         int ret;
1105         static DEFINE_MUTEX(hugetlb_instantiation_mutex);
1106
1107         ptep = huge_pte_alloc(mm, address);
1108         if (!ptep)
1109                 return VM_FAULT_OOM;
1110
1111         /*
1112          * Serialize hugepage allocation and instantiation, so that we don't
1113          * get spurious allocation failures if two CPUs race to instantiate
1114          * the same page in the page cache.
1115          */
1116         mutex_lock(&hugetlb_instantiation_mutex);
1117         entry = huge_ptep_get(ptep);
1118         if (huge_pte_none(entry)) {
1119                 ret = hugetlb_no_page(mm, vma, address, ptep, write_access);
1120                 mutex_unlock(&hugetlb_instantiation_mutex);
1121                 return ret;
1122         }
1123
1124         ret = 0;
1125
1126         spin_lock(&mm->page_table_lock);
1127         /* Check for a racing update before calling hugetlb_cow */
1128         if (likely(pte_same(entry, huge_ptep_get(ptep))))
1129                 if (write_access && !pte_write(entry))
1130                         ret = hugetlb_cow(mm, vma, address, ptep, entry);
1131         spin_unlock(&mm->page_table_lock);
1132         mutex_unlock(&hugetlb_instantiation_mutex);
1133
1134         return ret;
1135 }
1136
1137 int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
1138                         struct page **pages, struct vm_area_struct **vmas,
1139                         unsigned long *position, int *length, int i,
1140                         int write)
1141 {
1142         unsigned long pfn_offset;
1143         unsigned long vaddr = *position;
1144         int remainder = *length;
1145
1146         spin_lock(&mm->page_table_lock);
1147         while (vaddr < vma->vm_end && remainder) {
1148                 pte_t *pte;
1149                 struct page *page;
1150
1151                 /*
1152                  * Some archs (sparc64, sh*) have multiple pte_ts to
1153                  * each hugepage.  We have to make * sure we get the
1154                  * first, for the page indexing below to work.
1155                  */
1156                 pte = huge_pte_offset(mm, vaddr & HPAGE_MASK);
1157
1158                 if (!pte || huge_pte_none(huge_ptep_get(pte)) ||
1159                     (write && !pte_write(huge_ptep_get(pte)))) {
1160                         int ret;
1161
1162                         spin_unlock(&mm->page_table_lock);
1163                         ret = hugetlb_fault(mm, vma, vaddr, write);
1164                         spin_lock(&mm->page_table_lock);
1165                         if (!(ret & VM_FAULT_ERROR))
1166                                 continue;
1167
1168                         remainder = 0;
1169                         if (!i)
1170                                 i = -EFAULT;
1171                         break;
1172                 }
1173
1174                 pfn_offset = (vaddr & ~HPAGE_MASK) >> PAGE_SHIFT;
1175                 page = pte_page(huge_ptep_get(pte));
1176 same_page:
1177                 if (pages) {
1178                         get_page(page);
1179                         pages[i] = page + pfn_offset;
1180                 }
1181
1182                 if (vmas)
1183                         vmas[i] = vma;
1184
1185                 vaddr += PAGE_SIZE;
1186                 ++pfn_offset;
1187                 --remainder;
1188                 ++i;
1189                 if (vaddr < vma->vm_end && remainder &&
1190                                 pfn_offset < HPAGE_SIZE/PAGE_SIZE) {
1191                         /*
1192                          * We use pfn_offset to avoid touching the pageframes
1193                          * of this compound page.
1194                          */
1195                         goto same_page;
1196                 }
1197         }
1198         spin_unlock(&mm->page_table_lock);
1199         *length = remainder;
1200         *position = vaddr;
1201
1202         return i;
1203 }
1204
1205 void hugetlb_change_protection(struct vm_area_struct *vma,
1206                 unsigned long address, unsigned long end, pgprot_t newprot)
1207 {
1208         struct mm_struct *mm = vma->vm_mm;
1209         unsigned long start = address;
1210         pte_t *ptep;
1211         pte_t pte;
1212
1213         BUG_ON(address >= end);
1214         flush_cache_range(vma, address, end);
1215
1216         spin_lock(&vma->vm_file->f_mapping->i_mmap_lock);
1217         spin_lock(&mm->page_table_lock);
1218         for (; address < end; address += HPAGE_SIZE) {
1219                 ptep = huge_pte_offset(mm, address);
1220                 if (!ptep)
1221                         continue;
1222                 if (huge_pmd_unshare(mm, &address, ptep))
1223                         continue;
1224                 if (!huge_pte_none(huge_ptep_get(ptep))) {
1225                         pte = huge_ptep_get_and_clear(mm, address, ptep);
1226                         pte = pte_mkhuge(pte_modify(pte, newprot));
1227                         set_huge_pte_at(mm, address, ptep, pte);
1228                 }
1229         }
1230         spin_unlock(&mm->page_table_lock);
1231         spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
1232
1233         flush_tlb_range(vma, start, end);
1234 }
1235
1236 struct file_region {
1237         struct list_head link;
1238         long from;
1239         long to;
1240 };
1241
1242 static long region_add(struct list_head *head, long f, long t)
1243 {
1244         struct file_region *rg, *nrg, *trg;
1245
1246         /* Locate the region we are either in or before. */
1247         list_for_each_entry(rg, head, link)
1248                 if (f <= rg->to)
1249                         break;
1250
1251         /* Round our left edge to the current segment if it encloses us. */
1252         if (f > rg->from)
1253                 f = rg->from;
1254
1255         /* Check for and consume any regions we now overlap with. */
1256         nrg = rg;
1257         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
1258                 if (&rg->link == head)
1259                         break;
1260                 if (rg->from > t)
1261                         break;
1262
1263                 /* If this area reaches higher then extend our area to
1264                  * include it completely.  If this is not the first area
1265                  * which we intend to reuse, free it. */
1266                 if (rg->to > t)
1267                         t = rg->to;
1268                 if (rg != nrg) {
1269                         list_del(&rg->link);
1270                         kfree(rg);
1271                 }
1272         }
1273         nrg->from = f;
1274         nrg->to = t;
1275         return 0;
1276 }
1277
1278 static long region_chg(struct list_head *head, long f, long t)
1279 {
1280         struct file_region *rg, *nrg;
1281         long chg = 0;
1282
1283         /* Locate the region we are before or in. */
1284         list_for_each_entry(rg, head, link)
1285                 if (f <= rg->to)
1286                         break;
1287
1288         /* If we are below the current region then a new region is required.
1289          * Subtle, allocate a new region at the position but make it zero
1290          * size such that we can guarantee to record the reservation. */
1291         if (&rg->link == head || t < rg->from) {
1292                 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
1293                 if (!nrg)
1294                         return -ENOMEM;
1295                 nrg->from = f;
1296                 nrg->to   = f;
1297                 INIT_LIST_HEAD(&nrg->link);
1298                 list_add(&nrg->link, rg->link.prev);
1299
1300                 return t - f;
1301         }
1302
1303         /* Round our left edge to the current segment if it encloses us. */
1304         if (f > rg->from)
1305                 f = rg->from;
1306         chg = t - f;
1307
1308         /* Check for and consume any regions we now overlap with. */
1309         list_for_each_entry(rg, rg->link.prev, link) {
1310                 if (&rg->link == head)
1311                         break;
1312                 if (rg->from > t)
1313                         return chg;
1314
1315                 /* We overlap with this area, if it extends futher than
1316                  * us then we must extend ourselves.  Account for its
1317                  * existing reservation. */
1318                 if (rg->to > t) {
1319                         chg += rg->to - t;
1320                         t = rg->to;
1321                 }
1322                 chg -= rg->to - rg->from;
1323         }
1324         return chg;
1325 }
1326
1327 static long region_truncate(struct list_head *head, long end)
1328 {
1329         struct file_region *rg, *trg;
1330         long chg = 0;
1331
1332         /* Locate the region we are either in or before. */
1333         list_for_each_entry(rg, head, link)
1334                 if (end <= rg->to)
1335                         break;
1336         if (&rg->link == head)
1337                 return 0;
1338
1339         /* If we are in the middle of a region then adjust it. */
1340         if (end > rg->from) {
1341                 chg = rg->to - end;
1342                 rg->to = end;
1343                 rg = list_entry(rg->link.next, typeof(*rg), link);
1344         }
1345
1346         /* Drop any remaining regions. */
1347         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
1348                 if (&rg->link == head)
1349                         break;
1350                 chg += rg->to - rg->from;
1351                 list_del(&rg->link);
1352                 kfree(rg);
1353         }
1354         return chg;
1355 }
1356
1357 int hugetlb_reserve_pages(struct inode *inode,
1358                                         long from, long to,
1359                                         struct vm_area_struct *vma)
1360 {
1361         long ret, chg;
1362
1363         /*
1364          * Shared mappings base their reservation on the number of pages that
1365          * are already allocated on behalf of the file. Private mappings need
1366          * to reserve the full area even if read-only as mprotect() may be
1367          * called to make the mapping read-write. Assume !vma is a shm mapping
1368          */
1369         if (!vma || vma->vm_flags & VM_SHARED)
1370                 chg = region_chg(&inode->i_mapping->private_list, from, to);
1371         else {
1372                 chg = to - from;
1373                 set_vma_resv_huge_pages(vma, chg);
1374         }
1375
1376         if (chg < 0)
1377                 return chg;
1378
1379         if (hugetlb_get_quota(inode->i_mapping, chg))
1380                 return -ENOSPC;
1381         ret = hugetlb_acct_memory(chg);
1382         if (ret < 0) {
1383                 hugetlb_put_quota(inode->i_mapping, chg);
1384                 return ret;
1385         }
1386         if (!vma || vma->vm_flags & VM_SHARED)
1387                 region_add(&inode->i_mapping->private_list, from, to);
1388         return 0;
1389 }
1390
1391 void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
1392 {
1393         long chg = region_truncate(&inode->i_mapping->private_list, offset);
1394
1395         spin_lock(&inode->i_lock);
1396         inode->i_blocks -= BLOCKS_PER_HUGEPAGE * freed;
1397         spin_unlock(&inode->i_lock);
1398
1399         hugetlb_put_quota(inode->i_mapping, (chg - freed));
1400         hugetlb_acct_memory(-(chg - freed));
1401 }