hugetlb: move reservation region support earlier
[linux-2.6.git] / mm / hugetlb.c
1 /*
2  * Generic hugetlb support.
3  * (C) William Irwin, April 2004
4  */
5 #include <linux/gfp.h>
6 #include <linux/list.h>
7 #include <linux/init.h>
8 #include <linux/module.h>
9 #include <linux/mm.h>
10 #include <linux/sysctl.h>
11 #include <linux/highmem.h>
12 #include <linux/nodemask.h>
13 #include <linux/pagemap.h>
14 #include <linux/mempolicy.h>
15 #include <linux/cpuset.h>
16 #include <linux/mutex.h>
17
18 #include <asm/page.h>
19 #include <asm/pgtable.h>
20
21 #include <linux/hugetlb.h>
22 #include "internal.h"
23
24 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
25 static unsigned long nr_huge_pages, free_huge_pages, resv_huge_pages;
26 static unsigned long surplus_huge_pages;
27 static unsigned long nr_overcommit_huge_pages;
28 unsigned long max_huge_pages;
29 unsigned long sysctl_overcommit_huge_pages;
30 static struct list_head hugepage_freelists[MAX_NUMNODES];
31 static unsigned int nr_huge_pages_node[MAX_NUMNODES];
32 static unsigned int free_huge_pages_node[MAX_NUMNODES];
33 static unsigned int surplus_huge_pages_node[MAX_NUMNODES];
34 static gfp_t htlb_alloc_mask = GFP_HIGHUSER;
35 unsigned long hugepages_treat_as_movable;
36 static int hugetlb_next_nid;
37
38 /*
39  * Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
40  */
41 static DEFINE_SPINLOCK(hugetlb_lock);
42
43 /*
44  * Region tracking -- allows tracking of reservations and instantiated pages
45  *                    across the pages in a mapping.
46  */
47 struct file_region {
48         struct list_head link;
49         long from;
50         long to;
51 };
52
53 static long region_add(struct list_head *head, long f, long t)
54 {
55         struct file_region *rg, *nrg, *trg;
56
57         /* Locate the region we are either in or before. */
58         list_for_each_entry(rg, head, link)
59                 if (f <= rg->to)
60                         break;
61
62         /* Round our left edge to the current segment if it encloses us. */
63         if (f > rg->from)
64                 f = rg->from;
65
66         /* Check for and consume any regions we now overlap with. */
67         nrg = rg;
68         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
69                 if (&rg->link == head)
70                         break;
71                 if (rg->from > t)
72                         break;
73
74                 /* If this area reaches higher then extend our area to
75                  * include it completely.  If this is not the first area
76                  * which we intend to reuse, free it. */
77                 if (rg->to > t)
78                         t = rg->to;
79                 if (rg != nrg) {
80                         list_del(&rg->link);
81                         kfree(rg);
82                 }
83         }
84         nrg->from = f;
85         nrg->to = t;
86         return 0;
87 }
88
89 static long region_chg(struct list_head *head, long f, long t)
90 {
91         struct file_region *rg, *nrg;
92         long chg = 0;
93
94         /* Locate the region we are before or in. */
95         list_for_each_entry(rg, head, link)
96                 if (f <= rg->to)
97                         break;
98
99         /* If we are below the current region then a new region is required.
100          * Subtle, allocate a new region at the position but make it zero
101          * size such that we can guarantee to record the reservation. */
102         if (&rg->link == head || t < rg->from) {
103                 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
104                 if (!nrg)
105                         return -ENOMEM;
106                 nrg->from = f;
107                 nrg->to   = f;
108                 INIT_LIST_HEAD(&nrg->link);
109                 list_add(&nrg->link, rg->link.prev);
110
111                 return t - f;
112         }
113
114         /* Round our left edge to the current segment if it encloses us. */
115         if (f > rg->from)
116                 f = rg->from;
117         chg = t - f;
118
119         /* Check for and consume any regions we now overlap with. */
120         list_for_each_entry(rg, rg->link.prev, link) {
121                 if (&rg->link == head)
122                         break;
123                 if (rg->from > t)
124                         return chg;
125
126                 /* We overlap with this area, if it extends futher than
127                  * us then we must extend ourselves.  Account for its
128                  * existing reservation. */
129                 if (rg->to > t) {
130                         chg += rg->to - t;
131                         t = rg->to;
132                 }
133                 chg -= rg->to - rg->from;
134         }
135         return chg;
136 }
137
138 static long region_truncate(struct list_head *head, long end)
139 {
140         struct file_region *rg, *trg;
141         long chg = 0;
142
143         /* Locate the region we are either in or before. */
144         list_for_each_entry(rg, head, link)
145                 if (end <= rg->to)
146                         break;
147         if (&rg->link == head)
148                 return 0;
149
150         /* If we are in the middle of a region then adjust it. */
151         if (end > rg->from) {
152                 chg = rg->to - end;
153                 rg->to = end;
154                 rg = list_entry(rg->link.next, typeof(*rg), link);
155         }
156
157         /* Drop any remaining regions. */
158         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
159                 if (&rg->link == head)
160                         break;
161                 chg += rg->to - rg->from;
162                 list_del(&rg->link);
163                 kfree(rg);
164         }
165         return chg;
166 }
167
168 /*
169  * Convert the address within this vma to the page offset within
170  * the mapping, in base page units.
171  */
172 static pgoff_t vma_page_offset(struct vm_area_struct *vma,
173                                 unsigned long address)
174 {
175         return ((address - vma->vm_start) >> PAGE_SHIFT) +
176                                         (vma->vm_pgoff >> PAGE_SHIFT);
177 }
178
179 /*
180  * Convert the address within this vma to the page offset within
181  * the mapping, in pagecache page units; huge pages here.
182  */
183 static pgoff_t vma_pagecache_offset(struct vm_area_struct *vma,
184                                         unsigned long address)
185 {
186         return ((address - vma->vm_start) >> HPAGE_SHIFT) +
187                         (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));
188 }
189
190 #define HPAGE_RESV_OWNER    (1UL << (BITS_PER_LONG - 1))
191 #define HPAGE_RESV_UNMAPPED (1UL << (BITS_PER_LONG - 2))
192 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
193 /*
194  * These helpers are used to track how many pages are reserved for
195  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
196  * is guaranteed to have their future faults succeed.
197  *
198  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
199  * the reserve counters are updated with the hugetlb_lock held. It is safe
200  * to reset the VMA at fork() time as it is not in use yet and there is no
201  * chance of the global counters getting corrupted as a result of the values.
202  */
203 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
204 {
205         return (unsigned long)vma->vm_private_data;
206 }
207
208 static void set_vma_private_data(struct vm_area_struct *vma,
209                                                         unsigned long value)
210 {
211         vma->vm_private_data = (void *)value;
212 }
213
214 static unsigned long vma_resv_huge_pages(struct vm_area_struct *vma)
215 {
216         VM_BUG_ON(!is_vm_hugetlb_page(vma));
217         if (!(vma->vm_flags & VM_SHARED))
218                 return get_vma_private_data(vma) & ~HPAGE_RESV_MASK;
219         return 0;
220 }
221
222 static void set_vma_resv_huge_pages(struct vm_area_struct *vma,
223                                                         unsigned long reserve)
224 {
225         VM_BUG_ON(!is_vm_hugetlb_page(vma));
226         VM_BUG_ON(vma->vm_flags & VM_SHARED);
227
228         set_vma_private_data(vma,
229                 (get_vma_private_data(vma) & HPAGE_RESV_MASK) | reserve);
230 }
231
232 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
233 {
234         VM_BUG_ON(!is_vm_hugetlb_page(vma));
235         VM_BUG_ON(vma->vm_flags & VM_SHARED);
236
237         set_vma_private_data(vma, get_vma_private_data(vma) | flags);
238 }
239
240 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
241 {
242         VM_BUG_ON(!is_vm_hugetlb_page(vma));
243
244         return (get_vma_private_data(vma) & flag) != 0;
245 }
246
247 /* Decrement the reserved pages in the hugepage pool by one */
248 static void decrement_hugepage_resv_vma(struct vm_area_struct *vma)
249 {
250         if (vma->vm_flags & VM_SHARED) {
251                 /* Shared mappings always use reserves */
252                 resv_huge_pages--;
253         } else {
254                 /*
255                  * Only the process that called mmap() has reserves for
256                  * private mappings.
257                  */
258                 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
259                         unsigned long flags, reserve;
260                         resv_huge_pages--;
261                         flags = (unsigned long)vma->vm_private_data &
262                                                         HPAGE_RESV_MASK;
263                         reserve = (unsigned long)vma->vm_private_data - 1;
264                         vma->vm_private_data = (void *)(reserve | flags);
265                 }
266         }
267 }
268
269 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
270 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
271 {
272         VM_BUG_ON(!is_vm_hugetlb_page(vma));
273         if (!(vma->vm_flags & VM_SHARED))
274                 vma->vm_private_data = (void *)0;
275 }
276
277 /* Returns true if the VMA has associated reserve pages */
278 static int vma_has_private_reserves(struct vm_area_struct *vma)
279 {
280         if (vma->vm_flags & VM_SHARED)
281                 return 0;
282         if (!vma_resv_huge_pages(vma))
283                 return 0;
284         return 1;
285 }
286
287 static void clear_huge_page(struct page *page, unsigned long addr)
288 {
289         int i;
290
291         might_sleep();
292         for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); i++) {
293                 cond_resched();
294                 clear_user_highpage(page + i, addr + i * PAGE_SIZE);
295         }
296 }
297
298 static void copy_huge_page(struct page *dst, struct page *src,
299                            unsigned long addr, struct vm_area_struct *vma)
300 {
301         int i;
302
303         might_sleep();
304         for (i = 0; i < HPAGE_SIZE/PAGE_SIZE; i++) {
305                 cond_resched();
306                 copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE, vma);
307         }
308 }
309
310 static void enqueue_huge_page(struct page *page)
311 {
312         int nid = page_to_nid(page);
313         list_add(&page->lru, &hugepage_freelists[nid]);
314         free_huge_pages++;
315         free_huge_pages_node[nid]++;
316 }
317
318 static struct page *dequeue_huge_page(void)
319 {
320         int nid;
321         struct page *page = NULL;
322
323         for (nid = 0; nid < MAX_NUMNODES; ++nid) {
324                 if (!list_empty(&hugepage_freelists[nid])) {
325                         page = list_entry(hugepage_freelists[nid].next,
326                                           struct page, lru);
327                         list_del(&page->lru);
328                         free_huge_pages--;
329                         free_huge_pages_node[nid]--;
330                         break;
331                 }
332         }
333         return page;
334 }
335
336 static struct page *dequeue_huge_page_vma(struct vm_area_struct *vma,
337                                 unsigned long address, int avoid_reserve)
338 {
339         int nid;
340         struct page *page = NULL;
341         struct mempolicy *mpol;
342         nodemask_t *nodemask;
343         struct zonelist *zonelist = huge_zonelist(vma, address,
344                                         htlb_alloc_mask, &mpol, &nodemask);
345         struct zone *zone;
346         struct zoneref *z;
347
348         /*
349          * A child process with MAP_PRIVATE mappings created by their parent
350          * have no page reserves. This check ensures that reservations are
351          * not "stolen". The child may still get SIGKILLed
352          */
353         if (!vma_has_private_reserves(vma) &&
354                         free_huge_pages - resv_huge_pages == 0)
355                 return NULL;
356
357         /* If reserves cannot be used, ensure enough pages are in the pool */
358         if (avoid_reserve && free_huge_pages - resv_huge_pages == 0)
359                 return NULL;
360
361         for_each_zone_zonelist_nodemask(zone, z, zonelist,
362                                                 MAX_NR_ZONES - 1, nodemask) {
363                 nid = zone_to_nid(zone);
364                 if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask) &&
365                     !list_empty(&hugepage_freelists[nid])) {
366                         page = list_entry(hugepage_freelists[nid].next,
367                                           struct page, lru);
368                         list_del(&page->lru);
369                         free_huge_pages--;
370                         free_huge_pages_node[nid]--;
371
372                         if (!avoid_reserve)
373                                 decrement_hugepage_resv_vma(vma);
374
375                         break;
376                 }
377         }
378         mpol_cond_put(mpol);
379         return page;
380 }
381
382 static void update_and_free_page(struct page *page)
383 {
384         int i;
385         nr_huge_pages--;
386         nr_huge_pages_node[page_to_nid(page)]--;
387         for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) {
388                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
389                                 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
390                                 1 << PG_private | 1<< PG_writeback);
391         }
392         set_compound_page_dtor(page, NULL);
393         set_page_refcounted(page);
394         arch_release_hugepage(page);
395         __free_pages(page, HUGETLB_PAGE_ORDER);
396 }
397
398 static void free_huge_page(struct page *page)
399 {
400         int nid = page_to_nid(page);
401         struct address_space *mapping;
402
403         mapping = (struct address_space *) page_private(page);
404         set_page_private(page, 0);
405         BUG_ON(page_count(page));
406         INIT_LIST_HEAD(&page->lru);
407
408         spin_lock(&hugetlb_lock);
409         if (surplus_huge_pages_node[nid]) {
410                 update_and_free_page(page);
411                 surplus_huge_pages--;
412                 surplus_huge_pages_node[nid]--;
413         } else {
414                 enqueue_huge_page(page);
415         }
416         spin_unlock(&hugetlb_lock);
417         if (mapping)
418                 hugetlb_put_quota(mapping, 1);
419 }
420
421 /*
422  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
423  * balanced by operating on them in a round-robin fashion.
424  * Returns 1 if an adjustment was made.
425  */
426 static int adjust_pool_surplus(int delta)
427 {
428         static int prev_nid;
429         int nid = prev_nid;
430         int ret = 0;
431
432         VM_BUG_ON(delta != -1 && delta != 1);
433         do {
434                 nid = next_node(nid, node_online_map);
435                 if (nid == MAX_NUMNODES)
436                         nid = first_node(node_online_map);
437
438                 /* To shrink on this node, there must be a surplus page */
439                 if (delta < 0 && !surplus_huge_pages_node[nid])
440                         continue;
441                 /* Surplus cannot exceed the total number of pages */
442                 if (delta > 0 && surplus_huge_pages_node[nid] >=
443                                                 nr_huge_pages_node[nid])
444                         continue;
445
446                 surplus_huge_pages += delta;
447                 surplus_huge_pages_node[nid] += delta;
448                 ret = 1;
449                 break;
450         } while (nid != prev_nid);
451
452         prev_nid = nid;
453         return ret;
454 }
455
456 static struct page *alloc_fresh_huge_page_node(int nid)
457 {
458         struct page *page;
459
460         page = alloc_pages_node(nid,
461                 htlb_alloc_mask|__GFP_COMP|__GFP_THISNODE|
462                                                 __GFP_REPEAT|__GFP_NOWARN,
463                 HUGETLB_PAGE_ORDER);
464         if (page) {
465                 if (arch_prepare_hugepage(page)) {
466                         __free_pages(page, HUGETLB_PAGE_ORDER);
467                         return NULL;
468                 }
469                 set_compound_page_dtor(page, free_huge_page);
470                 spin_lock(&hugetlb_lock);
471                 nr_huge_pages++;
472                 nr_huge_pages_node[nid]++;
473                 spin_unlock(&hugetlb_lock);
474                 put_page(page); /* free it into the hugepage allocator */
475         }
476
477         return page;
478 }
479
480 static int alloc_fresh_huge_page(void)
481 {
482         struct page *page;
483         int start_nid;
484         int next_nid;
485         int ret = 0;
486
487         start_nid = hugetlb_next_nid;
488
489         do {
490                 page = alloc_fresh_huge_page_node(hugetlb_next_nid);
491                 if (page)
492                         ret = 1;
493                 /*
494                  * Use a helper variable to find the next node and then
495                  * copy it back to hugetlb_next_nid afterwards:
496                  * otherwise there's a window in which a racer might
497                  * pass invalid nid MAX_NUMNODES to alloc_pages_node.
498                  * But we don't need to use a spin_lock here: it really
499                  * doesn't matter if occasionally a racer chooses the
500                  * same nid as we do.  Move nid forward in the mask even
501                  * if we just successfully allocated a hugepage so that
502                  * the next caller gets hugepages on the next node.
503                  */
504                 next_nid = next_node(hugetlb_next_nid, node_online_map);
505                 if (next_nid == MAX_NUMNODES)
506                         next_nid = first_node(node_online_map);
507                 hugetlb_next_nid = next_nid;
508         } while (!page && hugetlb_next_nid != start_nid);
509
510         if (ret)
511                 count_vm_event(HTLB_BUDDY_PGALLOC);
512         else
513                 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
514
515         return ret;
516 }
517
518 static struct page *alloc_buddy_huge_page(struct vm_area_struct *vma,
519                                                 unsigned long address)
520 {
521         struct page *page;
522         unsigned int nid;
523
524         /*
525          * Assume we will successfully allocate the surplus page to
526          * prevent racing processes from causing the surplus to exceed
527          * overcommit
528          *
529          * This however introduces a different race, where a process B
530          * tries to grow the static hugepage pool while alloc_pages() is
531          * called by process A. B will only examine the per-node
532          * counters in determining if surplus huge pages can be
533          * converted to normal huge pages in adjust_pool_surplus(). A
534          * won't be able to increment the per-node counter, until the
535          * lock is dropped by B, but B doesn't drop hugetlb_lock until
536          * no more huge pages can be converted from surplus to normal
537          * state (and doesn't try to convert again). Thus, we have a
538          * case where a surplus huge page exists, the pool is grown, and
539          * the surplus huge page still exists after, even though it
540          * should just have been converted to a normal huge page. This
541          * does not leak memory, though, as the hugepage will be freed
542          * once it is out of use. It also does not allow the counters to
543          * go out of whack in adjust_pool_surplus() as we don't modify
544          * the node values until we've gotten the hugepage and only the
545          * per-node value is checked there.
546          */
547         spin_lock(&hugetlb_lock);
548         if (surplus_huge_pages >= nr_overcommit_huge_pages) {
549                 spin_unlock(&hugetlb_lock);
550                 return NULL;
551         } else {
552                 nr_huge_pages++;
553                 surplus_huge_pages++;
554         }
555         spin_unlock(&hugetlb_lock);
556
557         page = alloc_pages(htlb_alloc_mask|__GFP_COMP|
558                                         __GFP_REPEAT|__GFP_NOWARN,
559                                         HUGETLB_PAGE_ORDER);
560
561         spin_lock(&hugetlb_lock);
562         if (page) {
563                 /*
564                  * This page is now managed by the hugetlb allocator and has
565                  * no users -- drop the buddy allocator's reference.
566                  */
567                 put_page_testzero(page);
568                 VM_BUG_ON(page_count(page));
569                 nid = page_to_nid(page);
570                 set_compound_page_dtor(page, free_huge_page);
571                 /*
572                  * We incremented the global counters already
573                  */
574                 nr_huge_pages_node[nid]++;
575                 surplus_huge_pages_node[nid]++;
576                 __count_vm_event(HTLB_BUDDY_PGALLOC);
577         } else {
578                 nr_huge_pages--;
579                 surplus_huge_pages--;
580                 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
581         }
582         spin_unlock(&hugetlb_lock);
583
584         return page;
585 }
586
587 /*
588  * Increase the hugetlb pool such that it can accomodate a reservation
589  * of size 'delta'.
590  */
591 static int gather_surplus_pages(int delta)
592 {
593         struct list_head surplus_list;
594         struct page *page, *tmp;
595         int ret, i;
596         int needed, allocated;
597
598         needed = (resv_huge_pages + delta) - free_huge_pages;
599         if (needed <= 0) {
600                 resv_huge_pages += delta;
601                 return 0;
602         }
603
604         allocated = 0;
605         INIT_LIST_HEAD(&surplus_list);
606
607         ret = -ENOMEM;
608 retry:
609         spin_unlock(&hugetlb_lock);
610         for (i = 0; i < needed; i++) {
611                 page = alloc_buddy_huge_page(NULL, 0);
612                 if (!page) {
613                         /*
614                          * We were not able to allocate enough pages to
615                          * satisfy the entire reservation so we free what
616                          * we've allocated so far.
617                          */
618                         spin_lock(&hugetlb_lock);
619                         needed = 0;
620                         goto free;
621                 }
622
623                 list_add(&page->lru, &surplus_list);
624         }
625         allocated += needed;
626
627         /*
628          * After retaking hugetlb_lock, we need to recalculate 'needed'
629          * because either resv_huge_pages or free_huge_pages may have changed.
630          */
631         spin_lock(&hugetlb_lock);
632         needed = (resv_huge_pages + delta) - (free_huge_pages + allocated);
633         if (needed > 0)
634                 goto retry;
635
636         /*
637          * The surplus_list now contains _at_least_ the number of extra pages
638          * needed to accomodate the reservation.  Add the appropriate number
639          * of pages to the hugetlb pool and free the extras back to the buddy
640          * allocator.  Commit the entire reservation here to prevent another
641          * process from stealing the pages as they are added to the pool but
642          * before they are reserved.
643          */
644         needed += allocated;
645         resv_huge_pages += delta;
646         ret = 0;
647 free:
648         /* Free the needed pages to the hugetlb pool */
649         list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
650                 if ((--needed) < 0)
651                         break;
652                 list_del(&page->lru);
653                 enqueue_huge_page(page);
654         }
655
656         /* Free unnecessary surplus pages to the buddy allocator */
657         if (!list_empty(&surplus_list)) {
658                 spin_unlock(&hugetlb_lock);
659                 list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
660                         list_del(&page->lru);
661                         /*
662                          * The page has a reference count of zero already, so
663                          * call free_huge_page directly instead of using
664                          * put_page.  This must be done with hugetlb_lock
665                          * unlocked which is safe because free_huge_page takes
666                          * hugetlb_lock before deciding how to free the page.
667                          */
668                         free_huge_page(page);
669                 }
670                 spin_lock(&hugetlb_lock);
671         }
672
673         return ret;
674 }
675
676 /*
677  * When releasing a hugetlb pool reservation, any surplus pages that were
678  * allocated to satisfy the reservation must be explicitly freed if they were
679  * never used.
680  */
681 static void return_unused_surplus_pages(unsigned long unused_resv_pages)
682 {
683         static int nid = -1;
684         struct page *page;
685         unsigned long nr_pages;
686
687         /*
688          * We want to release as many surplus pages as possible, spread
689          * evenly across all nodes. Iterate across all nodes until we
690          * can no longer free unreserved surplus pages. This occurs when
691          * the nodes with surplus pages have no free pages.
692          */
693         unsigned long remaining_iterations = num_online_nodes();
694
695         /* Uncommit the reservation */
696         resv_huge_pages -= unused_resv_pages;
697
698         nr_pages = min(unused_resv_pages, surplus_huge_pages);
699
700         while (remaining_iterations-- && nr_pages) {
701                 nid = next_node(nid, node_online_map);
702                 if (nid == MAX_NUMNODES)
703                         nid = first_node(node_online_map);
704
705                 if (!surplus_huge_pages_node[nid])
706                         continue;
707
708                 if (!list_empty(&hugepage_freelists[nid])) {
709                         page = list_entry(hugepage_freelists[nid].next,
710                                           struct page, lru);
711                         list_del(&page->lru);
712                         update_and_free_page(page);
713                         free_huge_pages--;
714                         free_huge_pages_node[nid]--;
715                         surplus_huge_pages--;
716                         surplus_huge_pages_node[nid]--;
717                         nr_pages--;
718                         remaining_iterations = num_online_nodes();
719                 }
720         }
721 }
722
723 static struct page *alloc_huge_page(struct vm_area_struct *vma,
724                                     unsigned long addr, int avoid_reserve)
725 {
726         struct page *page;
727         struct address_space *mapping = vma->vm_file->f_mapping;
728         struct inode *inode = mapping->host;
729         unsigned int chg = 0;
730
731         /*
732          * Processes that did not create the mapping will have no reserves and
733          * will not have accounted against quota. Check that the quota can be
734          * made before satisfying the allocation
735          */
736         if (!(vma->vm_flags & VM_SHARED) &&
737                         !is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
738                 chg = 1;
739                 if (hugetlb_get_quota(inode->i_mapping, chg))
740                         return ERR_PTR(-ENOSPC);
741         }
742
743         spin_lock(&hugetlb_lock);
744         page = dequeue_huge_page_vma(vma, addr, avoid_reserve);
745         spin_unlock(&hugetlb_lock);
746
747         if (!page) {
748                 page = alloc_buddy_huge_page(vma, addr);
749                 if (!page) {
750                         hugetlb_put_quota(inode->i_mapping, chg);
751                         return ERR_PTR(-VM_FAULT_OOM);
752                 }
753         }
754
755         set_page_refcounted(page);
756         set_page_private(page, (unsigned long) mapping);
757
758         return page;
759 }
760
761 static int __init hugetlb_init(void)
762 {
763         unsigned long i;
764
765         if (HPAGE_SHIFT == 0)
766                 return 0;
767
768         for (i = 0; i < MAX_NUMNODES; ++i)
769                 INIT_LIST_HEAD(&hugepage_freelists[i]);
770
771         hugetlb_next_nid = first_node(node_online_map);
772
773         for (i = 0; i < max_huge_pages; ++i) {
774                 if (!alloc_fresh_huge_page())
775                         break;
776         }
777         max_huge_pages = free_huge_pages = nr_huge_pages = i;
778         printk("Total HugeTLB memory allocated, %ld\n", free_huge_pages);
779         return 0;
780 }
781 module_init(hugetlb_init);
782
783 static int __init hugetlb_setup(char *s)
784 {
785         if (sscanf(s, "%lu", &max_huge_pages) <= 0)
786                 max_huge_pages = 0;
787         return 1;
788 }
789 __setup("hugepages=", hugetlb_setup);
790
791 static unsigned int cpuset_mems_nr(unsigned int *array)
792 {
793         int node;
794         unsigned int nr = 0;
795
796         for_each_node_mask(node, cpuset_current_mems_allowed)
797                 nr += array[node];
798
799         return nr;
800 }
801
802 #ifdef CONFIG_SYSCTL
803 #ifdef CONFIG_HIGHMEM
804 static void try_to_free_low(unsigned long count)
805 {
806         int i;
807
808         for (i = 0; i < MAX_NUMNODES; ++i) {
809                 struct page *page, *next;
810                 list_for_each_entry_safe(page, next, &hugepage_freelists[i], lru) {
811                         if (count >= nr_huge_pages)
812                                 return;
813                         if (PageHighMem(page))
814                                 continue;
815                         list_del(&page->lru);
816                         update_and_free_page(page);
817                         free_huge_pages--;
818                         free_huge_pages_node[page_to_nid(page)]--;
819                 }
820         }
821 }
822 #else
823 static inline void try_to_free_low(unsigned long count)
824 {
825 }
826 #endif
827
828 #define persistent_huge_pages (nr_huge_pages - surplus_huge_pages)
829 static unsigned long set_max_huge_pages(unsigned long count)
830 {
831         unsigned long min_count, ret;
832
833         /*
834          * Increase the pool size
835          * First take pages out of surplus state.  Then make up the
836          * remaining difference by allocating fresh huge pages.
837          *
838          * We might race with alloc_buddy_huge_page() here and be unable
839          * to convert a surplus huge page to a normal huge page. That is
840          * not critical, though, it just means the overall size of the
841          * pool might be one hugepage larger than it needs to be, but
842          * within all the constraints specified by the sysctls.
843          */
844         spin_lock(&hugetlb_lock);
845         while (surplus_huge_pages && count > persistent_huge_pages) {
846                 if (!adjust_pool_surplus(-1))
847                         break;
848         }
849
850         while (count > persistent_huge_pages) {
851                 /*
852                  * If this allocation races such that we no longer need the
853                  * page, free_huge_page will handle it by freeing the page
854                  * and reducing the surplus.
855                  */
856                 spin_unlock(&hugetlb_lock);
857                 ret = alloc_fresh_huge_page();
858                 spin_lock(&hugetlb_lock);
859                 if (!ret)
860                         goto out;
861
862         }
863
864         /*
865          * Decrease the pool size
866          * First return free pages to the buddy allocator (being careful
867          * to keep enough around to satisfy reservations).  Then place
868          * pages into surplus state as needed so the pool will shrink
869          * to the desired size as pages become free.
870          *
871          * By placing pages into the surplus state independent of the
872          * overcommit value, we are allowing the surplus pool size to
873          * exceed overcommit. There are few sane options here. Since
874          * alloc_buddy_huge_page() is checking the global counter,
875          * though, we'll note that we're not allowed to exceed surplus
876          * and won't grow the pool anywhere else. Not until one of the
877          * sysctls are changed, or the surplus pages go out of use.
878          */
879         min_count = resv_huge_pages + nr_huge_pages - free_huge_pages;
880         min_count = max(count, min_count);
881         try_to_free_low(min_count);
882         while (min_count < persistent_huge_pages) {
883                 struct page *page = dequeue_huge_page();
884                 if (!page)
885                         break;
886                 update_and_free_page(page);
887         }
888         while (count < persistent_huge_pages) {
889                 if (!adjust_pool_surplus(1))
890                         break;
891         }
892 out:
893         ret = persistent_huge_pages;
894         spin_unlock(&hugetlb_lock);
895         return ret;
896 }
897
898 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
899                            struct file *file, void __user *buffer,
900                            size_t *length, loff_t *ppos)
901 {
902         proc_doulongvec_minmax(table, write, file, buffer, length, ppos);
903         max_huge_pages = set_max_huge_pages(max_huge_pages);
904         return 0;
905 }
906
907 int hugetlb_treat_movable_handler(struct ctl_table *table, int write,
908                         struct file *file, void __user *buffer,
909                         size_t *length, loff_t *ppos)
910 {
911         proc_dointvec(table, write, file, buffer, length, ppos);
912         if (hugepages_treat_as_movable)
913                 htlb_alloc_mask = GFP_HIGHUSER_MOVABLE;
914         else
915                 htlb_alloc_mask = GFP_HIGHUSER;
916         return 0;
917 }
918
919 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
920                         struct file *file, void __user *buffer,
921                         size_t *length, loff_t *ppos)
922 {
923         proc_doulongvec_minmax(table, write, file, buffer, length, ppos);
924         spin_lock(&hugetlb_lock);
925         nr_overcommit_huge_pages = sysctl_overcommit_huge_pages;
926         spin_unlock(&hugetlb_lock);
927         return 0;
928 }
929
930 #endif /* CONFIG_SYSCTL */
931
932 int hugetlb_report_meminfo(char *buf)
933 {
934         return sprintf(buf,
935                         "HugePages_Total: %5lu\n"
936                         "HugePages_Free:  %5lu\n"
937                         "HugePages_Rsvd:  %5lu\n"
938                         "HugePages_Surp:  %5lu\n"
939                         "Hugepagesize:    %5lu kB\n",
940                         nr_huge_pages,
941                         free_huge_pages,
942                         resv_huge_pages,
943                         surplus_huge_pages,
944                         HPAGE_SIZE/1024);
945 }
946
947 int hugetlb_report_node_meminfo(int nid, char *buf)
948 {
949         return sprintf(buf,
950                 "Node %d HugePages_Total: %5u\n"
951                 "Node %d HugePages_Free:  %5u\n"
952                 "Node %d HugePages_Surp:  %5u\n",
953                 nid, nr_huge_pages_node[nid],
954                 nid, free_huge_pages_node[nid],
955                 nid, surplus_huge_pages_node[nid]);
956 }
957
958 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
959 unsigned long hugetlb_total_pages(void)
960 {
961         return nr_huge_pages * (HPAGE_SIZE / PAGE_SIZE);
962 }
963
964 static int hugetlb_acct_memory(long delta)
965 {
966         int ret = -ENOMEM;
967
968         spin_lock(&hugetlb_lock);
969         /*
970          * When cpuset is configured, it breaks the strict hugetlb page
971          * reservation as the accounting is done on a global variable. Such
972          * reservation is completely rubbish in the presence of cpuset because
973          * the reservation is not checked against page availability for the
974          * current cpuset. Application can still potentially OOM'ed by kernel
975          * with lack of free htlb page in cpuset that the task is in.
976          * Attempt to enforce strict accounting with cpuset is almost
977          * impossible (or too ugly) because cpuset is too fluid that
978          * task or memory node can be dynamically moved between cpusets.
979          *
980          * The change of semantics for shared hugetlb mapping with cpuset is
981          * undesirable. However, in order to preserve some of the semantics,
982          * we fall back to check against current free page availability as
983          * a best attempt and hopefully to minimize the impact of changing
984          * semantics that cpuset has.
985          */
986         if (delta > 0) {
987                 if (gather_surplus_pages(delta) < 0)
988                         goto out;
989
990                 if (delta > cpuset_mems_nr(free_huge_pages_node)) {
991                         return_unused_surplus_pages(delta);
992                         goto out;
993                 }
994         }
995
996         ret = 0;
997         if (delta < 0)
998                 return_unused_surplus_pages((unsigned long) -delta);
999
1000 out:
1001         spin_unlock(&hugetlb_lock);
1002         return ret;
1003 }
1004
1005 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
1006 {
1007         unsigned long reserve = vma_resv_huge_pages(vma);
1008         if (reserve)
1009                 hugetlb_acct_memory(-reserve);
1010 }
1011
1012 /*
1013  * We cannot handle pagefaults against hugetlb pages at all.  They cause
1014  * handle_mm_fault() to try to instantiate regular-sized pages in the
1015  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
1016  * this far.
1017  */
1018 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1019 {
1020         BUG();
1021         return 0;
1022 }
1023
1024 struct vm_operations_struct hugetlb_vm_ops = {
1025         .fault = hugetlb_vm_op_fault,
1026         .close = hugetlb_vm_op_close,
1027 };
1028
1029 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
1030                                 int writable)
1031 {
1032         pte_t entry;
1033
1034         if (writable) {
1035                 entry =
1036                     pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
1037         } else {
1038                 entry = huge_pte_wrprotect(mk_pte(page, vma->vm_page_prot));
1039         }
1040         entry = pte_mkyoung(entry);
1041         entry = pte_mkhuge(entry);
1042
1043         return entry;
1044 }
1045
1046 static void set_huge_ptep_writable(struct vm_area_struct *vma,
1047                                    unsigned long address, pte_t *ptep)
1048 {
1049         pte_t entry;
1050
1051         entry = pte_mkwrite(pte_mkdirty(huge_ptep_get(ptep)));
1052         if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1)) {
1053                 update_mmu_cache(vma, address, entry);
1054         }
1055 }
1056
1057
1058 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
1059                             struct vm_area_struct *vma)
1060 {
1061         pte_t *src_pte, *dst_pte, entry;
1062         struct page *ptepage;
1063         unsigned long addr;
1064         int cow;
1065
1066         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
1067
1068         for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) {
1069                 src_pte = huge_pte_offset(src, addr);
1070                 if (!src_pte)
1071                         continue;
1072                 dst_pte = huge_pte_alloc(dst, addr);
1073                 if (!dst_pte)
1074                         goto nomem;
1075
1076                 /* If the pagetables are shared don't copy or take references */
1077                 if (dst_pte == src_pte)
1078                         continue;
1079
1080                 spin_lock(&dst->page_table_lock);
1081                 spin_lock_nested(&src->page_table_lock, SINGLE_DEPTH_NESTING);
1082                 if (!huge_pte_none(huge_ptep_get(src_pte))) {
1083                         if (cow)
1084                                 huge_ptep_set_wrprotect(src, addr, src_pte);
1085                         entry = huge_ptep_get(src_pte);
1086                         ptepage = pte_page(entry);
1087                         get_page(ptepage);
1088                         set_huge_pte_at(dst, addr, dst_pte, entry);
1089                 }
1090                 spin_unlock(&src->page_table_lock);
1091                 spin_unlock(&dst->page_table_lock);
1092         }
1093         return 0;
1094
1095 nomem:
1096         return -ENOMEM;
1097 }
1098
1099 void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
1100                             unsigned long end, struct page *ref_page)
1101 {
1102         struct mm_struct *mm = vma->vm_mm;
1103         unsigned long address;
1104         pte_t *ptep;
1105         pte_t pte;
1106         struct page *page;
1107         struct page *tmp;
1108         /*
1109          * A page gathering list, protected by per file i_mmap_lock. The
1110          * lock is used to avoid list corruption from multiple unmapping
1111          * of the same page since we are using page->lru.
1112          */
1113         LIST_HEAD(page_list);
1114
1115         WARN_ON(!is_vm_hugetlb_page(vma));
1116         BUG_ON(start & ~HPAGE_MASK);
1117         BUG_ON(end & ~HPAGE_MASK);
1118
1119         spin_lock(&mm->page_table_lock);
1120         for (address = start; address < end; address += HPAGE_SIZE) {
1121                 ptep = huge_pte_offset(mm, address);
1122                 if (!ptep)
1123                         continue;
1124
1125                 if (huge_pmd_unshare(mm, &address, ptep))
1126                         continue;
1127
1128                 /*
1129                  * If a reference page is supplied, it is because a specific
1130                  * page is being unmapped, not a range. Ensure the page we
1131                  * are about to unmap is the actual page of interest.
1132                  */
1133                 if (ref_page) {
1134                         pte = huge_ptep_get(ptep);
1135                         if (huge_pte_none(pte))
1136                                 continue;
1137                         page = pte_page(pte);
1138                         if (page != ref_page)
1139                                 continue;
1140
1141                         /*
1142                          * Mark the VMA as having unmapped its page so that
1143                          * future faults in this VMA will fail rather than
1144                          * looking like data was lost
1145                          */
1146                         set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
1147                 }
1148
1149                 pte = huge_ptep_get_and_clear(mm, address, ptep);
1150                 if (huge_pte_none(pte))
1151                         continue;
1152
1153                 page = pte_page(pte);
1154                 if (pte_dirty(pte))
1155                         set_page_dirty(page);
1156                 list_add(&page->lru, &page_list);
1157         }
1158         spin_unlock(&mm->page_table_lock);
1159         flush_tlb_range(vma, start, end);
1160         list_for_each_entry_safe(page, tmp, &page_list, lru) {
1161                 list_del(&page->lru);
1162                 put_page(page);
1163         }
1164 }
1165
1166 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
1167                           unsigned long end, struct page *ref_page)
1168 {
1169         /*
1170          * It is undesirable to test vma->vm_file as it should be non-null
1171          * for valid hugetlb area. However, vm_file will be NULL in the error
1172          * cleanup path of do_mmap_pgoff. When hugetlbfs ->mmap method fails,
1173          * do_mmap_pgoff() nullifies vma->vm_file before calling this function
1174          * to clean up. Since no pte has actually been setup, it is safe to
1175          * do nothing in this case.
1176          */
1177         if (vma->vm_file) {
1178                 spin_lock(&vma->vm_file->f_mapping->i_mmap_lock);
1179                 __unmap_hugepage_range(vma, start, end, ref_page);
1180                 spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
1181         }
1182 }
1183
1184 /*
1185  * This is called when the original mapper is failing to COW a MAP_PRIVATE
1186  * mappping it owns the reserve page for. The intention is to unmap the page
1187  * from other VMAs and let the children be SIGKILLed if they are faulting the
1188  * same region.
1189  */
1190 int unmap_ref_private(struct mm_struct *mm,
1191                                         struct vm_area_struct *vma,
1192                                         struct page *page,
1193                                         unsigned long address)
1194 {
1195         struct vm_area_struct *iter_vma;
1196         struct address_space *mapping;
1197         struct prio_tree_iter iter;
1198         pgoff_t pgoff;
1199
1200         /*
1201          * vm_pgoff is in PAGE_SIZE units, hence the different calculation
1202          * from page cache lookup which is in HPAGE_SIZE units.
1203          */
1204         address = address & huge_page_mask(hstate_vma(vma));
1205         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT)
1206                 + (vma->vm_pgoff >> PAGE_SHIFT);
1207         mapping = (struct address_space *)page_private(page);
1208
1209         vma_prio_tree_foreach(iter_vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
1210                 /* Do not unmap the current VMA */
1211                 if (iter_vma == vma)
1212                         continue;
1213
1214                 /*
1215                  * Unmap the page from other VMAs without their own reserves.
1216                  * They get marked to be SIGKILLed if they fault in these
1217                  * areas. This is because a future no-page fault on this VMA
1218                  * could insert a zeroed page instead of the data existing
1219                  * from the time of fork. This would look like data corruption
1220                  */
1221                 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
1222                         unmap_hugepage_range(iter_vma,
1223                                 address, address + HPAGE_SIZE,
1224                                 page);
1225         }
1226
1227         return 1;
1228 }
1229
1230 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
1231                         unsigned long address, pte_t *ptep, pte_t pte,
1232                         struct page *pagecache_page)
1233 {
1234         struct page *old_page, *new_page;
1235         int avoidcopy;
1236         int outside_reserve = 0;
1237
1238         old_page = pte_page(pte);
1239
1240 retry_avoidcopy:
1241         /* If no-one else is actually using this page, avoid the copy
1242          * and just make the page writable */
1243         avoidcopy = (page_count(old_page) == 1);
1244         if (avoidcopy) {
1245                 set_huge_ptep_writable(vma, address, ptep);
1246                 return 0;
1247         }
1248
1249         /*
1250          * If the process that created a MAP_PRIVATE mapping is about to
1251          * perform a COW due to a shared page count, attempt to satisfy
1252          * the allocation without using the existing reserves. The pagecache
1253          * page is used to determine if the reserve at this address was
1254          * consumed or not. If reserves were used, a partial faulted mapping
1255          * at the time of fork() could consume its reserves on COW instead
1256          * of the full address range.
1257          */
1258         if (!(vma->vm_flags & VM_SHARED) &&
1259                         is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
1260                         old_page != pagecache_page)
1261                 outside_reserve = 1;
1262
1263         page_cache_get(old_page);
1264         new_page = alloc_huge_page(vma, address, outside_reserve);
1265
1266         if (IS_ERR(new_page)) {
1267                 page_cache_release(old_page);
1268
1269                 /*
1270                  * If a process owning a MAP_PRIVATE mapping fails to COW,
1271                  * it is due to references held by a child and an insufficient
1272                  * huge page pool. To guarantee the original mappers
1273                  * reliability, unmap the page from child processes. The child
1274                  * may get SIGKILLed if it later faults.
1275                  */
1276                 if (outside_reserve) {
1277                         BUG_ON(huge_pte_none(pte));
1278                         if (unmap_ref_private(mm, vma, old_page, address)) {
1279                                 BUG_ON(page_count(old_page) != 1);
1280                                 BUG_ON(huge_pte_none(pte));
1281                                 goto retry_avoidcopy;
1282                         }
1283                         WARN_ON_ONCE(1);
1284                 }
1285
1286                 return -PTR_ERR(new_page);
1287         }
1288
1289         spin_unlock(&mm->page_table_lock);
1290         copy_huge_page(new_page, old_page, address, vma);
1291         __SetPageUptodate(new_page);
1292         spin_lock(&mm->page_table_lock);
1293
1294         ptep = huge_pte_offset(mm, address & HPAGE_MASK);
1295         if (likely(pte_same(huge_ptep_get(ptep), pte))) {
1296                 /* Break COW */
1297                 huge_ptep_clear_flush(vma, address, ptep);
1298                 set_huge_pte_at(mm, address, ptep,
1299                                 make_huge_pte(vma, new_page, 1));
1300                 /* Make the old page be freed below */
1301                 new_page = old_page;
1302         }
1303         page_cache_release(new_page);
1304         page_cache_release(old_page);
1305         return 0;
1306 }
1307
1308 /* Return the pagecache page at a given address within a VMA */
1309 static struct page *hugetlbfs_pagecache_page(struct vm_area_struct *vma,
1310                         unsigned long address)
1311 {
1312         struct address_space *mapping;
1313         pgoff_t idx;
1314
1315         mapping = vma->vm_file->f_mapping;
1316         idx = vma_pagecache_offset(vma, address);
1317
1318         return find_lock_page(mapping, idx);
1319 }
1320
1321 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
1322                         unsigned long address, pte_t *ptep, int write_access)
1323 {
1324         int ret = VM_FAULT_SIGBUS;
1325         pgoff_t idx;
1326         unsigned long size;
1327         struct page *page;
1328         struct address_space *mapping;
1329         pte_t new_pte;
1330
1331         /*
1332          * Currently, we are forced to kill the process in the event the
1333          * original mapper has unmapped pages from the child due to a failed
1334          * COW. Warn that such a situation has occured as it may not be obvious
1335          */
1336         if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
1337                 printk(KERN_WARNING
1338                         "PID %d killed due to inadequate hugepage pool\n",
1339                         current->pid);
1340                 return ret;
1341         }
1342
1343         mapping = vma->vm_file->f_mapping;
1344         idx = vma_pagecache_offset(vma, address);
1345
1346         /*
1347          * Use page lock to guard against racing truncation
1348          * before we get page_table_lock.
1349          */
1350 retry:
1351         page = find_lock_page(mapping, idx);
1352         if (!page) {
1353                 size = i_size_read(mapping->host) >> HPAGE_SHIFT;
1354                 if (idx >= size)
1355                         goto out;
1356                 page = alloc_huge_page(vma, address, 0);
1357                 if (IS_ERR(page)) {
1358                         ret = -PTR_ERR(page);
1359                         goto out;
1360                 }
1361                 clear_huge_page(page, address);
1362                 __SetPageUptodate(page);
1363
1364                 if (vma->vm_flags & VM_SHARED) {
1365                         int err;
1366                         struct inode *inode = mapping->host;
1367
1368                         err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
1369                         if (err) {
1370                                 put_page(page);
1371                                 if (err == -EEXIST)
1372                                         goto retry;
1373                                 goto out;
1374                         }
1375
1376                         spin_lock(&inode->i_lock);
1377                         inode->i_blocks += BLOCKS_PER_HUGEPAGE;
1378                         spin_unlock(&inode->i_lock);
1379                 } else
1380                         lock_page(page);
1381         }
1382
1383         spin_lock(&mm->page_table_lock);
1384         size = i_size_read(mapping->host) >> HPAGE_SHIFT;
1385         if (idx >= size)
1386                 goto backout;
1387
1388         ret = 0;
1389         if (!huge_pte_none(huge_ptep_get(ptep)))
1390                 goto backout;
1391
1392         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
1393                                 && (vma->vm_flags & VM_SHARED)));
1394         set_huge_pte_at(mm, address, ptep, new_pte);
1395
1396         if (write_access && !(vma->vm_flags & VM_SHARED)) {
1397                 /* Optimization, do the COW without a second fault */
1398                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page);
1399         }
1400
1401         spin_unlock(&mm->page_table_lock);
1402         unlock_page(page);
1403 out:
1404         return ret;
1405
1406 backout:
1407         spin_unlock(&mm->page_table_lock);
1408         unlock_page(page);
1409         put_page(page);
1410         goto out;
1411 }
1412
1413 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
1414                         unsigned long address, int write_access)
1415 {
1416         pte_t *ptep;
1417         pte_t entry;
1418         int ret;
1419         static DEFINE_MUTEX(hugetlb_instantiation_mutex);
1420
1421         ptep = huge_pte_alloc(mm, address);
1422         if (!ptep)
1423                 return VM_FAULT_OOM;
1424
1425         /*
1426          * Serialize hugepage allocation and instantiation, so that we don't
1427          * get spurious allocation failures if two CPUs race to instantiate
1428          * the same page in the page cache.
1429          */
1430         mutex_lock(&hugetlb_instantiation_mutex);
1431         entry = huge_ptep_get(ptep);
1432         if (huge_pte_none(entry)) {
1433                 ret = hugetlb_no_page(mm, vma, address, ptep, write_access);
1434                 mutex_unlock(&hugetlb_instantiation_mutex);
1435                 return ret;
1436         }
1437
1438         ret = 0;
1439
1440         spin_lock(&mm->page_table_lock);
1441         /* Check for a racing update before calling hugetlb_cow */
1442         if (likely(pte_same(entry, huge_ptep_get(ptep))))
1443                 if (write_access && !pte_write(entry)) {
1444                         struct page *page;
1445                         page = hugetlbfs_pagecache_page(vma, address);
1446                         ret = hugetlb_cow(mm, vma, address, ptep, entry, page);
1447                         if (page) {
1448                                 unlock_page(page);
1449                                 put_page(page);
1450                         }
1451                 }
1452         spin_unlock(&mm->page_table_lock);
1453         mutex_unlock(&hugetlb_instantiation_mutex);
1454
1455         return ret;
1456 }
1457
1458 int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
1459                         struct page **pages, struct vm_area_struct **vmas,
1460                         unsigned long *position, int *length, int i,
1461                         int write)
1462 {
1463         unsigned long pfn_offset;
1464         unsigned long vaddr = *position;
1465         int remainder = *length;
1466
1467         spin_lock(&mm->page_table_lock);
1468         while (vaddr < vma->vm_end && remainder) {
1469                 pte_t *pte;
1470                 struct page *page;
1471
1472                 /*
1473                  * Some archs (sparc64, sh*) have multiple pte_ts to
1474                  * each hugepage.  We have to make * sure we get the
1475                  * first, for the page indexing below to work.
1476                  */
1477                 pte = huge_pte_offset(mm, vaddr & HPAGE_MASK);
1478
1479                 if (!pte || huge_pte_none(huge_ptep_get(pte)) ||
1480                     (write && !pte_write(huge_ptep_get(pte)))) {
1481                         int ret;
1482
1483                         spin_unlock(&mm->page_table_lock);
1484                         ret = hugetlb_fault(mm, vma, vaddr, write);
1485                         spin_lock(&mm->page_table_lock);
1486                         if (!(ret & VM_FAULT_ERROR))
1487                                 continue;
1488
1489                         remainder = 0;
1490                         if (!i)
1491                                 i = -EFAULT;
1492                         break;
1493                 }
1494
1495                 pfn_offset = (vaddr & ~HPAGE_MASK) >> PAGE_SHIFT;
1496                 page = pte_page(huge_ptep_get(pte));
1497 same_page:
1498                 if (pages) {
1499                         get_page(page);
1500                         pages[i] = page + pfn_offset;
1501                 }
1502
1503                 if (vmas)
1504                         vmas[i] = vma;
1505
1506                 vaddr += PAGE_SIZE;
1507                 ++pfn_offset;
1508                 --remainder;
1509                 ++i;
1510                 if (vaddr < vma->vm_end && remainder &&
1511                                 pfn_offset < HPAGE_SIZE/PAGE_SIZE) {
1512                         /*
1513                          * We use pfn_offset to avoid touching the pageframes
1514                          * of this compound page.
1515                          */
1516                         goto same_page;
1517                 }
1518         }
1519         spin_unlock(&mm->page_table_lock);
1520         *length = remainder;
1521         *position = vaddr;
1522
1523         return i;
1524 }
1525
1526 void hugetlb_change_protection(struct vm_area_struct *vma,
1527                 unsigned long address, unsigned long end, pgprot_t newprot)
1528 {
1529         struct mm_struct *mm = vma->vm_mm;
1530         unsigned long start = address;
1531         pte_t *ptep;
1532         pte_t pte;
1533
1534         BUG_ON(address >= end);
1535         flush_cache_range(vma, address, end);
1536
1537         spin_lock(&vma->vm_file->f_mapping->i_mmap_lock);
1538         spin_lock(&mm->page_table_lock);
1539         for (; address < end; address += HPAGE_SIZE) {
1540                 ptep = huge_pte_offset(mm, address);
1541                 if (!ptep)
1542                         continue;
1543                 if (huge_pmd_unshare(mm, &address, ptep))
1544                         continue;
1545                 if (!huge_pte_none(huge_ptep_get(ptep))) {
1546                         pte = huge_ptep_get_and_clear(mm, address, ptep);
1547                         pte = pte_mkhuge(pte_modify(pte, newprot));
1548                         set_huge_pte_at(mm, address, ptep, pte);
1549                 }
1550         }
1551         spin_unlock(&mm->page_table_lock);
1552         spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
1553
1554         flush_tlb_range(vma, start, end);
1555 }
1556
1557 int hugetlb_reserve_pages(struct inode *inode,
1558                                         long from, long to,
1559                                         struct vm_area_struct *vma)
1560 {
1561         long ret, chg;
1562
1563         /*
1564          * Shared mappings base their reservation on the number of pages that
1565          * are already allocated on behalf of the file. Private mappings need
1566          * to reserve the full area even if read-only as mprotect() may be
1567          * called to make the mapping read-write. Assume !vma is a shm mapping
1568          */
1569         if (!vma || vma->vm_flags & VM_SHARED)
1570                 chg = region_chg(&inode->i_mapping->private_list, from, to);
1571         else {
1572                 chg = to - from;
1573                 set_vma_resv_huge_pages(vma, chg);
1574                 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
1575         }
1576
1577         if (chg < 0)
1578                 return chg;
1579
1580         if (hugetlb_get_quota(inode->i_mapping, chg))
1581                 return -ENOSPC;
1582         ret = hugetlb_acct_memory(chg);
1583         if (ret < 0) {
1584                 hugetlb_put_quota(inode->i_mapping, chg);
1585                 return ret;
1586         }
1587         if (!vma || vma->vm_flags & VM_SHARED)
1588                 region_add(&inode->i_mapping->private_list, from, to);
1589         return 0;
1590 }
1591
1592 void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
1593 {
1594         long chg = region_truncate(&inode->i_mapping->private_list, offset);
1595
1596         spin_lock(&inode->i_lock);
1597         inode->i_blocks -= BLOCKS_PER_HUGEPAGE * freed;
1598         spin_unlock(&inode->i_lock);
1599
1600         hugetlb_put_quota(inode->i_mapping, (chg - freed));
1601         hugetlb_acct_memory(-(chg - freed));
1602 }