[PATCH] hugepage: serialize hugepage allocation and instantiation
[linux-2.6.git] / mm / hugetlb.c
1 /*
2  * Generic hugetlb support.
3  * (C) William Irwin, April 2004
4  */
5 #include <linux/gfp.h>
6 #include <linux/list.h>
7 #include <linux/init.h>
8 #include <linux/module.h>
9 #include <linux/mm.h>
10 #include <linux/sysctl.h>
11 #include <linux/highmem.h>
12 #include <linux/nodemask.h>
13 #include <linux/pagemap.h>
14 #include <linux/mempolicy.h>
15 #include <linux/cpuset.h>
16 #include <linux/mutex.h>
17
18 #include <asm/page.h>
19 #include <asm/pgtable.h>
20
21 #include <linux/hugetlb.h>
22 #include "internal.h"
23
24 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
25 static unsigned long nr_huge_pages, free_huge_pages;
26 unsigned long max_huge_pages;
27 static struct list_head hugepage_freelists[MAX_NUMNODES];
28 static unsigned int nr_huge_pages_node[MAX_NUMNODES];
29 static unsigned int free_huge_pages_node[MAX_NUMNODES];
30 /*
31  * Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
32  */
33 static DEFINE_SPINLOCK(hugetlb_lock);
34
35 static void clear_huge_page(struct page *page, unsigned long addr)
36 {
37         int i;
38
39         might_sleep();
40         for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); i++) {
41                 cond_resched();
42                 clear_user_highpage(page + i, addr);
43         }
44 }
45
46 static void copy_huge_page(struct page *dst, struct page *src,
47                            unsigned long addr)
48 {
49         int i;
50
51         might_sleep();
52         for (i = 0; i < HPAGE_SIZE/PAGE_SIZE; i++) {
53                 cond_resched();
54                 copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE);
55         }
56 }
57
58 static void enqueue_huge_page(struct page *page)
59 {
60         int nid = page_to_nid(page);
61         list_add(&page->lru, &hugepage_freelists[nid]);
62         free_huge_pages++;
63         free_huge_pages_node[nid]++;
64 }
65
66 static struct page *dequeue_huge_page(struct vm_area_struct *vma,
67                                 unsigned long address)
68 {
69         int nid = numa_node_id();
70         struct page *page = NULL;
71         struct zonelist *zonelist = huge_zonelist(vma, address);
72         struct zone **z;
73
74         for (z = zonelist->zones; *z; z++) {
75                 nid = (*z)->zone_pgdat->node_id;
76                 if (cpuset_zone_allowed(*z, GFP_HIGHUSER) &&
77                     !list_empty(&hugepage_freelists[nid]))
78                         break;
79         }
80
81         if (*z) {
82                 page = list_entry(hugepage_freelists[nid].next,
83                                   struct page, lru);
84                 list_del(&page->lru);
85                 free_huge_pages--;
86                 free_huge_pages_node[nid]--;
87         }
88         return page;
89 }
90
91 static int alloc_fresh_huge_page(void)
92 {
93         static int nid = 0;
94         struct page *page;
95         page = alloc_pages_node(nid, GFP_HIGHUSER|__GFP_COMP|__GFP_NOWARN,
96                                         HUGETLB_PAGE_ORDER);
97         nid = (nid + 1) % num_online_nodes();
98         if (page) {
99                 page[1].lru.next = (void *)free_huge_page;      /* dtor */
100                 spin_lock(&hugetlb_lock);
101                 nr_huge_pages++;
102                 nr_huge_pages_node[page_to_nid(page)]++;
103                 spin_unlock(&hugetlb_lock);
104                 put_page(page); /* free it into the hugepage allocator */
105                 return 1;
106         }
107         return 0;
108 }
109
110 void free_huge_page(struct page *page)
111 {
112         BUG_ON(page_count(page));
113
114         INIT_LIST_HEAD(&page->lru);
115
116         spin_lock(&hugetlb_lock);
117         enqueue_huge_page(page);
118         spin_unlock(&hugetlb_lock);
119 }
120
121 struct page *alloc_huge_page(struct vm_area_struct *vma, unsigned long addr)
122 {
123         struct page *page;
124
125         spin_lock(&hugetlb_lock);
126         page = dequeue_huge_page(vma, addr);
127         if (!page) {
128                 spin_unlock(&hugetlb_lock);
129                 return NULL;
130         }
131         spin_unlock(&hugetlb_lock);
132         set_page_refcounted(page);
133         return page;
134 }
135
136 static int __init hugetlb_init(void)
137 {
138         unsigned long i;
139
140         if (HPAGE_SHIFT == 0)
141                 return 0;
142
143         for (i = 0; i < MAX_NUMNODES; ++i)
144                 INIT_LIST_HEAD(&hugepage_freelists[i]);
145
146         for (i = 0; i < max_huge_pages; ++i) {
147                 if (!alloc_fresh_huge_page())
148                         break;
149         }
150         max_huge_pages = free_huge_pages = nr_huge_pages = i;
151         printk("Total HugeTLB memory allocated, %ld\n", free_huge_pages);
152         return 0;
153 }
154 module_init(hugetlb_init);
155
156 static int __init hugetlb_setup(char *s)
157 {
158         if (sscanf(s, "%lu", &max_huge_pages) <= 0)
159                 max_huge_pages = 0;
160         return 1;
161 }
162 __setup("hugepages=", hugetlb_setup);
163
164 #ifdef CONFIG_SYSCTL
165 static void update_and_free_page(struct page *page)
166 {
167         int i;
168         nr_huge_pages--;
169         nr_huge_pages_node[page_zone(page)->zone_pgdat->node_id]--;
170         for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) {
171                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
172                                 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
173                                 1 << PG_private | 1<< PG_writeback);
174         }
175         page[1].lru.next = NULL;
176         set_page_refcounted(page);
177         __free_pages(page, HUGETLB_PAGE_ORDER);
178 }
179
180 #ifdef CONFIG_HIGHMEM
181 static void try_to_free_low(unsigned long count)
182 {
183         int i, nid;
184         for (i = 0; i < MAX_NUMNODES; ++i) {
185                 struct page *page, *next;
186                 list_for_each_entry_safe(page, next, &hugepage_freelists[i], lru) {
187                         if (PageHighMem(page))
188                                 continue;
189                         list_del(&page->lru);
190                         update_and_free_page(page);
191                         nid = page_zone(page)->zone_pgdat->node_id;
192                         free_huge_pages--;
193                         free_huge_pages_node[nid]--;
194                         if (count >= nr_huge_pages)
195                                 return;
196                 }
197         }
198 }
199 #else
200 static inline void try_to_free_low(unsigned long count)
201 {
202 }
203 #endif
204
205 static unsigned long set_max_huge_pages(unsigned long count)
206 {
207         while (count > nr_huge_pages) {
208                 if (!alloc_fresh_huge_page())
209                         return nr_huge_pages;
210         }
211         if (count >= nr_huge_pages)
212                 return nr_huge_pages;
213
214         spin_lock(&hugetlb_lock);
215         try_to_free_low(count);
216         while (count < nr_huge_pages) {
217                 struct page *page = dequeue_huge_page(NULL, 0);
218                 if (!page)
219                         break;
220                 update_and_free_page(page);
221         }
222         spin_unlock(&hugetlb_lock);
223         return nr_huge_pages;
224 }
225
226 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
227                            struct file *file, void __user *buffer,
228                            size_t *length, loff_t *ppos)
229 {
230         proc_doulongvec_minmax(table, write, file, buffer, length, ppos);
231         max_huge_pages = set_max_huge_pages(max_huge_pages);
232         return 0;
233 }
234 #endif /* CONFIG_SYSCTL */
235
236 int hugetlb_report_meminfo(char *buf)
237 {
238         return sprintf(buf,
239                         "HugePages_Total: %5lu\n"
240                         "HugePages_Free:  %5lu\n"
241                         "Hugepagesize:    %5lu kB\n",
242                         nr_huge_pages,
243                         free_huge_pages,
244                         HPAGE_SIZE/1024);
245 }
246
247 int hugetlb_report_node_meminfo(int nid, char *buf)
248 {
249         return sprintf(buf,
250                 "Node %d HugePages_Total: %5u\n"
251                 "Node %d HugePages_Free:  %5u\n",
252                 nid, nr_huge_pages_node[nid],
253                 nid, free_huge_pages_node[nid]);
254 }
255
256 int is_hugepage_mem_enough(size_t size)
257 {
258         return (size + ~HPAGE_MASK)/HPAGE_SIZE <= free_huge_pages;
259 }
260
261 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
262 unsigned long hugetlb_total_pages(void)
263 {
264         return nr_huge_pages * (HPAGE_SIZE / PAGE_SIZE);
265 }
266
267 /*
268  * We cannot handle pagefaults against hugetlb pages at all.  They cause
269  * handle_mm_fault() to try to instantiate regular-sized pages in the
270  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
271  * this far.
272  */
273 static struct page *hugetlb_nopage(struct vm_area_struct *vma,
274                                 unsigned long address, int *unused)
275 {
276         BUG();
277         return NULL;
278 }
279
280 struct vm_operations_struct hugetlb_vm_ops = {
281         .nopage = hugetlb_nopage,
282 };
283
284 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
285                                 int writable)
286 {
287         pte_t entry;
288
289         if (writable) {
290                 entry =
291                     pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
292         } else {
293                 entry = pte_wrprotect(mk_pte(page, vma->vm_page_prot));
294         }
295         entry = pte_mkyoung(entry);
296         entry = pte_mkhuge(entry);
297
298         return entry;
299 }
300
301 static void set_huge_ptep_writable(struct vm_area_struct *vma,
302                                    unsigned long address, pte_t *ptep)
303 {
304         pte_t entry;
305
306         entry = pte_mkwrite(pte_mkdirty(*ptep));
307         ptep_set_access_flags(vma, address, ptep, entry, 1);
308         update_mmu_cache(vma, address, entry);
309         lazy_mmu_prot_update(entry);
310 }
311
312
313 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
314                             struct vm_area_struct *vma)
315 {
316         pte_t *src_pte, *dst_pte, entry;
317         struct page *ptepage;
318         unsigned long addr;
319         int cow;
320
321         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
322
323         for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) {
324                 src_pte = huge_pte_offset(src, addr);
325                 if (!src_pte)
326                         continue;
327                 dst_pte = huge_pte_alloc(dst, addr);
328                 if (!dst_pte)
329                         goto nomem;
330                 spin_lock(&dst->page_table_lock);
331                 spin_lock(&src->page_table_lock);
332                 if (!pte_none(*src_pte)) {
333                         if (cow)
334                                 ptep_set_wrprotect(src, addr, src_pte);
335                         entry = *src_pte;
336                         ptepage = pte_page(entry);
337                         get_page(ptepage);
338                         add_mm_counter(dst, file_rss, HPAGE_SIZE / PAGE_SIZE);
339                         set_huge_pte_at(dst, addr, dst_pte, entry);
340                 }
341                 spin_unlock(&src->page_table_lock);
342                 spin_unlock(&dst->page_table_lock);
343         }
344         return 0;
345
346 nomem:
347         return -ENOMEM;
348 }
349
350 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
351                           unsigned long end)
352 {
353         struct mm_struct *mm = vma->vm_mm;
354         unsigned long address;
355         pte_t *ptep;
356         pte_t pte;
357         struct page *page;
358
359         WARN_ON(!is_vm_hugetlb_page(vma));
360         BUG_ON(start & ~HPAGE_MASK);
361         BUG_ON(end & ~HPAGE_MASK);
362
363         spin_lock(&mm->page_table_lock);
364
365         /* Update high watermark before we lower rss */
366         update_hiwater_rss(mm);
367
368         for (address = start; address < end; address += HPAGE_SIZE) {
369                 ptep = huge_pte_offset(mm, address);
370                 if (!ptep)
371                         continue;
372
373                 pte = huge_ptep_get_and_clear(mm, address, ptep);
374                 if (pte_none(pte))
375                         continue;
376
377                 page = pte_page(pte);
378                 put_page(page);
379                 add_mm_counter(mm, file_rss, (int) -(HPAGE_SIZE / PAGE_SIZE));
380         }
381
382         spin_unlock(&mm->page_table_lock);
383         flush_tlb_range(vma, start, end);
384 }
385
386 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
387                         unsigned long address, pte_t *ptep, pte_t pte)
388 {
389         struct page *old_page, *new_page;
390         int avoidcopy;
391
392         old_page = pte_page(pte);
393
394         /* If no-one else is actually using this page, avoid the copy
395          * and just make the page writable */
396         avoidcopy = (page_count(old_page) == 1);
397         if (avoidcopy) {
398                 set_huge_ptep_writable(vma, address, ptep);
399                 return VM_FAULT_MINOR;
400         }
401
402         page_cache_get(old_page);
403         new_page = alloc_huge_page(vma, address);
404
405         if (!new_page) {
406                 page_cache_release(old_page);
407                 return VM_FAULT_OOM;
408         }
409
410         spin_unlock(&mm->page_table_lock);
411         copy_huge_page(new_page, old_page, address);
412         spin_lock(&mm->page_table_lock);
413
414         ptep = huge_pte_offset(mm, address & HPAGE_MASK);
415         if (likely(pte_same(*ptep, pte))) {
416                 /* Break COW */
417                 set_huge_pte_at(mm, address, ptep,
418                                 make_huge_pte(vma, new_page, 1));
419                 /* Make the old page be freed below */
420                 new_page = old_page;
421         }
422         page_cache_release(new_page);
423         page_cache_release(old_page);
424         return VM_FAULT_MINOR;
425 }
426
427 int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
428                         unsigned long address, pte_t *ptep, int write_access)
429 {
430         int ret = VM_FAULT_SIGBUS;
431         unsigned long idx;
432         unsigned long size;
433         struct page *page;
434         struct address_space *mapping;
435         pte_t new_pte;
436
437         mapping = vma->vm_file->f_mapping;
438         idx = ((address - vma->vm_start) >> HPAGE_SHIFT)
439                 + (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));
440
441         /*
442          * Use page lock to guard against racing truncation
443          * before we get page_table_lock.
444          */
445 retry:
446         page = find_lock_page(mapping, idx);
447         if (!page) {
448                 if (hugetlb_get_quota(mapping))
449                         goto out;
450                 page = alloc_huge_page(vma, address);
451                 if (!page) {
452                         hugetlb_put_quota(mapping);
453                         ret = VM_FAULT_OOM;
454                         goto out;
455                 }
456                 clear_huge_page(page, address);
457
458                 if (vma->vm_flags & VM_SHARED) {
459                         int err;
460
461                         err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
462                         if (err) {
463                                 put_page(page);
464                                 hugetlb_put_quota(mapping);
465                                 if (err == -EEXIST)
466                                         goto retry;
467                                 goto out;
468                         }
469                 } else
470                         lock_page(page);
471         }
472
473         spin_lock(&mm->page_table_lock);
474         size = i_size_read(mapping->host) >> HPAGE_SHIFT;
475         if (idx >= size)
476                 goto backout;
477
478         ret = VM_FAULT_MINOR;
479         if (!pte_none(*ptep))
480                 goto backout;
481
482         add_mm_counter(mm, file_rss, HPAGE_SIZE / PAGE_SIZE);
483         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
484                                 && (vma->vm_flags & VM_SHARED)));
485         set_huge_pte_at(mm, address, ptep, new_pte);
486
487         if (write_access && !(vma->vm_flags & VM_SHARED)) {
488                 /* Optimization, do the COW without a second fault */
489                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte);
490         }
491
492         spin_unlock(&mm->page_table_lock);
493         unlock_page(page);
494 out:
495         return ret;
496
497 backout:
498         spin_unlock(&mm->page_table_lock);
499         hugetlb_put_quota(mapping);
500         unlock_page(page);
501         put_page(page);
502         goto out;
503 }
504
505 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
506                         unsigned long address, int write_access)
507 {
508         pte_t *ptep;
509         pte_t entry;
510         int ret;
511         static DEFINE_MUTEX(hugetlb_instantiation_mutex);
512
513         ptep = huge_pte_alloc(mm, address);
514         if (!ptep)
515                 return VM_FAULT_OOM;
516
517         /*
518          * Serialize hugepage allocation and instantiation, so that we don't
519          * get spurious allocation failures if two CPUs race to instantiate
520          * the same page in the page cache.
521          */
522         mutex_lock(&hugetlb_instantiation_mutex);
523         entry = *ptep;
524         if (pte_none(entry)) {
525                 ret = hugetlb_no_page(mm, vma, address, ptep, write_access);
526                 mutex_unlock(&hugetlb_instantiation_mutex);
527                 return ret;
528         }
529
530         ret = VM_FAULT_MINOR;
531
532         spin_lock(&mm->page_table_lock);
533         /* Check for a racing update before calling hugetlb_cow */
534         if (likely(pte_same(entry, *ptep)))
535                 if (write_access && !pte_write(entry))
536                         ret = hugetlb_cow(mm, vma, address, ptep, entry);
537         spin_unlock(&mm->page_table_lock);
538         mutex_unlock(&hugetlb_instantiation_mutex);
539
540         return ret;
541 }
542
543 int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
544                         struct page **pages, struct vm_area_struct **vmas,
545                         unsigned long *position, int *length, int i)
546 {
547         unsigned long vpfn, vaddr = *position;
548         int remainder = *length;
549
550         vpfn = vaddr/PAGE_SIZE;
551         spin_lock(&mm->page_table_lock);
552         while (vaddr < vma->vm_end && remainder) {
553                 pte_t *pte;
554                 struct page *page;
555
556                 /*
557                  * Some archs (sparc64, sh*) have multiple pte_ts to
558                  * each hugepage.  We have to make * sure we get the
559                  * first, for the page indexing below to work.
560                  */
561                 pte = huge_pte_offset(mm, vaddr & HPAGE_MASK);
562
563                 if (!pte || pte_none(*pte)) {
564                         int ret;
565
566                         spin_unlock(&mm->page_table_lock);
567                         ret = hugetlb_fault(mm, vma, vaddr, 0);
568                         spin_lock(&mm->page_table_lock);
569                         if (ret == VM_FAULT_MINOR)
570                                 continue;
571
572                         remainder = 0;
573                         if (!i)
574                                 i = -EFAULT;
575                         break;
576                 }
577
578                 if (pages) {
579                         page = &pte_page(*pte)[vpfn % (HPAGE_SIZE/PAGE_SIZE)];
580                         get_page(page);
581                         pages[i] = page;
582                 }
583
584                 if (vmas)
585                         vmas[i] = vma;
586
587                 vaddr += PAGE_SIZE;
588                 ++vpfn;
589                 --remainder;
590                 ++i;
591         }
592         spin_unlock(&mm->page_table_lock);
593         *length = remainder;
594         *position = vaddr;
595
596         return i;
597 }
598
599 void hugetlb_change_protection(struct vm_area_struct *vma,
600                 unsigned long address, unsigned long end, pgprot_t newprot)
601 {
602         struct mm_struct *mm = vma->vm_mm;
603         unsigned long start = address;
604         pte_t *ptep;
605         pte_t pte;
606
607         BUG_ON(address >= end);
608         flush_cache_range(vma, address, end);
609
610         spin_lock(&mm->page_table_lock);
611         for (; address < end; address += HPAGE_SIZE) {
612                 ptep = huge_pte_offset(mm, address);
613                 if (!ptep)
614                         continue;
615                 if (!pte_none(*ptep)) {
616                         pte = huge_ptep_get_and_clear(mm, address, ptep);
617                         pte = pte_mkhuge(pte_modify(pte, newprot));
618                         set_huge_pte_at(mm, address, ptep, pte);
619                         lazy_mmu_prot_update(pte);
620                 }
621         }
622         spin_unlock(&mm->page_table_lock);
623
624         flush_tlb_range(vma, start, end);
625 }
626