[PATCH] hugepages: fold find_or_alloc_pages into huge_no_page()
[linux-2.6.git] / mm / hugetlb.c
1 /*
2  * Generic hugetlb support.
3  * (C) William Irwin, April 2004
4  */
5 #include <linux/gfp.h>
6 #include <linux/list.h>
7 #include <linux/init.h>
8 #include <linux/module.h>
9 #include <linux/mm.h>
10 #include <linux/sysctl.h>
11 #include <linux/highmem.h>
12 #include <linux/nodemask.h>
13 #include <linux/pagemap.h>
14 #include <linux/mempolicy.h>
15
16 #include <asm/page.h>
17 #include <asm/pgtable.h>
18
19 #include <linux/hugetlb.h>
20
21 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
22 static unsigned long nr_huge_pages, free_huge_pages;
23 unsigned long max_huge_pages;
24 static struct list_head hugepage_freelists[MAX_NUMNODES];
25 static unsigned int nr_huge_pages_node[MAX_NUMNODES];
26 static unsigned int free_huge_pages_node[MAX_NUMNODES];
27
28 /*
29  * Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
30  */
31 static DEFINE_SPINLOCK(hugetlb_lock);
32
33 static void enqueue_huge_page(struct page *page)
34 {
35         int nid = page_to_nid(page);
36         list_add(&page->lru, &hugepage_freelists[nid]);
37         free_huge_pages++;
38         free_huge_pages_node[nid]++;
39 }
40
41 static struct page *dequeue_huge_page(struct vm_area_struct *vma,
42                                 unsigned long address)
43 {
44         int nid = numa_node_id();
45         struct page *page = NULL;
46         struct zonelist *zonelist = huge_zonelist(vma, address);
47         struct zone **z;
48
49         for (z = zonelist->zones; *z; z++) {
50                 nid = (*z)->zone_pgdat->node_id;
51                 if (!list_empty(&hugepage_freelists[nid]))
52                         break;
53         }
54
55         if (*z) {
56                 page = list_entry(hugepage_freelists[nid].next,
57                                   struct page, lru);
58                 list_del(&page->lru);
59                 free_huge_pages--;
60                 free_huge_pages_node[nid]--;
61         }
62         return page;
63 }
64
65 static struct page *alloc_fresh_huge_page(void)
66 {
67         static int nid = 0;
68         struct page *page;
69         page = alloc_pages_node(nid, GFP_HIGHUSER|__GFP_COMP|__GFP_NOWARN,
70                                         HUGETLB_PAGE_ORDER);
71         nid = (nid + 1) % num_online_nodes();
72         if (page) {
73                 spin_lock(&hugetlb_lock);
74                 nr_huge_pages++;
75                 nr_huge_pages_node[page_to_nid(page)]++;
76                 spin_unlock(&hugetlb_lock);
77         }
78         return page;
79 }
80
81 void free_huge_page(struct page *page)
82 {
83         BUG_ON(page_count(page));
84
85         INIT_LIST_HEAD(&page->lru);
86         page[1].mapping = NULL;
87
88         spin_lock(&hugetlb_lock);
89         enqueue_huge_page(page);
90         spin_unlock(&hugetlb_lock);
91 }
92
93 struct page *alloc_huge_page(struct vm_area_struct *vma, unsigned long addr)
94 {
95         struct page *page;
96         int i;
97
98         spin_lock(&hugetlb_lock);
99         page = dequeue_huge_page(vma, addr);
100         if (!page) {
101                 spin_unlock(&hugetlb_lock);
102                 return NULL;
103         }
104         spin_unlock(&hugetlb_lock);
105         set_page_count(page, 1);
106         page[1].mapping = (void *)free_huge_page;
107         for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); ++i)
108                 clear_highpage(&page[i]);
109         return page;
110 }
111
112 static int __init hugetlb_init(void)
113 {
114         unsigned long i;
115         struct page *page;
116
117         if (HPAGE_SHIFT == 0)
118                 return 0;
119
120         for (i = 0; i < MAX_NUMNODES; ++i)
121                 INIT_LIST_HEAD(&hugepage_freelists[i]);
122
123         for (i = 0; i < max_huge_pages; ++i) {
124                 page = alloc_fresh_huge_page();
125                 if (!page)
126                         break;
127                 spin_lock(&hugetlb_lock);
128                 enqueue_huge_page(page);
129                 spin_unlock(&hugetlb_lock);
130         }
131         max_huge_pages = free_huge_pages = nr_huge_pages = i;
132         printk("Total HugeTLB memory allocated, %ld\n", free_huge_pages);
133         return 0;
134 }
135 module_init(hugetlb_init);
136
137 static int __init hugetlb_setup(char *s)
138 {
139         if (sscanf(s, "%lu", &max_huge_pages) <= 0)
140                 max_huge_pages = 0;
141         return 1;
142 }
143 __setup("hugepages=", hugetlb_setup);
144
145 #ifdef CONFIG_SYSCTL
146 static void update_and_free_page(struct page *page)
147 {
148         int i;
149         nr_huge_pages--;
150         nr_huge_pages_node[page_zone(page)->zone_pgdat->node_id]--;
151         for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) {
152                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
153                                 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
154                                 1 << PG_private | 1<< PG_writeback);
155                 set_page_count(&page[i], 0);
156         }
157         set_page_count(page, 1);
158         __free_pages(page, HUGETLB_PAGE_ORDER);
159 }
160
161 #ifdef CONFIG_HIGHMEM
162 static void try_to_free_low(unsigned long count)
163 {
164         int i, nid;
165         for (i = 0; i < MAX_NUMNODES; ++i) {
166                 struct page *page, *next;
167                 list_for_each_entry_safe(page, next, &hugepage_freelists[i], lru) {
168                         if (PageHighMem(page))
169                                 continue;
170                         list_del(&page->lru);
171                         update_and_free_page(page);
172                         nid = page_zone(page)->zone_pgdat->node_id;
173                         free_huge_pages--;
174                         free_huge_pages_node[nid]--;
175                         if (count >= nr_huge_pages)
176                                 return;
177                 }
178         }
179 }
180 #else
181 static inline void try_to_free_low(unsigned long count)
182 {
183 }
184 #endif
185
186 static unsigned long set_max_huge_pages(unsigned long count)
187 {
188         while (count > nr_huge_pages) {
189                 struct page *page = alloc_fresh_huge_page();
190                 if (!page)
191                         return nr_huge_pages;
192                 spin_lock(&hugetlb_lock);
193                 enqueue_huge_page(page);
194                 spin_unlock(&hugetlb_lock);
195         }
196         if (count >= nr_huge_pages)
197                 return nr_huge_pages;
198
199         spin_lock(&hugetlb_lock);
200         try_to_free_low(count);
201         while (count < nr_huge_pages) {
202                 struct page *page = dequeue_huge_page(NULL, 0);
203                 if (!page)
204                         break;
205                 update_and_free_page(page);
206         }
207         spin_unlock(&hugetlb_lock);
208         return nr_huge_pages;
209 }
210
211 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
212                            struct file *file, void __user *buffer,
213                            size_t *length, loff_t *ppos)
214 {
215         proc_doulongvec_minmax(table, write, file, buffer, length, ppos);
216         max_huge_pages = set_max_huge_pages(max_huge_pages);
217         return 0;
218 }
219 #endif /* CONFIG_SYSCTL */
220
221 int hugetlb_report_meminfo(char *buf)
222 {
223         return sprintf(buf,
224                         "HugePages_Total: %5lu\n"
225                         "HugePages_Free:  %5lu\n"
226                         "Hugepagesize:    %5lu kB\n",
227                         nr_huge_pages,
228                         free_huge_pages,
229                         HPAGE_SIZE/1024);
230 }
231
232 int hugetlb_report_node_meminfo(int nid, char *buf)
233 {
234         return sprintf(buf,
235                 "Node %d HugePages_Total: %5u\n"
236                 "Node %d HugePages_Free:  %5u\n",
237                 nid, nr_huge_pages_node[nid],
238                 nid, free_huge_pages_node[nid]);
239 }
240
241 int is_hugepage_mem_enough(size_t size)
242 {
243         return (size + ~HPAGE_MASK)/HPAGE_SIZE <= free_huge_pages;
244 }
245
246 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
247 unsigned long hugetlb_total_pages(void)
248 {
249         return nr_huge_pages * (HPAGE_SIZE / PAGE_SIZE);
250 }
251
252 /*
253  * We cannot handle pagefaults against hugetlb pages at all.  They cause
254  * handle_mm_fault() to try to instantiate regular-sized pages in the
255  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
256  * this far.
257  */
258 static struct page *hugetlb_nopage(struct vm_area_struct *vma,
259                                 unsigned long address, int *unused)
260 {
261         BUG();
262         return NULL;
263 }
264
265 struct vm_operations_struct hugetlb_vm_ops = {
266         .nopage = hugetlb_nopage,
267 };
268
269 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
270                                 int writable)
271 {
272         pte_t entry;
273
274         if (writable) {
275                 entry =
276                     pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
277         } else {
278                 entry = pte_wrprotect(mk_pte(page, vma->vm_page_prot));
279         }
280         entry = pte_mkyoung(entry);
281         entry = pte_mkhuge(entry);
282
283         return entry;
284 }
285
286 static void set_huge_ptep_writable(struct vm_area_struct *vma,
287                                    unsigned long address, pte_t *ptep)
288 {
289         pte_t entry;
290
291         entry = pte_mkwrite(pte_mkdirty(*ptep));
292         ptep_set_access_flags(vma, address, ptep, entry, 1);
293         update_mmu_cache(vma, address, entry);
294         lazy_mmu_prot_update(entry);
295 }
296
297
298 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
299                             struct vm_area_struct *vma)
300 {
301         pte_t *src_pte, *dst_pte, entry;
302         struct page *ptepage;
303         unsigned long addr;
304         int cow;
305
306         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
307
308         for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) {
309                 src_pte = huge_pte_offset(src, addr);
310                 if (!src_pte)
311                         continue;
312                 dst_pte = huge_pte_alloc(dst, addr);
313                 if (!dst_pte)
314                         goto nomem;
315                 spin_lock(&dst->page_table_lock);
316                 spin_lock(&src->page_table_lock);
317                 if (!pte_none(*src_pte)) {
318                         if (cow)
319                                 ptep_set_wrprotect(src, addr, src_pte);
320                         entry = *src_pte;
321                         ptepage = pte_page(entry);
322                         get_page(ptepage);
323                         add_mm_counter(dst, file_rss, HPAGE_SIZE / PAGE_SIZE);
324                         set_huge_pte_at(dst, addr, dst_pte, entry);
325                 }
326                 spin_unlock(&src->page_table_lock);
327                 spin_unlock(&dst->page_table_lock);
328         }
329         return 0;
330
331 nomem:
332         return -ENOMEM;
333 }
334
335 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
336                           unsigned long end)
337 {
338         struct mm_struct *mm = vma->vm_mm;
339         unsigned long address;
340         pte_t *ptep;
341         pte_t pte;
342         struct page *page;
343
344         WARN_ON(!is_vm_hugetlb_page(vma));
345         BUG_ON(start & ~HPAGE_MASK);
346         BUG_ON(end & ~HPAGE_MASK);
347
348         spin_lock(&mm->page_table_lock);
349
350         /* Update high watermark before we lower rss */
351         update_hiwater_rss(mm);
352
353         for (address = start; address < end; address += HPAGE_SIZE) {
354                 ptep = huge_pte_offset(mm, address);
355                 if (!ptep)
356                         continue;
357
358                 pte = huge_ptep_get_and_clear(mm, address, ptep);
359                 if (pte_none(pte))
360                         continue;
361
362                 page = pte_page(pte);
363                 put_page(page);
364                 add_mm_counter(mm, file_rss, (int) -(HPAGE_SIZE / PAGE_SIZE));
365         }
366
367         spin_unlock(&mm->page_table_lock);
368         flush_tlb_range(vma, start, end);
369 }
370
371 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
372                         unsigned long address, pte_t *ptep, pte_t pte)
373 {
374         struct page *old_page, *new_page;
375         int i, avoidcopy;
376
377         old_page = pte_page(pte);
378
379         /* If no-one else is actually using this page, avoid the copy
380          * and just make the page writable */
381         avoidcopy = (page_count(old_page) == 1);
382         if (avoidcopy) {
383                 set_huge_ptep_writable(vma, address, ptep);
384                 return VM_FAULT_MINOR;
385         }
386
387         page_cache_get(old_page);
388         new_page = alloc_huge_page(vma, address);
389
390         if (!new_page) {
391                 page_cache_release(old_page);
392
393                 /* Logically this is OOM, not a SIGBUS, but an OOM
394                  * could cause the kernel to go killing other
395                  * processes which won't help the hugepage situation
396                  * at all (?) */
397                 return VM_FAULT_SIGBUS;
398         }
399
400         spin_unlock(&mm->page_table_lock);
401         for (i = 0; i < HPAGE_SIZE/PAGE_SIZE; i++)
402                 copy_user_highpage(new_page + i, old_page + i,
403                                    address + i*PAGE_SIZE);
404         spin_lock(&mm->page_table_lock);
405
406         ptep = huge_pte_offset(mm, address & HPAGE_MASK);
407         if (likely(pte_same(*ptep, pte))) {
408                 /* Break COW */
409                 set_huge_pte_at(mm, address, ptep,
410                                 make_huge_pte(vma, new_page, 1));
411                 /* Make the old page be freed below */
412                 new_page = old_page;
413         }
414         page_cache_release(new_page);
415         page_cache_release(old_page);
416         return VM_FAULT_MINOR;
417 }
418
419 int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
420                         unsigned long address, pte_t *ptep, int write_access)
421 {
422         int ret = VM_FAULT_SIGBUS;
423         unsigned long idx;
424         unsigned long size;
425         struct page *page;
426         struct address_space *mapping;
427         pte_t new_pte;
428
429         mapping = vma->vm_file->f_mapping;
430         idx = ((address - vma->vm_start) >> HPAGE_SHIFT)
431                 + (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));
432
433         /*
434          * Use page lock to guard against racing truncation
435          * before we get page_table_lock.
436          */
437 retry:
438         page = find_lock_page(mapping, idx);
439         if (!page) {
440                 if (hugetlb_get_quota(mapping))
441                         goto out;
442                 page = alloc_huge_page(vma, address);
443                 if (!page) {
444                         hugetlb_put_quota(mapping);
445                         goto out;
446                 }
447
448                 if (vma->vm_flags & VM_SHARED) {
449                         int err;
450
451                         err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
452                         if (err) {
453                                 put_page(page);
454                                 hugetlb_put_quota(mapping);
455                                 if (err == -EEXIST)
456                                         goto retry;
457                                 goto out;
458                         }
459                 } else
460                         lock_page(page);
461         }
462
463         spin_lock(&mm->page_table_lock);
464         size = i_size_read(mapping->host) >> HPAGE_SHIFT;
465         if (idx >= size)
466                 goto backout;
467
468         ret = VM_FAULT_MINOR;
469         if (!pte_none(*ptep))
470                 goto backout;
471
472         add_mm_counter(mm, file_rss, HPAGE_SIZE / PAGE_SIZE);
473         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
474                                 && (vma->vm_flags & VM_SHARED)));
475         set_huge_pte_at(mm, address, ptep, new_pte);
476
477         if (write_access && !(vma->vm_flags & VM_SHARED)) {
478                 /* Optimization, do the COW without a second fault */
479                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte);
480         }
481
482         spin_unlock(&mm->page_table_lock);
483         unlock_page(page);
484 out:
485         return ret;
486
487 backout:
488         spin_unlock(&mm->page_table_lock);
489         hugetlb_put_quota(mapping);
490         unlock_page(page);
491         put_page(page);
492         goto out;
493 }
494
495 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
496                         unsigned long address, int write_access)
497 {
498         pte_t *ptep;
499         pte_t entry;
500         int ret;
501
502         ptep = huge_pte_alloc(mm, address);
503         if (!ptep)
504                 return VM_FAULT_OOM;
505
506         entry = *ptep;
507         if (pte_none(entry))
508                 return hugetlb_no_page(mm, vma, address, ptep, write_access);
509
510         ret = VM_FAULT_MINOR;
511
512         spin_lock(&mm->page_table_lock);
513         /* Check for a racing update before calling hugetlb_cow */
514         if (likely(pte_same(entry, *ptep)))
515                 if (write_access && !pte_write(entry))
516                         ret = hugetlb_cow(mm, vma, address, ptep, entry);
517         spin_unlock(&mm->page_table_lock);
518
519         return ret;
520 }
521
522 int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
523                         struct page **pages, struct vm_area_struct **vmas,
524                         unsigned long *position, int *length, int i)
525 {
526         unsigned long vpfn, vaddr = *position;
527         int remainder = *length;
528
529         vpfn = vaddr/PAGE_SIZE;
530         spin_lock(&mm->page_table_lock);
531         while (vaddr < vma->vm_end && remainder) {
532                 pte_t *pte;
533                 struct page *page;
534
535                 /*
536                  * Some archs (sparc64, sh*) have multiple pte_ts to
537                  * each hugepage.  We have to make * sure we get the
538                  * first, for the page indexing below to work.
539                  */
540                 pte = huge_pte_offset(mm, vaddr & HPAGE_MASK);
541
542                 if (!pte || pte_none(*pte)) {
543                         int ret;
544
545                         spin_unlock(&mm->page_table_lock);
546                         ret = hugetlb_fault(mm, vma, vaddr, 0);
547                         spin_lock(&mm->page_table_lock);
548                         if (ret == VM_FAULT_MINOR)
549                                 continue;
550
551                         remainder = 0;
552                         if (!i)
553                                 i = -EFAULT;
554                         break;
555                 }
556
557                 if (pages) {
558                         page = &pte_page(*pte)[vpfn % (HPAGE_SIZE/PAGE_SIZE)];
559                         get_page(page);
560                         pages[i] = page;
561                 }
562
563                 if (vmas)
564                         vmas[i] = vma;
565
566                 vaddr += PAGE_SIZE;
567                 ++vpfn;
568                 --remainder;
569                 ++i;
570         }
571         spin_unlock(&mm->page_table_lock);
572         *length = remainder;
573         *position = vaddr;
574
575         return i;
576 }