[PATCH] cpuset: rework cpuset_zone_allowed api
[linux-2.6.git] / mm / hugetlb.c
index 075877b..089092d 100644 (file)
@@ -22,7 +22,7 @@
 #include "internal.h"
 
 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
-static unsigned long nr_huge_pages, free_huge_pages, reserved_huge_pages;
+static unsigned long nr_huge_pages, free_huge_pages, resv_huge_pages;
 unsigned long max_huge_pages;
 static struct list_head hugepage_freelists[MAX_NUMNODES];
 static unsigned int nr_huge_pages_node[MAX_NUMNODES];
@@ -72,8 +72,8 @@ static struct page *dequeue_huge_page(struct vm_area_struct *vma,
        struct zone **z;
 
        for (z = zonelist->zones; *z; z++) {
-               nid = (*z)->zone_pgdat->node_id;
-               if (cpuset_zone_allowed(*z, GFP_HIGHUSER) &&
+               nid = zone_to_nid(*z);
+               if (cpuset_zone_allowed_softwall(*z, GFP_HIGHUSER) &&
                    !list_empty(&hugepage_freelists[nid]))
                        break;
        }
@@ -105,9 +105,11 @@ static int alloc_fresh_huge_page(void)
        struct page *page;
        page = alloc_pages_node(nid, GFP_HIGHUSER|__GFP_COMP|__GFP_NOWARN,
                                        HUGETLB_PAGE_ORDER);
-       nid = (nid + 1) % num_online_nodes();
+       nid = next_node(nid, node_online_map);
+       if (nid == MAX_NUMNODES)
+               nid = first_node(node_online_map);
        if (page) {
-               page[1].lru.next = (void *)free_huge_page;      /* dtor */
+               set_compound_page_dtor(page, free_huge_page);
                spin_lock(&hugetlb_lock);
                nr_huge_pages++;
                nr_huge_pages_node[page_to_nid(page)]++;
@@ -121,39 +123,13 @@ static int alloc_fresh_huge_page(void)
 static struct page *alloc_huge_page(struct vm_area_struct *vma,
                                    unsigned long addr)
 {
-       struct inode *inode = vma->vm_file->f_dentry->d_inode;
        struct page *page;
-       int use_reserve = 0;
-       unsigned long idx;
 
        spin_lock(&hugetlb_lock);
-
-       if (vma->vm_flags & VM_MAYSHARE) {
-
-               /* idx = radix tree index, i.e. offset into file in
-                * HPAGE_SIZE units */
-               idx = ((addr - vma->vm_start) >> HPAGE_SHIFT)
-                       + (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));
-
-               /* The hugetlbfs specific inode info stores the number
-                * of "guaranteed available" (huge) pages.  That is,
-                * the first 'prereserved_hpages' pages of the inode
-                * are either already instantiated, or have been
-                * pre-reserved (by hugetlb_reserve_for_inode()). Here
-                * we're in the process of instantiating the page, so
-                * we use this to determine whether to draw from the
-                * pre-reserved pool or the truly free pool. */
-               if (idx < HUGETLBFS_I(inode)->prereserved_hpages)
-                       use_reserve = 1;
-       }
-
-       if (!use_reserve) {
-               if (free_huge_pages <= reserved_huge_pages)
-                       goto fail;
-       } else {
-               BUG_ON(reserved_huge_pages == 0);
-               reserved_huge_pages--;
-       }
+       if (vma->vm_flags & VM_MAYSHARE)
+               resv_huge_pages--;
+       else if (free_huge_pages <= resv_huge_pages)
+               goto fail;
 
        page = dequeue_huge_page(vma, addr);
        if (!page)
@@ -163,96 +139,11 @@ static struct page *alloc_huge_page(struct vm_area_struct *vma,
        set_page_refcounted(page);
        return page;
 
- fail:
-       WARN_ON(use_reserve); /* reserved allocations shouldn't fail */
+fail:
        spin_unlock(&hugetlb_lock);
        return NULL;
 }
 
-/* hugetlb_extend_reservation()
- *
- * Ensure that at least 'atleast' hugepages are, and will remain,
- * available to instantiate the first 'atleast' pages of the given
- * inode.  If the inode doesn't already have this many pages reserved
- * or instantiated, set aside some hugepages in the reserved pool to
- * satisfy later faults (or fail now if there aren't enough, rather
- * than getting the SIGBUS later).
- */
-int hugetlb_extend_reservation(struct hugetlbfs_inode_info *info,
-                              unsigned long atleast)
-{
-       struct inode *inode = &info->vfs_inode;
-       unsigned long change_in_reserve = 0;
-       int ret = 0;
-
-       spin_lock(&hugetlb_lock);
-       read_lock_irq(&inode->i_mapping->tree_lock);
-
-       if (info->prereserved_hpages >= atleast)
-               goto out;
-
-       /* Because we always call this on shared mappings, none of the
-        * pages beyond info->prereserved_hpages can have been
-        * instantiated, so we need to reserve all of them now. */
-       change_in_reserve = atleast - info->prereserved_hpages;
-
-       if ((reserved_huge_pages + change_in_reserve) > free_huge_pages) {
-               ret = -ENOMEM;
-               goto out;
-       }
-
-       reserved_huge_pages += change_in_reserve;
-       info->prereserved_hpages = atleast;
-
- out:
-       read_unlock_irq(&inode->i_mapping->tree_lock);
-       spin_unlock(&hugetlb_lock);
-
-       return ret;
-}
-
-/* hugetlb_truncate_reservation()
- *
- * This returns pages reserved for the given inode to the general free
- * hugepage pool.  If the inode has any pages prereserved, but not
- * instantiated, beyond offset (atmost << HPAGE_SIZE), then release
- * them.
- */
-void hugetlb_truncate_reservation(struct hugetlbfs_inode_info *info,
-                                 unsigned long atmost)
-{
-       struct inode *inode = &info->vfs_inode;
-       struct address_space *mapping = inode->i_mapping;
-       unsigned long idx;
-       unsigned long change_in_reserve = 0;
-       struct page *page;
-
-       spin_lock(&hugetlb_lock);
-       read_lock_irq(&inode->i_mapping->tree_lock);
-
-       if (info->prereserved_hpages <= atmost)
-               goto out;
-
-       /* Count pages which were reserved, but not instantiated, and
-        * which we can now release. */
-       for (idx = atmost; idx < info->prereserved_hpages; idx++) {
-               page = radix_tree_lookup(&mapping->page_tree, idx);
-               if (!page)
-                       /* Pages which are already instantiated can't
-                        * be unreserved (and in fact have already
-                        * been removed from the reserved pool) */
-                       change_in_reserve++;
-       }
-
-       BUG_ON(reserved_huge_pages < change_in_reserve);
-       reserved_huge_pages -= change_in_reserve;
-       info->prereserved_hpages = atmost;
-
- out:
-       read_unlock_irq(&inode->i_mapping->tree_lock);
-       spin_unlock(&hugetlb_lock);
-}
-
 static int __init hugetlb_init(void)
 {
        unsigned long i;
@@ -286,7 +177,7 @@ static void update_and_free_page(struct page *page)
 {
        int i;
        nr_huge_pages--;
-       nr_huge_pages_node[page_zone(page)->zone_pgdat->node_id]--;
+       nr_huge_pages_node[page_to_nid(page)]--;
        for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) {
                page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
                                1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
@@ -300,7 +191,8 @@ static void update_and_free_page(struct page *page)
 #ifdef CONFIG_HIGHMEM
 static void try_to_free_low(unsigned long count)
 {
-       int i, nid;
+       int i;
+
        for (i = 0; i < MAX_NUMNODES; ++i) {
                struct page *page, *next;
                list_for_each_entry_safe(page, next, &hugepage_freelists[i], lru) {
@@ -308,9 +200,8 @@ static void try_to_free_low(unsigned long count)
                                continue;
                        list_del(&page->lru);
                        update_and_free_page(page);
-                       nid = page_zone(page)->zone_pgdat->node_id;
                        free_huge_pages--;
-                       free_huge_pages_node[nid]--;
+                       free_huge_pages_node[page_to_nid(page)]--;
                        if (count >= nr_huge_pages)
                                return;
                }
@@ -332,6 +223,7 @@ static unsigned long set_max_huge_pages(unsigned long count)
                return nr_huge_pages;
 
        spin_lock(&hugetlb_lock);
+       count = max(count, resv_huge_pages);
        try_to_free_low(count);
        while (count < nr_huge_pages) {
                struct page *page = dequeue_huge_page(NULL, 0);
@@ -358,11 +250,11 @@ int hugetlb_report_meminfo(char *buf)
        return sprintf(buf,
                        "HugePages_Total: %5lu\n"
                        "HugePages_Free:  %5lu\n"
-                       "HugePages_Rsvd:  %5lu\n"
+                       "HugePages_Rsvd:  %5lu\n"
                        "Hugepagesize:    %5lu kB\n",
                        nr_huge_pages,
                        free_huge_pages,
-                       reserved_huge_pages,
+                       resv_huge_pages,
                        HPAGE_SIZE/1024);
 }
 
@@ -452,7 +344,6 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
                        entry = *src_pte;
                        ptepage = pte_page(entry);
                        get_page(ptepage);
-                       add_mm_counter(dst, file_rss, HPAGE_SIZE / PAGE_SIZE);
                        set_huge_pte_at(dst, addr, dst_pte, entry);
                }
                spin_unlock(&src->page_table_lock);
@@ -464,40 +355,66 @@ nomem:
        return -ENOMEM;
 }
 
-void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
-                         unsigned long end)
+void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
+                           unsigned long end)
 {
        struct mm_struct *mm = vma->vm_mm;
        unsigned long address;
        pte_t *ptep;
        pte_t pte;
        struct page *page;
+       struct page *tmp;
+       /*
+        * A page gathering list, protected by per file i_mmap_lock. The
+        * lock is used to avoid list corruption from multiple unmapping
+        * of the same page since we are using page->lru.
+        */
+       LIST_HEAD(page_list);
 
        WARN_ON(!is_vm_hugetlb_page(vma));
        BUG_ON(start & ~HPAGE_MASK);
        BUG_ON(end & ~HPAGE_MASK);
 
        spin_lock(&mm->page_table_lock);
-
-       /* Update high watermark before we lower rss */
-       update_hiwater_rss(mm);
-
        for (address = start; address < end; address += HPAGE_SIZE) {
                ptep = huge_pte_offset(mm, address);
                if (!ptep)
                        continue;
 
+               if (huge_pmd_unshare(mm, &address, ptep))
+                       continue;
+
                pte = huge_ptep_get_and_clear(mm, address, ptep);
                if (pte_none(pte))
                        continue;
 
                page = pte_page(pte);
-               put_page(page);
-               add_mm_counter(mm, file_rss, (int) -(HPAGE_SIZE / PAGE_SIZE));
+               list_add(&page->lru, &page_list);
        }
-
        spin_unlock(&mm->page_table_lock);
        flush_tlb_range(vma, start, end);
+       list_for_each_entry_safe(page, tmp, &page_list, lru) {
+               list_del(&page->lru);
+               put_page(page);
+       }
+}
+
+void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
+                         unsigned long end)
+{
+       /*
+        * It is undesirable to test vma->vm_file as it should be non-null
+        * for valid hugetlb area. However, vm_file will be NULL in the error
+        * cleanup path of do_mmap_pgoff. When hugetlbfs ->mmap method fails,
+        * do_mmap_pgoff() nullifies vma->vm_file before calling this function
+        * to clean up. Since no pte has actually been setup, it is safe to
+        * do nothing in this case.
+        */
+       if (vma->vm_file) {
+               spin_lock(&vma->vm_file->f_mapping->i_mmap_lock);
+               __unmap_hugepage_range(vma, start, end);
+               spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
+       }
 }
 
 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
@@ -562,6 +479,9 @@ int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
 retry:
        page = find_lock_page(mapping, idx);
        if (!page) {
+               size = i_size_read(mapping->host) >> HPAGE_SHIFT;
+               if (idx >= size)
+                       goto out;
                if (hugetlb_get_quota(mapping))
                        goto out;
                page = alloc_huge_page(vma, address);
@@ -596,7 +516,6 @@ retry:
        if (!pte_none(*ptep))
                goto backout;
 
-       add_mm_counter(mm, file_rss, HPAGE_SIZE / PAGE_SIZE);
        new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
                                && (vma->vm_flags & VM_SHARED)));
        set_huge_pte_at(mm, address, ptep, new_pte);
@@ -661,10 +580,10 @@ int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
                        struct page **pages, struct vm_area_struct **vmas,
                        unsigned long *position, int *length, int i)
 {
-       unsigned long vpfn, vaddr = *position;
+       unsigned long pfn_offset;
+       unsigned long vaddr = *position;
        int remainder = *length;
 
-       vpfn = vaddr/PAGE_SIZE;
        spin_lock(&mm->page_table_lock);
        while (vaddr < vma->vm_end && remainder) {
                pte_t *pte;
@@ -692,19 +611,29 @@ int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
                        break;
                }
 
+               pfn_offset = (vaddr & ~HPAGE_MASK) >> PAGE_SHIFT;
+               page = pte_page(*pte);
+same_page:
                if (pages) {
-                       page = &pte_page(*pte)[vpfn % (HPAGE_SIZE/PAGE_SIZE)];
                        get_page(page);
-                       pages[i] = page;
+                       pages[i] = page + pfn_offset;
                }
 
                if (vmas)
                        vmas[i] = vma;
 
                vaddr += PAGE_SIZE;
-               ++vpfn;
+               ++pfn_offset;
                --remainder;
                ++i;
+               if (vaddr < vma->vm_end && remainder &&
+                               pfn_offset < HPAGE_SIZE/PAGE_SIZE) {
+                       /*
+                        * We use pfn_offset to avoid touching the pageframes
+                        * of this compound page.
+                        */
+                       goto same_page;
+               }
        }
        spin_unlock(&mm->page_table_lock);
        *length = remainder;
@@ -724,11 +653,14 @@ void hugetlb_change_protection(struct vm_area_struct *vma,
        BUG_ON(address >= end);
        flush_cache_range(vma, address, end);
 
+       spin_lock(&vma->vm_file->f_mapping->i_mmap_lock);
        spin_lock(&mm->page_table_lock);
        for (; address < end; address += HPAGE_SIZE) {
                ptep = huge_pte_offset(mm, address);
                if (!ptep)
                        continue;
+               if (huge_pmd_unshare(mm, &address, ptep))
+                       continue;
                if (!pte_none(*ptep)) {
                        pte = huge_ptep_get_and_clear(mm, address, ptep);
                        pte = pte_mkhuge(pte_modify(pte, newprot));
@@ -737,7 +669,161 @@ void hugetlb_change_protection(struct vm_area_struct *vma,
                }
        }
        spin_unlock(&mm->page_table_lock);
+       spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
 
        flush_tlb_range(vma, start, end);
 }
 
+struct file_region {
+       struct list_head link;
+       long from;
+       long to;
+};
+
+static long region_add(struct list_head *head, long f, long t)
+{
+       struct file_region *rg, *nrg, *trg;
+
+       /* Locate the region we are either in or before. */
+       list_for_each_entry(rg, head, link)
+               if (f <= rg->to)
+                       break;
+
+       /* Round our left edge to the current segment if it encloses us. */
+       if (f > rg->from)
+               f = rg->from;
+
+       /* Check for and consume any regions we now overlap with. */
+       nrg = rg;
+       list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
+               if (&rg->link == head)
+                       break;
+               if (rg->from > t)
+                       break;
+
+               /* If this area reaches higher then extend our area to
+                * include it completely.  If this is not the first area
+                * which we intend to reuse, free it. */
+               if (rg->to > t)
+                       t = rg->to;
+               if (rg != nrg) {
+                       list_del(&rg->link);
+                       kfree(rg);
+               }
+       }
+       nrg->from = f;
+       nrg->to = t;
+       return 0;
+}
+
+static long region_chg(struct list_head *head, long f, long t)
+{
+       struct file_region *rg, *nrg;
+       long chg = 0;
+
+       /* Locate the region we are before or in. */
+       list_for_each_entry(rg, head, link)
+               if (f <= rg->to)
+                       break;
+
+       /* If we are below the current region then a new region is required.
+        * Subtle, allocate a new region at the position but make it zero
+        * size such that we can guarentee to record the reservation. */
+       if (&rg->link == head || t < rg->from) {
+               nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
+               if (nrg == 0)
+                       return -ENOMEM;
+               nrg->from = f;
+               nrg->to   = f;
+               INIT_LIST_HEAD(&nrg->link);
+               list_add(&nrg->link, rg->link.prev);
+
+               return t - f;
+       }
+
+       /* Round our left edge to the current segment if it encloses us. */
+       if (f > rg->from)
+               f = rg->from;
+       chg = t - f;
+
+       /* Check for and consume any regions we now overlap with. */
+       list_for_each_entry(rg, rg->link.prev, link) {
+               if (&rg->link == head)
+                       break;
+               if (rg->from > t)
+                       return chg;
+
+               /* We overlap with this area, if it extends futher than
+                * us then we must extend ourselves.  Account for its
+                * existing reservation. */
+               if (rg->to > t) {
+                       chg += rg->to - t;
+                       t = rg->to;
+               }
+               chg -= rg->to - rg->from;
+       }
+       return chg;
+}
+
+static long region_truncate(struct list_head *head, long end)
+{
+       struct file_region *rg, *trg;
+       long chg = 0;
+
+       /* Locate the region we are either in or before. */
+       list_for_each_entry(rg, head, link)
+               if (end <= rg->to)
+                       break;
+       if (&rg->link == head)
+               return 0;
+
+       /* If we are in the middle of a region then adjust it. */
+       if (end > rg->from) {
+               chg = rg->to - end;
+               rg->to = end;
+               rg = list_entry(rg->link.next, typeof(*rg), link);
+       }
+
+       /* Drop any remaining regions. */
+       list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
+               if (&rg->link == head)
+                       break;
+               chg += rg->to - rg->from;
+               list_del(&rg->link);
+               kfree(rg);
+       }
+       return chg;
+}
+
+static int hugetlb_acct_memory(long delta)
+{
+       int ret = -ENOMEM;
+
+       spin_lock(&hugetlb_lock);
+       if ((delta + resv_huge_pages) <= free_huge_pages) {
+               resv_huge_pages += delta;
+               ret = 0;
+       }
+       spin_unlock(&hugetlb_lock);
+       return ret;
+}
+
+int hugetlb_reserve_pages(struct inode *inode, long from, long to)
+{
+       long ret, chg;
+
+       chg = region_chg(&inode->i_mapping->private_list, from, to);
+       if (chg < 0)
+               return chg;
+       ret = hugetlb_acct_memory(chg);
+       if (ret < 0)
+               return ret;
+       region_add(&inode->i_mapping->private_list, from, to);
+       return 0;
+}
+
+void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
+{
+       long chg = region_truncate(&inode->i_mapping->private_list, offset);
+       hugetlb_acct_memory(freed - chg);
+}