mm: remove double indirection on tlb parameter to free_pgd_range() & Co
[linux-2.6.git] / arch / ia64 / mm / hugetlbpage.c
index 626258a..cd49e28 100644 (file)
@@ -8,15 +8,14 @@
  * Feb, 2004: dynamic hugetlb page size via boot parameter
  */
 
-#include <linux/config.h>
 #include <linux/init.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
 #include <linux/hugetlb.h>
 #include <linux/pagemap.h>
-#include <linux/smp_lock.h>
 #include <linux/slab.h>
 #include <linux/sysctl.h>
+#include <linux/log2.h>
 #include <asm/mman.h>
 #include <asm/pgalloc.h>
 #include <asm/tlb.h>
@@ -24,7 +23,7 @@
 
 unsigned int hpage_shift=HPAGE_SHIFT_DEFAULT;
 
-static pte_t *
+pte_t *
 huge_pte_alloc (struct mm_struct *mm, unsigned long addr)
 {
        unsigned long taddr = htlbpage_to_page(addr);
@@ -43,7 +42,7 @@ huge_pte_alloc (struct mm_struct *mm, unsigned long addr)
        return pte;
 }
 
-static pte_t *
+pte_t *
 huge_pte_offset (struct mm_struct *mm, unsigned long addr)
 {
        unsigned long taddr = htlbpage_to_page(addr);
@@ -65,108 +64,35 @@ huge_pte_offset (struct mm_struct *mm, unsigned long addr)
        return pte;
 }
 
-#define mk_pte_huge(entry) { pte_val(entry) |= _PAGE_P; }
-
-static void
-set_huge_pte (struct mm_struct *mm, struct vm_area_struct *vma,
-             struct page *page, pte_t * page_table, int write_access)
+int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
 {
-       pte_t entry;
-
-       add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE);
-       if (write_access) {
-               entry =
-                   pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
-       } else
-               entry = pte_wrprotect(mk_pte(page, vma->vm_page_prot));
-       entry = pte_mkyoung(entry);
-       mk_pte_huge(entry);
-       set_pte(page_table, entry);
-       return;
+       return 0;
 }
+
+#define mk_pte_huge(entry) { pte_val(entry) |= _PAGE_P; }
+
 /*
- * This function checks for proper alignment of input addr and len parameters.
+ * Don't actually need to do any preparation, but need to make sure
+ * the address is in the right region.
  */
-int is_aligned_hugepage_range(unsigned long addr, unsigned long len)
+int prepare_hugepage_range(unsigned long addr, unsigned long len)
 {
        if (len & ~HPAGE_MASK)
                return -EINVAL;
        if (addr & ~HPAGE_MASK)
                return -EINVAL;
-       if (REGION_NUMBER(addr) != REGION_HPAGE)
+       if (REGION_NUMBER(addr) != RGN_HPAGE)
                return -EINVAL;
 
        return 0;
 }
 
-int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
-                       struct vm_area_struct *vma)
-{
-       pte_t *src_pte, *dst_pte, entry;
-       struct page *ptepage;
-       unsigned long addr = vma->vm_start;
-       unsigned long end = vma->vm_end;
-
-       while (addr < end) {
-               dst_pte = huge_pte_alloc(dst, addr);
-               if (!dst_pte)
-                       goto nomem;
-               src_pte = huge_pte_offset(src, addr);
-               entry = *src_pte;
-               ptepage = pte_page(entry);
-               get_page(ptepage);
-               set_pte(dst_pte, entry);
-               add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE);
-               addr += HPAGE_SIZE;
-       }
-       return 0;
-nomem:
-       return -ENOMEM;
-}
-
-int
-follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
-                   struct page **pages, struct vm_area_struct **vmas,
-                   unsigned long *st, int *length, int i)
-{
-       pte_t *ptep, pte;
-       unsigned long start = *st;
-       unsigned long pstart;
-       int len = *length;
-       struct page *page;
-
-       do {
-               pstart = start & HPAGE_MASK;
-               ptep = huge_pte_offset(mm, start);
-               pte = *ptep;
-
-back1:
-               page = pte_page(pte);
-               if (pages) {
-                       page += ((start & ~HPAGE_MASK) >> PAGE_SHIFT);
-                       get_page(page);
-                       pages[i] = page;
-               }
-               if (vmas)
-                       vmas[i] = vma;
-               i++;
-               len--;
-               start += PAGE_SIZE;
-               if (((start & HPAGE_MASK) == pstart) && len &&
-                               (start < vma->vm_end))
-                       goto back1;
-       } while (len && start < vma->vm_end);
-       *length = len;
-       *st = start;
-       return i;
-}
-
 struct page *follow_huge_addr(struct mm_struct *mm, unsigned long addr, int write)
 {
        struct page *page;
        pte_t *ptep;
 
-       if (REGION_NUMBER(addr) != REGION_HPAGE)
+       if (REGION_NUMBER(addr) != RGN_HPAGE)
                return ERR_PTR(-EINVAL);
 
        ptep = huge_pte_offset(mm, addr);
@@ -186,88 +112,29 @@ follow_huge_pmd(struct mm_struct *mm, unsigned long address, pmd_t *pmd, int wri
        return NULL;
 }
 
-/*
- * Do nothing, until we've worked out what to do!  To allow build, we
- * must remove reference to clear_page_range since it no longer exists.
- */
-void hugetlb_free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *prev,
-       unsigned long start, unsigned long end)
+void hugetlb_free_pgd_range(struct mmu_gather *tlb,
+                       unsigned long addr, unsigned long end,
+                       unsigned long floor, unsigned long ceiling)
 {
-}
-
-void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
-{
-       struct mm_struct *mm = vma->vm_mm;
-       unsigned long address;
-       pte_t *pte;
-       struct page *page;
-
-       BUG_ON(start & (HPAGE_SIZE - 1));
-       BUG_ON(end & (HPAGE_SIZE - 1));
-
-       for (address = start; address < end; address += HPAGE_SIZE) {
-               pte = huge_pte_offset(mm, address);
-               if (pte_none(*pte))
-                       continue;
-               page = pte_page(*pte);
-               put_page(page);
-               pte_clear(mm, address, pte);
-       }
-       add_mm_counter(mm, rss, - ((end - start) >> PAGE_SHIFT));
-       flush_tlb_range(vma, start, end);
-}
-
-int hugetlb_prefault(struct address_space *mapping, struct vm_area_struct *vma)
-{
-       struct mm_struct *mm = current->mm;
-       unsigned long addr;
-       int ret = 0;
-
-       BUG_ON(vma->vm_start & ~HPAGE_MASK);
-       BUG_ON(vma->vm_end & ~HPAGE_MASK);
+       /*
+        * This is called to free hugetlb page tables.
+        *
+        * The offset of these addresses from the base of the hugetlb
+        * region must be scaled down by HPAGE_SIZE/PAGE_SIZE so that
+        * the standard free_pgd_range will free the right page tables.
+        *
+        * If floor and ceiling are also in the hugetlb region, they
+        * must likewise be scaled down; but if outside, left unchanged.
+        */
 
-       spin_lock(&mm->page_table_lock);
-       for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) {
-               unsigned long idx;
-               pte_t *pte = huge_pte_alloc(mm, addr);
-               struct page *page;
+       addr = htlbpage_to_page(addr);
+       end  = htlbpage_to_page(end);
+       if (REGION_NUMBER(floor) == RGN_HPAGE)
+               floor = htlbpage_to_page(floor);
+       if (REGION_NUMBER(ceiling) == RGN_HPAGE)
+               ceiling = htlbpage_to_page(ceiling);
 
-               if (!pte) {
-                       ret = -ENOMEM;
-                       goto out;
-               }
-               if (!pte_none(*pte))
-                       continue;
-
-               idx = ((addr - vma->vm_start) >> HPAGE_SHIFT)
-                       + (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));
-               page = find_get_page(mapping, idx);
-               if (!page) {
-                       /* charge the fs quota first */
-                       if (hugetlb_get_quota(mapping)) {
-                               ret = -ENOMEM;
-                               goto out;
-                       }
-                       page = alloc_huge_page();
-                       if (!page) {
-                               hugetlb_put_quota(mapping);
-                               ret = -ENOMEM;
-                               goto out;
-                       }
-                       ret = add_to_page_cache(page, mapping, idx, GFP_ATOMIC);
-                       if (! ret) {
-                               unlock_page(page);
-                       } else {
-                               hugetlb_put_quota(mapping);
-                               page_cache_release(page);
-                               goto out;
-                       }
-               }
-               set_huge_pte(mm, vma, page, pte, vma->vm_flags & VM_WRITE);
-       }
-out:
-       spin_unlock(&mm->page_table_lock);
-       return ret;
+       free_pgd_range(tlb, addr, end, floor, ceiling);
 }
 
 unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
@@ -279,8 +146,16 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, u
                return -ENOMEM;
        if (len & ~HPAGE_MASK)
                return -EINVAL;
-       /* This code assumes that REGION_HPAGE != 0. */
-       if ((REGION_NUMBER(addr) != REGION_HPAGE) || (addr & (HPAGE_SIZE - 1)))
+
+       /* Handle MAP_FIXED */
+       if (flags & MAP_FIXED) {
+               if (prepare_hugepage_range(addr, len))
+                       return -EINVAL;
+               return addr;
+       }
+
+       /* This code assumes that RGN_HPAGE != 0. */
+       if ((REGION_NUMBER(addr) != RGN_HPAGE) || (addr & (HPAGE_SIZE - 1)))
                addr = HPAGE_REGION_BASE;
        else
                addr = ALIGN(addr, HPAGE_SIZE);
@@ -306,7 +181,7 @@ static int __init hugetlb_setup_sz(char *str)
                tr_pages = 0x15557000UL;
 
        size = memparse(str, &str);
-       if (*str || (size & (size-1)) || !(tr_pages & size) ||
+       if (*str || !is_power_of_2(size) || !(tr_pages & size) ||
                size <= PAGE_SIZE ||
                size >= (1UL << PAGE_SHIFT << MAX_ORDER)) {
                printk(KERN_WARNING "Invalid huge page size specified\n");
@@ -319,6 +194,6 @@ static int __init hugetlb_setup_sz(char *str)
         * override here with new page shift.
         */
        ia64_set_rr(HPAGE_REGION_BASE, hpage_shift << 2);
-       return 1;
+       return 0;
 }
-__setup("hugepagesz=", hugetlb_setup_sz);
+early_param("hugepagesz", hugetlb_setup_sz);