mm: fix mbind vma merge problem
[linux-2.6.git] / mm / mempolicy.c
index e08e2c4..44dd9d1 100644 (file)
 #include <linux/seq_file.h>
 #include <linux/proc_fs.h>
 #include <linux/migrate.h>
+#include <linux/ksm.h>
 #include <linux/rmap.h>
 #include <linux/security.h>
 #include <linux/syscalls.h>
 #include <linux/ctype.h>
+#include <linux/mm_inline.h>
 
 #include <asm/tlbflush.h>
 #include <asm/uaccess.h>
@@ -191,25 +193,27 @@ static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)
  * Must be called holding task's alloc_lock to protect task's mems_allowed
  * and mempolicy.  May also be called holding the mmap_semaphore for write.
  */
-static int mpol_set_nodemask(struct mempolicy *pol, const nodemask_t *nodes)
+static int mpol_set_nodemask(struct mempolicy *pol,
+                    const nodemask_t *nodes, struct nodemask_scratch *nsc)
 {
-       nodemask_t cpuset_context_nmask;
        int ret;
 
        /* if mode is MPOL_DEFAULT, pol is NULL. This is right. */
        if (pol == NULL)
                return 0;
+       /* Check N_HIGH_MEMORY */
+       nodes_and(nsc->mask1,
+                 cpuset_current_mems_allowed, node_states[N_HIGH_MEMORY]);
 
        VM_BUG_ON(!nodes);
        if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes))
                nodes = NULL;   /* explicit local allocation */
        else {
                if (pol->flags & MPOL_F_RELATIVE_NODES)
-                       mpol_relative_nodemask(&cpuset_context_nmask, nodes,
-                                              &cpuset_current_mems_allowed);
+                       mpol_relative_nodemask(&nsc->mask2, nodes,&nsc->mask1);
                else
-                       nodes_and(cpuset_context_nmask, *nodes,
-                                 cpuset_current_mems_allowed);
+                       nodes_and(nsc->mask2, *nodes, nsc->mask1);
+
                if (mpol_store_user_nodemask(pol))
                        pol->w.user_nodemask = *nodes;
                else
@@ -217,8 +221,10 @@ static int mpol_set_nodemask(struct mempolicy *pol, const nodemask_t *nodes)
                                                cpuset_current_mems_allowed;
        }
 
-       ret = mpol_ops[pol->mode].create(pol,
-                               nodes ? &cpuset_context_nmask : NULL);
+       if (nodes)
+               ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);
+       else
+               ret = mpol_ops[pol->mode].create(pol, NULL);
        return ret;
 }
 
@@ -408,17 +414,11 @@ static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
                if (!page)
                        continue;
                /*
-                * The check for PageReserved here is important to avoid
-                * handling zero pages and other pages that may have been
-                * marked special by the system.
-                *
-                * If the PageReserved would not be checked here then f.e.
-                * the location of the zero page could have an influence
-                * on MPOL_MF_STRICT, zero pages would be counted for
-                * the per node stats, and there would be useless attempts
-                * to put zero pages on the migration list.
+                * vm_normal_page() filters out zero pages, but there might
+                * still be PageReserved pages to skip, perhaps in a VDSO.
+                * And we cannot move PageKsm pages sensibly or safely yet.
                 */
-               if (PageReserved(page))
+               if (PageReserved(page) || PageKsm(page))
                        continue;
                nid = page_to_nid(page);
                if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
@@ -563,24 +563,50 @@ static int policy_vma(struct vm_area_struct *vma, struct mempolicy *new)
 }
 
 /* Step 2: apply policy to a range and do splits. */
-static int mbind_range(struct vm_area_struct *vma, unsigned long start,
-                      unsigned long end, struct mempolicy *new)
+static int mbind_range(struct mm_struct *mm, unsigned long start,
+                      unsigned long end, struct mempolicy *new_pol)
 {
        struct vm_area_struct *next;
-       int err;
+       struct vm_area_struct *prev;
+       struct vm_area_struct *vma;
+       int err = 0;
+       pgoff_t pgoff;
+       unsigned long vmstart;
+       unsigned long vmend;
 
-       err = 0;
-       for (; vma && vma->vm_start < end; vma = next) {
+       vma = find_vma_prev(mm, start, &prev);
+       if (!vma || vma->vm_start > start)
+               return -EFAULT;
+
+       for (; vma && vma->vm_start < end; prev = vma, vma = next) {
                next = vma->vm_next;
-               if (vma->vm_start < start)
-                       err = split_vma(vma->vm_mm, vma, start, 1);
-               if (!err && vma->vm_end > end)
-                       err = split_vma(vma->vm_mm, vma, end, 0);
-               if (!err)
-                       err = policy_vma(vma, new);
+               vmstart = max(start, vma->vm_start);
+               vmend   = min(end, vma->vm_end);
+
+               pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
+               prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,
+                                 vma->anon_vma, vma->vm_file, pgoff, new_pol);
+               if (prev) {
+                       vma = prev;
+                       next = vma->vm_next;
+                       continue;
+               }
+               if (vma->vm_start != vmstart) {
+                       err = split_vma(vma->vm_mm, vma, vmstart, 1);
+                       if (err)
+                               goto out;
+               }
+               if (vma->vm_end != vmend) {
+                       err = split_vma(vma->vm_mm, vma, vmend, 0);
+                       if (err)
+                               goto out;
+               }
+               err = policy_vma(vma, new_pol);
                if (err)
-                       break;
+                       goto out;
        }
+
+ out:
        return err;
 }
 
@@ -620,12 +646,17 @@ static long do_set_mempolicy(unsigned short mode, unsigned short flags,
 {
        struct mempolicy *new, *old;
        struct mm_struct *mm = current->mm;
+       NODEMASK_SCRATCH(scratch);
        int ret;
 
-       new = mpol_new(mode, flags, nodes);
-       if (IS_ERR(new))
-               return PTR_ERR(new);
+       if (!scratch)
+               return -ENOMEM;
 
+       new = mpol_new(mode, flags, nodes);
+       if (IS_ERR(new)) {
+               ret = PTR_ERR(new);
+               goto out;
+       }
        /*
         * prevent changing our mempolicy while show_numa_maps()
         * is using it.
@@ -635,13 +666,13 @@ static long do_set_mempolicy(unsigned short mode, unsigned short flags,
        if (mm)
                down_write(&mm->mmap_sem);
        task_lock(current);
-       ret = mpol_set_nodemask(new, nodes);
+       ret = mpol_set_nodemask(new, nodes, scratch);
        if (ret) {
                task_unlock(current);
                if (mm)
                        up_write(&mm->mmap_sem);
                mpol_put(new);
-               return ret;
+               goto out;
        }
        old = current->mempolicy;
        current->mempolicy = new;
@@ -654,7 +685,10 @@ static long do_set_mempolicy(unsigned short mode, unsigned short flags,
                up_write(&mm->mmap_sem);
 
        mpol_put(old);
-       return 0;
+       ret = 0;
+out:
+       NODEMASK_SCRATCH_FREE(scratch);
+       return ret;
 }
 
 /*
@@ -797,6 +831,8 @@ static void migrate_page_add(struct page *page, struct list_head *pagelist,
        if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) {
                if (!isolate_lru_page(page)) {
                        list_add_tail(&page->lru, pagelist);
+                       inc_zone_page_state(page, NR_ISOLATED_ANON +
+                                           page_is_file_cache(page));
                }
        }
 }
@@ -824,7 +860,7 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest,
                        flags | MPOL_MF_DISCONTIG_OK, &pagelist);
 
        if (!list_empty(&pagelist))
-               err = migrate_pages(&pagelist, new_node_page, dest);
+               err = migrate_pages(&pagelist, new_node_page, dest, 0);
 
        return err;
 }
@@ -1012,17 +1048,24 @@ static long do_mbind(unsigned long start, unsigned long len,
 
                err = migrate_prep();
                if (err)
-                       return err;
+                       goto mpol_out;
        }
-       down_write(&mm->mmap_sem);
-       task_lock(current);
-       err = mpol_set_nodemask(new, nmask);
-       task_unlock(current);
-       if (err) {
-               up_write(&mm->mmap_sem);
-               mpol_put(new);
-               return err;
+       {
+               NODEMASK_SCRATCH(scratch);
+               if (scratch) {
+                       down_write(&mm->mmap_sem);
+                       task_lock(current);
+                       err = mpol_set_nodemask(new, nmask, scratch);
+                       task_unlock(current);
+                       if (err)
+                               up_write(&mm->mmap_sem);
+               } else
+                       err = -ENOMEM;
+               NODEMASK_SCRATCH_FREE(scratch);
        }
+       if (err)
+               goto mpol_out;
+
        vma = check_range(mm, start, end, nmask,
                          flags | MPOL_MF_INVERT, &pagelist);
 
@@ -1030,17 +1073,19 @@ static long do_mbind(unsigned long start, unsigned long len,
        if (!IS_ERR(vma)) {
                int nr_failed = 0;
 
-               err = mbind_range(vma, start, end, new);
+               err = mbind_range(mm, start, end, new);
 
                if (!list_empty(&pagelist))
                        nr_failed = migrate_pages(&pagelist, new_vma_page,
-                                               (unsigned long)vma);
+                                               (unsigned long)vma, 0);
 
                if (!err && nr_failed && (flags & MPOL_MF_STRICT))
                        err = -EIO;
-       }
+       } else
+               putback_lru_pages(&pagelist);
 
        up_write(&mm->mmap_sem);
+ mpol_out:
        mpol_put(new);
        return err;
 }
@@ -1544,6 +1589,53 @@ struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
        }
        return zl;
 }
+
+/*
+ * init_nodemask_of_mempolicy
+ *
+ * If the current task's mempolicy is "default" [NULL], return 'false'
+ * to indicate default policy.  Otherwise, extract the policy nodemask
+ * for 'bind' or 'interleave' policy into the argument nodemask, or
+ * initialize the argument nodemask to contain the single node for
+ * 'preferred' or 'local' policy and return 'true' to indicate presence
+ * of non-default mempolicy.
+ *
+ * We don't bother with reference counting the mempolicy [mpol_get/put]
+ * because the current task is examining it's own mempolicy and a task's
+ * mempolicy is only ever changed by the task itself.
+ *
+ * N.B., it is the caller's responsibility to free a returned nodemask.
+ */
+bool init_nodemask_of_mempolicy(nodemask_t *mask)
+{
+       struct mempolicy *mempolicy;
+       int nid;
+
+       if (!(mask && current->mempolicy))
+               return false;
+
+       mempolicy = current->mempolicy;
+       switch (mempolicy->mode) {
+       case MPOL_PREFERRED:
+               if (mempolicy->flags & MPOL_F_LOCAL)
+                       nid = numa_node_id();
+               else
+                       nid = mempolicy->v.preferred_node;
+               init_nodemask_of_node(mask, nid);
+               break;
+
+       case MPOL_BIND:
+               /* Fall through */
+       case MPOL_INTERLEAVE:
+               *mask =  mempolicy->v.nodes;
+               break;
+
+       default:
+               BUG();
+       }
+
+       return true;
+}
 #endif
 
 /* Allocate a page in interleaved policy.
@@ -1891,6 +1983,7 @@ restart:
  * Install non-NULL @mpol in inode's shared policy rb-tree.
  * On entry, the current task has a reference on a non-NULL @mpol.
  * This must be released on exit.
+ * This is called at get_inode() calls and we can use GFP_KERNEL.
  */
 void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
 {
@@ -1902,19 +1995,24 @@ void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
        if (mpol) {
                struct vm_area_struct pvma;
                struct mempolicy *new;
+               NODEMASK_SCRATCH(scratch);
 
+               if (!scratch)
+                       return;
                /* contextualize the tmpfs mount point mempolicy */
                new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
                if (IS_ERR(new)) {
                        mpol_put(mpol); /* drop our ref on sb mpol */
+                       NODEMASK_SCRATCH_FREE(scratch);
                        return;         /* no valid nodemask intersection */
                }
 
                task_lock(current);
-               ret = mpol_set_nodemask(new, &mpol->w.user_nodemask);
+               ret = mpol_set_nodemask(new, &mpol->w.user_nodemask, scratch);
                task_unlock(current);
                mpol_put(mpol); /* drop our ref on sb mpol */
                if (ret) {
+                       NODEMASK_SCRATCH_FREE(scratch);
                        mpol_put(new);
                        return;
                }
@@ -1924,6 +2022,7 @@ void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
                pvma.vm_end = TASK_SIZE;        /* policy covers entire file */
                mpol_set_shared_policy(sp, &pvma, new); /* adds ref */
                mpol_put(new);                  /* drop initial ref */
+               NODEMASK_SCRATCH_FREE(scratch);
        }
 }
 
@@ -2140,13 +2239,18 @@ int mpol_parse_str(char *str, struct mempolicy **mpol, int no_context)
                err = 1;
        else {
                int ret;
-
-               task_lock(current);
-               ret = mpol_set_nodemask(new, &nodes);
-               task_unlock(current);
-               if (ret)
+               NODEMASK_SCRATCH(scratch);
+               if (scratch) {
+                       task_lock(current);
+                       ret = mpol_set_nodemask(new, &nodes, scratch);
+                       task_unlock(current);
+               } else
+                       ret = -ENOMEM;
+               NODEMASK_SCRATCH_FREE(scratch);
+               if (ret) {
                        err = 1;
-               else if (no_context) {
+                       mpol_put(new);
+               } else if (no_context) {
                        /* save for contextualization */
                        new->w.user_nodemask = nodes;
                }