mm: improve page_lock_anon_vma() comment
[linux-2.6.git] / mm / migrate.c
index 11c6c56..34132f8 100644 (file)
@@ -21,6 +21,7 @@
 #include <linux/mm_inline.h>
 #include <linux/nsproxy.h>
 #include <linux/pagevec.h>
+#include <linux/ksm.h>
 #include <linux/rmap.h>
 #include <linux/topology.h>
 #include <linux/cpu.h>
 #include <linux/security.h>
 #include <linux/memcontrol.h>
 #include <linux/syscalls.h>
+#include <linux/hugetlb.h>
+#include <linux/gfp.h>
+
+#include <asm/tlbflush.h>
 
 #include "internal.h"
 
@@ -38,7 +43,8 @@
 
 /*
  * migrate_prep() needs to be called before we start compiling a list of pages
- * to be migrated using isolate_lru_page().
+ * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is
+ * undesirable, use migrate_prep_local()
  */
 int migrate_prep(void)
 {
@@ -53,31 +59,36 @@ int migrate_prep(void)
        return 0;
 }
 
+/* Do the necessary work of migrate_prep but not if it involves other CPUs */
+int migrate_prep_local(void)
+{
+       lru_add_drain();
+
+       return 0;
+}
+
 /*
  * Add isolated pages on the list back to the LRU under page lock
  * to avoid leaking evictable pages back onto unevictable list.
- *
- * returns the number of pages put back.
  */
-int putback_lru_pages(struct list_head *l)
+void putback_lru_pages(struct list_head *l)
 {
        struct page *page;
        struct page *page2;
-       int count = 0;
 
        list_for_each_entry_safe(page, page2, l, lru) {
                list_del(&page->lru);
+               dec_zone_page_state(page, NR_ISOLATED_ANON +
+                               page_is_file_cache(page));
                putback_lru_page(page);
-               count++;
        }
-       return count;
 }
 
 /*
  * Restore a potential migration pte to a working pte entry
  */
-static void remove_migration_pte(struct vm_area_struct *vma,
-               struct page *old, struct page *new)
+static int remove_migration_pte(struct page *new, struct vm_area_struct *vma,
+                                unsigned long addr, void *old)
 {
        struct mm_struct *mm = vma->vm_mm;
        swp_entry_t entry;
@@ -86,121 +97,75 @@ static void remove_migration_pte(struct vm_area_struct *vma,
        pmd_t *pmd;
        pte_t *ptep, pte;
        spinlock_t *ptl;
-       unsigned long addr = page_address_in_vma(new, vma);
 
-       if (addr == -EFAULT)
-               return;
+       if (unlikely(PageHuge(new))) {
+               ptep = huge_pte_offset(mm, addr);
+               if (!ptep)
+                       goto out;
+               ptl = &mm->page_table_lock;
+       } else {
+               pgd = pgd_offset(mm, addr);
+               if (!pgd_present(*pgd))
+                       goto out;
 
-       pgd = pgd_offset(mm, addr);
-       if (!pgd_present(*pgd))
-                return;
+               pud = pud_offset(pgd, addr);
+               if (!pud_present(*pud))
+                       goto out;
 
-       pud = pud_offset(pgd, addr);
-       if (!pud_present(*pud))
-                return;
+               pmd = pmd_offset(pud, addr);
+               if (pmd_trans_huge(*pmd))
+                       goto out;
+               if (!pmd_present(*pmd))
+                       goto out;
 
-       pmd = pmd_offset(pud, addr);
-       if (!pmd_present(*pmd))
-               return;
+               ptep = pte_offset_map(pmd, addr);
 
-       ptep = pte_offset_map(pmd, addr);
+               if (!is_swap_pte(*ptep)) {
+                       pte_unmap(ptep);
+                       goto out;
+               }
 
-       if (!is_swap_pte(*ptep)) {
-               pte_unmap(ptep);
-               return;
-       }
+               ptl = pte_lockptr(mm, pmd);
+       }
 
-       ptl = pte_lockptr(mm, pmd);
        spin_lock(ptl);
        pte = *ptep;
        if (!is_swap_pte(pte))
-               goto out;
+               goto unlock;
 
        entry = pte_to_swp_entry(pte);
 
-       if (!is_migration_entry(entry) || migration_entry_to_page(entry) != old)
-               goto out;
-
-       /*
-        * Yes, ignore the return value from a GFP_ATOMIC mem_cgroup_charge.
-        * Failure is not an option here: we're now expected to remove every
-        * migration pte, and will cause crashes otherwise.  Normally this
-        * is not an issue: mem_cgroup_prepare_migration bumped up the old
-        * page_cgroup count for safety, that's now attached to the new page,
-        * so this charge should just be another incrementation of the count,
-        * to keep in balance with rmap.c's mem_cgroup_uncharging.  But if
-        * there's been a force_empty, those reference counts may no longer
-        * be reliable, and this charge can actually fail: oh well, we don't
-        * make the situation any worse by proceeding as if it had succeeded.
-        */
-       mem_cgroup_charge(new, mm, GFP_ATOMIC);
+       if (!is_migration_entry(entry) ||
+           migration_entry_to_page(entry) != old)
+               goto unlock;
 
        get_page(new);
        pte = pte_mkold(mk_pte(new, vma->vm_page_prot));
        if (is_write_migration_entry(entry))
                pte = pte_mkwrite(pte);
+#ifdef CONFIG_HUGETLB_PAGE
+       if (PageHuge(new))
+               pte = pte_mkhuge(pte);
+#endif
        flush_cache_page(vma, addr, pte_pfn(pte));
        set_pte_at(mm, addr, ptep, pte);
 
-       if (PageAnon(new))
+       if (PageHuge(new)) {
+               if (PageAnon(new))
+                       hugepage_add_anon_rmap(new, vma, addr);
+               else
+                       page_dup_rmap(new);
+       } else if (PageAnon(new))
                page_add_anon_rmap(new, vma, addr);
        else
                page_add_file_rmap(new);
 
        /* No need to invalidate - it was non-present before */
-       update_mmu_cache(vma, addr, pte);
-
-out:
+       update_mmu_cache(vma, addr, ptep);
+unlock:
        pte_unmap_unlock(ptep, ptl);
-}
-
-/*
- * Note that remove_file_migration_ptes will only work on regular mappings,
- * Nonlinear mappings do not use migration entries.
- */
-static void remove_file_migration_ptes(struct page *old, struct page *new)
-{
-       struct vm_area_struct *vma;
-       struct address_space *mapping = page_mapping(new);
-       struct prio_tree_iter iter;
-       pgoff_t pgoff = new->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
-
-       if (!mapping)
-               return;
-
-       spin_lock(&mapping->i_mmap_lock);
-
-       vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff)
-               remove_migration_pte(vma, old, new);
-
-       spin_unlock(&mapping->i_mmap_lock);
-}
-
-/*
- * Must hold mmap_sem lock on at least one of the vmas containing
- * the page so that the anon_vma cannot vanish.
- */
-static void remove_anon_migration_ptes(struct page *old, struct page *new)
-{
-       struct anon_vma *anon_vma;
-       struct vm_area_struct *vma;
-       unsigned long mapping;
-
-       mapping = (unsigned long)new->mapping;
-
-       if (!mapping || (mapping & PAGE_MAPPING_ANON) == 0)
-               return;
-
-       /*
-        * We hold the mmap_sem lock. So no need to call page_lock_anon_vma.
-        */
-       anon_vma = (struct anon_vma *) (mapping - PAGE_MAPPING_ANON);
-       spin_lock(&anon_vma->lock);
-
-       list_for_each_entry(vma, &anon_vma->head, anon_vma_node)
-               remove_migration_pte(vma, old, new);
-
-       spin_unlock(&anon_vma->lock);
+out:
+       return SWAP_AGAIN;
 }
 
 /*
@@ -209,10 +174,7 @@ static void remove_anon_migration_ptes(struct page *old, struct page *new)
  */
 static void remove_migration_ptes(struct page *old, struct page *new)
 {
-       if (PageAnon(new))
-               remove_anon_migration_ptes(old, new);
-       else
-               remove_file_migration_ptes(old, new);
+       rmap_walk(new, remove_migration_pte, old);
 }
 
 /*
@@ -264,7 +226,7 @@ out:
  * The number of remaining references must be:
  * 1 for anonymous pages without a mapping
  * 2 for pages with a mapping
- * 3 for pages with a mapping and PagePrivate set.
+ * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
  */
 static int migrate_page_move_mapping(struct address_space *mapping,
                struct page *newpage, struct page *page)
@@ -284,9 +246,9 @@ static int migrate_page_move_mapping(struct address_space *mapping,
        pslot = radix_tree_lookup_slot(&mapping->page_tree,
                                        page_index(page));
 
-       expected_count = 2 + !!PagePrivate(page);
+       expected_count = 2 + page_has_private(page);
        if (page_count(page) != expected_count ||
-                       (struct page *)radix_tree_deref_slot(pslot) != page) {
+               radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) {
                spin_unlock_irq(&mapping->tree_lock);
                return -EAGAIN;
        }
@@ -300,12 +262,10 @@ static int migrate_page_move_mapping(struct address_space *mapping,
         * Now we know that no one else is looking at the page.
         */
        get_page(newpage);      /* add cache reference */
-#ifdef CONFIG_SWAP
        if (PageSwapCache(page)) {
                SetPageSwapCache(newpage);
                set_page_private(newpage, page_private(page));
        }
-#endif
 
        radix_tree_replace_slot(pslot, newpage);
 
@@ -328,20 +288,69 @@ static int migrate_page_move_mapping(struct address_space *mapping,
         */
        __dec_zone_page_state(page, NR_FILE_PAGES);
        __inc_zone_page_state(newpage, NR_FILE_PAGES);
-
+       if (PageSwapBacked(page)) {
+               __dec_zone_page_state(page, NR_SHMEM);
+               __inc_zone_page_state(newpage, NR_SHMEM);
+       }
        spin_unlock_irq(&mapping->tree_lock);
-       if (!PageSwapCache(newpage))
-               mem_cgroup_uncharge_cache_page(page);
 
        return 0;
 }
 
 /*
+ * The expected number of remaining references is the same as that
+ * of migrate_page_move_mapping().
+ */
+int migrate_huge_page_move_mapping(struct address_space *mapping,
+                                  struct page *newpage, struct page *page)
+{
+       int expected_count;
+       void **pslot;
+
+       if (!mapping) {
+               if (page_count(page) != 1)
+                       return -EAGAIN;
+               return 0;
+       }
+
+       spin_lock_irq(&mapping->tree_lock);
+
+       pslot = radix_tree_lookup_slot(&mapping->page_tree,
+                                       page_index(page));
+
+       expected_count = 2 + page_has_private(page);
+       if (page_count(page) != expected_count ||
+               radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) {
+               spin_unlock_irq(&mapping->tree_lock);
+               return -EAGAIN;
+       }
+
+       if (!page_freeze_refs(page, expected_count)) {
+               spin_unlock_irq(&mapping->tree_lock);
+               return -EAGAIN;
+       }
+
+       get_page(newpage);
+
+       radix_tree_replace_slot(pslot, newpage);
+
+       page_unfreeze_refs(page, expected_count);
+
+       __put_page(page);
+
+       spin_unlock_irq(&mapping->tree_lock);
+       return 0;
+}
+
+/*
  * Copy the page to its new location
  */
-static void migrate_page_copy(struct page *newpage, struct page *page)
+void migrate_page_copy(struct page *newpage, struct page *page)
 {
-       copy_highpage(newpage, page);
+       if (PageHuge(page))
+               copy_huge_page(newpage, page);
+       else
+               copy_highpage(newpage, page);
 
        if (PageError(page))
                SetPageError(newpage);
@@ -352,8 +361,8 @@ static void migrate_page_copy(struct page *newpage, struct page *page)
        if (TestClearPageActive(page)) {
                VM_BUG_ON(PageUnevictable(page));
                SetPageActive(newpage);
-       } else
-               unevictable_migrate_page(newpage, page);
+       } else if (TestClearPageUnevictable(page))
+               SetPageUnevictable(newpage);
        if (PageChecked(page))
                SetPageChecked(newpage);
        if (PageMappedToDisk(page))
@@ -366,16 +375,15 @@ static void migrate_page_copy(struct page *newpage, struct page *page)
                 * redo the accounting that clear_page_dirty_for_io undid,
                 * but we can't use set_page_dirty because that function
                 * is actually a signal that all of the page has become dirty.
-                * Wheras only part of our page may be dirty.
+                * Whereas only part of our page may be dirty.
                 */
                __set_page_dirty_nobuffers(newpage);
        }
 
        mlock_migrate_page(newpage, page);
+       ksm_migrate_page(newpage, page);
 
-#ifdef CONFIG_SWAP
        ClearPageSwapCache(page);
-#endif
        ClearPagePrivate(page);
        set_page_private(page, 0);
        page->mapping = NULL;
@@ -402,7 +410,7 @@ EXPORT_SYMBOL(fail_migrate_page);
 
 /*
  * Common logic to directly migrate a single page suitable for
- * pages that do not use PagePrivate.
+ * pages that do not use PagePrivate/PagePrivate2.
  *
  * Pages are locked upon entry and exit.
  */
@@ -493,7 +501,6 @@ static int writeout(struct address_space *mapping, struct page *page)
                .nr_to_write = 1,
                .range_start = 0,
                .range_end = LLONG_MAX,
-               .nonblocking = 1,
                .for_reclaim = 1
        };
        int rc;
@@ -517,15 +524,12 @@ static int writeout(struct address_space *mapping, struct page *page)
        remove_migration_ptes(page, page);
 
        rc = mapping->a_ops->writepage(page, &wbc);
-       if (rc < 0)
-               /* I/O Error writing */
-               return -EIO;
 
        if (rc != AOP_WRITEPAGE_ACTIVATE)
                /* unlocked. Relock */
                lock_page(page);
 
-       return -EAGAIN;
+       return (rc < 0) ? -EIO : -EAGAIN;
 }
 
 /*
@@ -541,7 +545,7 @@ static int fallback_migrate_page(struct address_space *mapping,
         * Buffers may be managed in a filesystem specific way.
         * We must have no buffers or drop them.
         */
-       if (PagePrivate(page) &&
+       if (page_has_private(page) &&
            !try_to_release_page(page, GFP_KERNEL))
                return -EAGAIN;
 
@@ -559,7 +563,8 @@ static int fallback_migrate_page(struct address_space *mapping,
  *   < 0 - error code
  *  == 0 - success
  */
-static int move_to_new_page(struct page *newpage, struct page *page)
+static int move_to_new_page(struct page *newpage, struct page *page,
+                                       int remap_swapcache, bool sync)
 {
        struct address_space *mapping;
        int rc;
@@ -581,23 +586,35 @@ static int move_to_new_page(struct page *newpage, struct page *page)
        mapping = page_mapping(page);
        if (!mapping)
                rc = migrate_page(mapping, newpage, page);
-       else if (mapping->a_ops->migratepage)
+       else {
                /*
-                * Most pages have a mapping and most filesystems
-                * should provide a migration function. Anonymous
-                * pages are part of swap space which also has its
-                * own migration function. This is the most common
-                * path for page migration.
+                * Do not writeback pages if !sync and migratepage is
+                * not pointing to migrate_page() which is nonblocking
+                * (swapcache/tmpfs uses migratepage = migrate_page).
                 */
-               rc = mapping->a_ops->migratepage(mapping,
-                                               newpage, page);
-       else
-               rc = fallback_migrate_page(mapping, newpage, page);
+               if (PageDirty(page) && !sync &&
+                   mapping->a_ops->migratepage != migrate_page)
+                       rc = -EBUSY;
+               else if (mapping->a_ops->migratepage)
+                       /*
+                        * Most pages have a mapping and most filesystems
+                        * should provide a migration function. Anonymous
+                        * pages are part of swap space which also has its
+                        * own migration function. This is the most common
+                        * path for page migration.
+                        */
+                       rc = mapping->a_ops->migratepage(mapping,
+                                                       newpage, page);
+               else
+                       rc = fallback_migrate_page(mapping, newpage, page);
+       }
 
-       if (!rc) {
-               remove_migration_ptes(page, newpage);
-       } else
+       if (rc) {
                newpage->mapping = NULL;
+       } else {
+               if (remap_swapcache)
+                       remove_migration_ptes(page, newpage);
+       }
 
        unlock_page(newpage);
 
@@ -609,13 +626,15 @@ static int move_to_new_page(struct page *newpage, struct page *page)
  * to the newly allocated page in newpage.
  */
 static int unmap_and_move(new_page_t get_new_page, unsigned long private,
-                       struct page *page, int force)
+                       struct page *page, int force, bool offlining, bool sync)
 {
        int rc = 0;
        int *result = NULL;
        struct page *newpage = get_new_page(page, private, &result);
-       int rcu_locked = 0;
+       int remap_swapcache = 1;
        int charge = 0;
+       struct mem_cgroup *mem;
+       struct anon_vma *anon_vma = NULL;
 
        if (!newpage)
                return -ENOMEM;
@@ -624,38 +643,110 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
                /* page was freed from under us. So we are done. */
                goto move_newpage;
        }
+       if (unlikely(PageTransHuge(page)))
+               if (unlikely(split_huge_page(page)))
+                       goto move_newpage;
 
-       charge = mem_cgroup_prepare_migration(page, newpage);
-       if (charge == -ENOMEM) {
-               rc = -ENOMEM;
-               goto move_newpage;
-       }
        /* prepare cgroup just returns 0 or -ENOMEM */
-       BUG_ON(charge);
-
        rc = -EAGAIN;
+
        if (!trylock_page(page)) {
-               if (!force)
+               if (!force || !sync)
                        goto move_newpage;
+
+               /*
+                * It's not safe for direct compaction to call lock_page.
+                * For example, during page readahead pages are added locked
+                * to the LRU. Later, when the IO completes the pages are
+                * marked uptodate and unlocked. However, the queueing
+                * could be merging multiple pages for one bio (e.g.
+                * mpage_readpages). If an allocation happens for the
+                * second or third page, the process can end up locking
+                * the same page twice and deadlocking. Rather than
+                * trying to be clever about what pages can be locked,
+                * avoid the use of lock_page for direct compaction
+                * altogether.
+                */
+               if (current->flags & PF_MEMALLOC)
+                       goto move_newpage;
+
                lock_page(page);
        }
 
+       /*
+        * Only memory hotplug's offline_pages() caller has locked out KSM,
+        * and can safely migrate a KSM page.  The other cases have skipped
+        * PageKsm along with PageReserved - but it is only now when we have
+        * the page lock that we can be certain it will not go KSM beneath us
+        * (KSM will not upgrade a page from PageAnon to PageKsm when it sees
+        * its pagecount raised, but only here do we take the page lock which
+        * serializes that).
+        */
+       if (PageKsm(page) && !offlining) {
+               rc = -EBUSY;
+               goto unlock;
+       }
+
+       /* charge against new page */
+       charge = mem_cgroup_prepare_migration(page, newpage, &mem, GFP_KERNEL);
+       if (charge == -ENOMEM) {
+               rc = -ENOMEM;
+               goto unlock;
+       }
+       BUG_ON(charge);
+
        if (PageWriteback(page)) {
+               /*
+                * For !sync, there is no point retrying as the retry loop
+                * is expected to be too short for PageWriteback to be cleared
+                */
+               if (!sync) {
+                       rc = -EBUSY;
+                       goto uncharge;
+               }
                if (!force)
-                       goto unlock;
+                       goto uncharge;
                wait_on_page_writeback(page);
        }
        /*
         * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
         * we cannot notice that anon_vma is freed while we migrates a page.
-        * This rcu_read_lock() delays freeing anon_vma pointer until the end
+        * This get_anon_vma() delays freeing anon_vma pointer until the end
         * of migration. File cache pages are no problem because of page_lock()
         * File Caches may use write_page() or lock_page() in migration, then,
         * just care Anon page here.
         */
        if (PageAnon(page)) {
-               rcu_read_lock();
-               rcu_locked = 1;
+               /*
+                * Only page_lock_anon_vma() understands the subtleties of
+                * getting a hold on an anon_vma from outside one of its mms.
+                */
+               anon_vma = page_lock_anon_vma(page);
+               if (anon_vma) {
+                       /*
+                        * Take a reference count on the anon_vma if the
+                        * page is mapped so that it is guaranteed to
+                        * exist when the page is remapped later
+                        */
+                       get_anon_vma(anon_vma);
+                       page_unlock_anon_vma(anon_vma);
+               } else if (PageSwapCache(page)) {
+                       /*
+                        * We cannot be sure that the anon_vma of an unmapped
+                        * swapcache page is safe to use because we don't
+                        * know in advance if the VMA that this page belonged
+                        * to still exists. If the VMA and others sharing the
+                        * data have been freed, then the anon_vma could
+                        * already be invalid.
+                        *
+                        * To avoid this possibility, swapcache pages get
+                        * migrated but are not remapped when migration
+                        * completes
+                        */
+                       remap_swapcache = 0;
+               } else {
+                       goto uncharge;
+               }
        }
 
        /*
@@ -671,34 +762,35 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
         * free the metadata, so the page can be freed.
         */
        if (!page->mapping) {
-               if (!PageAnon(page) && PagePrivate(page)) {
-                       /*
-                        * Go direct to try_to_free_buffers() here because
-                        * a) that's what try_to_release_page() would do anyway
-                        * b) we may be under rcu_read_lock() here, so we can't
-                        *    use GFP_KERNEL which is what try_to_release_page()
-                        *    needs to be effective.
-                        */
+               VM_BUG_ON(PageAnon(page));
+               if (page_has_private(page)) {
                        try_to_free_buffers(page);
+                       goto uncharge;
                }
-               goto rcu_unlock;
+               goto skip_unmap;
        }
 
        /* Establish migration ptes or remove ptes */
-       try_to_unmap(page, 1);
+       try_to_unmap(page, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
 
+skip_unmap:
        if (!page_mapped(page))
-               rc = move_to_new_page(newpage, page);
+               rc = move_to_new_page(newpage, page, remap_swapcache, sync);
 
-       if (rc)
+       if (rc && remap_swapcache)
                remove_migration_ptes(page, page);
-rcu_unlock:
-       if (rcu_locked)
-               rcu_read_unlock();
 
+       /* Drop an anon_vma reference if we took one */
+       if (anon_vma)
+               put_anon_vma(anon_vma);
+
+uncharge:
+       if (!charge)
+               mem_cgroup_end_migration(mem, page, newpage, rc == 0);
 unlock:
        unlock_page(page);
 
+move_newpage:
        if (rc != -EAGAIN) {
                /*
                 * A page that has been migrated has all references
@@ -707,13 +799,11 @@ unlock:
                 * restored.
                 */
                list_del(&page->lru);
+               dec_zone_page_state(page, NR_ISOLATED_ANON +
+                               page_is_file_cache(page));
                putback_lru_page(page);
        }
 
-move_newpage:
-       if (!charge)
-               mem_cgroup_end_migration(newpage);
-
        /*
         * Move the new page to the LRU. If migration was not successful
         * then this will free the page.
@@ -730,6 +820,81 @@ move_newpage:
 }
 
 /*
+ * Counterpart of unmap_and_move_page() for hugepage migration.
+ *
+ * This function doesn't wait the completion of hugepage I/O
+ * because there is no race between I/O and migration for hugepage.
+ * Note that currently hugepage I/O occurs only in direct I/O
+ * where no lock is held and PG_writeback is irrelevant,
+ * and writeback status of all subpages are counted in the reference
+ * count of the head page (i.e. if all subpages of a 2MB hugepage are
+ * under direct I/O, the reference of the head page is 512 and a bit more.)
+ * This means that when we try to migrate hugepage whose subpages are
+ * doing direct I/O, some references remain after try_to_unmap() and
+ * hugepage migration fails without data corruption.
+ *
+ * There is also no race when direct I/O is issued on the page under migration,
+ * because then pte is replaced with migration swap entry and direct I/O code
+ * will wait in the page fault for migration to complete.
+ */
+static int unmap_and_move_huge_page(new_page_t get_new_page,
+                               unsigned long private, struct page *hpage,
+                               int force, bool offlining, bool sync)
+{
+       int rc = 0;
+       int *result = NULL;
+       struct page *new_hpage = get_new_page(hpage, private, &result);
+       struct anon_vma *anon_vma = NULL;
+
+       if (!new_hpage)
+               return -ENOMEM;
+
+       rc = -EAGAIN;
+
+       if (!trylock_page(hpage)) {
+               if (!force || !sync)
+                       goto out;
+               lock_page(hpage);
+       }
+
+       if (PageAnon(hpage)) {
+               anon_vma = page_lock_anon_vma(hpage);
+               if (anon_vma) {
+                       get_anon_vma(anon_vma);
+                       page_unlock_anon_vma(anon_vma);
+               }
+       }
+
+       try_to_unmap(hpage, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
+
+       if (!page_mapped(hpage))
+               rc = move_to_new_page(new_hpage, hpage, 1, sync);
+
+       if (rc)
+               remove_migration_ptes(hpage, hpage);
+
+       if (anon_vma)
+               put_anon_vma(anon_vma);
+out:
+       unlock_page(hpage);
+
+       if (rc != -EAGAIN) {
+               list_del(&hpage->lru);
+               put_page(hpage);
+       }
+
+       put_page(new_hpage);
+
+       if (result) {
+               if (rc)
+                       *result = rc;
+               else
+                       *result = page_to_nid(new_hpage);
+       }
+       return rc;
+}
+
+/*
  * migrate_pages
  *
  * The function takes one list of pages to migrate and a function
@@ -738,13 +903,15 @@ move_newpage:
  *
  * The function returns after 10 attempts or if no pages
  * are movable anymore because to has become empty
- * or no retryable pages exist anymore. All pages will be
- * returned to the LRU or freed.
+ * or no retryable pages exist anymore.
+ * Caller should call putback_lru_pages to return pages to the LRU
+ * or free list only if ret != 0.
  *
  * Return: Number of pages not migrated or error code.
  */
 int migrate_pages(struct list_head *from,
-               new_page_t get_new_page, unsigned long private)
+               new_page_t get_new_page, unsigned long private, bool offlining,
+               bool sync)
 {
        int retry = 1;
        int nr_failed = 0;
@@ -764,7 +931,8 @@ int migrate_pages(struct list_head *from,
                        cond_resched();
 
                        rc = unmap_and_move(get_new_page, private,
-                                               page, pass > 2);
+                                               page, pass > 2, offlining,
+                                               sync);
 
                        switch(rc) {
                        case -ENOMEM:
@@ -786,8 +954,50 @@ out:
        if (!swapwrite)
                current->flags &= ~PF_SWAPWRITE;
 
-       putback_lru_pages(from);
+       if (rc)
+               return rc;
+
+       return nr_failed + retry;
+}
+
+int migrate_huge_pages(struct list_head *from,
+               new_page_t get_new_page, unsigned long private, bool offlining,
+               bool sync)
+{
+       int retry = 1;
+       int nr_failed = 0;
+       int pass = 0;
+       struct page *page;
+       struct page *page2;
+       int rc;
+
+       for (pass = 0; pass < 10 && retry; pass++) {
+               retry = 0;
 
+               list_for_each_entry_safe(page, page2, from, lru) {
+                       cond_resched();
+
+                       rc = unmap_and_move_huge_page(get_new_page,
+                                       private, page, pass > 2, offlining,
+                                       sync);
+
+                       switch(rc) {
+                       case -ENOMEM:
+                               goto out;
+                       case -EAGAIN:
+                               retry++;
+                               break;
+                       case 0:
+                               break;
+                       default:
+                               /* Permanent failure */
+                               nr_failed++;
+                               break;
+                       }
+               }
+       }
+       rc = 0;
+out:
        if (rc)
                return rc;
 
@@ -818,7 +1028,7 @@ static struct page *new_page_node(struct page *p, unsigned long private,
 
        *result = &pm->status;
 
-       return alloc_pages_node(pm->node,
+       return alloc_pages_exact_node(pm->node,
                                GFP_HIGHUSER_MOVABLE | GFP_THISNODE, 0);
 }
 
@@ -841,23 +1051,16 @@ static int do_move_page_to_node_array(struct mm_struct *mm,
        /*
         * Build a list of pages to migrate
         */
-       migrate_prep();
        for (pp = pm; pp->node != MAX_NUMNODES; pp++) {
                struct vm_area_struct *vma;
                struct page *page;
 
-               /*
-                * A valid page pointer that will not match any of the
-                * pages that will be moved.
-                */
-               pp->page = ZERO_PAGE(0);
-
                err = -EFAULT;
                vma = find_vma(mm, pp->addr);
-               if (!vma || !vma_migratable(vma))
+               if (!vma || pp->addr < vma->vm_start || !vma_migratable(vma))
                        goto set_status;
 
-               page = follow_page(vma, pp->addr, FOLL_GET);
+               page = follow_page(vma, pp->addr, FOLL_GET|FOLL_SPLIT);
 
                err = PTR_ERR(page);
                if (IS_ERR(page))
@@ -867,7 +1070,8 @@ static int do_move_page_to_node_array(struct mm_struct *mm,
                if (!page)
                        goto set_status;
 
-               if (PageReserved(page))         /* Check for zero page */
+               /* Use PageReserved to check for zero page */
+               if (PageReserved(page) || PageKsm(page))
                        goto put_and_set;
 
                pp->page = page;
@@ -885,8 +1089,11 @@ static int do_move_page_to_node_array(struct mm_struct *mm,
                        goto put_and_set;
 
                err = isolate_lru_page(page);
-               if (!err)
+               if (!err) {
                        list_add_tail(&page->lru, &pagelist);
+                       inc_zone_page_state(page, NR_ISOLATED_ANON +
+                                           page_is_file_cache(page));
+               }
 put_and_set:
                /*
                 * Either remove the duplicate refcount from
@@ -899,9 +1106,12 @@ set_status:
        }
 
        err = 0;
-       if (!list_empty(&pagelist))
+       if (!list_empty(&pagelist)) {
                err = migrate_pages(&pagelist, new_page_node,
-                               (unsigned long)pm);
+                               (unsigned long)pm, 0, true);
+               if (err)
+                       putback_lru_pages(&pagelist);
+       }
 
        up_read(&mm->mmap_sem);
        return err;
@@ -917,44 +1127,52 @@ static int do_pages_move(struct mm_struct *mm, struct task_struct *task,
                         const int __user *nodes,
                         int __user *status, int flags)
 {
-       struct page_to_node *pm = NULL;
+       struct page_to_node *pm;
        nodemask_t task_nodes;
-       int err = 0;
-       int i;
+       unsigned long chunk_nr_pages;
+       unsigned long chunk_start;
+       int err;
 
        task_nodes = cpuset_mems_allowed(task);
 
-       /* Limit nr_pages so that the multiplication may not overflow */
-       if (nr_pages >= ULONG_MAX / sizeof(struct page_to_node) - 1) {
-               err = -E2BIG;
+       err = -ENOMEM;
+       pm = (struct page_to_node *)__get_free_page(GFP_KERNEL);
+       if (!pm)
                goto out;
-       }
 
-       pm = vmalloc((nr_pages + 1) * sizeof(struct page_to_node));
-       if (!pm) {
-               err = -ENOMEM;
-               goto out;
-       }
+       migrate_prep();
 
        /*
-        * Get parameters from user space and initialize the pm
-        * array. Return various errors if the user did something wrong.
+        * Store a chunk of page_to_node array in a page,
+        * but keep the last one as a marker
         */
-       for (i = 0; i < nr_pages; i++) {
-               const void __user *p;
+       chunk_nr_pages = (PAGE_SIZE / sizeof(struct page_to_node)) - 1;
 
-               err = -EFAULT;
-               if (get_user(p, pages + i))
-                       goto out_pm;
+       for (chunk_start = 0;
+            chunk_start < nr_pages;
+            chunk_start += chunk_nr_pages) {
+               int j;
 
-               pm[i].addr = (unsigned long)p;
-               if (nodes) {
+               if (chunk_start + chunk_nr_pages > nr_pages)
+                       chunk_nr_pages = nr_pages - chunk_start;
+
+               /* fill the chunk pm with addrs and nodes from user-space */
+               for (j = 0; j < chunk_nr_pages; j++) {
+                       const void __user *p;
                        int node;
 
-                       if (get_user(node, nodes + i))
+                       err = -EFAULT;
+                       if (get_user(p, pages + j + chunk_start))
+                               goto out_pm;
+                       pm[j].addr = (unsigned long) p;
+
+                       if (get_user(node, nodes + j + chunk_start))
                                goto out_pm;
 
                        err = -ENODEV;
+                       if (node < 0 || node >= MAX_NUMNODES)
+                               goto out_pm;
+
                        if (!node_state(node, N_HIGH_MEMORY))
                                goto out_pm;
 
@@ -962,22 +1180,29 @@ static int do_pages_move(struct mm_struct *mm, struct task_struct *task,
                        if (!node_isset(node, task_nodes))
                                goto out_pm;
 
-                       pm[i].node = node;
-               } else
-                       pm[i].node = 0; /* anything to not match MAX_NUMNODES */
-       }
-       /* End marker */
-       pm[nr_pages].node = MAX_NUMNODES;
+                       pm[j].node = node;
+               }
+
+               /* End marker for this chunk */
+               pm[chunk_nr_pages].node = MAX_NUMNODES;
+
+               /* Migrate this chunk */
+               err = do_move_page_to_node_array(mm, pm,
+                                                flags & MPOL_MF_MOVE_ALL);
+               if (err < 0)
+                       goto out_pm;
 
-       err = do_move_page_to_node_array(mm, pm, flags & MPOL_MF_MOVE_ALL);
-       if (err >= 0)
                /* Return status information */
-               for (i = 0; i < nr_pages; i++)
-                       if (put_user(pm[i].status, status + i))
+               for (j = 0; j < chunk_nr_pages; j++)
+                       if (put_user(pm[j].status, status + j + chunk_start)) {
                                err = -EFAULT;
+                               goto out_pm;
+                       }
+       }
+       err = 0;
 
 out_pm:
-       vfree(pm);
+       free_page((unsigned long)pm);
 out:
        return err;
 }
@@ -985,28 +1210,21 @@ out:
 /*
  * Determine the nodes of an array of pages and store it in an array of status.
  */
-static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
-                        const void __user * __user *pages,
-                        int __user *status)
+static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
+                               const void __user **pages, int *status)
 {
        unsigned long i;
-       int err;
 
        down_read(&mm->mmap_sem);
 
        for (i = 0; i < nr_pages; i++) {
-               const void __user *p;
-               unsigned long addr;
+               unsigned long addr = (unsigned long)(*pages);
                struct vm_area_struct *vma;
                struct page *page;
-
-               err = -EFAULT;
-               if (get_user(p, pages+i))
-                       goto out;
-               addr = (unsigned long) p;
+               int err = -EFAULT;
 
                vma = find_vma(mm, addr);
-               if (!vma)
+               if (!vma || addr < vma->vm_start)
                        goto set_status;
 
                page = follow_page(vma, addr, 0);
@@ -1017,29 +1235,64 @@ static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
 
                err = -ENOENT;
                /* Use PageReserved to check for zero page */
-               if (!page || PageReserved(page))
+               if (!page || PageReserved(page) || PageKsm(page))
                        goto set_status;
 
                err = page_to_nid(page);
 set_status:
-               put_user(err, status+i);
+               *status = err;
+
+               pages++;
+               status++;
        }
-       err = 0;
 
-out:
        up_read(&mm->mmap_sem);
-       return err;
+}
+
+/*
+ * Determine the nodes of a user array of pages and store it in
+ * a user array of status.
+ */
+static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
+                        const void __user * __user *pages,
+                        int __user *status)
+{
+#define DO_PAGES_STAT_CHUNK_NR 16
+       const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
+       int chunk_status[DO_PAGES_STAT_CHUNK_NR];
+
+       while (nr_pages) {
+               unsigned long chunk_nr;
+
+               chunk_nr = nr_pages;
+               if (chunk_nr > DO_PAGES_STAT_CHUNK_NR)
+                       chunk_nr = DO_PAGES_STAT_CHUNK_NR;
+
+               if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages)))
+                       break;
+
+               do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);
+
+               if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
+                       break;
+
+               pages += chunk_nr;
+               status += chunk_nr;
+               nr_pages -= chunk_nr;
+       }
+       return nr_pages ? -EFAULT : 0;
 }
 
 /*
  * Move a list of pages in the address space of the currently executing
  * process.
  */
-asmlinkage long sys_move_pages(pid_t pid, unsigned long nr_pages,
-                       const void __user * __user *pages,
-                       const int __user *nodes,
-                       int __user *status, int flags)
+SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
+               const void __user * __user *, pages,
+               const int __user *, nodes,
+               int __user *, status, int, flags)
 {
+       const struct cred *cred = current_cred(), *tcred;
        struct task_struct *task;
        struct mm_struct *mm;
        int err;
@@ -1052,14 +1305,14 @@ asmlinkage long sys_move_pages(pid_t pid, unsigned long nr_pages,
                return -EPERM;
 
        /* Find the mm_struct */
-       read_lock(&tasklist_lock);
+       rcu_read_lock();
        task = pid ? find_task_by_vpid(pid) : current;
        if (!task) {
-               read_unlock(&tasklist_lock);
+               rcu_read_unlock();
                return -ESRCH;
        }
        mm = get_task_mm(task);
-       read_unlock(&tasklist_lock);
+       rcu_read_unlock();
 
        if (!mm)
                return -EINVAL;
@@ -1070,12 +1323,16 @@ asmlinkage long sys_move_pages(pid_t pid, unsigned long nr_pages,
         * capabilities, superuser privileges or the same
         * userid as the target process.
         */
-       if ((current->euid != task->suid) && (current->euid != task->uid) &&
-           (current->uid != task->suid) && (current->uid != task->uid) &&
+       rcu_read_lock();
+       tcred = __task_cred(task);
+       if (cred->euid != tcred->suid && cred->euid != tcred->uid &&
+           cred->uid  != tcred->suid && cred->uid  != tcred->uid &&
            !capable(CAP_SYS_NICE)) {
+               rcu_read_unlock();
                err = -EPERM;
                goto out;
        }
+       rcu_read_unlock();
 
        err = security_task_movememory(task);
        if (err)
@@ -1104,7 +1361,7 @@ int migrate_vmas(struct mm_struct *mm, const nodemask_t *to,
        struct vm_area_struct *vma;
        int err = 0;
 
-       for(vma = mm->mmap; vma->vm_next && !err; vma = vma->vm_next) {
+       for (vma = mm->mmap; vma && !err; vma = vma->vm_next) {
                if (vma->vm_ops && vma->vm_ops->migrate) {
                        err = vma->vm_ops->migrate(vma, to, from, flags);
                        if (err)