mm: improve page_lock_anon_vma() comment
[linux-2.6.git] / mm / migrate.c
index 1a531b7..34132f8 100644 (file)
@@ -248,7 +248,7 @@ static int migrate_page_move_mapping(struct address_space *mapping,
 
        expected_count = 2 + page_has_private(page);
        if (page_count(page) != expected_count ||
-                       (struct page *)radix_tree_deref_slot(pslot) != page) {
+               radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) {
                spin_unlock_irq(&mapping->tree_lock);
                return -EAGAIN;
        }
@@ -320,7 +320,7 @@ int migrate_huge_page_move_mapping(struct address_space *mapping,
 
        expected_count = 2 + page_has_private(page);
        if (page_count(page) != expected_count ||
-           (struct page *)radix_tree_deref_slot(pslot) != page) {
+               radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) {
                spin_unlock_irq(&mapping->tree_lock);
                return -EAGAIN;
        }
@@ -375,7 +375,7 @@ void migrate_page_copy(struct page *newpage, struct page *page)
                 * redo the accounting that clear_page_dirty_for_io undid,
                 * but we can't use set_page_dirty because that function
                 * is actually a signal that all of the page has become dirty.
-                * Wheras only part of our page may be dirty.
+                * Whereas only part of our page may be dirty.
                 */
                __set_page_dirty_nobuffers(newpage);
        }
@@ -564,7 +564,7 @@ static int fallback_migrate_page(struct address_space *mapping,
  *  == 0 - success
  */
 static int move_to_new_page(struct page *newpage, struct page *page,
-                                               int remap_swapcache)
+                                       int remap_swapcache, bool sync)
 {
        struct address_space *mapping;
        int rc;
@@ -586,18 +586,28 @@ static int move_to_new_page(struct page *newpage, struct page *page,
        mapping = page_mapping(page);
        if (!mapping)
                rc = migrate_page(mapping, newpage, page);
-       else if (mapping->a_ops->migratepage)
+       else {
                /*
-                * Most pages have a mapping and most filesystems
-                * should provide a migration function. Anonymous
-                * pages are part of swap space which also has its
-                * own migration function. This is the most common
-                * path for page migration.
+                * Do not writeback pages if !sync and migratepage is
+                * not pointing to migrate_page() which is nonblocking
+                * (swapcache/tmpfs uses migratepage = migrate_page).
                 */
-               rc = mapping->a_ops->migratepage(mapping,
-                                               newpage, page);
-       else
-               rc = fallback_migrate_page(mapping, newpage, page);
+               if (PageDirty(page) && !sync &&
+                   mapping->a_ops->migratepage != migrate_page)
+                       rc = -EBUSY;
+               else if (mapping->a_ops->migratepage)
+                       /*
+                        * Most pages have a mapping and most filesystems
+                        * should provide a migration function. Anonymous
+                        * pages are part of swap space which also has its
+                        * own migration function. This is the most common
+                        * path for page migration.
+                        */
+                       rc = mapping->a_ops->migratepage(mapping,
+                                                       newpage, page);
+               else
+                       rc = fallback_migrate_page(mapping, newpage, page);
+       }
 
        if (rc) {
                newpage->mapping = NULL;
@@ -622,9 +632,8 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
        int *result = NULL;
        struct page *newpage = get_new_page(page, private, &result);
        int remap_swapcache = 1;
-       int rcu_locked = 0;
        int charge = 0;
-       struct mem_cgroup *mem = NULL;
+       struct mem_cgroup *mem;
        struct anon_vma *anon_vma = NULL;
 
        if (!newpage)
@@ -642,7 +651,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
        rc = -EAGAIN;
 
        if (!trylock_page(page)) {
-               if (!force)
+               if (!force || !sync)
                        goto move_newpage;
 
                /*
@@ -679,7 +688,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
        }
 
        /* charge against new page */
-       charge = mem_cgroup_prepare_migration(page, newpage, &mem);
+       charge = mem_cgroup_prepare_migration(page, newpage, &mem, GFP_KERNEL);
        if (charge == -ENOMEM) {
                rc = -ENOMEM;
                goto unlock;
@@ -687,27 +696,41 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
        BUG_ON(charge);
 
        if (PageWriteback(page)) {
-               if (!force || !sync)
+               /*
+                * For !sync, there is no point retrying as the retry loop
+                * is expected to be too short for PageWriteback to be cleared
+                */
+               if (!sync) {
+                       rc = -EBUSY;
+                       goto uncharge;
+               }
+               if (!force)
                        goto uncharge;
                wait_on_page_writeback(page);
        }
        /*
         * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
         * we cannot notice that anon_vma is freed while we migrates a page.
-        * This rcu_read_lock() delays freeing anon_vma pointer until the end
+        * This get_anon_vma() delays freeing anon_vma pointer until the end
         * of migration. File cache pages are no problem because of page_lock()
         * File Caches may use write_page() or lock_page() in migration, then,
         * just care Anon page here.
         */
        if (PageAnon(page)) {
-               rcu_read_lock();
-               rcu_locked = 1;
-
-               /* Determine how to safely use anon_vma */
-               if (!page_mapped(page)) {
-                       if (!PageSwapCache(page))
-                               goto rcu_unlock;
-
+               /*
+                * Only page_lock_anon_vma() understands the subtleties of
+                * getting a hold on an anon_vma from outside one of its mms.
+                */
+               anon_vma = page_lock_anon_vma(page);
+               if (anon_vma) {
+                       /*
+                        * Take a reference count on the anon_vma if the
+                        * page is mapped so that it is guaranteed to
+                        * exist when the page is remapped later
+                        */
+                       get_anon_vma(anon_vma);
+                       page_unlock_anon_vma(anon_vma);
+               } else if (PageSwapCache(page)) {
                        /*
                         * We cannot be sure that the anon_vma of an unmapped
                         * swapcache page is safe to use because we don't
@@ -722,13 +745,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
                         */
                        remap_swapcache = 0;
                } else {
-                       /*
-                        * Take a reference count on the anon_vma if the
-                        * page is mapped so that it is guaranteed to
-                        * exist when the page is remapped later
-                        */
-                       anon_vma = page_anon_vma(page);
-                       get_anon_vma(anon_vma);
+                       goto uncharge;
                }
        }
 
@@ -745,16 +762,10 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
         * free the metadata, so the page can be freed.
         */
        if (!page->mapping) {
-               if (!PageAnon(page) && page_has_private(page)) {
-                       /*
-                        * Go direct to try_to_free_buffers() here because
-                        * a) that's what try_to_release_page() would do anyway
-                        * b) we may be under rcu_read_lock() here, so we can't
-                        *    use GFP_KERNEL which is what try_to_release_page()
-                        *    needs to be effective.
-                        */
+               VM_BUG_ON(PageAnon(page));
+               if (page_has_private(page)) {
                        try_to_free_buffers(page);
-                       goto rcu_unlock;
+                       goto uncharge;
                }
                goto skip_unmap;
        }
@@ -764,24 +775,22 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
 
 skip_unmap:
        if (!page_mapped(page))
-               rc = move_to_new_page(newpage, page, remap_swapcache);
+               rc = move_to_new_page(newpage, page, remap_swapcache, sync);
 
        if (rc && remap_swapcache)
                remove_migration_ptes(page, page);
-rcu_unlock:
 
        /* Drop an anon_vma reference if we took one */
        if (anon_vma)
-               drop_anon_vma(anon_vma);
+               put_anon_vma(anon_vma);
 
-       if (rcu_locked)
-               rcu_read_unlock();
 uncharge:
        if (!charge)
-               mem_cgroup_end_migration(mem, page, newpage);
+               mem_cgroup_end_migration(mem, page, newpage, rc == 0);
 unlock:
        unlock_page(page);
 
+move_newpage:
        if (rc != -EAGAIN) {
                /*
                 * A page that has been migrated has all references
@@ -795,8 +804,6 @@ unlock:
                putback_lru_page(page);
        }
 
-move_newpage:
-
        /*
         * Move the new page to the LRU. If migration was not successful
         * then this will free the page.
@@ -837,7 +844,6 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
        int rc = 0;
        int *result = NULL;
        struct page *new_hpage = get_new_page(hpage, private, &result);
-       int rcu_locked = 0;
        struct anon_vma *anon_vma = NULL;
 
        if (!new_hpage)
@@ -852,33 +858,23 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
        }
 
        if (PageAnon(hpage)) {
-               rcu_read_lock();
-               rcu_locked = 1;
-
-               if (page_mapped(hpage)) {
-                       anon_vma = page_anon_vma(hpage);
-                       atomic_inc(&anon_vma->external_refcount);
+               anon_vma = page_lock_anon_vma(hpage);
+               if (anon_vma) {
+                       get_anon_vma(anon_vma);
+                       page_unlock_anon_vma(anon_vma);
                }
        }
 
        try_to_unmap(hpage, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
 
        if (!page_mapped(hpage))
-               rc = move_to_new_page(new_hpage, hpage, 1);
+               rc = move_to_new_page(new_hpage, hpage, 1, sync);
 
        if (rc)
                remove_migration_ptes(hpage, hpage);
 
-       if (anon_vma && atomic_dec_and_lock(&anon_vma->external_refcount,
-                                           &anon_vma->lock)) {
-               int empty = list_empty(&anon_vma->head);
-               spin_unlock(&anon_vma->lock);
-               if (empty)
-                       anon_vma_free(anon_vma);
-       }
-
-       if (rcu_locked)
-               rcu_read_unlock();
+       if (anon_vma)
+               put_anon_vma(anon_vma);
 out:
        unlock_page(hpage);
 
@@ -909,7 +905,7 @@ out:
  * are movable anymore because to has become empty
  * or no retryable pages exist anymore.
  * Caller should call putback_lru_pages to return pages to the LRU
- * or free list.
+ * or free list only if ret != 0.
  *
  * Return: Number of pages not migrated or error code.
  */
@@ -1002,10 +998,6 @@ int migrate_huge_pages(struct list_head *from,
        }
        rc = 0;
 out:
-
-       list_for_each_entry_safe(page, page2, from, lru)
-               put_page(page);
-
        if (rc)
                return rc;
 
@@ -1313,14 +1305,14 @@ SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
                return -EPERM;
 
        /* Find the mm_struct */
-       read_lock(&tasklist_lock);
+       rcu_read_lock();
        task = pid ? find_task_by_vpid(pid) : current;
        if (!task) {
-               read_unlock(&tasklist_lock);
+               rcu_read_unlock();
                return -ESRCH;
        }
        mm = get_task_mm(task);
-       read_unlock(&tasklist_lock);
+       rcu_read_unlock();
 
        if (!mm)
                return -EINVAL;