UPSTREAM mm: memcg: enable memcg OOM killer only for user faults
[linux-3.10.git] / mm / page_isolation.c
index 6744235..383bdbb 100644 (file)
@@ -8,29 +8,7 @@
 #include <linux/memory.h>
 #include "internal.h"
 
-/* called while holding zone->lock */
-static void set_pageblock_isolate(struct page *page)
-{
-       if (get_pageblock_migratetype(page) == MIGRATE_ISOLATE)
-               return;
-
-       set_pageblock_migratetype(page, MIGRATE_ISOLATE);
-       page_zone(page)->nr_pageblock_isolate++;
-}
-
-/* called while holding zone->lock */
-static void restore_pageblock_isolate(struct page *page, int migratetype)
-{
-       struct zone *zone = page_zone(page);
-       if (WARN_ON(get_pageblock_migratetype(page) != MIGRATE_ISOLATE))
-               return;
-
-       BUG_ON(zone->nr_pageblock_isolate <= 0);
-       set_pageblock_migratetype(page, migratetype);
-       zone->nr_pageblock_isolate--;
-}
-
-int set_migratetype_isolate(struct page *page)
+int set_migratetype_isolate(struct page *page, bool skip_hwpoisoned_pages)
 {
        struct zone *zone;
        unsigned long flags, pfn;
@@ -66,7 +44,8 @@ int set_migratetype_isolate(struct page *page)
         * FIXME: Now, memory hotplug doesn't call shrink_slab() by itself.
         * We just check MOVABLE pages.
         */
-       if (!has_unmovable_pages(zone, page, arg.pages_found))
+       if (!has_unmovable_pages(zone, page, arg.pages_found,
+                                skip_hwpoisoned_pages))
                ret = 0;
 
        /*
@@ -79,7 +58,7 @@ out:
                unsigned long nr_pages;
                int migratetype = get_pageblock_migratetype(page);
 
-               set_pageblock_isolate(page);
+               set_pageblock_migratetype(page, MIGRATE_ISOLATE);
                nr_pages = move_freepages_block(zone, page, MIGRATE_ISOLATE);
 
                __mod_zone_freepage_state(zone, -nr_pages, migratetype);
@@ -102,7 +81,7 @@ void unset_migratetype_isolate(struct page *page, unsigned migratetype)
                goto out;
        nr_pages = move_freepages_block(zone, page, migratetype);
        __mod_zone_freepage_state(zone, nr_pages, migratetype);
-       restore_pageblock_isolate(page, migratetype);
+       set_pageblock_migratetype(page, migratetype);
 out:
        spin_unlock_irqrestore(&zone->lock, flags);
 }
@@ -134,7 +113,7 @@ __first_valid_page(unsigned long pfn, unsigned long nr_pages)
  * Returns 0 on success and -EBUSY if any part of range cannot be isolated.
  */
 int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
-                            unsigned migratetype)
+                            unsigned migratetype, bool skip_hwpoisoned_pages)
 {
        unsigned long pfn;
        unsigned long undo_pfn;
@@ -147,7 +126,8 @@ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
             pfn < end_pfn;
             pfn += pageblock_nr_pages) {
                page = __first_valid_page(pfn, pageblock_nr_pages);
-               if (page && set_migratetype_isolate(page)) {
+               if (page &&
+                   set_migratetype_isolate(page, skip_hwpoisoned_pages)) {
                        undo_pfn = pfn;
                        goto undo;
                }
@@ -190,7 +170,8 @@ int undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
  * Returns 1 if all pages in the range are isolated.
  */
 static int
-__test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn)
+__test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn,
+                                 bool skip_hwpoisoned_pages)
 {
        struct page *page;
 
@@ -201,13 +182,33 @@ __test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn)
                }
                page = pfn_to_page(pfn);
                if (PageBuddy(page)) {
-                       if (get_freepage_migratetype(page) != MIGRATE_ISOLATE)
-                               break;
+                       /*
+                        * If race between isolatation and allocation happens,
+                        * some free pages could be in MIGRATE_MOVABLE list
+                        * although pageblock's migratation type of the page
+                        * is MIGRATE_ISOLATE. Catch it and move the page into
+                        * MIGRATE_ISOLATE list.
+                        */
+                       if (get_freepage_migratetype(page) != MIGRATE_ISOLATE) {
+                               struct page *end_page;
+
+                               end_page = page + (1 << page_order(page)) - 1;
+                               move_freepages(page_zone(page), page, end_page,
+                                               MIGRATE_ISOLATE);
+                       }
                        pfn += 1 << page_order(page);
                }
                else if (page_count(page) == 0 &&
                        get_freepage_migratetype(page) == MIGRATE_ISOLATE)
                        pfn += 1;
+               else if (skip_hwpoisoned_pages && PageHWPoison(page)) {
+                       /*
+                        * The HWPoisoned page may be not in buddy
+                        * system, and page_count() is not 0.
+                        */
+                       pfn++;
+                       continue;
+               }
                else
                        break;
        }
@@ -216,7 +217,8 @@ __test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn)
        return 1;
 }
 
-int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn)
+int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn,
+                       bool skip_hwpoisoned_pages)
 {
        unsigned long pfn, flags;
        struct page *page;
@@ -239,7 +241,19 @@ int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn)
        /* Check all pages are free or Marked as ISOLATED */
        zone = page_zone(page);
        spin_lock_irqsave(&zone->lock, flags);
-       ret = __test_page_isolated_in_pageblock(start_pfn, end_pfn);
+       ret = __test_page_isolated_in_pageblock(start_pfn, end_pfn,
+                                               skip_hwpoisoned_pages);
        spin_unlock_irqrestore(&zone->lock, flags);
        return ret ? 0 : -EBUSY;
 }
+
+struct page *alloc_migrate_target(struct page *page, unsigned long private,
+                                 int **resultp)
+{
+       gfp_t gfp_mask = GFP_USER | __GFP_MOVABLE;
+
+       if (PageHighMem(page))
+               gfp_mask |= __GFP_HIGHMEM;
+
+       return alloc_page(gfp_mask);
+}