Merge git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty-2.6
[linux-2.6.git] / mm / vmscan.c
index 2ef59d5..3ff3311 100644 (file)
@@ -13,7 +13,7 @@
 
 #include <linux/mm.h>
 #include <linux/module.h>
-#include <linux/slab.h>
+#include <linux/gfp.h>
 #include <linux/kernel_stat.h>
 #include <linux/swap.h>
 #include <linux/pagemap.h>
@@ -262,27 +262,6 @@ unsigned long shrink_slab(unsigned long scanned, gfp_t gfp_mask,
        return ret;
 }
 
-/* Called without lock on whether page is mapped, so answer is unstable */
-static inline int page_mapping_inuse(struct page *page)
-{
-       struct address_space *mapping;
-
-       /* Page is in somebody's page tables. */
-       if (page_mapped(page))
-               return 1;
-
-       /* Be more reluctant to reclaim swapcache than pagecache */
-       if (PageSwapCache(page))
-               return 1;
-
-       mapping = page_mapping(page);
-       if (!mapping)
-               return 0;
-
-       /* File is mmap'd by somebody? */
-       return mapping_mapped(mapping);
-}
-
 static inline int is_page_cache_freeable(struct page *page)
 {
        /*
@@ -579,6 +558,65 @@ redo:
        put_page(page);         /* drop ref from isolate */
 }
 
+enum page_references {
+       PAGEREF_RECLAIM,
+       PAGEREF_RECLAIM_CLEAN,
+       PAGEREF_KEEP,
+       PAGEREF_ACTIVATE,
+};
+
+static enum page_references page_check_references(struct page *page,
+                                                 struct scan_control *sc)
+{
+       int referenced_ptes, referenced_page;
+       unsigned long vm_flags;
+
+       referenced_ptes = page_referenced(page, 1, sc->mem_cgroup, &vm_flags);
+       referenced_page = TestClearPageReferenced(page);
+
+       /* Lumpy reclaim - ignore references */
+       if (sc->order > PAGE_ALLOC_COSTLY_ORDER)
+               return PAGEREF_RECLAIM;
+
+       /*
+        * Mlock lost the isolation race with us.  Let try_to_unmap()
+        * move the page to the unevictable list.
+        */
+       if (vm_flags & VM_LOCKED)
+               return PAGEREF_RECLAIM;
+
+       if (referenced_ptes) {
+               if (PageAnon(page))
+                       return PAGEREF_ACTIVATE;
+               /*
+                * All mapped pages start out with page table
+                * references from the instantiating fault, so we need
+                * to look twice if a mapped file page is used more
+                * than once.
+                *
+                * Mark it and spare it for another trip around the
+                * inactive list.  Another page table reference will
+                * lead to its activation.
+                *
+                * Note: the mark is set for activated pages as well
+                * so that recently deactivated but used pages are
+                * quickly recovered.
+                */
+               SetPageReferenced(page);
+
+               if (referenced_page)
+                       return PAGEREF_ACTIVATE;
+
+               return PAGEREF_KEEP;
+       }
+
+       /* Reclaim if clean, defer dirty pages to writeback */
+       if (referenced_page)
+               return PAGEREF_RECLAIM_CLEAN;
+
+       return PAGEREF_RECLAIM;
+}
+
 /*
  * shrink_page_list() returns the number of reclaimed pages
  */
@@ -590,16 +628,15 @@ static unsigned long shrink_page_list(struct list_head *page_list,
        struct pagevec freed_pvec;
        int pgactivate = 0;
        unsigned long nr_reclaimed = 0;
-       unsigned long vm_flags;
 
        cond_resched();
 
        pagevec_init(&freed_pvec, 1);
        while (!list_empty(page_list)) {
+               enum page_references references;
                struct address_space *mapping;
                struct page *page;
                int may_enter_fs;
-               int referenced;
 
                cond_resched();
 
@@ -641,17 +678,16 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                                goto keep_locked;
                }
 
-               referenced = page_referenced(page, 1,
-                                               sc->mem_cgroup, &vm_flags);
-               /*
-                * In active use or really unfreeable?  Activate it.
-                * If page which have PG_mlocked lost isoltation race,
-                * try_to_unmap moves it to unevictable list
-                */
-               if (sc->order <= PAGE_ALLOC_COSTLY_ORDER &&
-                                       referenced && page_mapping_inuse(page)
-                                       && !(vm_flags & VM_LOCKED))
+               references = page_check_references(page, sc);
+               switch (references) {
+               case PAGEREF_ACTIVATE:
                        goto activate_locked;
+               case PAGEREF_KEEP:
+                       goto keep_locked;
+               case PAGEREF_RECLAIM:
+               case PAGEREF_RECLAIM_CLEAN:
+                       ; /* try to reclaim the page below */
+               }
 
                /*
                 * Anonymous process memory has backing store?
@@ -685,7 +721,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                }
 
                if (PageDirty(page)) {
-                       if (sc->order <= PAGE_ALLOC_COSTLY_ORDER && referenced)
+                       if (references == PAGEREF_RECLAIM_CLEAN)
                                goto keep_locked;
                        if (!may_enter_fs)
                                goto keep_locked;
@@ -1165,10 +1201,8 @@ static unsigned long shrink_inactive_list(unsigned long max_scan,
                __mod_zone_page_state(zone, NR_ISOLATED_ANON, nr_anon);
                __mod_zone_page_state(zone, NR_ISOLATED_FILE, nr_file);
 
-               reclaim_stat->recent_scanned[0] += count[LRU_INACTIVE_ANON];
-               reclaim_stat->recent_scanned[0] += count[LRU_ACTIVE_ANON];
-               reclaim_stat->recent_scanned[1] += count[LRU_INACTIVE_FILE];
-               reclaim_stat->recent_scanned[1] += count[LRU_ACTIVE_FILE];
+               reclaim_stat->recent_scanned[0] += nr_anon;
+               reclaim_stat->recent_scanned[1] += nr_file;
 
                spin_unlock_irq(&zone->lru_lock);
 
@@ -1352,9 +1386,7 @@ static void shrink_active_list(unsigned long nr_pages, struct zone *zone,
                        continue;
                }
 
-               /* page_referenced clears PageReferenced */
-               if (page_mapping_inuse(page) &&
-                   page_referenced(page, 0, sc->mem_cgroup, &vm_flags)) {
+               if (page_referenced(page, 0, sc->mem_cgroup, &vm_flags)) {
                        nr_rotated++;
                        /*
                         * Identify referenced, file-backed active pages and
@@ -1463,20 +1495,26 @@ static int inactive_file_is_low(struct zone *zone, struct scan_control *sc)
        return low;
 }
 
+static int inactive_list_is_low(struct zone *zone, struct scan_control *sc,
+                               int file)
+{
+       if (file)
+               return inactive_file_is_low(zone, sc);
+       else
+               return inactive_anon_is_low(zone, sc);
+}
+
 static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
        struct zone *zone, struct scan_control *sc, int priority)
 {
        int file = is_file_lru(lru);
 
-       if (lru == LRU_ACTIVE_FILE && inactive_file_is_low(zone, sc)) {
-               shrink_active_list(nr_to_scan, zone, sc, priority, file);
+       if (is_active_lru(lru)) {
+               if (inactive_list_is_low(zone, sc, file))
+                   shrink_active_list(nr_to_scan, zone, sc, priority, file);
                return 0;
        }
 
-       if (lru == LRU_ACTIVE_ANON && inactive_anon_is_low(zone, sc)) {
-               shrink_active_list(nr_to_scan, zone, sc, priority, file);
-               return 0;
-       }
        return shrink_inactive_list(nr_to_scan, zone, sc, priority, file);
 }
 
@@ -1690,8 +1728,7 @@ static void shrink_zones(int priority, struct zonelist *zonelist,
                                continue;
                        note_zone_scanning_priority(zone, priority);
 
-                       if (zone_is_all_unreclaimable(zone) &&
-                                               priority != DEF_PRIORITY)
+                       if (zone->all_unreclaimable && priority != DEF_PRIORITY)
                                continue;       /* Let kswapd poll it */
                        sc->all_unreclaimable = 0;
                } else {
@@ -1918,6 +1955,9 @@ static int sleeping_prematurely(pg_data_t *pgdat, int order, long remaining)
                if (!populated_zone(zone))
                        continue;
 
+               if (zone->all_unreclaimable)
+                       continue;
+
                if (!zone_watermark_ok(zone, order, high_wmark_pages(zone),
                                                                0, 0))
                        return 1;
@@ -2005,8 +2045,7 @@ loop_again:
                        if (!populated_zone(zone))
                                continue;
 
-                       if (zone_is_all_unreclaimable(zone) &&
-                           priority != DEF_PRIORITY)
+                       if (zone->all_unreclaimable && priority != DEF_PRIORITY)
                                continue;
 
                        /*
@@ -2049,13 +2088,9 @@ loop_again:
                        if (!populated_zone(zone))
                                continue;
 
-                       if (zone_is_all_unreclaimable(zone) &&
-                                       priority != DEF_PRIORITY)
+                       if (zone->all_unreclaimable && priority != DEF_PRIORITY)
                                continue;
 
-                       if (!zone_watermark_ok(zone, order,
-                                       high_wmark_pages(zone), end_zone, 0))
-                               all_zones_ok = 0;
                        temp_priority[i] = priority;
                        sc.nr_scanned = 0;
                        note_zone_scanning_priority(zone, priority);
@@ -2080,12 +2115,11 @@ loop_again:
                                                lru_pages);
                        sc.nr_reclaimed += reclaim_state->reclaimed_slab;
                        total_scanned += sc.nr_scanned;
-                       if (zone_is_all_unreclaimable(zone))
+                       if (zone->all_unreclaimable)
                                continue;
-                       if (nr_slab == 0 && zone->pages_scanned >=
-                                       (zone_reclaimable_pages(zone) * 6))
-                                       zone_set_flag(zone,
-                                                     ZONE_ALL_UNRECLAIMABLE);
+                       if (nr_slab == 0 &&
+                           zone->pages_scanned >= (zone_reclaimable_pages(zone) * 6))
+                               zone->all_unreclaimable = 1;
                        /*
                         * If we've done a decent amount of scanning and
                         * the reclaim ratio is low, start doing writepage
@@ -2095,13 +2129,18 @@ loop_again:
                            total_scanned > sc.nr_reclaimed + sc.nr_reclaimed / 2)
                                sc.may_writepage = 1;
 
-                       /*
-                        * We are still under min water mark. it mean we have
-                        * GFP_ATOMIC allocation failure risk. Hurry up!
-                        */
-                       if (!zone_watermark_ok(zone, order, min_wmark_pages(zone),
-                                             end_zone, 0))
-                               has_under_min_watermark_zone = 1;
+                       if (!zone_watermark_ok(zone, order,
+                                       high_wmark_pages(zone), end_zone, 0)) {
+                               all_zones_ok = 0;
+                               /*
+                                * We are still under min water mark.  This
+                                * means that we have a GFP_ATOMIC allocation
+                                * failure risk. Hurry up!
+                                */
+                               if (!zone_watermark_ok(zone, order,
+                                           min_wmark_pages(zone), end_zone, 0))
+                                       has_under_min_watermark_zone = 1;
+                       }
 
                }
                if (all_zones_ok)
@@ -2543,6 +2582,7 @@ static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
         * and RECLAIM_SWAP.
         */
        p->flags |= PF_MEMALLOC | PF_SWAPWRITE;
+       lockdep_set_current_reclaim_state(gfp_mask);
        reclaim_state.reclaimed_slab = 0;
        p->reclaim_state = &reclaim_state;
 
@@ -2586,6 +2626,7 @@ static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
 
        p->reclaim_state = NULL;
        current->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE);
+       lockdep_clear_current_reclaim_state();
        return sc.nr_reclaimed >= nr_pages;
 }
 
@@ -2608,7 +2649,7 @@ int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
            zone_page_state(zone, NR_SLAB_RECLAIMABLE) <= zone->min_slab_pages)
                return ZONE_RECLAIM_FULL;
 
-       if (zone_is_all_unreclaimable(zone))
+       if (zone->all_unreclaimable)
                return ZONE_RECLAIM_FULL;
 
        /*