Memory controller remove control_type feature
[linux-2.6.git] / mm / memcontrol.c
index 160101a..5c2c702 100644 (file)
@@ -21,6 +21,7 @@
 #include <linux/memcontrol.h>
 #include <linux/cgroup.h>
 #include <linux/mm.h>
+#include <linux/smp.h>
 #include <linux/page-flags.h>
 #include <linux/backing-dev.h>
 #include <linux/bit_spinlock.h>
@@ -28,6 +29,7 @@
 #include <linux/swap.h>
 #include <linux/spinlock.h>
 #include <linux/fs.h>
+#include <linux/seq_file.h>
 
 #include <asm/uaccess.h>
 
@@ -35,6 +37,78 @@ struct cgroup_subsys mem_cgroup_subsys;
 static const int MEM_CGROUP_RECLAIM_RETRIES = 5;
 
 /*
+ * Statistics for memory cgroup.
+ */
+enum mem_cgroup_stat_index {
+       /*
+        * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
+        */
+       MEM_CGROUP_STAT_CACHE,     /* # of pages charged as cache */
+       MEM_CGROUP_STAT_RSS,       /* # of pages charged as rss */
+
+       MEM_CGROUP_STAT_NSTATS,
+};
+
+struct mem_cgroup_stat_cpu {
+       s64 count[MEM_CGROUP_STAT_NSTATS];
+} ____cacheline_aligned_in_smp;
+
+struct mem_cgroup_stat {
+       struct mem_cgroup_stat_cpu cpustat[NR_CPUS];
+};
+
+/*
+ * For accounting under irq disable, no need for increment preempt count.
+ */
+static void __mem_cgroup_stat_add_safe(struct mem_cgroup_stat *stat,
+               enum mem_cgroup_stat_index idx, int val)
+{
+       int cpu = smp_processor_id();
+       stat->cpustat[cpu].count[idx] += val;
+}
+
+static s64 mem_cgroup_read_stat(struct mem_cgroup_stat *stat,
+               enum mem_cgroup_stat_index idx)
+{
+       int cpu;
+       s64 ret = 0;
+       for_each_possible_cpu(cpu)
+               ret += stat->cpustat[cpu].count[idx];
+       return ret;
+}
+
+/*
+ * per-zone information in memory controller.
+ */
+
+enum mem_cgroup_zstat_index {
+       MEM_CGROUP_ZSTAT_ACTIVE,
+       MEM_CGROUP_ZSTAT_INACTIVE,
+
+       NR_MEM_CGROUP_ZSTAT,
+};
+
+struct mem_cgroup_per_zone {
+       /*
+        * spin_lock to protect the per cgroup LRU
+        */
+       spinlock_t              lru_lock;
+       struct list_head        active_list;
+       struct list_head        inactive_list;
+       unsigned long count[NR_MEM_CGROUP_ZSTAT];
+};
+/* Macro for accessing counter */
+#define MEM_CGROUP_ZSTAT(mz, idx)      ((mz)->count[(idx)])
+
+struct mem_cgroup_per_node {
+       struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
+};
+
+struct mem_cgroup_lru_info {
+       struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
+};
+
+/*
  * The memory controller data structure. The memory controller controls both
  * page cache and RSS per cgroup. We would eventually like to provide
  * statistics based on the statistics developed by Rik Van Riel for clock-pro,
@@ -54,15 +128,14 @@ struct mem_cgroup {
        /*
         * Per cgroup active and inactive list, similar to the
         * per zone LRU lists.
-        * TODO: Consider making these lists per zone
         */
-       struct list_head active_list;
-       struct list_head inactive_list;
+       struct mem_cgroup_lru_info info;
+
+       int     prev_priority;  /* for recording reclaim priority */
        /*
-        * spin_lock to protect the per cgroup LRU
+        * statistics.
         */
-       spinlock_t lru_lock;
-       unsigned long control_type;     /* control RSS or RSS+Pagecache */
+       struct mem_cgroup_stat stat;
 };
 
 /*
@@ -86,6 +159,17 @@ struct page_cgroup {
        int      flags;
 };
 #define PAGE_CGROUP_FLAG_CACHE (0x1)   /* charged as cache */
+#define PAGE_CGROUP_FLAG_ACTIVE (0x2)  /* page is active in this cgroup */
+
+static inline int page_cgroup_nid(struct page_cgroup *pc)
+{
+       return page_to_nid(pc->page);
+}
+
+static inline enum zone_type page_cgroup_zid(struct page_cgroup *pc)
+{
+       return page_zonenum(pc->page);
+}
 
 enum {
        MEM_CGROUP_TYPE_UNSPEC = 0,
@@ -100,6 +184,56 @@ enum charge_type {
        MEM_CGROUP_CHARGE_TYPE_MAPPED,
 };
 
+
+/*
+ * Always modified under lru lock. Then, not necessary to preempt_disable()
+ */
+static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, int flags,
+                                       bool charge)
+{
+       int val = (charge)? 1 : -1;
+       struct mem_cgroup_stat *stat = &mem->stat;
+       VM_BUG_ON(!irqs_disabled());
+
+       if (flags & PAGE_CGROUP_FLAG_CACHE)
+               __mem_cgroup_stat_add_safe(stat,
+                                       MEM_CGROUP_STAT_CACHE, val);
+       else
+               __mem_cgroup_stat_add_safe(stat, MEM_CGROUP_STAT_RSS, val);
+}
+
+static inline struct mem_cgroup_per_zone *
+mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
+{
+       BUG_ON(!mem->info.nodeinfo[nid]);
+       return &mem->info.nodeinfo[nid]->zoneinfo[zid];
+}
+
+static inline struct mem_cgroup_per_zone *
+page_cgroup_zoneinfo(struct page_cgroup *pc)
+{
+       struct mem_cgroup *mem = pc->mem_cgroup;
+       int nid = page_cgroup_nid(pc);
+       int zid = page_cgroup_zid(pc);
+
+       return mem_cgroup_zoneinfo(mem, nid, zid);
+}
+
+static unsigned long mem_cgroup_get_all_zonestat(struct mem_cgroup *mem,
+                                       enum mem_cgroup_zstat_index idx)
+{
+       int nid, zid;
+       struct mem_cgroup_per_zone *mz;
+       u64 total = 0;
+
+       for_each_online_node(nid)
+               for (zid = 0; zid < MAX_NR_ZONES; zid++) {
+                       mz = mem_cgroup_zoneinfo(mem, nid, zid);
+                       total += MEM_CGROUP_ZSTAT(mz, idx);
+               }
+       return total;
+}
+
 static struct mem_cgroup init_mem_cgroup;
 
 static inline
@@ -174,8 +308,8 @@ static void __always_inline unlock_page_cgroup(struct page *page)
  * This can fail if the page has been tied to a page_cgroup.
  * If success, returns 0.
  */
-static inline int
-page_cgroup_assign_new_page_cgroup(struct page *page, struct page_cgroup *pc)
+static int page_cgroup_assign_new_page_cgroup(struct page *page,
+                                               struct page_cgroup *pc)
 {
        int ret = 0;
 
@@ -197,8 +331,8 @@ page_cgroup_assign_new_page_cgroup(struct page *page, struct page_cgroup *pc)
  *  clear_page_cgroup(page, pc) == pc
  */
 
-static inline struct page_cgroup *
-clear_page_cgroup(struct page *page, struct page_cgroup *pc)
+static struct page_cgroup *clear_page_cgroup(struct page *page,
+                                               struct page_cgroup *pc)
 {
        struct page_cgroup *ret;
        /* lock and clear */
@@ -210,13 +344,54 @@ clear_page_cgroup(struct page *page, struct page_cgroup *pc)
        return ret;
 }
 
+static void __mem_cgroup_remove_list(struct page_cgroup *pc)
+{
+       int from = pc->flags & PAGE_CGROUP_FLAG_ACTIVE;
+       struct mem_cgroup_per_zone *mz = page_cgroup_zoneinfo(pc);
+
+       if (from)
+               MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) -= 1;
+       else
+               MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) -= 1;
+
+       mem_cgroup_charge_statistics(pc->mem_cgroup, pc->flags, false);
+       list_del_init(&pc->lru);
+}
+
+static void __mem_cgroup_add_list(struct page_cgroup *pc)
+{
+       int to = pc->flags & PAGE_CGROUP_FLAG_ACTIVE;
+       struct mem_cgroup_per_zone *mz = page_cgroup_zoneinfo(pc);
+
+       if (!to) {
+               MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) += 1;
+               list_add(&pc->lru, &mz->inactive_list);
+       } else {
+               MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) += 1;
+               list_add(&pc->lru, &mz->active_list);
+       }
+       mem_cgroup_charge_statistics(pc->mem_cgroup, pc->flags, true);
+}
 
 static void __mem_cgroup_move_lists(struct page_cgroup *pc, bool active)
 {
-       if (active)
-               list_move(&pc->lru, &pc->mem_cgroup->active_list);
+       int from = pc->flags & PAGE_CGROUP_FLAG_ACTIVE;
+       struct mem_cgroup_per_zone *mz = page_cgroup_zoneinfo(pc);
+
+       if (from)
+               MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) -= 1;
        else
-               list_move(&pc->lru, &pc->mem_cgroup->inactive_list);
+               MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) -= 1;
+
+       if (active) {
+               MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) += 1;
+               pc->flags |= PAGE_CGROUP_FLAG_ACTIVE;
+               list_move(&pc->lru, &mz->active_list);
+       } else {
+               MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) += 1;
+               pc->flags &= ~PAGE_CGROUP_FLAG_ACTIVE;
+               list_move(&pc->lru, &mz->inactive_list);
+       }
 }
 
 int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
@@ -234,15 +409,99 @@ int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
  */
 void mem_cgroup_move_lists(struct page_cgroup *pc, bool active)
 {
-       struct mem_cgroup *mem;
+       struct mem_cgroup_per_zone *mz;
+       unsigned long flags;
+
        if (!pc)
                return;
 
-       mem = pc->mem_cgroup;
-
-       spin_lock(&mem->lru_lock);
+       mz = page_cgroup_zoneinfo(pc);
+       spin_lock_irqsave(&mz->lru_lock, flags);
        __mem_cgroup_move_lists(pc, active);
-       spin_unlock(&mem->lru_lock);
+       spin_unlock_irqrestore(&mz->lru_lock, flags);
+}
+
+/*
+ * Calculate mapped_ratio under memory controller. This will be used in
+ * vmscan.c for deteremining we have to reclaim mapped pages.
+ */
+int mem_cgroup_calc_mapped_ratio(struct mem_cgroup *mem)
+{
+       long total, rss;
+
+       /*
+        * usage is recorded in bytes. But, here, we assume the number of
+        * physical pages can be represented by "long" on any arch.
+        */
+       total = (long) (mem->res.usage >> PAGE_SHIFT) + 1L;
+       rss = (long)mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_RSS);
+       return (int)((rss * 100L) / total);
+}
+/*
+ * This function is called from vmscan.c. In page reclaiming loop. balance
+ * between active and inactive list is calculated. For memory controller
+ * page reclaiming, we should use using mem_cgroup's imbalance rather than
+ * zone's global lru imbalance.
+ */
+long mem_cgroup_reclaim_imbalance(struct mem_cgroup *mem)
+{
+       unsigned long active, inactive;
+       /* active and inactive are the number of pages. 'long' is ok.*/
+       active = mem_cgroup_get_all_zonestat(mem, MEM_CGROUP_ZSTAT_ACTIVE);
+       inactive = mem_cgroup_get_all_zonestat(mem, MEM_CGROUP_ZSTAT_INACTIVE);
+       return (long) (active / (inactive + 1));
+}
+
+/*
+ * prev_priority control...this will be used in memory reclaim path.
+ */
+int mem_cgroup_get_reclaim_priority(struct mem_cgroup *mem)
+{
+       return mem->prev_priority;
+}
+
+void mem_cgroup_note_reclaim_priority(struct mem_cgroup *mem, int priority)
+{
+       if (priority < mem->prev_priority)
+               mem->prev_priority = priority;
+}
+
+void mem_cgroup_record_reclaim_priority(struct mem_cgroup *mem, int priority)
+{
+       mem->prev_priority = priority;
+}
+
+/*
+ * Calculate # of pages to be scanned in this priority/zone.
+ * See also vmscan.c
+ *
+ * priority starts from "DEF_PRIORITY" and decremented in each loop.
+ * (see include/linux/mmzone.h)
+ */
+
+long mem_cgroup_calc_reclaim_active(struct mem_cgroup *mem,
+                                  struct zone *zone, int priority)
+{
+       long nr_active;
+       int nid = zone->zone_pgdat->node_id;
+       int zid = zone_idx(zone);
+       struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(mem, nid, zid);
+
+       nr_active = MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE);
+       return (nr_active >> priority);
+}
+
+long mem_cgroup_calc_reclaim_inactive(struct mem_cgroup *mem,
+                                       struct zone *zone, int priority)
+{
+       long nr_inactive;
+       int nid = zone->zone_pgdat->node_id;
+       int zid = zone_idx(zone);
+       struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(mem, nid, zid);
+
+       nr_inactive = MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE);
+
+       return (nr_inactive >> priority);
 }
 
 unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
@@ -258,13 +517,18 @@ unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
        LIST_HEAD(pc_list);
        struct list_head *src;
        struct page_cgroup *pc, *tmp;
+       int nid = z->zone_pgdat->node_id;
+       int zid = zone_idx(z);
+       struct mem_cgroup_per_zone *mz;
 
+       mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
        if (active)
-               src = &mem_cont->active_list;
+               src = &mz->active_list;
        else
-               src = &mem_cont->inactive_list;
+               src = &mz->inactive_list;
+
 
-       spin_lock(&mem_cont->lru_lock);
+       spin_lock(&mz->lru_lock);
        scan = 0;
        list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
                if (scan >= nr_to_scan)
@@ -284,13 +548,6 @@ unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
                        continue;
                }
 
-               /*
-                * Reclaim, per zone
-                * TODO: make the active/inactive lists per zone
-                */
-               if (page_zone(page) != z)
-                       continue;
-
                scan++;
                list_move(&pc->lru, &pc_list);
 
@@ -301,7 +558,7 @@ unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
        }
 
        list_splice(&pc_list, src);
-       spin_unlock(&mem_cont->lru_lock);
+       spin_unlock(&mz->lru_lock);
 
        *scanned = scan;
        return nr_taken;
@@ -320,6 +577,7 @@ static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
        struct page_cgroup *pc;
        unsigned long flags;
        unsigned long nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
+       struct mem_cgroup_per_zone *mz;
 
        /*
         * Should page_cgroup's go to their own slab?
@@ -329,39 +587,41 @@ static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
         * with it
         */
 retry:
-       lock_page_cgroup(page);
-       pc = page_get_page_cgroup(page);
-       /*
-        * The page_cgroup exists and the page has already been accounted
-        */
-       if (pc) {
-               if (unlikely(!atomic_inc_not_zero(&pc->ref_cnt))) {
-                       /* this page is under being uncharged ? */
-                       unlock_page_cgroup(page);
-                       cpu_relax();
-                       goto retry;
-               } else {
-                       unlock_page_cgroup(page);
-                       goto done;
+       if (page) {
+               lock_page_cgroup(page);
+               pc = page_get_page_cgroup(page);
+               /*
+                * The page_cgroup exists and
+                * the page has already been accounted.
+                */
+               if (pc) {
+                       if (unlikely(!atomic_inc_not_zero(&pc->ref_cnt))) {
+                               /* this page is under being uncharged ? */
+                               unlock_page_cgroup(page);
+                               cpu_relax();
+                               goto retry;
+                       } else {
+                               unlock_page_cgroup(page);
+                               goto done;
+                       }
                }
+               unlock_page_cgroup(page);
        }
 
-       unlock_page_cgroup(page);
-
        pc = kzalloc(sizeof(struct page_cgroup), gfp_mask);
        if (pc == NULL)
                goto err;
 
-       rcu_read_lock();
        /*
-        * We always charge the cgroup the mm_struct belongs to
-        * the mm_struct's mem_cgroup changes on task migration if the
+        * We always charge the cgroup the mm_struct belongs to.
+        * The mm_struct's mem_cgroup changes on task migration if the
         * thread group leader migrates. It's possible that mm is not
         * set, if so charge the init_mm (happens for pagecache usage).
         */
        if (!mm)
                mm = &init_mm;
 
+       rcu_read_lock();
        mem = rcu_dereference(mm->mem_cgroup);
        /*
         * For every charge from the cgroup, increment reference
@@ -375,12 +635,8 @@ retry:
         * the cgroup limit.
         */
        while (res_counter_charge(&mem->res, PAGE_SIZE)) {
-               bool is_atomic = gfp_mask & GFP_ATOMIC;
-               /*
-                * We cannot reclaim under GFP_ATOMIC, fail the charge
-                */
-               if (is_atomic)
-                       goto noreclaim;
+               if (!(gfp_mask & __GFP_WAIT))
+                       goto out;
 
                if (try_to_free_mem_cgroup_pages(mem, gfp_mask))
                        continue;
@@ -394,50 +650,45 @@ retry:
                 */
                if (res_counter_check_under_limit(&mem->res))
                        continue;
-                       /*
-                        * Since we control both RSS and cache, we end up with a
-                        * very interesting scenario where we end up reclaiming
-                        * memory (essentially RSS), since the memory is pushed
-                        * to swap cache, we eventually end up adding those
-                        * pages back to our list. Hence we give ourselves a
-                        * few chances before we fail
-                        */
-               else if (nr_retries--) {
-                       congestion_wait(WRITE, HZ/10);
-                       continue;
+
+               if (!nr_retries--) {
+                       mem_cgroup_out_of_memory(mem, gfp_mask);
+                       goto out;
                }
-noreclaim:
-               css_put(&mem->css);
-               if (!is_atomic)
-                       mem_cgroup_out_of_memory(mem, GFP_KERNEL);
-               goto free_pc;
+               congestion_wait(WRITE, HZ/10);
        }
 
        atomic_set(&pc->ref_cnt, 1);
        pc->mem_cgroup = mem;
        pc->page = page;
-       pc->flags = 0;
+       pc->flags = PAGE_CGROUP_FLAG_ACTIVE;
        if (ctype == MEM_CGROUP_CHARGE_TYPE_CACHE)
                pc->flags |= PAGE_CGROUP_FLAG_CACHE;
-       if (page_cgroup_assign_new_page_cgroup(page, pc)) {
+
+       if (!page || page_cgroup_assign_new_page_cgroup(page, pc)) {
                /*
-                * an another charge is added to this page already.
-                * we do take lock_page_cgroup(page) again and read
+                * Another charge has been added to this page already.
+                * We take lock_page_cgroup(page) again and read
                 * page->cgroup, increment refcnt.... just retry is OK.
                 */
                res_counter_uncharge(&mem->res, PAGE_SIZE);
                css_put(&mem->css);
                kfree(pc);
+               if (!page)
+                       goto done;
                goto retry;
        }
 
-       spin_lock_irqsave(&mem->lru_lock, flags);
-       list_add(&pc->lru, &mem->active_list);
-       spin_unlock_irqrestore(&mem->lru_lock, flags);
+       mz = page_cgroup_zoneinfo(pc);
+       spin_lock_irqsave(&mz->lru_lock, flags);
+       /* Update statistics vector */
+       __mem_cgroup_add_list(pc);
+       spin_unlock_irqrestore(&mz->lru_lock, flags);
 
 done:
        return 0;
-free_pc:
+out:
+       css_put(&mem->css);
        kfree(pc);
 err:
        return -ENOMEM;
@@ -457,55 +708,59 @@ int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
                                gfp_t gfp_mask)
 {
        int ret = 0;
-       struct mem_cgroup *mem;
        if (!mm)
                mm = &init_mm;
 
-       rcu_read_lock();
-       mem = rcu_dereference(mm->mem_cgroup);
-       css_get(&mem->css);
-       rcu_read_unlock();
-       if (mem->control_type == MEM_CGROUP_TYPE_ALL)
-               ret = mem_cgroup_charge_common(page, mm, gfp_mask,
+       ret = mem_cgroup_charge_common(page, mm, gfp_mask,
                                MEM_CGROUP_CHARGE_TYPE_CACHE);
-       css_put(&mem->css);
        return ret;
 }
 
 /*
  * Uncharging is always a welcome operation, we never complain, simply
- * uncharge.
+ * uncharge. This routine should be called with lock_page_cgroup held
  */
 void mem_cgroup_uncharge(struct page_cgroup *pc)
 {
        struct mem_cgroup *mem;
+       struct mem_cgroup_per_zone *mz;
        struct page *page;
        unsigned long flags;
 
        /*
-        * This can handle cases when a page is not charged at all and we
-        * are switching between handling the control_type.
+        * Check if our page_cgroup is valid
         */
        if (!pc)
                return;
 
        if (atomic_dec_and_test(&pc->ref_cnt)) {
                page = pc->page;
+               mz = page_cgroup_zoneinfo(pc);
                /*
                 * get page->cgroup and clear it under lock.
                 * force_empty can drop page->cgroup without checking refcnt.
                 */
+               unlock_page_cgroup(page);
                if (clear_page_cgroup(page, pc) == pc) {
                        mem = pc->mem_cgroup;
                        css_put(&mem->css);
                        res_counter_uncharge(&mem->res, PAGE_SIZE);
-                       spin_lock_irqsave(&mem->lru_lock, flags);
-                       list_del_init(&pc->lru);
-                       spin_unlock_irqrestore(&mem->lru_lock, flags);
+                       spin_lock_irqsave(&mz->lru_lock, flags);
+                       __mem_cgroup_remove_list(pc);
+                       spin_unlock_irqrestore(&mz->lru_lock, flags);
                        kfree(pc);
                }
+               lock_page_cgroup(page);
        }
 }
+
+void mem_cgroup_uncharge_page(struct page *page)
+{
+       lock_page_cgroup(page);
+       mem_cgroup_uncharge(page_get_page_cgroup(page));
+       unlock_page_cgroup(page);
+}
+
 /*
  * Returns non-zero if a page (under migration) has valid page_cgroup member.
  * Refcnt of page_cgroup is incremented.
@@ -525,8 +780,12 @@ int mem_cgroup_prepare_migration(struct page *page)
 
 void mem_cgroup_end_migration(struct page *page)
 {
-       struct page_cgroup *pc = page_get_page_cgroup(page);
+       struct page_cgroup *pc;
+
+       lock_page_cgroup(page);
+       pc = page_get_page_cgroup(page);
        mem_cgroup_uncharge(pc);
+       unlock_page_cgroup(page);
 }
 /*
  * We know both *page* and *newpage* are now not-on-LRU and Pg_locked.
@@ -537,16 +796,31 @@ void mem_cgroup_end_migration(struct page *page)
 void mem_cgroup_page_migration(struct page *page, struct page *newpage)
 {
        struct page_cgroup *pc;
+       struct mem_cgroup *mem;
+       unsigned long flags;
+       struct mem_cgroup_per_zone *mz;
 retry:
        pc = page_get_page_cgroup(page);
        if (!pc)
                return;
+       mem = pc->mem_cgroup;
+       mz = page_cgroup_zoneinfo(pc);
        if (clear_page_cgroup(page, pc) != pc)
                goto retry;
+       spin_lock_irqsave(&mz->lru_lock, flags);
+
+       __mem_cgroup_remove_list(pc);
+       spin_unlock_irqrestore(&mz->lru_lock, flags);
+
        pc->page = newpage;
        lock_page_cgroup(newpage);
        page_assign_page_cgroup(newpage, pc);
        unlock_page_cgroup(newpage);
+
+       mz = page_cgroup_zoneinfo(pc);
+       spin_lock_irqsave(&mz->lru_lock, flags);
+       __mem_cgroup_add_list(pc);
+       spin_unlock_irqrestore(&mz->lru_lock, flags);
        return;
 }
 
@@ -557,16 +831,26 @@ retry:
  */
 #define FORCE_UNCHARGE_BATCH   (128)
 static void
-mem_cgroup_force_empty_list(struct mem_cgroup *mem, struct list_head *list)
+mem_cgroup_force_empty_list(struct mem_cgroup *mem,
+                           struct mem_cgroup_per_zone *mz,
+                           int active)
 {
        struct page_cgroup *pc;
        struct page *page;
        int count;
        unsigned long flags;
+       struct list_head *list;
+
+       if (active)
+               list = &mz->active_list;
+       else
+               list = &mz->inactive_list;
 
+       if (list_empty(list))
+               return;
 retry:
        count = FORCE_UNCHARGE_BATCH;
-       spin_lock_irqsave(&mem->lru_lock, flags);
+       spin_lock_irqsave(&mz->lru_lock, flags);
 
        while (--count && !list_empty(list)) {
                pc = list_entry(list->prev, struct page_cgroup, lru);
@@ -576,12 +860,12 @@ retry:
                if (clear_page_cgroup(page, pc) == pc) {
                        css_put(&mem->css);
                        res_counter_uncharge(&mem->res, PAGE_SIZE);
-                       list_del_init(&pc->lru);
+                       __mem_cgroup_remove_list(pc);
                        kfree(pc);
                } else  /* being uncharged ? ...do relax */
                        break;
        }
-       spin_unlock_irqrestore(&mem->lru_lock, flags);
+       spin_unlock_irqrestore(&mz->lru_lock, flags);
        if (!list_empty(list)) {
                cond_resched();
                goto retry;
@@ -597,20 +881,25 @@ retry:
 int mem_cgroup_force_empty(struct mem_cgroup *mem)
 {
        int ret = -EBUSY;
+       int node, zid;
        css_get(&mem->css);
        /*
         * page reclaim code (kswapd etc..) will move pages between
 `       * active_list <-> inactive_list while we don't take a lock.
         * So, we have to do loop here until all lists are empty.
         */
-       while (!(list_empty(&mem->active_list) &&
-                list_empty(&mem->inactive_list))) {
+       while (mem->res.usage > 0) {
                if (atomic_read(&mem->css.cgroup->count) > 0)
                        goto out;
-               /* drop all page_cgroup in active_list */
-               mem_cgroup_force_empty_list(mem, &mem->active_list);
-               /* drop all page_cgroup in inactive_list */
-               mem_cgroup_force_empty_list(mem, &mem->inactive_list);
+               for_each_node_state(node, N_POSSIBLE)
+                       for (zid = 0; zid < MAX_NR_ZONES; zid++) {
+                               struct mem_cgroup_per_zone *mz;
+                               mz = mem_cgroup_zoneinfo(mem, node, zid);
+                               /* drop all page_cgroup in active_list */
+                               mem_cgroup_force_empty_list(mem, mz, 1);
+                               /* drop all page_cgroup in inactive_list */
+                               mem_cgroup_force_empty_list(mem, mz, 0);
+                       }
        }
        ret = 0;
 out:
@@ -651,61 +940,6 @@ static ssize_t mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
                                mem_cgroup_write_strategy);
 }
 
-static ssize_t mem_control_type_write(struct cgroup *cont,
-                       struct cftype *cft, struct file *file,
-                       const char __user *userbuf,
-                       size_t nbytes, loff_t *pos)
-{
-       int ret;
-       char *buf, *end;
-       unsigned long tmp;
-       struct mem_cgroup *mem;
-
-       mem = mem_cgroup_from_cont(cont);
-       buf = kmalloc(nbytes + 1, GFP_KERNEL);
-       ret = -ENOMEM;
-       if (buf == NULL)
-               goto out;
-
-       buf[nbytes] = 0;
-       ret = -EFAULT;
-       if (copy_from_user(buf, userbuf, nbytes))
-               goto out_free;
-
-       ret = -EINVAL;
-       tmp = simple_strtoul(buf, &end, 10);
-       if (*end != '\0')
-               goto out_free;
-
-       if (tmp <= MEM_CGROUP_TYPE_UNSPEC || tmp >= MEM_CGROUP_TYPE_MAX)
-               goto out_free;
-
-       mem->control_type = tmp;
-       ret = nbytes;
-out_free:
-       kfree(buf);
-out:
-       return ret;
-}
-
-static ssize_t mem_control_type_read(struct cgroup *cont,
-                               struct cftype *cft,
-                               struct file *file, char __user *userbuf,
-                               size_t nbytes, loff_t *ppos)
-{
-       unsigned long val;
-       char buf[64], *s;
-       struct mem_cgroup *mem;
-
-       mem = mem_cgroup_from_cont(cont);
-       s = buf;
-       val = mem->control_type;
-       s += sprintf(s, "%lu\n", val);
-       return simple_read_from_buffer((void __user *)userbuf, nbytes,
-                       ppos, buf, s - buf);
-}
-
-
 static ssize_t mem_force_empty_write(struct cgroup *cont,
                                struct cftype *cft, struct file *file,
                                const char __user *userbuf,
@@ -732,6 +966,60 @@ static ssize_t mem_force_empty_read(struct cgroup *cont,
 }
 
 
+static const struct mem_cgroup_stat_desc {
+       const char *msg;
+       u64 unit;
+} mem_cgroup_stat_desc[] = {
+       [MEM_CGROUP_STAT_CACHE] = { "cache", PAGE_SIZE, },
+       [MEM_CGROUP_STAT_RSS] = { "rss", PAGE_SIZE, },
+};
+
+static int mem_control_stat_show(struct seq_file *m, void *arg)
+{
+       struct cgroup *cont = m->private;
+       struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
+       struct mem_cgroup_stat *stat = &mem_cont->stat;
+       int i;
+
+       for (i = 0; i < ARRAY_SIZE(stat->cpustat[0].count); i++) {
+               s64 val;
+
+               val = mem_cgroup_read_stat(stat, i);
+               val *= mem_cgroup_stat_desc[i].unit;
+               seq_printf(m, "%s %lld\n", mem_cgroup_stat_desc[i].msg,
+                               (long long)val);
+       }
+       /* showing # of active pages */
+       {
+               unsigned long active, inactive;
+
+               inactive = mem_cgroup_get_all_zonestat(mem_cont,
+                                               MEM_CGROUP_ZSTAT_INACTIVE);
+               active = mem_cgroup_get_all_zonestat(mem_cont,
+                                               MEM_CGROUP_ZSTAT_ACTIVE);
+               seq_printf(m, "active %ld\n", (active) * PAGE_SIZE);
+               seq_printf(m, "inactive %ld\n", (inactive) * PAGE_SIZE);
+       }
+       return 0;
+}
+
+static const struct file_operations mem_control_stat_file_operations = {
+       .read = seq_read,
+       .llseek = seq_lseek,
+       .release = single_release,
+};
+
+static int mem_control_stat_open(struct inode *unused, struct file *file)
+{
+       /* XXX __d_cont */
+       struct cgroup *cont = file->f_dentry->d_parent->d_fsdata;
+
+       file->f_op = &mem_control_stat_file_operations;
+       return single_open(file, mem_control_stat_show, cont);
+}
+
+
+
 static struct cftype mem_cgroup_files[] = {
        {
                .name = "usage_in_bytes",
@@ -750,23 +1038,61 @@ static struct cftype mem_cgroup_files[] = {
                .read = mem_cgroup_read,
        },
        {
-               .name = "control_type",
-               .write = mem_control_type_write,
-               .read = mem_control_type_read,
-       },
-       {
                .name = "force_empty",
                .write = mem_force_empty_write,
                .read = mem_force_empty_read,
        },
+       {
+               .name = "stat",
+               .open = mem_control_stat_open,
+       },
 };
 
+static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
+{
+       struct mem_cgroup_per_node *pn;
+       struct mem_cgroup_per_zone *mz;
+       int zone;
+       /*
+        * This routine is called against possible nodes.
+        * But it's BUG to call kmalloc() against offline node.
+        *
+        * TODO: this routine can waste much memory for nodes which will
+        *       never be onlined. It's better to use memory hotplug callback
+        *       function.
+        */
+       if (node_state(node, N_HIGH_MEMORY))
+               pn = kmalloc_node(sizeof(*pn), GFP_KERNEL, node);
+       else
+               pn = kmalloc(sizeof(*pn), GFP_KERNEL);
+       if (!pn)
+               return 1;
+
+       mem->info.nodeinfo[node] = pn;
+       memset(pn, 0, sizeof(*pn));
+
+       for (zone = 0; zone < MAX_NR_ZONES; zone++) {
+               mz = &pn->zoneinfo[zone];
+               INIT_LIST_HEAD(&mz->active_list);
+               INIT_LIST_HEAD(&mz->inactive_list);
+               spin_lock_init(&mz->lru_lock);
+       }
+       return 0;
+}
+
+static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
+{
+       kfree(mem->info.nodeinfo[node]);
+}
+
+
 static struct mem_cgroup init_mem_cgroup;
 
 static struct cgroup_subsys_state *
 mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
 {
        struct mem_cgroup *mem;
+       int node;
 
        if (unlikely((cont->parent) == NULL)) {
                mem = &init_mem_cgroup;
@@ -778,16 +1104,38 @@ mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
                return NULL;
 
        res_counter_init(&mem->res);
-       INIT_LIST_HEAD(&mem->active_list);
-       INIT_LIST_HEAD(&mem->inactive_list);
-       spin_lock_init(&mem->lru_lock);
-       mem->control_type = MEM_CGROUP_TYPE_ALL;
+
+       memset(&mem->info, 0, sizeof(mem->info));
+
+       for_each_node_state(node, N_POSSIBLE)
+               if (alloc_mem_cgroup_per_zone_info(mem, node))
+                       goto free_out;
+
        return &mem->css;
+free_out:
+       for_each_node_state(node, N_POSSIBLE)
+               free_mem_cgroup_per_zone_info(mem, node);
+       if (cont->parent != NULL)
+               kfree(mem);
+       return NULL;
+}
+
+static void mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
+                                       struct cgroup *cont)
+{
+       struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
+       mem_cgroup_force_empty(mem);
 }
 
 static void mem_cgroup_destroy(struct cgroup_subsys *ss,
                                struct cgroup *cont)
 {
+       int node;
+       struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
+
+       for_each_node_state(node, N_POSSIBLE)
+               free_mem_cgroup_per_zone_info(mem, node);
+
        kfree(mem_cgroup_from_cont(cont));
 }
 
@@ -836,8 +1184,9 @@ struct cgroup_subsys mem_cgroup_subsys = {
        .name = "memory",
        .subsys_id = mem_cgroup_subsys_id,
        .create = mem_cgroup_create,
+       .pre_destroy = mem_cgroup_pre_destroy,
        .destroy = mem_cgroup_destroy,
        .populate = mem_cgroup_populate,
        .attach = mem_cgroup_move_task,
-       .early_init = 1,
+       .early_init = 0,
 };